diff --git a/docs/ax-conwip.md b/docs/ax-conwip.md new file mode 100644 index 000000000..e4be4e416 --- /dev/null +++ b/docs/ax-conwip.md @@ -0,0 +1,257 @@ +# ax-conwip: the CONWIP scheduler as a user service + +Module: `home/ax-conwip.nix`. Option namespace: `myAxConwip`. Check: +`checks.x86_64-linux.ax-conwip-topology` in `flake.nix`. Written 2026-09-23. + +**Status: declared and OFF. `myAxConwip.enable` is set nowhere on this fleet, so +this module defines no unit on any host today.** The flake check asserts exactly +that, and it is written so that it goes red on the flip. Flipping it is Tom's. + +## What the CONWIP is + +`/home/tom/mecattaf/ax-conwip` is a CONWIP scheduler. CONWIP is constant +work-in-progress: a fixed number of slots, and a new item is admitted only when +a finished one gives its slot back. It is a cap on how much is in flight, not a +rate limit and not a queue discipline. + +This one: + +1. reads Claude ultracode workflow run records (`wf_*.json`); +2. derives one work item per `workflow_agent` entry of `workflowProgress`, never + one per workflow. Phases become dependency edges between items; +3. resolves each item's full prompt out of the record's own `script` string, by + scanning for the matching `agent(...)` call. It never parses and never + evaluates that script: the dialect opens with `export const meta`, closes + with a top-level `return`, and uses top-level `await`, which together mean no + single JavaScript goal symbol will load it. A label that matches zero calls, + or two, is a refusal with a named reason, not a guess; +4. admits items under a fixed work-in-progress cap, only on a free slot and only + when every dependency edge into the item is satisfied; +5. dispatches each admitted item to an ax server as a Task over gRPC, using + `UpdateTask`, which is an upsert. ax v0.3.0 has no `CreateTask` rpc at all; +6. gives the slot back when the Task reaches a phase the CONWIP calls terminal, + which is `{Completed, Failed, Terminating}`. That is deliberately NOT ax's own + terminal set, `{Running, Completed, Failed}`: a Running Task still occupies + work-in-progress, so the end of the `WatchTask` stream is a signal to read the + phase again, not a release; +7. writes an append-only ledger, one JSON object per line, deterministically + serialized. + +Read that repository's `DESIGN.md` first. It is the authority on all of the +above; this page only says what the module would run. + +## Nothing is packaged, and why + +**The `ax-conwip` repository has no remote.** It is a local git repository on the +coordinator, created with `git init`, and nothing has been pushed anywhere. + +The consequences are the reason this module has the shape it has: + +- **There is no flake input.** There is nothing for `inputs.ax-conwip` to point + at. A `builtins.fetchGit` of a path that exists on one box is not a + declaration; it is a machine-local accident that would break every other + host's evaluation. +- **There is therefore no package.** There is no `pkgs/ax-conwip.nix` and no + `.#ax-conwip`, because packaging follows the input. +- **So the unit runs a checkout in place.** `WorkingDirectory` is the + `sourceDir` option, and `ExecStart` is `pnpm exec tsx src/serve.ts`, resolved + against the `node_modules` that checkout already carries. That is a + development shape, not a delivered one, and it is the single biggest reason + `enable` must stay false. + +This is the first thing to fix if the CONWIP is ever to be a real lane: give the +repository a remote, declare it as a flake input, package it, and change +`ExecStart` to name a store path. Until then, do not flip the gate on a host you +care about. + +## What the module would run + +One long-running user service, `ax-conwip.service`, on whichever host has the +gate set, running `src/serve.ts`. + +Until 2026-09-23 the unit's `ExecStart` ran `src/cli.ts`, which reads the records +directory once, runs the loop over what it found, prints its ledger and exits: a +`Type = oneshot` in everything but name. `src/serve.ts` is the program that makes +the unit's own claim true. It polls the records directory every +`--poll-interval-ms` (2000 by default, and this module passes no other value), +derives each new `wf_*.json` exactly once keyed by absolute path, admits under a +cap that HOLDS FOR THE LIFE OF THE PROCESS rather than per tick, reads +`--meters` for seat admission through the same pure refusal rule the seat dry run +uses, and exits 0 on SIGTERM after appending a final `stop` line to its ledger. +Dispatch is DRY RUN by default; the live path needs all three of `--live`, +`AX_CONWIP_LIVE_HALOGEN=1` and a seat on the `["halogen"]` allow list in the +program's own `src/seats.ts`, and this module never passes `--live`. + +Two startup refusals are worth knowing before flipping anything: a `recordsDir` +that does not exist is exit 2, not a watcher that polls nothing forever, and an +ax server unreachable at startup is exit 4, because `Restart = no` means a dead +process is a fact to read in the journal rather than a thing to paper over. + +No timer: the release signal is a `WatchTask` stream, not a poll, so +there is nothing to wake on a cadence. + +`Type=simple`, `Restart=no`, `Nice=10`. `Restart=no` is deliberate: the +scheduler holds slots for the life of the process, so restarting it silently +would re-derive and re-admit work that is already in flight. If it dies, that is +a fact to read in the journal. + +**The unit carries no `Install` section.** Flipping `enable` DECLARES the unit; +it does not arm it and it does not start it. Starting it is a second, separate, +deliberate act. There are two gates here, not one, and that is on purpose. + +### The options, and their defaults + +| option | type | default | what it is | +|---|---|---|---| +| `enable` | bool | `false` | the gate. Set nowhere on this fleet. | +| `serverUrl` | str | `"127.0.0.1:8080"` | the ax server to dispatch to, as `host:port` | +| `metersDir` | path | `~/.local/state/tally-rewrite/meters` | the seat meters, READ ONLY. Passed both as `AX_CONWIP_METERS` and as the program's `--meters` | +| `wipCap` | positive int | `1` | how many Tasks may be admitted at once | +| `sourceDir` | path | `/home/tom/mecattaf/ax-conwip` | where the program lives, because it is not packaged | +| `recordsDir` | path | `~/.local/state/ax-conwip/records` | the `wf_*.json` run records to derive from, and to WATCH | +| `stateDir` | path | `~/.local/state/ax-conwip` | this module's own state root | + +Three of those defaults are choices worth defending: + +- **`serverUrl` defaults to loopback**, specifically the address the mock stack + listens on by default (`ax-mockstack -addr 127.0.0.1:8080`). No default here + points at a live host and none ever should: an accidental enable on a box with + no mock stack running reaches nothing at all. On the coordinator the live + ax-server proxy listens on `myAxFleet.apiListen` (127.0.0.1:8099), never on + this default; ax-fleet-topology asserts that. Note that this is a gRPC target + and not a URL. The transport is plain h2c with insecure credentials, so there + is no scheme to write. +- **`wipCap` defaults to 1**, which is stricter than the program's own default + of 2 and stricter than the seat dry run's 3. A cap is the one number where the + conservative default costs only throughput. +- **`recordsDir` defaults to a path that does not exist**, so an accidental + enable fails legibly instead of passing silently over an empty directory. + +### The seat meters are an input and only an input + +`metersDir` defaults to the REWRITE's meters directory, +`~/.local/state/tally-rewrite/meters` — the one `home/seat-feeder.nix` declares +and its three timers write. Not `~/.local/state/tally/meters`, which is branch +(a)'s and is pinned by `SHA256SUMS`. The two estates never share a path. + +This module **never writes into either**. It declares no tmpfiles rule over the +meters directory and does not create it; `seat-feeder` owns that directory's +existence and its mode. The scheduler opens those files read-only. The flake +check asserts that no tmpfiles rule naming `ax-conwip` exists while the gate is +off. + +## The sequence to enable it + +Each step is separate and each is reversible. Do them in order. + +1. **Decide the host.** The seat meters are per-user and the feeder timers that + write them run on the coordinator only, so the coordinator is the only host + where the default `metersDir` has anything in it. + +2. **Put run records where the scheduler will look**, or point `recordsDir` at + where they already are. The default path does not exist; `src/serve.ts` exits + 2 without `--records`, and exits 2 again if the directory it is pointed at + does not exist. Records may also be dropped in AFTER the service is running: + that is the whole point of the serve entry point, and a file is derived + exactly once, so a record that is edited in place is not re-admitted. + +3. **Set the option**, in the host's home-manager configuration: + + ```nix + myAxConwip = { + enable = true; + serverUrl = "127.0.0.1:8099"; # or wherever the ax server actually listens + wipCap = 1; + }; + ``` + +4. **Edit `checks.x86_64-linux.ax-conwip-topology` in `flake.nix` in the same + commit.** Its `enable == false` assertion goes red on the flip, deliberately, + so that no gate on this fleet moves without a reviewer seeing it. Do not + delete the check; narrow it to the hosts that are still off. Keep the + flipped-unit assertions that follow it: they pin that `ExecStart` names + `src/serve.ts`, carries `--meters`, does NOT carry `--live`, and that the + flipped unit still has no `Install` section. + +5. **Check it evaluates before rebuilding anything:** + + ``` + nix build --no-link .#checks.x86_64-linux.ax-conwip-topology + nix eval .#nixosConfigurations.coordinator.config.home-manager.users.tom.systemd.user.services \ + --apply builtins.attrNames + ``` + + The second should now list `ax-conwip`. While the gate is off it does not, + which is this module's whole present claim. + +6. **Rebuild.** The usual switch, which is Tom's. + +7. **Look at the unit before starting it.** It has no `Install` section, so the + rebuild declares it and nothing wants it: + + ``` + systemctl --user cat ax-conwip.service + systemctl --user status ax-conwip.service # inactive (dead), as expected + ``` + +8. **Start it by hand, once, and watch it:** + + ``` + systemctl --user start ax-conwip.service + journalctl --user -u ax-conwip.service -f + ``` + + The ledger is printed to stdout and therefore lands in the journal. v1 keeps + it in memory and writes nothing to `stateDir`. + +9. **Stop it when you are done looking.** Nothing wants it, so a stop is final + until the next manual start. + +## What is deliberately not here + +- No flake input, because the repository has no remote. +- No package, because packaging follows the input. +- No timer, because the release signal is a stream. +- Nothing enabled and nothing armed: the gate is off, and even with the gate on + the unit has no `Install` section. +- No system-bus twin. This is a per-user scheduler reading per-user seat meters + and it must never acquire a system unit. The check asserts that on all four + hosts, the NAS included. +- No write to any meters directory, by this module or by the program. + +## Related + +- `home/seat-feeder.nix` — owns the rewrite's meters directory and its three + feeder timers. R44: read-only from here. +- `modules/ax-client.nix` — the `myAxClient` gate that puts `kubectl` and the + `ax` binaries on a host. Also off, also #454. +- dotfiles#455 — `nix flake check --no-build` aborts at + `nixosConfigurations.client` before the `checks` output is reached. Until that + is fixed, the check on this page must be built by hand: + `nix build --no-link .#checks.x86_64-linux.ax-conwip-topology`. + +## Unknowns and proposed defaults + +- **Whether the scheduler should read the seat meters at all in v1.** The + program's `DESIGN.md` section 10 lists "seat meters as an admission input" + under what was DROPPED for v1: it admits on slots and edges only, and the + meter refusal rule lives in the seat dry run, not in the loop. The module + passes `metersDir` in as `AX_CONWIP_METERS` anyway. Proposed default: keep + passing it, because the path is the thing worth declaring and reviewing, and + an unread environment variable costs nothing. Revisit when the loop grows a + meter gate. +- **Whether `recordsDir` should default to a real directory.** Proposed + default: no. A path that does not exist is a legible failure; an empty + directory that does exist is a silent pass. +- **Whether the ledger should be written to `stateDir` rather than printed.** + Proposed default: printed, for as long as v1 keeps it in memory. When it is + written, `stateDir` is where it goes and the tmpfiles rule already declared + under the gate is what creates the directory. +- **Whether `Restart` should stay `no`.** Proposed default: yes, until the + scheduler can reconcile against Tasks already in flight on the ax server. A + restart that re-derives and re-admits is worse than a process that stays dead + and legible. +- **Which host, if the gate is ever flipped.** Proposed default: the + coordinator, because it is the only host whose feeder timers write the meters + directory. Not measured against any real need; nobody has asked for this to + run yet. diff --git a/flake.lock b/flake.lock index a14843125..60712a1b5 100644 --- a/flake.lock +++ b/flake.lock @@ -737,6 +737,22 @@ "type": "github" } }, + "nixpkgs-go": { + "locked": { + "lastModified": 1790096505, + "narHash": "sha256-UbSc81aqsHU4mOficKbMitf3S7gFnJkYDAKfprVS1R8=", + "owner": "NixOS", + "repo": "nixpkgs", + "rev": "a251c42236bbff9f870fcdc513dac5873009c304", + "type": "github" + }, + "original": { + "owner": "NixOS", + "repo": "nixpkgs", + "rev": "a251c42236bbff9f870fcdc513dac5873009c304", + "type": "github" + } + }, "nixpkgs-lib": { "locked": { "lastModified": 1774748309, @@ -880,6 +896,7 @@ "nixos-hardware": "nixos-hardware", "nixpkgs": "nixpkgs_5", "nixpkgs-fresh": "nixpkgs-fresh", + "nixpkgs-go": "nixpkgs-go", "nixpkgs-paperless": "nixpkgs-paperless", "nixpkgs-stable": "nixpkgs-stable", "piri": "piri", diff --git a/flake.nix b/flake.nix index 9155f32a2..b385a43ca 100644 --- a/flake.nix +++ b/flake.nix @@ -27,6 +27,26 @@ # writing the lock. A plain local build uses the reviewed fallback revision. nixpkgs-fresh.url = "github:NixOS/nixpkgs/nixos-unstable-small"; + # nixpkgs-go — pins ONE attribute, `go_1_27`, for ONE package, pkgs/ax. + # Same shape and same reasoning as nixpkgs-paperless below: a single + # upstream that needs a version no pin this flake already carries can + # supply, named here so it is reviewable rather than hidden in the package. + # + # google/ax v0.3.0's go.mod opens `go 1.27.1`, and Go refuses outright to + # build a module whose `go` directive is newer than the running toolchain + # (`go: go.mod requires go >= 1.27.1 (running go 1.27.0; GOTOOLCHAIN=local)`), + # with no network in the sandbox to fetch one. MEASURED 2026-09-23: + # nixpkgs go 1.26.5, go_1_27 1.27rc2 too old + # nixpkgs-fresh go 1.26.7, go_1_27 1.27.0 too old, by one patch release + # this input go_1_27 1.27.1 exact + # + # Pinned BY REVISION, not by branch, deliberately: nixpkgs-fresh is a rolling + # resolver whose whole job is to advance, and a toolchain pin has the opposite + # job. Retire this input the moment nixpkgs-fresh's go_1_27 reaches 1.27.1 — + # `nix eval .#inputs.nixpkgs-fresh.legacyPackages.x86_64-linux.go_1_27.version` + # is the whole test — and point overlays/default.nix back at it. + nixpkgs-go.url = "github:NixOS/nixpkgs/a251c42236bbff9f870fcdc513dac5873009c304"; + # nixpkgs-stable — pins ONLY nixosConfigurations.nas (issue #135 ruling): # the NAS is a frozen self-sustaining appliance on standard stable nixpkgs, # maintained manually every few years. It never rides the unstable @@ -571,6 +591,9 @@ overlays.default = import ./overlays { torchRocm = inputs.nix-strix-halo.packages.${system}.torch-rocm; + # One attribute out of the nixpkgs-go input, for pkgs/ax only. An + # overlay cannot read `inputs`, so it is passed like torchRocm above. + go127 = inputs.nixpkgs-go.legacyPackages.${system}.go_1_27; }; nixosConfigurations = { @@ -646,6 +669,11 @@ speech-session parakeet-service academic-ocr + # `nix build .#ax` — the ax control plane's four binaries. Exposed + # because nothing installs it by default (modules/ax-client.nix + # lands with its gate OFF on every host), so this is the only way + # to build or inspect it without flipping a gate first. + ax brother-print-text call-diarize browser-desktop @@ -679,8 +707,39 @@ ; live-iso = strixAi.live-iso; nas-installer-iso = nasInstaller.config.system.build.isoImage; + # The second half of the ax-fleet kill switch (pkgs/ax-fleet-teardown). + ax-fleet-teardown = pkgs.callPackage ./pkgs/ax-fleet-teardown { + k3s = inputs.nixpkgs.legacyPackages.${system}.k3s_1_36; + }; + # The fleet's ax Task image (OCI layout plus `digest`), built exactly + # as modules/ax-fleet/ax.nix builds the one the NAS seeds: the flake's + # nixpkgs, the patched `ax`, pi from llm-agents. pi only, no claude-code. + ax-agent-image = inputs.nixpkgs.legacyPackages.${system}.callPackage ./pkgs/ax-agent-image { + inherit (self.packages.${system}) ax; + inherit (inputs.llm-agents.packages.${system}) pi; + }; + # Upstream Agent Substrate d277088b: the six component images, the + # third-party images by digest and the gVisor asset (pkgs/substrate). + substrate-images = pkgs.callPackage ./pkgs/substrate/images.nix { + substrate = pkgs.callPackage ./pkgs/substrate { + go_1_27 = inputs.nixpkgs-go.legacyPackages.${system}.go_1_27; + }; + }; }; + # The same teardown the control and harness roles keep on their PATH + # (`sudo ax-fleet-teardown` after `myAxFleet.enable = false` and a + # switch). This app is for a generation rollback from a checkout, where + # the older generation predates the package: on the coordinator + # `sudo nix run ~/dotfiles#ax-fleet-teardown`; for the NAS, which holds + # no checkout, `nix copy --to ssh-ng://nas .#ax-fleet-teardown`, then + # `ssh -t nas sudo /bin/ax-fleet-teardown`. + apps.${system}.ax-fleet-teardown = { + type = "app"; + program = "${self.packages.${system}.ax-fleet-teardown}/bin/ax-fleet-teardown"; + meta.description = "Tear down ax-fleet's k3s leftovers and restore the pre-k3s sysctls"; + }; + # `nix build .#models.` retired with the 2026-08-21 "weights leave # nix" ruling: weights are no longer derivations, so there is nothing to # build — the NAS Library and library-fetch own materialization now. @@ -704,6 +763,22 @@ # The RAW out-of-store dotfiles are never checked at switch, so check them here. checks.${system} = { + # ax on the fleet (modules/ax-fleet, DESIGN.md 12). ax-fleet is the + # 4-VM switch/rollback proof, ax-fleet-boot the NAS-from-boot proof, + # ax-fleet-topology the evaluation-only assertions over the real hosts. + ax-fleet = import ./tests/ax-fleet { + inherit pkgs inputs; + inherit (nixpkgs) lib; + }; + ax-fleet-boot = import ./tests/ax-fleet-boot { + inherit pkgs inputs; + inherit (nixpkgs) lib; + }; + ax-fleet-topology = import ./tests/ax-fleet-topology { + inherit pkgs self; + inherit (nixpkgs) lib; + }; + # DF-5: the vendored Cargo.lock must equal upstream's at the pinned rev. # A build-time comparison, not an evaluation-time read, so evaluation # never needs the fetched source (no import-from-derivation). @@ -753,6 +828,154 @@ python3 repo/tests/qwen-speech/test_speech.py touch "$out" ''; + # ax-client-topology — the gate's rendered shape (modules/ax-client.nix, + # #453). `nix flake check --no-build` on its own proves only that + # the tree EVALUATES, and it would stay green through a merge resolution + # that dropped ../../modules/ax-client.nix from a host's imports, that + # flipped a gate, or that let the module reach hosts/nas. Every assertion + # below is eval-time, so each runs under --no-build: + # - the option EXISTS on the three interactive hosts, which is what + # proves the import survived (a dropped import makes the option + # undefined, not false); + # - it is FALSE on all three. This is the line that goes red on the + # flip, deliberately: the flip edits this check in the same commit, + # so no gate on this fleet can move without a reviewer seeing it; + # - kubectl and ax are therefore absent from all three systemPackages, + # asserted directly rather than inferred from the gate; + # - the NAS carries no myAxClient option at all — it does not import + # the module, it is pinned to nixpkgs-stable, and it is an appliance. + ax-client-topology = + let + hostCfg = host: self.nixosConfigurations.${host}.config; + gated = [ + "coordinator" + "worker" + "client" + ]; + in + assert builtins.all (host: (hostCfg host) ? myAxClient) gated; + # ON on the coordinator only, as a mkDefault consequence of + # myAxFleet's harness role (modules/ax-fleet/default.nix); OFF on the + # worker (inference role) and the client (no fleet role). + assert (hostCfg "coordinator").myAxFleet.enable && (hostCfg "coordinator").myAxFleet.role == "harness"; + assert (hostCfg "coordinator").myAxClient.enable; + assert builtins.all (host: (hostCfg host).myAxClient.enable == false) [ + "worker" + "client" + ]; + assert builtins.all ( + host: + !(builtins.elem pkgs.kubectl (hostCfg host).environment.systemPackages) + && !(builtins.elem pkgs.ax (hostCfg host).environment.systemPackages) + ) [ + "worker" + "client" + ]; + assert !((hostCfg "nas") ? myAxClient); + pkgs.runCommand "ax-client-topology" { } '' + touch "$out" + ''; + + # ax-conwip-topology — the CONWIP gate's rendered shape + # (home/ax-conwip.nix, PR #454). Home Manager gives no `assertions` + # option, so the invariants over the RENDERED user units live here, + # the same reasoning as tally-filler-topology and tally-pump-topology + # further down this file. + # + # Every assertion below is eval-time and sits in front of the + # runCommand, so each runs under `--no-build`. What each one is for: + # - the option EXISTS on all three home-manager hosts, which is what + # proves the import survived: dropping ./ax-conwip.nix from + # home/home.nix's imports makes the option UNDEFINED, not false, + # and turns the first assert red. That is the mutation hint; + # - it is FALSE on all three, and this is the line that goes red on + # the flip, deliberately, so the flip edits this check in the same + # commit and no gate on this fleet moves without a reviewer; + # - NO `ax-conwip` user unit is rendered anywhere — service, timer or + # socket — asserted directly over the rendered attrsets rather than + # inferred from the gate, because `lib.mkIf false` removing the key + # is the property under test, not an assumption; + # - no system-bus twin: this is a per-user scheduler reading per-user + # seat meters, and it must never acquire a system unit; + # - it writes no tmpfiles rule while off, and in particular declares + # nothing over the REWRITE's meters directory, which belongs to + # home/seat-feeder.nix (R44) and is an input to this module and + # never an output; + # - the NAS carries no home-manager at all (flake.nix:603, + # `withHomeManager = false`), so it cannot carry this option; the + # assert pins that rather than leaving it to be rediscovered. + ax-conwip-topology = + let + homeHosts = [ + "coordinator" + "worker" + "client" + ]; + homeCfg = host: self.nixosConfigurations.${host}.config.home-manager.users.tom; + hostCfg = host: self.nixosConfigurations.${host}.config; + in + # the import survived, on every host that has home-manager. + assert builtins.all (host: (homeCfg host) ? myAxConwip) homeHosts; + # and the gate is OFF on every one of them. + assert builtins.all (host: (homeCfg host).myAxConwip.enable == false) homeHosts; + # therefore NOTHING is rendered: no service, no timer, no socket. + assert builtins.all (host: !((homeCfg host).systemd.user.services ? ax-conwip)) homeHosts; + assert builtins.all (host: !((homeCfg host).systemd.user.timers ? ax-conwip)) homeHosts; + assert builtins.all (host: !((homeCfg host).systemd.user.sockets ? ax-conwip)) homeHosts; + # no system-bus twin, on any host, including the NAS. + assert builtins.all ( + host: + !((hostCfg host).systemd.services ? ax-conwip) && !((hostCfg host).systemd.timers ? ax-conwip) + ) (homeHosts ++ [ "nas" ]); + # no tmpfiles rule of its own while off, and nothing at all naming + # the meters directory it only ever reads. + assert builtins.all ( + host: + !(builtins.any ( + r: nixpkgs.lib.hasInfix "ax-conwip" r + ) (homeCfg host).systemd.user.tmpfiles.rules) + ) homeHosts; + # the NAS has no home-manager, so it cannot carry the option. + assert !((hostCfg "nas") ? home-manager); + # AND, with the gate flipped IN MEMORY through extendModules — which + # changes no rendered byte on any host and writes nothing anywhere — + # the argument list the unit would run is the one the program accepts. + # + # Until 2026-09-23 `ExecStart` ran `src/cli.ts`, which reads the + # records directory once, prints its ledger and exits: this file + # declared a long-running service whose program was a oneshot in + # everything but name. It now runs `src/serve.ts`, which polls, + # admits under a cap that holds for the life of the process, reads + # the meters directory, and exits 0 on SIGTERM. These asserts are + # what keep the module's argument list and the program's contract + # from drifting apart again, and they are cheap and eval-time: + # - the entry point IS src/serve.ts and is NOT src/cli.ts; + # - `--meters` is passed, so the AX_CONWIP_METERS this unit already + # set is no longer read by nothing; + # - `--live` is ABSENT. The live dispatch path is gated three ways + # inside the program and this module must never be one of the + # ways in; + # - there is still NO Install section on the flipped unit, so + # declaring it and arming it stay two separate acts. + assert ( + let + flipped = self.nixosConfigurations.coordinator.extendModules { + modules = [ { home-manager.users.tom.myAxConwip.enable = true; } ]; + }; + unit = flipped.config.home-manager.users.tom.systemd.user.services.ax-conwip; + raw = unit.Service.ExecStart; + exec = if builtins.isList raw then builtins.concatStringsSep " " raw else raw; + in + nixpkgs.lib.hasInfix "src/serve.ts" exec + && !(nixpkgs.lib.hasInfix "src/cli.ts" exec) + && nixpkgs.lib.hasInfix "--meters" exec + && !(nixpkgs.lib.hasInfix "--live" exec) + && !(unit ? Install) + ); + pkgs.runCommand "ax-conwip-topology" { } '' + touch "$out" + ''; + qwen-speech-topology = let coord = self.nixosConfigurations.coordinator.config; diff --git a/home/ax-conwip.nix b/home/ax-conwip.nix new file mode 100644 index 000000000..706e165ef --- /dev/null +++ b/home/ax-conwip.nix @@ -0,0 +1,265 @@ +{ + config, + lib, + osConfig, + pkgs, + ... +}: +# ax-conwip — the CONWIP scheduler as ONE long-running user service, declared +# and OFF. +# +# UNIT: none yet; this is a skeleton, not a lane. ISSUE: dotfiles#455 is the +# gate's own bug (nix flake check never reaches `checks`), filed alongside this +# file; the module itself carries no issue of its own and rides PR #454. +# DATE: 2026-09-23. DOC: docs/ax-conwip.md. +# +# WHAT IT IS. `/home/tom/mecattaf/ax-conwip` is a CONWIP scheduler: it reads +# Claude ultracode workflow run records, derives one work item per agent entry, +# admits items under a fixed work-in-progress cap, dispatches each admitted item +# to an ax server as a Task over gRPC (`UpdateTask`, an upsert — ax v0.3.0 has +# no `CreateTask`), and gives the slot back when the Task reaches a phase the +# CONWIP calls terminal, which is `{Completed, Failed, Terminating}` and NOT +# ax's own `{Running, Completed, Failed}`. The output is an append-only ledger. +# Read that repository's DESIGN.md before touching anything here. +# +# WHY THIS FILE EXISTS. So the shape of running it is declared and reviewable +# BEFORE anything runs it, and so the argument list is data in this repository +# rather than a command line somebody types. Every number a running scheduler +# would obey — the cap, the server it dispatches to, the meters it reads — is an +# option below, set in one place, visible in one diff. +# +# THE SEAT METERS ARE AN INPUT AND ONLY AN INPUT. `metersDir` defaults to the +# REWRITE's meters directory, %h/.local/state/tally-rewrite/meters, the one +# home/seat-feeder.nix declares and its three timers write. Not +# ~/.local/state/tally/meters, which is branch (a)'s and is pinned by +# SHA256SUMS; the two estates never share a path. This module never writes into +# either: it declares no tmpfiles rule over the meters directory, it does not +# create it, and the scheduler opens those files read-only. seat-feeder owns +# that directory's existence and its mode, and R44 puts that file out of reach. +# +# FOUR THINGS THIS FILE DELIBERATELY DOES NOT DO. +# +# 1. NO FLAKE INPUT. `/home/tom/mecattaf/ax-conwip` is a local git repository +# with NO remote (MEASURED 2026-09-23: `git remote -v` is empty). There is +# nothing for `inputs.ax-conwip` to point at, and a `builtins.fetchGit` of +# a path on one box is not a declaration — it is a machine-local accident +# that would break every other host's eval. So the module names a +# DIRECTORY, `sourceDir`, and says out loud that it is a directory. +# 2. NO PACKAGE. There is no `pkgs/ax-conwip.nix` and no `.#ax-conwip`, +# because packaging follows the input, and there is no input. The unit +# below runs the checkout in place, through `pnpm exec tsx`, against the +# `node_modules` that checkout already carries. That is honest about what +# it is: a development shape, not a delivered one. It is the single +# biggest reason `enable` must stay false. +# 3. NO TIMER, and as of 2026-09-23 that is a statement about the PROGRAM and +# not only about this module. `ExecStart` runs `src/serve.ts`, which polls +# the records directory on its own interval and holds each admitted slot on +# a `WatchTask` stream until the CONWIP's release rule gives it back, so the +# cap holds for the life of the process. There is nothing for a timer to +# wake, and restarting on a cadence would re-derive and re-admit. Until this +# date the unit ran `src/cli.ts`, which reads the directory once and exits: +# a `Type = oneshot` in everything but name, and the mismatch this change +# closes. Compare home/seat-feeder.nix, which is all timers and no +# long-running anything, because its job is freshness. +# 4. NOTHING ENABLED, AND NOTHING ARMED EITHER. `enable` defaults to false and +# is set nowhere, so this module defines NO unit on any host today; the +# flake check `ax-conwip-topology` asserts exactly that. And even with the +# gate flipped the unit carries NO `Install` section, so it is declared and +# not wanted by anything: `systemctl --user start ax-conwip` is a second, +# separate, deliberate act. Both flips are Tom's. docs/ax-conwip.md carries +# the runbook. +# +# ONE OPTION BEYOND THE SIX THE BRIEF NAMED, said plainly: `recordsDir`. The +# scheduler's entry point refuses to start without `--records ` +# (src/serve.ts prints usage and exits 2), so a skeleton without it could not +# form a command line at all, and a skeleton that cannot form a command line is +# not a skeleton, it is a comment. Its default is deliberately a path that does +# not exist, and `src/serve.ts` exits 2 on a records directory that is absent +# rather than polling it forever, so an accidental enable produces a legible +# failure and not a silent pass over nothing. +# +# NO SEVENTH OPTION WAS ADDED. `src/serve.ts` also takes `--poll-interval-ms`, +# `--max-ticks`, `--staleness-bound-seconds` and `--live`, and this module +# passes none of them: every one has a conservative default in the program, and +# an option is a thing Tom has to review. `--live` in particular is not passed +# and must never grow a way to be. Dispatch is dry run by default, and the live +# path needs all three of `--live`, `AX_CONWIP_LIVE_HALOGEN=1` and a seat on the +# `["halogen"]` allow list in the program's own `src/seats.ts`. +let + cfg = config.myAxConwip; + + # A user unit inherits no interactive PATH and no profile. `node`, `pnpm` and + # `npm` are absent from PATH on both hosts anyway (MEASURED 2026-09-22), so + # naming the store paths here is not belt-and-braces, it is the only way the + # unit could ever run. + runtimePath = lib.makeBinPath [ + pkgs.nodejs_22 + pkgs.pnpm + pkgs.coreutils + ]; +in +{ + options.myAxConwip = { + enable = lib.mkEnableOption '' + the ax CONWIP scheduler as a user service. OFF, and set nowhere. Read + docs/ax-conwip.md before flipping this: the program is not packaged and + the unit runs a checkout in place + ''; + + serverUrl = lib.mkOption { + type = lib.types.str; + default = "127.0.0.1:8080"; + example = "127.0.0.1:8099"; + description = '' + The ax server the scheduler dispatches Tasks to, as `host:port`. This + is the program's own `--addr`, which is a gRPC target and NOT a URL: + the transport is plain h2c with insecure credentials, so there is no + scheme to write. The default is the LOOPBACK address the mock stack + listens on by default (`ax-mockstack -addr 127.0.0.1:8080`), chosen so + that an accidental enable on a box with no mock stack running reaches + nothing at all, and on a box with one running reaches only the mock. + No default here points at a live host, and none ever should. + On the coordinator the live ax-server proxy listens on + myAxFleet.apiListen (127.0.0.1:8099), never on this default + (ax-fleet-topology asserts they differ). + ''; + }; + + metersDir = lib.mkOption { + type = lib.types.path; + default = "${config.home.homeDirectory}/.local/state/tally-rewrite/meters"; + description = '' + The seat meter directory, read-only, the rewrite's and not branch + (a)'s. home/seat-feeder.nix declares it and its timers write it; this + module only reads it and declares no tmpfiles rule over it. + + It is passed BOTH as `AX_CONWIP_METERS` below and as the program's + `--meters` flag. Until 2026-09-23 only the environment variable was + set and nothing read it: `src/cli.ts` took no `--meters` and read no + variable. `src/serve.ts` reads `/.json` + once per tick and lets the pure refusal rule decide admission, and a + refusal costs throughput and never a slot. The scheduler opens those + files read-only and its ledger writer refuses any path under this + directory outright. + ''; + }; + + wipCap = lib.mkOption { + type = lib.types.ints.positive; + default = 1; + description = '' + The work-in-progress cap: how many Tasks may be admitted at once. The + cap is data and is never derived by the scheduler. One, deliberately, + which is stricter than the program's own default of 2 and stricter + than the seat dry run's 3: a cap is the one number where the + conservative default costs only throughput, and the expensive + direction is the other one. + ''; + }; + + sourceDir = lib.mkOption { + type = lib.types.path; + default = "/home/tom/mecattaf/ax-conwip"; + description = '' + Where the CONWIP program lives, as a DIRECTORY on this box, because it + is not packaged and there is no flake input to package it from: the + repository has no remote. The unit runs `pnpm exec tsx src/serve.ts` + with this as its working directory, against the `node_modules` that + checkout already carries. Nothing in the Nix store is built from it + and nothing here pretends otherwise. + ''; + }; + + recordsDir = lib.mkOption { + type = lib.types.path; + default = "${config.home.homeDirectory}/.local/state/ax-conwip/records"; + description = '' + The directory of `wf_*.json` ultracode run records the scheduler + derives work items from, and then WATCHES: `src/serve.ts` re-lists it + every poll interval and derives each new file exactly once, keyed by + absolute path. `src/serve.ts` exits 2 without this flag, and exits 2 + again if the directory does not exist, so the default below — a path + that does not exist today — makes an accidental enable fail legibly + instead of passing silently over nothing. + ''; + }; + + stateDir = lib.mkOption { + type = lib.types.path; + default = "${config.home.homeDirectory}/.local/state/ax-conwip"; + description = '' + This module's own state root, under %h/.local/state/. Nothing is + written here yet: `src/serve.ts` puts its jsonl ledgers under the + checkout's own gitignored `out/ledger/`, so run output never becomes + source. The directory is declared because the ledger belongs here once + the program is packaged, and because a missing directory turns a first + run into a silent no-op rather than a legible failure (the same + reasoning as dotfiles#292). + ''; + }; + }; + + # Gate OFF: mkIf false removes the attribute outright, so with `enable` + # unset this module contributes NO `systemd.user.services.ax-conwip` key at + # all, on any host. That is what `checks.ax-conwip-topology` asserts, and it + # is why adding this file to home/home.nix's imports changes no rendered + # byte on any host today. + config = lib.mkIf cfg.enable { + systemd.user.services.ax-conwip = { + Unit = { + Description = "ax CONWIP scheduler: admit ultracode work items under a fixed WIP cap"; + # Read back by `nix eval` the way home/seat-feeder.nix's X-Tally* keys + # are, so the declared cap and target cannot drift from whatever a + # future oracle reads. X- is systemd's extension space; the manager + # ignores these. + X-AxConwipCap = toString cfg.wipCap; + X-AxConwipServer = cfg.serverUrl; + X-AxConwipHost = osConfig.networking.hostName; + }; + + Service = { + # Long-running, not a oneshot, and `src/serve.ts` is the program that + # makes that true: it polls the records directory, admits under a cap + # that holds for the LIFE OF THE PROCESS, and gives a slot back only + # when the release rule says so. Restarting it silently would re-derive + # and re-admit. If it dies, that is a fact to read in the journal, not + # a thing to paper over. It exits 0 on SIGTERM, so a deliberate stop is + # not reported as a signal death. + Type = "simple"; + Restart = "no"; + Nice = 10; + WorkingDirectory = cfg.sourceDir; + Environment = [ + "PATH=${runtimePath}" + "AX_CONWIP_STATE=${cfg.stateDir}" + "AX_CONWIP_METERS=${cfg.metersDir}" + ]; + # The whole argument list, as data. Every value is an option above. + # Every flag here is one the program prints under `--print-flags`, and + # `checks.ax-conwip-topology` asserts the entry point and `--meters`. + # `--live` is absent and must stay absent. + ExecStart = lib.escapeShellArgs [ + "${pkgs.pnpm}/bin/pnpm" + "exec" + "tsx" + "src/serve.ts" + "--records" + "${cfg.recordsDir}" + "--addr" + "${cfg.serverUrl}" + "--cap" + "${toString cfg.wipCap}" + "--meters" + "${cfg.metersDir}" + ]; + }; + + # NO Install section, deliberately. See point 4 of the header: flipping + # the gate declares this unit; starting it is a separate act. + }; + + systemd.user.tmpfiles.rules = [ + "d ${cfg.stateDir} 0700 - - -" + ]; + }; +} diff --git a/home/home.nix b/home/home.nix index a71ca98ac..de197cb41 100644 --- a/home/home.nix +++ b/home/home.nix @@ -117,6 +117,7 @@ in imports = [ ./browser-trust.nix ./ai-memory.nix + ./ax-conwip.nix # gate OFF; defines no unit on any host. docs/ax-conwip.md ./client-apps.nix ./harness-records.nix ./herdr.nix diff --git a/hosts/client/default.nix b/hosts/client/default.nix index 17b326c0e..f30969747 100644 --- a/hosts/client/default.nix +++ b/hosts/client/default.nix @@ -79,6 +79,10 @@ inputs.nixos-hardware.nixosModules.common-pc-laptop inputs.nixos-hardware.nixosModules.common-pc-laptop-ssd ../../modules/zenbook-duo-daemon.nix + # kubectl + the google/ax binaries, behind myAxClient.enable. Imported on + # all three interactive hosts, OFF on all three; read that module's header + # for the runbook and for what it deliberately does not declare. + ../../modules/ax-client.nix ]; networking.hostName = "client"; @@ -94,6 +98,13 @@ # 2026-08-21 return. mySecrets.enable = true; + # OFF, and it lands OFF (modules/ax-client.nix). There is no cluster on this + # fleet to point kubectl at and no Agent Substrate for ax to delegate to, so + # flipping this today installs two binaries with nothing to talk to. The flip + # is Tom's, one host at a time, and ax-client-topology in flake.nix goes red + # on it by design. + myAxClient.enable = false; + # ── names ────────────────────────────────────────────────────────────────── # `nas` is fleet-wide (modules/common.nix). The two twins are pinned here # by hand because modules/fleet-hosts.nix is twins-only and also deletes diff --git a/hosts/coordinator/default.nix b/hosts/coordinator/default.nix index 9467e4440..f37e13a2b 100644 --- a/hosts/coordinator/default.nix +++ b/hosts/coordinator/default.nix @@ -75,15 +75,41 @@ # this: it carries its own pins in hosts/nas/network.nix and keeps the # stock loopback mapping. ../../modules/fleet-hosts.nix + # ax on the fleet (2026-09-23): the HARNESS node, see myAxFleet below. + ../../modules/ax-fleet # The REWRITE kernel (github.com/mecattaf/tally, U-B1…U-B13) as one system # service against ~/.local/state/tally-rewrite/, coexisting with the live # user-bus tally-daemon.service (U-D13). Declared here, installed by U-D19's # switch — never hand-started (DEFERRED.md DF-U-D13-1). ../../modules/tally-b.nix + # kubectl + the google/ax binaries, behind myAxClient.enable. Imported on + # all three interactive hosts, OFF on all three; read that module's header + # for the runbook and for what it deliberately does not declare. + ../../modules/ax-client.nix ]; networking.hostName = "coordinator"; + # ── ax on the fleet: THE kill switch for this host ───────────────────── + # The HARNESS node (modules/ax-fleet/harness.nix): a k3s agent tainted + # ate.dev/sandboxClass=gvisor:NoSchedule, so only atelet and the gVisor + # WorkerPool land here. "agent harnesses on coordinator" (Tom, 2026-09-23). + # Switch the NAS first. `false`, switch, then `sudo ax-fleet-teardown` + # (on PATH whatever the switch says) is the whole rollback. This + # also turns myAxClient (kubectl, ax) on by mkDefault. + myAxFleet = { + enable = true; + role = "harness"; + lan = { + interface = "wlp192s0"; + address = "10.42.0.2"; + # The wired port: "Wired connection 1" autoconnects with DHCP (MEASURED + # nmcli, 2026-09-23). The guard covers it and it never takes the LAN + # routes from the wifi (fix round 3). + extraInterfaces = [ "enp191s0" ]; + }; + }; + # gVisor runsc for direct rootless `runsc run` jobs (modules/gvisor.nix). # OFF until Tom flips it; the lane recommends the worker first. myGvisor.enable = false; @@ -115,6 +141,10 @@ # backend; flips with the NAS's myNas.paperless.enable (2026-09-13). myNasClient.relayPaperless = true; + # myAxClient (kubectl + ax) is ON here by mkDefault from myAxFleet's + # harness role (modules/ax-fleet/default.nix); ax-client-topology in + # flake.nix pins that. + # The rewrite's served kernel: ONE kernel, on the coordinator (spec §2.4 Q2 — # the worker twin is a ROW this kernel serves, not a second kernel), on the # system bus, against the rewrite's own state root. The live daemon on tom's diff --git a/hosts/nas/default.nix b/hosts/nas/default.nix index 7b6de7252..34895451a 100644 --- a/hosts/nas/default.nix +++ b/hosts/nas/default.nix @@ -46,6 +46,12 @@ ./tailscale-personal.nix # Additional isolated SaaS ingress; never enroll the lent laptops here ./personal-https.nix # Gated, NAS-scoped DNS-01 certificates for private media ./headscale-backup.nix # consistent identity backup before overseas handover + # ax on the fleet (2026-09-23): this box is the CONTROL node. k3s server + # with its kubelet, Substrate's and ax's control planes, the registry. + # "hypervisor on NAS" (Tom, 2026-09-23). The CIDR-overlap assertions in + # that module evaluate whether or not the switch below is on. + ../../modules/ax-fleet + ./substrate-link.nix # 2026-09-23: Cloudflare floor (Substrate) -> ax link, outbound only; gate OFF (LINK-DESIGN.md) ../../modules/adguardhome.nix inputs.nixos-hardware.nixosModules.common-cpu-amd inputs.nixos-hardware.nixosModules.common-pc @@ -121,6 +127,24 @@ myNas.tailscalePersonal.funnel.policyApproved = true; myNas.headscale.serverUrl = "https://nas-saas.tail8dd1.ts.net:8443"; myNas.headscale.backup.enable = true; + + # ── ax on the fleet: THE kill switch for this host ───────────────────── + # One line. `false`, switch (from the coordinator, --target-host nas), then + # `ssh -t nas sudo ax-fleet-teardown` (k3s-killall.sh, the guard table and + # the sysctl restore). The teardown stays on this host's PATH with the + # switch off; there is no dotfiles checkout here. It removes every trace but + # the data left on purpose under /mnt/fast/k3s and /mnt/nas/services/ax-fleet. + # The k3s credentials are the agenix secrets secrets/k3s-token.age (server, + # this host only) and secrets/k3s-agent-token.age (mySecrets is on here). + # Switch order: this host first, then the coordinator. + myAxFleet = { + enable = true; + role = "control"; + lan = { + interface = "enp1s0"; + address = "10.42.0.1"; + }; + }; # Retired 2026-09-16: the Dell belongs to its owner; Tom no longer # publishes or manages Omarchy updates. Keep historical receipts only. myNas.omarchyUpdateCenter.enable = false; diff --git a/hosts/nas/substrate-link.nix b/hosts/nas/substrate-link.nix new file mode 100644 index 000000000..7d3692137 --- /dev/null +++ b/hosts/nas/substrate-link.nix @@ -0,0 +1,261 @@ +{ + config, + options, + lib, + pkgs, + ... +}: +# substrate-link: the NAS side of the Cloudflare floor (Substrate) to ax link, declared and OFF. +# +# DESIGN: ~/today/evals-2026-09-23/link/LINK-DESIGN.md (2026-09-23), BUILD.md beside it. CODE: vendored +# in pkgs/substrate-link/src from ax-conwip eval/2026-09-23-link a021003 apps/link (pkgs/substrate-link/SYNC.md). +# Formerly conwip-link; renamed with the ax-conwip project (now "substrate", Tom 2026-09-23 E10). DATE: 2026-09-23. +# Critique items carried here: N1 (axServer from ax-fleet), N2 (Node 24, pkgs), N3 (guest URL), N4 (floorUrls). +# +# WHAT IT IS. One long-running process that dials OUT to the floor (a Durable Object behind a +# Cloudflare Worker), leases AgentJobs under the floor's CONWIP cap, creates one ax Task per lease +# (GetTask, then UpdateTask only on NotFound: ax has no CreateTask and UpdateTask is a blind upsert), +# watches the Tasks by ListTasks, and reports each verdict through a write-ahead outbox. Nothing +# dials in: no tunnel, no listener, no port opened in this file. +# +# WHY A SYSTEM UNIT ON THE NAS HOST AND NOT A POD. The link needs no Kubernetes API (ax is gRPC), it +# must keep reporting "ax is down" to the floor while k3s is down, its journal must outlive the +# cluster, and its bearer must never become a Kubernetes Secret: ax-controller's ClusterRole reads +# secrets cluster-wide (ax-fleet DESIGN section 9). ax-server's ClusterIP answers host processes on +# a k3s node through kube-proxy, the same path the coordinator's socket proxy uses (DESIGN D13). +# +# THE TOKEN IS A PATH, NEVER A VALUE. agenix decrypts floor-link-token.age for root only (0400); +# LoadCredential hands the unit a private copy under $CREDENTIALS_DIRECTORY. The program reads it +# once and never logs it. Minting the token (a Worker secret and this age file) is Tom's hand. +# +# THE GATE. `enable` defaults to false and nothing in this repository sets it. Arming needs, in +# order: the ax-fleet PR switched on the NAS, the sealed token, the floor URL, a guest Complete +# relay (completion = guest on zero-patch ax), and Tom's ack. Exit 78 (bad token or config) stops the +# restart loop; exit 75 (a second replica holds the session, or an endpoint switch to a URL in +# floorUrls was persisted) retries after RestartSec. +let + cfg = config.myNas.substrateLink; + inherit (lib) mkEnableOption mkIf mkOption types; + # N1: one source for ax-server's address. When modules/ax-fleet is imported its pinned ClusterIP wins. + axFleetIP = if options ? myAxFleet && options.myAxFleet ? axServerClusterIP then config.myAxFleet.axServerClusterIP else null; + # N3 (critique A5): a guest must post to a fleet-internal relay, never to a public Workers host. + publicWorkersHost = u: lib.hasInfix ".workers.dev" u || lib.hasInfix ".pages.dev" u; + # Review round 2: mirrors isFleetInternalUrl (apps/link/src/link.ts). Private ranges match only IPv4 literals + # (10.evil.example and 127.0.0.1.nip.io are names), names only under .internal/.lan/.local/.home.arpa, IPv6 only + # ::1 and ULA, a single label only when listed in internalHosts. Stricter than the program where they differ. + hostOf = u: let m = builtins.match "[a-z]+://([^@/]*@)?(\\[[^]]*]|[^/:?#]+).*" (lib.toLower u); in if m == null then "" else builtins.elemAt m 1; + ipv4Of = h: builtins.match "([0-9]{1,3})\\.([0-9]{1,3})\\.([0-9]{1,3})\\.([0-9]{1,3})" h; + privateV4 = o: let a = lib.toInt (builtins.elemAt o 0); b = lib.toInt (builtins.elemAt o 1); in + a == 127 || a == 10 || (a == 192 && b == 168) || (a == 172 && b >= 16 && b <= 31) || (a == 100 && b >= 64 && b <= 127); + fleetInternalUrl = u: let h = hostOf u; o = ipv4Of h; in + if o != null then privateV4 o + else if lib.hasPrefix "[" h then h == "[::1]" || builtins.match "\\[f[cd][0-9a-f]{2}:.*" h != null + else h == "localhost" || lib.elem h cfg.internalHosts || builtins.match "([a-z0-9-]+\\.)+(internal|lan|local|home\\.arpa)" h != null; +in +{ + options.myNas.substrateLink = { + enable = mkEnableOption "the Substrate floor-to-ax link (outbound only). OFF; see this file's header before flipping it"; + + package = mkOption { + type = types.package; + default = pkgs.callPackage ../../pkgs/substrate-link { }; + defaultText = lib.literalExpression "pkgs.callPackage ../../pkgs/substrate-link { }"; + description = "The link program, built from the source vendored in pkgs/substrate-link/src."; + }; + + floorUrl = mkOption { + type = types.str; + default = ""; + example = "https://substrate.example.dev"; + description = "The floor Worker's base URL. HTTPS only; the link appends /rpc."; + }; + + floorUrls = mkOption { + type = types.listOf types.str; + default = [ cfg.floorUrl ]; + defaultText = lib.literalExpression "[ config.myNas.substrateLink.floorUrl ]"; + description = "N4 (B18): the only floor URLs a Lease `endpoint` may move this link to, exact https matches. The switch is persisted in the state directory and the unit restarts onto it (exit 75)."; + }; + + holder = mkOption { + type = types.str; + default = "nas-link-1"; + description = "holderIdentity. The floor binds it to this link's token; one session per identity."; + }; + + axServer = mkOption { + type = types.str; + default = if axFleetIP != null then "${axFleetIP}:8080" else "10.201.0.80:8080"; + defaultText = lib.literalExpression ''"''${config.myAxFleet.axServerClusterIP}:8080" when modules/ax-fleet is imported, else "10.201.0.80:8080"''; + description = "ax-server as host:port (gRPC h2c, no auth upstream, #376). N1: read from ax-fleet's axServerClusterIP option (D13)."; + }; + + atespace = mkOption { + type = types.str; + default = "fleet"; + }; + + image = mkOption { + type = types.str; + default = "ax-agent"; + example = "localhost:5000/ax-agent@sha256:0000"; + description = "Task image; set it to the digest ax-fleet-image-ref prints."; + }; + + gateway = mkOption { + type = types.str; + default = "halogen"; + description = "The egress Gateway every Task names. A missing Gateway means open egress on v0.3.0, so the link leases nothing while GetGateway answers NotFound or the Gateway allows * (B10)."; + }; + + maxInFlight = mkOption { + type = types.ints.positive; + default = 2; + description = "Sandbox cap this link fills: the ateom-gvisor WorkerPool replicas (ax-fleet section 9)."; + }; + + servedLabels = mkOption { + type = types.listOf types.str; + default = [ + "seat:halogen" + "runtime:gvisor" + ]; + description = "runs-on labels this link accepts. Claude seats stay on the coordinator until Tom rules (W6)."; + }; + + seatCommands = mkOption { + type = types.attrsOf (types.listOf types.str); + default = { + halogen = [ + "ax-agent" + "pi" + ]; + }; + description = "Task command per seat (the ax-agent adapter's modes, ax-fleet section 10.2)."; + }; + + completion = mkOption { + type = types.enum [ + "auto" + "p1" + "guest" + ]; + # The fleet runs stock ax with no P1 (pkgs/ax: patches = [ ]), so "p1" and "auto" lease + # nothing (B6). "guest" is the only mode that completes on this fleet; it needs guestCompleteUrl, + # a fleet-internal relay that does not exist yet, so arming fails its assertion until then. + default = "guest"; + description = "p1: the controller writes the outcome (carried patch P1); the link probes GetTaskResult at start and whenever ax comes back, and leases nothing on a server without P1 (B6). auto: the same, never guest. guest: the L7 reserve, pre-P1, the guest completes with a per-lease token through a fleet-internal relay."; + }; + + guestCompleteUrl = mkOption { + type = types.nullOr types.str; + default = null; + description = "Only for completion = guest: the fleet-internal URL the guest posts its per-lease Complete to (N3: never a workers.dev host)."; + }; + + internalHosts = mkOption { + type = types.listOf types.str; + default = [ ]; + description = "Review round 2: single-label host names a guestCompleteUrl may use (fleet DNS names without a dot)."; + }; + + tokenAgeFile = mkOption { + type = types.path; + default = ../../secrets/floor-link-token.age; + defaultText = lib.literalExpression "../../secrets/floor-link-token.age"; + description = "The sealed per-link bearer (secrets.nix: editors ++ nasOnly). Evaluated only when enabled."; + }; + }; + + config = mkIf cfg.enable { + assertions = [ + { + assertion = lib.hasPrefix "https://" cfg.floorUrl; + message = "myNas.substrateLink.floorUrl must be an https:// URL (the bearer never travels in clear)."; + } + { + assertion = config.networking.hostName == "nas"; + message = "substrate-link runs on the hypervisor host only (placement ruling: NAS = k3s server, Substrate, ax-server)."; + } + { + assertion = cfg.completion != "guest" || (cfg.guestCompleteUrl != null && !(publicWorkersHost cfg.guestCompleteUrl) && fleetInternalUrl cfg.guestCompleteUrl); + message = "completion = guest needs a fleet-internal guestCompleteUrl: a private IPv4 literal, ::1 or ULA, or a name under .internal/.lan/.local/.home.arpa, never a workers.dev or pages.dev host (critique A5, review round 2)."; + } + { + assertion = lib.all (u: lib.hasPrefix "https://" u) cfg.floorUrls && lib.elem cfg.floorUrl cfg.floorUrls; + message = "myNas.substrateLink.floorUrls must be https:// URLs and include floorUrl."; + } + ]; + + age.secrets.floor-link-token = { + file = cfg.tokenAgeFile; + mode = "0400"; + }; + + systemd.services.substrate-link = { + description = "substrate-link: Cloudflare floor to ax, outbound only"; + wantedBy = [ "multi-user.target" ]; + wants = [ "network-online.target" ]; + after = [ + "network-online.target" + "k3s.service" + ]; + environment = { + LINK_FLOOR_URL = cfg.floorUrl; + LINK_FLOOR_URLS = lib.concatStringsSep "," cfg.floorUrls; + LINK_HOLDER = cfg.holder; + AX_SERVER = cfg.axServer; + AX_ATESPACE = cfg.atespace; + LINK_IMAGE = cfg.image; + LINK_GATEWAY = cfg.gateway; + LINK_MAX_IN_FLIGHT = toString cfg.maxInFlight; + LINK_SERVED_LABELS = lib.concatStringsSep "," cfg.servedLabels; + LINK_SEAT_COMMANDS = builtins.toJSON cfg.seatCommands; + LINK_COMPLETION = cfg.completion; + LINK_INTERNAL_HOSTS = lib.concatStringsSep "," cfg.internalHosts; + } + // lib.optionalAttrs (cfg.guestCompleteUrl != null) { LINK_GUEST_COMPLETE_URL = cfg.guestCompleteUrl; }; + serviceConfig = { + ExecStart = lib.getExe cfg.package; + DynamicUser = true; + StateDirectory = "substrate-link"; # journal.jsonl, session-id, floor-endpoint; a few fsynced lines per job + StateDirectoryMode = "0700"; + LoadCredential = [ "floor-link-token:${config.age.secrets.floor-link-token.path}" ]; + Restart = "on-failure"; + RestartSec = "10s"; + RestartPreventExitStatus = [ 78 ]; + TimeoutStopSec = "30s"; # SIGTERM drains: no new leases, outbox flushed; ax Tasks keep running + UMask = "0077"; + MemoryMax = "512M"; + NoNewPrivileges = true; + ProtectSystem = "strict"; + ProtectHome = true; + PrivateTmp = true; + PrivateDevices = true; + ProtectKernelTunables = true; + ProtectKernelModules = true; + ProtectKernelLogs = true; + ProtectControlGroups = true; + ProtectClock = true; + ProtectHostname = true; + LockPersonality = true; + RestrictRealtime = true; + RestrictSUIDSGID = true; + RestrictNamespaces = true; + SystemCallArchitectures = "native"; + SystemCallFilter = [ + "@system-service" + "~@privileged" + ]; + CapabilityBoundingSet = ""; + AmbientCapabilities = ""; + RestrictAddressFamilies = [ + "AF_INET" + "AF_INET6" + "AF_UNIX" + ]; + # MemoryDenyWriteExecute stays off: V8's JIT needs writable and executable pages. + }; + }; + }; +} diff --git a/hosts/worker/default.nix b/hosts/worker/default.nix index 8afa63dbb..53617cf89 100644 --- a/hosts/worker/default.nix +++ b/hosts/worker/default.nix @@ -80,10 +80,31 @@ # resolves to loopback, which every distributed library happily binds — the # rank-1-hangs-forever failure. The NAS must NOT import this. ../../modules/fleet-hosts.nix + # ax on the fleet (2026-09-23): the INFERENCE role, see myAxFleet below. + ../../modules/ax-fleet + # kubectl + the google/ax binaries, behind myAxClient.enable. Imported on + # all three interactive hosts, OFF on all three; read that module's header + # for the runbook and for what it deliberately does not declare. + ../../modules/ax-client.nix ]; networking.hostName = "worker"; + # ── ax on the fleet: the INFERENCE role ───────────────────────────────── + # "halogen inference mainly on worker" (Tom, 2026-09-23). Halogen stays this + # box's host service; ax sandboxes on the coordinator reach 10.42.0.5:8731 + # through Substrate's egress gateway on the NAS. This role renders nothing + # at runtime (no k3s, no unit): modules/ax-fleet/inference.nix only asserts + # that 8731 stays open on enp191s0. This host is not switched in the motion. + myAxFleet = { + enable = true; + role = "inference"; + lan = { + interface = "enp191s0"; + address = "10.42.0.5"; + }; + }; + # gVisor runsc for direct rootless `runsc run` jobs (modules/gvisor.nix). # OFF until Tom flips it; the lane recommends the worker first. myGvisor.enable = false; @@ -323,6 +344,13 @@ # first boot of the new closure — no flash, no host-key dance. mySecrets.enable = true; + # OFF, and it lands OFF (modules/ax-client.nix). There is no cluster on this + # fleet to point kubectl at and no Agent Substrate for ax to delegate to, so + # flipping this today installs two binaries with nothing to talk to. The flip + # is Tom's, one host at a time, and ax-client-topology in flake.nix goes red + # on it by design. + myAxClient.enable = false; + # ── Fleet candidate adoption (#354, 2026-09-13): ROLLING ───────────────── # The worker adopts the NAS's signed nightly candidate on its own when it is # safe to disturb (modules/update-adopt.nix). "Safe" here is one fact: the diff --git a/modules/ax-client.nix b/modules/ax-client.nix new file mode 100644 index 000000000..2c977169f --- /dev/null +++ b/modules/ax-client.nix @@ -0,0 +1,66 @@ +{ + config, + lib, + pkgs, + ... +}: +# ─── ax-client: kubectl and the ax binaries, on a host, behind a gate ─────── +# +# Issue #453, 2026-09-23. Puts `kubectl` and `pkgs.ax` (google/ax v0.3.0, +# pkgs/ax) into environment.systemPackages on the host that imports this module +# and sets `myAxClient.enable = true`. +# +# THE GATE LANDS OFF, on all three hosts that import it. Flipping it is a +# separate act and it is Tom's. Nothing here starts, schedules or contacts +# anything: both are client binaries that do nothing until invoked with a target. +# +# WHY IT EXISTS. `kubectl` is absent from every host on this fleet (MEASURED +# 2026-09-22), so the ax CLI has no way to reach a cluster even once ax itself is +# installed, and the two are useless apart. They are host-scoped rather than +# fleet-wide user packages because kubectl is: it reads /etc/kubernetes and a +# host kube context, and the NAS must never grow either. +# +# WHAT IS DELIBERATELY NOT DONE HERE, and each reason: +# - No cluster. No k3s, no kubelet, no API server, no control plane of any +# kind is declared by this module or anywhere else in this tree. The +# k3s-in-microVM route was NOT run on 2026-09-23; issue #453 records +# why (the 2026-09-20 precedent needed a MODIFIED copy of +# home/dot_local/bin/runtime-test carrying `--dev-bind /dev/kvm`, and the +# private global rules forbid running that experiment without the declared +# wrapper being changed first, reviewed, in its own change). +# - No Agent Substrate and no Redis. ax delegates every sandbox to Agent +# Substrate and is inert without it; Substrate has no flake and is not +# packaged (see pkgs/ax's header). This module installs the client half +# only, knowingly. +# - No kubeconfig, no context, no credential, no secret. Nothing in this +# module writes to /etc or to a home directory. +# - No service, no unit, no timer, no socket, no firewall hole. +# - hosts/nas does NOT import this module and must not: it is pinned to +# nixpkgs-stable with no home-manager, and it is an appliance. +# +# RUNBOOK, for the flip. On the host whose turn it is, one host at a time: +# 1. Set `myAxClient.enable = true;` in that host's hosts//default.nix, +# beside the `false` this module landed with. +# 2. `nix build --no-link .#ax` and `nix flake check --no-build` from the repo +# root. The ax-client-topology check in flake.nix asserts the gate's value +# per host, so it goes RED on the flip by design: update its expectation in +# the same commit, which is the point — the flip cannot be silent. +# 3. Tom switches. Agents do not. +# 4. `kubectl version --client` and `ax --help` on the box. Neither needs a +# cluster to answer, so both are safe first probes. +# 5. There is still no cluster to point either at. That is a later change and +# it is gated on issue #453 being resolved first. +let + cfg = config.myAxClient; +in +{ + options.myAxClient.enable = + lib.mkEnableOption "the ax control-plane client: kubectl plus the google/ax binaries, on this host"; + + config = lib.mkIf cfg.enable { + environment.systemPackages = [ + pkgs.kubectl + pkgs.ax + ]; + }; +} diff --git a/modules/ax-fleet/ax-fleet-smoke.sh b/modules/ax-fleet/ax-fleet-smoke.sh new file mode 100644 index 000000000..6d1bf7225 --- /dev/null +++ b/modules/ax-fleet/ax-fleet-smoke.sh @@ -0,0 +1,200 @@ +# ax-fleet-smoke: run one proof case as a real ax Task and print a JSON receipt. +# (Wrapped by writeShellApplication in ./ax.nix: strict mode, pinned PATH, and +# AX_FLEET_{ATESPACE,HALOGEN,HALOGEN_CIDR,API} from the module.) +# +# ax-fleet-smoke halogen [--hold N] one chat completion against Halogen +# ax-fleet-smoke pi [--hold N] pi against Halogen, schema-valid result +# ax-fleet-smoke exit N [--hold N] the command exits N (Failed ExitCode=N) +# ax-fleet-smoke egress-deny [URL] GET a non-allowlisted URL (default the +# coordinator's LAN address); must be refused +# ax-fleet-smoke floor N N Tasks in a row; none ResourceExhausted +# ax-fleet-smoke probe [--hold N] `claude --version` and a GET of Halogen's +# /v1/models, sandboxClass gvisor; needs an +# --image that carries claude-code (the +# fleet image does not, DESIGN.md 11) +# +# Options: --hold N (re-read the phase after N seconds), --timeout N (per Task, +# default 900), --keep (do not delete the Tasks), --image REF (default: the +# fleet image, ax-fleet-image-ref), --sandbox-class C (spec.sandboxClass; empty +# means gVisor), --gateway NAME (spec.gateway.name, default halogen; `none` +# omits the gateway; ax-fleet-gateway-default then points the Task at the +# atespace's default Gateway, since stock ax would give it allow-all). +# Exit 0 only when the case passes. The receipt carries no secret: Tasks carry none (DESIGN.md 11). + +export AX_SERVER="${AX_SERVER:-$AX_FLEET_API}" +ns="$AX_FLEET_ATESPACE" +hold=0 +timeout=900 +keep=0 +image="" +sandbox_class="" +gateway=halogen +args=() +while [ $# -gt 0 ]; do + case "$1" in + --hold) hold="$2"; shift 2 ;; + --timeout) timeout="$2"; shift 2 ;; + --keep) keep=1; shift ;; + --image) image="$2"; shift 2 ;; + --sandbox-class) sandbox_class="$2"; shift 2 ;; + --gateway) gateway="$2"; shift 2 ;; + *) args+=("$1"); shift ;; + esac +done +[ "${#args[@]}" -ge 1 ] || { sed -n '2,21p' "$0" >&2; exit 64; } +case_="${args[0]}" +[ -n "$image" ] || image="$(ax-fleet-image-ref)" +# Stock ax has no spec.sandboxClass and rejects the unknown field, so the +# probe sends none (ax v0.3.0 hardcodes gVisor). +run_id="$(date +%s)-$$" + +# The Gateway's `port` is carried for the record only: ax v0.3.0 copies only +# the host into a CIDR rule and Substrate's evaluator never compares ports +# (REPORTED ax client.go:457-490, egresspolicy.go:133-158; MEASURED by the +# round-3 review: worker:2222 answered through this Gateway). The port is +# enforced on the NAS instead: its pods reach ${AX_FLEET_HALOGEN} and no other +# private address (modules/ax-fleet/control.nix, podEgress). +ensure_gateway() { + ax -a "$ns" apply -f - >/dev/null </dev/null | yq -o json '.' 2>/dev/null || echo '{}' +} + +# run_task NAME CMD...: apply, wait for a terminal phase, hold, print a receipt. +run_task() { + local name="$1" + shift + local cmd_json + cmd_json="$(jq -cn '$ARGS.positional' --args -- "$@")" + local class_line="" + [ -z "$sandbox_class" ] || class_line=" sandboxClass: \"$sandbox_class\"" + local gateway_lines="" + [ "$gateway" = none ] || gateway_lines=" gateway: + name: \"$gateway\"" + ax -a "$ns" apply -f - >/dev/null </dev/null | jq -c '.' 2>/dev/null || echo null)" + fi + jq -cn --arg name "$name" --arg phase "$phase" --arg after "$after" \ + --argjson t "$t" --argjson result "$result" --argjson secs "$((settled - t0))" --argjson hold "$hold" \ + '{task:$name, phase:$phase, phase_after_hold:$after, hold_seconds:$hold, settle_seconds:$secs, + ready:(($t.status.conditions // []) | map(select(.type=="Ready")) | .[0] // null | if . then {reason, message} else null end), + gateway_ready:(($t.status.conditions // []) | map(select(.type=="GatewayReady")) | .[0] // null | if . then {status, reason} else null end), + exit_code:(if ($t.status.command.exited // false) then ($t.status.command.exitCode // 0) else null end), result_bytes:($t.status.command.resultBytes // null), + result_sha256:($t.status.command.resultSha256 // null), result:$result}' + if [ "$keep" -eq 0 ]; then ax -a "$ns" delete task "$name" >/dev/null 2>&1 || true; fi +} + +run_case() { +case "$case_" in +halogen) + r="$(run_task "smoke-halogen-$run_id" ax-agent halogen-smoke)" + jq -c --arg case "$case_" '. + {case:$case, pass:(.phase=="Completed" and .phase_after_hold=="Completed" and .exit_code==0)}' <<<"$r" + ;; +pi) + r="$(run_task "smoke-pi-$run_id" ax-agent pi)" + jq -c --arg case "$case_" '. + {case:$case, pass:(.phase=="Completed" and .phase_after_hold=="Completed" and (.result.valid == true))}' <<<"$r" + ;; +exit) + n="${args[1]:?usage: ax-fleet-smoke exit N}" + r="$(run_task "smoke-exit-$run_id" ax-agent exit "$n")" + if [ "$n" -eq 0 ]; then want=Completed; else want=Failed; fi + jq -c --arg case "exit $n" --arg want "$want" --argjson n "$n" \ + '. + {case:$case, pass:(.phase==$want and .phase_after_hold==$want and .exit_code==$n and .ready.reason=="CommandExited")}' <<<"$r" + ;; +egress-deny) + url="${args[1]:-http://10.42.0.2/}" + r="$(run_task "smoke-egress-$run_id" ax-agent fetch "$url")" + # Refused either at the Gateway (the connection fails: curl rc != 0) or, + # for an allowlisted host on a port the NAS drops (fix round 3), by the + # egress gateway's own upstream error (502/503/504, the target never + # answered), or by Substrate's egress router refusing the actor outright + # (403 "egress denied": no policy, or a policy with no rules). Any other + # HTTP status + # is the target answering: fail. The deny targets in the VM answer 200. + jq -c --arg case "$case_" --arg url "$url" --arg gw "$gateway" \ + '. + {case:$case, url:$url, gateway:$gw, + refused_by:(if (.result.curl_rc // 0) != 0 then "gateway-connection" + elif ((.result.http_code // "") | test("^50[234]$")) then "egress-upstream" + elif (.result.http_code // "") == "403" then "egress-policy" + else null end)} + | . + {pass:(.refused_by != null and .ready.reason=="CommandExited")}' <<<"$r" + ;; +floor) + n="${args[1]:?usage: ax-fleet-smoke floor N}" + all='[]' + for i in $(seq 1 "$n"); do + r="$(run_task "smoke-floor-$run_id-$i" ax-agent exit 0)" + all="$(jq -c --argjson r "$r" '. + [$r]' <<<"$all")" + done + jq -c --arg case "floor $n" --argjson n "$n" \ + '{case:$case, tasks:., pass:((length == $n) and all(.[]; .phase=="Completed" and ((.ready.message // "") | test("ResourceExhausted") | not)))}' <<<"$all" + ;; +probe) + r="$(run_task "smoke-probe-$run_id" ax-agent probe)" + jq -c --arg case "$case_" --arg class "$sandbox_class" --arg image "$image" \ + '. + {case:$case, sandbox_class:$class, image:$image, + pass:(.phase=="Completed" and .phase_after_hold=="Completed" and .exit_code==0 + and .result.claude_rc==0 and .result.http_code=="200")}' <<<"$r" + ;; +*) + echo "unknown case: $case_" >&2 + exit 64 + ;; +esac +} + +ensure_gateway +receipt="$(run_case)" +printf '%s\n' "$receipt" +jq -e '.pass' <<<"$receipt" >/dev/null diff --git a/modules/ax-fleet/ax.nix b/modules/ax-fleet/ax.nix new file mode 100644 index 000000000..becb1fa86 --- /dev/null +++ b/modules/ax-fleet/ax.nix @@ -0,0 +1,488 @@ +{ + config, + lib, + pkgs, + inputs, + ... +}: +# ax on the fleet, track ax (DESIGN.md sections 9, 10 and 14 A3). This module +# writes only the myAxFleet extension points (manifests, registrySeed) and, on +# the harness, the coordinator's ax scripts and AX_SERVER. Everything else +# (k3s, the registry, the bootstrap runner, ax-server-proxy.socket) belongs to +# the cluster modules next to this file. +# +# ONE image set for every host. The images are built from the flake's own +# nixpkgs pin and the flake's `ax` package, never from the host's `pkgs`: the +# NAS evaluates from nixpkgs-stable, and the digest the NAS seeds must be the +# digest the coordinator's `ax-fleet-image-ref` prints. +# +# Day one: pi against Halogen only. No Claude credential, no claude-code in the +# task image, no secret in any Task or manifest (DESIGN.md section 11). +let + cfg = config.myAxFleet; + system = "x86_64-linux"; + + fleetPkgs = inputs.nixpkgs.legacyPackages.${system}; + ax = inputs.self.packages.${system}.ax; + pi = inputs.llm-agents.packages.${system}.pi; + + images = fleetPkgs.callPackage ../../pkgs/ax/images.nix { inherit ax; }; + agentImage = fleetPkgs.callPackage ../../pkgs/ax-agent-image { inherit ax pi; }; + + seeds = { + ax-server = { + oci = images.ax-server; + repo = "ax/ax-server"; + tag = images.tag; + }; + ax-controller = { + oci = images.ax-controller; + repo = "ax/ax-controller"; + tag = images.tag; + }; + ax-redis = { + oci = images.ax-redis; + repo = "ax/ax-redis"; + tag = images.ax-redis.passthru.tag; + }; + ax-agent = { + oci = agentImage; + repo = "ax/ax-agent"; + tag = agentImage.passthru.tag; + }; + }; + + # Pod image references through the containerd mirror for cfg.registry. + # @digest:@ is replaced at build time from "/digest". + podImage = name: "${cfg.registry}/${seeds.${name}.repo}@@digest:${name}@"; + + controlSelector = { + "ax.mecattaf.dev/role" = "control"; + }; + + labels = name: { + "app.kubernetes.io/name" = name; + "app.kubernetes.io/part-of" = "ax"; + "app.kubernetes.io/managed-by" = "ax-fleet"; + }; + + # The ax-system objects, as data. Rendered to YAML below, then the digests + # are substituted in a build step (no import-from-derivation). + objects = [ + { + apiVersion = "v1"; + kind = "Namespace"; + metadata = { + name = "ax-system"; + labels = labels "ax-system"; + }; + } + + # ── ax-redis: AOF on a local-path volume (the data pool on the NAS) ── + { + apiVersion = "v1"; + kind = "PersistentVolumeClaim"; + metadata = { + name = "ax-redis-data"; + namespace = "ax-system"; + labels = labels "ax-redis"; + }; + spec = { + accessModes = [ "ReadWriteOnce" ]; + storageClassName = "local-path"; + resources.requests.storage = "5Gi"; + }; + } + { + apiVersion = "apps/v1"; + kind = "Deployment"; + metadata = { + name = "ax-redis"; + namespace = "ax-system"; + labels = labels "ax-redis"; + }; + spec = { + replicas = 1; + # One RWO volume: never two pods at once. + strategy.type = "Recreate"; + selector.matchLabels."app.kubernetes.io/name" = "ax-redis"; + template = { + metadata.labels = labels "ax-redis"; + spec = { + nodeSelector = controlSelector; + containers = [ + { + name = "redis"; + image = podImage "ax-redis"; + imagePullPolicy = "IfNotPresent"; + # No password: reachable only as a ClusterIP, and a sandbox + # reaches only its Gateway allowlist. If one is ever added, both + # daemons read REDIS_PASSWORD from the environment; never argv. + args = [ + "--appendonly" + "yes" + "--dir" + "/data" + "--protected-mode" + "no" + ]; + ports = [ + { + containerPort = 6379; + name = "redis"; + } + ]; + readinessProbe = { + tcpSocket.port = 6379; + periodSeconds = 5; + }; + resources = { + requests = { + cpu = "100m"; + memory = "128Mi"; + }; + limits.memory = "1Gi"; + }; + volumeMounts = [ + { + name = "data"; + mountPath = "/data"; + } + ]; + } + ]; + volumes = [ + { + name = "data"; + persistentVolumeClaim.claimName = "ax-redis-data"; + } + ]; + }; + }; + }; + } + { + apiVersion = "v1"; + kind = "Service"; + metadata = { + name = "ax-redis"; + namespace = "ax-system"; + labels = labels "ax-redis"; + }; + spec = { + type = "ClusterIP"; + selector."app.kubernetes.io/name" = "ax-redis"; + ports = [ + { + name = "redis"; + port = 6379; + targetPort = 6379; + } + ]; + }; + } + + # ── ax-server: ClusterIP pinned, never a NodePort (the API has no auth) ── + { + apiVersion = "apps/v1"; + kind = "Deployment"; + metadata = { + name = "ax-server"; + namespace = "ax-system"; + labels = labels "ax-server"; + }; + spec = { + replicas = 1; + selector.matchLabels."app.kubernetes.io/name" = "ax-server"; + template = { + metadata.labels = labels "ax-server"; + spec = { + nodeSelector = controlSelector; + containers = [ + { + name = "ax-server"; + image = podImage "ax-server"; + imagePullPolicy = "IfNotPresent"; + args = [ + "--addr=:8080" + "--redis-addr=ax-redis.ax-system.svc.cluster.local:6379" + ]; + ports = [ + { + containerPort = 8080; + name = "http"; + } + ]; + readinessProbe = { + httpGet = { + path = "/healthz"; + port = 8080; + }; + initialDelaySeconds = 2; + periodSeconds = 5; + }; + livenessProbe = { + httpGet = { + path = "/healthz"; + port = 8080; + }; + initialDelaySeconds = 5; + periodSeconds = 10; + }; + resources = { + requests = { + cpu = "100m"; + memory = "128Mi"; + }; + limits.memory = "1Gi"; + }; + securityContext = { + readOnlyRootFilesystem = true; + allowPrivilegeEscalation = false; + runAsNonRoot = true; + }; + } + ]; + }; + }; + }; + } + { + apiVersion = "v1"; + kind = "Service"; + metadata = { + name = "ax-server"; + namespace = "ax-system"; + labels = labels "ax-server"; + }; + spec = { + type = "ClusterIP"; + clusterIP = cfg.axServerClusterIP; + selector."app.kubernetes.io/name" = "ax-server"; + ports = [ + { + name = "http"; + port = 8080; + targetPort = 8080; + } + ]; + }; + } + + # ── ax-controller: upstream deploy/ax-controller.yaml, stock args ── + { + apiVersion = "v1"; + kind = "ServiceAccount"; + metadata = { + name = "ax-controller"; + namespace = "ax-system"; + labels = labels "ax-controller"; + }; + } + # No ClusterRole (fix round 1, 2026-09-23). Upstream grants get/list/watch + # on every Secret cluster-wide; the controller's only read is one GET of + # gemini-api-secret in the Task's atespace (reconciler.go lookupGeminiKey, + # REPORTED from the ax source), which falls back to env and then to "". + # Day one is pi on Halogen with no key, so the grant bought nothing. If a + # Gemini key is ever ruled in: a namespaced Role in that atespace with + # resourceNames [ "gemini-api-secret" ] and verbs [ "get" ] only. + { + apiVersion = "apps/v1"; + kind = "Deployment"; + metadata = { + name = "ax-controller"; + namespace = "ax-system"; + labels = labels "ax-controller"; + }; + spec = { + # One consumer per Redis group, as upstream ships it. + replicas = 1; + strategy.type = "Recreate"; + selector.matchLabels."app.kubernetes.io/name" = "ax-controller"; + template = { + metadata.labels = labels "ax-controller"; + spec = { + nodeSelector = controlSelector; + serviceAccountName = "ax-controller"; + # No API token in the pod: it holds no RBAC and needs none. The + # projected ateapi token below is a separate volume and stays. + automountServiceAccountToken = false; + containers = [ + { + name = "controller"; + image = podImage "ax-controller"; + imagePullPolicy = "IfNotPresent"; + args = [ + "--redis-addr=ax-redis.ax-system.svc.cluster.local:6379" + "--substrate-endpoint=api.ate-system.svc.cluster.local:443" + "--substrate-authority=api.ate-system.svc" + "--substrate-token-file=/var/run/secrets/ateapi/token" + "--substrate-ca-file=/run/servicedns-ca/trust-bundle.pem" + "--template=default-template" + "--template-atespace=ax-system" + ]; + env = [ + { + name = "ATENET_ROUTER_ADDR"; + value = "atenet-router.ate-system.svc.cluster.local:80"; + } + { + # MEASURED working on RustFS by the 2026-09-23 probe. + name = "AX_SNAPSHOTS_BUCKET"; + value = "gs://ate-snapshots/ax/"; + } + ]; + resources = { + requests = { + cpu = "100m"; + memory = "128Mi"; + }; + limits.memory = "512Mi"; + }; + securityContext = { + readOnlyRootFilesystem = true; + allowPrivilegeEscalation = false; + runAsNonRoot = true; + }; + volumeMounts = [ + { + mountPath = "/var/run/secrets/ateapi"; + name = "ate-token"; + readOnly = true; + } + { + mountPath = "/run/servicedns-ca"; + name = "servicedns-ca"; + readOnly = true; + } + ]; + } + ]; + volumes = [ + { + name = "ate-token"; + projected = { + defaultMode = 292; # 0444: the controller runs as uid 65532 + sources = [ + { + serviceAccountToken = { + audience = "api.ate-system.svc"; + expirationSeconds = 7200; + path = "token"; + }; + } + ]; + }; + } + { + name = "servicedns-ca"; + projected = { + defaultMode = 420; + sources = [ + { + clusterTrustBundle = { + signerName = "servicedns.podcert.ate.dev/identity"; + labelSelector.matchLabels."podcert.ate.dev/canarying" = "live"; + path = "trust-bundle.pem"; + }; + } + ]; + }; + } + ]; + }; + }; + }; + } + ]; + + manifestTemplate = fleetPkgs.writeText "ax-fleet-40-ax.yaml.in" ( + lib.concatMapStringsSep "\n---\n" builtins.toJSON objects + "\n" + ); + + # JSON documents are valid YAML; k3s's deploy controller reads multi-doc + # YAML. Digests are substituted from each image's `digest` file here, at + # build time, then every document is checked for a leftover placeholder. + manifest = + fleetPkgs.runCommand "ax-fleet-40-ax.yaml" + { + nativeBuildInputs = [ fleetPkgs.yq-go ]; + } + '' + cp ${manifestTemplate} $out + chmod u+w $out + ${lib.concatMapStrings (n: '' + substituteInPlace $out --replace-quiet '@digest:${n}@' "$(cat ${seeds.${n}.oci}/digest)" + '') (builtins.attrNames seeds)} + if grep -q '@digest:' $out; then echo "unsubstituted digest in $out" >&2; exit 1; fi + # Every document parses, and every image is pinned by digest. + yq -e 'select(.kind == "Deployment") | .spec.template.spec.containers[].image | test("@sha256:[0-9a-f]{64}$")' $out >/dev/null + test "$(yq -N 'select(.kind != null) | .kind' $out | wc -l)" -eq ${toString (builtins.length objects)} + ''; + + agentRef = fleetPkgs.writeShellScriptBin "ax-fleet-image-ref" '' + # The ax-agent Task image by digest. atelet rewrites localhost:5000. + printf 'localhost:5000/ax/ax-agent@%s\n' "$(cat ${agentImage}/digest)" + ''; + + smoke = fleetPkgs.writeShellApplication { + name = "ax-fleet-smoke"; + runtimeInputs = [ + ax + agentRef + fleetPkgs.coreutils + fleetPkgs.jq + fleetPkgs.yq-go + ]; + text = builtins.readFile ./ax-fleet-smoke.sh; + runtimeEnv = { + AX_FLEET_ATESPACE = cfg.ax.atespace; + AX_FLEET_HALOGEN = cfg.halogenEndpoint; + AX_FLEET_HALOGEN_CIDR = "${builtins.head (lib.splitString ":" cfg.halogenEndpoint)}/32"; + AX_FLEET_API = "http://${cfg.apiListen}"; + }; + }; +in +{ + options.myAxFleet.ax = { + atespace = lib.mkOption { + type = lib.types.str; + default = "fleet"; + description = "The ax atespace ax-fleet-smoke runs Tasks and the halogen Gateway in."; + }; + images = lib.mkOption { + type = lib.types.attrsOf lib.types.package; + readOnly = true; + internal = true; + default = { + inherit (images) ax-server ax-controller ax-redis; + ax-agent = agentImage; + manifest = manifest; + }; + description = "The ax images and the rendered manifest, for tests and the flake's packages."; + }; + }; + + config = lib.mkIf cfg.enable ( + lib.mkMerge [ + { + # Consumed on the NAS (control role) by the cluster modules; declared + # on every role so the digests any host names are the digests seeded. + myAxFleet.manifests.ax-fleet-40-ax.source = manifest; + myAxFleet.registrySeed = lib.mapAttrs (_: s: s) seeds; + } + + (lib.mkIf (cfg.role == "harness") { + environment.systemPackages = [ + agentRef + smoke + ]; + # ax-server-proxy.socket (harness.nix) listens here and forwards to + # the ax-server ClusterIP. The ax CLI honours it. ax-conwip does NOT + # read it: its serverUrl defaults to 127.0.0.1:8080, the mock stack's + # port, which the proxy no longer takes (fix round 3). + environment.sessionVariables.AX_SERVER = "http://${cfg.apiListen}"; + }) + ] + ); +} diff --git a/modules/ax-fleet/control.nix b/modules/ax-fleet/control.nix new file mode 100644 index 000000000..d6697da87 --- /dev/null +++ b/modules/ax-fleet/control.nix @@ -0,0 +1,442 @@ +{ + config, + lib, + pkgs, + ... +}: +# The control node: the NAS. "Hypervisor on NAS" (Tom, 2026-09-23) read as +# everything that schedules or remembers: the k3s server WITH its kubelet and +# NO taint (so CoreDNS, local-path, Substrate's and ax's control pods and every +# PersistentVolume land here by elimination, because the coordinator is +# tainted), the registry, and the two runners that seed it and bootstrap the +# cluster. DESIGN.md sections 6.2, 6.3, 7, 8, 9. +# +# Nothing churns on the 57 GB eMMC root: k3s's datastore, containerd and +# kubelet state are bind-mounted from the fast tier (stateRoot), and every +# PersistentVolume and the registry live on the data pool. The NAS's shared +# PostgreSQL (Paperless, Immich) is not touched. +let + cfg = config.myAxFleet; + # The guard table is the control ROLE's, whatever `enable` says (fix round + # 3): after the kill switch, pods (the egress gateway among them) outlive + # k3s until ax-fleet-teardown. Inert without cni0 and flannel.1. + roleOn = cfg.role == "control"; + on = cfg.enable && roleOn; + + halogenHost = lib.head (lib.splitString ":" cfg.halogenEndpoint); + halogenPort = lib.last (lib.splitString ":" cfg.halogenEndpoint); + + gates = "ClusterTrustBundle=true,ClusterTrustBundleProjection=true,PodCertificateRequest=true"; + + registryPort = lib.toInt (lib.last (lib.splitString ":" cfg.registry)); + registryHost = lib.head (lib.splitString ":" cfg.registry); + + serverFlags = [ + "--cluster-cidr=${cfg.podCidr}" + "--service-cidr=${cfg.serviceCidr}" + "--cluster-dns=${cfg.clusterDns}" + "--flannel-backend=vxlan" + "--advertise-address=${cfg.lan.address}" + "--tls-san=${cfg.lan.address}" + "--tls-san=nas" + "--disable-helm-controller" + # k3s's network-policy controller off: Substrate ships no NetworkPolicy + # (MEASURED at d277088b). Confining worker pods by policy is a follow-up. + "--disable-network-policy" + # local-storage stays ON: the kind install's PostgreSQL and RustFS claim + # volumes (MEASURED kind/rustfs.yaml:16, postgres/postgres.yaml:242). + "--default-local-storage-path=${cfg.localPathRoot}" + "--secrets-encryption" + "--service-node-port-range=${cfg.nodePortRange}" + "--kube-apiserver-arg=feature-gates=${gates}" + # The gates alone serve nothing; this is what serves the group (A5a, + # MEASURED; upstream's hack/create-kind-cluster.sh runtimeConfig). + "--kube-apiserver-arg=runtime-config=certificates.k8s.io/v1beta1=true" + "--kube-controller-manager-arg=feature-gates=${gates}" + ]; + + # k3s hot state on the fast tier, by bind mount (not --data-dir: the NixOS + # module links manifests and images into fixed /var/lib/rancher/k3s paths). + binds = { + "/var/lib/rancher" = "${cfg.stateRoot}/k3s/rancher"; + "/var/lib/kubelet" = "${cfg.stateRoot}/k3s/kubelet"; + "/var/log/pods" = "${cfg.stateRoot}/k3s/pod-logs"; + }; + + kubectl = "${cfg.k3sPackage}/bin/kubectl"; + + waitLanAddr = pkgs.writeShellScript "ax-fleet-wait-lan-addr" '' + for _ in $(${pkgs.coreutils}/bin/seq 30); do + ${pkgs.iproute2}/bin/ip -4 addr show dev ${cfg.lan.interface} 2>/dev/null \ + | ${pkgs.gnugrep}/bin/grep -qF 'inet ${cfg.lan.address}/' && exit 0 + ${pkgs.coreutils}/bin/sleep 1 + done + echo "ax-fleet: ${cfg.lan.address} not on ${cfg.lan.interface} after 30 s; starting anyway" >&2 + exit 0 + ''; + + # ── the cluster-range guard (fix round 1, 2026-09-23) ── + # The NAS is the house default gateway and forwards with policy accept, and + # kube-proxy's KUBE-SERVICES DNAT sits in PREROUTING for every interface. So + # without this, any LAN device that routes the pod or Service range via + # 10.42.0.1 reaches ClusterIPs and pods (MEASURED by the security review: + # ax-server 200, Redis INFO, RustFS 403; and, through VXLAN, pods on the + # coordinator). At priority raw, before any DNAT: destinations in the + # cluster ranges are accepted only from the pod-side interfaces. VXLAN + # outer packets target ${cfg.lan.address}, so flannel is unaffected, and + # traffic the NAS itself originates never passes prerouting. + rangeGuard = '' + chain prerouting { + type filter hook prerouting priority raw; policy accept; + iifname { "cni0", "flannel.1", "lo" } return + iifname "veth*" return + # The match carries the drop (never a bare drop): IPv6 and every other + # destination fall through to policy accept. + ip daddr { ${cfg.podCidr}, ${cfg.serviceCidr} } counter drop comment "ax-fleet: cluster ranges only from pods and flannel" + } + + # ── pod egress from the router (fix round 3) ── + # MEASURED by the round-3 review: postgres-0 reached worker:2222 and the + # egress gateway (atenet-egress, a pod on this node, which carries every + # sandbox's allowlisted traffic) reached every TCP port on the worker, + # sshd included: ax drops the Gateway's port and Substrate never compares + # one (REPORTED ax client.go:457-490, egresspolicy.go:133-158). Here the + # port IS enforced: from pods, the only private address is Halogen on its + # port; the tailnet and the containers are never reachable; the internet + # is. Pod-to-NAS-host traffic (DNS, the apiserver, the registry) is INPUT + # and unaffected. Same shape as the coordinator's guard (harness.nix). + chain forward { + type filter hook forward priority filter; policy accept; + iifname != { "cni0", "flannel.1" } return + ct state established,related return + oifname { "cni0", "flannel.1" } return + ip daddr ${halogenHost} tcp dport ${halogenPort} return + ip daddr { 10.0.0.0/8, 172.16.0.0/12, 192.168.0.0/16, 100.64.0.0/10, 169.254.0.0/16 } counter drop comment "ax-fleet: pods reach no private address but Halogen" + ip6 daddr { fc00::/7, fe80::/10 } counter drop comment "ax-fleet: pods reach no private address but Halogen" + oifname { ${lib.concatMapStringsSep ", " (i: ''"${i}"'') cfg.guardInterfaces} } counter drop comment "ax-fleet: pods never reach the tailnet" + oifname "ve-*" counter drop comment "ax-fleet: pods never reach the NAS's containers" + } + + # ── the cluster from the NAS's own processes: root only (fix round 4) ── + # The coordinator's owner match (harness.nix apiRules) had no counterpart + # here: prerouting never sees locally generated packets, so paperless, + # immich, atticd, headscale or nginx (MEASURED uid-map) could open the + # unauthenticated ax-server API and the password-less ax-redis, and, while + # a seed runs, push to the loopback writer. Only NEW connections are + # policed: replies from AdGuard or the registry to pods are established. + chain output { + type filter hook output priority filter; policy accept; + ct state != new return + meta skuid 0 return + ${lib.optionalString (cfg.clusterClientUids != [ ]) "meta skuid { ${lib.concatMapStringsSep ", " toString cfg.clusterClientUids} } return"} + ip daddr { ${cfg.podCidr}, ${cfg.serviceCidr} } counter reject comment "ax-fleet: the cluster ranges from this host, root only" + ip daddr 127.0.0.1 tcp dport ${lib.last (lib.splitString ":" seedAddr)} counter reject with tcp reset comment "ax-fleet: the seed's writable registry, root only" + } + ''; + + # ── the bootstrap steps this track owns ── + bootstrapApi = '' + # 10-api: wait for the apiserver and the admin kubeconfig. + for _ in $(seq 1 300); do + if [ -s /etc/rancher/k3s/k3s.yaml ] && ${kubectl} get --raw /readyz >/dev/null 2>&1; then + echo "apiserver ready" + exit 0 + fi + sleep 2 + done + echo "apiserver not ready" >&2 + exit 1 + ''; + + bootstrapKubeconfig = '' + # 60-kubeconfig: the admin kubeconfig for ax-fleet-kubeconfig on the + # coordinator (read over the existing ssh trust). 0640 root:wheel, never + # printed. + install -d -m 0755 /etc/ax-fleet + tmp=$(mktemp /etc/ax-fleet/.admin.kubeconfig.XXXXXX) + sed 's#https://127.0.0.1:6443#https://${cfg.serverAddress}:6443#' /etc/rancher/k3s/k3s.yaml > "$tmp" + chown root:wheel "$tmp" + chmod 0640 "$tmp" + mv "$tmp" /etc/ax-fleet/admin.kubeconfig + ''; + + stepNames = lib.sort (a: b: a < b) (lib.attrNames cfg.bootstrap); + stepScript = + name: + pkgs.writeShellScript "ax-fleet-step-${name}" '' + set -euo pipefail + ${cfg.bootstrap.${name}} + ''; + + # ── the registry: read-only to the network, writable only to the seed ── + # (fix round 2) The round-2 review MEASURED an unprivileged user on the + # coordinator pushing blobs (202), mounting across repos (201) and + # overwriting substrate/atelet:d277088b's tag (201): the source-address rule + # admits every uid and every pod on the coordinator (masqueraded to its LAN + # address). ate-setup resolves --image-tag to a digest at install time, so a + # rewritten tag is what would get pinned. Now the served instance on + # ${cfg.registry} runs with storage.maintenance.readonly, and delete is off; + # the seed pushes through a second, loopback-only instance on the same root + # directory that lives only for the seed run, as the registry user. + seedAddr = "127.0.0.1:5001"; + registryBase = { + version = "0.1"; + log.fields.service = "registry"; + storage = { + cache.blobdescriptor = "inmemory"; + delete.enabled = false; + filesystem.rootdirectory = cfg.registryRoot; + }; + http.headers.X-Content-Type-Options = [ "nosniff" ]; + }; + seedRegistryConfig = pkgs.writeText "ax-fleet-seed-registry.json" ( + builtins.toJSON (lib.recursiveUpdate registryBase { http.addr = seedAddr; }) + ); + + seedScript = '' + set -euo pipefail + # The writable instance, loopback only, gone when this script exits. + setpriv --reuid=docker-registry --regid=docker-registry --init-groups \ + ${lib.getExe config.services.dockerRegistry.package} serve ${seedRegistryConfig} & + writer=$! + trap 'kill $writer 2>/dev/null || true; wait $writer 2>/dev/null || true' EXIT + for _ in $(seq 1 120); do + curl -fsS -o /dev/null http://${cfg.registry}/v2/ && curl -fsS -o /dev/null http://${seedAddr}/v2/ && break + sleep 1 + done + curl -fsS -o /dev/null http://${cfg.registry}/v2/ + curl -fsS -o /dev/null http://${seedAddr}/v2/ + ${lib.concatStrings ( + lib.mapAttrsToList (name: s: '' + echo "seed ${name}: ${s.repo}:${s.tag}" + digest=$(tr -d '[:space:]' < ${s.oci}/digest) + skopeo --insecure-policy copy --all --preserve-digests --dest-tls-verify=false \ + oci:${s.oci} docker://${seedAddr}/${s.repo}:${s.tag} + # Read back through the served, read-only instance: what pods pull. + skopeo --insecure-policy inspect --raw --tls-verify=false \ + docker://${cfg.registry}/${s.repo}@"$digest" >/dev/null + echo "seeded ${s.repo}@$digest" + '') cfg.registrySeed + )} + echo "registry seed complete: ${toString (lib.length (lib.attrNames cfg.registrySeed))} image(s)" + ''; +in +{ + config = lib.mkMerge [ + (lib.mkIf roleOn { + assertions = [ + { + assertion = config.networking.nftables.enable; + message = "modules/ax-fleet/control.nix: the control role's cluster-range guard is an nftables table; the NAS runs nftables."; + } + ]; + networking.nftables.tables.ax-fleet-guard = { + family = "inet"; + content = rangeGuard; + }; + # The teardown leaves a guard the generation declares (pkgs/ax-fleet-teardown). + environment.etc."ax-fleet/guard-declared".text = "control\n"; + }) + (lib.mkIf on { + myAxFleet.kubelet = { + # Protects DNS, DHCP, headscale, Paperless and Immich on 8 cores / 22 GiB. + systemReserved = lib.mkDefault "cpu=2,memory=8Gi"; + }; + + myAxFleet.bootstrap = { + "10-api" = bootstrapApi; + "60-kubeconfig" = bootstrapKubeconfig; + }; + + services.k3s = { + role = "server"; + disable = [ + "traefik" + "servicelb" + "metrics-server" + ]; + # No nodeTaint: the NAS is untainted (DESIGN D5). The `none` version + # value keeps Substrate's version-keyed atelet DaemonSet off the NAS + # (ate-setup only labels nodes that lack the key). + nodeLabel = [ + "ax.mecattaf.dev/role=control" + "ate.dev/substrate-version=none" + ]; + extraFlags = serverFlags; + manifests = lib.mapAttrs (name: m: { + inherit (m) source; + target = "${name}.yaml"; + }) cfg.manifests; + }; + + # ── bind mounts: k3s state on the fast tier ── + # systemd mount units, not fileSystems: they stay out of local-fs.target + # (a failed bind cannot drop the router into emergency mode at boot; only + # k3s, which RequiresMountsFor them, would fail), and the VM test runs the + # exact same units (qemu-vm replaces `fileSystems` wholesale). + systemd.mounts = lib.mapAttrsToList (where: what: { + inherit what where; + type = "none"; + options = "bind"; + # Lazy: at the kill-switch switch, pods and shims outlive k3s + # (KillMode=process) and can keep /var/lib/kubelet busy, which made the + # rollback switch exit 4 (MEASURED, fix round 1 VM run 4). Detaching + # lazily leaves the data on /mnt/fast untouched; ax-fleet-teardown then + # stops what still holds it. + mountConfig.LazyUnmount = true; + requires = [ "ax-fleet-dirs.service" ]; + after = [ "ax-fleet-dirs.service" ]; + wantedBy = [ "k3s.service" ]; + before = [ "k3s.service" ]; + }) binds; + + systemd.services.ax-fleet-dirs = { + description = "ax-fleet: create the k3s state and data-pool directories"; + unitConfig = { + DefaultDependencies = false; + RequiresMountsFor = [ + cfg.stateRoot + cfg.localPathRoot + cfg.registryRoot + ]; + }; + serviceConfig = { + Type = "oneshot"; + RemainAfterExit = true; + }; + script = '' + ${lib.concatMapStringsSep "\n" (d: "${pkgs.coreutils}/bin/install -d -m 0711 ${d}") ( + lib.attrValues binds + )} + ${pkgs.coreutils}/bin/install -d -m 0711 ${cfg.localPathRoot} + ${pkgs.coreutils}/bin/install -d -m 0750 -o docker-registry -g docker-registry ${cfg.registryRoot} + ''; + }; + + systemd.services.k3s = { + wants = [ "ax-fleet-dirs.service" ]; + after = [ "ax-fleet-dirs.service" ]; + unitConfig.RequiresMountsFor = (lib.attrNames binds) ++ [ cfg.localPathRoot ]; + }; + + # ── the registry (moved from #446), on the data pool ── + services.dockerRegistry = { + enable = true; + listenAddress = registryHost; + port = registryPort; + storagePath = cfg.registryRoot; + # Read-only to the network (see seedAddr); garbage collection is + # offline and needs no delete API. + enableDelete = false; + extraConfig.storage.maintenance.readonly.enabled = true; + enableGarbageCollect = true; + garbageCollectDates = "weekly"; + # openFirewall NOT used: the source-scoped rule below is the access control. + }; + systemd.services.docker-registry = { + wants = [ "ax-fleet-dirs.service" ]; + after = [ "ax-fleet-dirs.service" ]; + unitConfig.RequiresMountsFor = [ cfg.registryRoot ]; + # The explicit ${registryHost} bind races NetworkManager at boot: the + # static address reaches ${cfg.lan.interface} seconds after + # network(-online).target (MEASURED on the NAS, 2026-09-23; the same race + # hosts/nas/headscale.nix and modules/adguardhome.nix already guard). + # Wait up to 30 s for the address, then start anyway and let Restart + # cover a genuinely late interface. + serviceConfig = { + ExecStartPre = waitLanAddr; + Restart = "on-failure"; + RestartSec = 5; + }; + }; + + + # ── firewall: only what the coordinator and the pods need ── + # #447 opened 6443 to the whole LAN and #446 opened 5432/9000/5000 to the + # whole LAN. Now: 6443, 5000 and VXLAN only from the coordinator; DNS and + # the apiserver from pods on cni0. 5432, 6379 and 9000 are never opened on + # the host. Nothing on tailscale0. + networking.firewall.extraInputRules = '' + iifname "${cfg.lan.interface}" ip saddr { ${lib.concatStringsSep ", " cfg.harnessAddresses} } tcp dport { 6443, ${toString registryPort} } accept comment "ax-fleet: kube API and registry, coordinator only" + iifname "${cfg.lan.interface}" ip saddr { ${lib.concatStringsSep ", " cfg.harnessAddresses} } udp dport 8472 accept comment "ax-fleet: flannel VXLAN, coordinator only" + iifname "cni0" ip saddr ${cfg.podCidr} tcp dport { 53, 6443 } accept comment "ax-fleet: pods to AdGuard and the apiserver" + iifname "cni0" ip saddr ${cfg.podCidr} udp dport 53 accept comment "ax-fleet: pods to AdGuard" + ''; + + # ── the registry seed: from store paths in the NAS closure ── + systemd.services.ax-fleet-registry-seed = { + description = "ax-fleet: seed the NAS registry from the store, digests preserved"; + wantedBy = [ "multi-user.target" ]; + # wants, not requires: a registry that fails its first start (late LAN + # address) must not fail the seed on dependency, which Restart= would + # never retry. The script itself waits for /v2/, and Restart retries. + wants = [ "docker-registry.service" ]; + after = [ "docker-registry.service" ]; + path = [ + pkgs.skopeo + pkgs.curl + pkgs.coreutils + pkgs.util-linux + ]; + environment.HOME = "/var/lib/ax-fleet"; + serviceConfig = { + Type = "oneshot"; + RemainAfterExit = true; + StateDirectory = "ax-fleet"; + Restart = "on-failure"; + RestartSec = 15; + }; + script = seedScript; + }; + + # ── the bootstrap: myAxFleet.bootstrap, in name order ── + systemd.services.ax-fleet-bootstrap = { + description = "ax-fleet: bootstrap the cluster (idempotent steps, in name order)"; + wantedBy = [ "multi-user.target" ]; + wants = [ + "k3s.service" + "docker-registry.service" + "ax-fleet-registry-seed.service" + ]; + after = [ + "k3s.service" + "docker-registry.service" + "ax-fleet-registry-seed.service" + ]; + path = [ + cfg.k3sPackage + pkgs.coreutils + pkgs.gnugrep + pkgs.gnused + pkgs.gawk + pkgs.findutils + pkgs.jq + pkgs.curl + pkgs.skopeo + pkgs.util-linux + pkgs.bash + ]; + environment = { + KUBECONFIG = "/etc/rancher/k3s/k3s.yaml"; + HOME = "/var/lib/ax-fleet"; + }; + serviceConfig = { + Type = "oneshot"; + RemainAfterExit = true; + Restart = "on-failure"; + RestartSec = 30; + TimeoutStartSec = "45min"; + StateDirectory = "ax-fleet"; + }; + script = '' + set -euo pipefail + ${lib.concatMapStringsSep "\n" (n: '' + echo "== ax-fleet-bootstrap step ${n}" + ${stepScript n} + '') stepNames} + echo "== ax-fleet-bootstrap complete: ${toString (lib.length stepNames)} step(s)" + ''; + }; + }) + ]; +} diff --git a/modules/ax-fleet/default.nix b/modules/ax-fleet/default.nix new file mode 100644 index 000000000..7c341d792 --- /dev/null +++ b/modules/ax-fleet/default.nix @@ -0,0 +1,130 @@ +{ + config, + lib, + options, + ... +}: +# ax on the fleet (DESIGN.md, ~/today/evals-2026-09-23/ax-fleet/). One import +# per host, one kill switch per host (`myAxFleet.enable`), three roles: +# +# control nas k3s server + kubelet (untainted), Substrate and ax +# control planes, the registry. "Hypervisor on NAS." +# harness coordinator k3s agent tainted ate.dev/sandboxClass=gvisor, the +# atelet DaemonSet and the gVisor WorkerPool. "Agent +# harnesses on coordinator." +# inference worker Halogen as today, as a host service. Nothing from +# this PR runs there; one evaluation assertion only. +# +# Folded in and deleted: modules/k3s-fleet.nix (#447; its CIDRs, assertions, +# feature gates and runtime-config are kept below and in ./k3s.nix; its Cilium, +# containerd template and runsc RuntimeClass are dropped because Substrate runs +# its own runsc inside the worker pods) and hosts/nas/state-services.nix (#446; +# only its registry is kept, in ./control.nix). +let + cfg = config.myAxFleet; + + # ── CIDR arithmetic (from #447), so the overlap check is arithmetic ── + ipToInt = + s: + let + o = map lib.toInt (lib.splitString "." s); + at = builtins.elemAt o; + in + (at 0) * 16777216 + (at 1) * 65536 + (at 2) * 256 + (at 3); + + cidrRange = + c: + let + parts = lib.splitString "/" c; + base = ipToInt (builtins.head parts); + bits = lib.toInt (builtins.elemAt parts 1); + size = builtins.foldl' (a: _: a * 2) 1 (lib.range 1 (32 - bits)); + in + { + lo = base; + hi = base + size - 1; + }; + + overlaps = + a: b: + let + x = cidrRange a; + y = cidrRange b; + in + x.lo <= y.hi && y.lo <= x.hi; + + inCidr = ip: c: overlaps "${ip}/32" c; + + roleHost = { + control = "nas"; + harness = "coordinator"; + inference = "worker"; + }; +in +{ + imports = [ + ./interface.nix + ./k3s.nix + ./control.nix + ./harness.nix + ./inference.nix + ./substrate.nix + ./ax.nix + ./gateways.nix + ]; + + config = lib.mkMerge [ + { + # ── OUTSIDE THE GATE, ON PURPOSE (kept from #447) ────────────────── + # k3s's own default pod range (10.42.0.0/16) contains the house LAN. + # These evaluate on every host whether or not the fleet is enabled, so + # an edit to a CIDR fails `nix eval`, not the house's DNS. + assertions = [ + { + assertion = !(overlaps cfg.podCidr cfg.lan.cidr); + message = "modules/ax-fleet: the pod CIDR ${cfg.podCidr} overlaps the house LAN ${cfg.lan.cidr}."; + } + { + assertion = !(overlaps cfg.serviceCidr cfg.lan.cidr); + message = "modules/ax-fleet: the service CIDR ${cfg.serviceCidr} overlaps the house LAN ${cfg.lan.cidr}."; + } + { + assertion = !(overlaps cfg.podCidr cfg.serviceCidr); + message = "modules/ax-fleet: the pod CIDR ${cfg.podCidr} and the service CIDR ${cfg.serviceCidr} overlap."; + } + { + assertion = inCidr cfg.clusterDns cfg.serviceCidr && inCidr cfg.axServerClusterIP cfg.serviceCidr; + message = "modules/ax-fleet: clusterDns and axServerClusterIP must sit inside the service CIDR ${cfg.serviceCidr}."; + } + ]; + } + + (lib.mkIf cfg.enable { + assertions = [ + { + # The test VMs carry the production hostnames, so this holds there too. + assertion = config.networking.hostName == roleHost.${cfg.role}; + message = "modules/ax-fleet: ${config.networking.hostName} has role \"${cfg.role}\", which belongs to ${roleHost.${cfg.role}}. nas is control, coordinator is harness, worker is inference."; + } + { + assertion = inCidr cfg.lan.address cfg.lan.cidr; + message = "modules/ax-fleet: lan.address ${cfg.lan.address} is not inside lan.cidr ${cfg.lan.cidr}."; + } + { + assertion = cfg.role != "control" || cfg.lan.address == cfg.serverAddress; + message = "modules/ax-fleet: the control node's LAN address (${cfg.lan.address}) must equal serverAddress (${cfg.serverAddress})."; + } + ]; + }) + + # One switch per host: the harness role brings the ax and kubectl clients + # (modules/ax-client.nix, #454) with it. mkDefault, so a host can still + # say no. Guarded on the option existing: the NAS does not import + # ax-client.nix. + (lib.mkIf (cfg.enable && cfg.role == "harness") ( + lib.optionalAttrs (options ? myAxClient) { + myAxClient.enable = lib.mkDefault true; + } + )) + ]; +} diff --git a/modules/ax-fleet/gateways.nix b/modules/ax-fleet/gateways.nix new file mode 100644 index 000000000..fd6f8a38c --- /dev/null +++ b/modules/ax-fleet/gateways.nix @@ -0,0 +1,188 @@ +{ + config, + lib, + inputs, + ... +}: +# ax-fleet gateways: the egress default for every atespace the fleet declares. +# +# Stock ax v0.3.0 fails open: a Task with no `gateway:`, or naming a Gateway +# its atespace lacks, gets an allow-all EgressPolicy (reconciler.go "Default +# to allow all egress"; the client turns "*" into EgressRule{All}, and +# Substrate ignores the port). The fleet does not patch ax (Tom, 2026-09-23 +# 08:45Z), so the default is closed from outside it, in three layers: +# +# 1. Declared: every atespace in myAxFleet.ax.atespaces names a default +# Gateway with a non-empty allowlist and no allow-all host (assertion +# below). The bootstrap step 80-ax-gateways applies every declared +# Gateway on the NAS, idempotently. +# 2. Enforced: ax-fleet-gateway-default (control role) polls each declared +# atespace and points any Task without a usable gateway at the default, +# through ax's own UpdateTask; the controller then replaces the actor's +# egress policy with the default's allowlist. The window between a +# gateway-less Task's first reconcile and the repoint is bounded by the +# poll interval plus one reconcile; checks.ax-fleet measures it +# (33-gateway-default, record gateway_default_repoint). +# 3. The link: capacity 0 on a missing gateway (B10), so the floor never +# dispatches a Task that would need layer 2. +# +# Below all three, the NAS forward chain (control.nix) keeps pods off every +# private range but Halogen, whatever policy ax writes. +let + cfg = config.myAxFleet; + system = "x86_64-linux"; + fleetPkgs = inputs.nixpkgs.legacyPackages.${system}; + ax = inputs.self.packages.${system}.ax; + enforcer = fleetPkgs.callPackage ../../pkgs/ax-gateway-default { inherit ax; }; + on = cfg.enable && cfg.role == "control"; + + halogenHost = builtins.head (lib.splitString ":" cfg.halogenEndpoint); + halogenPort = lib.toInt (lib.last (lib.splitString ":" cfg.halogenEndpoint)); + halogenRule = { + host = "${halogenHost}/32"; + port = halogenPort; + }; + allowAll = [ + "*" + "0.0.0.0/0" + "::/0" + ]; + + hostRule = lib.types.submodule { + options = { + host = lib.mkOption { + type = lib.types.str; + description = "A hostname pattern or a CIDR, as ax's HostRule.host."; + }; + port = lib.mkOption { + type = lib.types.port; + description = "Carried for the record; ax v0.3.0 drops it and the NAS chain enforces ports."; + }; + }; + }; + + atespaceType = lib.types.submodule { + options = { + defaultGateway = lib.mkOption { + type = lib.types.str; + default = "default"; + description = "The Gateway a Task without a usable gateway is pointed at."; + }; + gateways = lib.mkOption { + type = lib.types.attrsOf (lib.types.listOf hostRule); + default = { }; + description = "Gateway name -> egress allowlist, applied by the bootstrap."; + }; + }; + }; + + gatewayDoc = + ns: name: hosts: + builtins.toJSON { + apiVersion = "ax.io/v1alpha1"; + kind = "Gateway"; + metadata = { + inherit name; + atespace = ns; + }; + spec.egress.allowlist.hosts = hosts; + }; + + axServer = "${cfg.axServerClusterIP}:8080"; + spacesArg = lib.concatStringsSep "," ( + lib.mapAttrsToList (ns: a: "${ns}=${a.defaultGateway}") cfg.ax.atespaces + ); +in +{ + options.myAxFleet.ax = { + atespaces = lib.mkOption { + type = lib.types.attrsOf atespaceType; + default = { + ${cfg.ax.atespace}.gateways = { + default = [ halogenRule ]; + halogen = [ halogenRule ]; + }; + # ax's CLI default when -a is not given. + default.gateways.default = [ halogenRule ]; + }; + defaultText = lib.literalExpression ''{ fleet.gateways = { default = [ halogen ]; halogen = [ halogen ]; }; default.gateways.default = [ halogen ]; }''; + description = "Every atespace the fleet creates, each with its Gateways and the default one."; + }; + gatewayDefaultInterval = lib.mkOption { + type = lib.types.str; + default = "2s"; + description = "ax-fleet-gateway-default's poll interval (a Go duration). A trade between the fail-open window and API load, not an estimate."; + }; + }; + + config = lib.mkMerge [ + (lib.mkIf cfg.enable { + assertions = lib.flatten ( + lib.mapAttrsToList (ns: a: [ + { + assertion = a.gateways ? ${a.defaultGateway}; + message = "myAxFleet.ax.atespaces.${ns}: the default Gateway \"${a.defaultGateway}\" is not declared in its gateways; stock ax would give a gateway-less Task allow-all egress."; + } + { + assertion = (a.gateways.${a.defaultGateway} or [ ]) != [ ]; + message = "myAxFleet.ax.atespaces.${ns}: the default Gateway \"${a.defaultGateway}\" has an empty allowlist, which stock ax treats as allow-all."; + } + { + assertion = lib.all (r: !(lib.elem r.host allowAll)) (a.gateways.${a.defaultGateway} or [ ]); + message = "myAxFleet.ax.atespaces.${ns}: the default Gateway \"${a.defaultGateway}\" must not allow every host."; + } + ]) cfg.ax.atespaces + ) + ++ [ + { + assertion = cfg.ax.atespaces ? ${cfg.ax.atespace}; + message = "myAxFleet.ax.atespaces must declare myAxFleet.ax.atespace (\"${cfg.ax.atespace}\")."; + } + ]; + }) + + (lib.mkIf on { + myAxFleet.bootstrap."80-ax-gateways" = '' + # 80-ax-gateways: every declared Gateway, default ones included + # (modules/ax-fleet/gateways.nix). Idempotent: ax apply updates. + export AX_SERVER=http://${axServer} + for _ in $(seq 1 300); do + ${lib.getExe fleetPkgs.curl} -fsS -o /dev/null --max-time 5 "$AX_SERVER/healthz" && break + sleep 2 + done + ${lib.concatStrings ( + lib.flatten ( + lib.mapAttrsToList ( + ns: a: + lib.mapAttrsToList (name: hosts: '' + echo "gateway ${ns}/${name}" + ${ax}/bin/ax -a ${ns} apply -f ${fleetPkgs.writeText "gateway-${ns}-${name}.json" (gatewayDoc ns name hosts)} + '') a.gateways + ) cfg.ax.atespaces + ) + )} + echo "ax gateways declared: ${spacesArg}" + ''; + + systemd.services.ax-fleet-gateway-default = { + description = "ax-fleet: point Tasks without a usable gateway at their atespace's default Gateway"; + wantedBy = [ "multi-user.target" ]; + after = [ + "k3s.service" + "ax-fleet-bootstrap.service" + ]; + serviceConfig = { + ExecStart = "${lib.getExe enforcer} -server ${axServer} -atespaces ${spacesArg} -interval ${cfg.ax.gatewayDefaultInterval}"; + Restart = "always"; + RestartSec = 5; + # Root: the NAS output chain lets only root reach the Service range. + NoNewPrivileges = true; + ProtectSystem = "strict"; + ProtectHome = true; + PrivateTmp = true; + CapabilityBoundingSet = ""; + }; + }; + }) + ]; +} diff --git a/modules/ax-fleet/harness.nix b/modules/ax-fleet/harness.nix new file mode 100644 index 000000000..b9c3f2bd4 --- /dev/null +++ b/modules/ax-fleet/harness.nix @@ -0,0 +1,381 @@ +{ + config, + lib, + pkgs, + ... +}: +# The harness node: the coordinator. "Agent harnesses on coordinator" (Tom, +# 2026-09-23). A k3s agent tainted with upstream Substrate's own key, so only +# atelet and the WorkerPool's gVisor worker pods land here; nothing that +# schedules or remembers does (the wifi leg is the least available link). +# DESIGN.md sections 6.2, 6.4, 6.5, 6.6, 9. +# +# What must NOT change for Tom at switch, and how this file keeps it: +# - NetworkManager is not restarted. `networking.networkmanager.unmanaged` +# rewrites NetworkManager.conf, NetworkManager's restart trigger with +# stopIfChanged = true (MEASURED nix eval), which would drop the desk's +# wifi. A conf.d drop-in plus `nmcli general reload conf` instead. +# - the tailnet: flannel and kube-proxy bind the LAN leg only, nothing is +# published, and the guard chain below keeps pods, wifi and the tailnet +# apart even though k3s turns ip_forward on (judge 1's second risk). The +# guard chain polices FORWARD only; pods reaching the coordinator HOST +# (sshd, which accepts passwords) go through INPUT, so pod interfaces get +# their own refusal at the head of nixos-fw (fix round 2, see podInput). The +# chain covers the direct path; the path routed through the NAS into +# VXLAN is closed twice, by the NAS's prerouting range guard +# (control.nix) and by the flannel.1 source rule here. Plain VXLAN on the +# wifi leg is accepted by source address only (spoofable over wifi); see +# DESIGN.md Unknowns. +# - the kernel's panic behaviour: kubelet's kernel.panic / panic_on_oops / +# overcommit values are put back after it starts +# (myAxFleet.kubelet.keepHostKernelTunables, k3s.nix). +# - herdr and every user unit: only system units are added. +# - no containerd template, no runsc on PATH, no RuntimeClass (Substrate +# runs its own runsc in the worker pods). +let + cfg = config.myAxFleet; + # The firewall half (guard chain, pod-input refusal, API owner match) is + # rendered for the harness ROLE, whatever `enable` says (fix round 3): the + # kill switch stops k3s with KillMode=process, so gVisor pods, cni0 and + # flannel.1 outlive the switch until ax-fleet-teardown runs. The rules are + # inert once those interfaces are gone. + roleOn = cfg.role == "harness"; + on = cfg.enable && roleOn; + + lan = cfg.lan.interface; + # Every NIC that can reach the house LAN (fix round 3): the desk's wired + # port enp191s0 has an autoconnecting DHCP profile (MEASURED nmcli). + lans = [ lan ] ++ cfg.lan.extraInterfaces; + apiPort = lib.last (lib.splitString ":" cfg.apiListen); + podIfs = [ + "cni0" + "flannel.1" + ]; + + ipt = "${pkgs.iptables}/bin/iptables -w"; + ip6t = "${pkgs.iptables}/bin/ip6tables -w"; + registryPort = lib.last (lib.splitString ":" cfg.registry); + + # ── the guard chain (DESIGN 6.4), in mangle FORWARD, position 1 ── + # It runs before the filter rules kube-proxy and flannel insert. + # + # First line, a correction to the design's list: established and related + # traffic RETURNs. Without it the "wifi into pods only from the NAS" line + # also drops every REPLY to a coordinator pod's own outbound connection + # (atelet's anonymous GCS fetch, a worker pod reaching the LAN), because the + # reply arrives on the LAN leg from a source that is not the NAS. Only NEW + # flows are policed, which is the property the guard exists for. + # + # Deny by default (fix round 3). Round 2 listed interfaces (`-o ${lan}`), + # so a second LAN leg (the wired port, a podman bridge) matched no DROP. + # Now: pod egress is policed by DESTINATION on every output interface, and + # nothing enters a pod interface unless a rule below lets it. + guardRules = [ + "-m conntrack --ctstate ESTABLISHED,RELATED -j RETURN" + # Into pods over VXLAN only from the pod network (fix round 1). Real + # peers, the NAS host included (its flannel.1 address), are sourced + # from the pod CIDR. Defence in depth, not the fix: LAN traffic the NAS + # routes into VXLAN (MEASURED bypass, worker -> nas -> flannel.1 -> + # harness pod, rc=0) is likely masqueraded by flannel's own rule to the + # NAS's flannel.1 address (INFERRED), so the NAS's prerouting range + # guard (control.nix) is what closes that path. This rule drops VXLAN + # payloads whose inner source is outside the pod CIDR. + "-i flannel.1 ! -s ${cfg.podCidr} -j DROP" + # In-cluster: bridge-local and across nodes. + "-i cni0 -o cni0 -j RETURN" + "-i cni0 -o flannel.1 -j RETURN" + "-i flannel.1 -o cni0 -j RETURN" + ] + # Into pods from outside the cluster: only the NAS host (hostPorts, the + # apiserver), on a LAN leg. Everything else, any interface, is dropped. + ++ map (l: "-i ${l} -o cni0 -s ${cfg.serverAddress} -j RETURN") lans + ++ [ + "-o cni0 -j DROP" + "-o flannel.1 -j DROP" + "-i flannel.1 -j DROP" + # Out of pods. Pod traffic leaving a LAN leg is masqueraded to this + # host's address and would inherit every NAS rule that trusts the + # coordinator (ssh, NFS, media, paperless). From pods, the LAN gets only + # the apiserver and the registry on the NAS; the internet (atelet's GCS + # fetch) is unaffected. Everything else in-cluster rides flannel.1. + "-i cni0 -d ${cfg.serverAddress} -p tcp -m multiport --dports 6443,${registryPort} -j RETURN" + ] + # Every private range, not only the house /24 (fix round 1), on ANY output + # interface (fix round 3): when NetworkManager falls back to the Freebox + # profile (hosts/coordinator/uplink-nas.nix) the leg is a DHCP subnet this + # module does not know, and 100.64/10 is the tailnet's range. + ++ map (r: "-i cni0 -d ${r} -j DROP") privateRanges + # The internet, over a LAN leg only; tailscale0, podman0 and anything else + # a pod could be routed to are dropped. + ++ map (l: "-i cni0 -o ${l} -j RETURN") lans + ++ [ "-i cni0 -j DROP" ] + # The tailnet and the house LAN never forward into each other, on any leg. + ++ lib.concatMap ( + g: + lib.concatMap (l: [ + "-i ${l} -o ${g} -j DROP" + "-i ${g} -o ${l} -j DROP" + ]) lans + ) cfg.guardInterfaces; + + privateRanges = lib.unique [ + cfg.lan.cidr + "10.0.0.0/8" + "172.16.0.0/12" + "192.168.0.0/16" + "100.64.0.0/10" + ]; + + # ── pods never open a connection to the coordinator host (fix round 2) ── + # MEASURED by the round-2 review: from a pod, `nc 10.200.0.1 22` and the + # node's LAN address answered SSH-2.0-OpenSSH, because nixos-fw accepts 22 on + # every interface and the guard chain above sees FORWARD only. Nothing on + # the Substrate path needs a NEW pod-to-host flow: hostPorts and ClusterIPs + # are DNATed through FORWARD, kubelet reaches pods (OUTPUT, replies are + # ESTABLISHED), atelet talks to kubelet and containerd over unix sockets, and + # kubectl exec/logs ride the agent tunnel. First rules of nixos-fw, so they + # run before its ESTABLISHED accept and every port rule; IPv6 too (link-local + # addresses on cni0 and the veths). + # Idempotent (fix round 4): deleted, then inserted, so guardApply can re-run + # it over a live nixos-fw without duplicating a rule. + podInputSpecs = map ( + p: "-i ${p} -m conntrack --ctstate NEW -m comment --comment ax-fleet-pod-input -j nixos-fw-refuse" + ) podIfs; + podInputCmds = + t: + lib.concatMapStringsSep "\n" (spec: '' + while ${t} -D nixos-fw ${spec} 2>/dev/null; do :; done + ${t} -I nixos-fw 1 ${spec}'') podInputSpecs; + + # ── the ax API and the cluster ranges: root, apiUsers and the proxy only ── + # (fix round 3) The round-3 review MEASURED an unprivileged user applying a + # Gateway with host 0.0.0.0/0 through the loopback proxy (the API has no + # authentication, upstream #376). The same user could reach the ax-server + # ClusterIP or pod directly through kube-proxy's OUTPUT DNAT, so the match + # covers the cluster ranges as well as the proxy's port. filter OUTPUT sees + # the post-DNAT address. Packets without a full socket (kernel replies, + # TIME_WAIT) pass. + apiUsersAll = [ "root" ] ++ cfg.apiUsers ++ [ "ax-server-proxy" ]; + apiRules = [ + "-m owner ! --socket-exists -j RETURN" + ] + ++ map (u: "-m owner --uid-owner ${u} -j RETURN") apiUsersAll + ++ [ + "-d 127.0.0.1 -p tcp --dport ${apiPort} -j REJECT --reject-with tcp-reset" + "-d ${cfg.podCidr} -j REJECT" + "-d ${cfg.serviceCidr} -j REJECT" + ]; + + guardStart = '' + # ax-fleet guard chain (idempotent) + ${ipt} -t mangle -N ax-fleet-guard 2>/dev/null || true + ${ipt} -t mangle -F ax-fleet-guard + while ${ipt} -t mangle -D FORWARD -j ax-fleet-guard 2>/dev/null; do :; done + ${ipt} -t mangle -I FORWARD 1 -j ax-fleet-guard + ${lib.concatMapStringsSep "\n" (r: "${ipt} -t mangle -A ax-fleet-guard ${r}") guardRules} + # pods to the host: refused (podInputSpecs) + ${podInputCmds ipt} + ${lib.optionalString config.networking.enableIPv6 (podInputCmds ip6t)} + # the ax API and the cluster ranges from this host: owner match (fix round 3) + ${ipt} -N ax-fleet-api 2>/dev/null || true + ${ipt} -F ax-fleet-api + while ${ipt} -D OUTPUT -j ax-fleet-api 2>/dev/null; do :; done + ${ipt} -I OUTPUT 1 -j ax-fleet-api + ${lib.concatMapStringsSep "\n" (r: "${ipt} -A ax-fleet-api ${r}") apiRules} + ''; + + # (fix round 4) The same rules as a script the teardown runs after + # k3s-killall.sh: the killall's `iptables-save | grep -iv flannel | + # iptables-restore` (REPORTED k3s-killall.sh:90 in the pinned k3s) deletes + # every rule naming flannel.1, the guard's and the pod-input refusal's + # included, and nothing reloads the firewall before k3s starts again. + guardApply = pkgs.writeShellScript "ax-fleet-guard-apply" ('' + set -eu + '' + guardStart); + + guardStop = '' + while ${ipt} -t mangle -D FORWARD -j ax-fleet-guard 2>/dev/null; do :; done + ${ipt} -t mangle -F ax-fleet-guard 2>/dev/null || true + ${ipt} -t mangle -X ax-fleet-guard 2>/dev/null || true + while ${ipt} -D OUTPUT -j ax-fleet-api 2>/dev/null; do :; done + ${ipt} -F ax-fleet-api 2>/dev/null || true + ${ipt} -X ax-fleet-api 2>/dev/null || true + ''; + + nmDropIn = '' + # ax-fleet (modules/ax-fleet/harness.nix): k3s's own interfaces are never + # NetworkManager's. Appended (+=) to whatever NetworkManager.conf lists. + [keyfile] + unmanaged-devices+=interface-name:cni0;interface-name:flannel*;interface-name:veth* + '' + # (fix round 3) The other LAN-capable NICs never take the house routes from + # ${lan} while it is up: a profile that leaves ipv4.route-metric at -1 (the + # desk's "Wired connection 1", MEASURED) takes this default instead of + # ethernet's 100, which beat the wifi's 600. The NAS admits the harness to + # 6443, the registry and VXLAN by ${lan}'s address only. + + lib.optionalString (cfg.lan.extraInterfaces != [ ]) '' + + [connection-ax-fleet-extra-lan] + match-device=${lib.concatMapStringsSep ";" (i: "interface-name:${i}") cfg.lan.extraInterfaces} + ipv4.route-metric=${toString cfg.lan.extraRouteMetric} + ipv6.route-metric=${toString cfg.lan.extraRouteMetric} + ''; + + kubeconfigScript = pkgs.writeShellApplication { + name = "ax-fleet-kubeconfig"; + runtimeInputs = [ + pkgs.openssh + pkgs.coreutils + ]; + text = '' + # Fetch the cluster admin kubeconfig from the NAS over the existing ssh + # trust into ~/.kube/config (0600). The content is never printed. + umask 077 + mkdir -p "$HOME/.kube" + tmp=$(mktemp "$HOME/.kube/.config.XXXXXX") + trap 'rm -f "$tmp"' EXIT + ssh "''${AX_FLEET_NAS:-nas}" cat /etc/ax-fleet/admin.kubeconfig > "$tmp" + [ -s "$tmp" ] || { echo "ax-fleet-kubeconfig: empty kubeconfig from the NAS" >&2; exit 1; } + mv "$tmp" "$HOME/.kube/config" + trap - EXIT + echo "wrote $HOME/.kube/config" + ''; + }; +in +{ + config = lib.mkMerge [ + (lib.mkIf roleOn { + networking.firewall.extraCommands = guardStart; + networking.firewall.extraStopCommands = guardStop; + # A fixed uid for the proxy, so the owner match can name it (DynamicUser + # allocates from a range any other DynamicUser unit shares). + users.users.ax-server-proxy = { + isSystemUser = true; + group = "ax-server-proxy"; + }; + users.groups.ax-server-proxy = { }; + # The teardown leaves a guard the generation declares (see pkgs/ax-fleet-teardown). + environment.etc."ax-fleet/guard-declared".text = "harness\n"; + environment.etc."ax-fleet/guard-apply".source = guardApply; + assertions = [ + { + assertion = builtins.all (u: config.users.users ? ${u}) cfg.apiUsers; + message = "modules/ax-fleet/harness.nix: every myAxFleet.apiUsers entry must be a declared user (iptables resolves the name when the firewall starts)."; + } + { + assertion = !(builtins.elem lan cfg.lan.extraInterfaces); + message = "modules/ax-fleet/harness.nix: myAxFleet.lan.extraInterfaces must not repeat lan.interface."; + } + ]; + }) + (lib.mkIf on { + myAxFleet.kubelet = { + # Memory: Tom's seats, Chrome and a coordinator Halogen feel pressure + # after the sandboxes are evicted, never before. CPU is the slices below: + # system-reserved only shrinks kubepods.slice's weight, it protects + # nothing. + systemReserved = lib.mkDefault "cpu=8,memory=32Gi"; + kubeReserved = lib.mkDefault "cpu=1,memory=2Gi"; + # Every signal, not only memory (fix round 4): a set --eviction-hard + # REPLACES kubelet's whole default map (MEASURED in the round-3 VM log: + # HardEvictionThresholds=[memory.available] only), which dropped k3s's + # nodefs/imagefs defaults. / here is also /nix/store, journald and the + # coordinator's postgres (MEASURED findmnt: one nvme partition), so a + # sandbox filling its writable layer must be evicted before they ENOSPC. + evictionHard = lib.mkDefault "memory.available<8Gi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<15%,imagefs.inodesFree<5%"; + }; + + # ── CPU: the desk outweighs the sandboxes (fix round 2) ── + # kubelet gives kubepods.slice cpu.weight = 1 + ((allocatable_mcpu * 1024 + # / 1000 - 2) * 9999) / 262142: MEASURED 274 for 7 allocatable CPUs in the + # review VM, INFERRED 899 for the desk's 32 - 8 - 1 = 23. user.slice and + # system.slice are 100 on the live box (MEASURED), so under contention the + # gVisor workers (no CPU limit, substrate.nix) would take about 90 % of the + # CPU from niri, herdr, the seats and Chrome. Weights are work-conserving: + # idle desk CPU still goes to the sandboxes. Both slices get the same + # weight, so their ratio to each other is unchanged. switch-to-configuration + # never restarts a slice; daemon-reload applies the property. + systemd.slices.user.sliceConfig.CPUWeight = lib.mkDefault cfg.kubelet.deskCpuWeight; + systemd.slices.system.sliceConfig.CPUWeight = lib.mkDefault cfg.kubelet.deskCpuWeight; + + services.k3s = { + role = "agent"; + # The IP, not the name `nas`. + serverAddr = "https://${cfg.serverAddress}:6443"; + nodeTaint = [ cfg.harnessTaint ]; + # Registered WITH the version label: ate-setup only labels nodes that + # exist when it runs (MEASURED version.go:113-131), and the coordinator + # joins after the NAS. Without it atelet (version-keyed) never lands. + nodeLabel = [ + "ax.mecattaf.dev/role=harness" + "ate.dev/substrate-version=${cfg.substrateVersion}" + ]; + }; + + # k3s turns ip_forward on at start anyway; the guard chain is what keeps + # it safe. IPv6 forwarding stays 0 (the wifi leg keeps accepting RAs). + # flannel VXLAN from the NAS only; no TCP port is opened. With the switch + # on only: an accept, unlike the guards above. + networking.firewall.extraCommands = '' + ${ipt} -A nixos-fw -i ${lan} -s ${cfg.serverAddress} -p udp --dport 8472 -j nixos-fw-accept + ''; + + boot.kernel.sysctl."net.ipv4.ip_forward" = 1; + # Proxy ARP on the pod interfaces only (fix round 3). Round 2 set + # `conf.default`, but every NIC created after systemd-sysctl copies + # `default`, and the desk's NICs are renamed after it runs (MEASURED boot + # journal: sysctl 6.356 s, enp191s0 6.497 s, wlp192s0 7.493 s), so + # wlp192s0 would have answered ARP for the whole LAN after a reboot. + # systemd's udev rule (99-systemd.rules) runs systemd-sysctl for each new + # interface, which applies these by name and glob as each one appears. + # `flannel/1` is sysctl.d's spelling of flannel.1. + boot.kernel.sysctl."net.ipv4.conf.cni0.proxy_arp" = 1; + boot.kernel.sysctl."net.ipv4.conf.flannel/1.proxy_arp" = 1; + boot.kernel.sysctl."net.ipv4.conf.veth*.proxy_arp" = 1; + + # ── NetworkManager: a drop-in and a config reload, never a restart ── + environment.etc."NetworkManager/conf.d/90-ax-fleet.conf".text = nmDropIn; + systemd.services.ax-fleet-nm-unmanaged = { + description = "ax-fleet: make NetworkManager re-read its conf.d (k3s interfaces unmanaged)"; + wantedBy = [ "multi-user.target" ]; + after = [ "NetworkManager.service" ]; + before = [ "k3s.service" ]; + restartTriggers = [ nmDropIn ]; + serviceConfig = { + Type = "oneshot"; + RemainAfterExit = true; + # Tolerated: with NetworkManager down there is nothing to reload. + ExecStart = "-${pkgs.networkmanager}/bin/nmcli general reload conf"; + }; + }; + systemd.services.k3s.wants = [ "ax-fleet-nm-unmanaged.service" ]; + + # ── ax-server on ${cfg.apiListen}, through kube-proxy's OUTPUT rules ── + # The ClusterIP is never a NodePort: ax's API has no authentication + # (upstream #376). Loopback only, and only root, apiUsers and this proxy + # may connect (apiRules). Not 127.0.0.1:8080 (fix round 3): that is + # ax-conwip's default and ax-mockstack's, which must reach nothing live. + systemd.sockets.ax-server-proxy = { + description = "ax-fleet: ax-server on ${cfg.apiListen}"; + wantedBy = [ "sockets.target" ]; + listenStreams = [ cfg.apiListen ]; + }; + systemd.services.ax-server-proxy = { + description = "ax-fleet: proxy ${cfg.apiListen} to the ax-server ClusterIP"; + requires = [ "ax-server-proxy.socket" ]; + after = [ "ax-server-proxy.socket" ]; + serviceConfig = { + ExecStart = "${config.systemd.package}/lib/systemd/systemd-socket-proxyd ${cfg.axServerClusterIP}:8080"; + User = "ax-server-proxy"; + Group = "ax-server-proxy"; + PrivateTmp = true; + NoNewPrivileges = true; + ProtectSystem = "strict"; + ProtectHome = true; + }; + }; + + environment.systemPackages = [ kubeconfigScript ]; + }) + ]; +} diff --git a/modules/ax-fleet/inference.nix b/modules/ax-fleet/inference.nix new file mode 100644 index 000000000..fd44083d7 --- /dev/null +++ b/modules/ax-fleet/inference.nix @@ -0,0 +1,31 @@ +{ + config, + lib, + ... +}: +# The inference node: the worker. "Halogen inference mainly on worker" (Tom, +# 2026-09-23). Halogen stays the worker's host service; sandboxes reach it at +# halogenEndpoint through Substrate's egress gateway on the NAS (SNAT to the +# NAS's LAN address). Nothing from this PR runs here at runtime and the worker +# is not switched in this motion: this file is one evaluation assertion, that +# Halogen's port stays open on the LAN leg. +let + cfg = config.myAxFleet; + on = cfg.enable && cfg.role == "inference"; + port = lib.toInt (lib.last (lib.splitString ":" cfg.halogenEndpoint)); + lanPorts = config.networking.firewall.interfaces.${cfg.lan.interface}.allowedTCPPorts or [ ]; +in +{ + config = lib.mkIf on { + assertions = [ + { + assertion = builtins.elem port lanPorts; + message = "modules/ax-fleet/inference.nix: Halogen's port ${toString port} must stay open on ${cfg.lan.interface}; ax sandboxes reach ${cfg.halogenEndpoint} through the egress gateway on the NAS."; + } + { + assertion = !config.services.k3s.enable; + message = "modules/ax-fleet/inference.nix: the worker is not a k3s node in this motion."; + } + ]; + }; +} diff --git a/modules/ax-fleet/interface.nix b/modules/ax-fleet/interface.nix new file mode 100644 index 000000000..668b80444 --- /dev/null +++ b/modules/ax-fleet/interface.nix @@ -0,0 +1,314 @@ +{ + lib, + inputs, + ... +}: +# ax on the fleet: the options, and nothing else. +# +# Design: ~/today/evals-2026-09-23/ax-fleet/DESIGN.md section 5. Tom's ruling +# (2026-09-23, verbatim): "this is a dotfiles task to do on my nixos fleet. the +# decisions there were already made: hypervisor on NAS, agent harnesses on +# coordinator, halogen inference mainly on worker (can also run on coordinator +# if we need redundancy or a second parallel halogen task)." +# +# `enable` is THE kill switch. Its default is false, so a host that imports +# this module and says nothing renders nothing from it. The substrate and ax +# tracks write only the three extension points at the bottom (`manifests`, +# `registrySeed`, `bootstrap`); everything above them is read by the cluster +# modules next to this file. +let + inherit (lib) mkOption mkEnableOption types; +in +{ + options.myAxFleet = { + enable = mkEnableOption "this host's part of ax on the fleet (k3s, Substrate, ax). THE kill switch"; + + role = mkOption { + type = types.enum [ + "control" + "harness" + "inference" + ]; + description = '' + control = the NAS (k3s server, Substrate and ax control planes, the + registry). harness = the coordinator (k3s agent, gVisor sandboxes). + inference = the worker (Halogen as a host service; nothing from this + PR at runtime). Asserted against the hostname in ./default.nix. + ''; + }; + + lan = { + interface = mkOption { + type = types.str; + example = "enp1s0"; + description = "The LAN leg: enp1s0 (nas), wlp192s0 (coordinator), enp191s0 (worker). Tests use eth1."; + }; + address = mkOption { + type = types.str; + example = "10.42.0.1"; + description = "This host's address on the LAN leg."; + }; + cidr = mkOption { + type = types.str; + default = "10.42.0.0/24"; + }; + extraInterfaces = mkOption { + type = types.listOf types.str; + default = [ ]; + example = [ "enp191s0" ]; + description = '' + Other NICs that can reach the house LAN (fix round 3): the + coordinator's wired port enp191s0 has an autoconnecting DHCP profile. + On the harness role the guard chain treats them as LAN legs, and a + NetworkManager drop-in gives them `extraRouteMetric`, so the LAN + routes stay on `interface` while it is up. Tests use [ "eth3" ]. + ''; + }; + extraRouteMetric = mkOption { + type = types.int; + default = 700; + description = "Route metric for `extraInterfaces` (NetworkManager's wifi default is 600, ethernet 100)."; + }; + }; + + apiListen = mkOption { + type = types.str; + default = "127.0.0.1:8099"; + description = '' + Loopback address of ax-server-proxy.socket on the harness (fix round 3: + not 127.0.0.1:8080, which ax-conwip's default and ax-mockstack own). + AX_SERVER points here. + ''; + }; + apiUsers = mkOption { + type = types.listOf types.str; + default = [ "tom" ]; + description = '' + Local users (besides root) that may open connections to `apiListen` + and to the cluster ranges from the harness host (fix round 3). The ax + API has no authentication (upstream #376); everyone else is refused by + an owner match in OUTPUT. + ''; + }; + + guardInterfaces = mkOption { + type = types.listOf types.str; + default = [ "tailscale0" ]; + description = "Interfaces the coordinator guard chain isolates from pods and from the LAN leg (the tailnet). Tests use [ \"eth2\" ]."; + }; + + serverAddress = mkOption { + type = types.str; + default = "10.42.0.1"; + description = "The k3s server the harness agent dials: the IP, never the name."; + }; + harnessAddresses = mkOption { + type = types.listOf types.str; + default = [ "10.42.0.2" ]; + description = "LAN addresses of harness nodes: the only sources the NAS admits to 6443, the registry and VXLAN."; + }; + podCidr = mkOption { + type = types.str; + default = "10.200.0.0/16"; + }; + serviceCidr = mkOption { + type = types.str; + default = "10.201.0.0/16"; + }; + clusterDns = mkOption { + type = types.str; + default = "10.201.0.10"; + }; + axServerClusterIP = mkOption { + type = types.str; + default = "10.201.0.80"; + }; + nodePortRange = mkOption { + type = types.str; + default = "30000-30999"; + description = "Kept below 32400 (Plex on the NAS, a listener on the coordinator)."; + }; + + k3sPackage = mkOption { + type = types.package; + default = inputs.nixpkgs.legacyPackages.x86_64-linux.k3s_1_36; + defaultText = lib.literalExpression "inputs.nixpkgs.legacyPackages.x86_64-linux.k3s_1_36"; + description = "One k3s derivation for every node: the one the 2026-09-23 probe ran (1.36.2+k3s1 from the dotfiles nixpkgs pin, not stable's)."; + }; + + k3sTokenFile = mkOption { + type = types.nullOr types.str; + default = null; + description = '' + null (the default) means the agenix secret secrets/k3s-token.age, + declared by ./k3s.nix. Tests set a path to a plain file instead. + ''; + }; + + k3sAgentTokenFile = mkOption { + type = types.nullOr types.str; + default = null; + description = '' + The agent join credential (server --agent-token-file, agent + --token-file). null (the default) means the agenix secret + secrets/k3s-agent-token.age, declared by ./k3s.nix. Tests set a path + to a plain file instead. + ''; + }; + + stateRoot = mkOption { + type = types.str; + default = "/mnt/fast"; + description = "The NAS's fast tier. k3s hot state (/var/lib/rancher, /var/lib/kubelet, /var/log/pods) is bind-mounted from /k3s."; + }; + localPathRoot = mkOption { + type = types.str; + default = "/mnt/nas/services/ax-fleet/local-path"; + description = "Every PersistentVolume (k3s local-path), on the data pool."; + }; + registryRoot = mkOption { + type = types.str; + default = "/mnt/nas/services/ax-fleet/registry"; + }; + registry = mkOption { + type = types.str; + default = "10.42.0.1:5000"; + description = "The NAS registry, plain HTTP, LAN only, scoped to the coordinator by nftables."; + }; + + substrateVersion = mkOption { + type = types.str; + default = "d277088b"; + description = "One string: the harness node label value, the component image tag and ate-setup's VERSION."; + }; + harnessTaint = mkOption { + type = types.str; + default = "ate.dev/sandboxClass=gvisor:NoSchedule"; + description = "Upstream Substrate's own taint key (atelet tolerates it)."; + }; + + kubelet = { + systemReserved = mkOption { + type = types.nullOr types.str; + default = null; + description = "kubelet --system-reserved. Defaults per role in control.nix / harness.nix; tests shrink it."; + }; + kubeReserved = mkOption { + type = types.nullOr types.str; + default = null; + }; + evictionHard = mkOption { + type = types.nullOr types.str; + default = null; + }; + deskCpuWeight = mkOption { + type = types.ints.between 1 10000; + default = 10000; + description = '' + Harness role: cpu.weight for user.slice and system.slice, against + kubepods.slice's kubelet-computed weight (INFERRED 899 on the desk). + The desk wins CPU contention; idle CPU still goes to the sandboxes. + ''; + }; + keepHostKernelTunables = mkOption { + type = types.bool; + default = true; + description = '' + kubelet sets kernel.panic=10, kernel.panic_on_oops=1 and + vm.overcommit_memory=1 when it starts (REPORTED by the VM receipt). + On the desk that turns any kernel oops into a reboot 10 s later, + dropping herdr and every seat. true: ax-fleet-kernel-tunables puts + the three keys back to the values recorded before k3s first ran, + after every kubelet start. false: kubelet's values stay. Tom's + ruling is pending (Waiting on you, 2026-09-23); true is the default + because it changes nothing for Tom at switch. + ''; + }; + }; + + workerPool = { + replicas = mkOption { + type = types.ints.positive; + default = 2; + }; + memoryLimit = mkOption { + type = types.str; + default = "16Gi"; + }; + # (fix round 4) Each worker pod's writable layer, emptyDirs and logs + # live on the harness node's / (the desk's /nix/store disk). The limit + # evicts one runaway sandbox's pod before the node-level nodefs + # threshold (kubelet.evictionHard) has to evict everything. + ephemeralStorageRequest = mkOption { + type = types.str; + default = "1Gi"; + }; + ephemeralStorageLimit = mkOption { + type = types.str; + default = "32Gi"; + }; + unreachableTolerationSeconds = mkOption { + type = types.ints.unsigned; + default = 3600; + description = "A configuration value, not an estimate: how long worker pods tolerate an unreachable or not-ready coordinator."; + }; + }; + + clusterClientUids = mkOption { + type = types.listOf types.ints.unsigned; + default = [ ]; + description = '' + Control role (fix round 4): numeric uids, besides root, that may open + connections from the NAS host to the pod and Service ranges. Numeric + because nftables resolves user names at load time and the ruleset is + checked in the build sandbox. Default: none. + ''; + }; + + halogenEndpoint = mkOption { + type = types.str; + default = "10.42.0.5:8731"; + }; + + # ── Extension points. The substrate and ax tracks write ONLY these. ── + manifests = mkOption { + type = types.attrsOf ( + types.submodule { + options.source = mkOption { + type = types.path; + description = "A YAML file; linked into k3s's auto-deploy directory on the NAS as .yaml."; + }; + } + ); + default = { }; + description = "Our own ax-system objects, auto-deployed by k3s on the NAS. Substrate itself is installed by ate-setup (bootstrap), not here."; + }; + + registrySeed = mkOption { + type = types.attrsOf ( + types.submodule { + options = { + oci = mkOption { + type = types.package; + description = "An OCI image layout at the output root (oci-layout, index.json, blobs/) plus a `digest` file holding the manifest digest (sha256:...)."; + }; + repo = mkOption { type = types.str; }; + tag = mkOption { type = types.str; }; + }; + } + ); + default = { }; + description = "Images the NAS copies into its own registry (skopeo --preserve-digests) before the bootstrap runs."; + }; + + bootstrap = mkOption { + type = types.attrsOf types.lines; + default = { }; + description = '' + "NN-name" -> idempotent bash, run by ax-fleet-bootstrap.service on the + NAS in name order, with KUBECONFIG set to the k3s admin kubeconfig. + The cluster track owns 10-api and 60-kubeconfig. + ''; + }; + }; +} diff --git a/modules/ax-fleet/k3s.nix b/modules/ax-fleet/k3s.nix new file mode 100644 index 000000000..a2f37b2cd --- /dev/null +++ b/modules/ax-fleet/k3s.nix @@ -0,0 +1,329 @@ +{ + config, + lib, + pkgs, + options, + ... +}: +# k3s, common to the two cluster nodes (control = nas, harness = coordinator). +# DESIGN.md sections 6.2, 6.5 and 7. What the 2026-09-23 probe MEASURED +# working on k3s 1.36.2 (probe-build/vm/flake.nix) is the core: k3s_1_36 from +# the dotfiles nixpkgs pin, the three feature gates on every component, the +# certificates.k8s.io/v1beta1 runtime-config, overlay and br_netfilter, the +# inotify raises, a registry mirror in registries.yaml. +# +# What is deliberately NOT here (DESIGN.md D2, D4, 6.2): +# - Cilium. flannel VXLAN plus kube-proxy (iptables), bound to the LAN leg +# only. No BPF socket-LB next to tailscale0. +# - a containerd template, runsc on the unit PATH, a RuntimeClass. Substrate +# runs its own runsc inside the worker pods and uses no RuntimeClass +# (MEASURED, read-substrate.md 2); containerd keeps k3s's generated config. +# - --data-dir. The NixOS module links manifests and images into fixed +# paths under /var/lib/rancher/k3s, so state moves by bind mount instead +# (./control.nix). +let + cfg = config.myAxFleet; + clusterRole = options.myAxFleet.role.isDefined && (cfg.role == "control" || cfg.role == "harness"); + cluster = cfg.enable && clusterRole; + + teardown = pkgs.callPackage ../../pkgs/ax-fleet-teardown { k3s = cfg.k3sPackage; }; + + # The feature gates Substrate's podcertcontroller needs, on all three + # components (A5a and the probe MEASURED that kubelet accepts all three). + gates = "ClusterTrustBundle=true,ClusterTrustBundleProjection=true,PodCertificateRequest=true"; + + kubeletArgs = [ + "feature-gates=${gates}" + ] + ++ lib.optional (cfg.kubelet.systemReserved != null) "system-reserved=${cfg.kubelet.systemReserved}" + ++ lib.optional (cfg.kubelet.kubeReserved != null) "kube-reserved=${cfg.kubelet.kubeReserved}" + ++ lib.optional (cfg.kubelet.evictionHard != null) "eviction-hard=${cfg.kubelet.evictionHard}"; + + commonFlags = [ + "--resolv-conf=/etc/ax-fleet/resolv.conf" + "--flannel-iface=${cfg.lan.interface}" + # NodePorts (none are declared) could only ever bind the LAN /32, and + # kube-proxy never sets route_localnet. + "--kube-proxy-arg=nodeport-addresses=${cfg.lan.address}/32" + "--kube-proxy-arg=iptables-localhost-nodeports=false" + # kube-proxy leaves the host's conntrack table alone (fix round 2). Its + # defaults set nf_conntrack_tcp_timeout_established to 86400 and + # close_wait to 3600 (MEASURED in the review VM; the live NAS and desk + # read 432000 and 60), which on the house router would drop the SNAT + # entry of any NATed flow idle for a day. 0 = do not touch. + "--kube-proxy-arg=conntrack-tcp-timeout-established=0s" + "--kube-proxy-arg=conntrack-tcp-timeout-close-wait=0s" + "--kube-proxy-arg=conntrack-max-per-core=0" + ] + ++ map (a: "--kubelet-arg=${a}") kubeletArgs; + + # containerd pulls every pod image through the NAS registry; for anything + # the mirror lacks it falls back to the upstream registry, so a missing + # seed degrades rather than breaks on the fleet. The offline VM test is what + # proves that nothing is missing. + registriesYaml = '' + mirrors: + "${cfg.registry}": + endpoint: + - "http://${cfg.registry}" + "docker.io": + endpoint: + - "http://${cfg.registry}" + "registry.k8s.io": + endpoint: + - "http://${cfg.registry}" + ''; + + # Values recorded before k3s first runs on this host, restored by + # ax-fleet-teardown. The kubelet sets the three kernel keys itself + # (INFERRED from upstream kubelet behaviour; the VM test records them). + snapshotKeys = [ + "net.ipv4.ip_forward" + "net.ipv6.conf.all.forwarding" + "net.ipv4.conf.all.proxy_arp" + "net.ipv4.conf.default.proxy_arp" + "kernel.panic" + "kernel.panic_on_oops" + "vm.overcommit_memory" + # kube-proxy's conntrack keys (fix round 2): the args above leave them, + # the snapshot and the teardown make sure. Absent when nf_conntrack is + # not loaded yet (boot), then simply not recorded. + "net.netfilter.nf_conntrack_max" + "net.netfilter.nf_conntrack_tcp_timeout_established" + "net.netfilter.nf_conntrack_tcp_timeout_close_wait" + ]; + + # kubelet's values for the three kernel keys (upstream kubelet + # setupKernelTunables; MEASURED 10 1 1 in the VM tests). + kubeletKernel = { + "kernel.panic" = "10"; + "kernel.panic_on_oops" = "1"; + "vm.overcommit_memory" = "1"; + }; + + # ── the host's own declared value for a snapshot key (fix round 2) ── + # At boot the activation script runs before systemd-sysctl, so /proc holds + # kernel defaults, not the generation's values: the round-2 review MEASURED + # a snapshot of ip_forward = 0 on a NAS that declares 1 (the house router), + # which the teardown would then have applied. So at boot, a key some module + # OUTSIDE modules/ax-fleet declares is recorded from that declaration; the + # keys only this module declares (the harness's ip_forward and proxy_arp) + # and the undeclared ones are read from /proc, where they still hold the + # pre-ax value. null = no non-ax declaration, or conflicting ones. + unwrap = + v: + if builtins.isAttrs v && (v._type or null) == "override" then + unwrap v.content + else if builtins.isAttrs v && (v._type or null) == "if" then + (if v.condition then unwrap v.content else null) + else + v; + sysctlString = + v: + if builtins.isBool v then + (if v then "1" else "0") + else if v == null then + null + else + toString v; + hostDeclared = + k: + let + defs = lib.filter ( + d: !(lib.hasInfix "/modules/ax-fleet/" (toString d.file)) + ) options.boot.kernel.sysctl.definitionsWithLocations; + vals = lib.unique ( + lib.filter (v: v != null) ( + map (d: sysctlString (unwrap (d.value.${k} or null))) ( + lib.filter (d: builtins.isAttrs d.value) defs + ) + ) + ); + in + if builtins.length vals == 1 then builtins.head vals else null; + + serverToken = + if cfg.k3sTokenFile != null then cfg.k3sTokenFile else config.age.secrets.k3s-token.path; + agentToken = + if cfg.k3sAgentTokenFile != null then + cfg.k3sAgentTokenFile + else + config.age.secrets.k3s-agent-token.path; +in +{ + config = lib.mkMerge [ + # The kill switch's second half stays on the host whatever the switch says + # (fix round 2): the documented rollback is `enable = false`, switch, then + # `sudo ax-fleet-teardown` ON the host, and the NAS holds no dotfiles + # checkout (MEASURED). Inert until run by hand. `KillMode=process` means + # pods and shims outlive k3s, so the teardown is a real step. + (lib.mkIf clusterRole { environment.systemPackages = [ teardown ]; }) + + (lib.mkIf cluster ( + lib.mkMerge [ + { + services.k3s = { + enable = true; + package = cfg.k3sPackage; + # Two credentials (fix round 2). The server token is the NAS's + # alone; with no agent token k3s gives agents the server password + # (MEASURED deps.go getNodePass), i.e. the k3s:server role on + # /v1-k3s/token, /cacerts and /encrypt/config. The coordinator joins + # with the agent token only. + tokenFile = if cfg.role == "control" then serverToken else agentToken; + agentTokenFile = if cfg.role == "control" then agentToken else null; + nodeIP = cfg.lan.address; + # k3s core images (pause, coredns, local-path and its helper) come + # from the pinned airgap tarball, never from the network. + images = [ cfg.k3sPackage.airgap-images ]; + extraFlags = commonFlags; + }; + + environment.etc."rancher/k3s/registries.yaml".text = registriesYaml; + # The kubelet's (and so CoreDNS's) upstream resolver: AdGuard on the + # NAS, never the coordinator's systemd-resolved stub. + environment.etc."ax-fleet/resolv.conf".text = "nameserver ${cfg.serverAddress}\n"; + + boot.kernelModules = [ + "overlay" + "br_netfilter" + ]; + # Both hosts define 512 / 524288 today (modules/common.nix and the + # nixpkgs default); a plain definition would be an evaluation + # conflict. mkOverride 99 raises them to the probe recipe's values. + boot.kernel.sysctl."fs.inotify.max_user_instances" = lib.mkOverride 99 8192; + boot.kernel.sysctl."fs.inotify.max_user_watches" = lib.mkOverride 99 1048576; + + systemd.services.k3s = { + # Re-create the manifest and image links AFTER the bind mounts are + # up, whatever order a live switch ran tmpfiles and mounts in. + serviceConfig.ExecStartPre = [ + "${config.systemd.package}/bin/systemd-tmpfiles --create --prefix=/var/lib/rancher/k3s" + # --node-ip and --flannel-iface name the LAN address, which + # NetworkManager adds after network.target at boot (MEASURED on the + # NAS). Bounded wait, then start anyway (k3s's Restart covers it). + "${pkgs.writeShellScript "ax-fleet-k3s-wait-lan-addr" '' + for _ in $(${pkgs.coreutils}/bin/seq 30); do + ${pkgs.iproute2}/bin/ip -4 addr show dev ${cfg.lan.interface} 2>/dev/null \ + | ${pkgs.gnugrep}/bin/grep -qF 'inet ${cfg.lan.address}/' && exit 0 + ${pkgs.coreutils}/bin/sleep 1 + done + echo "ax-fleet: ${cfg.lan.address} not on ${cfg.lan.interface} after 30 s; starting anyway" >&2 + exit 0 + ''}" + ]; + }; + + # ── kubelet's kernel tunables, put back (myAxFleet.kubelet.keepHostKernelTunables) ── + # kubelet sets these whenever its container manager starts, which is + # NOT tied to a k3s restart: an agent whose server is unreachable is + # `active` and runs no kubelet, and sets 10 1 1 whenever the server + # answers, with NRestarts 0 (MEASURED by the round-2 review). So a + # watcher, bound to k3s, not a one-shot after it (the round-1 unit gave + # up after 900 s). Per key: only a value equal to kubelet's that differs + # from the snapshot is put back, so a deliberate host change to any + # other value is left alone. + systemd.services.ax-fleet-kernel-tunables = lib.mkIf cfg.kubelet.keepHostKernelTunables { + description = "ax-fleet: keep the host's kernel.panic, kernel.panic_on_oops, vm.overcommit_memory against kubelet"; + wantedBy = [ "k3s.service" ]; + after = [ "k3s.service" ]; + partOf = [ "k3s.service" ]; + path = [ + pkgs.procps + pkgs.coreutils + pkgs.gnugrep + ]; + serviceConfig = { + Type = "simple"; + Restart = "always"; + RestartSec = 5; + }; + script = '' + snap=/var/lib/ax-fleet/sysctl-before.conf + [ -s "$snap" ] || { echo "no $snap; leaving kernel tunables as they are"; exec sleep infinity; } + while :; do + ${lib.concatStrings ( + lib.mapAttrsToList (k: kv: '' + want=$(grep -E '^${k} = ' "$snap" | cut -d' ' -f3) + cur=$(sysctl -n ${k}) + if [ -n "$want" ] && [ "$cur" = ${kv} ] && [ "$cur" != "$want" ]; then + sysctl -w "${k}=$want" + fi + '') kubeletKernel + )} + sleep 5 + done + ''; + }; + + # ── sysctl snapshot, taken BEFORE this generation's sysctls apply ── + # An activation snippet, not a unit: at a live switch the activation + # script runs before systemd-sysctl is restarted with the new values; + # a unit ordered before k3s would record the ip_forward=1 this very + # generation just set on the harness. switch-to-configuration exports + # NIXOS_ACTION (switch, test) to the activation; the boot activation + # (initrd-nixos-activation on the systemd-initrd hosts, stage 2 + # otherwise) does not, and there /proc holds kernel defaults, so + # declared keys come from hostDeclared (fix round 2). /run/systemd/system + # is no signal: it exists in the systemd initrd. Written once; later + # generations never overwrite it. + system.activationScripts.ax-fleet-sysctl-snapshot = { + text = '' + if [ ! -e /var/lib/ax-fleet/sysctl-before.conf ]; then + mkdir -p /var/lib/ax-fleet + { + ${lib.concatMapStringsSep "\n" ( + k: + let + proc = "/proc/sys/${lib.replaceStrings [ "." ] [ "/" ] k}"; + d = hostDeclared k; + in + if d == null then + "v=$(cat ${proc} 2>/dev/null) && echo \"${k} = $v\" || true" + else + '' + if [ -n "''${NIXOS_ACTION:-}" ]; then + v=$(cat ${proc} 2>/dev/null) && echo "${k} = $v" || true + else + echo "${k} = ${d}" + fi'' + ) snapshotKeys} + } > /var/lib/ax-fleet/sysctl-before.conf.tmp + mv /var/lib/ax-fleet/sysctl-before.conf.tmp /var/lib/ax-fleet/sysctl-before.conf + fi + ''; + }; + } + + # The tokens: agenix secrets, never printed. k3s-token.age is the + # server's (recipients editors ++ nasOnly); k3s-agent-token.age is the + # agent credential (editors ++ coordinatorOnly ++ nasOnly). Only + # declared where agenix is imported and no test token is given. + (lib.optionalAttrs (options ? age) { + age.secrets = lib.mkMerge [ + (lib.mkIf (cfg.role == "control" && cfg.k3sTokenFile == null) { + k3s-token = { + file = ../../secrets/k3s-token.age; + mode = "0400"; + }; + }) + (lib.mkIf (cfg.k3sAgentTokenFile == null) { + k3s-agent-token = { + file = ../../secrets/k3s-agent-token.age; + mode = "0400"; + }; + }) + ]; + assertions = [ + { + assertion = + (cfg.k3sAgentTokenFile != null && (cfg.role != "control" || cfg.k3sTokenFile != null)) + || config.mySecrets.enable or false; + message = "myAxFleet needs agenix delivery (mySecrets.enable) for secrets/k3s-token.age and secrets/k3s-agent-token.age on ${config.networking.hostName}."; + } + ]; + }) + ] + )) + ]; +} diff --git a/modules/ax-fleet/substrate.nix b/modules/ax-fleet/substrate.nix new file mode 100644 index 000000000..d0178bb41 --- /dev/null +++ b/modules/ax-fleet/substrate.nix @@ -0,0 +1,325 @@ +{ + config, + lib, + pkgs, + inputs, + ... +}: +# Track substrate (DESIGN.md sections 8, 9 and 14 S3): upstream Agent Substrate +# d277088b on the fleet's k3s, installed by its own `ate-setup --kind` exactly +# as the 2026-09-23 probe ran it (MEASURED rc 0, probe-build.md section 4), with +# --image-repo/--image-tag pointing at nix-built images in the NAS registry +# instead of ko. +# +# This file writes ONLY the myAxFleet extension points: +# registrySeed the six component images, the eight third-party images +# bootstrap 20-registry-svc, 30-substrate, 40-gvisor-asset, 50-workerpool +# and an assertion that the version is one string everywhere. It renders +# nothing on a host where myAxFleet.enable is false, and the extension points +# are consumed only on the control role (the NAS), so the coordinator and the +# worker see an unchanged closure from this file. +let + cfg = config.myAxFleet; + on = cfg.enable && cfg.role == "control"; + + inherit (pkgs.stdenv.hostPlatform) system; + + # ONE image set for every evaluator (fix round 2), as ax.nix already does: + # built from the flake's own nixpkgs pin, never the host's `pkgs`. The NAS + # evaluates on nixpkgs-stable, and the round-2 review MEASURED that the six + # component images the real NAS would seed (ateapi, atecontroller, atelet, + # atenet, ateom-gvisor, podcertcontroller) had other digests than the ones + # the VM test ran. The Go toolchain comes from the nixpkgs-go input, the + # same derivation pkgs/ax uses. ax-fleet-topology asserts the parity. + fleetPkgs = inputs.nixpkgs.legacyPackages.${system}; + substrate = fleetPkgs.callPackage ../../pkgs/substrate { + go_1_27 = inputs.nixpkgs-go.legacyPackages.${system}.go_1_27; + }; + images = fleetPkgs.callPackage ../../pkgs/substrate/images.nix { inherit substrate; }; + + kubectl = "${cfg.k3sPackage}/bin/kubectl"; + jq = "${pkgs.jq}/bin/jq"; + curl = "${pkgs.curl}/bin/curl"; + + ns = "ate-system"; + + # 10.42.0.1:5000 -> host and port, for the kind-registry EndpointSlice. + registryParts = lib.splitString ":" cfg.registry; + registryHost = lib.head registryParts; + registryPort = lib.toInt (lib.last registryParts); + + # "ate.dev/sandboxClass=gvisor:NoSchedule" -> key, value, effect. + taint = + let + m = builtins.match "([^=]+)=([^:]*):(.+)" cfg.harnessTaint; + in + { + key = lib.elemAt m 0; + value = lib.elemAt m 1; + effect = lib.elemAt m 2; + }; + + # ── 20: the Service atelet's --localhost-registry-replacement names ── + # The kind overlay runs atelet with --localhost-registry-replacement= + # kind-registry:5000 (manifests/ate-install/kind/atelet/kustomization.yaml), + # so every localhost:5000/... image atelet pulls itself (the pause image, the + # Task images ax names) resolves to this Service. The probe needed exactly + # this object pointing at its node's registry (MEASURED probe-build.md 4). + registrySvc = pkgs.writeText "ax-fleet-20-registry-svc.yaml" ( + builtins.toJSON { + apiVersion = "v1"; + kind = "List"; + items = [ + { + apiVersion = "v1"; + kind = "Namespace"; + metadata.name = ns; + } + { + apiVersion = "v1"; + kind = "Service"; + metadata = { + name = "kind-registry"; + namespace = ns; + labels."app.kubernetes.io/managed-by" = "ax-fleet"; + }; + spec.ports = [ + { + name = "registry"; + port = 5000; + targetPort = registryPort; + protocol = "TCP"; + } + ]; + } + { + apiVersion = "discovery.k8s.io/v1"; + kind = "EndpointSlice"; + metadata = { + name = "kind-registry-nas"; + namespace = ns; + labels = { + "kubernetes.io/service-name" = "kind-registry"; + "endpointslice.kubernetes.io/managed-by" = "ax-fleet"; + }; + }; + addressType = "IPv4"; + ports = [ + { + name = "registry"; + port = registryPort; + protocol = "TCP"; + } + ]; + endpoints = [ + { + addresses = [ registryHost ]; + conditions.ready = true; + } + ]; + } + ]; + } + ); + + # The install is re-run only when what it installs changes: the installer, + # the patched manifests and every image are all in these three paths. The + # component binaries themselves are not in the NAS closure, only the images. + stamp = "${substrate.ate-setup}|${substrate.installTree}|${images}"; + + # ── 50: the gVisor WorkerPool, on the harness only ── + # nodeSelector and tolerations go through spec.template (MEASURED + # pkg/api/v1alpha1/workerpool_types.go). Resources: limits.memory caps each + # worker pod's cgroup, which holds the gVisor sandbox (DESIGN.md section 2, + # replacing ax patch P3). Capacity is read from limits only and an unreported + # dimension is unconstrained (cmd/ateapi/internal/scheduling HasRoom), so no + # CPU limit. The requests are set explicitly: with limits alone Kubernetes + # defaults requests to limits, and 2 x 16Gi would be reserved up front. + workerPoolSpec = { + replicas = cfg.workerPool.replicas; + sandboxClass = "gvisor"; + workerImage = "@WORKER_IMAGE@"; + template = { + labels."ax.mecattaf.dev/pool" = "ateom-gvisor"; + nodeSelector = { + "ax.mecattaf.dev/role" = "harness"; + "ate.dev/substrate-version" = cfg.substrateVersion; + }; + tolerations = [ + { + inherit (taint) key value effect; + operator = "Equal"; + } + { + key = "node.kubernetes.io/unreachable"; + operator = "Exists"; + effect = "NoExecute"; + tolerationSeconds = cfg.workerPool.unreachableTolerationSeconds; + } + { + key = "node.kubernetes.io/not-ready"; + operator = "Exists"; + effect = "NoExecute"; + tolerationSeconds = cfg.workerPool.unreachableTolerationSeconds; + } + ]; + # ephemeral-storage (fix round 4): passed through unchanged by + # atecontroller's applyWorkerPoolPodTemplate (REPORTED + # workerpool_apply.go:522-528 at d277088b). + resources = { + limits = { + memory = cfg.workerPool.memoryLimit; + ephemeral-storage = cfg.workerPool.ephemeralStorageLimit; + }; + requests = { + cpu = "250m"; + memory = "1Gi"; + ephemeral-storage = cfg.workerPool.ephemeralStorageRequest; + }; + }; + }; + }; + workerPoolTemplate = pkgs.writeText "ax-fleet-50-workerpool.json" ( + builtins.toJSON { + apiVersion = "ate.dev/v1alpha1"; + kind = "WorkerPool"; + metadata = { + name = "ateom-gvisor"; + namespace = ns; + labels."app.kubernetes.io/managed-by" = "ax-fleet"; + }; + spec = workerPoolSpec; + } + ); +in +{ + config = lib.mkIf on { + assertions = [ + { + assertion = cfg.substrateVersion == substrate.version; + message = "myAxFleet.substrateVersion (${cfg.substrateVersion}) must equal pkgs/substrate's version (${substrate.version}): it is the node label, the image tag and ate-setup's VERSION at once."; + } + { + assertion = builtins.match "([^=]+)=([^:]*):(.+)" cfg.harnessTaint != null; + message = "myAxFleet.harnessTaint must read key=value:Effect."; + } + { + assertion = builtins.length registryParts == 2; + message = "myAxFleet.registry must read host:port."; + } + ]; + + # Components under substrate/:; third-party images under + # their upstream repositories, which the docker.io and registry.k8s.io + # mirrors in registries.yaml resolve to. The pause image under `pause`, + # which atelet reaches as kind-registry:5000/pause. + myAxFleet.registrySeed = images.seed; + + myAxFleet.bootstrap = { + "20-registry-svc" = '' + # 20-registry-svc: Namespace ${ns} and the kind-registry Service -> + # ${cfg.registry}, before ate-setup starts atelet. + ${kubectl} apply -f ${registrySvc} + ''; + + "25-rustfs-secret" = '' + # 25-rustfs-secret: the RustFS credential RustFS, ate-api and atelet + # read through secretKeyRef (pkgs/substrate patch 0003), in place of + # the kind overlay's literal default published upstream. Generated + # once on this host into /var/lib/ax-fleet/rustfs.env (0600 root), + # applied before ate-setup starts those pods. Never printed. + f=/var/lib/ax-fleet/rustfs.env + if [ ! -s "$f" ]; then + ( + umask 077 + tmp=$(mktemp /var/lib/ax-fleet/.rustfs.env.XXXXXX) + rnd() { head -c "$1" /dev/urandom | od -An -tx1 | tr -d ' \n'; } + printf 'access-key=ax%s\nsecret-key=%s\n' "$(rnd 10)" "$(rnd 32)" > "$tmp" + mv "$tmp" "$f" + ) + echo "generated $f" + fi + ${kubectl} -n ${ns} create secret generic ax-fleet-rustfs --from-env-file="$f" \ + --dry-run=client -o yaml | ${kubectl} apply -f - >/dev/null + echo "secret ${ns}/ax-fleet-rustfs applied" + ''; + + "30-substrate" = '' + # 30-substrate: upstream's own installer, once per (installer, images) + # pair. The stamp lives in the cluster, so a wiped cluster re-installs. + want='${stamp}' + have=$(${kubectl} -n kube-system get configmap ax-fleet-substrate \ + -o jsonpath='{.data.stamp}' 2>/dev/null || true) + if [ "$have" = "$want" ]; then + echo "substrate ${substrate.version} already installed from this closure" + else + # ate-setup walks up from its working directory to go.mod and reads + # manifests/ from there; it writes nothing under that root. + cd ${substrate.installTree} + env -u GCE_REGION -u CLUSTER_LOCATION -u NETWORK -u SUBNETWORK \ + -u MEMORYSTORE_INSTANCE -u PROJECT_ID \ + VERSION=${cfg.substrateVersion} BUCKET_NAME=ate-snapshots \ + HOME="''${HOME:-/var/lib/ax-fleet}" \ + ${substrate.ate-setup}/bin/ate-setup --kind --no-dev-env \ + --kubeconfig "$KUBECONFIG" --context default \ + --rollout-timeout 10m \ + --image-repo ${cfg.registry}/substrate \ + --image-tag ${cfg.substrateVersion} \ + deploy ate-system + cd / + ${kubectl} -n kube-system create configmap ax-fleet-substrate \ + --from-literal=stamp="$want" \ + --from-literal=version=${cfg.substrateVersion} \ + --dry-run=client -o yaml | ${kubectl} apply -f - + echo "substrate ${substrate.version} installed" + fi + ''; + + "40-gvisor-asset" = '' + # 40-gvisor-asset: put the pinned runsc tarball where atelet's S3 + # fallback looks for gs://${images.gvisor.bucket}/${images.gvisor.key} + # (same bucket and key; the scheme is ignored). The credential is + # read from the Secret 25-rustfs-secret applied and kept off argv. + ${kubectl} -n ${ns} rollout status deploy/rustfs --timeout=10m + ip=$(${kubectl} -n ${ns} get svc rustfs -o jsonpath='{.spec.clusterIP}') + base="http://$ip:9000" + creds=$(mktemp) + trap 'rm -f "$creds"' EXIT + chmod 600 "$creds" + ${kubectl} -n ${ns} get secret ax-fleet-rustfs -o json | ${jq} -r ' + .data + | "user = \"\(.["access-key"] | @base64d):\(.["secret-key"] | @base64d)\""' > "$creds" + s3() { ${curl} -sS --aws-sigv4 "aws:amz:us-east-1:s3" -K "$creds" "$@"; } + code=$(s3 -o /dev/null -w '%{http_code}' -I "$base/${images.gvisor.bucket}") + if [ "$code" != 200 ]; then + s3 -f -X PUT "$base/${images.gvisor.bucket}" -o /dev/null + echo "created bucket ${images.gvisor.bucket}" + fi + obj="$base/${images.gvisor.bucket}/${images.gvisor.key}" + code=$(s3 -o /dev/null -w '%{http_code}' -I "$obj") + if [ "$code" != 200 ]; then + s3 -f -T ${images.gvisor} \ + -H "x-amz-content-sha256: ${images.gvisor.sha256}" \ + -H "Content-Type: application/zstd" "$obj" -o /dev/null + echo "uploaded ${images.gvisor.key}" + fi + got=$(s3 -f "$obj" | sha256sum | cut -d' ' -f1) + if [ "$got" != "${images.gvisor.sha256}" ]; then + echo "gvisor asset sha256 mismatch in RustFS: $got" >&2 + exit 1 + fi + echo "gvisor asset verified (sha256 ${images.gvisor.sha256})" + ''; + + "50-workerpool" = '' + # 50-workerpool: WorkerPool ateom-gvisor on the harness node(s), by + # digest (the CRD's CEL rule wants an @). The digest is read from the + # store at run time, so evaluation needs no IFD. + digest=$(tr -d '[:space:]' < ${images.components.ateom-gvisor}/digest) + ref="${cfg.registry}/substrate/ateom-gvisor:${cfg.substrateVersion}@$digest" + sed "s|@WORKER_IMAGE@|$ref|" ${workerPoolTemplate} | ${kubectl} apply -f - + echo "workerpool ateom-gvisor: ${toString cfg.workerPool.replicas} replica(s), $ref" + ''; + }; + }; +} diff --git a/overlays/default.nix b/overlays/default.nix index eec5f8be1..426f1a853 100644 --- a/overlays/default.nix +++ b/overlays/default.nix @@ -1,4 +1,4 @@ -{ torchRocm }: +{ torchRocm, go127 }: final: prev: { qwentts = final.callPackage ../pkgs/qwentts.nix { }; parakeet-service = final.callPackage ../pkgs/parakeet-service { }; @@ -37,6 +37,15 @@ final: prev: { mactahoe-claude-gtk-theme = final.callPackage ../pkgs/mactahoe-gtk-theme.nix { variant = "claude"; }; mactahoe-icon-theme = final.callPackage ../pkgs/mactahoe-icon-theme.nix { }; + # google/ax — the Kubernetes control plane for agent Tasks (v0.3.0, pinned by + # commit). Not in nixpkgs under any name, in any channel: upstream ships no nix + # packaging, no flake and no published image, so there is nothing to take. + # go_1_27 is threaded in from the nixpkgs-go input rather than resolved from + # this fixpoint because ax's go.mod requires exactly 1.27.1 and neither the + # main pin (1.27rc2) nor nixpkgs-fresh (1.27.0) has it — see that input's + # comment in flake.nix, and the `let` in pkgs/ax/default.nix. + ax = final.callPackage ../pkgs/ax { go_1_27 = go127; }; + # Backlog.md — markdown-native task manager CLI (`backlog`). Not in nixpkgs; # packaged from the upstream release binary (Bun compile). See pkgs/backlog-md.nix. backlog-md = final.callPackage ../pkgs/backlog-md.nix { }; diff --git a/pkgs/ax-agent-image/ax-agent.sh b/pkgs/ax-agent-image/ax-agent.sh new file mode 100644 index 000000000..0959c2707 --- /dev/null +++ b/pkgs/ax-agent-image/ax-agent.sh @@ -0,0 +1,111 @@ +#!/usr/bin/env bash +# ax-agent: the command an ax Task runs in the fleet's task image. +# +# ax-agent halogen-smoke one chat completion against Halogen with curl +# ax-agent pi pi against Halogen, result validated against a +# JSON Schema (the probe's adapter-pi.sh, e1) +# ax-agent fetch URL GET URL; exits with curl's code (egress checks) +# ax-agent probe what a sandbox can do without a credential: +# `claude --version` (only in an image that carries +# claude-code; the fleet image does not) and one GET +# of Halogen's /v1/models; records /proc/version, +# which names gVisor's kernel inside a sandbox +# ax-agent exit N exit with code N +# +# Every mode writes its result to $AX_RESULT_PATH (default .ax/result.json +# under the workspace), which P1's runner serves to the controller once the +# command exits. No mode reads or needs a secret: Halogen has no auth, and no +# Claude credential exists in this image (DESIGN.md section 11). +set -uo pipefail + +mode="${1:-}" +[ $# -gt 0 ] && shift +out="${AX_RESULT_PATH:-${AX_CONWIP_RESULT_PATH:-.ax/result.json}}" +mkdir -p "$(dirname "$out")" +halogen="${HALOGEN_URL:-http://10.42.0.5:8731}" +model="${AX_AGENT_MODEL:-${AX_CONWIP_MODEL:-halogen-qwen3.8-flash-next}}" +share="@share@" + +case "$mode" in +halogen-smoke) + body=$(jq -nc --arg m "$model" \ + '{model:$m, max_tokens:32, messages:[{role:"user", content:"Reply with the single word: ok"}]}') + curl -sS --max-time 300 -H 'content-type: application/json' \ + -d "$body" "$halogen/v1/chat/completions" >"$out.raw" + rc=$? + content=$(jq -r '.choices[0].message.content // empty' "$out.raw" 2>/dev/null) + jq -n --arg mode "$mode" --arg model "$model" --arg url "$halogen" --argjson rc "$rc" \ + --arg content "$content" \ + '{mode:$mode, model:$model, halogen:$url, curl_rc:$rc, ok:($rc == 0 and ($content | length) > 0), content:$content}' >"$out" + echo "ax-agent halogen-smoke: curl rc=$rc content_bytes=${#content}" + [ "$rc" -eq 0 ] && [ -n "$content" ] + ;; + +pi) + # pi reads its providers from ~/.pi/agent/models.json (pi README). HOME is + # /workspace/.home, which survives a suspend. + mkdir -p "$HOME/.pi/agent" + sed "s#@HALOGEN_URL@#${halogen}#" "$share/pi-models.json" >"$HOME/.pi/agent/models.json" + prompt="${AX_CONWIP_PROMPT:-What is 6 times 7? Answer with the number.}" + default_schema='{"type":"object","required":["answer"],"properties":{"answer":{"type":"integer"}}}' + schema="${AX_CONWIP_SCHEMA_JSON:-$default_schema}" + sys="Return only one JSON object, no prose and no code fence, that validates against this JSON Schema: ${schema}" + t0=$(date +%s.%N) + raw=$(pi -p --provider halogen --model "$model" --thinking "${AX_CONWIP_EFFORT:-low}" \ + --no-session --no-tools --no-context-files --no-skills --no-extensions --no-prompt-templates --offline \ + --append-system-prompt "$sys" "$prompt" 2>"$out.stderr") + rc=$? + t1=$(date +%s.%N) + printf '%s' "$raw" >"$out.raw" + verdict=$(printf '%s' "$raw" | python3 "$share/validate.py" "$schema") + vrc=$? + jq -n --arg label "${AX_CONWIP_LABEL:-}" --arg model "$model" --arg effort "${AX_CONWIP_EFFORT:-low}" \ + --argjson rc "$rc" --argjson v "$verdict" --arg secs "$(python3 -c "print($t1-$t0)")" --arg cwd "$PWD" \ + --arg err "$(tail -c 2000 "$out.stderr" 2>/dev/null)" \ + '{label:$label, model:$model, effort:$effort, harness:"pi", harness_rc:$rc, + valid:$v.valid, errors:$v.errors, result:$v.value, seconds:($secs|tonumber), cwd:$cwd, + stderr_tail:$err}' >"$out" + echo "ax-agent pi: valid=$(jq .valid "$out") rc=$rc vrc=$vrc" + [ "$rc" -eq 0 ] && [ "$vrc" -eq 0 ] + ;; + +fetch) + url="${1:?usage: ax-agent fetch URL}" + code=$(curl -sS -o /dev/null -w '%{http_code}' --max-time 20 "$url") + rc=$? + jq -n --arg url "$url" --argjson rc "$rc" --arg code "$code" '{mode:"fetch", url:$url, curl_rc:$rc, http_code:$code}' >"$out" + echo "ax-agent fetch: $url rc=$rc http=$code" + exit "$rc" + ;; + +probe) + kernel=$(cat /proc/version 2>/dev/null) + if command -v claude >/dev/null 2>&1; then + cv=$(claude --version 2>&1) + crc=$? + else + cv="claude: not in this image" + crc=127 + fi + code=$(curl -sS -o "$out.models" -w '%{http_code}' --max-time 60 "$halogen/v1/models") + hrc=$? + seen=$(jq -r '.data[0].id // empty' "$out.models" 2>/dev/null) + jq -n --arg cv "$cv" --argjson crc "$crc" --arg url "$halogen" --argjson hrc "$hrc" --arg code "$code" \ + --arg seen "$seen" --arg kernel "$kernel" \ + '{mode:"probe", claude_version:$cv, claude_rc:$crc, halogen:$url, curl_rc:$hrc, http_code:$code, + model:$seen, proc_version:$kernel, ok:($crc == 0 and $hrc == 0 and $code == "200")}' >"$out" + echo "ax-agent probe: claude_rc=$crc curl_rc=$hrc http=$code" + [ "$crc" -eq 0 ] && [ "$hrc" -eq 0 ] && [ "$code" = 200 ] + ;; + +exit) + n="${1:-0}" + jq -n --argjson n "$n" '{mode:"exit", code:$n}' >"$out" + exit "$n" + ;; + +*) + echo "usage: ax-agent halogen-smoke | pi | fetch URL | probe | exit N" >&2 + exit 64 + ;; +esac diff --git a/pkgs/ax-agent-image/default.nix b/pkgs/ax-agent-image/default.nix new file mode 100644 index 000000000..c93ad2886 --- /dev/null +++ b/pkgs/ax-agent-image/default.nix @@ -0,0 +1,169 @@ +{ + lib, + callPackage, + dockerTools, + runCommand, + writeTextDir, + buildEnv, + bashInteractive, + coreutils, + findutils, + gnugrep, + gnused, + gawk, + curl, + jq, + python3, + cacert, + gnutar, + gzip, + git, + shellcheck, + # The patched ax (sandbox-class + P1): its ax-task-runner is PID 1. + ax, + # pi from llm-agents. Halogen only on day one; no claude-code in this image + # until Tom rules on Claude credentials in sandboxes (DESIGN.md section 11). + pi, + # Test-only variants (tests/ax-fleet): extra store paths linked into /bin and + # a name suffix. The fleet image passes neither: pi only, no claude-code. + extraPaths ? [ ], + variant ? "", +}: +# The ax Task image for the fleet (DESIGN.md section 10.2): `ax-agent`. +# +# Substrate starts /usr/local/bin/ax-task-runner (ax's DefaultGuestCommand) as +# PID 1; the runner sets up /workspace and runs the Task's spec.command, here +# `ax-agent `. Output: an OCI layout plus `digest`, seeded into the NAS +# registry as ax/ax-agent and named by Tasks as +# localhost:5000/ax/ax-agent@sha256: (atelet rewrites localhost:5000). +# +# No secret is baked in: pi-models.json names Halogen, which has no auth, and +# the endpoint is filled at run time from $HALOGEN_URL (default the worker, +# http://10.42.0.5:8731), so one image digest serves every host. +let + ociLayout = callPackage ../ax/oci-layout.nix { }; + + share = runCommand "ax-agent-share" { } '' + mkdir -p $out/share/ax-agent + cp ${./pi-models.json} $out/share/ax-agent/pi-models.json + cp ${./validate.py} $out/share/ax-agent/validate.py + ''; + + ax-agent = runCommand "ax-agent" { nativeBuildInputs = [ shellcheck ]; } '' + mkdir -p $out/bin + substitute ${./ax-agent.sh} $out/bin/ax-agent \ + --replace-fail '@share@' '${share}/share/ax-agent' \ + --replace-fail '#!/usr/bin/env bash' '#!${bashInteractive}/bin/bash' + chmod +x $out/bin/ax-agent + shellcheck -S warning $out/bin/ax-agent + ''; + + # The runner at the path Substrate's template names. + # pi with its ELF program headers in the order the ELF spec asks for. pi + # 0.85.1's bun binary lists PT_LOAD out of p_vaddr order; Linux runs it, but + # gVisor's loader refuses it (ENOEXEC, exit 126 inside the sandbox, MEASURED + # in the ax-fleet VM test, INTEGRATE.md). Only table entries move; segments, + # addresses and bytes stay. The name keeps pi's, so the store path length is + # unchanged; only the text wrapper in bin/ is repointed. + piSandbox = runCommand pi.name { nativeBuildInputs = [ python3 ]; } '' + cp -a ${pi} $out + chmod -R u+w $out + find $out -type f -print0 | while IFS= read -r -d "" f; do + if [ "$(head -c 4 "$f" | od -An -c | tr -d ' ')" = '177ELF' ]; then + python3 ${./elf-sort-load.py} "$f" + fi + done + for f in $out/bin/*; do + substituteInPlace "$f" --replace-quiet ${pi} $out + done + chmod -R a-w $out + ''; + + runner = runCommand "ax-task-runner-usr-local" { } '' + mkdir -p $out/usr/local/bin + ln -s ${ax}/bin/ax-task-runner $out/usr/local/bin/ax-task-runner + ln -s ${ax-agent}/bin/ax-agent $out/usr/local/bin/ax-agent + ''; + + etc = [ + (writeTextDir "etc/passwd" '' + root:x:0:0:root:/root:/bin/bash + agent:x:1000:1000:agent:/workspace/.home:/bin/bash + nobody:x:65534:65534:nobody:/var/empty:/bin/false + '') + (writeTextDir "etc/group" '' + root:x:0: + agent:x:1000: + nobody:x:65534: + '') + ]; + + env = buildEnv { + name = "ax-agent-env"; + paths = [ + bashInteractive + coreutils + findutils + gnugrep + gnused + gawk + curl + jq + python3 + cacert + gnutar + gzip + git + piSandbox + ax-agent + ] + ++ extraPaths; + pathsToLink = [ + "/bin" + "/etc/ssl" + "/share/ax-agent" + ]; + }; + + image = dockerTools.buildLayeredImage { + name = "ax/ax-agent${variant}"; + tag = "v${ax.version}-p1"; + contents = [ + env + runner + share + ] + ++ etc; + extraCommands = '' + mkdir -p tmp workspace usr/bin etc/ax-agent/pi + chmod 1777 tmp + ln -s ${coreutils}/bin/env usr/bin/env + cp ${./pi-models.json} etc/ax-agent/pi/models.json + ''; + config = { + Cmd = [ "/usr/local/bin/ax-task-runner" ]; + WorkingDir = "/workspace"; + Env = [ + "PATH=/usr/local/bin:/bin" + "HOME=/workspace/.home" + "SSL_CERT_FILE=${cacert}/etc/ssl/certs/ca-bundle.crt" + "HALOGEN_URL=http://10.42.0.5:8731" + ]; + }; + }; +in +(ociLayout { + name = "ax/ax-agent${variant}"; + tag = "v${ax.version}-p1"; + inherit image; +}).overrideAttrs + (old: { + passthru = old.passthru // { + inherit ax-agent image piSandbox; + }; + meta = { + description = "ax Task image for the fleet: ax-task-runner, pi, and the ax-agent adapter (OCI layout)"; + platforms = [ "x86_64-linux" ]; + license = lib.licenses.asl20; + }; + }) diff --git a/pkgs/ax-agent-image/elf-sort-load.py b/pkgs/ax-agent-image/elf-sort-load.py new file mode 100644 index 000000000..0af001f78 --- /dev/null +++ b/pkgs/ax-agent-image/elf-sort-load.py @@ -0,0 +1,51 @@ +#!/usr/bin/env python3 +"""Reorder an ELF64 program header table so gVisor's loader accepts it. + +The ELF spec says loadable segments appear in ascending p_vaddr order, and +PT_PHDR and PT_INTERP precede every PT_LOAD. Linux tolerates a table that +breaks this; gVisor's loader refuses it with ENOEXEC, which bash reports as +exit 126. pi 0.85.1 from llm-agents ships such a table: its first PT_LOAD maps +vaddr 0x6431000 and a later one 0x1ff000 (MEASURED 2026-09-23, ax-fleet +INTEGRATE.md). The fix moves table entries only: every segment keeps its file +offset, address, size and flags, and the table keeps its place and length. + +Usage: elf-sort-load.py FILE... (files that are not ELF64 LE, or are already +in order, are left alone; prints one line per file it changed) +""" +import struct +import sys + +PT_LOAD, PT_INTERP, PT_PHDR = 1, 3, 6 + + +def fix(path): + with open(path, "r+b") as f: + ident = f.read(64) + if len(ident) < 64 or ident[:4] != b"\x7fELF" or ident[4] != 2 or ident[5] != 1: + return False + (phoff,) = struct.unpack_from(" {order}") + return True + + +if __name__ == "__main__": + for p in sys.argv[1:]: + fix(p) diff --git a/pkgs/ax-agent-image/pi-models.json b/pkgs/ax-agent-image/pi-models.json new file mode 100644 index 000000000..282b8fa89 --- /dev/null +++ b/pkgs/ax-agent-image/pi-models.json @@ -0,0 +1,26 @@ +{ + "providers": { + "halogen": { + "api": "openai-completions", + "apiKey": "none", + "authHeader": false, + "baseUrl": "@HALOGEN_URL@/v1", + "compat": { + "supportsDeveloperRole": false, + "supportsReasoningEffort": false, + "supportsStore": false + }, + "models": [ + { + "id": "halogen-qwen3.8-flash-next", + "name": "Qwen3.8-Flash-Next (Halogen, worker)", + "contextWindow": 262144, + "maxTokens": 32768, + "input": ["text", "image"], + "reasoning": true, + "cost": { "input": 0, "output": 0, "cacheRead": 0, "cacheWrite": 0 } + } + ] + } + } +} diff --git a/pkgs/ax-agent-image/validate.py b/pkgs/ax-agent-image/validate.py new file mode 100755 index 000000000..7a7fe3d1b --- /dev/null +++ b/pkgs/ax-agent-image/validate.py @@ -0,0 +1,36 @@ +#!/usr/bin/env python3 +"""Minimal JSON Schema subset check (type, required, properties, enum, items, +additionalProperties:false) for the probe. Reads schema from $1 (JSON text) and +candidate text from stdin; extracts the first JSON object; prints a verdict JSON.""" +import json, sys, re +schema = json.loads(sys.argv[1]) +raw = sys.stdin.read() +T = {"object": dict, "array": list, "string": str, "integer": int, "number": (int, float), "boolean": bool, "null": type(None)} +def check(s, v, p="$"): + errs = [] + t = s.get("type") + if t and not isinstance(v, T[t]) or (t == "integer" and isinstance(v, bool)): + return [f"{p}: expected {t}"] + if "enum" in s and v not in s["enum"]: + errs.append(f"{p}: not in enum") + if isinstance(v, dict): + for r in s.get("required", []): + if r not in v: errs.append(f"{p}.{r}: missing") + props = s.get("properties", {}) + for k, sub in props.items(): + if k in v: errs += check(sub, v[k], f"{p}.{k}") + if s.get("additionalProperties") is False: + errs += [f"{p}.{k}: extra" for k in v if k not in props] + if isinstance(v, list) and "items" in s: + for i, x in enumerate(v): errs += check(s["items"], x, f"{p}[{i}]") + return errs +m = re.search(r"\{.*\}", raw, re.S) +if not m: + print(json.dumps({"valid": False, "errors": ["no JSON object in output"], "value": None})); sys.exit(3) +try: + val = json.loads(m.group(0)) +except Exception as e: + print(json.dumps({"valid": False, "errors": [f"parse: {e}"], "value": None})); sys.exit(3) +errs = check(schema, val) +print(json.dumps({"valid": not errs, "errors": errs, "value": val})) +sys.exit(0 if not errs else 4) diff --git a/pkgs/ax-fleet-teardown/default.nix b/pkgs/ax-fleet-teardown/default.nix new file mode 100644 index 000000000..383700dc4 --- /dev/null +++ b/pkgs/ax-fleet-teardown/default.nix @@ -0,0 +1,123 @@ +{ + writeShellApplication, + k3s, + iptables, + nftables, + iproute2, + procps, + coreutils, + gnugrep, + systemd, +}: +# ax-fleet-teardown: the second half of the kill switch (DESIGN.md section 13). +# +# The k3s unit runs with KillMode=process, so pods and containerd shims outlive +# it; switching `myAxFleet.enable = false` alone leaves them, plus cni0, +# flannel.1, the KUBE-/FLANNEL-/CNI- rules and ip_forward=1. This wraps the +# pinned package's own k3s-killall.sh, removes the guard chains only when the +# running generation does not declare them (a pre-ax generation), then +# restores the sysctls recorded before k3s first ran on this host +# (/var/lib/ax-fleet/sysctl-before.conf, written by the activation snippet in +# modules/ax-fleet/k3s.nix). +# +# PATH is pinned (inheritPath = false). The upstream killall's +# remove_interfaces runs `tailscale set --advertise-routes=` whenever a +# `tailscale` binary is on PATH; on the NAS that would withdraw the +# 10.42.0.0/24 subnet route (hosts/nas/headscale.nix). With the caller's PATH +# cut off, `command -v tailscale` fails and the call is skipped; the killall +# wrapper still prefixes its own dependencies. The explicit check below makes +# the teardown refuse to run if tailscale ever becomes reachable anyway. +# +# Left on disk on purpose: /mnt/fast/k3s, the data-pool directories and +# /var/lib/ax-fleet. Deleting them is Tom's call. +writeShellApplication { + name = "ax-fleet-teardown"; + inheritPath = false; + runtimeInputs = [ + k3s + iptables + nftables + iproute2 + procps + coreutils + gnugrep + systemd + ]; + text = '' + if [ "$(id -u)" -ne 0 ]; then + echo "ax-fleet-teardown: run as root (sudo)" >&2 + exit 1 + fi + + if command -v tailscale >/dev/null 2>&1; then + echo "ax-fleet-teardown: tailscale is on PATH; k3s-killall.sh would clear the advertised routes. Refusing." >&2 + exit 1 + fi + + if systemctl is-enabled --quiet k3s.service 2>/dev/null; then + echo "ax-fleet-teardown: note: k3s.service is still part of this generation;" >&2 + echo " k3s-killall.sh stops it now, and it starts again at the next boot or switch." >&2 + fi + + guard_failed=0 + echo "== k3s-killall.sh (${k3s.version})" + ${k3s}/bin/k3s-killall.sh || echo "k3s-killall.sh exited $?; continuing" >&2 + + # Fix round 3: a harness or control generation declares the guards + # whatever `enable` says (the kill switch leaves pods running until this + # script), so they stay; a generation from before ax never had them, and + # then they go. + if [ -e /etc/ax-fleet/guard-declared ]; then + # Fix round 4: the killall's `grep -iv flannel` pipeline just deleted + # every guard rule naming flannel.1 (the VXLAN source rule, the + # cross-node returns, the pod-input refusal). Re-apply the generation's + # guards rather than trust them; the NAS's are an nftables table the + # iptables pipeline never touches. + if [ -x /etc/ax-fleet/guard-apply ]; then + echo "== guards: re-applying this generation's ($(cat /etc/ax-fleet/guard-declared))" + if ! /etc/ax-fleet/guard-apply; then + echo "ax-fleet-teardown: WARNING: re-applying the guards failed; pods of a restarted k3s would reach this host over flannel.1. Reload the firewall." >&2 + guard_failed=1 + fi + fi + echo "== guards: declared by this generation ($(cat /etc/ax-fleet/guard-declared)); left in place" + else + echo "== guard chains" + while iptables -w -t mangle -D FORWARD -j ax-fleet-guard 2>/dev/null; do :; done + iptables -w -t mangle -F ax-fleet-guard 2>/dev/null || true + iptables -w -t mangle -X ax-fleet-guard 2>/dev/null || true + while iptables -w -D OUTPUT -j ax-fleet-api 2>/dev/null; do :; done + iptables -w -F ax-fleet-api 2>/dev/null || true + iptables -w -X ax-fleet-api 2>/dev/null || true + # The pod-input refusal, if this generation's firewall reload left it. + for ifc in cni0 flannel.1; do + for t in iptables ip6tables; do + while $t -w -D nixos-fw -i "$ifc" -m conntrack --ctstate NEW -m comment --comment ax-fleet-pod-input -j nixos-fw-refuse 2>/dev/null; do :; done + done + done + + echo "== NAS guard table" + nft delete table inet ax-fleet-guard 2>/dev/null || true + fi + + snap=/var/lib/ax-fleet/sysctl-before.conf + if [ -s "$snap" ]; then + echo "== sysctl restore from $snap" + # -e: a conntrack key recorded while nf_conntrack was loaded may be + # absent now; skip it rather than abort the restore. + sysctl -e -p "$snap" + else + echo "ax-fleet-teardown: no $snap; sysctls left as they are" >&2 + fi + + echo "== left behind (should be empty)" + ip -br link show 2>/dev/null | grep -E '^(cni0|flannel\.1|veth)' || true + iptables-save 2>/dev/null | grep -cE 'KUBE-|FLANNEL|CNI-' || true + pgrep -a containerd-shim || true + if [ "$guard_failed" -ne 0 ]; then + echo "ax-fleet-teardown: done, with the guard re-apply FAILED (see above)" >&2 + exit 1 + fi + echo "ax-fleet-teardown: done" + ''; +} diff --git a/pkgs/ax-gateway-default/default.nix b/pkgs/ax-gateway-default/default.nix new file mode 100644 index 000000000..a2a50f4ee --- /dev/null +++ b/pkgs/ax-gateway-default/default.nix @@ -0,0 +1,17 @@ +# ax-gateway-default: a separate client binary built from stock ax's module +# tree (same vendorHash, no ax patch). See main.go and +# modules/ax-fleet/gateways.nix. +{ ax }: +ax.overrideAttrs (old: { + pname = "ax-gateway-default"; + postPatch = (old.postPatch or "") + '' + mkdir -p cmd/ax-gateway-default + cp ${./main.go} cmd/ax-gateway-default/main.go + ''; + subPackages = [ "cmd/ax-gateway-default" ]; + doCheck = false; + meta = (old.meta or { }) // { + description = "Points ax Tasks without a usable gateway at their atespace's default Gateway"; + mainProgram = "ax-gateway-default"; + }; +}) diff --git a/pkgs/ax-gateway-default/main.go b/pkgs/ax-gateway-default/main.go new file mode 100644 index 000000000..d1587e6fc --- /dev/null +++ b/pkgs/ax-gateway-default/main.go @@ -0,0 +1,114 @@ +// ax-gateway-default: stock ax v0.3.0 gives a Task with no gateway, or one +// naming a Gateway its atespace lacks, an allow-all EgressPolicy +// (internal/controller/reconciler.go, "Default to allow all egress"). This +// fleet does not patch ax (Tom, 2026-09-23), so the default lives outside it: +// every declared atespace carries a default Gateway (the bootstrap applies +// it), and this loop points any Task in those atespaces that has no usable +// gateway at that default through ax's own UpdateTask. The controller then +// reconciles the Task again and replaces the actor's egress policy with the +// default Gateway's allowlist. +// +// It is a client of ax's public gRPC API, built from the same module tree; +// it changes nothing inside ax. +package main + +import ( + "context" + "flag" + "fmt" + "log/slog" + "os" + "strings" + "time" + + "github.com/google/ax/pkg/apis/v1alpha1" + "google.golang.org/grpc" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/credentials/insecure" + "google.golang.org/grpc/status" +) + +func main() { + server := flag.String("server", os.Getenv("AX_SERVER"), "ax-server address, host:port or http://host:port") + spaces := flag.String("atespaces", "", "comma-separated atespace=defaultGateway pairs") + every := flag.Duration("interval", 2*time.Second, "poll interval") + once := flag.Bool("once", false, "one pass, then exit (tests)") + flag.Parse() + + defaults := map[string]string{} + for _, kv := range strings.Split(*spaces, ",") { + if kv == "" { + continue + } + ns, gw, ok := strings.Cut(kv, "=") + if !ok || ns == "" || gw == "" { + fmt.Fprintf(os.Stderr, "bad -atespaces entry %q (want atespace=gateway)\n", kv) + os.Exit(64) + } + defaults[ns] = gw + } + if len(defaults) == 0 { + fmt.Fprintln(os.Stderr, "no atespaces declared") + os.Exit(64) + } + target := strings.TrimPrefix(strings.TrimPrefix(*server, "http://"), "https://") + conn, err := grpc.NewClient(target, grpc.WithTransportCredentials(insecure.NewCredentials())) + if err != nil { + slog.Error("connect", "target", target, "error", err) + os.Exit(1) + } + defer conn.Close() + client := v1alpha1.NewAXClient(conn) + + for { + for ns, gw := range defaults { + if err := pass(client, ns, gw); err != nil { + slog.Warn("pass failed", "atespace", ns, "error", err) + } + } + if *once { + return + } + time.Sleep(*every) + } +} + +func pass(client v1alpha1.AXClient, ns, def string) error { + ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second) + defer cancel() + if _, err := client.GetGateway(ctx, &v1alpha1.GetGatewayRequest{Atespace: ns, Name: def}); err != nil { + // Without the default there is nothing safe to point at; the + // bootstrap re-applies it. Log loudly, change nothing. + return fmt.Errorf("default gateway %s/%s: %w", ns, def, err) + } + resp, err := client.ListTasks(ctx, &v1alpha1.ListTasksRequest{Atespace: ns}) + if err != nil { + return fmt.Errorf("list tasks: %w", err) + } + for _, t := range resp.Tasks { + if t.Spec == nil || t.Metadata == nil { + continue + } + name := "" + if t.Spec.Gateway != nil { + name = t.Spec.Gateway.Name + } + if name != "" { + _, gerr := client.GetGateway(ctx, &v1alpha1.GetGatewayRequest{Atespace: ns, Name: name}) + if gerr == nil { + continue + } + if status.Code(gerr) != codes.NotFound { + slog.Warn("gateway lookup", "task", t.Metadata.Name, "gateway", name, "error", gerr) + continue + } + } + t.Spec.Gateway = &v1alpha1.GatewayRef{Name: def} + if _, err := client.UpdateTask(ctx, &v1alpha1.UpdateTaskRequest{Task: t}); err != nil { + slog.Warn("repoint", "task", t.Metadata.Name, "error", err) + continue + } + slog.Info("pointed at the default gateway", "atespace", ns, "task", t.Metadata.Name, "was", name, "now", def) + } + return nil +} diff --git a/pkgs/ax/default.nix b/pkgs/ax/default.nix new file mode 100644 index 000000000..c9c4bacc2 --- /dev/null +++ b/pkgs/ax/default.nix @@ -0,0 +1,93 @@ +{ + lib, + buildGoModule, + fetchFromGitHub, + git, + # The Go toolchain, passed in by overlays/default.nix rather than taken from + # this pkgs fixpoint. See the `go` binding below for why. + go_1_27, +}: +# google/ax — a Kubernetes control plane for agent Tasks, which delegates the +# sandbox to Agent Substrate rather than running workloads itself. Four commands +# ship in cmd/: `ax` (CLI), `ax-controller`, `ax-server`, `ax-task-runner`. +# +# NOT in nixpkgs, in any channel, under any name (checked 2026-09-22): upstream +# tagged v0.3.0 and has no nix packaging of its own, no flake, and no container +# publishing path a fleet could consume. This expression is the whole of it. +# +# Pinned by commit, not by tag, so a retag upstream cannot move what this fleet +# builds. d8ed0fe38bceb7842d3c47817d53d16ccdfcb601 IS tag v0.3.0 as of 2026-09-22. +# +# NO CARRIED PATCHES: stock v0.3.0, see the `patches` entry below. The vendored +# Substrate client stays at 672533541dbf. +let + # go.mod's first directive is `go 1.27.1` (MEASURED). Go refuses to build a + # module whose `go` line is newer than the running toolchain, and the sandbox + # has no network to fetch one, so the toolchain must be at least 1.27.1: + # this flake's main nixpkgs pin go = 1.26.5, go_1_27 = 1.27rc2 too old + # nixpkgs-fresh (flake.nix:28) go = 1.26.7, go_1_27 = 1.27.0 too old + # so overlays/default.nix resolves go_1_27 from the `nixpkgs-go` input, which + # exists for this one attribute and nothing else. All three versions MEASURED + # 2026-09-23; see that input's comment in flake.nix. + buildGo127Module = buildGoModule.override { go = go_1_27; }; +in +buildGo127Module { + pname = "ax"; + version = "0.3.0"; + + src = fetchFromGitHub { + owner = "google"; + repo = "ax"; + rev = "d8ed0fe38bceb7842d3c47817d53d16ccdfcb601"; # = tag v0.3.0 + hash = "sha256-mGSQ4QsYLdeKDtVMBODCulqQQ0Ze0NjeADPhB6edaYU="; + }; + + # Obtained the ordinary way: build once with lib.fakeHash, read the "got:" + # line off the failure, paste it back. + vendorHash = "sha256-iC/X6Bg1M7Pn3dT1zWs2YxuPfgl9ZKNEYQsBisIQguY="; + + # Stock google/ax v0.3.0, no carried patches (Tom, 2026-09-23 08:45Z: "i + # prefer not to patch ax itself unless we really have to"). The two patches + # the bring-up carried were measured unnecessary on the 4-VM test + # (evals-2026-09-23/zero-patch/zero-patch-combined.md, run 1 rc 0): + # - sandbox-class.patch: stock v0.3.0 already hardcodes + # SANDBOX_CLASS_GVISOR / gvisor-default (no-sandboxclass.md); + # - p1-completion.patch: a Task reports its own completion to the floor and + # the link deletes the Task, which frees the worker (no-p1.md). Stock ax + # keeps a finished Task Running until that delete. + # Egress for a Task with no gateway is closed outside ax: the bootstrap + # declares a default Gateway in every fleet atespace and + # ax-fleet-gateway-default points gateway-less Tasks at it + # (modules/ax-fleet/gateways.nix); the link refuses capacity on a missing + # gateway (B10). Keep this list empty. + patches = [ ]; + + # subPackages left unset so all four commands build, matching upstream's + # `make build-binaries` plus the cross-compiled runner. -s -w mirrors the + # Makefile's ldflags. + ldflags = [ + "-s" + "-w" + ]; + + # Upstream's `make test` is `go test ./...`, and per docs/development.md it + # "Runs everything, including the mock Substrate gRPC server, in-memory store + # validation, and API server tests". It needs no cluster and no network, so it + # is the one cluster-free regression proof this package has. Kept ON. + doCheck = true; + + # MEASURED 2026-09-22: without git on PATH, two internal/workspace tests fail + # with `exit status 127 ... git: command not found` (TestSetupWorkspace_GitSubdir + # at setup_test.go:104, TestSetupWorkspace_GitDepth at setup_test.go:227). They + # shell out to git to build a throwaway repo. This is a packaging fix, not a + # source patch: nothing upstream is wrong. + nativeCheckInputs = [ git ]; + + meta = { + description = "Kubernetes control plane for agent Tasks, delegating sandboxes to Agent Substrate"; + homepage = "https://github.com/google/ax"; + license = lib.licenses.asl20; + platforms = lib.platforms.linux; + mainProgram = "ax"; + }; +} diff --git a/pkgs/ax/images.nix b/pkgs/ax/images.nix new file mode 100644 index 000000000..ff146a41d --- /dev/null +++ b/pkgs/ax/images.nix @@ -0,0 +1,64 @@ +{ + callPackage, + dockerTools, + cacert, + redis, + # The patched ax (pkgs/ax: sandbox-class + P1). Passed in so every host seeds + # and references the same derivation. + ax, +}: +# The ax control-plane images for the fleet (DESIGN.md section 8): ax-server, +# ax-controller and ax-redis, each an OCI layout with a `digest` file, seeded +# into the NAS registry as ax/: and referenced by digest in the +# ax-fleet-40-ax manifest (modules/ax-fleet/ax.nix). +# +# Upstream publishes no image (ko:// references only, deploy/*.yaml), so these +# are built here from the same pkgs/ax the CLI uses. +let + ociLayout = callPackage ./oci-layout.nix { }; + tag = "v${ax.version}-p1"; + + axImage = + cmd: + ociLayout { + name = "ax/${cmd}"; + inherit tag; + image = dockerTools.buildLayeredImage { + name = "ax/${cmd}"; + inherit tag; + contents = [ cacert ]; + config = { + Entrypoint = [ "${ax}/bin/${cmd}" ]; + Env = [ "SSL_CERT_FILE=${cacert}/etc/ssl/certs/ca-bundle.crt" ]; + User = "65532:65532"; + }; + }; + }; +in +{ + inherit tag ociLayout; + + ax-server = axImage "ax-server"; + ax-controller = axImage "ax-controller"; + + # nixpkgs redis, not upstream's redis:7-alpine. Upstream's image patches + # protected mode off; a stock redis-server refuses non-loopback clients when + # it has no password, so the manifest passes --protected-mode no. It is + # reachable only as a ClusterIP (DESIGN.md section 9). + ax-redis = ociLayout { + name = "ax/ax-redis"; + tag = "${redis.version}"; + image = dockerTools.buildLayeredImage { + name = "ax/ax-redis"; + tag = "${redis.version}"; + contents = [ ]; + extraCommands = '' + mkdir -p data + ''; + config = { + Entrypoint = [ "${redis}/bin/redis-server" ]; + WorkingDir = "/data"; + }; + }; + }; +} diff --git a/pkgs/ax/oci-layout.nix b/pkgs/ax/oci-layout.nix new file mode 100644 index 000000000..06ab9b92c --- /dev/null +++ b/pkgs/ax/oci-layout.nix @@ -0,0 +1,36 @@ +{ + lib, + runCommand, + skopeo, + jq, +}: +# Turn a dockerTools image (a docker-archive tarball) into an OCI image layout +# at the output root (oci-layout, index.json, blobs/) plus a `digest` file with +# the manifest digest (sha256:...). That is the shape myAxFleet.registrySeed +# takes: the NAS pushes it with `skopeo copy --preserve-digests`, so the digest +# a manifest or a Task names is the digest the registry serves. +# +# The digest is read at BUILD time, inside this derivation, and consumers read +# it from "${layout}/digest" in their own build steps. No import-from-derivation. +{ + name, + image, + tag ? "latest", +}: +runCommand "${lib.replaceStrings [ "/" ] [ "-" ] name}-oci" + { + nativeBuildInputs = [ + skopeo + jq + ]; + passthru = { inherit image tag; }; + } + '' + export HOME=$TMPDIR + # gzip layers: these images cross the coordinator's wifi leg when pulled. + skopeo --insecure-policy --tmpdir "$TMPDIR" copy --quiet \ + --dest-compress --dest-compress-format gzip \ + docker-archive:${image} oci:$out:${tag} + jq -r '.manifests[0].digest' $out/index.json > $out/digest + grep -Eq '^sha256:[0-9a-f]{64}$' $out/digest + '' diff --git a/pkgs/substrate-link/SYNC.md b/pkgs/substrate-link/SYNC.md new file mode 100644 index 000000000..f7a19de5d --- /dev/null +++ b/pkgs/substrate-link/SYNC.md @@ -0,0 +1,22 @@ +# pkgs/substrate-link/src: vendored source + +- Upstream: the ax-conwip repository (now named "substrate", Tom 2026-09-23 E10), local only at the time of + vendoring: `/home/tom/mecattaf/ax-conwip-wt-link`, branch `eval/2026-09-23-link`. +- Source sha: `a02100344205fcd20c93c0f9bdc87902adbbcd4e` (`a021003`), "link: final pass, fence every journaled + earlier attempt and re-time the heartbeat sleep". +- Taken with `git archive a021003 package.json pnpm-lock.yaml pnpm-workspace.yaml tsconfig.json apps/link`, unchanged. + The root `src/`, `test/` and `proto/` (the CONWIP scheduler) are not vendored: `apps/link` imports none of them. +- Verified upstream at that sha (REPORTED `evals-2026-09-23/link/LINK-FINAL-2026-09-23.md`): `tsc --noEmit` rc 0, + link suite 134 pass. +- The bundle uses `apps/link/proto/ax-p1.proto`, a superset of stock ax v0.3.0's API (it adds P1's + `GetTaskResult`). On this fleet ax carries no patch, so that RPC answers Unimplemented and the link's + `completion` defaults to `guest` (hosts/nas/substrate-link.nix). + +To resync: `git -C archive package.json pnpm-lock.yaml pnpm-workspace.yaml tsconfig.json +apps/link | tar -x -C pkgs/substrate-link/src`, update the sha above, then refresh `pnpmDeps.hash` in +`default.nix` (set it to `lib.fakeHash`, build, paste the `got:` value). + +## Unknowns and proposed defaults + +- When the substrate repository gets its remote (agency-agency/substrate), whether to switch this vendored copy to a + flake input. Default: keep the vendored copy (E1: the code that wraps ax lives in dotfiles) and resync by sha. diff --git a/pkgs/substrate-link/default.nix b/pkgs/substrate-link/default.nix new file mode 100644 index 000000000..e64c2a788 --- /dev/null +++ b/pkgs/substrate-link/default.nix @@ -0,0 +1,57 @@ +{ + lib, + stdenvNoCC, + nodejs_24, + pnpm_10, + esbuild, + makeWrapper, +}: +# substrate-link as one bundled file, built from the source vendored in ./src (SYNC.md names the +# upstream sha). Formerly conwip-link. +# N2: Node 24, the line the link suite ran on (24.18.0, 24.20.0 and 22.23.1 all 43/43 on 2026-09-23); the pinned +# nixpkgs b6c98e9e has nodejs_24 = 24.20.0 (read from its all-packages.nix and v24.nix). +# The bundling step itself was run by hand in the link worktree (esbuild, one 2,030,521 B .mjs that +# starts from / with only AX_CONWIP_PROTO_PATH set, drains on SIGTERM and exits 0). +stdenvNoCC.mkDerivation (finalAttrs: { + pname = "substrate-link"; + version = "0.1.0-unstable-2026-09-23"; + src = ./src; + + nativeBuildInputs = [ + nodejs_24 + pnpm_10.configHook + esbuild + makeWrapper + ]; + + pnpmDeps = pnpm_10.fetchDeps { + inherit (finalAttrs) pname version src; + # nix build with lib.fakeHash, then the "got:" value (MEASURED 2026-09-23). + hash = "sha256-7i4RxnkDbAV/tVZd8P+zXbhVAi7DfnZHDHq0BjnC0M4="; + fetcherVersion = 2; + }; + + buildPhase = '' + runHook preBuild + esbuild apps/link/src/main.ts --bundle --platform=node --format=esm --target=node22 \ + --outfile=dist/substrate-link.mjs \ + "--banner:js=import{createRequire as __cr}from'node:module';const require=__cr(import.meta.url);" + runHook postBuild + ''; + + installPhase = '' + runHook preInstall + install -Dm444 dist/substrate-link.mjs $out/lib/substrate-link/substrate-link.mjs + install -Dm444 apps/link/proto/ax-p1.proto $out/share/substrate-link/ax-p1.proto + makeWrapper ${lib.getExe nodejs_24} $out/bin/substrate-link \ + --add-flags $out/lib/substrate-link/substrate-link.mjs \ + --set LINK_AX_PROTO_PATH $out/share/substrate-link/ax-p1.proto + runHook postInstall + ''; + + meta = { + description = "Outbound-only link from the Cloudflare Substrate floor to ax on the NAS"; + mainProgram = "substrate-link"; + platforms = lib.platforms.linux; + }; +}) diff --git a/pkgs/substrate-link/src/apps/link/proto/ax-p1.proto b/pkgs/substrate-link/src/apps/link/proto/ax-p1.proto new file mode 100644 index 000000000..134a0d244 --- /dev/null +++ b/pkgs/substrate-link/src/apps/link/proto/ax-p1.proto @@ -0,0 +1,406 @@ +// RUNTIME PROTO of conwip-link (round 2): the vendored ax.proto (v0.3.0 d8ed0fe) plus the three hunks of carried patch +// P1 that the link reads: rpc GetTaskResult, messages GetTaskResultRequest and TaskResult, UsageStats.tool_calls. +// Copied from dotfiles branch ax/fleet-bringup pkgs/ax/patches/p1-completion.patch (lines 3240-3300) on 2026-09-23. +// A superset of the stock proto: against a stock server GetTaskResult reaches the server, which answers UNIMPLEMENTED, +// so the capability probe (B6) is decided by the server, never by the client's proto. Keep in sync with P1. +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +syntax = "proto3"; + +package ax.v1alpha1; + +import "google/protobuf/struct.proto"; +import "google/protobuf/timestamp.proto"; + +option go_package = "github.com/google/ax/pkg/apis/v1alpha1"; + +// AX defines the core control plane gRPC service for autonomous agent orchestration. +service AX { + // Manifests are parsed by the client (see `ax apply`) and submitted through the + // typed Update* RPCs below; the server never receives raw YAML. + + // Tasks + rpc GetTask(GetTaskRequest) returns (Task); + rpc ListTasks(ListTasksRequest) returns (ListTasksResponse); + rpc UpdateTask(UpdateTaskRequest) returns (Task); + rpc DeleteTask(DeleteTaskRequest) returns (DeleteTaskResponse); + rpc SuspendTask(SuspendTaskRequest) returns (Task); + rpc ResumeTask(ResumeTaskRequest) returns (Task); + rpc WatchTask(WatchTaskRequest) returns (stream WatchTaskResponse); + // GetTaskResult returns the result file the task's command wrote, as copied + // by the controller when the command exited. NotFound until then. + rpc GetTaskResult(GetTaskResultRequest) returns (TaskResult); + + // Gateways + rpc GetGateway(GetGatewayRequest) returns (Gateway); + rpc ListGateways(ListGatewaysRequest) returns (ListGatewaysResponse); + rpc UpdateGateway(UpdateGatewayRequest) returns (Gateway); + rpc DeleteGateway(DeleteGatewayRequest) returns (DeleteGatewayResponse); + + // Workspaces + rpc GetWorkspace(GetWorkspaceRequest) returns (Workspace); + rpc ListWorkspaces(ListWorkspacesRequest) returns (ListWorkspacesResponse); + rpc UpdateWorkspace(UpdateWorkspaceRequest) returns (Workspace); + rpc DeleteWorkspace(DeleteWorkspaceRequest) returns (DeleteWorkspaceResponse); + + // Models + rpc GetModel(GetModelRequest) returns (Model); + rpc ListModels(ListModelsRequest) returns (ListModelsResponse); + rpc UpdateModel(UpdateModelRequest) returns (Model); + rpc DeleteModel(DeleteModelRequest) returns (DeleteModelResponse); + +} + +// ObjectMeta is metadata for AX resources. +message ObjectMeta { + string name = 1; + string atespace = 2; + google.protobuf.Timestamp creation_timestamp = 3; +} + +// --- Task --- + +message Task { + string api_version = 1; + string kind = 2; + ObjectMeta metadata = 3; + TaskSpec spec = 4; + TaskStatus status = 5; +} + +message TaskSpec { + // Field 1 was `goal`, removed; the workspace goal lives on WorkspaceRef. + reserved 1; + reserved "goal"; + bool suspend = 2; + string image = 3; + repeated string command = 4; + repeated EnvVar env = 5; + ResourceReqs resources = 6; + // workspaces binds one or more Workspaces, each mounted at its own path + // under /workspace. The first entry is the task command's working directory. + repeated WorkspaceRef workspaces = 7; + GatewayRef gateway = 8; + // Field 9 was `policies` (budget and approval config), removed for now. + reserved 9; + reserved "policies"; + // debug enables the in-container guest services (process execution and file + // access) that back `ax ssh`. Off by default. + bool debug = 10; +} + +message EnvVar { + string name = 1; + string value = 2; +} + +message ResourceReqs { + ResourceList requests = 1; + ResourceList limits = 2; +} + +message ResourceList { + string cpu = 1; + string memory = 2; +} + +message WorkspaceRef { + string name = 1; + string path = 2; + string goal = 3; +} + +message GatewayRef { + string name = 1; +} + +message TaskStatus { + string phase = 1; + string id = 2; + string actor = 3; + // json_name keeps the established `workerIP` spelling in YAML and JSON. + string worker_ip = 4 [json_name = "workerIP"]; + PendingApproval pending_approval = 5; + UsageStats usage = 6; + repeated Condition conditions = 7; +} + +message PendingApproval { + string id = 1; + string action = 2; + google.protobuf.Timestamp requested_at = 3; +} + +message UsageStats { + int32 prompt_tokens = 1; + int32 completion_tokens = 2; + int32 tool_calls = 3; +} + +message Condition { + string type = 1; + string status = 2; + google.protobuf.Timestamp last_transition_time = 3; + string reason = 4; + string message = 5; +} + +// --- Gateway --- + +message Gateway { + string api_version = 1; + string kind = 2; + ObjectMeta metadata = 3; + GatewaySpec spec = 4; +} + +message GatewaySpec { + repeated Listener listeners = 1; + EgressConfig egress = 2; +} + +message Listener { + string name = 1; + int32 port = 2; + string protocol = 3; +} + +message EgressConfig { + EgressAllowlist allowlist = 1; +} + +message EgressAllowlist { + repeated HostRule hosts = 1; +} + +message HostRule { + string host = 1; + int32 port = 2; +} + +// --- Workspace --- + +message Workspace { + string api_version = 1; + string kind = 2; + ObjectMeta metadata = 3; + WorkspaceSpec spec = 4; +} + +message WorkspaceSpec { + repeated GitRepo git = 1; + MCPConfig mcp = 2; + SkillsConfig skills = 3; +} + +message GitRepo { + string name = 1; + string repo = 2; + string branch = 3; + string dir = 4; + int32 depth = 5; +} + +message MCPConfig { + repeated MCPRegistry registries = 1; + repeated MCPServer servers = 2; +} + +message MCPRegistry { + string provider = 1; + string project = 2; + string query = 3; + repeated MCPServer servers = 4; +} + +message MCPServer { + string name = 1; + string endpoint = 2; + string command = 3; + repeated string args = 4; +} + +message SkillsConfig { + repeated SkillRegistry registries = 1; + string path = 2; +} + +message SkillRegistry { + string provider = 1; + string project = 2; + string query = 3; +} + +// --- Model --- + +message Model { + string api_version = 1; + string kind = 2; + ObjectMeta metadata = 3; + ModelSpec spec = 4; +} + +message ModelSpec { + string provider = 1; + string model = 2; + // Fields 3-5 were typed temperature, max_tokens, and system_instruction; + // provider settings now live in the free-form `parameters` map. + reserved 3, 4, 5; + reserved "temperature", "max_tokens", "system_instruction"; + SecretKeyRef secret_key = 6; + // parameters are provider-specific generation settings passed through to the + // model API as-is, for example temperature or maxOutputTokens for Gemini. + google.protobuf.Struct parameters = 7; +} + +message SecretKeyRef { + string name = 1; + string key = 2; +} + +// --- RPC Request & Response Messages --- + +// Tasks +message GetTaskRequest { + string atespace = 1; + string name = 2; +} + +message GetTaskResultRequest { + string atespace = 1; + string name = 2; +} + +message TaskResult { + bytes content = 1; + string sha256 = 2; +} + +message ListTasksRequest { + string atespace = 1; + int64 limit = 2; + int64 offset = 3; +} + +message ListTasksResponse { + repeated Task tasks = 1; +} + +message UpdateTaskRequest { + Task task = 1; +} + +message DeleteTaskRequest { + string atespace = 1; + string name = 2; +} + +message DeleteTaskResponse {} + +message SuspendTaskRequest { + string atespace = 1; + string name = 2; +} + +message ResumeTaskRequest { + string atespace = 1; + string name = 2; +} + +message WatchTaskRequest { + string atespace = 1; + string name = 2; +} + +message WatchTaskResponse { + Task task = 1; + string action = 2; +} + +// Gateways +message GetGatewayRequest { + string atespace = 1; + string name = 2; +} + +message ListGatewaysRequest { + string atespace = 1; +} + +message ListGatewaysResponse { + repeated Gateway gateways = 1; +} + +message UpdateGatewayRequest { + Gateway gateway = 1; +} + +message DeleteGatewayRequest { + string atespace = 1; + string name = 2; +} + +message DeleteGatewayResponse {} + +// Workspaces +message GetWorkspaceRequest { + string atespace = 1; + string name = 2; +} + +message ListWorkspacesRequest { + string atespace = 1; +} + +message ListWorkspacesResponse { + repeated Workspace workspaces = 1; +} + +message UpdateWorkspaceRequest { + Workspace workspace = 1; +} + +message DeleteWorkspaceRequest { + string atespace = 1; + string name = 2; +} + +message DeleteWorkspaceResponse {} + +// Models +message GetModelRequest { + string atespace = 1; + string name = 2; +} + +message ListModelsRequest { + string atespace = 1; +} + +message ListModelsResponse { + repeated Model models = 1; +} + +message UpdateModelRequest { + Model model = 1; +} + +message DeleteModelRequest { + string atespace = 1; + string name = 2; +} + +message DeleteModelResponse {} + diff --git a/pkgs/substrate-link/src/apps/link/src/ax.ts b/pkgs/substrate-link/src/apps/link/src/ax.ts new file mode 100644 index 000000000..cdff0d5d9 --- /dev/null +++ b/pkgs/substrate-link/src/apps/link/src/ax.ts @@ -0,0 +1,121 @@ +// The link's view of ax: five calls over plain gRPC (h2c), no CreateTask (ax has none; UpdateTask is an upsert, so +// the link only ever calls it after GetTask answered NotFound: decision L4, create-only). +// ListTasks is paged: ax-server answers limit 0 with 50 rows, newest first (MEASURED upstream +// internal/server/server.go:91-104), so `listAll` walks pages until a short one (B1, critique C1). +import { createHash } from "node:crypto" +import { isIPv4, isIPv6 } from "node:net" +import { dirname, resolve } from "node:path" +import { fileURLToPath } from "node:url" +import * as grpc from "@grpc/grpc-js" +import * as protoLoader from "@grpc/proto-loader" +import { Effect } from "effect" +import type { AxTask } from "./contract.ts" + +export class AxError { + readonly _tag = "AxError" + constructor(readonly code: number, readonly message: string) {} + get unavailable() { return this.code === grpc.status.UNAVAILABLE || this.code === grpc.status.DEADLINE_EXCEEDED || this.code === grpc.status.INTERNAL } + get resourceExhausted() { return this.code === grpc.status.RESOURCE_EXHAUSTED } +} +export const AX_PAGE = 50 // ax-server's own default page size + +export interface Condition { readonly type?: string; readonly status?: string; readonly reason?: string; readonly message?: string } +export interface AxObserved { + readonly name: string + readonly phase: string // "" and Pending before the controller acts; Running; Completed/Failed (P1); Terminating + readonly spec: { image?: string; command?: ReadonlyArray; env?: ReadonlyArray<{ name?: string; value?: string }>; gateway?: { name?: string } | null } + readonly conditions: ReadonlyArray + readonly usage?: { readonly promptTokens?: number; readonly completionTokens?: number; readonly toolCalls?: number } | null +} +/** `digestOk` is false when the server's sha256 does not match the bytes received, undefined when it sent none (B4). */ +export interface AxResult { readonly content: string; readonly sha256: string; readonly digestOk?: boolean } +/** A Gateway's egress, as ax applies it (MEASURED upstream internal/controller/reconciler.go:188-198 and + * internal/substrate/client.go:450-462): no egress or no allowlist means `*:443`, an empty host list applies no + * policy, and a `*` or `0.0.0.0/0` host allows everything. */ +export interface AxGateway { readonly hosts: ReadonlyArray<{ readonly host: string; readonly port: number }>; readonly hasAllowlist: boolean } +/** Round 2: ax sends every host containing "/" to Substrate as a CIDR rule (MEASURED upstream client.go:456-491), so an + * allowlist can be open by arithmetic (`::/0`, `0.0.0.0/1` + `128.0.0.0/1`). Fail closed: a CIDR that does not parse, + * or one at or wider than /8 (IPv4) or /16 (IPv6), counts as open. */ +export const cidrIsOpen = (host: string): boolean => { + const [addr = "", len, ...rest] = host.split("/") + if (rest.length > 0 || len === undefined || !/^\d{1,3}$/.test(len)) return true + const n = Number(len) + if (isIPv4(addr)) return n > 32 || n <= 8 + if (isIPv6(addr)) return n > 128 || n <= 16 + return true +} +export const gatewayAllowsAll = (g: AxGateway) => + !g.hasAllowlist || g.hosts.length === 0 || g.hosts.some((x) => { + const h = x.host.trim().toLowerCase() + return h === "*" || h === "" || (h.includes("/") && cidrIsOpen(h)) + }) + +/** Round 2: the link's own runtime proto, P1 included (see its header). `LINK_AX_PROTO_PATH` overrides it. */ +export const LINK_PROTO_PATH = resolve(dirname(fileURLToPath(import.meta.url)), "..", "proto", "ax-p1.proto") + +export interface AxApi { + readonly getTask: (name: string) => Effect.Effect // NotFound -> undefined + readonly createTask: (task: AxTask) => Effect.Effect // UpdateTask, called only after NotFound + /** One page of the executor's list (Buildkite informer); `limit` 0 means the server's 50. */ + readonly listTasks: (limit: number, offset: number) => Effect.Effect, AxError> + readonly getGateway: (name: string) => Effect.Effect // NotFound -> undefined + readonly deleteTask: (name: string) => Effect.Effect // NotFound counts as done + /** P1's GetTaskResult. `unimplemented` means the server predates P1 (stock v0.3.0): completion is the guest's (L7). */ + readonly getTaskResult: (name: string) => Effect.Effect +} + +/** Every page, until a short one. Rows can shift between pages while ax inserts or deletes; the link therefore + * confirms any "absent" Task with GetTask before acting on it (B1). */ +export const listAll = (ax: AxApi, page = AX_PAGE, maxPages = 400): Effect.Effect, AxError> => Effect.gen(function*() { + const out = new Map() + for (let i = 0; i < maxPages; i++) { + const rows = yield* ax.listTasks(page, i * page) + for (const t of rows) out.set(t.name, t) + if (rows.length < page) return [...out.values()] + } + return yield* Effect.fail(new AxError(grpc.status.OUT_OF_RANGE, `more than ${maxPages * page} Tasks in the atespace`)) +}) + +const observe = (t: any): AxObserved => ({ + name: t?.metadata?.name ?? "", + phase: t?.status?.phase ?? "", + spec: t?.spec ?? {}, + conditions: t?.status?.conditions ?? [], + usage: t?.status?.usage ?? null +}) + +/** The production AxApi: `address` is ax-server's `host:port` (the ClusterIP 10.201.0.80:8080 on the NAS). */ +export function grpcAx(address: string, atespace: string, deadlineMs = 10_000, protoPath: string = LINK_PROTO_PATH): AxApi & { close: () => void } { + const def = protoLoader.loadSync(protoPath, { keepCase: false, longs: String, enums: String, defaults: true, oneofs: true }) + const Ctor = (grpc.loadPackageDefinition(def) as any).ax?.v1alpha1?.AX + if (typeof Ctor !== "function") throw new Error(`ax.v1alpha1.AX not found in ${protoPath}`) + // Round 2: a proto without GetTaskResult would answer the P1 probe "unimplemented" locally, never asking the server + if (typeof Ctor.prototype?.GetTaskResult !== "function") throw new Error(`${protoPath} declares no GetTaskResult; the link needs its P1 proto`) + const client = new Ctor(address, grpc.credentials.createInsecure()) + const unary = (method: string, req: unknown) => Effect.callback((resume) => { + if (typeof client[method] !== "function") return resume(Effect.fail(new AxError(grpc.status.UNIMPLEMENTED, `${method} not in proto`))) + client[method](req, { deadline: new Date(Date.now() + deadlineMs) }, (err: grpc.ServiceError | null, res: T) => + resume(err ? Effect.fail(new AxError(err.code ?? grpc.status.UNKNOWN, err.details ?? err.message)) : Effect.succeed(res))) + }) + const notFound = (e: Effect.Effect, dflt: A) => + e.pipe(Effect.catchIf((x: AxError) => x.code === grpc.status.NOT_FOUND, () => Effect.succeed(dflt))) + return { + getTask: (name) => notFound(unary("GetTask", { atespace, name }).pipe(Effect.map(observe)), undefined), + createTask: (task) => unary("UpdateTask", { task }).pipe(Effect.asVoid), + listTasks: (limit, offset) => unary("ListTasks", { atespace, limit, offset }).pipe(Effect.map((r) => (r?.tasks ?? []).map(observe))), + getGateway: (name) => notFound(unary("GetGateway", { atespace, name }).pipe(Effect.map((g): AxGateway => { + const al = g?.spec?.egress?.allowlist + return { hasAllowlist: al != null, hosts: (al?.hosts ?? []).map((h: any) => ({ host: String(h?.host ?? ""), port: Number(h?.port ?? 0) })) } + })), undefined), + deleteTask: (name) => notFound(unary("DeleteTask", { atespace, name }).pipe(Effect.asVoid), undefined), + getTaskResult: (name) => notFound(unary("GetTaskResult", { atespace, name }).pipe( + Effect.map((r): AxResult => { + const bytes = Buffer.from(r?.content ?? "") + const sha256 = String(r?.sha256 ?? "") + return { content: bytes.toString("utf8"), sha256, ...(sha256 ? { digestOk: createHash("sha256").update(bytes).digest("hex") === sha256.toLowerCase() } : {}) } + }), + Effect.catchIf((x: AxError) => x.code === grpc.status.UNIMPLEMENTED, () => Effect.succeed("unimplemented" as const)) + ), undefined), + close: () => client.close() + } +} diff --git a/pkgs/substrate-link/src/apps/link/src/config.ts b/pkgs/substrate-link/src/apps/link/src/config.ts new file mode 100644 index 000000000..24198b656 --- /dev/null +++ b/pkgs/substrate-link/src/apps/link/src/config.ts @@ -0,0 +1,103 @@ +// conwip-link configuration from the environment (round 1, finding "config"). Every numeric key must be a plain +// non-negative integer inside its range and every enum key one of its literals; anything else is ConfigInvalid, which +// main.ts turns into exit 78 so systemd's RestartPreventExitStatus holds the unit down instead of letting it spin +// (Effect.sleep(NaN) is a zero sleep) or crash-loop. Errors name the key and the rule, never the value. + +export class ConfigInvalid extends Error { + readonly _tag = "ConfigInvalid" + constructor(readonly key: string, readonly why: string) { super(`${key}: ${why}`) } +} + +type Env = Readonly> + +const str = (env: Env, key: string, d?: string): string => { + const v = env[key] + if (v !== undefined && v !== "") return v + if (d !== undefined) return d + throw new ConfigInvalid(key, "missing") +} + +export const int = (env: Env, key: string, d: number, min: number, max = 1_000_000): number => { + const v = str(env, key, String(d)).trim() + if (!/^\d+$/.test(v)) throw new ConfigInvalid(key, "not a non-negative integer") + const n = Number(v) + if (n < min || n > max) throw new ConfigInvalid(key, `outside ${min}..${max}`) + return n +} + +export const literal = (env: Env, key: string, d: L, allowed: ReadonlyArray): L => { + const v = str(env, key, d) + if (!(allowed as ReadonlyArray).includes(v)) throw new ConfigInvalid(key, `not one of ${allowed.join(", ")}`) + return v as L +} + +export const seatCommands = (env: Env, key: string, d: string): Record> => { + let parsed: unknown + try { parsed = JSON.parse(str(env, key, d)) } catch { throw new ConfigInvalid(key, "not JSON") } + if (parsed === null || typeof parsed !== "object" || Array.isArray(parsed)) throw new ConfigInvalid(key, "not a JSON object") + const out: Record> = {} + for (const [seat, cmd] of Object.entries(parsed)) { + if (!Array.isArray(cmd) || cmd.length === 0 || !cmd.every((c) => typeof c === "string" && c !== "")) + throw new ConfigInvalid(key, `seat ${JSON.stringify(seat)} is not a non-empty array of non-empty strings`) + out[seat] = cmd as Array + } + return out +} + +export interface LinkEnv { + readonly holder: string + readonly maxInFlight: number + readonly servedLabels: Array + readonly atespace: string + readonly image: string + readonly gateway: string + readonly axServer: string + readonly guestCompleteUrl: string | undefined + readonly seatCommands: Record> + readonly completion: "auto" | "p1" | "guest" + readonly resyncSeconds: number + readonly pendingTimeoutSeconds: number + readonly deleteAfterSeconds: number + readonly deadlineBackstopSeconds: number + readonly createAttempts: number + readonly outboxBackoffSeconds: readonly [number, number] + readonly fenceTimeoutSeconds: number + readonly resultReadTries: number + readonly verdictAttempts: number // round 2 + readonly maxOutputBytes: number // round 2: below the floor's 2 MB DO row limit + readonly internalHosts: Array // round 2: single-label names a guest Complete URL may use + readonly axProtoPath: string | undefined // round 2: the link's ax proto (must declare GetTaskResult) +} + +/** Throws ConfigInvalid on the first bad key. Defaults are the NixOS module's. */ +export const readLinkEnv = (env: Env): LinkEnv => { + const outMin = int(env, "LINK_OUTBOX_BACKOFF_MIN_SECONDS", 60, 1, 86_400) + const outMax = int(env, "LINK_OUTBOX_BACKOFF_MAX_SECONDS", 900, 1, 86_400) + if (outMax < outMin) throw new ConfigInvalid("LINK_OUTBOX_BACKOFF_MAX_SECONDS", "below LINK_OUTBOX_BACKOFF_MIN_SECONDS") + const servedLabels = str(env, "LINK_SERVED_LABELS", "seat:halogen,runtime:gvisor").split(",").map((l) => l.trim()).filter((l) => l !== "") + if (servedLabels.length === 0) throw new ConfigInvalid("LINK_SERVED_LABELS", "empty") + return { + holder: str(env, "LINK_HOLDER", "nas-link-1"), + maxInFlight: int(env, "LINK_MAX_IN_FLIGHT", 2, 1, 1000), + servedLabels, + atespace: str(env, "AX_ATESPACE", "fleet"), + image: str(env, "LINK_IMAGE", "ax-agent"), + gateway: str(env, "LINK_GATEWAY", "halogen"), + axServer: str(env, "AX_SERVER", "10.201.0.80:8080"), + guestCompleteUrl: env.LINK_GUEST_COMPLETE_URL ? env.LINK_GUEST_COMPLETE_URL : undefined, + seatCommands: seatCommands(env, "LINK_SEAT_COMMANDS", '{"halogen":["ax-agent","pi"]}'), + completion: literal(env, "LINK_COMPLETION", "auto", ["auto", "p1", "guest"]), + resyncSeconds: int(env, "LINK_RESYNC_SECONDS", 15, 1, 3600), + pendingTimeoutSeconds: int(env, "LINK_PENDING_TIMEOUT_SECONDS", 900, 1, 86_400), + deleteAfterSeconds: int(env, "LINK_DELETE_AFTER_SECONDS", 600, 0, 604_800), + deadlineBackstopSeconds: int(env, "LINK_DEADLINE_BACKSTOP_SECONDS", 300, 0, 86_400), + createAttempts: int(env, "LINK_CREATE_ATTEMPTS", 7, 1, 100), + outboxBackoffSeconds: [outMin, outMax], + fenceTimeoutSeconds: int(env, "LINK_FENCE_TIMEOUT_SECONDS", 120, 1, 86_400), + resultReadTries: int(env, "LINK_RESULT_READ_TRIES", 5, 1, 1000), + verdictAttempts: int(env, "LINK_VERDICT_ATTEMPTS", 8, 1, 1000), + maxOutputBytes: int(env, "LINK_MAX_OUTPUT_BYTES", 1_000_000, 1024, 2_000_000), + internalHosts: (env.LINK_INTERNAL_HOSTS ?? "").split(",").map((h) => h.trim()).filter((h) => h !== ""), + axProtoPath: env.LINK_AX_PROTO_PATH ? env.LINK_AX_PROTO_PATH : undefined + } +} diff --git a/pkgs/substrate-link/src/apps/link/src/contract.ts b/pkgs/substrate-link/src/apps/link/src/contract.ts new file mode 100644 index 000000000..40e93d8b1 --- /dev/null +++ b/pkgs/substrate-link/src/apps/link/src/contract.ts @@ -0,0 +1,181 @@ +// The floor <-> link wire contract (LINK-DESIGN.md section 3). Starting point: the option-A prototype +// (/home/tom/today/evals-2026-09-23/arc/proto/contract.ts, decision L1). Every L1 name is kept as it was. +// Five keys are ADDED, each optional so an L1-only peer ignores them, each named after its source: +// Grant.supersedes L3 (two-stage expiry): the old attempt the link must DeleteTask first +// Grant.leaseToken L7 (completion before ax P1): a per-lease token for the guest (Buildkite job token) +// Grant.deadline Temporal start-to-close, as an absolute epoch ms the link can hold across a restart +// Lease.endpoint Buildkite ping `endpoint`: move the link to a new floor URL after a test call +// Complete.withdrew round 2: the leaseId of attempt n+1 that rule 4b withdrew when it accepted attempt n's verdict. +// The link releases n+1 only when it is named here; it never infers a withdrawal (a duplicate +// answer or a delivered retryable failure withdrew nothing) +// Complete.withdrewTransitions round 3: the leaseTransitions of the generation rule 4b withdrew. A floor that +// requeues the withdrawn attempt under the same leaseId grants it again with a higher value; the +// link treats a grant as withdrawn only when its generation matches +// Nothing here is GitHub-hosted at runtime. +import { Schema } from "effect" +import { Rpc, RpcGroup } from "effect/unstable/rpc" + +// Actions job result values (GitHub/Gitea job `result`). +export const Result = Schema.Literals(["success", "failure", "cancelled", "skipped"]) +export type Result = typeof Result.Type + +// ---- AgentJob: an ultracode agent() node as an Actions-style job in a Kubernetes object envelope (unchanged) ---- +const Labels = Schema.Record(Schema.String, Schema.String) +export const PromptRef = Schema.Struct({ + sha256: Schema.String, // content address; the guest refuses on mismatch (FIELD-MAP 5a AX_CONWIP_PROMPT_SHA256) + bytes: Schema.Int, + uri: Schema.String // journal:////prompt.md +}) +export const AgentJob = Schema.Struct({ + apiVersion: Schema.Literal("ultracode.mecattaf.dev/v1alpha1"), + kind: Schema.Literal("AgentJob"), + metadata: Schema.Struct({ name: Schema.String, labels: Schema.optionalKey(Labels), annotations: Schema.optionalKey(Labels) }), + spec: Schema.Struct({ + // CONWIP admission key, e.g. ["seat:halogen","runtime:gvisor"]. B7 asked for Schema.NonEmptyArray here; it stays + // Array on the wire because one malformed job would then fail the decode of a whole Lease reply and strand every + // grant in it (INFERRED). The link refuses it per job instead (`validRunsOn`, jobs.ts), and the floor at enqueue. + "runs-on": Schema.Array(Schema.String), + "timeout-minutes": Schema.optionalKey(Schema.Int), // Temporal start-to-close, enforced by the floor + "continue-on-error": Schema.optionalKey(Schema.Boolean), + environment: Schema.optionalKey(Schema.NullOr(Schema.String)), // Tom's ack gate + with: Schema.Struct({ + prompt: Schema.optionalKey(Schema.String), + prompt_ref: PromptRef, + model: Schema.String, + effort: Schema.optionalKey(Schema.String), + schema: Schema.optionalKey(Schema.Unknown), + isolation: Schema.optionalKey(Schema.String), + "agent-type": Schema.optionalKey(Schema.String) + }), + needs: Schema.optionalKey(Schema.Record(Schema.String, Schema.Struct({ result: Result, outputs_ref: Schema.String }))), + outputs: Schema.optionalKey(Schema.Array(Schema.String)) + }) +}) +export type AgentJob = typeof AgentJob.Type + +// ---- ax Task JSON (protojson camelCase of ax.proto v0.3.0 d8ed0fe). The link emits it; it never rides the wire ---- +// apiVersion is ax's own constant `ax.io/v1alpha1` (ax pkg/apis/v1alpha1/types.go:31); the prototype's +// `ax/v1alpha1` passed only because ax stores what it is given. +export const AxTask = Schema.Struct({ + apiVersion: Schema.Literal("ax.io/v1alpha1"), + kind: Schema.Literal("Task"), + metadata: Schema.Struct({ name: Schema.String, atespace: Schema.String }), + spec: Schema.Struct({ + image: Schema.String, + command: Schema.Array(Schema.String), + env: Schema.Array(Schema.Struct({ name: Schema.String, value: Schema.String })), + gateway: Schema.optionalKey(Schema.Struct({ name: Schema.String })) + }) +}) +export type AxTask = typeof AxTask.Type + +// ---- the link contract ---- +// Kubernetes coordination.k8s.io/v1 LeaseSpec field names. +export const LeaseSpec = Schema.Struct({ + holderIdentity: Schema.String, leaseDurationSeconds: Schema.Int, + acquireTime: Schema.Number, renewTime: Schema.Number, leaseTransitions: Schema.Int +}) +// Absolute counts on every response plus an increasing seq (ARC scaleset statistics). +export const Stats = Schema.Struct({ seq: Schema.Int, cap: Schema.Int, wip: Schema.Int, queued: Schema.Int, done: Schema.Int }) +export const Grant = Schema.Struct({ + leaseId: Schema.String, // `-a`; it is also the ax Task name (the idempotency key on ax) + attempt: Schema.Int, + job: AgentJob, + lease: LeaseSpec, + supersedes: Schema.optionalKey(Schema.Array(Schema.String)), // L3 + leaseToken: Schema.optionalKey(Schema.String), // L7, only while the guest completes (pre-P1) + deadline: Schema.optionalKey(Schema.Number) // epoch ms; acquireTime + timeout-minutes +}) +export type Grant = typeof Grant.Type +export const Usage = Schema.Struct({ prompt_tokens: Schema.Int, completion_tokens: Schema.Int, tool_calls: Schema.Int }) +export type Usage = typeof Usage.Type + +// Temporal PollActivityTaskQueue + ARC free capacity on every poll + Forgejo request key + Buildkite intervals. +export const Lease = Rpc.make("Lease", { + payload: { holderIdentity: Schema.String, capacity: Schema.Int, requestKey: Schema.String }, + success: Schema.Struct({ + grants: Schema.Array(Grant), stats: Stats, nextPollSeconds: Schema.Int, heartbeatSeconds: Schema.Int, + endpoint: Schema.optionalKey(Schema.String) // Buildkite ping endpoint switch + }) +}) +// Temporal RecordActivityTaskHeartbeat with cancel_requested, batched per session. `leaseIds` is the COMPLETE set +// the holder believes it holds (level-triggered, ARC): an orphaned lease it omits is requeued (L3 a). +// Round 4: `pendingRequestKey` (Forgejo request key, B8) is the Lease requestKey the holder journaled and has not +// seen answered. The holder cannot list leaseIds it never received, so it vouches for the key instead: the floor +// renews every lease granted under that key and never releases one as an omitted orphan (L3 a) while it is named. +export const Heartbeat = Rpc.make("Heartbeat", { + payload: { holderIdentity: Schema.String, leaseIds: Schema.Array(Schema.String), pendingRequestKey: Schema.optionalKey(Schema.String) }, + success: Schema.Struct({ + renewed: Schema.Array(Schema.String), lost: Schema.Array(Schema.String), + cancelRequested: Schema.Array(Schema.String), stats: Stats + }) +}) +// Temporal RespondActivityTask{Completed,Failed,Canceled}. The verdict travels here, never as an exit code (ARC). +// The two errors are terminal: stop retrying (Buildkite 422). +export const CompleteError = Schema.Struct({ code: Schema.Literals(["unknown-lease", "stale-attempt"]) }) +export const Complete = Rpc.make("Complete", { + payload: { + leaseId: Schema.String, attempt: Schema.Int, result: Result, + output: Schema.optionalKey(Schema.Unknown), usage: Schema.optionalKey(Usage) + }, + success: Schema.Struct({ duplicate: Schema.Boolean, stats: Stats, withdrew: Schema.optionalKey(Schema.String), + withdrewTransitions: Schema.optionalKey(Schema.Int) }), + error: CompleteError +}) +export class FloorLink extends RpcGroup.make(Lease, Heartbeat, Complete) {} + +// The link's own view of the same wire (round 1, finding "poison grant"): identical tags, payloads and JSON, but each +// grant is decoded by the link one at a time, so one malformed job fails only its own lease (pre-start/invalid-spec) +// instead of turning the whole Lease reply into a decode defect that the journaled requestKey would replay for ever. +// Round 3: the whole envelope is lenient too. A drifted interval (the prototype floor sends leaseMs / 3000, e.g. +// 3.333), a null endpoint or a changed stats shape would otherwise fail the decode of every replay of the journaled +// requestKey and strand the grants in it. The link bounds the intervals itself (link.ts `boundedSeconds`). +export const LeaseLenient = Rpc.make("Lease", { + payload: { holderIdentity: Schema.String, capacity: Schema.Int, requestKey: Schema.String }, + success: Schema.Struct({ + grants: Schema.Array(Schema.Unknown), stats: Schema.optionalKey(Schema.Unknown), + nextPollSeconds: Schema.optionalKey(Schema.Unknown), heartbeatSeconds: Schema.optionalKey(Schema.Unknown), + endpoint: Schema.optionalKey(Schema.Unknown) + }) +}) +// Round 2: the link decodes Complete's error code as any string, so a code the floor adds later is a permanent refusal +// (dropped, logged with the code), never a decode defect retried for ever at the head of the outbox. +// Round 3: its success is lenient as well. Any 2xx Exit Success means the floor applied the verdict; a drifted key +// (`withdrew: null`, a missing stats) must not keep an applied verdict at the head of the outbox. +export const CompleteLenient = Rpc.make("Complete", { + payload: Complete.payloadSchema.fields, + success: Schema.Struct({ duplicate: Schema.optionalKey(Schema.Unknown), stats: Schema.optionalKey(Schema.Unknown), + withdrew: Schema.optionalKey(Schema.Unknown), withdrewTransitions: Schema.optionalKey(Schema.Unknown) }), + error: Schema.Struct({ code: Schema.String }) +}) +// Round 4: Heartbeat is lenient as well. The strict reply schema turned a drifted `stats` into a heartbeat-error on +// every call, so `cancelRequested`, `lost` and `renewed` were never read while the floor kept renewing (a cancel was +// never carried out, a journaled grant never resumed). Every key is optional Unknown; floor.ts keeps the strings. +export const HeartbeatLenient = Rpc.make("Heartbeat", { + payload: Heartbeat.payloadSchema.fields, + success: Schema.Struct({ renewed: Schema.optionalKey(Schema.Unknown), lost: Schema.optionalKey(Schema.Unknown), + cancelRequested: Schema.optionalKey(Schema.Unknown), stats: Schema.optionalKey(Schema.Unknown) }) +}) +export class FloorLinkClient extends RpcGroup.make(LeaseLenient, HeartbeatLenient, CompleteLenient) {} + +// On result "failure" the link's `output` is this shape (Buildkite finish signal_reason, ARC failure reason). +// The reasons in RETRYABLE are the floor's to retry as attempt+1 (Temporal retry policy, Buildkite finish -1 on a +// pre-start failure); every other reason is final: a deterministic refusal, or the agent's own failure, which goes +// to the interpreter (null-on-failure). +export const FailureOutput = Schema.Struct({ + reason: Schema.String, // pre-start/env-over-budget, pre-start/name-conflict, infra/task-lost, deadline-exceeded, ... + message: Schema.optionalKey(Schema.String), + phase: Schema.optionalKey(Schema.String), + exitCode: Schema.optionalKey(Schema.Int) +}) +export type FailureOutput = typeof FailureOutput.Type +export const RETRYABLE = new Set([ + "pre-start/ax-unavailable", "pre-start/pending-timeout", "pre-start/resource-exhausted", + "infra/task-lost", "infra/actor-crashed", "infra/result-unreadable", + "pre-start/superseded-verdict-pending", // round 1: n+1 refused while attempt n's verdict could not be delivered + "infra/link-defect" // round 4: a dispatch fiber died on a defect before the Task was created; the floor requeues +]) +// Round 2, final (not in RETRYABLE): `infra/verdict-undeliverable` (the floor answered every delivery of the real +// verdict with a server error; a rerun would most likely produce the same bytes) and `agent/output-too-large` (the +// result is over the link's cap, below the floor's 2 MB row limit). +export const isRetryableReason = (reason: string) => RETRYABLE.has(reason) diff --git a/pkgs/substrate-link/src/apps/link/src/floor.ts b/pkgs/substrate-link/src/apps/link/src/floor.ts new file mode 100644 index 000000000..0884c3192 --- /dev/null +++ b/pkgs/substrate-link/src/apps/link/src/floor.ts @@ -0,0 +1,169 @@ +// The link's view of the floor: the three L1 RPCs over Effect's stock HTTP RPC client (decision L2), outbound only. +// HTTP statuses the floor's Worker entry answers before any RPC runs are classified here (Buildkite's retry +// classifier, api/retryable.go): 401/403 fatal (token), 409 fatal (a second session for this identity, L5), +// 429 back off honouring Retry-After. Round 2: any 3xx is fatal (`redirect`: redirects are never followed, so the only +// way to another floor URL is the Lease `endpoint` checked against the Nix list, B18); 404 and 405 are fatal +// (`misrouted`: a wrong route answers every call so, and dropping verdicts for it would lose them); any other 4xx except 408 is +// `rejected` (permanent for these bytes: Complete drops the verdict); a 500, 501 or 505+ and an RPC Defect in the reply +// are `server-error` (the floor answered and refused these bytes; the outbox counts them per verdict); 502, 503, 504, +// 408, network errors and timeouts stay `transient`. Every call has a ceiling (B9). +import { Effect, Layer, Schema, Scope } from "effect" +import { FetchHttpClient, HttpClient, HttpClientRequest } from "effect/unstable/http" +import { RpcClient, RpcSerialization } from "effect/unstable/rpc" +import { Complete, FloorLinkClient, Grant } from "./contract.ts" +import type { Usage, Result } from "./contract.ts" + +/** A grant the link could not decode. `leaseId` and `attempt` are set when they alone decode, so the lease can be + * failed back to the floor as pre-start/invalid-spec; otherwise the floor's own expiry is the only way out. */ +export interface InvalidGrant { readonly leaseId?: string; readonly attempt?: number; readonly message: string } +/** Round 3: the reply as the link uses it, after the lenient envelope decode. Intervals and endpoint are kept only + * when they have the right JSON type; the link bounds the intervals (link.ts `boundedSeconds`). */ +export type LeaseReply = { + readonly grants: ReadonlyArray; readonly invalid?: ReadonlyArray + readonly stats?: unknown; readonly nextPollSeconds?: number; readonly heartbeatSeconds?: number; readonly endpoint?: string +} +/** Round 4: the Heartbeat reply after the lenient decode: each list keeps only its strings, a missing one is []. */ +type HeartbeatReply = { readonly renewed: ReadonlyArray; readonly lost: ReadonlyArray; readonly cancelRequested: ReadonlyArray; readonly stats?: unknown } +const strings = (v: unknown): Array => Array.isArray(v) ? v.filter((x): x is string => typeof x === "string") : [] +const heartbeatReply = (r: { readonly renewed?: unknown; readonly lost?: unknown; readonly cancelRequested?: unknown; readonly stats?: unknown }): HeartbeatReply => + ({ renewed: strings(r.renewed), lost: strings(r.lost), cancelRequested: strings(r.cancelRequested), stats: r.stats }) +type CompleteReply = { readonly duplicate: boolean; readonly withdrew?: string; readonly withdrewTransitions?: number; readonly stats?: unknown } +const num = (v: unknown) => typeof v === "number" ? v : undefined +const leaseReply = (r: { readonly grants: ReadonlyArray; readonly nextPollSeconds?: unknown; readonly heartbeatSeconds?: unknown; readonly endpoint?: unknown; readonly stats?: unknown }): LeaseReply => { + const s = splitGrants(r) + const poll = num(r.nextPollSeconds), hb = num(r.heartbeatSeconds) + return { grants: s.grants, invalid: s.invalid, stats: r.stats, ...(poll !== undefined ? { nextPollSeconds: poll } : {}), + ...(hb !== undefined ? { heartbeatSeconds: hb } : {}), ...(typeof r.endpoint === "string" ? { endpoint: r.endpoint } : {}) } +} +const completeReply = (r: { readonly duplicate?: unknown; readonly withdrew?: unknown; readonly withdrewTransitions?: unknown; readonly stats?: unknown }): CompleteReply => { + const wt = num(r.withdrewTransitions) + return { duplicate: r.duplicate === true, stats: r.stats, ...(typeof r.withdrew === "string" ? { withdrew: r.withdrew } : {}), + ...(wt !== undefined && Number.isInteger(wt) ? { withdrewTransitions: wt } : {}) } +} +const encodeComplete = Schema.encodeUnknownExit(Complete.payloadSchema) + +export type FloorErrorKind = "transient" | "auth" | "session-conflict" | "rate-limited" | "redirect" | "misrouted" | "rejected" | "server-error" +export class FloorError { + readonly _tag = "FloorError" + constructor(readonly kind: FloorErrorKind, readonly message: string, readonly retryAfterMs?: number, readonly status?: number) {} + get fatal() { return this.kind === "auth" || this.kind === "session-conflict" || this.kind === "redirect" || this.kind === "misrouted" } +} +/** The floor refused these bytes for good: one of its CompleteError codes (any string, round 2), or an HTTP 4xx. */ +export class CompleteRefused { + readonly _tag = "CompleteRefused" + constructor(readonly code: string) {} +} + +export interface CompletePayload { leaseId: string; attempt: number; result: Result; output?: unknown; usage?: Usage } +export interface FloorApi { + readonly lease: (p: { holderIdentity: string; capacity: number; requestKey: string }) => Effect.Effect + readonly heartbeat: (p: { holderIdentity: string; leaseIds: ReadonlyArray; pendingRequestKey?: string }) => Effect.Effect + readonly complete: (p: CompletePayload) => Effect.Effect +} + +export interface FloorOptions { + readonly url: string // e.g. https://conwip-floor..workers.dev (no trailing /rpc) + readonly token: string // the per-link bearer, read from $CREDENTIALS_DIRECTORY; never logged + readonly sessionId: string // stable per state directory: a restart is the same session, a second replica is not + readonly fetch?: typeof globalThis.fetch // tests hand in an in-process handler; production uses globalThis.fetch + /** B9: a ceiling on every call (Buildkite's 330 s on acquire, shortened): Lease and Heartbeat 20 s, Complete 60 s. */ + readonly timeoutsMs?: { readonly lease: number; readonly heartbeat: number; readonly complete: number } +} +export const DEFAULT_FLOOR_TIMEOUTS_MS = { lease: 20_000, heartbeat: 20_000, complete: 60_000 } as const + +/** B9 (critique R9): a non-2xx answer is thrown by the fetch wrapper of the call that received it, so its status + * travels inside that call's own error chain; nothing is shared between concurrent calls. */ +class HttpStatus extends Error { + readonly linkHttpStatus = true + constructor(readonly status: number, readonly retryAfterMs: number | undefined) { super(`floor answered ${status}`) } +} +const statusIn = (e: unknown): HttpStatus | undefined => { + let x: any = e + for (let i = 0; i < 10 && x != null && typeof x === "object"; i++) { + if (x instanceof HttpStatus || x.linkHttpStatus === true) return x as HttpStatus + x = x.reason ?? x.cause ?? x.error + } + return undefined +} +const decodeGrant = Schema.decodeUnknownExit(Grant) +const Envelope = Schema.decodeUnknownExit(Schema.Struct({ leaseId: Schema.String, attempt: Schema.Int })) +/** Decode each grant on its own: the good ones are dispatched, the bad ones are refused one by one. */ +export const splitGrants = }>(r: R): Omit & { grants: Array; invalid: Array } => { + const grants: Array = [], invalid: Array = [] + for (const raw of r.grants) { + const g = decodeGrant(raw) + if (g._tag === "Success") { grants.push(g.value); continue } + const env = Envelope(raw) + const message = String(g.cause).slice(0, 500) + invalid.push(env._tag === "Success" ? { leaseId: env.value.leaseId, attempt: env.value.attempt, message } : { message }) + } + return { ...r, grants, invalid } +} + +export const classifyFloorError = (e: unknown): FloorError => { + const st = statusIn(e) + const s = st?.status ?? 0 + const msg = e instanceof Error ? e.message : String((e as any)?.message ?? e) + if (s >= 300 && s < 400) return new FloorError("redirect", `floor answered ${s}: redirects are never followed; fix LINK_FLOOR_URL`, undefined, s) + if (s === 401 || s === 403) return new FloorError("auth", `floor answered ${s}`, undefined, s) + if (s === 409) return new FloorError("session-conflict", "floor answered 409: another session holds this link identity", undefined, s) + if (s === 429) return new FloorError("rate-limited", "floor answered 429", st?.retryAfterMs, s) + // 404/405: the route is wrong for every call, not these bytes; fatal (exit 78), so no verdict is dropped for it + if (s === 404 || s === 405) return new FloorError("misrouted", `floor answered ${s}: fix LINK_FLOOR_URL or the Worker route`, undefined, s) + if (s >= 400 && s < 500 && s !== 408) return new FloorError("rejected", `floor answered ${s}`, undefined, s) + if (s >= 500 && s !== 502 && s !== 503 && s !== 504) return new FloorError("server-error", `floor answered ${s}`, undefined, s) + return new FloorError("transient", s ? `floor answered ${s}` : msg, undefined, s || undefined) +} +/** Round 3: a defect raised by a call is always the reply's: the link encodes the Complete payload itself before it + * sends (a failure there is `transient`, the link's own bug), and the envelopes it decodes are lenient. A defect is + * then the floor's RPC `Defect` reply (e.g. SQLITE_TOOBIG) or a reply that is not the protocol at all: the server + * refusing these bytes (`server-error`), counted per verdict on Complete and per requestKey on Lease. */ +export const rpcFloor = (o: FloorOptions): Effect.Effect => Effect.gen(function*() { + const base = o.fetch ?? globalThis.fetch + const t = o.timeoutsMs ?? DEFAULT_FLOOR_TIMEOUTS_MS + const observing: typeof globalThis.fetch = async (input, init) => { + // Round 2: never follow a redirect. A followed 307 lands on an origin the Nix list never declared, and the link + // would act on its replies (B18 bypass); an Access 302 to a login page would read as a transient decode error. + const res = await base(input, { ...init, redirect: "manual" }) + if (res.type === "opaqueredirect") throw new HttpStatus(307, undefined) + if (!res.ok) { + const ra = Number(res.headers.get("retry-after")) + throw new HttpStatus(res.status, Number.isFinite(ra) && ra > 0 ? ra * 1000 : undefined) + } + return res + } + const protocol = RpcClient.layerProtocolHttp({ + url: `${o.url.replace(/\/$/, "")}/rpc`, + transformClient: HttpClient.mapRequest((r) => r.pipe( + HttpClientRequest.bearerToken(o.token), + HttpClientRequest.setHeader("x-link-session", o.sessionId))) + }).pipe(Layer.provide([FetchHttpClient.layer, RpcSerialization.layerJson]), + Layer.provide(Layer.succeed(FetchHttpClient.Fetch, observing))) + const client = yield* RpcClient.make(FloorLinkClient).pipe(Effect.provide(protocol)) + // The timeout interrupts the call, which aborts its fetch (FetchHttpClient passes an AbortSignal). + const within = (ms: number, what: string) => (e: Effect.Effect) => + e.pipe(Effect.timeoutOrElse({ duration: ms, orElse: () => Effect.fail(new FloorError("transient", `${what} timed out after ${ms} ms`)) })) + return { + // Round 1: a reply that does not decode is a typed transient failure, never a defect that kills the link. + lease: (p) => client.Lease(p).pipe(Effect.catchDefect((d) => Effect.fail(new FloorError("server-error", `Lease reply undecodable: ${String(d).slice(0, 300)}`))), + Effect.mapError((e) => e instanceof FloorError ? e : classifyFloorError(e)), Effect.map(leaseReply), within(t.lease, "Lease")), + // Round 4: lenient like Lease and Complete; a reply that still does not decode is the server's (server-error). + heartbeat: (p) => client.Heartbeat({ holderIdentity: p.holderIdentity, leaseIds: p.leaseIds, ...(p.pendingRequestKey !== undefined ? { pendingRequestKey: p.pendingRequestKey } : {}) }).pipe( + Effect.catchDefect((d) => Effect.fail(new FloorError("server-error", `Heartbeat reply undecodable: ${String(d).slice(0, 300)}`))), + Effect.mapError((e) => e instanceof FloorError ? e : classifyFloorError(e)), Effect.map(heartbeatReply), within(t.heartbeat, "Heartbeat")), + // L9: omit absent optional keys; `usage: undefined` in an optionalKey field is an encode defect, not a failure. + complete: (p) => { + const payload = { leaseId: p.leaseId, attempt: p.attempt, result: p.result, + ...(p.output !== undefined ? { output: p.output } : {}), ...(p.usage !== undefined ? { usage: p.usage } : {}) } + const enc = encodeComplete(payload) + if (enc._tag === "Failure") return Effect.fail(new FloorError("transient", `encode defect: ${String(enc.cause).slice(0, 300)}`)) + return client.Complete(payload).pipe(Effect.map(completeReply), + Effect.catchDefect((d) => Effect.fail(new FloorError("server-error", `server defect: ${String((d as any)?.message ?? d).slice(0, 300)}`))), + Effect.mapError((e: any) => e instanceof FloorError ? e + : e && typeof e === "object" && "code" in e && !("_tag" in e) ? new CompleteRefused(String(e.code)) : classifyFloorError(e)), + // Round 2: a 4xx other than 401/403/408/409/429 refuses these bytes for good (Buildkite: not retryable) + Effect.mapError((e) => e instanceof FloorError && e.kind === "rejected" ? new CompleteRefused(`http-${e.status}`) : e), + within(t.complete, "Complete")) + } + } +}) diff --git a/pkgs/substrate-link/src/apps/link/src/jobs.ts b/pkgs/substrate-link/src/apps/link/src/jobs.ts new file mode 100644 index 000000000..c8b4753c1 --- /dev/null +++ b/pkgs/substrate-link/src/apps/link/src/jobs.ts @@ -0,0 +1,120 @@ +// Pure mapping AgentJob -> ax Task JSON (FIELD-MAP-2026-09-23.md 5a keys). Ported from the option-A prototype +// (arc/proto/jobs.ts) with ARC-DECISION X1 applied: atespace `fleet`, image `ax-agent`, Gateway `halogen`, the names +// ax-fleet DESIGN.md 10.2 uses. No I/O: the link and any converter call the same function. +import { createHash } from "node:crypto" +import type { AgentJob, AxTask, Grant } from "./contract.ts" + +export const ENV_BUDGET = 16384 // FIELD-MAP 5a hard pre-dispatch check (stock v0.3.0, before P5) +const A = "ultracode.mecattaf.dev/" +const utf8Len = (s: string) => new TextEncoder().encode(s).length +/** FIELD-MAP 5a journal key: `:` (ax-conwip taskspec.ts journalKey). */ +export const JOURNAL_KEY = /^[0-9a-f]{64}:\d+$/ + +export interface TaskShape { + readonly atespace: string // `fleet` + readonly image: string // `localhost:5000/ax-agent@sha256:...` (ax-fleet-image-ref) + readonly gateway: string // `halogen` + readonly command: (job: AgentJob) => ReadonlyArray | undefined // e.g. ["ax-agent", "pi"] for seat:halogen + readonly completeUrl?: string // set only in guest-completion mode (L7, pre-P1) + readonly holder?: string // round 1: AX_CONWIP_HOLDER, the ownership mark a journal-less link checks before a fence +} + +/** B7: no default seat. `undefined` unless `runs-on` names exactly one `seat:` label. */ +export const seatOf = (job: AgentJob): string | undefined => { + const seats = job.spec["runs-on"].filter((l) => l.startsWith("seat:")) + return seats.length === 1 ? seats[0]!.slice(5) : undefined +} + +/** B7: `runs-on` must be non-empty, name exactly one `seat:` label, and be wholly served by this link. */ +export const validRunsOn = (job: AgentJob, served: ReadonlyArray): { reason: string; message: string } | undefined => { + const on = job.spec["runs-on"] + if (on.length === 0) return { reason: "pre-start/runs-on-invalid", message: "runs-on is empty" } + const seats = on.filter((l) => l.startsWith("seat:")) + if (seats.length !== 1) return { reason: "pre-start/runs-on-invalid", message: `runs-on names ${seats.length} seat: labels, not 1` } + const unserved = on.filter((l) => !served.includes(l)) + if (unserved.length > 0) return { reason: "pre-start/label-not-served", message: unserved.join(",") } + return undefined +} + +export type Built = { readonly _tag: "ok"; readonly task: AxTask } | { readonly _tag: "refused"; readonly reason: string; readonly message: string } + +export function axTaskFromGrant(g: Grant, shape: TaskShape): Built { + const job = g.job + // Round 1 (fail closed): a guest Task without its per-lease token has no completion route; never create it. + if (shape.completeUrl !== undefined && (g.leaseToken === undefined || g.leaseToken === "")) + return { _tag: "refused", reason: "pre-start/lease-token-missing", message: "guest completion needs a leaseToken; the floor's token binding for this holder is not guest" } + const ann = job.metadata.annotations ?? {} + const lab = job.metadata.labels ?? {} + if (ann[A + "prompt-unresolved"] !== undefined) // L8: never dispatch an item whose prompt was not resolved + return { _tag: "refused", reason: "pre-start/prompt-unresolved", message: ann[A + "prompt-unresolved"]! } + // Round 4 (FIELD-MAP 5a): the journal key is sha256 of canonical [prompt, opts minus label] plus the occurrence, and + // only the floor-side converter can compute it (it holds opts and the occurrence). It rides the AgentJob as an + // annotation; without it the grant is refused, never replaced by the prompt digest (two calls sharing one prompt). + const journalKey = ann[A + "journal-key"] + if (journalKey === undefined || !JOURNAL_KEY.test(journalKey)) + return { _tag: "refused", reason: "pre-start/invalid-spec", message: journalKey === undefined ? `annotation ${A}journal-key is missing` : `annotation ${A}journal-key is not :` } + const w = job.spec.with + const seat = seatOf(job) + if (seat === undefined) return { _tag: "refused", reason: "pre-start/runs-on-invalid", message: "runs-on does not name exactly one seat: label" } + const command = shape.command(job) + if (command === undefined || command.length === 0) return { _tag: "refused", reason: "pre-start/seat-command-missing", message: `no command for seat ${seat}` } + const env: Array<{ name: string; value: string }> = [ + { name: "AX_CONWIP_RUN_ID", value: ann[A + "run-id-raw"] ?? "" }, + { name: "AX_CONWIP_LABEL", value: ann[A + "label"] ?? "" }, + { name: "AX_CONWIP_MODEL", value: w.model }, + { name: "AX_CONWIP_ITEM_KEY", value: ann[A + "item-key"] ?? "" }, + { name: "AX_CONWIP_JOURNAL_KEY", value: journalKey }, + { name: "AX_CONWIP_WORKFLOW", value: lab[A + "workflow"] ?? "" }, + { name: "AX_CONWIP_PHASE_INDEX", value: lab[A + "phase-index"] ?? "" }, + { name: "AX_CONWIP_PHASE_TITLE", value: ann[A + "phase-title"] ?? "" }, + { name: "AX_CONWIP_SEAT", value: seat }, + { name: "AX_CONWIP_ATTEMPT", value: String(g.attempt) }, + { name: "AX_CONWIP_LEASE_ID", value: g.leaseId }, + ...(shape.holder !== undefined ? [{ name: "AX_CONWIP_HOLDER", value: shape.holder }] : []), + { name: "AX_CONWIP_PROMPT_SHA256", value: w.prompt_ref.sha256 }, + { name: "AX_CONWIP_RESULT_PATH", value: "/workspace/.ax/result.json" }, + { name: "AX_CONWIP_USAGE_PATH", value: "/workspace/.ax/usage.json" } + ] + if (w.effort !== undefined) env.push({ name: "AX_CONWIP_EFFORT", value: w.effort }) + if (w.schema !== undefined) env.push({ name: "AX_CONWIP_SCHEMA_JSON", value: JSON.stringify(w.schema) }) + if (w.isolation !== undefined) env.push({ name: "AX_CONWIP_ISOLATION", value: w.isolation }) + if (w["agent-type"] !== undefined) env.push({ name: "AX_CONWIP_AGENT_TYPE", value: w["agent-type"] }) + if (job.spec["timeout-minutes"] !== undefined) + env.push({ name: "AX_CONWIP_TIMEOUT_MINUTES", value: String(job.spec["timeout-minutes"]) }) + if (shape.completeUrl !== undefined && g.leaseToken !== undefined) { // L7 (a missing token was refused above): a per-lease token, never the fleet bearer + env.push({ name: "AX_CONWIP_COMPLETE_URL", value: shape.completeUrl }) + env.push({ name: "AX_CONWIP_LEASE_TOKEN", value: g.leaseToken }) + } + const used = env.reduce((n, e) => n + utf8Len(e.value), 0) + // B12 (critique R2): the file route (AX_CONWIP_PROMPT_FILE) needs a Workspace that writes the file, and the Task + // carries none yet. Until that route exists a prompt that does not fit inline is refused, never sent to a dead path. + if (used > ENV_BUDGET) // FIELD-MAP D4 and probe test 4: over budget ax falls back silently, so refuse here + return { _tag: "refused", reason: "pre-start/env-over-budget", message: `env values ${used} B > ${ENV_BUDGET} B before the prompt` } + if (w.prompt === undefined || used + utf8Len(w.prompt) > ENV_BUDGET) + return { _tag: "refused", reason: "pre-start/prompt-file-route-missing", + message: w.prompt === undefined ? "no inline prompt and no Workspace route" : `prompt ${utf8Len(w.prompt)} B does not fit the ${ENV_BUDGET - used} B left inline` } + env.push({ name: "AX_CONWIP_PROMPT", value: w.prompt }) + const total = env.reduce((n, e) => n + utf8Len(e.value), 0) + if (total > ENV_BUDGET) // FIELD-MAP D4 and probe test 4: over budget ax falls back silently, so refuse here + return { _tag: "refused", reason: "pre-start/env-over-budget", message: `env values ${total} B > ${ENV_BUDGET} B` } + return { + _tag: "ok", + task: { + apiVersion: "ax.io/v1alpha1", + kind: "Task", + metadata: { name: g.leaseId, atespace: shape.atespace }, + spec: { image: shape.image, command: [...command], env, gateway: { name: shape.gateway } } + } + } +} + +// The spec digest used by create-only adoption (L4): found with the same digest = adopt, another = conflict. +// Only the fields the link writes are hashed, in a fixed order, so ax defaults added on the server do not count. +export const specDigest = (spec: { image?: string; command?: ReadonlyArray; env?: ReadonlyArray<{ name?: string; value?: string }>; gateway?: { name?: string } | null }) => + createHash("sha256").update(JSON.stringify([ + spec.image ?? "", [...(spec.command ?? [])], + [...(spec.env ?? [])].map((e) => [e.name ?? "", e.value ?? ""]), + spec.gateway?.name ?? "" + ])).digest("hex") + +export const envBytes = (t: AxTask) => t.spec.env.reduce((n, e) => n + utf8Len(e.value), 0) diff --git a/pkgs/substrate-link/src/apps/link/src/journal.ts b/pkgs/substrate-link/src/apps/link/src/journal.ts new file mode 100644 index 000000000..8ee3e03bc --- /dev/null +++ b/pkgs/substrate-link/src/apps/link/src/journal.ts @@ -0,0 +1,187 @@ +// The link's crash-safe local state: one append-only JSONL file, fsynced per line, written BEFORE the action it +// records (the tally uplink's "offline is a queue" outbox, kept verbatim in spirit: done / keep / drop). The ax +// store is the ledger of Tasks (Buildkite: the cluster is the ledger); this file only covers what ax cannot know: +// grants not yet created, and Complete reports not yet acknowledged by the floor. +import { appendFileSync, closeSync, existsSync, fsyncSync, mkdirSync, openSync, readFileSync, renameSync, writeSync } from "node:fs" +import { join } from "node:path" +import type { Grant, Result, Usage } from "./contract.ts" + +export type Entry = + | { ev: "grant"; leaseId: string; grant: Grant; regrant?: true } // round 2: `regrant` replaces a released record + | { ev: "creating"; leaseId: string; digest?: string } // round 1: written BEFORE the first UpdateTask; the Task may exist from here on. Round 3: the spec digest the link will write, so a cleanup can tell its own Task from a foreign one + | { ev: "not-mine"; leaseId: string } // round 3: a Task of this name exists and is not the link's; `creating` no longer holds + | { ev: "created"; leaseId: string; digest: string } + | { ev: "grant-invalid"; leaseId: string; attempt: number; message: string } // round 1: a grant that did not decode + | { ev: "report"; leaseId: string; attempt: number; result: Result; output?: unknown; usage?: Usage; replace?: true } // outbox; round 2: `replace` swaps an undeliverable verdict + | { ev: "reported"; leaseId: string; duplicate: boolean; withdrew?: string; withdrewGen?: Gen } // done: 2xx from the floor; round 2: what rule 4b withdrew; round 3: which generation of it + | { ev: "dropped"; leaseId: string; code: string } // drop: the floor refused these bytes for good + | { ev: "released"; leaseId: string; why: string } // no longer held (lost, cancelled, superseded) + | { ev: "deleting"; leaseId: string; why: string } // DeleteTask accepted; waiting for NotFound (ax deletes in two phases) + | { ev: "deleted"; leaseId: string } + | { ev: "lease-key"; requestKey: string } // B8: a Lease is in flight with this key; reuse it until its reply lands + | { ev: "lease-replied"; requestKey: string; abandoned?: true } // the reply's grants are journaled; the next Lease draws a new key + | { ev: "tomb"; leaseId: string; gen: Gen } // round 3: written by compaction for a finished lease; refuses its redelivery + +/** Round 3: a grant generation. `acquireTime` is absent when only the floor's `withdrewTransitions` is known. */ +export interface Gen { readonly leaseTransitions: number; readonly acquireTime?: number } +export const genOf = (g: Grant): Gen => ({ leaseTransitions: g.lease.leaseTransitions, acquireTime: g.lease.acquireTime }) +/** `g` is a strictly later generation than `than` (leaseTransitions first, then acquireTime). */ +export const laterGen = (g: Gen, than: Gen) => g.leaseTransitions > than.leaseTransitions || + (g.leaseTransitions === than.leaseTransitions && g.acquireTime !== undefined && than.acquireTime !== undefined && g.acquireTime > than.acquireTime) +/** `g` is the generation `w` names: equal leaseTransitions, and equal acquireTime when both are known. */ +export const sameGen = (g: Gen, w: Gen) => g.leaseTransitions === w.leaseTransitions && + (g.acquireTime === undefined || w.acquireTime === undefined || g.acquireTime === w.acquireTime) + +export interface LeaseRec { + grant: Grant + created?: string // spec digest + creating?: boolean // an UpdateTask may have landed (write-ahead); cleanup must DeleteTask by name and see NotFound + creatingDigest?: string // round 3: the spec digest the link was about to write + notMine?: boolean // round 3: the Task under this name is someone else's; never read, delete or report it + createdAt?: number + report?: Extract + reported?: boolean + duplicate?: boolean // round 2: the floor answered this verdict as a duplicate (rule 4c) + withdrew?: string // round 2: the leaseId the floor withdrew when it accepted this verdict (rule 4b), never inferred + withdrewGen?: Gen // round 3: the generation withdrawn; undefined when this link never held it (it then matches nothing) + dropped?: string + released?: string + deleting?: string // why: cancel, lost, dropped, janitor, pre-start, deadline, superseded + deleted?: boolean + at: number // when this record was last touched (ms) +} + +export class Journal { + readonly path: string + readonly recs = new Map() + /** Grants that did not decode: refused to the floor as pre-start/invalid-spec through the same outbox. */ + readonly invalid = new Map() + pendingLeaseKey: string | undefined + /** Round 3: finished leases, kept through compaction for TOMB_MS so a redelivered grant of one is never run again. */ + readonly tombs = new Map() + static TOMB_MS = 24 * 3600_000 // well past the floor's replay window (rule 2b: lease + grace, 390 s in production) + private readonly dir: string + private constructor(dir: string) { this.dir = dir; this.path = join(dir, "journal.jsonl") } + + static open(dir: string, now = Date.now()): Journal { + mkdirSync(dir, { recursive: true, mode: 0o700 }) + const j = new Journal(dir) + if (existsSync(j.path)) { + for (const line of readFileSync(j.path, "utf8").split("\n")) { + if (line.trim() === "") continue + let e: Entry & { t?: number } + try { e = JSON.parse(line) } catch { continue } // a torn last line from a crash mid-write is skipped + j.apply(e, e.t ?? now) + } + j.compact() + } + return j + } + + private apply(e: Entry, t: number) { + if (e.ev === "lease-key") { this.pendingLeaseKey = e.requestKey; return } + if (e.ev === "lease-replied") { if (this.pendingLeaseKey === e.requestKey) this.pendingLeaseKey = undefined; return } + if (e.ev === "tomb") { this.tombs.set(e.leaseId, { gen: e.gen, at: t }); return } + if (e.ev === "grant") { + if (this.recs.has(e.leaseId) && !e.regrant) return + this.recs.set(e.leaseId, { grant: e.grant, at: t }) + // round 2: the withdrawal recorded against the previous generation of this leaseId does not apply to the new one + if (e.regrant) for (const x of this.recs.values()) if (x.withdrew === e.leaseId) { delete x.withdrew; delete x.withdrewGen } + return + } + if (e.ev === "grant-invalid") { if (!this.recs.has(e.leaseId) && !this.invalid.has(e.leaseId)) this.invalid.set(e.leaseId, { attempt: e.attempt, message: e.message, at: t }); return } + const r = this.recs.get(e.leaseId) + if (!r) { + const x = this.invalid.get(e.leaseId) + if (x && e.ev === "reported") x.reported = true + if (x && e.ev === "dropped") x.dropped = e.code + if (x && e.ev === "released") x.released ??= e.why + return + } + r.at = t + switch (e.ev) { + case "creating": r.creating = true; delete r.notMine; if (e.digest !== undefined) r.creatingDigest = e.digest; break + case "not-mine": r.notMine = true; delete r.creating; break + case "created": r.created = e.digest; r.createdAt ??= t; break + case "report": if (!r.report || e.replace) r.report = e; break // first verdict wins (the floor dedupes as well), unless replaced + case "reported": r.reported = true; r.duplicate = e.duplicate; if (e.withdrew !== undefined) { r.withdrew = e.withdrew; if (e.withdrewGen !== undefined) r.withdrewGen = e.withdrewGen } break + case "dropped": r.dropped = e.code; break + case "released": r.released ??= e.why; break + case "deleting": r.deleting ??= e.why; break + case "deleted": r.deleted = true; break + } + } + + append(e: Entry, t = Date.now()) { + appendFileSync(this.path, JSON.stringify({ t, ...e }) + "\n", { mode: 0o600 }) + const fd = openSync(this.path, "r") + try { fsyncSync(fd) } finally { closeSync(fd) } + this.apply(e, t) + if (++this.appends >= Journal.COMPACT_EVERY) this.compact() // idle Lease keys must not grow the file for ever + } + static COMPACT_EVERY = 2000 + private appends = 0 + + /** Finished = nothing left to tell the floor and nothing left in ax. */ + static finished(r: LeaseRec) { + // Round 2: a verdict read after the lease was released (lost, superseded) is still owed to the floor + if (Journal.pending(r)) return false + const toldFloor = r.reported === true || r.dropped !== undefined || r.released !== undefined + const axClean = !Journal.maybeCreated(r) || r.deleted === true + return toldFloor && axClean + } + /** Round 1: a Task may exist in ax once `creating` is journaled, even if `created` never was (a crash, or a + * DEADLINE_EXCEEDED after the write landed). Every cleanup path treats such a record as created. */ + static maybeCreated(r: LeaseRec) { return r.created !== undefined || r.creating === true } + /** A verdict is journaled and neither acknowledged nor dropped. */ + static pending(r: LeaseRec) { return r.report !== undefined && !r.reported && r.dropped === undefined } + + /** Rewrite the file with only unfinished leases (atomic rename), so it never grows without bound. */ + compact() { + const tmp = this.path + ".tmp" + const fd = openSync(tmp, "w", 0o600) + try { + if (this.pendingLeaseKey !== undefined) writeSync(fd, JSON.stringify({ t: Date.now(), ev: "lease-key", requestKey: this.pendingLeaseKey }) + "\n") + for (const [id, r] of this.recs) { + if (Journal.finished(r)) { this.recs.delete(id); this.tombs.set(id, { gen: genOf(r.grant), at: r.at }); continue } + const lines: Array = [{ ev: "grant", leaseId: id, grant: r.grant }] + if (r.creating) lines.push({ ev: "creating", leaseId: id, ...(r.creatingDigest !== undefined ? { digest: r.creatingDigest } : {}) }) + if (r.notMine) lines.push({ ev: "not-mine", leaseId: id }) + if (r.created !== undefined) lines.push({ ev: "created", leaseId: id, digest: r.created }) + if (r.report) lines.push(r.report) + if (r.reported) lines.push({ ev: "reported", leaseId: id, duplicate: r.duplicate ?? false, ...(r.withdrew !== undefined ? { withdrew: r.withdrew } : {}), ...(r.withdrewGen !== undefined ? { withdrewGen: r.withdrewGen } : {}) }) + if (r.dropped !== undefined) lines.push({ ev: "dropped", leaseId: id, code: r.dropped }) + if (r.released !== undefined) lines.push({ ev: "released", leaseId: id, why: r.released }) + if (r.deleting !== undefined) lines.push({ ev: "deleting", leaseId: id, why: r.deleting }) + for (const l of lines) writeSync(fd, JSON.stringify({ t: l.ev === "created" || l.ev === "grant" ? (r.createdAt ?? r.at) : r.at, ...l }) + "\n") + } + const now = Date.now() + for (const [id, x] of this.tombs) { + if (now - x.at > Journal.TOMB_MS) { this.tombs.delete(id); continue } + writeSync(fd, JSON.stringify({ t: x.at, ev: "tomb", leaseId: id, gen: x.gen }) + "\n") + } + for (const [id, x] of this.invalid) { // never created, so finished once the floor is told + if (x.reported || x.dropped !== undefined || x.released !== undefined) { this.invalid.delete(id); continue } + writeSync(fd, JSON.stringify({ t: x.at, ev: "grant-invalid", leaseId: id, attempt: x.attempt, message: x.message }) + "\n") + } + fsyncSync(fd) + } finally { closeSync(fd) } + renameSync(tmp, this.path) + this.appends = 0 + const d = openSync(this.dir, "r") // B20: make the rename itself durable + try { fsyncSync(d) } finally { closeSync(d) } + } + + /** The complete set this holder believes it holds: sent in every Heartbeat (level-triggered). A lease whose + * verdict is still in the outbox stays held, so a heartbeat that races the outbox drain cannot orphan it. */ + held(): Array { + return [...[...this.recs].filter(([, r]) => !r.reported && r.dropped === undefined && r.released === undefined).map(([id]) => id), + ...[...this.invalid].filter(([, x]) => !x.reported && x.dropped === undefined && x.released === undefined).map(([id]) => id)] + } + /** Leases whose verdict is written but not yet acknowledged; they are still held until the floor says done. */ + outbox(): Array> { + return [...[...this.recs.values()].filter(Journal.pending).map((r) => r.report!), + ...[...this.invalid].filter(([, x]) => !x.reported && x.dropped === undefined && x.released === undefined) + .map(([leaseId, x]) => ({ ev: "report" as const, leaseId, attempt: x.attempt, result: "failure" as const, output: { reason: "pre-start/invalid-spec", message: x.message } }))] + } +} diff --git a/pkgs/substrate-link/src/apps/link/src/link.ts b/pkgs/substrate-link/src/apps/link/src/link.ts new file mode 100644 index 000000000..d8de5b991 --- /dev/null +++ b/pkgs/substrate-link/src/apps/link/src/link.ts @@ -0,0 +1,921 @@ +// The NAS link (LINK-DESIGN.md section 5): one outbound-only process between the Cloudflare floor and ax. +// Lease (free capacity on every poll, ARC; the outbox drains first, B15; the requestKey is journaled, B8) +// -> journal the grant (write-ahead) -> fence every superseded attempt: DeleteTask and wait for NotFound (L3, B2) +// -> wait for a sandbox slot (B11) -> GetTask, UpdateTask only on NotFound (L4, create-only), read it back (B12) +// -> resync by paging ListTasks (Buildkite: the executor's list is the ledger; B1) and confirming every absence with +// GetTask -> read the outcome (P1: terminal phase + GetTaskResult, retried and digest-checked, B4; guest mode only +// when configured, L7, B6) -> Complete through the journal's outbox (L6) -> DeleteTask after the floor acknowledged. +// One batched Heartbeat per session carries the COMPLETE held set; its reply carries cancel and lost (Temporal). +// Nothing is leased while ax is unreachable, the completion mode is unproven (B6) or the egress Gateway is missing or +// open (B10). +import { createHash, randomUUID } from "node:crypto" +import { isIPv4, isIPv6 } from "node:net" +import { Cause, Deferred, Effect, Schedule, Semaphore } from "effect" +import { gatewayAllowsAll, listAll } from "./ax.ts" +import type { AxApi, AxError, AxObserved } from "./ax.ts" +import type { FailureOutput, Grant, Result, Usage } from "./contract.ts" +import type { FloorApi, FloorError, LeaseReply } from "./floor.ts" +import { axTaskFromGrant, specDigest, validRunsOn } from "./jobs.ts" +import type { TaskShape } from "./jobs.ts" +import { genOf, Journal, laterGen, sameGen } from "./journal.ts" +import type { Gen, LeaseRec } from "./journal.ts" + +export interface LinkConfig { + readonly holder: string // holderIdentity; bound to this link's token on the floor + readonly maxInFlight: number // sandbox cap: the WorkerPool replicas this link may fill + readonly servedLabels: ReadonlyArray // defence in depth: the floor already routes by holder + readonly shape: TaskShape + /** `p1` and `auto` both require P1 on the server, proven by a probe (B6); `auto` never selects guest. `guest` is + * the L7 reserve and needs a fleet-internal `shape.completeUrl` (critique A5). */ + readonly completion: "auto" | "p1" | "guest" + readonly secondMs: number // 1000 in production; tests shrink every server-set interval with it + readonly resyncMs: number // 15 s, the P1 --running-resync period + readonly pendingTimeoutMs: number // 15 min, agent-stack-k8s: a Pod stuck Pending fails the job + readonly deleteAfterMs: number // 10 min, agent-stack-k8s TTLSecondsAfterFinished + readonly deadlineBackstopMs: number // local stop past the floor's deadline, if the floor cannot say cancel + readonly createAttempts: number // UpdateTask tries while ax is unavailable before pre-start failure + readonly outboxBackoffMs: readonly [number, number] // uplink shut-door backoff: min, max + readonly fenceTimeoutMs?: number // B2: how long a superseded attempt may take to reach NotFound (default 120 s) + readonly resultReadTries?: number // B4: resyncs a Completed Task's result may fail to read (default 5) + readonly floorUrls?: ReadonlyArray // B18: the only URLs a Lease `endpoint` may move this link to + readonly initialPollSeconds?: number // until the first Lease reply sets them (defaults 15 and 30, section 3.7) + readonly initialHeartbeatSeconds?: number + /** Round 2: a result larger than this is refused `agent/output-too-large` before it is reported (the floor's DO row + * limit is 2 MB, MEASURED as SQLITE_TOOBIG at 2.2 MB; default 1 000 000 B). */ + readonly maxOutputBytes?: number + /** Round 2: server errors one verdict may collect before it is replaced by `infra/verdict-undeliverable`, and that + * replacement before it is dead-lettered (journaled `dropped`). Default 8. */ + readonly verdictAttempts?: number + /** Round 2: single-label hosts a guest Complete URL may name (fleet DNS names without a dot). Default none. */ + readonly internalHosts?: ReadonlyArray + /** Round 3: consecutive undecodable or server-error Lease replies for one journaled requestKey before the key is + * abandoned (its grants are then released by the floor's own expiry and requeued). Default 5. */ + readonly leaseKeyAttempts?: number +} + +export type Log = (ev: string, fields?: Record) => void +export interface LinkDeps { + readonly ax: AxApi; readonly floor: FloorApi; readonly journal: Journal; readonly log: Log; readonly stop: Deferred.Deferred + readonly floorUrl?: string // the URL in use, to tell a real endpoint switch from an echo + readonly onEndpoint?: (url: string) => void // B18: called once for an allowed switch; main persists it and restarts +} + +const TERMINAL = new Set(["Completed", "Failed"]) +const jitter = (ms: number) => Math.round(ms * (0.8 + 0.4 * Math.random())) +const NOT_FOUND = 5 // gRPC status: DeleteTask of a name that is already gone +const MAX_POLL_SECONDS = 300, MAX_HEARTBEAT_SECONDS = 300 +/** Round 3: a server-driven interval (Buildkite: the agent falls back to its defaults). Missing, non-finite or not + * positive gives `def`; anything else is clamped to [1, hi]. `Effect.sleep(0)` would otherwise be a tight loop. */ +export const boundedSeconds = (v: number | undefined, def: number, hi: number) => + v === undefined || !Number.isFinite(v) || v <= 0 ? Math.min(def, hi) : Math.max(1, Math.min(v, hi)) + +/** Critique A5 and N3: a guest's Complete URL must stay inside the fleet, never a public Workers host. + * Round 2: private ranges are matched only on IP literals (after WHATWG normalisation, so `10.evil.example` and + * `127.0.0.1.nip.io` are names, not addresses); a name must end in a non-public suffix; a single-label name only when + * it is listed. */ +export const isFleetInternalUrl = (raw: string | undefined, internalHosts: ReadonlyArray = []): boolean => { + if (raw === undefined) return false + let u: URL + try { u = new URL(raw) } catch { return false } + let h = u.hostname.toLowerCase() + if (h.endsWith(".workers.dev") || h.endsWith(".pages.dev")) return false + if (h.startsWith("[") && h.endsWith("]")) h = h.slice(1, -1) + if (isIPv4(h)) { + const [a, b] = h.split(".").map(Number) as [number, number] + return a === 127 || a === 10 || (a === 192 && b === 168) || (a === 172 && b >= 16 && b <= 31) || (a === 100 && b >= 64 && b <= 127) + } + if (isIPv6(h)) return h === "::1" || /^f[cd][0-9a-f]{2}:/.test(h) // loopback, ULA fc00::/7 (the tailnet's fd7a:115c:a1e0::/48) + if (h === "localhost" || internalHosts.map((x) => x.toLowerCase()).includes(h)) return true + return /^([a-z0-9-]+\.)+(internal|lan|local|home\.arpa)$/.test(h) +} + +export const runLink = (cfg: LinkConfig, deps: LinkDeps) => Effect.gen(function*() { + const { ax, floor, journal: j, log } = deps + const fatal = yield* Deferred.make() + const fenceTimeoutMs = cfg.fenceTimeoutMs ?? 120_000 + const resultReadTries = cfg.resultReadTries ?? 5 + const maxOutputBytes = cfg.maxOutputBytes ?? 1_000_000 + const verdictAttempts = cfg.verdictAttempts ?? 8 + const defaultPoll = cfg.initialPollSeconds ?? 15, defaultHb = cfg.initialHeartbeatSeconds ?? 30 + let pollSeconds = defaultPoll, hbAsked: number | undefined // round 3: the floor's raw heartbeatSeconds, bounded at use + let clampLogged = "" + const leaseKeyAttempts = cfg.leaseKeyAttempts ?? 5 + let leaseKeyFailures = 0 + let axUp = false, resynced = false + let serverP1: boolean | undefined // the probe's answer; cleared on every ax-up transition (B6) + let gatewayOk: boolean | undefined // B10 + const guestUrlOk = cfg.completion !== "guest" || isFleetInternalUrl(cfg.shape.completeUrl, cfg.internalHosts) + if (!guestUrlOk) log("guest-url-invalid", { completion: "guest" }) + let last = new Map() + let outboxWaitUntil = 0, outboxBackoff = cfg.outboxBackoffMs[0] + let outboxNotBefore = 0 // round 3: the floor's own Retry-After on Complete; Lease does not cut it short (B15 does not) + const dispatching = new Map() // round 2: keyed by leaseId, valued by the grant generation it runs + const admitted = new Set() // B11: grants that passed the slot gate in this process + const abandoned = new Map() // B3: set by the Heartbeat reply, read by dispatch + const absent = new Map() // B1: consecutive resyncs a held Task answered NotFound + const resultFailures = new Map() // B4 + const verdictFailures = new Map() // round 2: server errors per verdict (a restart grants a fresh budget) + const verdictNotBefore = new Map() // round 2: per-verdict backoff after a server error + const pendingResume = new Set() // B13: journaled, never created; resumed only once the floor renews them + /** Round 4 (Buildkite reserve-with-expiry): the local time of the last call the floor answered by renewing this + * grant generation (the Lease that granted it, then every Heartbeat listing it in `renewed`). The send time is taken, + * so it is never later than the floor's own renewTime. Keyed by the grant object: a regrant starts afresh. */ + const renewedAt = new WeakMap() + /** Round 4: a create may start only while the floor still holds the lease: the last renewal is less than 3/4 of a + * lease old. A grant never renewed in this process (journaled by a previous one) is not live until a Heartbeat says so. */ + const leaseLive = (g: Grant) => { + const at = renewedAt.get(g) + if (at === undefined) return false + const ms = g.lease.leaseDurationSeconds * cfg.secondMs + return Date.now() < at + ms - ms / 4 + } + const deadlinePassed = (g: Grant) => g.deadline !== undefined && Date.now() >= g.deadline + let wake = yield* Deferred.make() // B17: a verdict that frees a slot leases at once + let hbWake = yield* Deferred.make() // final pass (R3-1): a Lease reply re-times a heartbeat sleep already begun + const failFatal = (e: FloorError) => Deferred.fail(fatal, e).pipe(Effect.asVoid) + + /** The mode the link may create in, or undefined while it is unproven (capacity 0). */ + const mode = (): "p1" | "guest" | undefined => { + if (cfg.completion === "guest") return guestUrlOk ? "guest" : undefined + return serverP1 === true ? "p1" : undefined + } + const canCreate = () => axUp && resynced && gatewayOk === true && mode() !== undefined + + // ---------------------------------------------------------------- verdicts (write-ahead, then the outbox drains) + /** `salvage` (round 2): the verdict was read from a Task whose lease is already released (lost, superseded). It is + * still owed to the floor, which accepts it by rule 4b or answers stale-attempt (then it is dropped). */ + /** Round 4: `g`, when given, is the grant generation the verdict is about; a record that holds another generation + * by now (a regrant of the same leaseId) is not given it. */ + const report = (id: string, result: Result, output?: unknown, usage?: Usage, salvage = false, g?: Grant) => Effect.gen(function*() { + const r = j.recs.get(id) + if (g !== undefined && r?.grant !== g) return log("stale-generation-verdict", { leaseId: id, result }) + if (!r || r.report || (r.released !== undefined && !salvage)) return + j.append({ ev: "report", leaseId: id, attempt: r.grant.attempt, result, ...(output !== undefined ? { output } : {}), ...(usage !== undefined ? { usage } : {}) }) + log("verdict", { leaseId: id, result, ...(result === "failure" ? { reason: (output as FailureOutput | undefined)?.reason } : {}), ...(salvage ? { salvaged: r.released } : {}) }) + yield* Deferred.succeed(wake, undefined) + }) + const fail = (id: string, o: FailureOutput) => report(id, "failure", o) + const failG = (g: Grant, o: FailureOutput) => report(g.leaseId, "failure", o, undefined, false, g) // round 4 + + // Round 1: single-flight. The main loop, the resync fiber and a dispatch waiting on a superseded verdict all drain; + // two concurrent drains read the same outbox before either journals `reported` and send every verdict twice. + const outboxLock = Semaphore.makeUnsafe(1) + const drainOutbox = outboxLock.withPermit(Effect.gen(function*() { + if (Date.now() < outboxWaitUntil || Date.now() < outboxNotBefore) return + for (const p of j.outbox()) { + if ((verdictNotBefore.get(p.leaseId) ?? 0) > Date.now()) continue + // Round 4: the generation of attempt n+1 this link held when the verdict was SENT. A regrant of n+1 taken while + // the reply is in flight is later than anything this Complete can have withdrawn. + const succId = `${p.leaseId.replace(/-a\d+$/, "")}-a${p.attempt + 1}` + const succAtSend = j.recs.get(succId)?.grant + const r = yield* floor.complete({ leaseId: p.leaseId, attempt: p.attempt, result: p.result, ...(p.output !== undefined ? { output: p.output } : {}), ...(p.usage !== undefined ? { usage: p.usage } : {}) }).pipe(Effect.result) + if (r._tag === "Success") { + const withdrew = r.success.withdrew + // Round 3: a withdrawal names a GENERATION, not a leaseId. The floor may requeue the withdrawn attempt under the + // same leaseId (a retryable verdict of n), and that later grant is live. The generation is the floor's + // `withdrewTransitions` when it sends one, else the one this link holds; a generation this link never received + // (n+1 was still queued) matches no later grant. + const w = withdrew !== undefined ? j.recs.get(withdrew) : undefined + const withdrewGen: Gen | undefined = withdrew === undefined ? undefined + : r.success.withdrewTransitions !== undefined ? { leaseTransitions: r.success.withdrewTransitions } + : succAtSend !== undefined && withdrew === succId ? genOf(succAtSend) + : w !== undefined ? genOf(w.grant) : undefined + j.append({ ev: "reported", leaseId: p.leaseId, duplicate: r.success.duplicate, ...(withdrew !== undefined ? { withdrew } : {}), ...(withdrewGen !== undefined ? { withdrewGen } : {}) }) + log("complete", { leaseId: p.leaseId, result: p.result, duplicate: r.success.duplicate, ...(withdrew !== undefined ? { withdrew } : {}) }) + if (withdrew !== undefined && withdrewGen === undefined) log("withdrawn-unheld", { leaseId: withdrew, by: p.leaseId }) + if (w !== undefined && withdrewGen !== undefined && sameGen(genOf(w.grant), withdrewGen) && w.released === undefined && !Journal.maybeCreated(w)) { + j.append({ ev: "released", leaseId: withdrew!, why: "withdrawn" }) // round 2: released at once, so a regrant + log("withdrawn", { leaseId: withdrew, by: p.leaseId }) // of it is not taken for a duplicate + } + outboxBackoff = cfg.outboxBackoffMs[0] + verdictFailures.delete(p.leaseId); verdictNotBefore.delete(p.leaseId) + } else if (r.failure._tag === "CompleteRefused") { // done/keep/drop: the floor refused these bytes for good + j.append({ ev: "dropped", leaseId: p.leaseId, code: r.failure.code }) + log("complete-dropped", { leaseId: p.leaseId, code: r.failure.code }) + } else if (r.failure.kind === "server-error") { + // Round 2: the floor answered and refused THESE bytes (a 500, an RPC Defect such as SQLITE_TOOBIG). Counted per + // verdict; the rest of the outbox still goes out. At the ceiling the verdict is replaced by a small final + // failure the floor can store; if that fails as well it is dead-lettered, so one verdict never gates Lease. + const n = (verdictFailures.get(p.leaseId) ?? 0) + 1 + verdictFailures.set(p.leaseId, n) + if (n < verdictAttempts) { + const wait = jitter(Math.min(cfg.outboxBackoffMs[0] * 2 ** (n - 1), cfg.outboxBackoffMs[1])) + verdictNotBefore.set(p.leaseId, Date.now() + wait) + log("outbox-keep", { leaseId: p.leaseId, kind: r.failure.kind, try: n, waitMs: wait }) + continue + } + verdictFailures.delete(p.leaseId); verdictNotBefore.delete(p.leaseId) + const rec = j.recs.get(p.leaseId) + if (rec !== undefined && !p.replace) { + const digest = createHash("sha256").update(JSON.stringify(p.output ?? null)).digest("hex") + j.append({ ev: "report", leaseId: p.leaseId, attempt: p.attempt, result: "failure", replace: true, + output: { reason: "infra/verdict-undeliverable", message: `${p.result} verdict (output sha256 ${digest}) refused ${n} times: ${r.failure.message.slice(0, 200)}` } }) + log("verdict-replaced", { leaseId: p.leaseId, result: p.result, tries: n, outputSha256: digest }) + } else { + j.append({ ev: "dropped", leaseId: p.leaseId, code: "undeliverable" }) + log("verdict-dead-letter", { leaseId: p.leaseId, tries: n, message: r.failure.message.slice(0, 200) }) + } + } else { + if (r.failure.fatal) return yield* failFatal(r.failure) + outboxWaitUntil = Date.now() + Math.min(r.failure.retryAfterMs ?? jitter(outboxBackoff), MAX_POLL_SECONDS * cfg.secondMs) + if (r.failure.retryAfterMs !== undefined) outboxNotBefore = outboxWaitUntil + outboxBackoff = Math.min(outboxBackoff * 2, cfg.outboxBackoffMs[1]) + log("outbox-keep", { leaseId: p.leaseId, kind: r.failure.kind, waitMs: outboxWaitUntil - Date.now() }) + return // keep: oldest first, stop at the first transient failure + } + } + })) + + // ---------------------------------------------------------------- ax calls + // Round 3 (token scope): a record that is only `creating` names a Task the link MAY have written. The floor chose + // that name, so before anything reads or deletes it the Task must carry this link's write: the spec digest the + // link built for this grant, or this lease id and this holder in its env. Anything else is someone else's Task. + const envMarks = (id: string, t: AxObserved) => { + const env = new Map((t.spec.env ?? []).map((e) => [e.name ?? "", e.value ?? ""])) + return env.get("AX_CONWIP_LEASE_ID") === id && env.get("AX_CONWIP_HOLDER") === cfg.holder + } + const owns = (id: string, r: LeaseRec, t: AxObserved) => + r.created !== undefined || (r.creatingDigest !== undefined && specDigest(t.spec) === r.creatingDigest) || envMarks(id, t) + /** Journal that the Task under this name is not the link's: nothing reads, deletes or reports it any more. A cancel + * already asked for is answered `cancelled`, since nothing of this lease runs in ax. */ + const markNotMine = (id: string) => Effect.gen(function*() { + const r = j.recs.get(id) + if (!r || r.notMine) return + j.append({ ev: "not-mine", leaseId: id }) + log("not-mine", { leaseId: id }) + if (!r.report && r.released === undefined && (abandoned.get(id) === "cancel" || r.deleting === "cancel")) yield* report(id, "cancelled") + }) + /** GetTask a `creating`-only record's name: owned, absent, foreign (journaled not-mine) or unknown (ax did not answer). */ + const ownership = (id: string, r: LeaseRec) => Effect.gen(function*() { + const got = yield* ax.getTask(id).pipe(Effect.result) + if (got._tag === "Failure") return "unknown" as const + if (got.success === undefined) return "absent" as const + if (owns(id, r, got.success)) return "owned" as const + yield* markNotMine(id) + return "foreign" as const + }) + + // Round 3: the intent is journaled BEFORE DeleteTask (`deleting`), so a DeleteTask that lands and whose reply is lost + // (DEADLINE_EXCEEDED) still reads as a delete on the next resync (a cancel is reported `cancelled`, not task-lost), + // and one that did not land is sent again by reconcile. NotFound means already gone. + const del = (id: string, why: string): Effect.Effect => Effect.gen(function*() { + const rec = j.recs.get(id) + if (rec?.notMine) return true + if (rec !== undefined && rec.created === undefined && rec.creating) { + const o = yield* ownership(id, rec) + if (o === "unknown") { log("delete-deferred", { task: id, why, until: "ownership" }); return false } + if (o === "foreign") return true + } + if (rec !== undefined && rec.deleting === undefined) j.append({ ev: "deleting", leaseId: id, why }) + return yield* ax.deleteTask(id).pipe( + Effect.tap(() => Effect.sync(() => log("delete", { task: id, why }))), + Effect.as(true), + Effect.catch((e: AxError) => Effect.sync(() => { + if (e.code === NOT_FOUND) { log("delete", { task: id, why, absent: true }); return true } + log("delete-deferred", { task: id, why, code: e.code }); return false + }))) + }) + + /** B2: an old attempt is gone only when GetTask answers NotFound; until then attempt n+1 is not created. */ + const fence = (old: string) => Effect.gen(function*() { + const until = Date.now() + fenceTimeoutMs + let asked = false + for (;;) { + if (j.recs.get(old)?.notMine) return true // round 3: not this link's Task; nothing of attempt n runs under it + const r = yield* ax.getTask(old).pipe(Effect.result) + if (r._tag === "Success" && r.success === undefined) { + if (j.recs.has(old) && !j.recs.get(old)!.deleted) j.append({ ev: "deleted", leaseId: old }) + return true + } + if (r._tag === "Success" && (!asked || r.success!.phase !== "Terminating")) asked = (yield* del(old, "superseded")) || asked + if (Date.now() >= until) return false + yield* Effect.sleep(Math.min(cfg.resyncMs, 2 * cfg.secondMs)) + } + }) + + /** B11: never more live Tasks than maxInFlight. Busy = live Tasks in the atespace (anyone's) U admitted leases + * not known finished. Synchronous, so two dispatch fibers cannot both take the last slot. */ + const busy = () => { + const b = new Set() + for (const [name, t] of last) if (!TERMINAL.has(t.phase)) b.add(name) + for (const id of admitted) { + const r = j.recs.get(id), t = last.get(id) + if (!r || r.report || r.released !== undefined || r.dropped !== undefined || r.deleted || (t && TERMINAL.has(t.phase))) { admitted.delete(id); b.delete(id); continue } + b.add(id) + } + return b + } + const admit = (id: string) => Effect.sync(() => { + if (!canCreate()) return false + const b = busy() + if (b.has(id) || b.size < cfg.maxInFlight) { admitted.add(id); return true } + return false + }) + + /** A grant this link will not create any more (B3): a cancel is reported once any Task that may exist is gone. */ + const abandon = (id: string, why: "cancel" | "lost", where: string) => Effect.gen(function*() { + log(where, { leaseId: id, why }) + const rec = j.recs.get(id) + if (rec && Journal.maybeCreated(rec)) return yield* del(id, why) // round 1: reconcile reports cancelled on NotFound + if (why === "cancel") yield* report(id, "cancelled") + }) + + let guestTokenWarned = false + const createOnly = (g: Grant) => Effect.gen(function*() { + const id = g.leaseId + let waited = false + let unconfirmed = false + for (;;) { // round 1: the capacity gate is re-checked before every UpdateTask; a closed gate goes back to waiting + // Round 4: and the lease must still be live (renewed within the lease) before the slot is taken + while (!leaseLive(g) || !(yield* admit(id))) { // B11: journaled and heartbeated while it waits + if (abandoned.has(id) || deadlinePassed(g)) break + if (!leaseLive(g)) { if (!unconfirmed) { log("lease-unconfirmed", { leaseId: id }); unconfirmed = true } } + else if (!waited) { log("waiting-for-slot", { leaseId: id }); waited = true } + yield* Effect.sleep(cfg.resyncMs) + } + const early = abandoned.get(id) + if (early !== undefined) return yield* abandon(id, early, "abandoned-before-create") + const now = j.recs.get(id) // round 2: withdrawn, or replaced by a newer grant of the same leaseId: never create + if (now?.grant !== g || now.released !== undefined) return log("grant-superseded-before-create", { leaseId: id }) + if (deadlinePassed(g)) return yield* failG(g, { reason: "deadline-exceeded", message: "the grant's deadline passed before its Task was created" }) // round 4 + const m = mode() + const built = axTaskFromGrant(g, { ...cfg.shape, holder: cfg.holder, ...(m === "guest" ? {} : { completeUrl: undefined }) }) + if (built._tag === "refused") { + if (built.reason === "pre-start/lease-token-missing" && !guestTokenWarned) { + guestTokenWarned = true + log("guest-token-missing", { leaseId: id, hint: "floor and link disagree: the link completes as guest, the floor mints no lease token for this holder" }) + } + return yield* failG(g, { reason: built.reason, message: built.message }) + } + const digest = specDigest(built.task.spec) + for (let i = 1; ; i++) { + const why = abandoned.get(id) // B3: checked before every UpdateTask + if (why !== undefined) return yield* abandon(id, why, "abandoned-before-create") + const cur = j.recs.get(id) + if (cur?.grant !== g || cur.released !== undefined) return log("grant-superseded-before-create", { leaseId: id }) + if (!canCreate() || mode() !== m) { log("gate-closed-before-create", { leaseId: id, try: i }); break } // round 1 + if (deadlinePassed(g)) return yield* failG(g, { reason: "deadline-exceeded", message: "the grant's deadline passed before its Task was created" }) + if (!leaseLive(g)) { log("lease-unconfirmed-before-create", { leaseId: id, try: i }); break } // round 4: back to waiting + if (!j.recs.get(id)?.creating) j.append({ ev: "creating", leaseId: id, digest }) // round 1: write-ahead; round 3: with the digest + const r = yield* Effect.gen(function*() { + const found = yield* ax.getTask(id) + if (found === undefined) { yield* ax.createTask(built.task); return "created" as const } + if (specDigest(found.spec) === digest) return "adopted" as const + return envMarks(id, found) ? "conflict" as const : "foreign" as const + }).pipe(Effect.result) + if (r._tag === "Success") { + // L4: never overwrite a Task this link did not write. Round 3: a Task without this link's marks is journaled + // not-mine first, so no cleanup path (janitor, lost, cancel, salvage) ever reads or deletes it. + if (r.success === "foreign") yield* markNotMine(id) + if (r.success === "conflict" || r.success === "foreign") + return yield* failG(g, { reason: "pre-start/name-conflict", message: "a Task with this name and another spec exists" }) + j.append({ ev: "created", leaseId: id, digest }) + log(r.success, { leaseId: id, attempt: g.attempt }) + const after = abandoned.get(id) // B3: and again once UpdateTask returned + if (after !== undefined) { yield* del(id, after); return } + const back = yield* ax.getTask(id).pipe(Effect.result) // B12: FIELD-MAP 5a read-back + if (back._tag === "Success" && back.success !== undefined && specDigest(back.success.spec) !== digest) { + yield* del(id, "pre-start") + return yield* failG(g, { reason: "pre-start/readback-mismatch", message: "ax stored another spec than the link wrote" }) + } + return + } + const e = r.failure + if (e.resourceExhausted) return yield* failG(g, { reason: "pre-start/resource-exhausted", message: e.message }) // B5 + if (!e.unavailable) return yield* failG(g, { reason: "pre-start/invalid-spec", message: e.message }) + if (i >= cfg.createAttempts) return yield* failG(g, { reason: "pre-start/ax-unavailable", message: e.message }) + yield* Effect.sleep(jitter(2 ** i * 100)) + } + } + }) + + /** Round 1: an unjournaled name may be fenced only if it is `-a` with k < this attempt and the Task's own + * env says this lease id and this holder (AX_CONWIP_HOLDER, written by this link at create). */ + const ownEarlierAttempt = (old: string, g: Grant, t: AxObserved) => { + const base = g.leaseId.replace(/-a\d+$/, ""), m = /^(.*)-a(\d+)$/.exec(old) + if (m === null || m[1] !== base || Number(m[2]) >= g.attempt) return false + return envMarks(old, t) + } + + const dispatch = (g: Grant) => Effect.gen(function*() { + if (j.recs.get(g.leaseId)?.grant === g) dispatching.set(g.leaseId, g) // round 2: only the current generation + const bad = validRunsOn(g.job, cfg.servedLabels) // B7 + if (bad) return yield* failG(g, bad) + // Final pass (VERIFY-LINK V1): the fence set is `supersedes` plus every earlier attempt of the same job this link + // journaled, that may exist in ax and is not confirmed deleted. A floor that omits `supersedes` (or a Heartbeat + // whose `lost` arrives after this grant) no longer lets attempt n+1 start beside a running attempt n. + const olds = [...new Set([...(g.supersedes ?? []), ...[...j.recs.values()] + .filter((r) => r.grant.leaseId !== g.leaseId && r.grant.job.metadata.name === g.job.metadata.name && r.grant.attempt < g.attempt + && Journal.maybeCreated(r) && r.deleted !== true && !r.notMine) + .map((r) => r.grant.leaseId)])] + if (olds.length > (g.supersedes ?? []).length) log("fence-set-extended", { leaseId: g.leaseId, supersedes: g.supersedes ?? [], fence: olds }) + // Round 2: an earlier attempt that may exist and has no verdict yet is READ before anything fences it. A Task that + // finished while its lease expired carries a result the floor can still accept (rule 4b); deleting it first loses + // the success and runs the job again. + for (const o of olds) { + const until = Date.now() + fenceTimeoutMs + for (;;) { + const s = yield* salvage(o) + if (s !== "unknown") break + const why = abandoned.get(g.leaseId) + if (why !== undefined) return yield* abandon(g.leaseId, why, "abandoned-before-create") + if (Date.now() >= until) + return yield* failG(g, { reason: "pre-start/ax-unavailable", message: `superseded attempt ${o} could not be read before its fence` }) + yield* Effect.sleep(Math.min(cfg.resyncMs, 2 * cfg.secondMs)) + } + } + // Round 1 (B15 at dispatch): an earlier attempt whose verdict is still in the outbox is delivered BEFORE it is + // fenced. The floor accepts it (rule 4b: n+1 is queued, or leased to this holder and not created) and withdraws + // this grant; deleting the finished Task first would throw its result away and rerun the job. + const undelivered = () => olds.filter((o) => { const r = j.recs.get(o); return r?.report !== undefined && !r.reported && r.dropped === undefined }) + if (undelivered().length > 0) { + log("supersede-waits-for-verdict", { leaseId: g.leaseId, old: undelivered() }) + const until = Date.now() + fenceTimeoutMs + for (;;) { + if (undelivered().length === 0) break // delivered (maybe by another fiber): decided below + const mine = j.recs.get(g.leaseId) + if (mine?.grant !== g || mine.released !== undefined) return log("grant-superseded-before-create", { leaseId: g.leaseId }) + const why = abandoned.get(g.leaseId) + if (why !== undefined) return yield* abandon(g.leaseId, why, "abandoned-before-create") + yield* drainOutbox + if (undelivered().length === 0) break + if (Date.now() >= until) + return yield* failG(g, { reason: "pre-start/superseded-verdict-pending", message: `the verdict of ${undelivered().join(",")} is not delivered yet` }) + yield* Effect.sleep(Math.min(cfg.resyncMs, 2 * cfg.secondMs)) + } + } + // Round 2: withdrawn only when the floor SAID so (Complete.withdrew names this grant). A duplicate answer (rule 4c) + // or a delivered retryable failure withdrew nothing: this grant is the retry, so fence the old attempt and create. + // Round 3: and only for the generation it withdrew (see drainOutbox). + const by = olds.find((o) => { const x = j.recs.get(o); return x?.withdrew === g.leaseId && x.withdrewGen !== undefined && sameGen(genOf(g), x.withdrewGen) }) + if (by !== undefined) { + // Round 4: only this generation's record; a regrant taken meanwhile is the floor's newer word and stays live + const cur = j.recs.get(g.leaseId) + if (cur?.grant !== g) return log("grant-superseded-before-create", { leaseId: g.leaseId }) + if (cur.released === undefined) j.append({ ev: "released", leaseId: g.leaseId, why: "withdrawn" }) + return log("withdrawn", { leaseId: g.leaseId, by }) + } + const self = j.recs.get(g.leaseId) + if (self?.grant !== g || self.released !== undefined) return log("grant-superseded-before-create", { leaseId: g.leaseId }) + for (const old of olds) { // L3 and B2: the old attempt is gone before attempt n+1 is created + const rec = j.recs.get(old) + if (rec && rec.released === undefined) j.append({ ev: "released", leaseId: old, why: "superseded" }) + if (rec && Journal.maybeCreated(rec)) { + if (!(yield* fence(old))) { + log("fence-failed", { leaseId: g.leaseId, old }) + return yield* failG(g, { reason: "pre-start/ax-unavailable", message: `superseded attempt ${old} not confirmed deleted` }) + } + continue + } + // Round 1 (token scope): a name this link has no journal record for is never deleted on the floor's word alone. + // Absent is fine. Present is fenced only when it is an earlier attempt of THIS job and the Task itself carries + // this link's marks (the journal-lost case, F8); anything else is someone else's Task and this grant is refused. + const r = yield* ax.getTask(old).pipe(Effect.result) + if (r._tag === "Failure") + return yield* failG(g, { reason: "pre-start/ax-unavailable", message: `superseded attempt ${old} not confirmed absent` }) + if (r.success !== undefined && ownEarlierAttempt(old, g, r.success)) { + log("supersedes-unjournaled-own", { leaseId: g.leaseId, old }) + if (!(yield* fence(old))) { + log("fence-failed", { leaseId: g.leaseId, old }) + return yield* failG(g, { reason: "pre-start/ax-unavailable", message: `superseded attempt ${old} not confirmed deleted` }) + } + continue + } + if (r.success !== undefined) { + log("supersedes-foreign", { leaseId: g.leaseId, old }) + return yield* failG(g, { reason: "pre-start/supersedes-foreign", message: `supersedes names ${old}, a Task this link did not create` }) + } + } + yield* createOnly(g) + }).pipe( + Effect.ensuring(Effect.sync(() => { if (dispatching.get(g.leaseId) === g) dispatching.delete(g.leaseId) })), + // L9. Round 4: a defect is not swallowed while the lease is held. The grant is failed back (infra/link-defect, + // retryable: the floor requeues it and the next attempt fences anything this one may have created); if even that + // cannot be journaled, the process dies (exit 1) and a restart resumes from the journal (B13). + Effect.catchCause((c) => Cause.hasInterruptsOnly(c) ? Effect.interrupt : Effect.gen(function*() { + log("job-fiber-died", { leaseId: g.leaseId, cause: Cause.pretty(c) }) + const rec = j.recs.get(g.leaseId) + if (rec?.grant !== g || rec.report !== undefined || rec.released !== undefined || rec.notMine) return + yield* failG(g, { reason: "infra/link-defect", message: Cause.pretty(c).slice(0, 300) }) + }).pipe(Effect.catchCause((c2) => Cause.hasInterruptsOnly(c2) ? Effect.interrupt + : Effect.sync(() => log("link-defect-fatal", { leaseId: g.leaseId, cause: Cause.pretty(c2).slice(0, 300) })).pipe( + Effect.andThen(Deferred.die(fatal, Cause.squash(c2))), Effect.asVoid))))) + + // ---------------------------------------------------------------- resync: the executor's list is the ledger + const outcomeOf = (id: string, t: AxObserved, salvage = false) => Effect.gen(function*() { + const fail = (id: string, o: FailureOutput) => report(id, "failure", o, undefined, salvage) + if (t.phase === "Completed") { + const res = yield* ax.getTaskResult(id).pipe(Effect.result) + // Round 4: UNIMPLEMENTED is the server losing P1 (a rollout), not this Task's result: close the gate at once and + // keep the Task, uncharged, to read it once P1 is back. Guest mode does not read results through ax. + if (res._tag === "Success" && res.success === "unimplemented" && cfg.completion !== "guest") { + setServerP1(false) + return log("result-read-deferred", { leaseId: id, why: "server without P1" }) + } + const unreadable = res._tag === "Failure" ? `GetTaskResult: ${res.failure.message}` + : res.success === undefined ? "no result stored" + : res.success === "unimplemented" ? "server without P1" + : res.success.digestOk === false ? "sha256 mismatch" : undefined + if (unreadable !== undefined) { // B4: a transient read is retried at the next resync, bounded + const n = (resultFailures.get(id) ?? 0) + 1 + resultFailures.set(id, n) + const permanent = res._tag === "Failure" && !res.failure.unavailable + if (!permanent && n < resultReadTries) return log("result-read-retry", { leaseId: id, try: n, why: unreadable }) + return yield* fail(id, { reason: "infra/result-unreadable", phase: t.phase, message: unreadable }) + } + const ok = (res as { success: { content: string; sha256: string } }).success + const bytes = Buffer.byteLength(ok.content, "utf8") + if (bytes > maxOutputBytes) // round 2: refused here, typed, before the floor answers SQLITE_TOOBIG for ever + return yield* fail(id, { reason: "agent/output-too-large", phase: t.phase, + message: `result ${bytes} B > ${maxOutputBytes} B (sha256 ${ok.sha256 || createHash("sha256").update(ok.content).digest("hex")})` }) + let output: unknown + try { output = JSON.parse(ok.content) } catch { return yield* fail(id, { reason: "agent/result-not-json", phase: t.phase }) } + const u = t.usage + return yield* report(id, "success", output, u ? { prompt_tokens: u.promptTokens ?? 0, completion_tokens: u.completionTokens ?? 0, tool_calls: u.toolCalls ?? 0 } : undefined, salvage) + } + const ready = t.conditions.find((c) => c.type === "Ready") + const why = `${ready?.reason ?? ""} ${ready?.message ?? ""}` + const code = /ExitCode=(\d+)/.exec(why) + const reason = /ResourceExhausted/i.test(why) ? "pre-start/resource-exhausted" // B5: a capacity race, retryable (#367) + : /ActorTemplateRejected/.test(why) ? "pre-start/actor-template-rejected" + : /CRASH/i.test(why) ? "infra/actor-crashed" : code ? "agent/exit-code" : "agent/failed" + yield* fail(id, { reason, phase: t.phase, ...(ready?.message ? { message: ready.message } : {}), ...(code ? { exitCode: Number(code[1]) } : {}) }) + }) + + /** B1: a Task missing from the list is confirmed with GetTask. `gone` is true only on NotFound. */ + const confirm = (id: string) => ax.getTask(id).pipe(Effect.map((t) => ({ t, gone: t === undefined })), Effect.orElseSucceed(() => undefined)) + + /** Round 2: before any DeleteTask of a Task that may exist and whose verdict is not journaled (heartbeat `lost`, the + * janitor, a supersede fence), GetTask it; a terminal Task has its outcome read and journaled first. `unknown` means + * ax did not answer or the result read is being retried: delete nothing yet. */ + const salvage = (id: string) => Effect.gen(function*() { + const r = j.recs.get(id) + if (!r || r.report !== undefined || !Journal.maybeCreated(r) || r.deleting !== undefined || r.deleted) return "none" as const + const t = yield* ax.getTask(id).pipe(Effect.result) + if (t._tag === "Failure") return "unknown" as const + if (t.success === undefined) return "gone" as const + if (!owns(id, r, t.success)) { yield* markNotMine(id); return "foreign" as const } // round 3: never read another's result + if (!TERMINAL.has(t.success.phase)) return "live" as const + yield* outcomeOf(id, t.success, true) + return j.recs.get(id)?.report !== undefined ? "read" as const : "unknown" as const + }) + + const reconcile = (id: string, r: LeaseRec, listed: AxObserved | undefined, now: number) => Effect.gen(function*() { + if (!Journal.maybeCreated(r)) return // not created yet: dispatch owns it (or a restart resumes it) + let t = listed + let gone = false + if (t === undefined && !r.deleted) { + const c = yield* confirm(id) + if (c === undefined) return // ax did not answer; decide nothing this round + t = c.t; gone = c.gone + } + if (!gone) absent.delete(id) + if (t !== undefined && !gone && !owns(id, r, t)) return yield* markNotMine(id) // round 3: token scope + if (r.deleting !== undefined) { + if (gone) { + j.append({ ev: "deleted", leaseId: id }) + if (r.deleting === "cancel" && !r.report && r.released === undefined) yield* report(id, "cancelled") + } else if (t !== undefined && t.phase !== "Terminating") { // round 3: a journaled DeleteTask that did not land + if (r.released !== undefined && r.report === undefined && !r.reported && r.dropped === undefined && TERMINAL.has(t.phase)) { + yield* outcomeOf(id, t, true) // a released lease's finished Task is read before it is deleted + if (j.recs.get(id)?.report === undefined) return + } + yield* del(id, r.deleting) + } + return + } + if (r.reported || r.dropped !== undefined || r.released !== undefined) { // the floor is told: janitor only + if (gone) return j.append({ ev: "deleted", leaseId: id }) + if (t === undefined) return + if (r.released !== undefined && r.report === undefined && !r.reported && r.dropped === undefined && TERMINAL.has(t.phase)) { + yield* outcomeOf(id, t, true) // round 2: a released lease's finished Task is read before the janitor deletes it + if (j.recs.get(id)?.report === undefined) return // read retried at the next resync; nothing deleted before + } + const live = !TERMINAL.has(t.phase) // B3: a told lease's sandbox that still runs goes at once + if (live || r.released !== undefined || r.dropped !== undefined || now - r.at >= cfg.deleteAfterMs) + yield* del(id, r.released ? "released" : r.dropped ? "dropped" : live ? "told-still-running" : "janitor") + return + } + if (r.created === undefined) return // round 1: `creating` only: cleanup above, otherwise dispatch or a resume owns it + if (r.report) return // verdict written, waiting for the outbox + if (gone) { + const n = (absent.get(id) ?? 0) + 1 + absent.set(id, n) + if (n < 2) return log("task-absent", { leaseId: id, resyncs: n }) // B1: two NotFound resyncs in a row + return yield* fail(id, { reason: "infra/task-lost", message: "created by this link, NotFound on two resyncs" }) + } + if (t === undefined) return + if (TERMINAL.has(t.phase)) return yield* outcomeOf(id, t) + if ((t.phase === "" || t.phase === "Pending") && now - (r.createdAt ?? r.at) > cfg.pendingTimeoutMs) { + yield* del(id, "pre-start") + return yield* fail(id, { reason: "pre-start/pending-timeout", phase: t.phase || "Pending" }) + } + const deadline = r.grant.deadline + if (t.phase === "Running" && deadline !== undefined && now > deadline + cfg.deadlineBackstopMs) { + yield* del(id, "deadline") + return yield* fail(id, { reason: "deadline-exceeded", phase: t.phase }) + } + }) + + /** Round 4: "P1 proven" is a live condition. Logged on every change; p1-missing closes the gate (capacity 0). */ + const setServerP1 = (v: boolean) => { + if (v === serverP1) return + serverP1 = v + log("completion-probe", { serverP1, completion: cfg.completion, mode: mode() ?? "none" }) + if (!v && cfg.completion !== "guest") log("p1-missing", { completion: cfg.completion }) + } + const probe = Effect.gen(function*() { // P1 adds GetTaskResult; stock v0.3.0 answers UNIMPLEMENTED + const p = yield* ax.getTaskResult("conwip-link-capability-probe").pipe(Effect.result) + if (p._tag === "Failure") return // not answered: keep what is known (undefined after an outage: capacity 0) + setServerP1(p.success !== "unimplemented") + }) + + const checkGateway = Effect.gen(function*() { // B10 + const g = yield* ax.getGateway(cfg.shape.gateway).pipe(Effect.result) + const ok = g._tag === "Success" && g.success !== undefined && !gatewayAllowsAll(g.success) + if (ok !== gatewayOk) log(ok ? "gateway-ok" : "gateway-missing", { gateway: cfg.shape.gateway, ...(g._tag === "Failure" ? { code: g.failure.code } : g.success === undefined ? { found: false } : { allowsAll: true }) }) + gatewayOk = ok + }) + + const resync = Effect.gen(function*() { + const listed = yield* listAll(ax).pipe(Effect.result) + if (listed._tag === "Failure") { + if (axUp) log("ax-down", { code: listed.failure.code }) + axUp = false + // Round 2: nothing read before the outage opens the gate after it. Guest mode does not depend on serverP1, so + // the Gateway verdict and the resync flag are cleared too; canCreate() opens only after a full resync. + resynced = false + gatewayOk = undefined + return + } + const wasDown = !axUp + if (wasDown) serverP1 = undefined // B6: nothing proven before the outage holds after it + yield* probe // round 4: on every resync, since a rollout replaces ax-server with no failed ListTasks + yield* checkGateway + last = new Map(listed.success.map((t) => [t.name, t])) + if (wasDown) log("ax-up", { tasks: listed.success.length }) + axUp = true // round 2: only once probe, Gateway and the occupancy snapshot are all from this side of the outage + const now = Date.now() + for (const [id, r] of [...j.recs]) { + if (dispatching.has(id)) continue + yield* reconcile(id, r, last.get(id), now) + } + resynced = true + }) + + // Occupancy = |live Tasks in the atespace (ours or not) U leases held and not known terminal| (limiter rebuilt from + // the executor's list, agent-stack-k8s limiter.go). A P1-terminal Task is suspended and holds no worker. + const occupancy = () => { + const b = new Set() + for (const [name, t] of last) if (!TERMINAL.has(t.phase)) b.add(name) + for (const id of j.held()) { const t = last.get(id); if (!(t && TERMINAL.has(t.phase))) b.add(id) } + return b.size + } + + // ---------------------------------------------------------------- heartbeat: level-triggered, complete held set + const heartbeat = Effect.gen(function*() { + const ids = j.held() + // Round 4: the reply is about the generations this call vouched for. A reply that lands after a regrant of the + // same leaseId (gen n+1) says nothing about it: `lost`, `cancelRequested` and `renewed` apply only while the record + // still holds the grant object that was sent. + const sent = new Map(ids.map((id) => [id, j.recs.get(id)?.grant])) + const current = (id: string) => sent.has(id) && j.recs.get(id)?.grant === sent.get(id) + const sentAt = Date.now() + const pendingRequestKey = j.pendingLeaseKey // round 4: vouch for grants under a key whose reply never landed (B8) + const r = yield* floor.heartbeat({ holderIdentity: cfg.holder, leaseIds: ids, ...(pendingRequestKey !== undefined ? { pendingRequestKey } : {}) }).pipe(Effect.result) + if (r._tag === "Failure") { + if (r.failure.fatal) { yield* failFatal(r.failure); return undefined } + log("heartbeat-error", { kind: r.failure.kind }) + return undefined + } + for (const id of r.success.renewed) { const g = sent.get(id); if (g !== undefined && current(id)) renewedAt.set(g, Math.max(renewedAt.get(g) ?? 0, sentAt)) } + for (const id of r.success.lost) { // fencing: the lease is gone, so the Task goes; no report + if (!current(id)) { log("stale-heartbeat-reply", { leaseId: id, about: "lost" }); continue } + const rec = j.recs.get(id) + pendingResume.delete(id) + if (!rec) { const x = j.invalid.get(id); if (x && x.released === undefined) j.append({ ev: "released", leaseId: id, why: "lost" }); continue } + if (rec.released !== undefined) continue + j.append({ ev: "released", leaseId: id, why: "lost" }) + abandoned.set(id, "lost") + log("lost", { leaseId: id }) + if (Journal.maybeCreated(rec)) { // round 1: `creating` counts; the janitor confirms NotFound + const s = yield* salvage(id) // round 2: a finished Task's verdict is read and journaled before the delete + if (s === "unknown") log("delete-deferred", { task: id, why: "lost", until: "verdict-read" }) // the janitor reads, then deletes + else if (s !== "gone" && s !== "foreign") yield* del(id, "lost") + } + } + for (const id of r.success.cancelRequested) { // Temporal cancel_requested; ax cancel is DeleteTask, two-phase + if (!current(id)) { log("stale-heartbeat-reply", { leaseId: id, about: "cancel" }); continue } + const rec = j.recs.get(id) + if (!rec || rec.report || rec.deleting !== undefined) continue + log("cancel", { leaseId: id }) + pendingResume.delete(id) + abandoned.set(id, "cancel") // B3: a running dispatch reports or deletes; no later resume creates it + if (dispatching.has(id)) continue + // Round 1: a Task that may exist (`creating`) is deleted and seen NotFound before `cancelled` is reported + if (Journal.maybeCreated(rec)) yield* del(id, "cancel") + else yield* report(id, "cancelled") + } + const renewed = new Set(r.success.renewed) + for (const id of [...pendingResume]) { // B13: resume only what the floor still says is ours + if (!renewed.has(id) || !current(id)) continue + pendingResume.delete(id) + const rec = j.recs.get(id) + if (rec && rec.created === undefined && !rec.report && rec.released === undefined && rec.dropped === undefined) { + log("resume", { leaseId: id }) + yield* Effect.forkScoped(dispatch(rec.grant)) + } + } + return renewed + }) + + // ---------------------------------------------------------------- the loops + const every = (name: string, ms: () => number, body: Effect.Effect) => Effect.forever( + Effect.suspend(() => Effect.sleep(ms())).pipe(Effect.andThen(body), // re-read: the floor may change the interval + Effect.catchCause((c) => Cause.hasInterruptsOnly(c) ? Effect.interrupt : Effect.sync(() => log(`${name}-died`, { cause: Cause.pretty(c) }))))) + + /** Round 3: the heartbeat interval, at most a third of the shortest lease held, so a large floor value can never let + * a held lease expire between two heartbeats. */ + const hbSeconds = () => { + let hi = MAX_HEARTBEAT_SECONDS + for (const id of j.held()) { + const d = j.recs.get(id)?.grant.lease.leaseDurationSeconds + if (d !== undefined && Number.isFinite(d) && d > 0) hi = Math.min(hi, d / 3) + } + return boundedSeconds(hbAsked, defaultHb, Math.max(1, hi)) + } + + /** Final pass (VERIFY-LINK R3-1). The heartbeat sleep is measured from the last heartbeat and is cut short when a + * Lease reply shrinks the interval or journals a grant with a shorter lease: a sleep begun at the 30 s default while + * nothing was held no longer outlives a 12 s lease granted during it. */ + let lastBeatAt = Date.now() + const heartbeatLoop = Effect.forever(Effect.gen(function*() { + let target = jitter(hbSeconds() * cfg.secondMs) + for (;;) { + const left = lastBeatAt + target - Date.now() + if (left <= 0) break + const w = hbWake + yield* Effect.raceFirst(Effect.sleep(left), Deferred.await(w)) + if (yield* Deferred.isDone(w)) { + hbWake = yield* Deferred.make() + target = Math.min(target, jitter(hbSeconds() * cfg.secondMs)) + } + } + lastBeatAt = Date.now() + yield* heartbeat + }).pipe(Effect.catchCause((c) => Cause.hasInterruptsOnly(c) ? Effect.interrupt : Effect.sync(() => log("heartbeat-died", { cause: Cause.pretty(c) }))))) + + let endpointSeen: string | undefined + const leaseOnce = Effect.gen(function*() { + outboxWaitUntil = Math.min(outboxWaitUntil, Date.now()) // B15: a verdict goes out before new work comes in + yield* drainOutbox + // Round 1 (B15): while any verdict is still undelivered, ask for nothing. A grant that arrived now could supersede + // that verdict's attempt, and once n+1 is leased the floor answers n's verdict stale-attempt. + // Round 2: a verdict the floor has answered with a server error is not an outage; it no longer gates Lease (its + // own budget replaces or dead-letters it, and a grant that supersedes it waits at dispatch, bounded). + const undelivered = j.outbox().filter((p) => !verdictFailures.has(p.leaseId)).length + if (undelivered > 0) log("lease-gated-by-outbox", { undelivered }) + const free = undelivered === 0 && canCreate() ? Math.max(0, cfg.maxInFlight - occupancy()) : 0 + // B8: a key is journaled before the call and reused until a reply is journaled, so a lost reply is replayed, + // not stranded. A capacity-0 call can grant nothing, so it is not journaled. + let requestKey = j.pendingLeaseKey + if (requestKey === undefined) { + requestKey = randomUUID() + if (free > 0) j.append({ ev: "lease-key", requestKey }) + } + const sentAt = Date.now() + const r = yield* callLease(free, requestKey) + if (r._tag === "Failure") { + if (r.failure.fatal) return yield* failFatal(r.failure) + leaseFailed(r.failure, requestKey) + // Round 3: bounded, and only this loop waits; the heartbeat and resync fibers run on their own intervals + return yield* Effect.sleep(Math.min(r.failure.retryAfterMs ?? jitter(pollSeconds * cfg.secondMs), MAX_POLL_SECONDS * cfg.secondMs)) + } + yield* applyLease(r.success, requestKey, sentAt) + }) + + const callLease = (capacity: number, requestKey: string) => floor.lease({ holderIdentity: cfg.holder, capacity, requestKey }).pipe( + Effect.retry({ times: 2, while: (e: FloorError) => e.kind === "transient", schedule: Schedule.exponential(Math.max(1, cfg.secondMs / 5)).pipe(Schedule.jittered) }), Effect.result) + const leaseFailed = (e: FloorError, requestKey: string) => { + log("lease-error", { kind: e.kind, pendingKey: j.pendingLeaseKey !== undefined, message: e.message.slice(0, 200) }) + // Round 3: a journaled key is replayed through outages (B8), but not through replies the floor keeps answering + // with a server error or that do not decode: those would strand the key's grants for ever. After + // `leaseKeyAttempts` in a row the key is abandoned; the floor's expiry requeues what it granted under it. + if (e.kind === "server-error" && j.pendingLeaseKey === requestKey) { + if (++leaseKeyFailures >= leaseKeyAttempts) { + j.append({ ev: "lease-replied", requestKey, abandoned: true }) + log("lease-key-abandoned", { tries: leaseKeyFailures, message: e.message.slice(0, 200) }) + leaseKeyFailures = 0 + } + } + } + /** Round 4 (B8 at startup): a key journaled by a previous process whose reply never landed is replayed BEFORE the + * first Heartbeat, so its grants are journaled and in the held set when the Heartbeat re-adopts orphans; replayed + * after it, an orphan granted under the key is omitted and released (L3 a). Capacity 0: a key the floor never saw + * grants nothing new. A failed replay leaves the key pending; the Heartbeat then vouches for it. */ + const replayPendingLease = Effect.gen(function*() { + const requestKey = j.pendingLeaseKey + if (requestKey === undefined) return + const sentAt = Date.now() + const r = yield* callLease(0, requestKey) + if (r._tag === "Failure") { + if (r.failure.fatal) return yield* failFatal(r.failure) + return leaseFailed(r.failure, requestKey) + } + log("lease-key-replayed", { grants: r.success.grants.length }) + yield* applyLease(r.success, requestKey, sentAt) + }) + + const applyLease = (reply: LeaseReply, requestKey: string, sentAt: number) => Effect.gen(function*() { + leaseKeyFailures = 0 + // Round 3: the server-driven intervals are bounded (Buildkite falls back to its defaults): 0, a negative or a + // missing value would make both loops tight loops of billed requests. + pollSeconds = boundedSeconds(reply.nextPollSeconds, defaultPoll, MAX_POLL_SECONDS) + hbAsked = reply.heartbeatSeconds + const hbNow = hbSeconds() + const clamp = `${reply.nextPollSeconds}->${pollSeconds} ${reply.heartbeatSeconds}->${hbNow}` + if ((pollSeconds !== reply.nextPollSeconds || hbNow !== reply.heartbeatSeconds) && clamp !== clampLogged) { + clampLogged = clamp + log("interval-clamped", { nextPollSeconds: reply.nextPollSeconds ?? null, usedPoll: pollSeconds, heartbeatSeconds: reply.heartbeatSeconds ?? null, usedHeartbeat: hbNow }) + } + for (const g of reply.grants) { + const prior = j.recs.get(g.leaseId) + // Round 3: a grant of a lease this link finished (compacted away since) is at-least-once redelivery, not work. + const tomb = j.tombs.get(g.leaseId) + if (prior === undefined && tomb !== undefined && !laterGen(genOf(g), tomb.gen)) { log("duplicate-grant", { leaseId: g.leaseId, finished: true }); continue } + if (prior !== undefined) { + // The floor already said it withdrew this generation (Complete.withdrew journaled on an earlier attempt) but its + // dispatch has not run yet: release it now, so its regrant is not taken for a duplicate. + const pg = genOf(prior.grant) + if (prior.released === undefined && !Journal.maybeCreated(prior) && [...j.recs.values()].some((x) => x.withdrew === g.leaseId && x.withdrewGen !== undefined && sameGen(pg, x.withdrewGen))) { + j.append({ ev: "released", leaseId: g.leaseId, why: "withdrawn" }) + log("withdrawn", { leaseId: g.leaseId, by: "journal" }) + } + // Round 2: dedupe on the grant's identity, not its leaseId. A floor that withdrew n+1 (rule 4b) and then requeued + // it for n's retryable verdict grants the same leaseId again with a new acquireTime and leaseTransitions; that + // is a new grant generation, dispatched, when the old record is released and holds nothing in ax or the outbox. + const newer = laterGen(genOf(g), pg) // round 3: strictly later, so a stale redelivery is never a regrant + // Round 4: the floor's generation is authoritative. A strictly later grant means the floor already withdrew + // the one held (its Complete reply may still be in flight, or lost). If the held one never reached ax and owes + // no verdict, it is released now and the regrant taken; the floor does not redeliver it. Its dispatch sees the + // record change and stops (grant-superseded-before-create). + if (newer && prior.released === undefined && !Journal.maybeCreated(prior) && !Journal.pending(prior)) { + j.append({ ev: "released", leaseId: g.leaseId, why: "superseded-by-regrant" }) + log("superseded-by-regrant", { leaseId: g.leaseId, was: pg.leaseTransitions, now: g.lease.leaseTransitions }) + } + const reusable = prior.released !== undefined && !Journal.pending(prior) && (!Journal.maybeCreated(prior) || prior.deleted === true) + if (!(newer && reusable)) { log("duplicate-grant", { leaseId: g.leaseId }); continue } // at-least-once delivery + for (const m of [abandoned, absent, resultFailures, verdictFailures, verdictNotBefore]) m.delete(g.leaseId) + admitted.delete(g.leaseId); pendingResume.delete(g.leaseId) + j.append({ ev: "grant", leaseId: g.leaseId, grant: g, regrant: true }) + log("regrant", { leaseId: g.leaseId, was: prior.released }) + } else j.append({ ev: "grant", leaseId: g.leaseId, grant: g }) + renewedAt.set(g, sentAt) // round 4: the floor acquired it no earlier than this call was sent + log("leased", { leaseId: g.leaseId, attempt: g.attempt, supersedes: g.supersedes ?? [] }) + yield* Effect.forkScoped(dispatch(g)) + } + for (const b of reply.invalid ?? []) { // round 1: refuse one bad grant, keep the rest of the reply + if (b.leaseId === undefined || b.attempt === undefined) { log("grant-undecodable", { message: b.message }); continue } // the floor's expiry frees it + if (j.recs.has(b.leaseId) || j.invalid.has(b.leaseId)) continue + j.append({ ev: "grant-invalid", leaseId: b.leaseId, attempt: b.attempt, message: b.message }) + log("grant-invalid", { leaseId: b.leaseId, attempt: b.attempt }) + yield* Deferred.succeed(wake, undefined) + } + if (j.pendingLeaseKey === requestKey) j.append({ ev: "lease-replied", requestKey }) + yield* Deferred.succeed(hbWake, undefined) // final pass (R3-1): the interval or the held set may have shrunk + const ep = reply.endpoint // B18: only an exact match in the Nix-declared list, and only once + if (ep !== undefined && ep !== deps.floorUrl && ep !== endpointSeen) { + endpointSeen = ep + if ((cfg.floorUrls ?? []).includes(ep) && ep.startsWith("https://")) { log("endpoint-accepted", { endpoint: ep }); deps.onEndpoint?.(ep) } + else log("endpoint-ignored", { endpoint: ep }) + } + }) + + const body = Effect.gen(function*() { + yield* resync // limiter and outcomes rebuilt before the first heartbeat or lease + for (const [id, r] of j.recs) // grants journaled by a previous process but never created (B13: resumed on renewal) + if (r.created === undefined && !r.report && r.released === undefined && r.dropped === undefined && !r.notMine) pendingResume.add(id) + yield* replayPendingLease // round 4: a key left pending by a kill is answered before the first Heartbeat + yield* heartbeat // vouch first: re-adopt or release orphans (L3), then resume only renewed grants + lastBeatAt = Date.now() + // Round 3: renewal and resync start now, before any Complete or Lease can stall (a 429 Retry-After, a timeout + // chain): held leases keep being renewed whatever the first Lease does. Both re-read their interval every time. + yield* Effect.forkScoped(heartbeatLoop) + yield* Effect.forkScoped(every("resync", () => cfg.resyncMs, resync.pipe(Effect.andThen(drainOutbox)))) + yield* drainOutbox // replay before new work (uplink) + yield* Effect.raceFirst(leaseOnce, Deferred.await(deps.stop)) // its reply sets the server-driven intervals + while (!(yield* Deferred.isDone(deps.stop))) { + const w = wake + yield* Effect.raceFirst(Effect.raceFirst(Effect.suspend(() => Effect.sleep(jitter(pollSeconds * cfg.secondMs))), Deferred.await(w)), Deferred.await(deps.stop)) + if (yield* Deferred.isDone(deps.stop)) break + if (yield* Deferred.isDone(w)) wake = yield* Deferred.make() + yield* Effect.raceFirst(leaseOnce, Deferred.await(deps.stop)) + } + // drain (Buildkite stop before disconnect): no more leases; ax Tasks keep running and are re-adopted on restart + outboxWaitUntil = 0 + yield* drainOutbox + log("drained", { held: j.held().length }) + }) + + return yield* Effect.raceFirst(Effect.scoped(body), Deferred.await(fatal)) +}) diff --git a/pkgs/substrate-link/src/apps/link/src/main.ts b/pkgs/substrate-link/src/apps/link/src/main.ts new file mode 100644 index 000000000..fa4306e5e --- /dev/null +++ b/pkgs/substrate-link/src/apps/link/src/main.ts @@ -0,0 +1,105 @@ +// conwip-link: the NAS-side process. Configuration is environment only (the NixOS module sets it); the bearer is a +// file under $CREDENTIALS_DIRECTORY (systemd LoadCredential from the agenix secret), read once and never logged. +// Exit codes: 0 drained on SIGTERM; 75 another session holds this identity (L5), or an allowed endpoint switch (B18) +// was persisted and the unit should restart onto it; 78 the floor refused the token, answered a redirect (round 2), or +// the configuration is invalid. +import { randomUUID } from "node:crypto" +import { existsSync, readFileSync, writeFileSync } from "node:fs" +import { join } from "node:path" +import { Deferred, Effect } from "effect" +import { grpcAx } from "./ax.ts" +import { ConfigInvalid, readLinkEnv } from "./config.ts" +import type { LinkEnv } from "./config.ts" +import { rpcFloor } from "./floor.ts" +import { seatOf } from "./jobs.ts" +import { Journal } from "./journal.ts" +import { runLink } from "./link.ts" + +const env = (k: string, d?: string) => { + const v = process.env[k] + if (v !== undefined && v !== "") return v + if (d !== undefined) return d + console.error(JSON.stringify({ ev: "config-missing", key: k })) + process.exit(78) +} +const log = (ev: string, f: Record = {}) => console.log(JSON.stringify({ t: new Date().toISOString(), ev, ...f })) +// Round 1: every numeric and enum key is validated up front; a bad one is exit 78 naming the key, never the value. +let c: LinkEnv +try { c = readLinkEnv(process.env) } catch (e) { + if (e instanceof ConfigInvalid) { console.error(JSON.stringify({ ev: "config-invalid", key: e.key, why: e.why })); process.exit(78) } + throw e +} + +const stateDir = env("LINK_STATE_DIR", process.env.STATE_DIRECTORY ?? "/var/lib/conwip-link") +const tokenPath = env("LINK_TOKEN_FILE", join(process.env.CREDENTIALS_DIRECTORY ?? "/nonexistent", "floor-link-token")) +if (!existsSync(tokenPath)) { log("token-missing", { path: tokenPath }); process.exit(78) } +const token = readFileSync(tokenPath, "utf8").trim() +const sessionPath = join(stateDir, "session-id") // stable across restarts of this unit, distinct for a second replica +const journal = Journal.open(stateDir) +if (!existsSync(sessionPath)) writeFileSync(sessionPath, randomUUID() + "\n", { mode: 0o600 }) +const sessionId = readFileSync(sessionPath, "utf8").trim() + +// B18: the floor URLs this link may use, declared in Nix. A switch the floor asked for is persisted here and used on +// the next start only while it is still in the declared list. +const declaredUrl = env("LINK_FLOOR_URL") +const floorUrls = (process.env.LINK_FLOOR_URLS ?? declaredUrl).split(",").map((u) => u.trim()).filter((u) => u !== "") +const endpointPath = join(stateDir, "floor-endpoint") +const persisted = existsSync(endpointPath) ? readFileSync(endpointPath, "utf8").trim() : "" +const floorUrl = persisted !== "" && floorUrls.includes(persisted) ? persisted : declaredUrl +let restartForEndpoint = false + +// Round 2: the link loads its own P1 proto (apps/link/proto/ax-p1.proto), never the stock one: with the stock proto +// GetTaskResult is answered "unimplemented" locally and completion auto/p1 would lease nothing for ever. +let ax: ReturnType +try { ax = grpcAx(c.axServer, c.atespace, undefined, c.axProtoPath) } catch (e) { + console.error(JSON.stringify({ ev: "config-invalid", key: "LINK_AX_PROTO_PATH", why: String((e as Error)?.message ?? e).slice(0, 200) })) + process.exit(78) +} +const program = Effect.gen(function*() { + const stop = yield* Deferred.make() + for (const sig of ["SIGTERM", "SIGINT"] as const) + process.once(sig, () => { log("stop-requested", { signal: sig }); Effect.runFork(Deferred.succeed(stop, undefined)) }) + const floor = yield* rpcFloor({ url: floorUrl, token, sessionId }) + yield* runLink({ + holder: c.holder, + maxInFlight: c.maxInFlight, + servedLabels: c.servedLabels, + shape: { + atespace: c.atespace, + image: c.image, + gateway: c.gateway, + command: (job) => { const seat = seatOf(job); return seat === undefined ? undefined : c.seatCommands[seat] }, // B7: no default seat + ...(c.guestCompleteUrl !== undefined ? { completeUrl: c.guestCompleteUrl } : {}) + }, + completion: c.completion, + secondMs: 1000, + resyncMs: c.resyncSeconds * 1000, + pendingTimeoutMs: c.pendingTimeoutSeconds * 1000, + deleteAfterMs: c.deleteAfterSeconds * 1000, + deadlineBackstopMs: c.deadlineBackstopSeconds * 1000, + createAttempts: c.createAttempts, + outboxBackoffMs: [c.outboxBackoffSeconds[0] * 1000, c.outboxBackoffSeconds[1] * 1000], + fenceTimeoutMs: c.fenceTimeoutSeconds * 1000, + resultReadTries: c.resultReadTries, + verdictAttempts: c.verdictAttempts, + maxOutputBytes: c.maxOutputBytes, + internalHosts: c.internalHosts, + floorUrls + }, { + ax, floor, journal, log, stop, floorUrl, + onEndpoint: (url) => { // B18: persist, drain, exit 75 so systemd restarts onto the new URL + writeFileSync(endpointPath, url + "\n", { mode: 0o600 }) + restartForEndpoint = true + Effect.runFork(Deferred.succeed(stop, undefined)) + } + }) +}) + +Effect.runPromise(Effect.scoped(program)).then( + () => { ax.close(); process.exit(restartForEndpoint ? 75 : 0) }, + (e) => { + ax.close() + const kind = (e as { kind?: string })?.kind + log("fatal", { kind: kind ?? "defect", message: String((e as { message?: string })?.message ?? e) }) + process.exit(kind === "session-conflict" ? 75 : kind === "auth" || kind === "redirect" || kind === "misrouted" ? 78 : 1) + }) diff --git a/pkgs/substrate-link/src/apps/link/test/ax-grpc.test.ts b/pkgs/substrate-link/src/apps/link/test/ax-grpc.test.ts new file mode 100644 index 000000000..32a137f37 --- /dev/null +++ b/pkgs/substrate-link/src/apps/link/test/ax-grpc.test.ts @@ -0,0 +1,91 @@ +// The production AxApi against a real gRPC server loaded from the vendored ax.proto (v0.3.0 d8ed0fe), on loopback. +// Proves the wire mapping the doubles skip: protojson field names survive UpdateTask, NotFound maps to undefined, +// DeleteTask NotFound is success. Round 2: the link loads its own P1 proto (apps/link/proto/ax-p1.proto), so the +// capability probe is decided by the SERVER: a stock server answers UNIMPLEMENTED over the wire (pre-P1), a P1 server +// answers; a proto without GetTaskResult is refused at construction instead of answering "unimplemented" locally. +import { createHash } from "node:crypto" +import * as grpc from "@grpc/grpc-js" +import * as protoLoader from "@grpc/proto-loader" +import { Effect } from "effect" +import { afterAll, beforeAll, expect, it } from "vitest" +import { resolveProtoPath, VENDORED_PROTO_PATH } from "../../../src/axclient.ts" +import { gatewayAllowsAll, grpcAx, LINK_PROTO_PATH, listAll } from "../src/ax.ts" +import { axTaskFromGrant } from "../src/jobs.ts" + +const stored = new Map() +const gateways = new Map([ + ["halogen", { apiVersion: "ax.io/v1alpha1", kind: "Gateway", metadata: { name: "halogen", atespace: "fleet" }, spec: { egress: { allowlist: { hosts: [{ host: "worker", port: 8731 }] } } } }], + ["open", { apiVersion: "ax.io/v1alpha1", kind: "Gateway", metadata: { name: "open", atespace: "fleet" }, spec: {} }] +]) +let server: grpc.Server, port = 0 +const nf = (name: string) => ({ code: grpc.status.NOT_FOUND, details: `task "${name}" not found` }) + +beforeAll(async () => { + const def = protoLoader.loadSync(resolveProtoPath(), { keepCase: false, longs: String, enums: String, defaults: true, oneofs: true }) + const svc = (grpc.loadPackageDefinition(def) as any).ax.v1alpha1.AX.service + server = new grpc.Server() + const impl: Record> = { + GetTask: (c, cb) => { const t = stored.get(c.request.name); t ? cb(null, t) : cb(nf(c.request.name)) }, + UpdateTask: (c, cb) => { stored.set(c.request.task.metadata.name, { ...c.request.task, status: { phase: "Pending" } }); cb(null, c.request.task) }, + // upstream server.go:91-104: limit <= 0 means 50, newest first (store.go ZRevRange) + ListTasks: (c, cb) => { const n = Number(c.request.limit) > 0 ? Number(c.request.limit) : 50, o = Number(c.request.offset) || 0; cb(null, { tasks: [...stored.values()].reverse().slice(o, o + n) }) }, + GetGateway: (c, cb) => { const g = gateways.get(c.request.name); g ? cb(null, g) : cb({ code: grpc.status.NOT_FOUND, details: "gateway not found" }) }, + DeleteTask: (c, cb) => stored.delete(c.request.name) ? cb(null, {}) : cb(nf(c.request.name)) + } + server.addService(svc, impl) + port = await new Promise((res, rej) => server.bindAsync("127.0.0.1:0", grpc.ServerCredentials.createInsecure(), (e, p) => e ? rej(e) : res(p))) +}) +afterAll(() => { server.forceShutdown() }) + +it("G1 grpcAx speaks ax.proto: create-only round trip, NotFound as undefined, a stock server answers GetTaskResult UNIMPLEMENTED", async () => { + const ax = grpcAx(`127.0.0.1:${port}`, "fleet", 3000) + const job = { + apiVersion: "ultracode.mecattaf.dev/v1alpha1" as const, kind: "AgentJob" as const, + metadata: { name: "wf-g-1-abababab", annotations: { "ultracode.mecattaf.dev/item-key": "wf_g#1", "ultracode.mecattaf.dev/journal-key": `${"cd".repeat(32)}:1` } }, + spec: { "runs-on": ["seat:halogen"], with: { prompt: "hi", prompt_ref: { sha256: "ab".repeat(32), bytes: 2, uri: "journal://wf_g/1/prompt.md" }, model: "halogen-qwen3.8-flash-next" } } + } + const built = axTaskFromGrant({ leaseId: "wf-g-1-abababab-a1", attempt: 1, job, lease: { holderIdentity: "nas-link-1", leaseDurationSeconds: 90, acquireTime: 0, renewTime: 0, leaseTransitions: 0 } }, + { atespace: "fleet", image: "localhost:5000/ax-agent@sha256:00", gateway: "halogen", command: () => ["ax-agent", "pi"] }) + if (built._tag !== "ok") throw new Error(built.reason) + const run = (e: Effect.Effect) => Effect.runPromise(e) + expect(await run(ax.getTask("wf-g-1-abababab-a1"))).toBeUndefined() + await run(ax.createTask(built.task)) + const back = stored.get("wf-g-1-abababab-a1") + expect([back.apiVersion, back.metadata.atespace, back.spec.gateway.name, back.spec.command]).toEqual(["ax.io/v1alpha1", "fleet", "halogen", ["ax-agent", "pi"]]) + expect(back.spec.env.find((e: any) => e.name === "AX_CONWIP_ITEM_KEY").value).toBe("wf_g#1") + const seen = await run(ax.getTask("wf-g-1-abababab-a1")) + expect(seen?.phase).toBe("Pending") + expect((await run(ax.listTasks(0, 0))).map((t) => t.name)).toEqual(["wf-g-1-abababab-a1"]) + for (let i = 0; i < 120; i++) stored.set(`bulk-${i}`, { apiVersion: "ax.io/v1alpha1", kind: "Task", metadata: { name: `bulk-${i}`, atespace: "fleet" }, spec: {}, status: { phase: "Completed" } }) + expect((await run(ax.listTasks(0, 0))).length).toBe(50) // B1: one call is one 50-row page + const all = await run(listAll(ax)) + expect([all.length, all.some((t) => t.name === "wf-g-1-abababab-a1")]).toEqual([121, true]) + for (let i = 0; i < 120; i++) stored.delete(`bulk-${i}`) + const gw = await run(ax.getGateway("halogen")) // B10 wire mapping + expect([gw?.hasAllowlist, gw?.hosts, gatewayAllowsAll(gw!)]).toEqual([true, [{ host: "worker", port: 8731 }], false]) + const open = await run(ax.getGateway("open")) + expect([open?.hasAllowlist, gatewayAllowsAll(open!)]).toEqual([false, true]) // no egress: ax applies *:443 + expect(await run(ax.getGateway("missing"))).toBeUndefined() + await run(ax.deleteTask("wf-g-1-abababab-a1")) + await run(ax.deleteTask("wf-g-1-abababab-a1")) // NotFound is success: delete is idempotent for the janitor + expect(await run(ax.getTaskResult("anything"))).toBe("unimplemented") // v0.3.0 server has no GetTaskResult: pre-P1 mode + ax.close() +}) + +it("R2-10 the packaged link (default proto) proves P1 against a P1 server and reads a result; a stock proto is refused", async () => { + let calls = 0 + const def = protoLoader.loadSync(LINK_PROTO_PATH, { keepCase: false, longs: String, enums: String, defaults: true, oneofs: true }) + const svc = (grpc.loadPackageDefinition(def) as any).ax.v1alpha1.AX.service + const p1 = new grpc.Server() + const body = Buffer.from('{"ok":true}') + p1.addService(svc, { + GetTaskResult: (c: any, cb: any) => { calls++; c.request.name === "t1" ? cb(null, { content: body, sha256: createHash("sha256").update(body).digest("hex") }) : cb({ code: grpc.status.NOT_FOUND, details: "nf" }) } + }) + const p1Port = await new Promise((res, rej) => p1.bindAsync("127.0.0.1:0", grpc.ServerCredentials.createInsecure(), (e, p) => e ? rej(e) : res(p))) + const ax = grpcAx(`127.0.0.1:${p1Port}`, "fleet", 3000) // exactly what main.ts builds when LINK_AX_PROTO_PATH is unset + expect(await Effect.runPromise(ax.getTaskResult("conwip-link-capability-probe"))).toBeUndefined() // NotFound: P1 present + expect(await Effect.runPromise(ax.getTaskResult("t1"))).toEqual({ content: '{"ok":true}', sha256: createHash("sha256").update(body).digest("hex"), digestOk: true }) + expect(calls).toBe(2) + ax.close(); p1.forceShutdown() + expect(() => grpcAx("127.0.0.1:1", "fleet", 1000, resolveProtoPath({ AX_CONWIP_PROTO_PATH: VENDORED_PROTO_PATH }))).toThrow(/GetTaskResult/) +}) diff --git a/pkgs/substrate-link/src/apps/link/test/critique-fixed.test.ts b/pkgs/substrate-link/src/apps/link/test/critique-fixed.test.ts new file mode 100644 index 000000000..12a7799a7 --- /dev/null +++ b/pkgs/substrate-link/src/apps/link/test/critique-fixed.test.ts @@ -0,0 +1,406 @@ +// The nine defects the 2026-09-23 critique reproduced (LINK-DESIGN.md "Critique applied", section B), each INVERTED +// into the fixed behaviour now that its build item is in (commit history: the characterization versions are in +// 6342476 and b900624). Plus the new cases the build list names for B1, B6, B9 to B13, B15, B17 and B18. +// Same doubles as link.test.ts; the floor double implements section H's amended rules (B16). +import { mkdtempSync, rmSync } from "node:fs" +import { tmpdir } from "node:os" +import { join } from "node:path" +import { Deferred, Effect, Fiber } from "effect" +import { afterEach, beforeEach, describe, expect, it } from "vitest" +import { AxError } from "../src/ax.ts" +import type { AgentJob, AxTask } from "../src/contract.ts" +import { rpcFloor } from "../src/floor.ts" +import type { FloorOptions } from "../src/floor.ts" +import { Journal } from "../src/journal.ts" +import { isFleetInternalUrl, runLink } from "../src/link.ts" +import type { LinkConfig } from "../src/link.ts" +import { FakeAx } from "./fake-ax.ts" +import { FakeFloor } from "./fake-floor.ts" +import type { FloorConfig } from "./fake-floor.ts" + +const SEC = 20 +const UNAVAILABLE = 14 +const A = "ultracode.mecattaf.dev/" +const JK = `${"cd".repeat(32)}:1` // round 4: FIELD-MAP 5a journal key (jobs.ts refuses a grant without it) +const job = (n: string, spec: Partial = {}): AgentJob => ({ + apiVersion: "ultracode.mecattaf.dev/v1alpha1", kind: "AgentJob", + metadata: { + name: `wf-test-${n}`, + labels: { [A + "run-id"]: "wf-test", [A + "workflow"]: "link-test", [A + "phase-index"]: "1" }, + annotations: { [A + "run-id-raw"]: "wf_test", [A + "label"]: `probe:${n}`, [A + "item-key"]: `wf_test#${n}`, [A + "journal-key"]: JK, [A + "phase-title"]: "Probe" } + }, + spec: { + "runs-on": ["seat:halogen", "runtime:gvisor"], + with: { prompt: `say ${n}`, prompt_ref: { sha256: "ab".repeat(32), bytes: 5, uri: `journal://wf_test/${n}/prompt.md` }, model: "halogen-qwen3.8-flash-next" }, + ...spec + } +}) +const world = (o: Partial = {}) => ({ + floor: new FakeFloor({ cap: 2, leaseSeconds: 6, graceSeconds: 15, pollSeconds: 1, heartbeatSeconds: 2, maxAttempts: 3, secondMs: SEC, tokens: { "tok-nas": "nas-link-1" }, ...o }), + ax: new FakeAx() +}) +type World = ReturnType +const foreign = (name: string): AxTask => ({ apiVersion: "ax.io/v1alpha1", kind: "Task", metadata: { name, atespace: "fleet" }, spec: { image: "someone-else", command: ["true"], env: [] } }) + +interface Extra { fetch?: typeof globalThis.fetch; timeoutsMs?: FloorOptions["timeoutsMs"]; onEndpoint?: (u: string) => void } +function start(w: World, dir: string, over: Partial = {}, x: Extra = {}) { + const logs: Array> = [] + const stop = Effect.runSync(Deferred.make()) + const cfg: LinkConfig = { + holder: "nas-link-1", maxInFlight: 2, servedLabels: ["seat:halogen", "runtime:gvisor"], + shape: { atespace: "fleet", image: "ax-agent", gateway: "halogen", command: () => ["ax-agent", "pi"] }, + completion: "auto", secondMs: SEC, resyncMs: 30, pendingTimeoutMs: 1500, deleteAfterMs: 0, deadlineBackstopMs: 300, + createAttempts: 3, outboxBackoffMs: [20, 100], initialPollSeconds: 1, initialHeartbeatSeconds: 2, fenceTimeoutMs: 300, resultReadTries: 3, ...over + } + const fiber = Effect.runFork(Effect.scoped(Effect.gen(function*() { + const floor = yield* rpcFloor({ url: "http://floor.test", token: "tok-nas", sessionId: `s:${dir}`, fetch: x.fetch ?? w.floor.fetch, ...(x.timeoutsMs ? { timeoutsMs: x.timeoutsMs } : {}) }) + return yield* runLink(cfg, { ax: w.ax, floor, journal: Journal.open(dir), log: (ev, f) => logs.push({ ev, ...f }), stop, floorUrl: "http://floor.test", ...(x.onEndpoint ? { onEndpoint: x.onEndpoint } : {}) }) + }))) + return { + logs, + has: (ev: string, leaseId?: string) => logs.some((l) => l.ev === ev && (leaseId === undefined || l.leaseId === leaseId)), + verdict: (reason: string) => logs.some((l) => l.ev === "verdict" && l.reason === reason), + crash: () => Effect.runPromise(Fiber.interrupt(fiber)), + drain: () => { Effect.runSync(Deferred.succeed(stop, undefined)); return Promise.race([Effect.runPromise(Fiber.await(fiber)), sleep(1500)]) } + } +} +const until = async (what: string, pred: () => boolean, ms = 5000) => { + const t0 = Date.now() + while (!pred()) { if (Date.now() - t0 > ms) throw new Error(`timeout waiting for: ${what}`); await new Promise((r) => setTimeout(r, 5)) } +} +const sleep = (ms: number) => new Promise((r) => setTimeout(r, ms)) +const J = (w: World, n = "1") => w.floor.jobs.get(`wf-test-${n}`)! +const liveOwn = (w: World) => [...w.ax.tasks.keys()].filter((k) => k.startsWith("wf-test-") && w.ax.live(k) && !["Completed", "Failed"].includes(w.ax.tasks.get(k)!.phase)).length + +describe("critique 2026-09-23: the defects, fixed", () => { + let dir: string + let worlds: Array = [] + const W = (o: Partial = {}) => { const w = world(o); worlds.push(w); return w } + beforeEach(() => { dir = mkdtempSync(join(tmpdir(), "conwip-link-critique-")) }) + afterEach(() => { for (const w of worlds) w.floor.close(); worlds = []; rmSync(dir, { recursive: true, force: true }) }) + + it("C1 (B1) a live Task off ListTasks' first 50-row page is neither lost nor rerun", async () => { + const w = W(); w.floor.enqueue(job("1")) + const l = start(w, dir) + await until("running", () => w.ax.tasks.get("wf-test-1-a1")?.phase === "Running") + for (let i = 0; i < 50; i++) w.ax.put(foreign(`foreign-${i}`), "Completed") // newer than ours: ours is on page 2 + const calls = w.ax.listCalls + await until("ten resyncs", () => w.ax.listCalls >= calls + 10) + expect([l.verdict("infra/task-lost"), l.has("task-absent"), w.ax.tasks.has("wf-test-1-a2"), w.ax.deletes]).toEqual([false, false, false, []]) + w.ax.finish("wf-test-1-a1", 0, { answer: 42 }) + await until("floor done", () => J(w).state === "done") + expect([J(w).result, J(w).output, J(w).attempt]).toEqual(["success", { answer: 42 }, 1]) + await l.drain() + }) + + it("C1b (B1) 120 Tasks: occupancy counts a live foreign Task on page 3, and a truly gone Task needs two NotFound resyncs", async () => { + const w = W() + for (let i = 0; i < 2; i++) w.ax.put(foreign(`busy-${i}`), "Running") // oldest: on the last page + for (let i = 0; i < 118; i++) w.ax.put(foreign(`done-${i}`), "Completed") + w.floor.enqueue(job("1")) + const l = start(w, dir) + await until("a Lease was sent", () => w.floor.calls.some((c) => c.rpc === "Lease")) + await sleep(10 * SEC) + expect(w.floor.calls.filter((c) => c.rpc === "Lease").every((c) => c.capacity === 0)).toBe(true) // 2 busy of 2 + w.ax.tasks.delete("busy-0"); w.ax.tasks.delete("busy-1") + await until("created once the sandboxes free", () => w.ax.tasks.get("wf-test-1-a1")?.phase === "Running") + w.ax.tasks.delete("wf-test-1-a1") // the store forgets it (F19) + await until("task-lost after two NotFound resyncs", () => l.verdict("infra/task-lost")) + const absent = l.logs.filter((x) => x.ev === "task-absent" && x.leaseId === "wf-test-1-a1") + expect(absent.length).toBe(1) + await until("attempt 2 created", () => w.ax.tasks.has("wf-test-1-a2")) + await l.drain() + }) + + it("C2 (B2) a superseded attempt that will not delete blocks attempt n+1: retryable pre-start/ax-unavailable, never two at once", async () => { + const w = W(); w.floor.enqueue(job("1")); w.ax.holdPending.add("wf-test-1-a1") + const del = w.ax.deleteTask + w.ax.deleteTask = (name) => name === "wf-test-1-a1" ? Effect.fail(new AxError(UNAVAILABLE, "unavailable")) : del(name) + const l = start(w, dir, { pendingTimeoutMs: 200 }) + await until("fence failed", () => l.has("fence-failed", "wf-test-1-a2"), 8000) + expect(l.verdict("pre-start/ax-unavailable")).toBe(true) + expect(w.ax.updates.has("wf-test-1-a2")).toBe(false) + await until("the infra budget is spent", () => J(w).state === "done", 10000) + expect([J(w).result, (J(w).output as { reason: string }).reason]).toEqual(["failure", "infra/retry-budget-spent"]) + expect([...w.ax.updates.keys()]).toEqual(["wf-test-1-a1"]) // only ever one attempt in ax + expect(J(w).supersedes.length).toBeGreaterThan(1) // rule 7 amended: every unconfirmed old attempt is listed + await l.drain() + }) + + it("C3 (B3) a cancel that lands while UpdateTask is retrying stops the create; nothing runs on", async () => { + const w = W(); w.floor.enqueue(job("1")) + let refusals = 2 + const create = w.ax.createTask + w.ax.createTask = (t) => refusals-- > 0 ? Effect.fail(new AxError(UNAVAILABLE, "unavailable")) : create(t) + const l = start(w, dir, { deleteAfterMs: 60_000, createAttempts: 5 }) + await until("leased", () => l.has("leased", "wf-test-1-a1")) + w.floor.cancel("wf-test-1", "tom") + await until("floor done", () => J(w).state === "done") + expect(J(w).result).toBe("cancelled") + await sleep(600) + expect([w.ax.live("wf-test-1-a1"), l.has("abandoned-before-create", "wf-test-1-a1")]).toEqual([false, true]) + await l.drain() + }) + + it("C3b (B3) a cancel that lands while UpdateTask is in flight deletes the Task as soon as it returns, then reports cancelled", async () => { + const w = W(); w.floor.enqueue(job("1")) + let gate!: () => void + const create = w.ax.createTask + w.ax.createTask = (t) => create(t).pipe(Effect.andThen(Effect.callback((resume) => { gate = () => resume(Effect.void) }))) + const l = start(w, dir, { deleteAfterMs: 60_000 }) + await until("create in flight", () => w.ax.tasks.has("wf-test-1-a1") && gate !== undefined) + w.floor.cancel("wf-test-1", "tom") + await until("cancel seen", () => l.has("cancel", "wf-test-1-a1")) + gate() + await until("floor done", () => J(w).state === "done") + expect([J(w).result, w.ax.live("wf-test-1-a1")]).toEqual(["cancelled", false]) + await l.drain() + }) + + it("C4 (B4) one transient GetTaskResult error is retried at the next resync: the success lands, nothing reruns", async () => { + const w = W(); w.floor.enqueue(job("1")) + let blips = 1 + const res = w.ax.getTaskResult + w.ax.getTaskResult = (n) => n === "wf-test-1-a1" && blips-- > 0 ? Effect.fail(new AxError(UNAVAILABLE, "unavailable")) : res(n) + const l = start(w, dir) + await until("running", () => w.ax.tasks.get("wf-test-1-a1")?.phase === "Running") + w.ax.finish("wf-test-1-a1", 0, { answer: 42 }) + await until("floor done", () => J(w).state === "done") + expect([J(w).result, J(w).output, J(w).attempt, l.has("result-read-retry", "wf-test-1-a1")]).toEqual(["success", { answer: 42 }, 1, true]) + expect(J(w).history.some((h) => h.startsWith("requeue"))).toBe(false) + await l.drain() + }) + + it("C4b (B4) a result whose sha256 never matches is infra/result-unreadable after the bounded tries", async () => { + const w = W(); w.floor.enqueue(job("1")) + const res = w.ax.getTaskResult + w.ax.getTaskResult = (n) => res(n).pipe(Effect.map((r) => r && r !== "unimplemented" && n === "wf-test-1-a1" ? { ...r, digestOk: false } : r)) + const l = start(w, dir) + await until("running", () => w.ax.tasks.get("wf-test-1-a1")?.phase === "Running") + w.ax.finish("wf-test-1-a1", 0, { answer: 42 }) + await until("attempt 2 created", () => w.ax.tasks.has("wf-test-1-a2")) + expect(J(w).history).toContain("requeue:infra/result-unreadable") + expect(l.logs.filter((x) => x.ev === "result-read-retry" && x.leaseId === "wf-test-1-a1").length).toBe(2) + await l.drain() + }) + + it("C5 (B5) Failed with ResourceExhausted is the retryable pre-start/resource-exhausted", async () => { + const w = W(); w.floor.enqueue(job("1")) + const l = start(w, dir) + await until("running", () => w.ax.tasks.get("wf-test-1-a1")?.phase === "Running") + const t = w.ax.tasks.get("wf-test-1-a1")! + t.phase = "Failed" + t.conditions = [{ type: "Ready", status: "False", reason: "ActorCreateFailed", message: "ResourceExhausted: no free workers available" }] + await until("attempt 2 created", () => w.ax.tasks.has("wf-test-1-a2")) + expect(J(w).history).toContain("requeue:pre-start/resource-exhausted") + await l.drain() + }) + + it("C6 (B6) configured p1 on a server without P1 leases nothing and says p1-missing", async () => { + const w = W(); w.floor.enqueue(job("1")); w.ax.p1 = false + const l = start(w, dir, { completion: "p1" }) + await until("probe", () => l.has("p1-missing")) + await sleep(10 * SEC) + expect(w.floor.calls.filter((c) => c.rpc === "Lease").every((c) => c.capacity === 0)).toBe(true) + expect([w.ax.updates.size, J(w).state]).toEqual([0, "queued"]) + await l.drain() + }) + + it("C6b (B6) auto never selects guest, and re-probes when ax comes back: P1 appears, work starts", async () => { + const w = W(); w.floor.enqueue(job("1")); w.ax.p1 = false + const l = start(w, dir, { completion: "auto" }) + await until("probe", () => l.has("p1-missing")) + await sleep(5 * SEC) + expect(w.ax.updates.size).toBe(0) + w.ax.up = false + await until("ax down", () => l.has("ax-down")) + w.ax.p1 = true; w.ax.up = true // the operator rolled out P1 + await until("created in p1 mode", () => w.ax.tasks.has("wf-test-1-a1")) + const env = Object.fromEntries(w.ax.tasks.get("wf-test-1-a1")!.task.spec.env.map((e) => [e.name, e.value])) + expect(env.AX_CONWIP_COMPLETE_URL).toBeUndefined() + await l.drain() + }) + + it("C6c (B6, A5) guest mode with a workers.dev Complete URL leases nothing", async () => { + expect([isFleetInternalUrl("https://conwip-floor.x.workers.dev/guest"), isFleetInternalUrl("http://link.fleet.internal/guest"), isFleetInternalUrl("http://10.201.0.9:8732/g"), isFleetInternalUrl("https://example.com/")]).toEqual([false, true, true, false]) + const w = W(); w.floor.enqueue(job("1")); w.ax.p1 = false + const l = start(w, dir, { completion: "guest", shape: { atespace: "fleet", image: "ax-agent", gateway: "halogen", command: () => ["ax-agent", "pi"], completeUrl: "https://conwip-floor.x.workers.dev/guest" } }) + await until("refused", () => l.has("guest-url-invalid")) + await sleep(5 * SEC) + expect(w.floor.calls.filter((c) => c.rpc === "Lease").every((c) => c.capacity === 0)).toBe(true) + await l.drain() + }) + + it("C7 (B7) an empty runs-on, or two seats, is refused by the link (and by the floor's enqueue)", async () => { + const w = W({ cap: 3 }) + expect(w.floor.enqueue(job("1", { "runs-on": [] }))).toBe(false) + w.floor.enqueueUnchecked(job("1", { "runs-on": [] })) + w.floor.o.tokens["tok-nas"] = { holder: "nas-link-1", labels: ["seat:halogen", "seat:cc", "runtime:gvisor"] } + w.floor.enqueueUnchecked(job("2", { "runs-on": ["seat:halogen", "seat:cc"] })) + const l = start(w, dir, { maxInFlight: 3, servedLabels: ["seat:halogen", "seat:cc", "runtime:gvisor"] }) + await until("both done", () => J(w).state === "done" && J(w, "2").state === "done") + expect([(J(w).output as { reason: string }).reason, (J(w, "2").output as { reason: string }).reason]).toEqual(["pre-start/runs-on-invalid", "pre-start/runs-on-invalid"]) + expect(w.ax.updates.size).toBe(0) + await l.drain() + }) + + it("C8 (B8) a Lease reply lost on all three tries is replayed by the journaled requestKey: attempt 1 runs", async () => { + const w = W(); w.floor.enqueue(job("1")) + let lose = 3 + const lossy: typeof globalThis.fetch = async (input, init) => { + const res = await w.floor.fetch(input, init) + if (lose > 0 && (await res.clone().text()).includes("\"leaseId\":\"wf-test-1-a1\"")) { lose--; throw new TypeError("fetch failed: reply lost") } + return res + } + const l = start(w, dir, {}, { fetch: lossy }) + await until("attempt 1 created", () => w.ax.tasks.has("wf-test-1-a1"), 8000) + expect(J(w).history.some((h) => h.startsWith("requeue"))).toBe(false) + expect(w.ax.tasks.has("wf-test-1-a2")).toBe(false) + await l.drain() + }) + + it("C9 (B9) a hung Heartbeat times out; the next one renews; the running attempt survives", async () => { + const w = W(); w.floor.enqueue(job("1")) + let hang = false + const hanging: typeof globalThis.fetch = async (input, init) => { + const req = new Request(input as string | URL | Request, init) + if (hang && (await req.clone().text()).includes("\"Heartbeat\"")) { hang = false; return new Promise(() => {}) } + return w.floor.fetch(req) + } + const l = start(w, dir, {}, { fetch: hanging, timeoutsMs: { lease: 150, heartbeat: 150, complete: 300 } }) + await until("running", () => w.ax.tasks.get("wf-test-1-a1")?.phase === "Running") + hang = true + await until("the hang timed out", () => l.logs.some((x) => x.ev === "heartbeat-error")) + await sleep(25 * SEC) // past lease plus grace + expect([J(w).history.includes("requeue:grace"), w.ax.deletes.includes("wf-test-1-a1"), J(w).attempt]).toEqual([false, false, 1]) + w.ax.finish("wf-test-1-a1", 0, { ok: 1 }) + await until("floor done", () => J(w).state === "done") + expect([J(w).result, J(w).attempt]).toEqual(["success", 1]) + await l.drain() + }) + + it("C9b (B9) concurrent 401 and 503 are each charged to their own call", async () => { + const w = W() + const f: typeof globalThis.fetch = async (input, init) => { + const body = await new Request(input as string | URL | Request, init).text() + if (body.includes("\"Lease\"")) { await sleep(30); return new Response("busy", { status: 503 }) } + return new Response("no", { status: 401 }) + } + const out = await Effect.runPromise(Effect.scoped(Effect.gen(function*() { + const floor = yield* rpcFloor({ url: "http://floor.test", token: "tok-nas", sessionId: "s", fetch: f }) + return yield* Effect.all([ + floor.lease({ holderIdentity: "nas-link-1", capacity: 1, requestKey: "k" }).pipe(Effect.flip), + floor.heartbeat({ holderIdentity: "nas-link-1", leaseIds: [] }).pipe(Effect.flip) + ], { concurrency: "unbounded" }) + }))) + expect(out.map((e) => e.kind)).toEqual(["transient", "auth"]) + void w + }) + + it("B10 a missing Gateway, or one that allows everything, leases nothing; a restricted one does", async () => { + const w = W(); w.floor.enqueue(job("1")) + const halogen = w.ax.gateways.get("halogen")! + w.ax.gateways.delete("halogen") + const l = start(w, dir) + await until("gateway-missing", () => l.has("gateway-missing")) + w.ax.gateways.set("halogen", { hasAllowlist: true, hosts: [{ host: "*", port: 443 }] }) + await sleep(5 * SEC) + expect([w.ax.updates.size, w.floor.calls.filter((c) => c.rpc === "Lease").every((c) => c.capacity === 0)]).toEqual([0, true]) + w.ax.gateways.set("halogen", halogen) + await until("created", () => w.ax.tasks.has("wf-test-1-a1")) + expect(l.has("gateway-ok")).toBe(true) + await l.drain() + }) + + it("B11 a floor that over-grants never makes the link over-create: extra grants wait, journaled and heartbeated", async () => { + const w = W({ cap: 10, overGrant: 2 }) + for (const n of ["1", "2", "3"]) w.floor.enqueue(job(n)) + const l = start(w, dir, { maxInFlight: 1 }) + let peak = 0 + const sample = setInterval(() => { peak = Math.max(peak, liveOwn(w)) }, 2) + await until("three grants", () => ["1", "2", "3"].every((n) => J(w, n).state === "leased")) + await until("waiting", () => l.has("waiting-for-slot")) + for (const n of ["1", "2", "3"]) { + await until(`job ${n} running`, () => w.ax.tasks.get(`wf-test-${n}-a1`)?.phase === "Running") + w.ax.finish(`wf-test-${n}-a1`, 0, { n }) + await until(`job ${n} done`, () => J(w, n).state === "done") + } + clearInterval(sample) + expect(peak).toBe(1) + expect(["1", "2", "3"].map((n) => [J(w, n).result, J(w, n).attempt])).toEqual([["success", 1], ["success", 1], ["success", 1]]) + await l.drain() + }) + + it("B12 a Task that reads back with another spec is deleted and refused", async () => { + const w = W(); w.floor.enqueue(job("1")) + const create = w.ax.createTask + w.ax.createTask = (t) => create({ ...t, spec: { ...t.spec, env: t.spec.env.slice(0, 3) } }) // the store dropped env rows + const l = start(w, dir) + await until("floor done", () => J(w).state === "done") + expect([(J(w).output as { reason: string }).reason, w.ax.live("wf-test-1-a1")]).toEqual(["pre-start/readback-mismatch", false]) + await l.drain() + }) + + it("B13 a journaled grant the floor released while the link was dead is never created on restart", async () => { + const w = W(); w.floor.enqueue(job("1")); w.ax.blockCreate = true + const l1 = start(w, dir) + await until("grant journaled", () => l1.has("leased", "wf-test-1-a1")) + await l1.crash() + w.ax.blockCreate = false + await until("requeued after the grace", () => J(w).history.includes("requeue:grace"), 3000) + const l2 = start(w, dir) + await until("attempt 2 created", () => w.ax.tasks.has("wf-test-1-a2")) + expect([w.ax.updates.has("wf-test-1-a1"), l2.has("resume", "wf-test-1-a1")]).toEqual([false, false]) + w.ax.tick(); w.ax.finish("wf-test-1-a2", 0, { ok: 2 }) + await until("floor done", () => J(w).state === "done") + expect([J(w).result, J(w).attempt]).toEqual(["success", 2]) + await l2.drain() + }) + + it("B15 floor down past lease plus grace: the queued success goes out before the next Lease and is accepted (rule 4b)", async () => { + const w = W(); w.floor.enqueue(job("1")) + const l = start(w, dir) + await until("running", () => w.ax.tasks.get("wf-test-1-a1")?.phase === "Running") + w.floor.down = true + w.ax.finish("wf-test-1-a1", 0, { v: 1 }) + await until("outbox keeps", () => l.has("outbox-keep", "wf-test-1-a1")) + await until("the alarm requeued it while the floor was unreachable", () => J(w).history.includes("requeue:grace"), 3000) + w.floor.down = false + await until("floor done", () => J(w).state === "done") + expect([J(w).result, J(w).output, J(w).attempt]).toEqual(["success", { v: 1 }, 1]) + expect(J(w).history).toContain("withdrawn:a2") + expect(w.ax.tasks.has("wf-test-1-a2")).toBe(false) + await l.drain() + }) + + it("B17 a verdict that frees a slot leases at once, not at the next poll", async () => { + const w = W({ pollSeconds: 100 }) // 2 s between polls here + w.floor.enqueue(job("1")); w.floor.enqueue(job("2")) + const l = start(w, dir, { maxInFlight: 1 }) + await until("running", () => w.ax.tasks.get("wf-test-1-a1")?.phase === "Running", 4000) + await sleep(200) + const t0 = Date.now() + w.ax.finish("wf-test-1-a1", 0, { ok: 1 }) + await until("job 2 created", () => w.ax.tasks.has("wf-test-2-a1"), 1500) + expect(Date.now() - t0).toBeLessThan(1000) + await l.drain() + }) + + it("B18 a Lease endpoint is followed only when it is in the declared list", async () => { + const w = W() + let ep = "https://evil.example/floor" + const f: typeof globalThis.fetch = async (input, init) => { + const res = await w.floor.fetch(input, init) + const text = await res.text() + const h = new Headers(res.headers); h.delete("content-length") + return new Response(text.replace("\"nextPollSeconds\":", `"endpoint":"${ep}","nextPollSeconds":`), { status: res.status, headers: h }) + } + const seen: Array = [] + const l = start(w, dir, { floorUrls: ["https://conwip-floor-2.example/"] }, { fetch: f, onEndpoint: (u) => seen.push(u) }) + await until("ignored", () => l.has("endpoint-ignored")) + ep = "https://conwip-floor-2.example/" + await until("accepted", () => l.has("endpoint-accepted")) + expect(seen).toEqual(["https://conwip-floor-2.example/"]) + await l.drain() + }) +}) diff --git a/pkgs/substrate-link/src/apps/link/test/fake-ax.ts b/pkgs/substrate-link/src/apps/link/test/fake-ax.ts new file mode 100644 index 000000000..5cc096e80 --- /dev/null +++ b/pkgs/substrate-link/src/apps/link/test/fake-ax.ts @@ -0,0 +1,84 @@ +// Test double for ax-server, with stock v0.3.0 semantics where they matter to the link (MEASURED in ax d8ed0fe: +// internal/server/server.go UpdateTask is a blind upsert that also overwrites status; DeleteTask is two-phase; +// FIELD-MAP D3: a save re-runs the reconcile and resumes the actor). `p1` switches on the carried patch P1: +// a terminal phase is written by the controller and GetTaskResult exists. ListTasks pages like ax's Redis store +// (ZRevRange: newest first, limit 0 means 50; MEASURED upstream server.go:91-104, store.go:209-212). GetGateway +// answers from `gateways`; the default `halogen` Gateway allows one host, as ax-fleet's does. +import { createHash } from "node:crypto" +import { Effect } from "effect" +import { AxError } from "../src/ax.ts" +import type { AxApi, AxGateway, AxObserved, AxResult } from "../src/ax.ts" +import type { AxTask } from "../src/contract.ts" + +const NOT_FOUND = 5, UNAVAILABLE = 14 +type T = { task: AxTask; phase: string; conditions: Array<{ type: string; status: string; reason: string; message: string }>; result?: string; usage?: { promptTokens: number; completionTokens: number; toolCalls: number }; exited?: boolean } + +export class FakeAx implements AxApi { + readonly tasks = new Map() + readonly updates = new Map() // UpdateTask calls per name + upsertsOnExisting = 0 // the hazard: an UpdateTask on a name that already existed (re-runs the actor on v0.3.0) + resumedAfterExit = 0 // how many times that hazard restarted a finished agent + deletes: Array = [] + events: Array = [] // update: and delete:, in call order + up = true + p1 = true + blockCreate = false // createTask never returns (a crash window before the Task exists) + hangAfterCreate = false // createTask writes the Task, then never returns (a crash window after it exists) + holdPending = new Set() // names the controller never starts + gateways = new Map([["halogen", { hasAllowlist: true, hosts: [{ host: "worker", port: 8731 }] }]]) + listCalls = 0 + + private guard = (): Effect.Effect => this.up ? Effect.void : Effect.fail(new AxError(UNAVAILABLE, "connection refused")) + private obs = (name: string, t: T): AxObserved => ({ name, phase: t.phase, spec: t.task.spec, conditions: t.conditions, usage: t.usage ?? null }) + + /** The controller's work between two resyncs: start Pending Tasks, finish two-phase deletes. */ + tick() { + for (const [name, t] of this.tasks) { + if (t.phase === "Terminating") this.tasks.delete(name) + else if (t.phase === "Pending" && !this.holdPending.has(name)) t.phase = "Running" + } + } + /** The agent exits. With P1 the controller writes the terminal phase; without it the Task stays Running. */ + finish(name: string, exitCode: number, result?: unknown, usage = { promptTokens: 11, completionTokens: 7, toolCalls: 3 }) { + const t = this.tasks.get(name) + if (!t) throw new Error(`no task ${name}`) + t.exited = true + if (!this.p1) return + t.phase = exitCode === 0 ? "Completed" : "Failed" + t.conditions = [{ type: "Ready", status: "False", reason: "CommandExited", message: `ExitCode=${exitCode}` }] + if (result !== undefined) t.result = JSON.stringify(result) + t.usage = usage + } + put(task: AxTask, phase = "Running") { this.tasks.set(task.metadata.name, { task, phase, conditions: [] }) } + live(name: string) { const t = this.tasks.get(name); return t !== undefined && t.phase !== "Terminating" } + + getTask = (name: string) => this.guard().pipe(Effect.map(() => { const t = this.tasks.get(name); return t ? this.obs(name, t) : undefined })) + createTask = (task: AxTask) => this.guard().pipe(Effect.andThen(() => { + if (this.blockCreate) return Effect.never + const name = task.metadata.name + this.updates.set(name, (this.updates.get(name) ?? 0) + 1) + this.events.push(`update:${name}`) + const prior = this.tasks.get(name) + if (prior) { this.upsertsOnExisting++; if (prior.exited) this.resumedAfterExit++ } // v0.3.0: status reset, actor resumed + this.tasks.set(name, { task, phase: "Pending", conditions: [] }) + return this.hangAfterCreate ? Effect.never : Effect.void + })) + listTasks = (limit: number, offset: number) => this.guard().pipe(Effect.map(() => { + if (offset === 0) { this.tick(); this.listCalls++ } + const n = limit > 0 ? limit : 50 + return [...this.tasks].reverse().slice(offset, offset + n).map(([name, t]) => this.obs(name, t)) + })) + getGateway = (name: string) => this.guard().pipe(Effect.map(() => this.gateways.get(name))) + deleteTask = (name: string) => this.guard().pipe(Effect.map(() => { + const t = this.tasks.get(name) + if (t) { t.phase = "Terminating"; this.deletes.push(name); this.events.push(`delete:${name}`) } + })) + getTaskResult = (name: string) => this.guard().pipe(Effect.andThen((): Effect.Effect => { + if (!this.p1) return Effect.succeed("unimplemented" as const) + const t = this.tasks.get(name) + if (!t) return Effect.succeed(undefined) + const content = t.result ?? "null" + return Effect.succeed({ content, sha256: createHash("sha256").update(content).digest("hex"), digestOk: true }) + })) + static notFound = NOT_FOUND +} diff --git a/pkgs/substrate-link/src/apps/link/test/fake-floor.ts b/pkgs/substrate-link/src/apps/link/test/fake-floor.ts new file mode 100644 index 000000000..179354ea9 --- /dev/null +++ b/pkgs/substrate-link/src/apps/link/test/fake-floor.ts @@ -0,0 +1,304 @@ +// Test double for the Cloudflare floor: the FloorLink group served by Effect's stock HTTP RPC server through a web +// handler (the same `RpcServer.layerHttp` + `HttpRouter.toWebHandler` pair the prototype DO runs), called in-process +// through the link's `fetch`. State is in memory. It implements the floor rules of LINK-DESIGN.md section 4 as amended +// by "Critique applied", section H (B16): +// rule 1 states queued, leased, orphaned, done; WIP = leased + orphaned +// rule 2 route by runs-on against the holder's labels, cap per label, an admission stub, refuse at enqueue a job +// whose runs-on is empty or does not name exactly one seat: label +// rule 2b a replayed requestKey returns the same grants (kept for the life of the double, at least lease + grace) +// rule 4 stale-attempt, unknown-lease, duplicate; rule 4b: a verdict for attempt n while n+1 is still queued is +// accepted and n+1 withdrawn +// round 1 rule 4c: the verdict applied is recorded per leaseId, and a repeat for the same (leaseId, attempt) answers +// duplicate: true before rule 4b is considered (a lost reply or a second sender never spends a second +// release); rule 4b also accepts n's verdict while n+1 is leased to the SAME holder, which does not create +// n+1 while n's verdict is undelivered (link.ts dispatch) +// round 2 rule 4b names the grant it withdrew in the answer (`withdrew`); the link never infers a withdrawal. Rule +// 4c's duplicate answer replays the original answer's `withdrew`, so a lost reply is answered the same way +// round 3 rule 4b also names the generation withdrawn (`withdrewTransitions`: the leaseTransitions a grant of it +// carried or would have carried); `sendWithdrewGen: false` is a floor that omits it +// rule 5 a timer (the DO alarm): leased past renew + lease becomes orphaned, even with a cancel pending +// rule 6 the holder's heartbeat renews and re-adopts; lost for anything else; cancelRequested +// rule 7 an omitted orphan is released, so is one past the grace; supersedes lists EVERY earlier attempt +// round 4 rule 6b: a Heartbeat's `pendingRequestKey` vouches for every lease granted under that key (renewed and +// re-adopted, never released as omitted) while the holder has not seen that Lease answered +// rule 8 two budgets: infrastructure releases and agent retries, each final reason naming its budget +// rule 9 a per-lease token, minted only for guest-mode holders (A3), single use +// rule 10 at the deadline the floor sets cancel with reason deadline +// rule 11 holder = f(bearer): a payload holderIdentity that differs answers 403; Complete only for that holder's +// leases; one live session per holder (409); pause per holder +// It is NOT the production floor. +import { randomUUID } from "node:crypto" +import { Effect, Layer } from "effect" +import { HttpRouter } from "effect/unstable/http" +import { RpcSerialization, RpcServer } from "effect/unstable/rpc" +import { FloorLink, FloorLinkClient } from "../src/contract.ts" +import type { AgentJob, FailureOutput } from "../src/contract.ts" + +type Job = { + name: string; job: AgentJob; state: "queued" | "leased" | "orphaned" | "done"; attempt: number + leaseId?: string; holder?: string; acquire?: number; renew?: number; orphanedAt?: number; transitions: number + cancel?: string; result?: string; output?: unknown; usage?: unknown; supersedes: Array; token?: string; deadline?: number + history: Array; infraSpent: number; agentSpent: number; holders: Record +} +export type TokenBinding = string | { holder: string; labels?: Array; guest?: boolean } +export interface FloorConfig { + cap: number; leaseSeconds: number; graceSeconds: number; pollSeconds: number; heartbeatSeconds: number + maxAttempts: number // rule 8: agent retries (actor-crashed, result-unreadable) + infraAttempts?: number // rule 8: infrastructure releases (default 5) + secondMs: number + tokens: Record + labelCaps?: Record // rule 2: cap per label + timer?: boolean // rule 5: the DO alarm as a real timer (default true) + overGrant?: number // a broken floor that grants this many more than asked (B11's test) + skewedEncode?: boolean // round 1: a floor whose schema drifted from the link's; grants go out without validation + sendWithdrewGen?: boolean // round 3: Complete names the generation withdrawn (default true) +} +const DEFAULT_LABELS = ["seat:halogen", "runtime:gvisor"] +const INFRA = new Set(["omitted", "grace", "infra/task-lost", "pre-start/pending-timeout", "pre-start/ax-unavailable", "pre-start/resource-exhausted", "pre-start/superseded-verdict-pending", "infra/link-defect"]) +const AGENT = new Set(["infra/actor-crashed", "infra/result-unreadable"]) + +export class FakeFloor { + readonly jobs = new Map() + readonly grants = new Map>() + readonly sessions = new Map() + readonly calls: Array<{ rpc: string; holder?: string; capacity?: number; granted?: number; leaseIds?: ReadonlyArray; status?: number }> = [] + readonly paused = new Set() + readonly verdicts = new Map() // rule 4c + admit: (job: AgentJob) => boolean = () => true // rule 3 stub (seat readings) + seq = 0 + down = false + redeliver: Array = [] // at-least-once: the next Lease hands these leases out again + private readonly timer: ReturnType | undefined + constructor(readonly o: FloorConfig) { + if (o.timer !== false) { this.timer = setInterval(() => this.sweep(), o.secondMs); this.timer.unref?.() } + } + close() { if (this.timer) clearInterval(this.timer) } + private ms = (s: number) => s * this.o.secondMs + private binding(token: string) { + const b = this.o.tokens[token] + if (b === undefined) return undefined + return typeof b === "string" ? { holder: b, labels: DEFAULT_LABELS, guest: false } : { holder: b.holder, labels: b.labels ?? DEFAULT_LABELS, guest: b.guest ?? false } + } + private bindingOf(holder: string) { + for (const t of Object.keys(this.o.tokens)) { const b = this.binding(t)!; if (b.holder === holder) return b } + return undefined + } + + /** Rule 2 (amended): refuses a job whose runs-on is empty or does not name exactly one seat: label. */ + enqueue(job: AgentJob): boolean { + const seats = job.spec["runs-on"].filter((l) => l.startsWith("seat:")) + if (job.spec["runs-on"].length === 0 || seats.length !== 1) return false + this.enqueueUnchecked(job) + return true + } + /** A floor without rule 2's enqueue check, to test the link's own defence (B7). */ + enqueueUnchecked(job: AgentJob) { + if (!this.jobs.has(job.metadata.name)) this.jobs.set(job.metadata.name, { name: job.metadata.name, job, state: "queued", attempt: 1, transitions: 0, supersedes: [], history: [], infraSpent: 0, agentSpent: 0, holders: {} }) + } + cancel(name: string, why = "tom") { + const j = this.jobs.get(name)! + if (j.state === "queued") this.finish(j, "cancelled", { reason: why }) + else if (j.state !== "done") j.cancel = why + } + wip(label?: string) { return [...this.jobs.values()].filter((j) => (j.state === "leased" || j.state === "orphaned") && (label === undefined || j.job.spec["runs-on"].includes(label))).length } + stats() { return { seq: ++this.seq, cap: this.o.cap, wip: this.wip(), queued: [...this.jobs.values()].filter((j) => j.state === "queued").length, done: [...this.jobs.values()].filter((j) => j.state === "done").length } } + private finish(j: Job, result: string, output?: unknown, usage?: unknown) { + Object.assign(j, { state: "done", result, output, usage, token: undefined }); j.history.push(`done:${result}`) + } + /** An orphan the fleet side no longer vouches for: a cancelled job ends cancelled, any other goes back to the queue. */ + private release(j: Job, why: string) { + if (j.cancel) { j.history.push(`released:${why}`); return this.finish(j, "cancelled", { reason: j.cancel }) } + this.requeue(j, why) + } + private requeue(j: Job, why: string) { + j.history.push(`requeue:${why}`) + const infra = INFRA.has(why) || !AGENT.has(why) + if (infra) j.infraSpent++; else j.agentSpent++ + if (infra && j.infraSpent >= (this.o.infraAttempts ?? 5)) return this.finish(j, "failure", { reason: "infra/retry-budget-spent", message: why }) + if (!infra && j.agentSpent >= this.o.maxAttempts) return this.finish(j, "failure", { reason: "agent/retry-budget-spent", message: why }) + const old = j.leaseId! + Object.assign(j, { state: "queued", attempt: j.attempt + 1, transitions: j.transitions + 1, supersedes: [...j.supersedes, old], leaseId: undefined, holder: undefined, token: undefined, orphanedAt: undefined }) + } + /** The DO storage alarm's work (rules 5, 7 b and 10). Runs on the timer and on every request. */ + sweep(now = Date.now()) { + for (const j of this.jobs.values()) { + if (j.state === "leased" && j.deadline !== undefined && now >= j.deadline && !j.cancel) j.cancel = "deadline" + if (j.state === "leased" && j.renew! + this.ms(this.o.leaseSeconds) <= now) { // L3: expiry never frees a live slot, + j.state = "orphaned"; j.orphanedAt = now; j.history.push("orphaned") // even when a cancel is pending + } + if (j.state === "orphaned" && j.orphanedAt! + this.ms(this.o.graceSeconds) <= now) this.release(j, "grace") // L3 b + } + } + private grantable(j: Job, labels: Array, extra: Map) { + if (j.state !== "queued") return false + if (!j.job.spec["runs-on"].every((l) => labels.includes(l))) return false // rule 2: routed by label + for (const l of j.job.spec["runs-on"]) { + const cap = this.o.labelCaps?.[l] + if (cap !== undefined && this.wip(l) + (extra.get(l) ?? 0) >= cap) return false + } + return this.admit(j.job) + } + lease(p: { holderIdentity: string; capacity: number; requestKey: string }) { + this.sweep() + const b = this.bindingOf(p.holderIdentity) + let rows: Array + const prior = this.grants.get(p.requestKey) + if (prior) rows = prior.map((id) => [...this.jobs.values()].find((j) => j.leaseId === id && j.state !== "done")).filter((j): j is Job => j !== undefined) + else if (this.paused.has(p.holderIdentity) || b === undefined) rows = [] + else { + const free = Math.max(0, Math.min(p.capacity, this.o.cap - this.wip())) + (this.o.overGrant ?? 0) + const extra = new Map() + rows = [] + for (const j of this.jobs.values()) { + if (rows.length >= free) break + if (!this.grantable(j, b.labels, extra)) continue + rows.push(j) + for (const l of j.job.spec["runs-on"]) extra.set(l, (extra.get(l) ?? 0) + 1) + } + const now = Date.now() + for (const j of rows) { + const tm = j.job.spec["timeout-minutes"] + const leaseId = `${j.name}-a${j.attempt}` + Object.assign(j, { state: "leased", leaseId, holder: p.holderIdentity, acquire: now, renew: now, token: b.guest ? randomUUID() : undefined, + deadline: tm === undefined ? undefined : now + this.ms(tm * 60) }) + j.holders[leaseId] = p.holderIdentity + j.history.push(`leased:a${j.attempt}`) + } + this.grants.set(p.requestKey, rows.map((j) => j.leaseId!)) + } + if (this.redeliver.length) { + rows = [...rows, ...this.redeliver.map((id) => [...this.jobs.values()].find((j) => j.leaseId === id)).filter((j): j is Job => j !== undefined)] + this.redeliver = [] + } + this.calls.push({ rpc: "Lease", holder: p.holderIdentity, capacity: p.capacity, granted: rows.length }) + return { + grants: rows.map((j) => ({ + leaseId: j.leaseId!, attempt: j.attempt, job: j.job, + lease: { holderIdentity: j.holder!, leaseDurationSeconds: this.o.leaseSeconds, acquireTime: j.acquire!, renewTime: j.renew!, leaseTransitions: j.transitions }, + ...(j.supersedes.length ? { supersedes: [...j.supersedes] } : {}), ...(j.token ? { leaseToken: j.token } : {}), ...(j.deadline !== undefined ? { deadline: j.deadline } : {}) + })), + stats: this.stats(), nextPollSeconds: this.paused.has(p.holderIdentity) ? this.o.pollSeconds * 10 : this.o.pollSeconds, heartbeatSeconds: this.o.heartbeatSeconds + } + } + vouchByKey = true // round 4 rule 6b; false is an L1-only floor that ignores pendingRequestKey + heartbeat(p: { holderIdentity: string; leaseIds: ReadonlyArray; pendingRequestKey?: string }) { + this.sweep() + this.calls.push({ rpc: "Heartbeat", holder: p.holderIdentity, leaseIds: p.leaseIds }) + const byKey = new Set(this.vouchByKey && p.pendingRequestKey !== undefined ? this.grants.get(p.pendingRequestKey) ?? [] : []) + for (const id of byKey) { // rule 6b: vouched for by key, not listed (the holder never saw these ids) + if (p.leaseIds.includes(id)) continue + const j = [...this.jobs.values()].find((x) => x.leaseId === id && (x.state === "leased" || x.state === "orphaned") && x.holder === p.holderIdentity) + if (!j) continue + if (j.state === "orphaned") { j.state = "leased"; j.history.push("re-adopted") } + j.renew = Date.now() + } + const renewed: Array = [], lost: Array = [], cancelRequested: Array = [] + for (const id of p.leaseIds) { + const j = [...this.jobs.values()].find((x) => x.leaseId === id && (x.state === "leased" || x.state === "orphaned") && x.holder === p.holderIdentity) + if (!j) { lost.push(id); continue } + if (j.state === "orphaned") { j.state = "leased"; j.history.push("re-adopted") } // the holder vouches again + j.renew = Date.now(); renewed.push(id) + if (j.cancel) cancelRequested.push(id) + } + for (const j of this.jobs.values()) // L3 a: an orphaned lease its holder no longer lists is gone on the fleet side + if (j.state === "orphaned" && j.holder === p.holderIdentity && !p.leaseIds.includes(j.leaseId!) && !byKey.has(j.leaseId!)) this.release(j, "omitted") + return { renewed, lost, cancelRequested, stats: this.stats() } + } + /** `holder` is the bearer's holder (rule 11); undefined only for the guest path, which its lease token authorises. */ + complete(p: { leaseId: string; attempt: number; result: string; output?: unknown; usage?: unknown }, holder?: string) { + this.sweep() + this.calls.push({ rpc: "Complete", leaseIds: [p.leaseId] }) + const seen = this.verdicts.get(p.leaseId) // rule 4c: at most one verdict per (leaseId, attempt) + if (seen !== undefined && seen.attempt === p.attempt) return { duplicate: true, stats: this.stats(), ...this.w(seen.withdrew, seen.withdrewTransitions) } + const name = p.leaseId.replace(/-a\d+$/, "") + let j = [...this.jobs.values()].find((x) => x.leaseId === p.leaseId) + let withdrew: string | undefined, withdrewTransitions: number | undefined + if (!j) { + const q = this.jobs.get(name) + if (!q) return { code: "unknown-lease" as const } + // rule 4b: attempt n's verdict while n+1 is still queued: accept it and withdraw n+1 + // rule 4b (round 1): n+1 queued, or leased to the holder of n (that holder has not created it) + const nHolder = q.holders[p.leaseId] + const withdrawable = q.state === "queued" || (q.state === "leased" && q.holder === nHolder) + if (withdrawable && q.attempt === p.attempt + 1 && q.supersedes.includes(p.leaseId) && (holder === undefined || nHolder === holder)) { + q.history.push(`withdrawn:a${q.attempt}`) + withdrew = `${q.name}-a${q.attempt}`; withdrewTransitions = q.transitions + Object.assign(q, { state: "leased", attempt: p.attempt, leaseId: p.leaseId, holder: nHolder, token: undefined, renew: Date.now(), supersedes: q.supersedes.filter((x) => x !== p.leaseId) }) + j = q + } else return { code: "stale-attempt" as const } + } + if (holder !== undefined && j.holders[p.leaseId] !== holder) return { code: "stale-attempt" as const } // rule 11 + if (j.state === "done") return { duplicate: true, stats: this.stats() } + if ((j.state !== "leased" && j.state !== "orphaned") || j.attempt !== p.attempt) return { code: "stale-attempt" as const } + const reason = (p.output as FailureOutput | undefined)?.reason + this.verdicts.set(p.leaseId, { attempt: p.attempt, result: p.result, ...(reason !== undefined ? { reason } : {}), ...(withdrew !== undefined ? { withdrew, withdrewTransitions } : {}) }) + const w = this.w(withdrew, withdrewTransitions) + if (p.result === "failure" && reason !== undefined && (INFRA.has(reason) || AGENT.has(reason))) { this.requeue(j, reason); return { duplicate: false, stats: this.stats(), ...w } } + this.finish(j, p.result, p.output ?? (j.cancel ? { reason: j.cancel } : null), p.usage ?? null) + return { duplicate: false, stats: this.stats(), ...w } + } + private w(withdrew: string | undefined, t: number | undefined) { + if (withdrew === undefined) return {} + return this.o.sendWithdrewGen === false || t === undefined ? { withdrew } : { withdrew, withdrewTransitions: t } + } + /** L7: the guest's own Complete, authorised by its per-lease token alone (single use, bound to one lease). */ + guestComplete(leaseId: string, token: string, result: string, output?: unknown) { + const j = [...this.jobs.values()].find((x) => x.leaseId === leaseId) + if (!j || j.token === undefined || j.token !== token) return { refused: "token" as const } + j.token = undefined // single use + return this.complete({ leaseId, attempt: j.attempt, result, output }) + } + + private handlerCache: ((r: Request) => Promise) | undefined + get handler(): (r: Request) => Promise { return this.handlerCache ??= this.skewed() ? this.lenientHandler() : this.strictHandler() } + private skewed() { return this.o.skewedEncode === true } + /** Round 1: the same handlers served through the link's lenient view of Lease, so a grant is sent as the floor + * holds it, without the strict encode (a floor whose contract drifted). */ + private lenientHandler() { + return HttpRouter.toWebHandler( + RpcServer.layerHttp({ group: FloorLinkClient, path: "/rpc", protocol: "http" }).pipe( + Layer.provide(FloorLinkClient.toLayer({ + Lease: (p) => Effect.sync(() => this.lease(p)), + Heartbeat: (p) => Effect.sync(() => this.heartbeat(p)), + Complete: (p, o) => Effect.suspend(() => { + const r = this.complete(p, String((o.headers as Record)["x-link-holder"] ?? "")) + return r.code !== undefined ? Effect.fail({ code: r.code }) : Effect.succeed({ duplicate: r.duplicate!, stats: r.stats!, ...(r.withdrew !== undefined ? { withdrew: r.withdrew } : {}), ...(r.withdrewTransitions !== undefined ? { withdrewTransitions: r.withdrewTransitions } : {}) }) + }) + })), + Layer.provide(RpcSerialization.layerJson)), { disableLogger: true }).handler as (r: Request) => Promise + } + private strictHandler() { return HttpRouter.toWebHandler( + RpcServer.layerHttp({ group: FloorLink, path: "/rpc", protocol: "http" }).pipe( + Layer.provide(FloorLink.toLayer({ + Lease: (p) => Effect.sync(() => this.lease(p)), + Heartbeat: (p) => Effect.sync(() => this.heartbeat(p)), + Complete: (p, o) => Effect.suspend(() => { + const r = this.complete(p, String((o.headers as Record)["x-link-holder"] ?? "")) + return r.code !== undefined ? Effect.fail({ code: r.code }) : Effect.succeed({ duplicate: r.duplicate!, stats: r.stats!, ...(r.withdrew !== undefined ? { withdrew: r.withdrew } : {}), ...(r.withdrewTransitions !== undefined ? { withdrewTransitions: r.withdrewTransitions } : {}) }) + }) + })), + Layer.provide(RpcSerialization.layerJson)), { disableLogger: true }).handler as (r: Request) => Promise } + + /** The Worker entry: network, bearer (per-link token bound to a holder), holder binding (403), then one session per + * holder (409, L5). The resolved holder reaches the handlers as `x-link-holder`; a client's own copy is dropped. */ + readonly fetch: typeof globalThis.fetch = async (input, init) => { + if (this.down) throw new TypeError("fetch failed") + const req = new Request(input as string | URL | Request, init) + const b = this.binding((req.headers.get("authorization") ?? "").replace(/^Bearer /, "")) + const answer = (status: number, body: string) => { this.calls.push({ rpc: "entry", status }); return new Response(body, { status }) } + if (b === undefined) return answer(401, "unauthorized") + const body = await req.clone().text() + let msgs: Array = [] + try { const m = JSON.parse(body); msgs = Array.isArray(m) ? m : [m] } catch { /* not JSON: the RPC server answers */ } + if (msgs.some((m) => m?.payload && typeof m.payload.holderIdentity === "string" && m.payload.holderIdentity !== b.holder)) return answer(403, "holder mismatch") + const sid = req.headers.get("x-link-session") ?? "" + const s = this.sessions.get(b.holder), now = Date.now() + if (s && s.id !== sid && s.until > now) return answer(409, "session held") + this.sessions.set(b.holder, { id: sid, until: now + this.ms(3 * this.o.heartbeatSeconds) }) + const u = new URL(req.url) + if (u.pathname === "/rpc/") u.pathname = "/rpc" // Effect's client posts to "/" (PROTO.md section 4) + const h = new Headers(req.headers) + h.set("x-link-holder", b.holder) + return this.handler(new Request(u, { method: req.method, headers: h, body })) + } +} diff --git a/pkgs/substrate-link/src/apps/link/test/final-pass.test.ts b/pkgs/substrate-link/src/apps/link/test/final-pass.test.ts new file mode 100644 index 000000000..2530486bc --- /dev/null +++ b/pkgs/substrate-link/src/apps/link/test/final-pass.test.ts @@ -0,0 +1,70 @@ +// Final pass (2026-09-23): regression tests for the two link-side findings of wave/VERIFY-LINK.md that were still open at +// 3b036da, both reproduced there by the chaos harness against the prototype floor under wrangler dev --local +// (link/final-verify/chaos-fresh-3b036da: f1 ran as a1 and a2, OVERLAP 8 s). +// FP-1 (R3-1): a heartbeat sleep begun at the 30 s default while nothing was held outlived a short lease granted +// during it; the floor released the orphan and the job ran twice. +// FP-2 (V1): the link fenced only what Grant.supersedes named; a floor that omits it got attempt n+1 created beside +// a still Running attempt n. +import { expect, it } from "vitest" +import { bodyOf, job, sleep, start, until, world } from "./review-r4-harness.ts" + +it("FP-1 a short lease granted during the first long heartbeat sleep is renewed in time", async () => { + // Lease 6 s and grace 2 s at SEC = 20 ms; the link starts with the production default heartbeat (30 s = 600 ms). + const w = world({ leaseSeconds: 6, graceSeconds: 2, heartbeatSeconds: 2 }) + // Every Lease reply is held 30 ms, as a real HTTPS round trip is (the chaos run: wrangler dev --local), so the heartbeat + // fiber sizes its first sleep before the first reply has set the server's interval. With an in-process reply the fiber + // happened to start after it, which hid the defect from every earlier test. + const slowLease: typeof globalThis.fetch = async (input, init) => { + const body = await bodyOf(input, init) + if (body.includes('"Lease"')) await sleep(30) + return w.floor.fetch(input as any, { ...init, body }) + } + const l = start(w, undefined, { initialHeartbeatSeconds: 30 }, slowLease) + await until("first Lease answered", () => w.floor.jobs.size === 0 && l.logs.some((x) => x.ev === "ax-up"), 2000).catch(() => {}) + await sleep(60) // the first heartbeat sleep has begun, sized from the 30 s default + w.floor.enqueue(job("1")) + const J = () => w.floor.jobs.get("wf-test-1")! + await until("a1 running", () => w.ax.tasks.get("wf-test-1-a1") !== undefined, 3000) + await sleep(1200) // twice the default heartbeat: without the fix the lease expires and the grace runs out first + const out = { history: J().history, attempt: J().attempt, a2: w.ax.updates.get("wf-test-1-a2") ?? 0 } + console.log(JSON.stringify(out)) + await l.drain(); w.floor.close() + expect(out.attempt).toBe(1) + expect(out.a2).toBe(0) + expect(out.history).not.toContain("requeue:omitted") + expect(out.history.filter((h: string) => h.startsWith("requeue")).length).toBe(0) +}) + +it("FP-2 attempt n+1 is never created beside a live attempt n, even when the floor omits supersedes", async () => { + const w = world({ leaseSeconds: 6, graceSeconds: 2, heartbeatSeconds: 2 }) + let failHeartbeat = false + const strip = (s: string) => s.replace(/,"supersedes":\[[^\]]*\]/g, "").replace(/"supersedes":\[[^\]]*\],/g, "") + const fetch: typeof globalThis.fetch = async (input, init) => { + const body = await bodyOf(input, init) + if (failHeartbeat && body.includes('"Heartbeat"')) throw new TypeError("fetch failed") + const res = await w.floor.fetch(input as any, { ...init, body }) + const text = await res.text() + return new Response(strip(text), { status: res.status, headers: res.headers }) + } + let a1LiveAtA2Create: boolean | undefined + const create = w.ax.createTask + ;(w.ax as any).createTask = (task: any) => { + if (task.metadata.name === "wf-test-1-a2" && a1LiveAtA2Create === undefined) a1LiveAtA2Create = w.ax.live("wf-test-1-a1") + return create(task) + } + w.floor.enqueue(job("1")) + const l = start(w, undefined, {}, fetch) + const J = () => w.floor.jobs.get("wf-test-1")! + await until("a1 running", () => w.ax.live("wf-test-1-a1") && l.has("created", "wf-test-1-a1"), 3000) + failHeartbeat = true // a1's lease expires and the grace runs out; the floor requeues attempt 2 and grants it on Lease + await until("floor requeued attempt 2", () => J().attempt === 2, 4000) + await until("a2 leased", () => l.has("leased", "wf-test-1-a2"), 4000) + failHeartbeat = false + await until("a2 created", () => (w.ax.updates.get("wf-test-1-a2") ?? 0) > 0, 4000) + const out = { a1LiveAtA2Create, extended: l.has("fence-set-extended", "wf-test-1-a2"), events: (w.ax as any).events?.filter((e: string) => e.includes("wf-test-1")) } + console.log(JSON.stringify(out)) + await l.drain(); w.floor.close() + expect(out.extended).toBe(true) + expect(out.a1LiveAtA2Create).toBe(false) + expect(w.ax.upsertsOnExisting).toBe(0) +}) diff --git a/pkgs/substrate-link/src/apps/link/test/fixtures/ax-p1.proto b/pkgs/substrate-link/src/apps/link/test/fixtures/ax-p1.proto new file mode 100644 index 000000000..a18e4a43a --- /dev/null +++ b/pkgs/substrate-link/src/apps/link/test/fixtures/ax-p1.proto @@ -0,0 +1,405 @@ +// TEST FIXTURE: the vendored ax.proto (v0.3.0 d8ed0fe) plus the three hunks of carried patch P1 that the link reads: +// rpc GetTaskResult, messages GetTaskResultRequest and TaskResult, UsageStats.tool_calls. Copied from dotfiles branch +// ax/fleet-bringup pkgs/ax/patches/p1-completion.patch (lines 3240-3300) on 2026-09-23. Used only by the workerd +// integration test; production vendors the P1 proto when ax-fleet ships it (LINK-DESIGN section 9). +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +syntax = "proto3"; + +package ax.v1alpha1; + +import "google/protobuf/struct.proto"; +import "google/protobuf/timestamp.proto"; + +option go_package = "github.com/google/ax/pkg/apis/v1alpha1"; + +// AX defines the core control plane gRPC service for autonomous agent orchestration. +service AX { + // Manifests are parsed by the client (see `ax apply`) and submitted through the + // typed Update* RPCs below; the server never receives raw YAML. + + // Tasks + rpc GetTask(GetTaskRequest) returns (Task); + rpc ListTasks(ListTasksRequest) returns (ListTasksResponse); + rpc UpdateTask(UpdateTaskRequest) returns (Task); + rpc DeleteTask(DeleteTaskRequest) returns (DeleteTaskResponse); + rpc SuspendTask(SuspendTaskRequest) returns (Task); + rpc ResumeTask(ResumeTaskRequest) returns (Task); + rpc WatchTask(WatchTaskRequest) returns (stream WatchTaskResponse); + // GetTaskResult returns the result file the task's command wrote, as copied + // by the controller when the command exited. NotFound until then. + rpc GetTaskResult(GetTaskResultRequest) returns (TaskResult); + + // Gateways + rpc GetGateway(GetGatewayRequest) returns (Gateway); + rpc ListGateways(ListGatewaysRequest) returns (ListGatewaysResponse); + rpc UpdateGateway(UpdateGatewayRequest) returns (Gateway); + rpc DeleteGateway(DeleteGatewayRequest) returns (DeleteGatewayResponse); + + // Workspaces + rpc GetWorkspace(GetWorkspaceRequest) returns (Workspace); + rpc ListWorkspaces(ListWorkspacesRequest) returns (ListWorkspacesResponse); + rpc UpdateWorkspace(UpdateWorkspaceRequest) returns (Workspace); + rpc DeleteWorkspace(DeleteWorkspaceRequest) returns (DeleteWorkspaceResponse); + + // Models + rpc GetModel(GetModelRequest) returns (Model); + rpc ListModels(ListModelsRequest) returns (ListModelsResponse); + rpc UpdateModel(UpdateModelRequest) returns (Model); + rpc DeleteModel(DeleteModelRequest) returns (DeleteModelResponse); + +} + +// ObjectMeta is metadata for AX resources. +message ObjectMeta { + string name = 1; + string atespace = 2; + google.protobuf.Timestamp creation_timestamp = 3; +} + +// --- Task --- + +message Task { + string api_version = 1; + string kind = 2; + ObjectMeta metadata = 3; + TaskSpec spec = 4; + TaskStatus status = 5; +} + +message TaskSpec { + // Field 1 was `goal`, removed; the workspace goal lives on WorkspaceRef. + reserved 1; + reserved "goal"; + bool suspend = 2; + string image = 3; + repeated string command = 4; + repeated EnvVar env = 5; + ResourceReqs resources = 6; + // workspaces binds one or more Workspaces, each mounted at its own path + // under /workspace. The first entry is the task command's working directory. + repeated WorkspaceRef workspaces = 7; + GatewayRef gateway = 8; + // Field 9 was `policies` (budget and approval config), removed for now. + reserved 9; + reserved "policies"; + // debug enables the in-container guest services (process execution and file + // access) that back `ax ssh`. Off by default. + bool debug = 10; +} + +message EnvVar { + string name = 1; + string value = 2; +} + +message ResourceReqs { + ResourceList requests = 1; + ResourceList limits = 2; +} + +message ResourceList { + string cpu = 1; + string memory = 2; +} + +message WorkspaceRef { + string name = 1; + string path = 2; + string goal = 3; +} + +message GatewayRef { + string name = 1; +} + +message TaskStatus { + string phase = 1; + string id = 2; + string actor = 3; + // json_name keeps the established `workerIP` spelling in YAML and JSON. + string worker_ip = 4 [json_name = "workerIP"]; + PendingApproval pending_approval = 5; + UsageStats usage = 6; + repeated Condition conditions = 7; +} + +message PendingApproval { + string id = 1; + string action = 2; + google.protobuf.Timestamp requested_at = 3; +} + +message UsageStats { + int32 prompt_tokens = 1; + int32 completion_tokens = 2; + int32 tool_calls = 3; +} + +message Condition { + string type = 1; + string status = 2; + google.protobuf.Timestamp last_transition_time = 3; + string reason = 4; + string message = 5; +} + +// --- Gateway --- + +message Gateway { + string api_version = 1; + string kind = 2; + ObjectMeta metadata = 3; + GatewaySpec spec = 4; +} + +message GatewaySpec { + repeated Listener listeners = 1; + EgressConfig egress = 2; +} + +message Listener { + string name = 1; + int32 port = 2; + string protocol = 3; +} + +message EgressConfig { + EgressAllowlist allowlist = 1; +} + +message EgressAllowlist { + repeated HostRule hosts = 1; +} + +message HostRule { + string host = 1; + int32 port = 2; +} + +// --- Workspace --- + +message Workspace { + string api_version = 1; + string kind = 2; + ObjectMeta metadata = 3; + WorkspaceSpec spec = 4; +} + +message WorkspaceSpec { + repeated GitRepo git = 1; + MCPConfig mcp = 2; + SkillsConfig skills = 3; +} + +message GitRepo { + string name = 1; + string repo = 2; + string branch = 3; + string dir = 4; + int32 depth = 5; +} + +message MCPConfig { + repeated MCPRegistry registries = 1; + repeated MCPServer servers = 2; +} + +message MCPRegistry { + string provider = 1; + string project = 2; + string query = 3; + repeated MCPServer servers = 4; +} + +message MCPServer { + string name = 1; + string endpoint = 2; + string command = 3; + repeated string args = 4; +} + +message SkillsConfig { + repeated SkillRegistry registries = 1; + string path = 2; +} + +message SkillRegistry { + string provider = 1; + string project = 2; + string query = 3; +} + +// --- Model --- + +message Model { + string api_version = 1; + string kind = 2; + ObjectMeta metadata = 3; + ModelSpec spec = 4; +} + +message ModelSpec { + string provider = 1; + string model = 2; + // Fields 3-5 were typed temperature, max_tokens, and system_instruction; + // provider settings now live in the free-form `parameters` map. + reserved 3, 4, 5; + reserved "temperature", "max_tokens", "system_instruction"; + SecretKeyRef secret_key = 6; + // parameters are provider-specific generation settings passed through to the + // model API as-is, for example temperature or maxOutputTokens for Gemini. + google.protobuf.Struct parameters = 7; +} + +message SecretKeyRef { + string name = 1; + string key = 2; +} + +// --- RPC Request & Response Messages --- + +// Tasks +message GetTaskRequest { + string atespace = 1; + string name = 2; +} + +message GetTaskResultRequest { + string atespace = 1; + string name = 2; +} + +message TaskResult { + bytes content = 1; + string sha256 = 2; +} + +message ListTasksRequest { + string atespace = 1; + int64 limit = 2; + int64 offset = 3; +} + +message ListTasksResponse { + repeated Task tasks = 1; +} + +message UpdateTaskRequest { + Task task = 1; +} + +message DeleteTaskRequest { + string atespace = 1; + string name = 2; +} + +message DeleteTaskResponse {} + +message SuspendTaskRequest { + string atespace = 1; + string name = 2; +} + +message ResumeTaskRequest { + string atespace = 1; + string name = 2; +} + +message WatchTaskRequest { + string atespace = 1; + string name = 2; +} + +message WatchTaskResponse { + Task task = 1; + string action = 2; +} + +// Gateways +message GetGatewayRequest { + string atespace = 1; + string name = 2; +} + +message ListGatewaysRequest { + string atespace = 1; +} + +message ListGatewaysResponse { + repeated Gateway gateways = 1; +} + +message UpdateGatewayRequest { + Gateway gateway = 1; +} + +message DeleteGatewayRequest { + string atespace = 1; + string name = 2; +} + +message DeleteGatewayResponse {} + +// Workspaces +message GetWorkspaceRequest { + string atespace = 1; + string name = 2; +} + +message ListWorkspacesRequest { + string atespace = 1; +} + +message ListWorkspacesResponse { + repeated Workspace workspaces = 1; +} + +message UpdateWorkspaceRequest { + Workspace workspace = 1; +} + +message DeleteWorkspaceRequest { + string atespace = 1; + string name = 2; +} + +message DeleteWorkspaceResponse {} + +// Models +message GetModelRequest { + string atespace = 1; + string name = 2; +} + +message ListModelsRequest { + string atespace = 1; +} + +message ListModelsResponse { + repeated Model models = 1; +} + +message UpdateModelRequest { + Model model = 1; +} + +message DeleteModelRequest { + string atespace = 1; + string name = 2; +} + +message DeleteModelResponse {} + diff --git a/pkgs/substrate-link/src/apps/link/test/link.test.ts b/pkgs/substrate-link/src/apps/link/test/link.test.ts new file mode 100644 index 000000000..d095d6fbb --- /dev/null +++ b/pkgs/substrate-link/src/apps/link/test/link.test.ts @@ -0,0 +1,357 @@ +// The failure matrix of LINK-DESIGN.md section 8, one test per row, against the two doubles (fake ax with v0.3.0 +// upsert semantics and an optional P1; fake floor served through Effect's HTTP RPC server). Every server-set +// interval is scaled so one "second" is SEC ms. A crash is a fiber interrupt: nothing after it runs, and only what the +// journal fsynced survives, which is what kill -9 leaves. +import { mkdtempSync, rmSync } from "node:fs" +import { tmpdir } from "node:os" +import { join } from "node:path" +import { Deferred, Effect, Exit, Fiber } from "effect" +import { afterEach, beforeEach, describe, expect, it } from "vitest" +import type { AgentJob } from "../src/contract.ts" +import { rpcFloor } from "../src/floor.ts" +import { Journal } from "../src/journal.ts" +import { runLink } from "../src/link.ts" +import type { LinkConfig } from "../src/link.ts" +import { FakeAx } from "./fake-ax.ts" +import { FakeFloor } from "./fake-floor.ts" +import type { FloorConfig } from "./fake-floor.ts" + +const SEC = 20 +const A = "ultracode.mecattaf.dev/" +const JK = `${"cd".repeat(32)}:1` // round 4: FIELD-MAP 5a journal key (jobs.ts refuses a grant without it) +const job = (n: string, spec: Partial = {}, ann: Record = {}, w: Partial = {}): AgentJob => ({ + apiVersion: "ultracode.mecattaf.dev/v1alpha1", kind: "AgentJob", + metadata: { + name: `wf-test-${n}`, + labels: { [A + "run-id"]: "wf-test", [A + "workflow"]: "link-test", [A + "phase-index"]: "1" }, + annotations: { [A + "run-id-raw"]: "wf_test", [A + "label"]: `probe:${n}`, [A + "item-key"]: `wf_test#${n}`, [A + "journal-key"]: JK, [A + "phase-title"]: "Probe", ...ann } + }, + spec: { + "runs-on": ["seat:halogen", "runtime:gvisor"], + with: { prompt: `say ${n}`, prompt_ref: { sha256: "ab".repeat(32), bytes: 5, uri: `journal://wf_test/${n}/prompt.md` }, model: "halogen-qwen3.8-flash-next", ...w }, + ...spec + } +}) +const world = (o: Partial = {}) => ({ + floor: new FakeFloor({ cap: 2, leaseSeconds: 6, graceSeconds: 15, pollSeconds: 1, heartbeatSeconds: 2, maxAttempts: 3, secondMs: SEC, tokens: { "tok-nas": "nas-link-1" }, ...o }), + ax: new FakeAx() +}) +type World = ReturnType + +const GUEST = { completion: "guest" as const, shape: { atespace: "fleet", image: "ax-agent", gateway: "halogen", command: () => ["ax-agent", "pi"], completeUrl: "http://link.fleet.internal/guest" } } +const guestWorld = () => { const w = world({ tokens: { "tok-nas": { holder: "nas-link-1", guest: true } } }); w.ax.p1 = false; return w } + +function start(w: World, dir: string, over: Partial = {}, session = `s:${dir}`) { + const logs: Array> = [] + const stop = Effect.runSync(Deferred.make()) + const cfg: LinkConfig = { + holder: "nas-link-1", maxInFlight: 2, servedLabels: ["seat:halogen", "runtime:gvisor"], + shape: { atespace: "fleet", image: "ax-agent", gateway: "halogen", command: () => ["ax-agent", "pi"] }, + completion: "auto", secondMs: SEC, resyncMs: 30, pendingTimeoutMs: 1500, deleteAfterMs: 0, deadlineBackstopMs: 300, + createAttempts: 3, outboxBackoffMs: [20, 100], initialPollSeconds: 1, initialHeartbeatSeconds: 2, ...over + } + const fiber = Effect.runFork(Effect.scoped(Effect.gen(function*() { + const floor = yield* rpcFloor({ url: "http://floor.test", token: "tok-nas", sessionId: session, fetch: w.floor.fetch }) + return yield* runLink(cfg, { ax: w.ax, floor, journal: Journal.open(dir), log: (ev, f) => logs.push({ ev, ...f }), stop }) + }))) + return { + logs, + has: (ev: string, leaseId?: string) => logs.some((l) => l.ev === ev && (leaseId === undefined || l.leaseId === leaseId)), + crash: () => Effect.runPromise(Fiber.interrupt(fiber)), + drain: () => { Effect.runSync(Deferred.succeed(stop, undefined)); return Effect.runPromise(Fiber.await(fiber)) }, + exit: () => Effect.runPromise(Fiber.await(fiber)) + } +} +const until = async (what: string, pred: () => boolean, ms = 5000) => { + const t0 = Date.now() + while (!pred()) { if (Date.now() - t0 > ms) throw new Error(`timeout waiting for: ${what}`); await new Promise((r) => setTimeout(r, 5)) } +} +const sleep = (ms: number) => new Promise((r) => setTimeout(r, ms)) +const J = (w: World, n = "1") => w.floor.jobs.get(`wf-test-${n}`)! + +describe("conwip link failure matrix", () => { + let dir: string + beforeEach(() => { dir = mkdtempSync(join(tmpdir(), "conwip-link-")) }) + afterEach(() => rmSync(dir, { recursive: true, force: true })) + + it("F0 happy path on P1: one create, typed verdict with usage, Task deleted after the floor acknowledged", async () => { + const w = world(); w.floor.enqueue(job("1")) + const l = start(w, dir) + await until("running", () => w.ax.tasks.get("wf-test-1-a1")?.phase === "Running") + const env = Object.fromEntries(w.ax.tasks.get("wf-test-1-a1")!.task.spec.env.map((e) => [e.name, e.value])) + expect(env.AX_CONWIP_ITEM_KEY).toBe("wf_test#1"); expect(env.AX_CONWIP_ATTEMPT).toBe("1"); expect(env.AX_CONWIP_PROMPT).toBe("say 1") + expect(env.AX_CONWIP_LEASE_TOKEN).toBeUndefined() // P1 present: the guest carries no token at all + expect(w.ax.tasks.get("wf-test-1-a1")!.task.apiVersion).toBe("ax.io/v1alpha1") + w.ax.finish("wf-test-1-a1", 0, { answer: 42 }) + await until("floor done", () => J(w).state === "done") + expect([J(w).result, J(w).output, J(w).usage]).toEqual(["success", { answer: 42 }, { prompt_tokens: 11, completion_tokens: 7, tool_calls: 3 }]) + await until("janitor deleted the Task", () => !w.ax.tasks.has("wf-test-1-a1")) + expect(w.ax.updates.get("wf-test-1-a1")).toBe(1); expect(w.ax.upsertsOnExisting).toBe(0) + await l.drain() + }) + + it("F1 duplicate delivery: a redelivered grant creates nothing twice", async () => { + const w = world(); w.floor.enqueue(job("1")) + const l = start(w, dir) + await until("running", () => w.ax.tasks.get("wf-test-1-a1")?.phase === "Running") + w.floor.redeliver = ["wf-test-1-a1"] + await until("duplicate seen", () => l.has("duplicate-grant", "wf-test-1-a1")) + w.ax.finish("wf-test-1-a1", 0, { ok: true }) + await until("floor done", () => J(w).state === "done") + expect(w.ax.updates.get("wf-test-1-a1")).toBe(1); expect(w.ax.upsertsOnExisting).toBe(0) + await l.drain() + }) + + it("F2 crash after Lease, before UpdateTask: the restart creates exactly once", async () => { + const w = world(); w.floor.enqueue(job("1")); w.ax.blockCreate = true + const l1 = start(w, dir) + await until("grant journaled", () => l1.has("leased", "wf-test-1-a1")) + await l1.crash() + w.ax.blockCreate = false + const l2 = start(w, dir) + await until("running", () => w.ax.tasks.get("wf-test-1-a1")?.phase === "Running") + w.ax.finish("wf-test-1-a1", 0, { ok: 1 }) + await until("floor done", () => J(w).state === "done") + expect([J(w).result, J(w).attempt, w.ax.updates.get("wf-test-1-a1")]).toEqual(["success", 1, 1]) + await l2.drain() + }) + + it("F3 crash after UpdateTask, before the journal said created: the restart adopts, no second UpdateTask", async () => { + const w = world(); w.floor.enqueue(job("1")); w.ax.hangAfterCreate = true + const l1 = start(w, dir) + await until("task exists", () => w.ax.tasks.has("wf-test-1-a1")) + await l1.crash() + w.ax.hangAfterCreate = false + const l2 = start(w, dir) + await until("adopted", () => l2.has("adopted", "wf-test-1-a1")) + w.ax.finish("wf-test-1-a1", 0, { ok: 1 }) + await until("floor done", () => J(w).state === "done") + expect([J(w).result, w.ax.updates.get("wf-test-1-a1"), w.ax.upsertsOnExisting]).toEqual(["success", 1, 0]) + await l2.drain() + }) + + it("F4 crash while running; the agent finishes while the link is down; the restart reports once", async () => { + const w = world(); w.floor.enqueue(job("1")) + const l1 = start(w, dir) + await until("running", () => w.ax.tasks.get("wf-test-1-a1")?.phase === "Running") + await l1.crash() + w.ax.finish("wf-test-1-a1", 0, { late: true }) + const l2 = start(w, dir) + await until("floor done", () => J(w).state === "done") + expect([J(w).result, J(w).output, J(w).attempt]).toEqual(["success", { late: true }, 1]) + expect(w.floor.calls.filter((c) => c.rpc === "Complete").length).toBe(1) + await l2.drain() + }) + + it("F5 floor unreachable: the verdict waits in the outbox across a crash, the lease is orphaned not requeued, then lands", async () => { + const w = world(); w.floor.enqueue(job("1")) + const l1 = start(w, dir) + await until("running", () => w.ax.tasks.get("wf-test-1-a1")?.phase === "Running") + w.floor.down = true + w.ax.finish("wf-test-1-a1", 0, { v: 1 }) + await until("outbox keeps", () => l1.has("outbox-keep", "wf-test-1-a1")) + await l1.crash() + await sleep(7 * SEC) // past the 6 s lease, well inside the 15 s grace + const l2 = start(w, dir) + await sleep(3 * SEC) + w.floor.down = false + w.floor.sweep() // the DO alarm fires + await until("floor done", () => J(w).state === "done") + expect([J(w).result, J(w).attempt]).toEqual(["success", 1]) + expect(J(w).history.some((h) => h.startsWith("requeue"))).toBe(false) + await l2.drain() + }) + + it("F6 ax unreachable: capacity 0 on every poll, held leases still renewed, work resumes when ax returns", async () => { + const w = world(); w.floor.enqueue(job("1")) + const l = start(w, dir) + await until("running", () => w.ax.tasks.get("wf-test-1-a1")?.phase === "Running") + w.ax.up = false + await until("link saw ax down", () => l.has("ax-down")) + const mark = w.floor.calls.length + w.floor.enqueue(job("2")) + await sleep(10 * SEC) // longer than the 6 s lease + if (process.env.LINK_DEBUG) console.log(JSON.stringify({ history: J(w).history, calls: w.floor.calls.slice(mark).map((c) => [c.rpc, c.capacity, c.leaseIds]), logs: l.logs.slice(-15) })) + const during = w.floor.calls.slice(mark).filter((c) => c.rpc === "Lease") + expect(during.length).toBeGreaterThan(0) + expect(during.every((c) => c.capacity === 0 && c.granted === 0)).toBe(true) + expect([J(w).state, J(w, "2").state]).toEqual(["leased", "queued"]) + w.ax.up = true + await until("second job created", () => w.ax.tasks.has("wf-test-2-a1")) + await l.drain() + }) + + it("F7 lease expiry, link back inside the grace: the orphaned lease is re-adopted, same attempt, no second Task", async () => { + const w = world(); w.floor.enqueue(job("1")) + const l1 = start(w, dir) + await until("running", () => w.ax.tasks.get("wf-test-1-a1")?.phase === "Running") + await l1.crash() + await sleep(7 * SEC); w.floor.sweep() + expect([J(w).state, w.floor.wip()]).toEqual(["orphaned", 1]) // L3: never reclaim a live slot + const l2 = start(w, dir) + await until("re-adopted", () => J(w).history.includes("re-adopted")) + w.ax.finish("wf-test-1-a1", 0, { ok: 1 }) + await until("floor done", () => J(w).state === "done") + expect([J(w).result, J(w).attempt, w.ax.updates.get("wf-test-1-a1")]).toEqual(["success", 1, 1]) + await l2.drain() + }) + + it("F8 journal lost: the holder's complete heartbeat omits the orphan, attempt 2 supersedes it, old Task deleted first", async () => { + const w = world(); w.floor.enqueue(job("1")) + const l1 = start(w, dir) + await until("running", () => w.ax.tasks.get("wf-test-1-a1")?.phase === "Running") + await l1.crash() + rmSync(dir, { recursive: true, force: true }) + await sleep(7 * SEC); w.floor.sweep() + const l2 = start(w, dir, {}, "s:fresh-state") + await until("attempt 2 created", () => w.ax.tasks.has("wf-test-1-a2")) + expect(J(w).history).toContain("requeue:omitted") + expect(w.ax.events.indexOf("delete:wf-test-1-a1")).toBeGreaterThanOrEqual(0) + expect(w.ax.events.indexOf("delete:wf-test-1-a1")).toBeLessThan(w.ax.events.indexOf("update:wf-test-1-a2")) + w.ax.tick(); w.ax.finish("wf-test-1-a2", 0, { ok: 2 }) + await until("floor done", () => J(w).state === "done") + expect([J(w).result, J(w).attempt]).toEqual(["success", 2]) + await l2.drain() + }) + + it("F9 link dead past the grace: the alarm requeues; the returning link drops attempt 1 (lost) and runs attempt 2", async () => { + const w = world(); w.floor.enqueue(job("1")) + const l1 = start(w, dir) + await until("running", () => w.ax.tasks.get("wf-test-1-a1")?.phase === "Running") + await l1.crash() + await sleep(7 * SEC); w.floor.sweep() + await sleep(16 * SEC); w.floor.sweep() + expect(J(w).history).toContain("requeue:grace") + const l2 = start(w, dir) + await until("attempt 2 created", () => w.ax.tasks.has("wf-test-1-a2")) + expect(w.ax.deletes).toContain("wf-test-1-a1") + w.ax.tick(); w.ax.finish("wf-test-1-a2", 0, { ok: 2 }) + await until("floor done", () => J(w).state === "done") + expect([J(w).result, J(w).attempt]).toEqual(["success", 2]) + // the late verdict of attempt 1 can never be recorded: the floor fences on the attempt + expect(w.floor.complete({ leaseId: "wf-test-1-a1", attempt: 1, result: "success" })).toEqual({ code: "stale-attempt" }) + await l2.drain() + }) + + it("F10 cancel: carried in the heartbeat reply, DeleteTask, then Complete(cancelled) once the Task is gone", async () => { + const w = world(); w.floor.enqueue(job("1")) + const l = start(w, dir) + await until("running", () => w.ax.tasks.get("wf-test-1-a1")?.phase === "Running") + w.floor.cancel("wf-test-1") + await until("floor done", () => J(w).state === "done") + if (process.env.LINK_DEBUG) console.log(JSON.stringify({ history: J(w).history, events: w.ax.events, phase: w.ax.tasks.get("wf-test-1-a1")?.phase, logs: l.logs })) + expect([J(w).result, J(w).attempt, w.ax.tasks.has("wf-test-1-a1")]).toEqual(["cancelled", 1, false]) + await l.drain() + }) + + it("F11 stock v0.3.0 (no P1), guest mode configured: the Task stays Running after exit; the guest completes with its lease token; the link deletes on lost", async () => { + const w = guestWorld(); w.floor.enqueue(job("1")) + const l = start(w, dir, GUEST) + await until("running", () => w.ax.tasks.get("wf-test-1-a1")?.phase === "Running") + const env = Object.fromEntries(w.ax.tasks.get("wf-test-1-a1")!.task.spec.env.map((e) => [e.name, e.value])) + expect(env.AX_CONWIP_COMPLETE_URL).toBe("http://link.fleet.internal/guest") + expect(Object.values(env).some((v) => v.includes("tok-nas"))).toBe(false) // the fleet bearer never enters the sandbox + w.ax.finish("wf-test-1-a1", 0) + await sleep(5 * SEC) + expect(w.ax.tasks.get("wf-test-1-a1")!.phase).toBe("Running") // FIELD-MAP D3: no terminal phase without P1 + expect(w.floor.guestComplete("wf-test-1-a1", env.AX_CONWIP_LEASE_TOKEN!, "success", { guest: true })).toMatchObject({ duplicate: false }) + expect(w.floor.guestComplete("wf-test-1-a1", env.AX_CONWIP_LEASE_TOKEN!, "success", { guest: true })).toEqual({ refused: "token" }) + await until("link deleted the still-Running Task", () => !w.ax.tasks.has("wf-test-1-a1")) + expect([J(w).result, J(w).output]).toEqual(["success", { guest: true }]) + await l.drain() + }) + + it("F12 stuck Running (no P1, the guest never reports): the floor's deadline cancels it and ax frees the worker", async () => { + const w = guestWorld(); w.floor.enqueue(job("1", { "timeout-minutes": 1 })) // 60 s = 1200 ms here + const l = start(w, dir, GUEST) + await until("running", () => w.ax.tasks.get("wf-test-1-a1")?.phase === "Running") + w.ax.finish("wf-test-1-a1", 0) // exited, invisible to ax without P1 + await until("floor done", () => J(w).state === "done", 8000) + expect([J(w).result, J(w).output]).toEqual(["cancelled", { reason: "deadline" }]) + await until("Task gone", () => !w.ax.tasks.has("wf-test-1-a1")) + await l.drain() + }) + + it("F13 deadline passes while the floor is unreachable: the local backstop deletes the Task and queues the verdict", async () => { + const w = guestWorld(); w.floor.enqueue(job("1", { "timeout-minutes": 1 })) + const l = start(w, dir, GUEST) + await until("running", () => w.ax.tasks.get("wf-test-1-a1")?.phase === "Running") + w.floor.down = true + await until("backstop verdict", () => l.logs.some((x) => x.ev === "verdict" && x.reason === "deadline-exceeded"), 8000) + await until("Task deleted", () => !w.ax.tasks.has("wf-test-1-a1") || w.ax.tasks.get("wf-test-1-a1")!.phase === "Terminating") + await l.crash() + }) + + it("F14 stuck Pending: pre-start failure at the timeout, retried by the floor as attempt 2 superseding attempt 1", async () => { + const w = world(); w.floor.enqueue(job("1")); w.ax.holdPending.add("wf-test-1-a1") + const l = start(w, dir, { pendingTimeoutMs: 200 }) + await until("attempt 2 created", () => w.ax.tasks.has("wf-test-1-a2"), 8000) + expect(J(w).history).toContain("requeue:pre-start/pending-timeout") + expect(w.ax.deletes).toContain("wf-test-1-a1") + w.ax.tick(); w.ax.finish("wf-test-1-a2", 0, { ok: 2 }) + await until("floor done", () => J(w).state === "done") + expect([J(w).result, J(w).attempt]).toEqual(["success", 2]) + await l.drain() + }) + + it("F15 name conflict: a foreign Task with the lease's name is never overwritten (create-only, L4)", async () => { + const w = world(); w.floor.enqueue(job("1")) + w.ax.put({ apiVersion: "ax.io/v1alpha1", kind: "Task", metadata: { name: "wf-test-1-a1", atespace: "fleet" }, spec: { image: "someone-else", command: ["sleep"], env: [] } }) + const l = start(w, dir) + await until("floor done", () => J(w).state === "done") + expect([J(w).result, (J(w).output as { reason: string }).reason]).toEqual(["failure", "pre-start/name-conflict"]) + expect([w.ax.updates.get("wf-test-1-a1"), w.ax.tasks.get("wf-test-1-a1")?.task.spec.image]).toEqual([undefined, "someone-else"]) + await l.drain() + }) + + it("F16 deterministic refusals are final and touch nothing in ax: env over budget, unresolved prompt, prompt too big to inline (B12)", async () => { + const w = world({ cap: 3 }) + w.floor.enqueue(job("1", {}, {}, { schema: { description: "x".repeat(20000) } })) + w.floor.enqueue(job("2", {}, { [A + "prompt-unresolved"]: "no agent() call carries label" })) + w.floor.enqueue(job("3", {}, {}, { prompt: "p".repeat(17000) })) + const l = start(w, dir, { maxInFlight: 3 }) + await until("all done", () => J(w).state === "done" && J(w, "2").state === "done" && J(w, "3").state === "done") + expect([(J(w, "3").output as { reason: string }).reason, J(w, "3").attempt]).toEqual(["pre-start/prompt-file-route-missing", 1]) + expect([(J(w).output as { reason: string }).reason, J(w).attempt]).toEqual(["pre-start/env-over-budget", 1]) + expect([(J(w, "2").output as { reason: string }).reason, J(w, "2").attempt]).toEqual(["pre-start/prompt-unresolved", 1]) + expect(w.ax.updates.size).toBe(0) + await l.drain() + }) + + it("F17 a second replica with the same identity is refused (409) and exits; the first keeps its session", async () => { + const w = world(); w.floor.enqueue(job("1")) + const l1 = start(w, dir) + await until("running", () => w.ax.tasks.get("wf-test-1-a1")?.phase === "Running") + const dir2 = mkdtempSync(join(tmpdir(), "conwip-link-b-")) + const l2 = start(w, dir2, {}, "s:second-replica") + const exit = await l2.exit() + rmSync(dir2, { recursive: true, force: true }) + expect(Exit.isFailure(exit)).toBe(true) + expect(JSON.stringify(exit)).toContain("session-conflict") + w.ax.finish("wf-test-1-a1", 0, { ok: 1 }) + await until("floor done", () => J(w).state === "done") + await l1.drain() + }) + + it("F19 ax lost a Task it had created (Redis loss): infra/task-lost, the floor retries as attempt 2", async () => { + const w = world(); w.floor.enqueue(job("1")) + const l = start(w, dir) + await until("running", () => w.ax.tasks.get("wf-test-1-a1")?.phase === "Running") + w.ax.tasks.delete("wf-test-1-a1") // gone without a DeleteTask: the store forgot it + await until("attempt 2 created", () => w.ax.tasks.has("wf-test-1-a2")) + expect(J(w).history).toContain("requeue:infra/task-lost") + w.ax.tick(); w.ax.finish("wf-test-1-a2", 0, { ok: 2 }) + await until("floor done", () => J(w).state === "done") + expect([J(w).result, J(w).attempt]).toEqual(["success", 2]) + await l.drain() + }) + + it("F18 drain on SIGTERM: stop leasing, flush the outbox, leave running Tasks to be re-adopted", async () => { + const w = world(); w.floor.enqueue(job("1")) + const l = start(w, dir) + await until("running", () => w.ax.tasks.get("wf-test-1-a1")?.phase === "Running") + const exit = await l.drain() + expect(Exit.isSuccess(exit)).toBe(true) + expect([w.ax.tasks.get("wf-test-1-a1")?.phase, J(w).state]).toEqual(["Running", "leased"]) + }) +}) diff --git a/pkgs/substrate-link/src/apps/link/test/review-r1.test.ts b/pkgs/substrate-link/src/apps/link/test/review-r1.test.ts new file mode 100644 index 000000000..5e49eeb77 --- /dev/null +++ b/pkgs/substrate-link/src/apps/link/test/review-r1.test.ts @@ -0,0 +1,391 @@ +// Review round 1 (2026-09-23): the ten findings, each as a regression test that failed before its fix. The scratch +// repros are in /home/tom/today/evals-2026-09-23/link/scratch-r1-{0,1,2}; the log is link/REVIEW-LOG.md. +import { mkdtempSync, rmSync } from "node:fs" +import { tmpdir } from "node:os" +import { join } from "node:path" +import { Deferred, Effect, Fiber } from "effect" +import { afterEach, beforeEach, describe, expect, it } from "vitest" +import { AxError } from "../src/ax.ts" +import type { AgentJob, AxTask } from "../src/contract.ts" +import { rpcFloor } from "../src/floor.ts" +import type { FloorOptions } from "../src/floor.ts" +import { Journal } from "../src/journal.ts" +import { runLink } from "../src/link.ts" +import type { LinkConfig } from "../src/link.ts" +import { FakeAx } from "./fake-ax.ts" +import { FakeFloor } from "./fake-floor.ts" +import type { FloorConfig } from "./fake-floor.ts" +import { ConfigInvalid, readLinkEnv } from "../src/config.ts" +import type { FloorApi } from "../src/floor.ts" +import { axTaskFromGrant } from "../src/jobs.ts" + +const SEC = 20 +const UNAVAILABLE = 14 +const A = "ultracode.mecattaf.dev/" +const JK = `${"cd".repeat(32)}:1` // round 4: FIELD-MAP 5a journal key (jobs.ts refuses a grant without it) +const job = (n: string, spec: Partial = {}): AgentJob => ({ + apiVersion: "ultracode.mecattaf.dev/v1alpha1", kind: "AgentJob", + metadata: { + name: `wf-test-${n}`, + labels: { [A + "run-id"]: "wf-test", [A + "workflow"]: "link-test", [A + "phase-index"]: "1" }, + annotations: { [A + "run-id-raw"]: "wf_test", [A + "label"]: `probe:${n}`, [A + "item-key"]: `wf_test#${n}`, [A + "journal-key"]: JK, [A + "phase-title"]: "Probe" } + }, + spec: { + "runs-on": ["seat:halogen", "runtime:gvisor"], + with: { prompt: `say ${n}`, prompt_ref: { sha256: "ab".repeat(32), bytes: 5, uri: `journal://wf_test/${n}/prompt.md` }, model: "halogen-qwen3.8-flash-next" }, + ...spec + } +}) +const world = (o: Partial = {}) => ({ + floor: new FakeFloor({ cap: 2, leaseSeconds: 6, graceSeconds: 15, pollSeconds: 1, heartbeatSeconds: 2, maxAttempts: 3, secondMs: SEC, tokens: { "tok-nas": "nas-link-1" }, ...o }), + ax: new FakeAx() +}) +type World = ReturnType +const foreign = (name: string): AxTask => ({ apiVersion: "ax.io/v1alpha1", kind: "Task", metadata: { name, atespace: "fleet" }, spec: { image: "someone-else", command: ["true"], env: [] } }) + +interface Extra { fetch?: typeof globalThis.fetch; timeoutsMs?: FloorOptions["timeoutsMs"]; onEndpoint?: (u: string) => void } +function start(w: World, dir: string, over: Partial = {}, x: Extra = {}) { + const logs: Array> = [] + const stop = Effect.runSync(Deferred.make()) + const cfg: LinkConfig = { + holder: "nas-link-1", maxInFlight: 2, servedLabels: ["seat:halogen", "runtime:gvisor"], + shape: { atespace: "fleet", image: "ax-agent", gateway: "halogen", command: () => ["ax-agent", "pi"] }, + completion: "auto", secondMs: SEC, resyncMs: 30, pendingTimeoutMs: 1500, deleteAfterMs: 0, deadlineBackstopMs: 300, + createAttempts: 3, outboxBackoffMs: [20, 100], initialPollSeconds: 1, initialHeartbeatSeconds: 2, fenceTimeoutMs: 300, resultReadTries: 3, ...over + } + const fiber = Effect.runFork(Effect.scoped(Effect.gen(function*() { + const floor = yield* rpcFloor({ url: "http://floor.test", token: "tok-nas", sessionId: `s:${dir}`, fetch: x.fetch ?? w.floor.fetch, ...(x.timeoutsMs ? { timeoutsMs: x.timeoutsMs } : {}) }) + return yield* runLink(cfg, { ax: w.ax, floor, journal: Journal.open(dir), log: (ev, f) => logs.push({ ev, ...f }), stop, floorUrl: "http://floor.test", ...(x.onEndpoint ? { onEndpoint: x.onEndpoint } : {}) }) + }))) + return { + logs, + has: (ev: string, leaseId?: string) => logs.some((l) => l.ev === ev && (leaseId === undefined || l.leaseId === leaseId)), + verdict: (reason: string) => logs.some((l) => l.ev === "verdict" && l.reason === reason), + crash: () => Effect.runPromise(Fiber.interrupt(fiber)), + drain: () => { Effect.runSync(Deferred.succeed(stop, undefined)); return Promise.race([Effect.runPromise(Fiber.await(fiber)), sleep(1500)]) } + } +} +const until = async (what: string, pred: () => boolean, ms = 5000) => { + const t0 = Date.now() + while (!pred()) { if (Date.now() - t0 > ms) throw new Error(`timeout waiting for: ${what}`); await new Promise((r) => setTimeout(r, 5)) } +} +const sleep = (ms: number) => new Promise((r) => setTimeout(r, ms)) +const J = (w: World, n = "1") => w.floor.jobs.get(`wf-test-${n}`)! +const liveOwn = (w: World) => [...w.ax.tasks.keys()].filter((k) => k.startsWith("wf-test-") && w.ax.live(k) && !["Completed", "Failed"].includes(w.ax.tasks.get(k)!.phase)).length + + +const bodyOf = async (input: unknown, init?: RequestInit) => { + const b = init?.body + if (typeof b === "string") return b + if (b instanceof Uint8Array) return new TextDecoder().decode(b) + if (b) return await new Response(b as ConstructorParameters[0]).text() + return (input instanceof Request) ? await input.clone().text() : "" +} +const isComplete = async (input: unknown, init?: RequestInit) => (await bodyOf(input, init)).includes('"Complete"') +const stats = { seq: 1, cap: 2, wip: 0, queued: 0, done: 0 } +const grantOf = (n: string, extra: Record = {}) => ({ + leaseId: `wf-test-${n}-a1`, attempt: 1, job: job(n), + lease: { holderIdentity: "nas-link-1", leaseDurationSeconds: 30, acquireTime: Date.now(), renewTime: Date.now(), leaseTransitions: 0 }, ...extra +}) as any +/** A floor that hands out `grants` once and renews everything: for link-side properties the double cannot stage. */ +const stubFloor = (grants: Array) => { + let sent = false + const completes: Array = [] + const api: FloorApi = { + lease: (p) => Effect.sync(() => { const g = !sent && p.capacity > 0 ? grants : []; if (g.length) sent = true; return { grants: g, stats, nextPollSeconds: 1, heartbeatSeconds: 2 } }), + heartbeat: (p) => Effect.succeed({ renewed: [...p.leaseIds], lost: [], cancelRequested: [], stats }), + complete: (p) => Effect.sync(() => { completes.push(p); return { duplicate: false, stats } }) + } + return { api, completes } +} +const cfgOf = (over: Partial = {}): LinkConfig => ({ + holder: "nas-link-1", maxInFlight: 2, servedLabels: ["seat:halogen", "runtime:gvisor"], + shape: { atespace: "fleet", image: "ax-agent", gateway: "halogen", command: () => ["ax-agent", "pi"] }, + completion: "auto", secondMs: SEC, resyncMs: 30, pendingTimeoutMs: 60_000, deleteAfterMs: 60_000, deadlineBackstopMs: 300, + createAttempts: 3, outboxBackoffMs: [20, 100], initialPollSeconds: 1, initialHeartbeatSeconds: 2, fenceTimeoutMs: 300, resultReadTries: 3, ...over +}) +const runStub = async (cfg: LinkConfig, ax: FakeAx, floor: FloorApi, ms: number) => { + const d = mkdtempSync(join(tmpdir(), "conwip-link-r1s-")) + const logs: Array> = [] + const stop = Effect.runSync(Deferred.make()) + const f = Effect.runFork(runLink(cfg, { ax, floor, journal: Journal.open(d), log: (ev, x) => logs.push({ ev, ...x }), stop })) + await sleep(ms) + Effect.runSync(Deferred.succeed(stop, undefined)); await Effect.runPromise(Fiber.await(f)); rmSync(d, { recursive: true, force: true }) + return logs +} + +describe("review round 1: fixed", () => { + let dir: string + let worlds: Array = [] + const W = (o: Partial = {}) => { const w = world(o); worlds.push(w); return w } + beforeEach(() => { dir = mkdtempSync(join(tmpdir(), "conwip-link-r1-")) }) + afterEach(() => { for (const w of worlds) w.floor.close(); worlds = []; rmSync(dir, { recursive: true, force: true }) }) + + // ------------------------------------------------------------------ findings 1 and 2: one verdict, one application + it("R1-1 no fault: every verdict costs exactly one Complete (single-flight outbox)", async () => { + const w = W({ cap: 8 }) + for (let i = 1; i <= 8; i++) w.floor.enqueue(job(String(i))) + const l = start(w, dir, { maxInFlight: 8, resyncMs: 5 }) + await until("all running", () => [1, 2, 3, 4, 5, 6, 7, 8].every((i) => w.ax.tasks.get(`wf-test-${i}-a1`)?.phase === "Running")) + for (let i = 1; i <= 8; i++) w.ax.finish(`wf-test-${i}-a1`, 0, { i }) + await until("all done", () => [1, 2, 3, 4, 5, 6, 7, 8].every((i) => J(w, String(i)).state === "done")) + await sleep(100) + const per = new Map() + for (const c of w.floor.calls.filter((c) => c.rpc === "Complete")) per.set(c.leaseIds![0]!, (per.get(c.leaseIds![0]!) ?? 0) + 1) + expect([...per.values()]).toEqual([1, 1, 1, 1, 1, 1, 1, 1]) + expect(l.logs.filter((x) => x.ev === "complete").length).toBe(8) + await l.drain() + }) + + it("R1-1 (D5) one retryable failure with a budget of two releases leases attempt 2", async () => { + const w = W({ infraAttempts: 2 }); w.floor.enqueue(job("1")); w.ax.holdPending.add("wf-test-1-a1") + const l = start(w, dir, { pendingTimeoutMs: 150 }) + await until("attempt 2 leased", () => J(w).state === "leased" && J(w).attempt === 2, 4000) + expect(J(w).infraSpent).toBe(1) + expect(J(w).history).not.toContain("withdrawn:a2") + await l.drain() + }) + + it("R1-2 (D2) a Complete whose reply is lost after the floor committed is resent and answered duplicate, not applied twice", async () => { + const w = W({ infraAttempts: 2 }); w.floor.enqueue(job("1")); w.ax.holdPending.add("wf-test-1-a1") + let dropped = 0 + const fetch: typeof globalThis.fetch = async (input, init) => { + const c = await isComplete(input, init) + const res = await w.floor.fetch(input, init) + if (c && dropped === 0) { dropped++; throw new TypeError("fetch failed: reply lost after the floor committed") } + return res + } + const l = start(w, dir, { pendingTimeoutMs: 150 }, { fetch }) + await until("attempt 2 leased", () => J(w).state === "leased" && J(w).attempt === 2, 4000) + await until("the resend is acknowledged", () => l.logs.some((x) => x.ev === "complete" && x.leaseId === "wf-test-1-a1")) + expect(dropped).toBe(1) + expect(J(w).infraSpent).toBe(1) + expect(l.logs.find((x) => x.ev === "complete" && x.leaseId === "wf-test-1-a1")!.duplicate).toBe(true) + await l.drain() + }) + + // ------------------------------------------------------------------ finding 3: the create window is write-ahead + it("R1-3 (D1) crash after UpdateTask landed, before `created`, then a cancel: the Task is deleted and NotFound precedes `cancelled`", async () => { + const w = W(); w.floor.enqueue(job("1")); w.ax.hangAfterCreate = true + const l1 = start(w, dir) + await until("Task written to ax", () => w.ax.tasks.has("wf-test-1-a1")) + await l1.crash() + w.ax.hangAfterCreate = false + w.floor.cancel("wf-test-1") + const l2 = start(w, dir) + await until("floor done", () => J(w).state === "done") + expect(J(w).result).toBe("cancelled") + expect(w.ax.deletes).toContain("wf-test-1-a1") + expect(w.ax.tasks.has("wf-test-1-a1")).toBe(false) // NotFound before the verdict (reconcile reports on gone) + const evs = l2.logs.filter((x) => x.leaseId === "wf-test-1-a1" || x.task === "wf-test-1-a1").map((x) => x.ev) + expect(evs.indexOf("delete")).toBeLessThan(evs.indexOf("verdict")) + await l2.drain() + expect(Journal.open(dir).recs.has("wf-test-1-a1")).toBe(false) // compacted only once ax is clean + }) + + it("R1-3 (D1 lost variant) crash in the create window, the floor gives up on the job: `lost` deletes the maybe-created Task", async () => { + const w = W({ infraAttempts: 1 }); w.floor.enqueue(job("1")); w.ax.hangAfterCreate = true + const l1 = start(w, dir) + await until("Task written to ax", () => w.ax.tasks.has("wf-test-1-a1")) + await l1.crash() + w.ax.hangAfterCreate = false + await sleep(7 * SEC); w.floor.sweep(); await sleep(16 * SEC); w.floor.sweep() + expect(J(w).state).toBe("done") // budget spent at the first release: no re-grant + const l2 = start(w, dir) + await until("lost", () => l2.has("lost", "wf-test-1-a1")) + await until("Task gone", () => !w.ax.tasks.has("wf-test-1-a1")) + expect(w.ax.deletes).toContain("wf-test-1-a1") + await l2.drain() + }) + + // ------------------------------------------------------------------ findings 4 and 8: B15 + it("R1-8 (B15 deterministic) the first Complete after the floor returns is lost in transit: no Lease runs before it lands", async () => { + const w = W(); w.floor.enqueue(job("1")) + let failCompletes = 0 + const fetch: typeof globalThis.fetch = async (input, init) => { + if (failCompletes > 0 && await isComplete(input, init)) { failCompletes--; throw new TypeError("fetch failed") } + return w.floor.fetch(input, init) + } + const l = start(w, dir, {}, { fetch }) + await until("running", () => w.ax.tasks.get("wf-test-1-a1")?.phase === "Running") + w.floor.down = true + w.ax.finish("wf-test-1-a1", 0, { v: 1 }) + await until("outbox keeps", () => l.has("outbox-keep", "wf-test-1-a1")) + await until("requeued", () => J(w).history.includes("requeue:grace"), 3000) + failCompletes = 1 + const mark = w.floor.calls.length + w.floor.down = false + await until("floor done", () => J(w).state === "done", 4000) + expect([J(w).result, J(w).output, J(w).attempt]).toEqual(["success", { v: 1 }, 1]) + // round 3 (finding 12): the gate is asserted on the floor's side, not on the log line + const after = w.floor.calls.slice(mark), landed = after.findIndex((c) => c.rpc === "Complete") + expect(after.slice(0, landed).filter((c) => c.rpc === "Lease").map((c) => c.capacity).filter((c) => c !== 0)).toEqual([]) + expect(J(w).history).not.toContain("leased:a2") + expect(w.ax.updates.has("wf-test-1-a2")).toBe(false) + expect(l.has("complete-dropped", "wf-test-1-a1")).toBe(false) + await l.drain() + }) + + it("R1-4 (D3) Complete answers 503 while Lease answers: the gate asks for nothing until the success lands", async () => { + const w = W(); w.floor.enqueue(job("1")) + let failComplete = false + const fetch: typeof globalThis.fetch = async (input, init) => { + if (failComplete && await isComplete(input, init)) return new Response("upstream timeout", { status: 503 }) + return w.floor.fetch(input, init) + } + const l = start(w, dir, {}, { fetch }) + await until("running", () => w.ax.tasks.get("wf-test-1-a1")?.phase === "Running") + w.floor.down = true + w.ax.finish("wf-test-1-a1", 0, { answer: 42 }) + await until("verdict written", () => l.has("verdict", "wf-test-1-a1")) + await until("floor requeued attempt 2", () => J(w).state === "queued" && J(w).attempt === 2, 4000) + failComplete = true + const mark = w.floor.calls.length + w.floor.down = false + await sleep(400) // several polls while only Complete fails + expect(l.has("lease-gated-by-outbox")).toBe(true) + // round 3 (finding 12): every Lease while the success is undelivered asks for nothing, and nothing is leased + const leases = w.floor.calls.slice(mark).filter((c) => c.rpc === "Lease") + expect([leases.length > 0, leases.filter((c) => c.capacity !== 0).length, J(w).history.includes("leased:a2")]).toEqual([true, 0, false]) + expect(w.ax.tasks.has("wf-test-1-a2")).toBe(false) + // a1 may be deleted as `lost` (its verdict is already journaled); it is never fenced as `superseded` + expect(l.logs.some((x) => x.ev === "delete" && x.task === "wf-test-1-a1" && x.why === "superseded")).toBe(false) + failComplete = false + await until("floor done", () => J(w).state === "done", 4000) + expect([J(w).result, J(w).output, J(w).attempt]).toEqual(["success", { answer: 42 }, 1]) + await l.drain() + }) + + it("R1-4 (D3 at dispatch) a grant that supersedes an undelivered success is withdrawn, never fenced or created", async () => { + const w = W(); w.floor.enqueue(job("1")) + let failComplete = false + const fetch: typeof globalThis.fetch = async (input, init) => { + if (failComplete && await isComplete(input, init)) return new Response("upstream timeout", { status: 503 }) + return w.floor.fetch(input, init) + } + const l = start(w, dir, {}, { fetch, }) + await until("running", () => w.ax.tasks.get("wf-test-1-a1")?.phase === "Running") + w.floor.down = true + w.ax.finish("wf-test-1-a1", 0, { answer: 42 }) + await until("verdict written", () => l.has("verdict", "wf-test-1-a1")) + await until("floor requeued attempt 2", () => J(w).state === "queued" && J(w).attempt === 2, 4000) + failComplete = true + w.floor.o.overGrant = 1 // a floor that grants on a capacity-0 Lease: the link's own defence must hold + w.floor.down = false + await until("a2 granted", () => l.has("leased", "wf-test-1-a2"), 4000) + w.floor.o.overGrant = 0 + await until("dispatch waits for the verdict", () => l.has("supersede-waits-for-verdict", "wf-test-1-a2")) + failComplete = false + await until("floor done", () => J(w).state === "done", 4000) + expect([J(w).result, J(w).output, J(w).attempt]).toEqual(["success", { answer: 42 }, 1]) + expect(J(w).history).toContain("withdrawn:a2") + await until("a2 withdrawn locally", () => l.has("withdrawn", "wf-test-1-a2")) + expect(w.ax.updates.has("wf-test-1-a2")).toBe(false) + // the finished Task was not fenced before its verdict (a `lost` delete is fine: the verdict is already journaled) + expect(l.logs.some((x) => x.ev === "delete" && x.task === "wf-test-1-a1" && x.why === "superseded")).toBe(false) + await l.drain() + }) + + // ------------------------------------------------------------------ finding 5: supersedes is scoped to own Tasks + it("R1-5 supersedes naming a Task this link never created: no DeleteTask, the grant is refused", async () => { + const ax = new FakeAx() + ax.put(foreign("someone-elses-task"), "Running") + const f = stubFloor([grantOf("1", { attempt: 2, leaseId: "wf-test-1-a2", supersedes: ["someone-elses-task"] })]) + const logs = await runStub(cfgOf(), ax, f.api, 600) + expect(ax.deletes).toEqual([]) + expect(ax.live("someone-elses-task")).toBe(true) + expect(ax.tasks.has("wf-test-1-a2")).toBe(false) + expect(f.completes.map((c) => [c.leaseId, c.output?.reason])).toEqual([["wf-test-1-a2", "pre-start/supersedes-foreign"]]) + expect(logs.some((x) => x.ev === "supersedes-foreign")).toBe(true) + }) + + it("R1-5 an earlier attempt of the same job that carries another holder's mark is not deleted either", async () => { + const ax = new FakeAx() + const built = axTaskFromGrant(grantOf("1"), { atespace: "fleet", image: "ax-agent", gateway: "halogen", command: () => ["ax-agent", "pi"], holder: "other-link" }) + if (built._tag !== "ok") throw new Error("build") + ax.put(built.task, "Running") + const f = stubFloor([grantOf("1", { attempt: 2, leaseId: "wf-test-1-a2", supersedes: ["wf-test-1-a1"] })]) + await runStub(cfgOf(), ax, f.api, 600) + expect(ax.deletes).toEqual([]) + expect(f.completes.map((c) => c.output?.reason)).toEqual(["pre-start/supersedes-foreign"]) + }) + + // ------------------------------------------------------------------ finding 6: the gate is checked per UpdateTask + it("R1-6 the Gateway disappears during the UpdateTask retry window: nothing is created while it is missing", async () => { + const ax = new FakeAx() + let failed = false + const orig = ax.createTask + ax.createTask = (task) => { if (!failed) { failed = true; ax.gateways.delete("halogen"); return Effect.fail(new AxError(UNAVAILABLE, "transient")) } return orig(task) } + const f = stubFloor([grantOf("3")]) + const logs = await runStub(cfgOf(), ax, f.api, 1000) + expect(logs.some((x) => x.ev === "gateway-missing")).toBe(true) + expect(ax.tasks.has("wf-test-3-a1")).toBe(false) + expect(logs.some((x) => x.ev === "gate-closed-before-create")).toBe(true) + }) + + it("R1-6 ax returns without P1 during the retry window: nothing is created in p1 mode", async () => { + const ax = new FakeAx() + let failed = false + const orig = ax.createTask + ax.createTask = (task) => { + if (!failed) { failed = true; ax.up = false; setTimeout(() => { ax.p1 = false; ax.up = true }, 60); return Effect.fail(new AxError(UNAVAILABLE, "transient")) } + return orig(task) + } + const f = stubFloor([grantOf("4")]) + const logs = await runStub(cfgOf(), ax, f.api, 1200) + expect(logs.some((x) => x.ev === "p1-missing")).toBe(true) + expect(ax.tasks.has("wf-test-4-a1")).toBe(false) + }) + + // ------------------------------------------------------------------ finding 7: guest mode fails closed + it("R1-7 guest mode with no leaseToken in the grant: refused before UpdateTask, logged once", async () => { + const ax = new FakeAx(); ax.p1 = false + const f = stubFloor([grantOf("2")]) + const logs = await runStub(cfgOf({ completion: "guest", shape: { atespace: "fleet", image: "ax-agent", gateway: "halogen", command: () => ["ax-agent", "pi"], completeUrl: "http://link.fleet.internal/guest" } }), ax, f.api, 600) + expect(ax.tasks.has("wf-test-2-a1")).toBe(false) + expect(f.completes.map((c) => c.output?.reason)).toEqual(["pre-start/lease-token-missing"]) + expect(logs.filter((x) => x.ev === "guest-token-missing").length).toBe(1) + }) + + // ------------------------------------------------------------------ finding 9: one bad grant is refused alone + it("R1-9 one undecodable grant next to a good one: the good one runs, the bad one is refused, no crash loop", async () => { + const w = W({ skewedEncode: true }) // the floor sends what the link's contract calls malformed + w.floor.enqueue(job("good")) + w.floor.enqueue(job("bad", { "timeout-minutes": 1.5 } as any)) // a float where the contract says Schema.Int + const l = start(w, dir) + await until("good created", () => w.ax.tasks.has("wf-test-good-a1")) + await until("bad refused", () => w.floor.jobs.get("wf-test-bad")!.state === "done") + expect((w.floor.jobs.get("wf-test-bad")!.output as any).reason).toBe("pre-start/invalid-spec") + expect(l.has("grant-invalid", "wf-test-bad-a1")).toBe(true) + await l.drain() + const j = Journal.open(dir) + expect(j.pendingLeaseKey).toBeUndefined() + expect(j.invalid.size).toBe(0) // told and compacted + }) + + it("R1-9 a Lease reply the floor cannot even encode is a typed transient error: the link stays up and heartbeats", async () => { + const w = W() // strict floor: its own encode of the float fails + w.floor.enqueue(job("bad", { "timeout-minutes": 1.5 } as any)) + const l = start(w, dir) + await until("lease errors", () => l.logs.filter((x) => x.ev === "lease-error").length >= 2) + await until("still heartbeating", () => w.floor.calls.filter((c) => c.rpc === "Heartbeat").length >= 2) + expect(l.logs.some((x) => String(x.ev).endsWith("-died"))).toBe(false) + expect(await l.drain()).toBeDefined() // exits on the stop signal, not on a defect + }) + + // ------------------------------------------------------------------ finding 10: configuration fails closed + it("R1-10 numeric and enum configuration is validated: a typo is ConfigInvalid naming the key, never a NaN interval", () => { + const bad = (env: Record) => { try { readLinkEnv(env); return undefined } catch (e) { return e instanceof ConfigInvalid ? e.key : "other" } } + expect(bad({ LINK_RESYNC_SECONDS: "15s" })).toBe("LINK_RESYNC_SECONDS") + expect(bad({ LINK_MAX_IN_FLIGHT: "0" })).toBe("LINK_MAX_IN_FLIGHT") + expect(bad({ LINK_CREATE_ATTEMPTS: "NaN" })).toBe("LINK_CREATE_ATTEMPTS") + expect(bad({ LINK_PENDING_TIMEOUT_SECONDS: "1.5" })).toBe("LINK_PENDING_TIMEOUT_SECONDS") + expect(bad({ LINK_COMPLETION: "gues" })).toBe("LINK_COMPLETION") + expect(bad({ LINK_SEAT_COMMANDS: "{halogen:[ax-agent]}" })).toBe("LINK_SEAT_COMMANDS") + expect(bad({ LINK_SEAT_COMMANDS: '{"halogen":[]}' })).toBe("LINK_SEAT_COMMANDS") + expect(bad({ LINK_OUTBOX_BACKOFF_MIN_SECONDS: "900", LINK_OUTBOX_BACKOFF_MAX_SECONDS: "60" })).toBe("LINK_OUTBOX_BACKOFF_MAX_SECONDS") + const ok = readLinkEnv({}) + expect([ok.resyncSeconds, ok.maxInFlight, ok.createAttempts, ok.completion, ok.seatCommands]).toEqual([15, 2, 7, "auto", { halogen: ["ax-agent", "pi"] }]) + try { readLinkEnv({ LINK_RESYNC_SECONDS: "secret-looking-value" }) } catch (e) { expect(String((e as Error).message)).not.toContain("secret-looking-value") } + }) +}) diff --git a/pkgs/substrate-link/src/apps/link/test/review-r2.test.ts b/pkgs/substrate-link/src/apps/link/test/review-r2.test.ts new file mode 100644 index 000000000..710d332c5 --- /dev/null +++ b/pkgs/substrate-link/src/apps/link/test/review-r2.test.ts @@ -0,0 +1,419 @@ +// Review round 2 (2026-09-23): the twelve findings, each as a regression test. The scratch repros they restate are in +// /home/tom/today/evals-2026-09-23/link/scratch-r2-{0,1,2}; they failed against c5ed453. The log is link/REVIEW-LOG.md. +import { mkdtempSync, rmSync } from "node:fs" +import { tmpdir } from "node:os" +import { join } from "node:path" +import { Deferred, Effect, Fiber } from "effect" +import { afterEach, beforeEach, describe, expect, it } from "vitest" +import { cidrIsOpen, gatewayAllowsAll } from "../src/ax.ts" +import type { AgentJob } from "../src/contract.ts" +import { rpcFloor } from "../src/floor.ts" +import { Journal } from "../src/journal.ts" +import { isFleetInternalUrl, runLink } from "../src/link.ts" +import type { LinkConfig } from "../src/link.ts" +import { FakeAx } from "./fake-ax.ts" +import { FakeFloor } from "./fake-floor.ts" +import type { FloorConfig } from "./fake-floor.ts" + +const SEC = 20 +const A = "ultracode.mecattaf.dev/" +const JK = `${"cd".repeat(32)}:1` // round 4: FIELD-MAP 5a journal key (jobs.ts refuses a grant without it) +const job = (n: string): AgentJob => ({ + apiVersion: "ultracode.mecattaf.dev/v1alpha1", kind: "AgentJob", + metadata: { name: `wf-test-${n}`, labels: { [A + "run-id"]: "wf-test", [A + "workflow"]: "link-test", [A + "phase-index"]: "1" }, + annotations: { [A + "run-id-raw"]: "wf_test", [A + "label"]: `probe:${n}`, [A + "item-key"]: `wf_test#${n}`, [A + "journal-key"]: JK, [A + "phase-title"]: "Probe" } }, + spec: { "runs-on": ["seat:halogen", "runtime:gvisor"], + with: { prompt: `say ${n}`, prompt_ref: { sha256: "ab".repeat(32), bytes: 5, uri: `journal://wf_test/${n}/prompt.md` }, model: "halogen-qwen3.8-flash-next" } } +}) +const world = (o: Partial = {}) => ({ + floor: new FakeFloor({ cap: 2, leaseSeconds: 6, graceSeconds: 15, pollSeconds: 1, heartbeatSeconds: 2, maxAttempts: 3, secondMs: SEC, tokens: { "tok-nas": "nas-link-1" }, ...o }), + ax: new FakeAx() +}) +type World = ReturnType +function start(w: World, dir: string, over: Partial = {}, fetch?: typeof globalThis.fetch) { + const logs: Array> = [] + const stop = Effect.runSync(Deferred.make()) + const cfg: LinkConfig = { + holder: "nas-link-1", maxInFlight: 2, servedLabels: ["seat:halogen", "runtime:gvisor"], + shape: { atespace: "fleet", image: "ax-agent", gateway: "halogen", command: () => ["ax-agent", "pi"] }, + completion: "auto", secondMs: SEC, resyncMs: 30, pendingTimeoutMs: 1500, deleteAfterMs: 0, deadlineBackstopMs: 300, + createAttempts: 3, outboxBackoffMs: [20, 100], initialPollSeconds: 1, initialHeartbeatSeconds: 2, fenceTimeoutMs: 300, resultReadTries: 3, ...over + } + const fiber = Effect.runFork(Effect.scoped(Effect.gen(function*() { + const floor = yield* rpcFloor({ url: "http://floor.test", token: "tok-nas", sessionId: `s:${dir}`, fetch: fetch ?? w.floor.fetch }) + return yield* runLink(cfg, { ax: w.ax, floor, journal: Journal.open(dir), log: (ev, f) => logs.push({ t: Date.now(), ev, ...f }), stop, floorUrl: "http://floor.test" }) + }))) + return { + logs, fiber, + has: (ev: string, leaseId?: string) => logs.some((l) => l.ev === ev && (leaseId === undefined || l.leaseId === leaseId)), + trace: () => logs.filter((x) => typeof x.leaseId === "string" || typeof x.task === "string").map((x) => `${x.ev}:${x.leaseId ?? x.task}${x.why ? ":" + x.why : ""}`), + crash: () => Effect.runPromise(Fiber.interrupt(fiber)), + drain: () => { Effect.runSync(Deferred.succeed(stop, undefined)); return Promise.race([Effect.runPromise(Fiber.await(fiber)), sleep(1500)]) } + } +} +const until = async (what: string, pred: () => boolean, ms = 5000) => { + const t0 = Date.now() + while (!pred()) { if (Date.now() - t0 > ms) throw new Error(`timeout waiting for: ${what}`); await new Promise((r) => setTimeout(r, 5)) } +} +const sleep = (ms: number) => new Promise((r) => setTimeout(r, ms)) +const J = (w: World, n = "1") => w.floor.jobs.get(`wf-test-${n}`)! +const bodyOf = async (input: unknown, init?: RequestInit) => { + const b = init?.body + if (typeof b === "string") return b + if (b instanceof Uint8Array) return new TextDecoder().decode(b) + if (b) return await new Response(b as ConstructorParameters[0]).text() + return (input instanceof Request) ? await input.clone().text() : "" +} +const rpcOf = async (input: unknown, init?: RequestInit) => { const b = await bodyOf(input, init); return b.includes('"Complete"') ? "Complete" : b.includes('"Lease"') ? "Lease" : b.includes('"Heartbeat"') ? "Heartbeat" : "?" } +/** A capacity race: Failed with ResourceExhausted, which the link maps to retryable pre-start/resource-exhausted. */ +const failRetryable = (w: World, name: string) => { + const t = w.ax.tasks.get(name)! + t.exited = true; t.phase = "Failed" + t.conditions = [{ type: "Ready", status: "False", reason: "ResourceExhausted", message: "ResourceExhausted: no free worker" }] +} +const why = (w: World, l: ReturnType) => JSON.stringify({ floor: J(w).history, link: l.trace() }) + +// A floor stub for the fail-closed tests (scratch-r2-1/repro.mts): hands out its grants once, renews everything. +const stats = { seq: 1, cap: 2, wip: 0, queued: 0, done: 0 } +const grantOf = (n: string, extra: Record = {}) => ({ leaseId: `wf-test-${n}-a1`, attempt: 1, job: job(n), + lease: { holderIdentity: "nas-link-1", leaseDurationSeconds: 30, acquireTime: Date.now(), renewTime: Date.now(), leaseTransitions: 0 }, ...extra }) +const stubFloor = (grants: Array, gate: () => boolean = () => true) => { + let sent = false + return { + lease: (p: any) => Effect.sync(() => { const g = !sent && p.capacity > 0 && gate() ? grants : []; if (g.length) sent = true; return { grants: g, invalid: [], stats, nextPollSeconds: 1, heartbeatSeconds: 2 } }), + heartbeat: (p: any) => Effect.succeed({ renewed: [...p.leaseIds], lost: [], cancelRequested: [], stats }), + complete: (_p: any) => Effect.succeed({ duplicate: false, stats }) + } +} +const runStub = async (over: Partial, ax: FakeAx, floor: any, ms: number) => { + const dir = mkdtempSync(join(tmpdir(), "conwip-link-r2s-")) + const logs: Array> = [] + const stop = Effect.runSync(Deferred.make()) + const cfg: LinkConfig = { holder: "nas-link-1", maxInFlight: 2, servedLabels: ["seat:halogen", "runtime:gvisor"], + shape: { atespace: "fleet", image: "ax-agent", gateway: "halogen", command: () => ["ax-agent", "pi"] }, + completion: "auto", secondMs: SEC, resyncMs: 30, pendingTimeoutMs: 60_000, deleteAfterMs: 60_000, deadlineBackstopMs: 300, + createAttempts: 3, outboxBackoffMs: [20, 100], initialPollSeconds: 1, initialHeartbeatSeconds: 2, ...over } + const f = Effect.runFork(runLink(cfg, { ax, floor, journal: Journal.open(dir), log: (ev, x) => logs.push({ t: Date.now(), ev, ...x }), stop })) + await sleep(ms) + Effect.runSync(Deferred.succeed(stop, undefined)); await Effect.runPromise(Fiber.await(f)); rmSync(dir, { recursive: true, force: true }) + return logs +} + +describe("review round 2: fixed", () => { + let dir: string + let worlds: Array = [] + const W = (o: Partial = {}) => { const w = world(o); worlds.push(w); return w } + beforeEach(() => { dir = mkdtempSync(join(tmpdir(), "conwip-link-r2-")) }) + afterEach(() => { for (const w of worlds) w.floor.close(); worlds = []; rmSync(dir, { recursive: true, force: true }) }) + + // ---------------------------------------------------------------- R2-1: heartbeat `lost` and the janitor read first + it("R2-1a kill -9, the agent finishes, the floor requeues past grace, restart while ax is down: the success is delivered, a2 never runs", async () => { + const w = W(); w.floor.enqueue(job("1")) + const l1 = start(w, dir) + await until("running", () => w.ax.tasks.get("wf-test-1-a1")?.phase === "Running") + await l1.crash() + w.ax.finish("wf-test-1-a1", 0, { answer: 42 }) + await until("requeued by the alarm", () => J(w).history.includes("requeue:grace"), 3000) + w.ax.up = false + const l2 = start(w, dir) + await until("heartbeat says lost", () => l2.has("lost", "wf-test-1-a1")) + w.ax.up = true + await until("floor settles", () => J(w).state === "done" || w.ax.updates.has("wf-test-1-a2"), 4000) + await sleep(200) + expect(w.ax.updates.has("wf-test-1-a2"), why(w, l2)).toBe(false) + expect([J(w).result, J(w).output, J(w).attempt]).toEqual(["success", { answer: 42 }, 1]) + expect(l2.logs.findIndex((x) => x.ev === "verdict")).toBeLessThan(l2.logs.findIndex((x) => x.ev === "delete" && x.task === "wf-test-1-a1")) + await l2.drain() + }) + + it("R2-1b no restart: the agent finishes as the partition heals and the heartbeat's lost beats the resync: the success is delivered", async () => { + const w = W(); w.floor.enqueue(job("1")) + const l = start(w, dir, { resyncMs: 300 }) + await until("running", () => w.ax.tasks.get("wf-test-1-a1")?.phase === "Running") + w.floor.down = true + await until("requeued by the alarm", () => J(w).history.includes("requeue:grace"), 3000) + w.ax.finish("wf-test-1-a1", 0, { answer: 42 }) + w.floor.down = false + await until("floor settles", () => J(w).state === "done" || w.ax.updates.has("wf-test-1-a2"), 4000) + await sleep(200) + expect(w.ax.updates.has("wf-test-1-a2"), why(w, l)).toBe(false) + expect([J(w).result, J(w).output, J(w).attempt]).toEqual(["success", { answer: 42 }, 1]) + await l.drain() + }) + + // ---------------------------------------------------------------- R2-2: the supersede fence reads first + it("R2-2 the Lease wins the race (heartbeats failing): a2's fence reads a1's Completed result, the floor withdraws a2", async () => { + const w = W(); w.floor.enqueue(job("1")) + let failHeartbeat = false + const fetch: typeof globalThis.fetch = async (input, init) => { + if (failHeartbeat && await rpcOf(input, init) === "Heartbeat") throw new TypeError("fetch failed") + return w.floor.fetch(input, init) + } + const l = start(w, dir, { resyncMs: 300 }, fetch) + await until("running", () => w.ax.tasks.get("wf-test-1-a1")?.phase === "Running") + w.floor.down = true + await until("requeued by the alarm", () => J(w).history.includes("requeue:grace"), 3000) + failHeartbeat = true + w.ax.finish("wf-test-1-a1", 0, { answer: 42 }) + w.floor.down = false + await until("floor settles", () => J(w).state === "done" || w.ax.updates.has("wf-test-1-a2"), 4000) + failHeartbeat = false + await sleep(200) + expect(w.ax.updates.has("wf-test-1-a2"), why(w, l)).toBe(false) + expect([J(w).result, J(w).output, J(w).attempt]).toEqual(["success", { answer: 42 }, 1]) + await l.drain() + }) + + // ---------------------------------------------------------------- R2-3: one verdict never wedges the link + it("R2-3a a result over the link's output cap is refused agent/output-too-large; later jobs are leased", async () => { + const w = W({ cap: 4 }); w.floor.enqueue(job("1")) + const l = start(w, dir, { maxInFlight: 2, outboxBackoffMs: [20, 60] }) + await until("a1 running", () => w.ax.tasks.get("wf-test-1-a1")?.phase === "Running") + w.ax.finish("wf-test-1-a1", 0, { report: "x".repeat(2 * 1024 * 1024 + 1000) }) + await until("verdict written", () => l.has("verdict", "wf-test-1-a1")) + w.floor.enqueue(job("2")); w.floor.enqueue(job("3")) + await until("job 2 leased", () => J(w, "2").state !== "queued", 3000) + expect([J(w).result, (J(w).output as any)?.reason]).toEqual(["failure", "agent/output-too-large"]) + await l.drain() + }) + + it("R2-3b a verdict the floor answers 500 every time is replaced by infra/verdict-undeliverable, and never gates Lease", async () => { + const w = W({ cap: 4 }); w.floor.enqueue(job("1")) + let refused = 0 + const fetch: typeof globalThis.fetch = async (input, init) => { + const b = await bodyOf(input, init) + if (b.includes('"Complete"') && b.includes('"success"')) { refused++; return new Response("internal error", { status: 500 }) } + return w.floor.fetch(input, init) + } + const l = start(w, dir, { outboxBackoffMs: [20, 60], verdictAttempts: 4 }, fetch) + await until("a1 running", () => w.ax.tasks.get("wf-test-1-a1")?.phase === "Running") + w.ax.finish("wf-test-1-a1", 0, { answer: 42 }) + await until("verdict written", () => l.has("verdict", "wf-test-1-a1")) + w.floor.enqueue(job("2")) + await until("job 2 leased", () => J(w, "2").state !== "queued", 3000) + await until("job 1 closed", () => J(w).state === "done", 3000) + expect([J(w).result, (J(w).output as any)?.reason, refused]).toEqual(["failure", "infra/verdict-undeliverable", 4]) + expect(l.has("verdict-replaced", "wf-test-1-a1")).toBe(true) + await l.drain() + }) + + it("R2-3c an RPC Defect reply (SQLITE_TOOBIG) is a server-error, not an encode defect", async () => { + const fetch: typeof globalThis.fetch = async (input, init) => { + const req = JSON.parse(await bodyOf(input, init)), id = (Array.isArray(req) ? req[0] : req).id + return new Response(JSON.stringify([{ _tag: "Exit", requestId: id, exit: { _tag: "Failure", cause: [{ _tag: "Die", defect: { message: "string or blob too big: SQLITE_TOOBIG" } }] } }]), { status: 200, headers: { "content-type": "application/json" } }) + } + const e = await Effect.runPromise(Effect.scoped(Effect.gen(function*() { + const floor = yield* rpcFloor({ url: "http://floor.test", token: "t", sessionId: "s", fetch }) + return yield* floor.complete({ leaseId: "x-a1", attempt: 1, result: "success", output: {} }).pipe(Effect.flip) + }))) + expect([e._tag, (e as any).kind, /SQLITE_TOOBIG/.test((e as any).message ?? "")]).toEqual(["FloorError", "server-error", true]) + }) + + it("R2-3d Complete classification: 413 and 422 drop the verdict, 404 is fatal misrouted (verdict kept), 500 is server-error, 503 transient", async () => { + const out: Record = {} + for (const status of [413, 422, 404, 500, 503, 429]) { + const e = await Effect.runPromise(Effect.scoped(Effect.gen(function*() { + const floor = yield* rpcFloor({ url: "http://floor.test", token: "t", sessionId: "s", fetch: async () => new Response("x", { status, headers: { "retry-after": "7" } }) }) + return yield* floor.complete({ leaseId: "x-a1", attempt: 1, result: "success" }).pipe(Effect.flip) + }))) + out[status] = e._tag === "CompleteRefused" ? `refused:${e.code}` : `${e.kind}:${e.fatal}${e.kind === "rate-limited" ? `:${e.retryAfterMs}` : ""}` + } + // round 3 (finding 13): 429 is rate-limited with its Retry-After, the verdict kept (not `rejected`, which drops it) + expect(out).toEqual({ 413: "refused:http-413", 422: "refused:http-422", 404: "misrouted:true", 500: "server-error:false", 503: "transient:false", 429: "rate-limited:false:7000" }) + }) + + // ---------------------------------------------------------------- R2-4: withdrawal is explicit on the wire + it("R2-4a a Lease crosses a1's retryable verdict, whose reply is lost; the resend answers duplicate: a2 is created", async () => { + const w = W(); w.floor.enqueue(job("1")) + let holdLease = false, loseCompleteReply = false, failComplete = false + const fetch: typeof globalThis.fetch = async (input, init) => { + const rpc = await rpcOf(input, init) + if (rpc === "Lease") while (holdLease) await sleep(2) + if (rpc === "Complete" && failComplete) return new Response("upstream timeout", { status: 503 }) + if (rpc === "Complete" && loseCompleteReply) { await w.floor.fetch(input, init); loseCompleteReply = false; failComplete = true; throw new TypeError("fetch failed") } + return w.floor.fetch(input, init) + } + const l = start(w, dir, {}, fetch) + await until("running", () => w.ax.tasks.get("wf-test-1-a1")?.phase === "Running") + holdLease = true + await sleep(60) + loseCompleteReply = true + failRetryable(w, "wf-test-1-a1") + await until("floor applied a1 and requeued a2", () => J(w).state === "queued" && J(w).attempt === 2, 4000) + holdLease = false + await until("a2 granted", () => l.has("leased", "wf-test-1-a2"), 4000) + await until("dispatch waits for the verdict", () => l.has("supersede-waits-for-verdict", "wf-test-1-a2")) + failComplete = false + await until("a2 created or withdrawn", () => w.ax.updates.has("wf-test-1-a2") || l.has("withdrawn", "wf-test-1-a2"), 4000) + await sleep(300) + expect(l.has("withdrawn", "wf-test-1-a2"), why(w, l)).toBe(false) + expect(w.ax.updates.get("wf-test-1-a2")).toBe(1) + expect(J(w).history).not.toContain("requeue:omitted") + await l.drain() + }) + + it("R2-4b the same race with an infrastructure budget of 2: the retry runs once, the job is not ended retry-budget-spent", async () => { + const w = W({ infraAttempts: 2 }); w.floor.enqueue(job("1")) + let armed = false, dropped = 0, committed!: () => void + const done = new Promise((r) => { committed = r }) + const fetch: typeof globalThis.fetch = async (input, init) => { + const body = await bodyOf(input, init) + if (armed && body.includes('"Lease"') && /"capacity":[1-9]/.test(body)) { armed = false; await done } + if (body.includes('"Complete"') && body.includes("wf-test-1-a1") && dropped > 0) await sleep(200) + const res = await w.floor.fetch(input, init) + if (body.includes('"Complete"') && body.includes("wf-test-1-a1") && dropped === 0) { dropped++; committed(); throw new TypeError("fetch failed: reply lost") } + return res + } + const l = start(w, dir, { outboxBackoffMs: [150, 150], fenceTimeoutMs: 3000 }, fetch) + await until("a1 running", () => w.ax.tasks.get("wf-test-1-a1")?.phase === "Running") + armed = true + await until("a Lease with capacity is held", () => !armed, 3000) + failRetryable(w, "wf-test-1-a1") + await until("a2 leased", () => l.has("leased", "wf-test-1-a2"), 4000) + await until("a2 created", () => w.ax.updates.has("wf-test-1-a2"), 4000).catch(() => {}) + expect(w.ax.updates.get("wf-test-1-a2") ?? 0, why(w, l)).toBe(1) + expect(l.has("withdrawn", "wf-test-1-a2")).toBe(false) + expect((J(w).output as any)?.reason).not.toBe("infra/retry-budget-spent") + await l.drain() + }) + + // ---------------------------------------------------------------- R2-5: a regrant of a withdrawn leaseId is new work + it("R2-5a rule 4b withdraws a2 for a1's retryable verdict and the floor grants a2 again: the new generation is created", async () => { + const w = W(); w.floor.enqueue(job("1")) + let holdLease = false, failHeartbeat = false, failComplete = false + const fetch: typeof globalThis.fetch = async (input, init) => { + const rpc = await rpcOf(input, init) + if (rpc === "Heartbeat" && failHeartbeat) throw new TypeError("fetch failed") + if (rpc === "Complete" && failComplete) return new Response("upstream timeout", { status: 503 }) + if (rpc === "Lease") while (holdLease) await sleep(2) + return w.floor.fetch(input, init) + } + const l = start(w, dir, {}, fetch) + await until("running", () => w.ax.tasks.get("wf-test-1-a1")?.phase === "Running") + w.floor.down = true + await until("floor requeued attempt 2", () => J(w).state === "queued" && J(w).attempt === 2, 4000) + failHeartbeat = true; holdLease = true; failComplete = true + w.floor.down = false + await sleep(80) + failRetryable(w, "wf-test-1-a1") + await until("verdict written", () => l.has("verdict", "wf-test-1-a1")) + holdLease = false + await until("a2 received", () => l.has("leased", "wf-test-1-a2"), 4000) + await until("dispatch waits for the verdict", () => l.has("supersede-waits-for-verdict", "wf-test-1-a2")) + failComplete = false; failHeartbeat = false + await until("a2 created, or the floor gives up on a2", () => w.ax.updates.has("wf-test-1-a2") || J(w).attempt >= 3 || J(w).state === "done", 4000).catch(() => {}) + expect(w.ax.updates.get("wf-test-1-a2") ?? 0, why(w, l)).toBe(1) + expect(l.has("duplicate-grant", "wf-test-1-a2")).toBe(false) + await l.drain() + }) + + it("R2-5b the Lease reply carrying a2 is slow; a1's retryable verdict lands first and withdraws it; the regrant is created", async () => { + const w = W(); w.floor.enqueue(job("1")) + let holdLeaseReply = false, failHeartbeat = false + const fetch: typeof globalThis.fetch = async (input, init) => { + const rpc = await rpcOf(input, init) + if (rpc === "Heartbeat" && failHeartbeat) throw new TypeError("fetch failed") + if (rpc === "Lease" && holdLeaseReply) { const r = await w.floor.fetch(input, init); while (holdLeaseReply) await sleep(2); return r } + return w.floor.fetch(input, init) + } + const l = start(w, dir, {}, fetch) + await until("running", () => w.ax.tasks.get("wf-test-1-a1")?.phase === "Running") + w.floor.down = true + await until("floor requeued attempt 2", () => J(w).state === "queued" && J(w).attempt === 2, 4000) + failHeartbeat = true; holdLeaseReply = true + w.floor.down = false + await until("floor leased a2 to this link", () => J(w).state === "leased" && J(w).attempt === 2, 4000) + failRetryable(w, "wf-test-1-a1") + await until("verdict written", () => l.has("verdict", "wf-test-1-a1")) + holdLeaseReply = false + await until("a2 received", () => l.has("leased", "wf-test-1-a2"), 4000) + failHeartbeat = false + await until("a2 created, or the floor gives up", () => w.ax.updates.has("wf-test-1-a2") || J(w).attempt >= 3 || J(w).state === "done", 4000).catch(() => {}) + expect(w.ax.updates.get("wf-test-1-a2") ?? 0, why(w, l)).toBe(1) + await l.drain() + }) + + // ---------------------------------------------------------------- R2-6: redirects are never followed + it("R2-6 the floor answers 307 to another origin: nothing is leased or created and the link stops fatal (redirect)", async () => { + const w = W(); w.floor.enqueue(job("1")) + let rogue = 0 + const fetch: typeof globalThis.fetch = async (input, init) => { + if ((init as RequestInit | undefined)?.redirect !== "manual") { rogue++; return w.floor.fetch(input, init) } // a follower would land on the rogue origin + return new Response(null, { status: 307, headers: { location: "http://127.0.0.1:9/rpc" } }) + } + const l = start(w, dir, {}, fetch) + const exit = await Promise.race([Effect.runPromise(Fiber.await(l.fiber)), sleep(1500).then(() => undefined)]) + expect([rogue, w.ax.tasks.size, l.has("leased")]).toEqual([0, 0, false]) + expect(exit?._tag).toBe("Failure") + expect(JSON.stringify(exit)).toContain("redirect") + }) + + // ---------------------------------------------------------------- R2-7, R2-8: fail-closed checks + it("R2-7 isFleetInternalUrl matches private ranges on IP literals only", () => { + for (const u of ["https://10.evil.example/guest", "https://127.0.0.1.nip.io/guest", "https://192.168.0.1.attacker.com/c", "https://link/guest", "https://x.workers.dev/", "https://10.internal.example.com/"]) + expect([u, isFleetInternalUrl(u)]).toEqual([u, false]) + for (const u of ["http://10.201.0.80:8080/c", "http://127.0.0.1/c", "http://100.64.0.7/c", "http://[fd7a:115c:a1e0::1]/c", "http://[::1]/c", "http://link.fleet.internal/guest", "http://nas.lan/c", "http://localhost:1/c", "http://0x7f.1/c"]) + expect([u, isFleetInternalUrl(u)]).toEqual([u, true]) + expect(isFleetInternalUrl("http://link/guest", ["link"])).toBe(true) + }) + + it("R2-7b guest mode with a public name that starts with a private prefix: no Task, no token handed out", async () => { + const ax = new FakeAx(); ax.p1 = false + const logs = await runStub({ completion: "guest", shape: { atespace: "fleet", image: "ax-agent", gateway: "halogen", command: () => ["ax-agent", "pi"], completeUrl: "https://10.evil.example/guest" } }, + ax, stubFloor([grantOf("g", { leaseToken: "tok" })]), 400) + expect([logs.some((l) => l.ev === "guest-url-invalid"), ax.tasks.has("wf-test-g-a1")]).toEqual([true, false]) + }) + + it("R2-8 a Gateway open by CIDR arithmetic is refused: ::/0, 0.0.0.0/1 + 128.0.0.0/1, and unparseable CIDRs", async () => { + const gw = (hosts: Array) => ({ hasAllowlist: true, hosts: hosts.map((host) => ({ host, port: 443 })) }) + expect(gatewayAllowsAll(gw(["worker", "::/0"]))).toBe(true) + expect(gatewayAllowsAll(gw(["0.0.0.0/1", "128.0.0.0/1"]))).toBe(true) + expect(gatewayAllowsAll(gw([" * "]))).toBe(true) + expect(gatewayAllowsAll(gw(["10.0.0.0/x"]))).toBe(true) + expect(gatewayAllowsAll(gw(["worker", "10.201.0.0/24", "fd7a:115c:a1e0::/48"]))).toBe(false) + expect([cidrIsOpen("8.0.0.0/8"), cidrIsOpen("10.1.0.0/16"), cidrIsOpen("2000::/3"), cidrIsOpen("fd00::/64")]).toEqual([true, false, true, false]) + const ax = new FakeAx() + ax.gateways.set("halogen", gw(["worker", "::/0"])) + const logs = await runStub({}, ax, stubFloor([grantOf("w")]), 400) + expect([logs.some((l) => l.ev === "gateway-missing"), ax.tasks.has("wf-test-w-a1")]).toEqual([true, false]) + }) + + // ---------------------------------------------------------------- R2-9: nothing from before an ax outage opens the gate + it("R2-9 guest mode: ax returns without the Gateway and GetGateway is slow; nothing is created on pre-outage state", async () => { + const ax = new FakeAx(); ax.p1 = false + const origGw = ax.getGateway + let outageOver = false, release = false + ax.getGateway = (name: string) => outageOver ? Effect.sleep(300).pipe(Effect.andThen(origGw(name))) : origGw(name) + setTimeout(() => { ax.up = false }, 150) + setTimeout(() => { release = true }, 170) + setTimeout(() => { ax.gateways.delete("halogen"); outageOver = true; ax.up = true }, 400) + const logs = await runStub({ completion: "guest", shape: { atespace: "fleet", image: "ax-agent", gateway: "halogen", command: () => ["ax-agent", "pi"], completeUrl: "http://link.fleet.internal/guest" } }, + ax, stubFloor([grantOf("u", { leaseToken: "tok" })], () => release), 1200) + expect(ax.tasks.has("wf-test-u-a1"), JSON.stringify(logs.map((l) => l.ev))).toBe(false) + expect(logs.some((l) => l.ev === "gateway-missing")).toBe(true) + }) + + // ---------------------------------------------------------------- R2-11: a permanent Complete refusal is dropped + for (const variant of ["http-413", "unknown-error-code"] as const) + it(`R2-11 one Complete refused for good (${variant}) is dropped and job 2 is leased`, async () => { + const w = W(); w.floor.enqueue(job("1")) + const fetch: typeof globalThis.fetch = async (input, init) => { + const body = await bodyOf(input, init) + if (body.includes('"Complete"') && body.includes("wf-test-1-a1")) { + if (variant === "http-413") return new Response("payload too large", { status: 413 }) + const req = JSON.parse(body), id = (Array.isArray(req) ? req[0] : req).id + return new Response(JSON.stringify([{ _tag: "Exit", requestId: id, exit: { _tag: "Failure", cause: [{ _tag: "Fail", error: { code: "output-too-large" } }] } }]), { status: 200, headers: { "content-type": "application/json" } }) + } + return w.floor.fetch(input, init) + } + const l = start(w, dir, { outboxBackoffMs: [20, 40] }, fetch) + await until("a1 running", () => w.ax.tasks.get("wf-test-1-a1")?.phase === "Running") + w.ax.finish("wf-test-1-a1", 0, { big: "x" }) + await until("verdict", () => l.has("verdict", "wf-test-1-a1")) + w.floor.enqueue(job("2")) + await until("job 2 leased", () => J(w, "2").state !== "queued", 3000) + const d = l.logs.find((x) => x.ev === "complete-dropped" && x.leaseId === "wf-test-1-a1") + expect(d?.code).toBe(variant === "http-413" ? "http-413" : "output-too-large") + await l.drain() + }) +}) diff --git a/pkgs/substrate-link/src/apps/link/test/review-r3.test.ts b/pkgs/substrate-link/src/apps/link/test/review-r3.test.ts new file mode 100644 index 000000000..057b6faa2 --- /dev/null +++ b/pkgs/substrate-link/src/apps/link/test/review-r3.test.ts @@ -0,0 +1,484 @@ +// Review round 3 (2026-09-23): the thirteen findings, each as a regression test. The scratch repros they restate are +// in /home/tom/today/evals-2026-09-23/link/scratch-r3-{0,1,2}; they failed against d582ead. The log is +// link/REVIEW-LOG.md. Test adequacy findings 11 to 13 also strengthen R1-4, R1-8 and R2-3d in place. +import { appendFileSync, mkdtempSync, rmSync } from "node:fs" +import { tmpdir } from "node:os" +import { join } from "node:path" +import { Deferred, Effect, Fiber } from "effect" +import { afterEach, beforeEach, describe, expect, it } from "vitest" +import { AxError } from "../src/ax.ts" +import type { AgentJob } from "../src/contract.ts" +import { rpcFloor } from "../src/floor.ts" +import { Journal } from "../src/journal.ts" +import { boundedSeconds, isFleetInternalUrl, runLink } from "../src/link.ts" +import type { LinkConfig } from "../src/link.ts" +import { FakeAx } from "./fake-ax.ts" +import { FakeFloor } from "./fake-floor.ts" +import type { FloorConfig } from "./fake-floor.ts" + +const SEC = 20 +const A = "ultracode.mecattaf.dev/" +const JK = `${"cd".repeat(32)}:1` // round 4: FIELD-MAP 5a journal key (jobs.ts refuses a grant without it) +const job = (n: string, spec: Record = {}): AgentJob => ({ + apiVersion: "ultracode.mecattaf.dev/v1alpha1", kind: "AgentJob", + metadata: { name: `wf-test-${n}`, labels: { [A + "run-id"]: "wf-test", [A + "workflow"]: "link-test", [A + "phase-index"]: "1" }, + annotations: { [A + "run-id-raw"]: "wf_test", [A + "label"]: `probe:${n}`, [A + "item-key"]: `wf_test#${n}`, [A + "journal-key"]: JK, [A + "phase-title"]: "Probe" } }, + spec: { "runs-on": ["seat:halogen", "runtime:gvisor"], + with: { prompt: `say ${n}`, prompt_ref: { sha256: "ab".repeat(32), bytes: 5, uri: `journal://wf_test/${n}/prompt.md` }, model: "halogen-qwen3.8-flash-next" }, ...spec } as AgentJob["spec"] +}) +const world = (o: Partial = {}) => ({ + floor: new FakeFloor({ cap: 2, leaseSeconds: 6, graceSeconds: 15, pollSeconds: 1, heartbeatSeconds: 2, maxAttempts: 3, secondMs: SEC, tokens: { "tok-nas": "nas-link-1" }, ...o }), + ax: new FakeAx() +}) +type World = ReturnType +function start(w: World, dir: string, over: Partial = {}, fetch?: typeof globalThis.fetch) { + const logs: Array> = [] + const stop = Effect.runSync(Deferred.make()) + const cfg: LinkConfig = { + holder: "nas-link-1", maxInFlight: 2, servedLabels: ["seat:halogen", "runtime:gvisor"], + shape: { atespace: "fleet", image: "ax-agent", gateway: "halogen", command: () => ["ax-agent", "pi"] }, + completion: "auto", secondMs: SEC, resyncMs: 30, pendingTimeoutMs: 1500, deleteAfterMs: 0, deadlineBackstopMs: 300, + createAttempts: 3, outboxBackoffMs: [20, 100], initialPollSeconds: 1, initialHeartbeatSeconds: 2, fenceTimeoutMs: 300, resultReadTries: 3, ...over + } + const fiber = Effect.runFork(Effect.scoped(Effect.gen(function*() { + const floor = yield* rpcFloor({ url: "http://floor.test", token: "tok-nas", sessionId: `s:${dir}`, fetch: fetch ?? w.floor.fetch }) + return yield* runLink(cfg, { ax: w.ax, floor, journal: Journal.open(dir), log: (ev, f) => logs.push({ t: Date.now(), ev, ...f }), stop, floorUrl: "http://floor.test" }) + }))) + return { + logs, fiber, + has: (ev: string, leaseId?: string) => logs.some((l) => l.ev === ev && (leaseId === undefined || l.leaseId === leaseId)), + trace: () => logs.filter((x) => typeof x.leaseId === "string" || typeof x.task === "string").map((x) => `${x.ev}:${x.leaseId ?? x.task}${x.why ? ":" + x.why : ""}`), + crash: () => Effect.runPromise(Fiber.interrupt(fiber)), + drain: () => { Effect.runSync(Deferred.succeed(stop, undefined)); return Promise.race([Effect.runPromise(Fiber.await(fiber)), sleep(1500)]) } + } +} +const until = async (what: string, pred: () => boolean, ms = 5000) => { + const t0 = Date.now() + while (!pred()) { if (Date.now() - t0 > ms) throw new Error(`timeout waiting for: ${what}`); await new Promise((r) => setTimeout(r, 5)) } +} +const sleep = (ms: number) => new Promise((r) => setTimeout(r, ms)) +const J = (w: World, n = "1") => w.floor.jobs.get(`wf-test-${n}`)! +const bodyOf = async (input: unknown, init?: RequestInit) => { + const b = init?.body + if (typeof b === "string") return b + if (b instanceof Uint8Array) return new TextDecoder().decode(b) + if (b) return await new Response(b as ConstructorParameters[0]).text() + return (input instanceof Request) ? await input.clone().text() : "" +} +const rpcOf = async (input: unknown, init?: RequestInit) => { const b = await bodyOf(input, init); return b.includes('"Complete"') ? "Complete" : b.includes('"Lease"') ? "Lease" : b.includes('"Heartbeat"') ? "Heartbeat" : "?" } +const failRetryable = (w: World, name: string) => { + const t = w.ax.tasks.get(name)! + t.exited = true; t.phase = "Failed" + t.conditions = [{ type: "Ready", status: "False", reason: "ResourceExhausted", message: "ResourceExhausted: no free worker" }] +} +const why = (w: World, l: ReturnType) => JSON.stringify({ floor: J(w).history, link: l.trace() }) +/** Rewrite every JSON reply of one RPC with `patch` (a floor whose wire drifted from the link's). */ +const rewriting = (w: World, rpc: string, patch: (x: any) => any): typeof globalThis.fetch => async (input, init) => { + const b = await bodyOf(input, init) + const res = await w.floor.fetch(input, init) + if (!b.includes(`"${rpc}"`)) return res + const txt = await res.text() + let j: unknown; try { j = JSON.parse(txt) } catch { return new Response(txt, { status: res.status, headers: res.headers }) } + return new Response(JSON.stringify(patch(j)), { status: res.status, headers: { "content-type": "application/json" } }) +} +const deep = (f: (o: Record) => void) => { const walk = (x: any): any => { + if (Array.isArray(x)) return x.map(walk) + if (x && typeof x === "object") { const o: Record = {}; for (const [k, v] of Object.entries(x)) o[k] = walk(v); f(o); return o } + return x +}; return walk } + +// A floor stub that grants one lease named after a Task someone else owns (scratch-r3-1). +const stats = { seq: 1, cap: 2, wip: 0, queued: 0, done: 0 } +const foreignTask = (name: string) => ({ apiVersion: "ax.io/v1alpha1" as const, kind: "Task" as const, metadata: { name, atespace: "fleet" }, + spec: { image: "someone-else", command: ["long-job"], env: [{ name: "AX_CONWIP_HOLDER", value: "nas-link-2" }], gateway: { name: "halogen" } } }) +const foreignGrant = (name: string) => ({ leaseId: name, attempt: 1, job: job("x"), + lease: { holderIdentity: "nas-link-1", leaseDurationSeconds: 30, acquireTime: Date.now(), renewTime: Date.now(), leaseTransitions: 0 } }) +const runStub = async (dir: string, ax: FakeAx, floor: any, ms: number, over: Partial = {}) => { + const logs: Array> = [] + const stop = Effect.runSync(Deferred.make()) + const cfg: LinkConfig = { holder: "nas-link-1", maxInFlight: 2, servedLabels: ["seat:halogen", "runtime:gvisor"], + shape: { atespace: "fleet", image: "ax-agent", gateway: "halogen", command: () => ["ax-agent", "pi"] }, + completion: "auto", secondMs: SEC, resyncMs: 30, pendingTimeoutMs: 60_000, deleteAfterMs: 600_000, deadlineBackstopMs: 300, + createAttempts: 3, outboxBackoffMs: [20, 100], initialPollSeconds: 1, initialHeartbeatSeconds: 2, ...over } + const f = Effect.runFork(runLink(cfg, { ax, floor, journal: Journal.open(dir), log: (ev, x) => logs.push({ t: Date.now(), ev, ...x }), stop })) + await sleep(ms) + Effect.runSync(Deferred.succeed(stop, undefined)); await Effect.runPromise(Fiber.await(f)) + return logs +} + +describe("review round 3", () => { + let dir: string + let worlds: Array = [] + const W = (o: Partial = {}) => { const w = world(o); worlds.push(w); return w } + beforeEach(() => { dir = mkdtempSync(join(tmpdir(), "conwip-link-r3-")) }) + afterEach(() => { for (const w of worlds) w.floor.close(); worlds = []; rmSync(dir, { recursive: true, force: true }) }) + + // ---------------------------------------------------------------- R3-1: a withdrawal names a generation + for (const sendWithdrewGen of [true, false]) { + it(`R3-1a (E) rule 4b withdraws a QUEUED a2 for a1's retryable verdict; the requeued a2 is created (floor names the generation: ${sendWithdrewGen})`, async () => { + const w = W({ sendWithdrewGen }); w.floor.enqueue(job("1")) + // Deterministic E: no Lease reaches the floor between its return and a1's verdict landing, so a2 gen 1 is withdrawn + // while still QUEUED and this link never receives it (a Lease retried across the return could carry a capacity + // computed before the verdict and receive gen 1 first: that is R2-5a's interleaving, not this one). + let gate = false, landed = false + const fetch: typeof globalThis.fetch = async (input, init) => { + const rpc = await rpcOf(input, init) + if (rpc === "Lease" && gate && !landed) throw new TypeError("fetch failed") + const r = await w.floor.fetch(input, init) + if (rpc === "Complete" && gate) landed = true + return r + } + const l = start(w, dir, {}, fetch) + await until("running", () => w.ax.tasks.get("wf-test-1-a1")?.phase === "Running") + gate = true + w.floor.down = true + await until("floor requeued attempt 2", () => J(w).state === "queued" && J(w).attempt === 2, 4000) + failRetryable(w, "wf-test-1-a1") + await until("verdict written", () => l.has("verdict", "wf-test-1-a1")) + w.floor.down = false + await until("a2 created", () => w.ax.updates.has("wf-test-1-a2"), 4000).catch(() => { throw new Error(why(w, l)) }) + expect(J(w).history).toContain("withdrawn:a2") + expect(l.logs.some((x) => x.ev === "leased" && x.leaseId === "wf-test-1-a2" && l.logs.indexOf(x) < l.logs.findIndex((y) => y.ev === "complete"))).toBe(false) + w.ax.finish("wf-test-1-a2", 0, { ok: 2 }) + await until("floor done", () => J(w).state === "done", 4000) + expect([J(w).result, J(w).attempt, w.ax.updates.get("wf-test-1-a2")]).toEqual(["success", 2, 1]) + await l.drain() + }) + + it(`R3-1b (C) kill -9 between the withdrawal and the regrant: the regrant of a2 is created after restart (floor names the generation: ${sendWithdrewGen})`, async () => { + const w = W({ sendWithdrewGen }); w.floor.enqueue(job("1")) + let holdLease = false, failHeartbeat = false, failComplete = false, holdAfterComplete = false + const fetch: typeof globalThis.fetch = async (input, init) => { + const rpc = await rpcOf(input, init) + if (rpc === "Heartbeat" && failHeartbeat) throw new TypeError("fetch failed") + if (rpc === "Complete" && failComplete) return new Response("upstream timeout", { status: 503 }) + if (rpc === "Lease") while (holdLease) await sleep(2) + const r = await w.floor.fetch(input, init) + if (rpc === "Complete" && holdAfterComplete) holdLease = true // the regrant never reaches this process + return r + } + const over = { deleteAfterMs: 60_000 } // a1's record survives the restart's compaction; a2 gen 1 does not + const l1 = start(w, dir, over, fetch) + await until("running", () => w.ax.tasks.get("wf-test-1-a1")?.phase === "Running") + w.floor.down = true + await until("floor requeued attempt 2", () => J(w).state === "queued" && J(w).attempt === 2, 4000) + failHeartbeat = true; holdLease = true; failComplete = true + w.floor.down = false + await sleep(80) + failRetryable(w, "wf-test-1-a1") + await until("verdict written", () => l1.has("verdict", "wf-test-1-a1")) + holdLease = false + await until("a2 received", () => l1.has("leased", "wf-test-1-a2"), 4000) + await until("dispatch waits for the verdict", () => l1.has("supersede-waits-for-verdict", "wf-test-1-a2")) + holdAfterComplete = true; failComplete = false; failHeartbeat = false + await until("floor withdrew a2 and the link journaled it", () => l1.has("withdrawn", "wf-test-1-a2"), 4000) + await l1.crash() + const l2 = start(w, dir, over) + await until("a2 created", () => w.ax.updates.has("wf-test-1-a2"), 4000).catch(() => { throw new Error(why(w, l2)) }) + expect(w.ax.updates.get("wf-test-1-a2")).toBe(1) + expect(J(w).history.filter((h) => h === "leased:a2").length).toBe(2) // gen 1 withdrawn, gen 2 run + await l2.drain() + }) + } + + // ---------------------------------------------------------------- R3-2: the cancel intent is journaled first + it("R3-2 (A) a cancel whose DeleteTask lands but whose reply is lost is reported cancelled; no attempt 2 is started", async () => { + const w = W({ leaseSeconds: 20, heartbeatSeconds: 6, graceSeconds: 40 }); w.floor.enqueue(job("1")) + const l = start(w, dir, { initialHeartbeatSeconds: 6 }) + await until("running", () => w.ax.tasks.get("wf-test-1-a1")?.phase === "Running") + const real = w.ax.deleteTask + let lostReplies = 0 + w.ax.deleteTask = (name: string) => real(name).pipe(Effect.andThen(() => lostReplies++ === 0 ? Effect.fail(new AxError(4, "deadline exceeded")) : Effect.void)) + w.floor.cancel("wf-test-1") + await until("floor done", () => J(w).state === "done", 4000).catch(() => { throw new Error(why(w, l)) }) + await sleep(200) + const v = l.logs.find((x) => x.ev === "verdict" && x.leaseId === "wf-test-1-a1")! + expect([v.result, J(w).result, J(w).attempt, J(w).infraSpent]).toEqual(["cancelled", "cancelled", 1, 0]) + expect(w.ax.updates.has("wf-test-1-a2")).toBe(false) + expect(l.logs.some((x) => x.ev === "delete-deferred" && x.task === "wf-test-1-a1")).toBe(true) + await l.drain() + }) + + it("R3-2b a journaled DeleteTask that did not land is sent again by the resync", async () => { + const w = W({ leaseSeconds: 20, heartbeatSeconds: 6, graceSeconds: 40 }); w.floor.enqueue(job("1")) + const l = start(w, dir, { initialHeartbeatSeconds: 6 }) + await until("running", () => w.ax.tasks.get("wf-test-1-a1")?.phase === "Running") + const real = w.ax.deleteTask + let refused = 0 + w.ax.deleteTask = (name: string) => refused++ === 0 ? Effect.fail(new AxError(14, "unavailable")) : real(name) + w.floor.cancel("wf-test-1") + await until("floor done", () => J(w).state === "done", 4000).catch(() => { throw new Error(why(w, l)) }) + expect([J(w).result, w.ax.deletes.filter((d) => d === "wf-test-1-a1").length]).toEqual(["cancelled", 1]) + await l.drain() + }) + + // ---------------------------------------------------------------- R3-3: finished leases are tombstoned + for (const restart of [false, true]) it(`R3-3 (D) a redelivered grant of a finished lease is never run again (restart=${restart})`, async () => { + const w = W(); w.floor.enqueue(job("1")) + const l1 = start(w, dir) + await until("running", () => w.ax.tasks.get("wf-test-1-a1")?.phase === "Running") + w.ax.finish("wf-test-1-a1", 0, { answer: 42 }) + await until("done", () => J(w).state === "done") + await until("janitor deleted a1", () => !w.ax.tasks.has("wf-test-1-a1"), 3000) + await sleep(100) + let l = l1 + if (restart) { await l1.crash(); l = start(w, dir); await sleep(100) } + w.floor.redeliver = ["wf-test-1-a1"] + await until("duplicate-grant", () => l.has("duplicate-grant", "wf-test-1-a1"), 2000) + await sleep(200) + expect(w.ax.updates.get("wf-test-1-a1")).toBe(1) + await l.drain() + }) + + it("R3-3b the tombstone survives compaction and expires after TOMB_MS; a later generation of the same leaseId is not refused", () => { + const j = Journal.open(dir) + const g = { leaseId: "x-a1", attempt: 1, job: job("x"), lease: { holderIdentity: "h", leaseDurationSeconds: 30, acquireTime: 1000, renewTime: 1000, leaseTransitions: 0 } } + j.append({ ev: "grant", leaseId: "x-a1", grant: g }) + j.append({ ev: "report", leaseId: "x-a1", attempt: 1, result: "success" }) + j.append({ ev: "reported", leaseId: "x-a1", duplicate: false }) + j.compact() + expect(j.recs.has("x-a1")).toBe(false) + expect(Journal.open(dir).tombs.get("x-a1")?.gen).toEqual({ leaseTransitions: 0, acquireTime: 1000 }) + const keep = Journal.TOMB_MS + try { Journal.TOMB_MS = -1; Journal.open(dir).compact(); expect(Journal.open(dir).tombs.has("x-a1")).toBe(false) } finally { Journal.TOMB_MS = keep } + }) + + // ---------------------------------------------------------------- R3-4: renewal does not wait for the first Lease + it("R3-4 (B) a restart whose first Lease is rate-limited keeps heartbeating; the running a1 is not requeued", async () => { + const w = W(); w.floor.enqueue(job("1")) + const l1 = start(w, dir) + await until("running", () => w.ax.tasks.get("wf-test-1-a1")?.phase === "Running") + await l1.crash() + let leases = 0 + const hbAt: Array = [] + const f: typeof globalThis.fetch = async (input, init) => { + const rpc = await rpcOf(input, init) + if (rpc === "Heartbeat") hbAt.push(Date.now()) + if (rpc === "Lease" && leases++ === 0) return new Response("slow down", { status: 429, headers: { "retry-after": "2" } }) + return w.floor.fetch(input, init) + } + const t0 = Date.now() + const l2 = start(w, dir, {}, f) + await sleep(2300) + expect(hbAt.filter((t) => t - t0 > 200 && t - t0 < 2000).length).toBeGreaterThan(5) + expect(J(w).history).not.toContain("requeue:grace") + expect([J(w).attempt, w.ax.tasks.get("wf-test-1-a1")?.phase]).toEqual([1, "Running"]) + await l2.drain() + }) + + // ---------------------------------------------------------------- R3-5: a Task the link did not write is never touched + for (const variant of ["janitor", "lost", "cancel"] as const) for (const phase of ["Running", "Completed"] as const) + it(`R3-5a a grant named after a foreign ${phase} Task: refused name-conflict, never deleted or read (${variant})`, async () => { + const ax = new FakeAx() + ax.put(foreignTask("other-team-build"), "Running") + if (phase === "Completed") ax.finish("other-team-build", 0, { secret: "other tenant's output" }) + let sent = false + const completes: Array = [] + const floor = { + lease: (p: any) => Effect.sync(() => { const g = !sent && p.capacity > 0 ? [foreignGrant("other-team-build")] : []; if (g.length) sent = true; return { grants: g, invalid: [], stats, nextPollSeconds: 1, heartbeatSeconds: 2 } }), + heartbeat: (p: any) => Effect.sync(() => { + const lost = variant === "lost" ? p.leaseIds.filter((x: string) => x === "other-team-build") : [] + const cancel = variant === "cancel" ? p.leaseIds.filter((x: string) => x === "other-team-build") : [] + return { renewed: p.leaseIds.filter((x: string) => !lost.includes(x)), lost, cancelRequested: cancel, stats } + }), + complete: (p: any) => Effect.sync(() => { completes.push(p); return { duplicate: false, stats } }) + } + const logs = await runStub(dir, ax, floor, 800) + expect(ax.deletes).toEqual([]) + expect(ax.tasks.get("other-team-build")?.phase).toBe(phase) + expect(completes.every((c) => c.result !== "success" && JSON.stringify(c).includes("secret") === false)).toBe(true) + expect(logs.some((x) => x.ev === "not-mine" && x.leaseId === "other-team-build")).toBe(true) + }) + + it("R3-5b (salvage) a crash in the create window, then `lost`: a foreign finished Task's result is never reported, the Task never deleted", async () => { + const g = foreignGrant("other-team-build") + const t = Date.now() + appendFileSync(join(dir, "journal.jsonl"), JSON.stringify({ t, ev: "grant", leaseId: g.leaseId, grant: g }) + "\n" + JSON.stringify({ t, ev: "creating", leaseId: g.leaseId }) + "\n") + const ax = new FakeAx() + ax.put(foreignTask("other-team-build"), "Running") + ax.finish("other-team-build", 0, { secret: "other tenant's output" }) + const completes: Array = [] + const floor = { + lease: (_p: any) => Effect.succeed({ grants: [], invalid: [], stats, nextPollSeconds: 1, heartbeatSeconds: 2 }), + heartbeat: (p: any) => Effect.succeed({ renewed: [], lost: p.leaseIds.filter((x: string) => x === "other-team-build"), cancelRequested: [], stats }), + complete: (p: any) => Effect.sync(() => { completes.push(p); return { duplicate: false, stats } }) + } + const logs = await runStub(dir, ax, floor, 600) + expect([completes, ax.deletes]).toEqual([[], []]) + expect(logs.some((x) => x.ev === "not-mine")).toBe(true) + }) + + it("R3-5c the link's own Task under a `creating`-only record (crash before `created`) is still salvaged and deleted", async () => { + const w = W(); w.floor.enqueue(job("1")) + w.ax.hangAfterCreate = true + const l1 = start(w, dir) + await until("created in ax", () => w.ax.tasks.has("wf-test-1-a1")) + await l1.crash() + w.ax.hangAfterCreate = false + w.ax.tick() + w.ax.finish("wf-test-1-a1", 0, { mine: true }) + const l2 = start(w, dir) + await until("floor done", () => J(w).state === "done", 4000).catch(() => { throw new Error(why(w, l2)) }) + expect([J(w).result, J(w).output]).toEqual(["success", { mine: true }]) + await until("deleted", () => !w.ax.tasks.has("wf-test-1-a1"), 3000) + expect(l2.has("not-mine")).toBe(false) + await l2.drain() + }) + + // ---------------------------------------------------------------- R3-6 and R3-7: server-driven intervals are bounded + for (const v of [0, -5]) it(`R3-6 a floor answering nextPollSeconds=${v} and heartbeatSeconds=${v} gets a bounded call rate (production secondMs)`, async () => { + const w = W({ pollSeconds: v, heartbeatSeconds: v, secondMs: 1000, leaseSeconds: 60, graceSeconds: 60 }) + w.floor.enqueue(job("1")) + const l = start(w, dir, { secondMs: 1000, resyncMs: 1000, initialPollSeconds: 15, initialHeartbeatSeconds: 30 }) + await sleep(1500) + const n = (rpc: string) => w.floor.calls.filter((c) => c.rpc === rpc).length + expect(n("Lease")).toBeLessThanOrEqual(4) + expect(n("Heartbeat")).toBeLessThanOrEqual(4) + expect(l.has("interval-clamped")).toBe(true) + await l.drain() + }) + + it("R3-7 boundedSeconds: default for missing, non-finite and not positive; clamped to [1, hi]", () => { + expect([boundedSeconds(undefined, 15, 300), boundedSeconds(0, 15, 300), boundedSeconds(-5, 15, 300), boundedSeconds(Number.NaN, 15, 300), + boundedSeconds(0.2, 15, 300), boundedSeconds(10 / 3, 15, 300), boundedSeconds(1e9, 15, 300), boundedSeconds(60, 30, 10)]).toEqual([15, 15, 15, 15, 1, 10 / 3, 300, 10]) + }) + + // ---------------------------------------------------------------- R3-8: the Lease envelope decodes leniently + it("R3-8 a Lease reply with heartbeatSeconds 3.333, endpoint null and no stats is used; the job runs", async () => { + const w = W(); w.floor.enqueue(job("1")) + const fetch = rewriting(w, "Lease", deep((o) => { if ("heartbeatSeconds" in o) { o.heartbeatSeconds = 10 / 3; o.endpoint = null; delete o.stats } })) + const l = start(w, dir, {}, fetch) + await until("created", () => w.ax.updates.has("wf-test-1-a1"), 4000).catch(() => { throw new Error(why(w, l)) }) + w.ax.finish("wf-test-1-a1", 0, { ok: 1 }) + await until("floor done", () => J(w).state === "done", 4000) + expect([J(w).result, J(w).attempt, l.has("lease-error")]).toEqual(["success", 1, false]) + await l.drain() + }) + + it("R3-8b a Lease key whose replies never decode is abandoned after leaseKeyAttempts, not replayed for ever; the job then runs", async () => { + const w = W(); w.floor.enqueue(job("1")) + let broken = true + const keys: Array = [] + const inner = rewriting(w, "Lease", deep((o) => { if (broken && "grants" in o && "nextPollSeconds" in o) o.grants = "not-an-array" })) + const fetch: typeof globalThis.fetch = async (input, init) => { + const b = await bodyOf(input, init) + const k = /"requestKey":"([^"]+)"/.exec(b)?.[1] + if (k && b.includes('"Lease"') && broken) keys.push(k) + return inner(input, init) + } + const l = start(w, dir, { leaseKeyAttempts: 3 }, fetch) + await until("key abandoned twice", () => l.logs.filter((x) => x.ev === "lease-key-abandoned").length >= 2, 4000).catch(() => { throw new Error(why(w, l)) }) + broken = false + expect(new Set(keys).size).toBeGreaterThan(1) // before round 3: one key, `transient`, for as long as the floor answers so + expect(l.logs.filter((x) => x.ev === "lease-error").every((x) => x.kind === "server-error")).toBe(true) + await until("created", () => w.ax.updates.size > 0, 4000).catch(() => { throw new Error(why(w, l)) }) + await l.drain() + }) + + it("R3-8c (R1-9b scenario) a floor that cannot encode one job: the link stays up and a job enqueued later runs", async () => { + // The good job co-granted with the bad one is only ever sent inside replies the strict floor cannot encode, so no + // link-side change can deliver it (REFUTED for the link; the floor port must encode per grant, round 1 H). + const w = W() + w.floor.enqueue(job("bad", { "timeout-minutes": 1.5 })) + w.floor.enqueue(job("good")) + const l = start(w, dir) + await sleep(500) + w.floor.enqueue(job("later")) + await until("later created", () => w.ax.updates.has("wf-test-later-a1"), 8000).catch(() => { throw new Error(why(w, l)) }) + expect(l.logs.filter((x) => x.ev === "lease-error").every((x) => x.kind === "server-error")).toBe(true) + await l.drain() + }) + + // ---------------------------------------------------------------- R3-9: Complete's success decodes leniently + it("R3-9 a Complete success reply with `withdrew: null` is reported once; the next jobs run", async () => { + const w = W(); w.floor.enqueue(job("1")); w.floor.enqueue(job("2")) + let completes = 0 + const patch = deep((o) => { if ("duplicate" in o && "stats" in o && !("withdrew" in o)) o.withdrew = null }) + const inner = rewriting(w, "Complete", patch) + const fetch: typeof globalThis.fetch = async (input, init) => { if (await rpcOf(input, init) === "Complete") completes++; return inner(input, init) } + const l = start(w, dir, { maxInFlight: 1 }, fetch) + await until("a1", () => w.ax.tasks.has("wf-test-1-a1")) + w.ax.finish("wf-test-1-a1", 0, { ok: true }) + await until("job 2 created", () => w.ax.updates.has("wf-test-2-a1"), 4000).catch(() => { throw new Error(why(w, l)) }) + expect([J(w).state, J(w).result, completes, l.has("outbox-keep")]).toEqual(["done", "success", 1, false]) + await l.drain() + }) + + it("R3-9b a Complete success reply that is not the protocol at all is a server-error: bounded by verdictAttempts, never gates Lease", async () => { + const w = W(); w.floor.enqueue(job("1")); w.floor.enqueue(job("2")) + // the floor applies the verdict; its reply's success value is replaced by a number + const fetch = rewriting(w, "Complete", deep((o) => { if (o.value && typeof o.value === "object" && "duplicate" in (o.value as object)) o.value = 7 })) + const l = start(w, dir, { maxInFlight: 1, verdictAttempts: 2 }, fetch) + await until("a1", () => w.ax.tasks.has("wf-test-1-a1")) + w.ax.finish("wf-test-1-a1", 0, { ok: true }) + await until("job 2 created", () => w.ax.updates.has("wf-test-2-a1"), 4000).catch(() => { throw new Error(why(w, l)) }) + expect(l.logs.filter((x) => x.ev === "outbox-keep").every((x) => x.kind === "server-error")).toBe(true) + await l.drain() + }) + + // ---------------------------------------------------------------- R3-10: B2 (n+1 only after NotFound for n) + it("R3-10 (F8 shape) attempt n+1 is written only once attempt n is absent from ax; while n stays Terminating it is never written", async () => { + const w = W() + const create = w.ax.createTask + const seenAtCreate: Array = [] + w.ax.createTask = (t) => { if (t.metadata.name === "wf-test-1-a2") seenAtCreate.push(...[...w.ax.tasks].map(([n, x]) => `${n}:${x.phase}`)); return create(t) } + w.floor.enqueue(job("1")) + const l1 = start(w, dir) + await until("running", () => w.ax.tasks.get("wf-test-1-a1")?.phase === "Running") + await l1.crash() + rmSync(dir, { recursive: true, force: true }) // the journal is lost: a2 is fenced on the Task's own marks + await sleep(7 * SEC); w.floor.sweep() + const tick = w.ax.tick.bind(w.ax) + let hold = true + w.ax.tick = () => { if (hold) { for (const t of w.ax.tasks.values()) if (t.phase === "Pending") t.phase = "Running" } else tick() } + const l2 = start(w, mkdtempSync(join(tmpdir(), "conwip-link-r3-")), { fenceTimeoutMs: 4000 }) + await until("a1 Terminating", () => w.ax.tasks.get("wf-test-1-a1")?.phase === "Terminating", 4000).catch(() => { throw new Error(why(w, l2)) }) + await sleep(250) + expect(w.ax.updates.has("wf-test-1-a2")).toBe(false) // the delete was asked, NotFound not yet seen + hold = false + await until("a2 written", () => w.ax.updates.has("wf-test-1-a2"), 6000).catch(() => { throw new Error(why(w, l2)) }) + expect(seenAtCreate.filter((x) => x.startsWith("wf-test-1-a1"))).toEqual([]) + await l2.drain() + }) + + // ---------------------------------------------------------------- R3-11: 429, Retry-After, endpoint and address rows + it("R3-11 a Complete answered 429 Retry-After: 1 keeps the verdict and waits at least Retry-After before the next try", async () => { + const w = W(); w.floor.enqueue(job("1")) + const at: Array = [] + const fetch: typeof globalThis.fetch = async (input, init) => { + if (await rpcOf(input, init) !== "Complete") return w.floor.fetch(input, init) + at.push(Date.now()) + if (at.length === 1) return new Response("slow down", { status: 429, headers: { "retry-after": "1" } }) + return w.floor.fetch(input, init) + } + const l = start(w, dir, {}, fetch) + await until("a1", () => w.ax.tasks.get("wf-test-1-a1")?.phase === "Running") + w.ax.finish("wf-test-1-a1", 0, { v: 1 }) + await until("floor done", () => J(w).state === "done", 5000).catch(() => { throw new Error(why(w, l)) }) + expect([J(w).result, J(w).attempt, l.has("complete-dropped")]).toEqual(["success", 1, false]) + expect(l.logs.some((x) => x.ev === "outbox-keep" && x.kind === "rate-limited")).toBe(true) + expect(at[1]! - at[0]!).toBeGreaterThanOrEqual(950) + await l.drain() + }) + + it("R3-11b an http:// endpoint is ignored even when it is in the declared list; 100.x outside 100.64/10 is not fleet-internal", async () => { + const w = W() + const fetch = rewriting(w, "Lease", deep((o) => { if ("nextPollSeconds" in o) o.endpoint = "http://floor2.test" })) + const switched: Array = [] + const logs: Array> = [] + const stop = Effect.runSync(Deferred.make()) + const cfg: LinkConfig = { holder: "nas-link-1", maxInFlight: 2, servedLabels: ["seat:halogen", "runtime:gvisor"], + shape: { atespace: "fleet", image: "ax-agent", gateway: "halogen", command: () => ["ax-agent", "pi"] }, + completion: "auto", secondMs: SEC, resyncMs: 30, pendingTimeoutMs: 1500, deleteAfterMs: 0, deadlineBackstopMs: 300, + createAttempts: 3, outboxBackoffMs: [20, 100], initialPollSeconds: 1, initialHeartbeatSeconds: 2, floorUrls: ["http://floor.test", "http://floor2.test"] } + const fiber = Effect.runFork(Effect.scoped(Effect.gen(function*() { + const floor = yield* rpcFloor({ url: "http://floor.test", token: "tok-nas", sessionId: "s", fetch }) + return yield* runLink(cfg, { ax: w.ax, floor, journal: Journal.open(dir), log: (ev, f) => logs.push({ ev, ...f }), stop, floorUrl: "http://floor.test", onEndpoint: (u) => switched.push(u) }) + }))) + await until("endpoint seen", () => logs.some((x) => x.ev === "endpoint-ignored" || x.ev === "endpoint-accepted"), 3000) + Effect.runSync(Deferred.succeed(stop, undefined)); await Promise.race([Effect.runPromise(Fiber.await(fiber)), sleep(1500)]) + expect([switched, logs.some((x) => x.ev === "endpoint-ignored")]).toEqual([[], true]) + expect(["http://100.1.2.3/c", "http://100.128.0.1/c", "http://100.63.255.255/c", "http://100.64.0.1/c", "http://100.127.255.254/c"].map((u) => isFleetInternalUrl(u))) + .toEqual([false, false, false, true, true]) + }) +}) diff --git a/pkgs/substrate-link/src/apps/link/test/review-r4-expired.test.ts b/pkgs/substrate-link/src/apps/link/test/review-r4-expired.test.ts new file mode 100644 index 000000000..c2b499f56 --- /dev/null +++ b/pkgs/substrate-link/src/apps/link/test/review-r4-expired.test.ts @@ -0,0 +1,86 @@ +// Round 4 regression test, ported from /home/tom/today/evals-2026-09-23/link/scratch-r4-1/expired.repro.ts (assertions kept; additions marked). +// Round 4, lens fail-closed and capacity gate: a grant that waits at the slot gate (B11: gate closed, e.g. the Gateway +// check failing) is created whenever the gate reopens, with no check that its lease is still live. The floor, which +// the link could not reach (heartbeats failing), has already expired it, requeued it, and may grant attempt 2. +// Buildkite's pattern the design adopts (row 8, "reserve with expiry, finish -1 before start") would refuse it. +import { expect, it } from "vitest" +import { bodyOf, job, sleep, start, until, world } from "./review-r4-harness.ts" + +it("a grant whose lease expired while it waited at the gate is still created", async () => { + const w = world({ leaseSeconds: 6, graceSeconds: 15 }) // at secondMs 20: lease 120 ms, grace 300 ms + w.floor.enqueue(job("1")) + const gw = w.ax.gateways.get("halogen")! + let holdOnce = true + let l: ReturnType + const fetch: typeof globalThis.fetch = async (input, init) => { + const b = await bodyOf(input, init) + const res = await w.floor.fetch(input, init) + if (holdOnce && b.includes('"Lease"')) { + const txt = await res.clone().text() + if (txt.includes("wf-test-1-a1")) { + holdOnce = false + // the Gateway is edited away while the reply is in flight; the next resync closes the gate (B10) + w.ax.gateways.delete("halogen") + await until("gateway-missing", () => l.has("gateway-missing"), 3000) + } + } + return res + } + l = start(w, undefined, { maxInFlight: 1 }, fetch) + await until("waiting-for-slot", () => l.has("waiting-for-slot", "wf-test-1-a1"), 3000) + // the NAS loses its route to Cloudflare; ax stays reachable + w.floor.down = true + const J = w.floor.jobs.get("wf-test-1")! + await until("floor requeued a1 after lease + grace", () => J.state === "queued" && J.attempt === 2, 3000) + const floorStateAtReopen = { state: J.state, attempt: J.attempt, history: [...J.history] } + w.ax.gateways.set("halogen", gw) // the Gateway is back; the floor is still unreachable + await until("a1 created", () => (w.ax.updates.get("wf-test-1-a1") ?? 0) > 0, 3000).catch(() => {}) + const createdA1WhileExpired = (w.ax.updates.get("wf-test-1-a1") ?? 0) > 0 + await sleep(200) + const a1BeforeFloorReturned = w.ax.updates.get("wf-test-1-a1") ?? 0 // round 4 addition + w.floor.down = false + await until("a2 created", () => w.ax.tasks.has("wf-test-1-a2"), 3000).catch(() => {}) + await sleep(300) + const out = { + floorStateAtReopen, createdA1WhileExpired, axEvents: w.ax.events, + linkTrace: l!.logs.filter((x) => typeof x.leaseId === "string" || typeof x.task === "string").map((x) => `${x.ev}:${x.leaseId ?? x.task}${x.why ? ":" + x.why : ""}`), + heartbeatErrors: l!.logs.filter((x) => x.ev === "heartbeat-error").length, floorHistory: J.history + } + console.log(JSON.stringify(out)) + await l!.drain(); w.floor.close() + expect(createdA1WhileExpired).toBe(false) // fail closed: a lease known to be past renewTime + duration is not started + expect(a1BeforeFloorReturned).toBe(0) // round 4 addition: nothing of a1 before the floor returns + expect([...w.ax.updates.keys()]).toEqual(["wf-test-1-a2"]) // exactly one created attempt, a2, afterwards + expect(l!.has("lease-unconfirmed", "wf-test-1-a1")).toBe(true) +}) + +// Round 4 addition (finding 5, deadline half): a grant whose deadline has passed is refused with no create. The gate +// closes while the grant is in flight and the Heartbeats fail, so the floor cannot cancel at its deadline (the lease is +// orphaned, grace is long); only the link's own check can end it. +it("R4-5b: a grant past its deadline is failed deadline-exceeded before any UpdateTask", async () => { + const w = world({ graceSeconds: 400 }) + w.floor.enqueue(job("d", { "timeout-minutes": 1 })) // 60 s x 20 ms = 1.2 s deadline + const gw = w.ax.gateways.get("halogen")! + let hbDown = false + let l: ReturnType + const fetch: typeof globalThis.fetch = async (input, init) => { + const b = await bodyOf(input, init) + if (hbDown && b.includes('"Heartbeat"')) throw new TypeError("fetch failed") + const res = await w.floor.fetch(input, init) + if (!hbDown && b.includes('"Lease"') && (await res.clone().text()).includes("wf-test-d-a1")) { + hbDown = true + w.ax.gateways.delete("halogen") + await until("gateway-missing", () => l.has("gateway-missing"), 3000) + } + return res + } + l = start(w, undefined, { maxInFlight: 1 }, fetch) + await until("waiting", () => l.has("waiting-for-slot", "wf-test-d-a1") || l.has("lease-unconfirmed", "wf-test-d-a1"), 3000) + await sleep(1400) + w.ax.gateways.set("halogen", gw) + await until("verdict", () => l.has("verdict", "wf-test-d-a1"), 3000) + const v = l.logs.find((x) => x.ev === "verdict" && x.leaseId === "wf-test-d-a1")! + await l.drain(); w.floor.close() + expect(v.reason).toBe("deadline-exceeded") + expect(w.ax.updates.get("wf-test-d-a1") ?? 0).toBe(0) +}) diff --git a/pkgs/substrate-link/src/apps/link/test/review-r4-fiber-defect.test.ts b/pkgs/substrate-link/src/apps/link/test/review-r4-fiber-defect.test.ts new file mode 100644 index 000000000..f68fcaa37 --- /dev/null +++ b/pkgs/substrate-link/src/apps/link/test/review-r4-fiber-defect.test.ts @@ -0,0 +1,96 @@ +// Round 4 regression test, ported from /home/tom/today/evals-2026-09-23/link/scratch-r4-2/fiber-defect.repro.ts (assertions kept; additions marked). +// Effect idiom: dispatch ends in `Effect.catchCause(... log("job-fiber-died"))`, so a defect in one job fiber is logged +// and swallowed while the process lives on. Here one journal write fails once (ENOSPC, then the disk recovers). The +// grant stays in `held()`, every Heartbeat renews it on the floor, nothing re-dispatches it, and nothing reports it: +// the job never runs and never ends, and it holds a floor WIP slot and a local slot until the process restarts. +import { mkdtempSync } from "node:fs" +import { tmpdir } from "node:os" +import { join } from "node:path" +import { Deferred, Effect, Fiber } from "effect" +import { expect, it } from "vitest" +import { rpcFloor } from "../src/floor.ts" +import { Journal } from "../src/journal.ts" +import { runLink } from "../src/link.ts" +import type { LinkConfig } from "../src/link.ts" +import { job, sleep, until, world, SEC } from "./review-r4-harness.ts" + +it("one failed journal write in a dispatch fiber strands the lease for the life of the process", async () => { + const w = world({ cap: 1 }) + w.floor.enqueue(job("1")) + w.floor.enqueue(job("2")) + const dir = mkdtempSync(join(tmpdir(), "r4-2-defect-")) + const j = Journal.open(dir) + const orig = j.append.bind(j) + let failed = 0 + ;(j as any).append = (e: any, t?: number) => { + if (e.ev === "creating" && failed === 0) { failed++; throw Object.assign(new Error("ENOSPC: no space left on device, write"), { code: "ENOSPC" }) } + return orig(e, t) + } + const logs: Array = [] + const stop = Effect.runSync(Deferred.make()) + const cfg: LinkConfig = { + holder: "nas-link-1", maxInFlight: 1, servedLabels: ["seat:halogen", "runtime:gvisor"], + shape: { atespace: "fleet", image: "ax-agent", gateway: "halogen", command: () => ["ax-agent", "pi"] }, + completion: "auto", secondMs: SEC, resyncMs: 30, pendingTimeoutMs: 1500, deleteAfterMs: 0, deadlineBackstopMs: 300, + createAttempts: 3, outboxBackoffMs: [20, 100], initialPollSeconds: 1, initialHeartbeatSeconds: 2, fenceTimeoutMs: 300, resultReadTries: 3 + } + const fiber = Effect.runFork(Effect.scoped(Effect.gen(function*() { + const floor = yield* rpcFloor({ url: "http://floor.test", token: "tok-nas", sessionId: "s:defect", fetch: w.floor.fetch }) + return yield* runLink(cfg, { ax: w.ax, floor, journal: j, log: (ev, f) => logs.push({ ev, ...f }), stop, floorUrl: "http://floor.test" }) + }))) + try { + await until("fiber died", () => logs.some((x) => x.ev === "job-fiber-died")) + await sleep(3000) // 25 lease durations (6 s x 20 ms); the disk is fine again + const J1 = w.floor.jobs.get("wf-test-1")!, J2 = w.floor.jobs.get("wf-test-2")! + const out = { + linkAlive: fiber.pollUnsafe() === undefined, failedWrites: failed, + job1: { state: J1.state, attempt: J1.attempt, history: J1.history, task: w.ax.tasks.get("wf-test-1-a1")?.phase ?? "absent" }, + job2: { state: J2.state, history: J2.history }, + held: j.held(), heartbeats: w.floor.calls.filter((c) => c.rpc === "Heartbeat").length, + completes: w.floor.calls.filter((c) => c.rpc === "Complete").length, + events: logs.map((x) => x.ev).filter((e, i, a) => a.indexOf(e) === i) + } + console.log(JSON.stringify(out)) + // Expected: the lease is failed back (retryable) or the process crashes and systemd restarts it; either frees job 1. + expect(J1.state === "done" || J1.attempt > 1 || w.ax.tasks.has("wf-test-1-a1") || !out.linkAlive).toBe(true) + } finally { + Effect.runSync(Deferred.succeed(stop, undefined)) + await Promise.race([Effect.runPromise(Fiber.await(fiber)), sleep(1500)]) + w.floor.close() + } +}) + +// Round 4 addition (finding 7): when even the fail-back cannot be journaled, the process dies (exit 1 under systemd) +// instead of renewing a lease nothing will ever run. +it("R4-7b: a dispatch defect whose fail-back also fails ends the link", async () => { + const w = world({ cap: 1 }) + w.floor.enqueue(job("1")) + const dir = mkdtempSync(join(tmpdir(), "r4-fix-defect-")) + const j = Journal.open(dir) + const orig = j.append.bind(j) + ;(j as any).append = (e: any, t?: number) => { + if (e.ev === "creating" || e.ev === "report") throw Object.assign(new Error("ENOSPC: no space left on device, write"), { code: "ENOSPC" }) + return orig(e, t) + } + const logs: Array = [] + const stop = Effect.runSync(Deferred.make()) + const cfg: LinkConfig = { + holder: "nas-link-1", maxInFlight: 1, servedLabels: ["seat:halogen", "runtime:gvisor"], + shape: { atespace: "fleet", image: "ax-agent", gateway: "halogen", command: () => ["ax-agent", "pi"] }, + completion: "auto", secondMs: SEC, resyncMs: 30, pendingTimeoutMs: 1500, deleteAfterMs: 0, deadlineBackstopMs: 300, + createAttempts: 3, outboxBackoffMs: [20, 100], initialPollSeconds: 1, initialHeartbeatSeconds: 2, fenceTimeoutMs: 300, resultReadTries: 3 + } + const fiber = Effect.runFork(Effect.scoped(Effect.gen(function*() { + const floor = yield* rpcFloor({ url: "http://floor.test", token: "tok-nas", sessionId: "s:defect2", fetch: w.floor.fetch }) + return yield* runLink(cfg, { ax: w.ax, floor, journal: j, log: (ev, f) => logs.push({ ev, ...f }), stop, floorUrl: "http://floor.test" }) + }))) + try { + const exit = await Promise.race([Effect.runPromise(Fiber.await(fiber)), sleep(3000).then(() => undefined)]) + expect(exit).toBeDefined() + expect(exit!._tag).toBe("Failure") + expect(logs.some((x) => x.ev === "link-defect-fatal")).toBe(true) + } finally { + Effect.runSync(Deferred.succeed(stop, undefined)) + w.floor.close() + } +}) diff --git a/pkgs/substrate-link/src/apps/link/test/review-r4-harness.ts b/pkgs/substrate-link/src/apps/link/test/review-r4-harness.ts new file mode 100644 index 000000000..94e35fc0b --- /dev/null +++ b/pkgs/substrate-link/src/apps/link/test/review-r4-harness.ts @@ -0,0 +1,59 @@ +// Round 4 fix pass: the reviewers' scratch harness (scratch-r4-0/1/2, identical but for the temp prefix), moved into +// the worktree. The only change: the job carries the FIELD-MAP 5a journal-key annotation jobs.ts now requires. +import { mkdtempSync } from "node:fs" +import { tmpdir } from "node:os" +import { join } from "node:path" +import { Deferred, Effect, Fiber } from "effect" +import type { AgentJob } from "../src/contract.ts" +import { rpcFloor } from "../src/floor.ts" +import { Journal } from "../src/journal.ts" +import { runLink } from "../src/link.ts" +import type { LinkConfig } from "../src/link.ts" +import { FakeAx } from "./fake-ax.ts" +import { FakeFloor } from "./fake-floor.ts" +import type { FloorConfig } from "./fake-floor.ts" +export const SEC = 20 +const A = "ultracode.mecattaf.dev/" +export const job = (n: string, spec: Record = {}): AgentJob => ({ + apiVersion: "ultracode.mecattaf.dev/v1alpha1", kind: "AgentJob", + metadata: { name: `wf-test-${n}`, labels: { [A + "run-id"]: "wf-test", [A + "workflow"]: "link-test", [A + "phase-index"]: "1" }, + annotations: { [A + "run-id-raw"]: "wf_test", [A + "label"]: `probe:${n}`, [A + "item-key"]: `wf_test#${n}`, [A + "journal-key"]: `${"cd".repeat(32)}:1`, [A + "phase-title"]: "Probe" } }, + spec: { "runs-on": ["seat:halogen", "runtime:gvisor"], + with: { prompt: `say ${n}`, prompt_ref: { sha256: "ab".repeat(32), bytes: 5, uri: `journal://wf_test/${n}/prompt.md` }, model: "halogen-qwen3.8-flash-next" }, ...spec } as any +}) +export const world = (o: Partial = {}) => ({ + floor: new FakeFloor({ cap: 2, leaseSeconds: 6, graceSeconds: 15, pollSeconds: 1, heartbeatSeconds: 2, maxAttempts: 3, secondMs: SEC, tokens: { "tok-nas": "nas-link-1" }, ...o }), + ax: new FakeAx() +}) +export type World = ReturnType +export function start(w: World, dir = mkdtempSync(join(tmpdir(), "r4-fix-")), over: Partial = {}, fetch?: typeof globalThis.fetch) { + const logs: Array> = [] + const stop = Effect.runSync(Deferred.make()) + const cfg: LinkConfig = { + holder: "nas-link-1", maxInFlight: 2, servedLabels: ["seat:halogen", "runtime:gvisor"], + shape: { atespace: "fleet", image: "ax-agent", gateway: "halogen", command: () => ["ax-agent", "pi"] }, + completion: "auto", secondMs: SEC, resyncMs: 30, pendingTimeoutMs: 1500, deleteAfterMs: 0, deadlineBackstopMs: 300, + createAttempts: 3, outboxBackoffMs: [20, 100], initialPollSeconds: 1, initialHeartbeatSeconds: 2, fenceTimeoutMs: 300, resultReadTries: 3, ...over + } + const fiber = Effect.runFork(Effect.scoped(Effect.gen(function*() { + const floor = yield* rpcFloor({ url: "http://floor.test", token: "tok-nas", sessionId: `s:${dir}`, fetch: fetch ?? w.floor.fetch }) + return yield* runLink(cfg, { ax: w.ax, floor, journal: Journal.open(dir), log: (ev, f) => logs.push({ t: Date.now(), ev, ...f }), stop, floorUrl: "http://floor.test" }) + }))) + return { + dir, logs, fiber, + has: (ev: string, leaseId?: string) => logs.some((l) => l.ev === ev && (leaseId === undefined || l.leaseId === leaseId)), + drain: () => { Effect.runSync(Deferred.succeed(stop, undefined)); return Promise.race([Effect.runPromise(Fiber.await(fiber)), sleep(1500)]) } + } +} +export const sleep = (ms: number) => new Promise((r) => setTimeout(r, ms)) +export const until = async (what: string, pred: () => boolean, ms = 5000) => { + const t0 = Date.now() + while (!pred()) { if (Date.now() - t0 > ms) throw new Error(`timeout waiting for: ${what}`); await new Promise((r) => setTimeout(r, 5)) } +} +export const bodyOf = async (input: unknown, init?: RequestInit) => { + const b = init?.body + if (typeof b === "string") return b + if (b instanceof Uint8Array) return new TextDecoder().decode(b) + if (b) return await new Response(b as any).text() + return (input instanceof Request) ? await input.clone().text() : "" +} diff --git a/pkgs/substrate-link/src/apps/link/test/review-r4-heartbeat-envelope.test.ts b/pkgs/substrate-link/src/apps/link/test/review-r4-heartbeat-envelope.test.ts new file mode 100644 index 000000000..81b5de396 --- /dev/null +++ b/pkgs/substrate-link/src/apps/link/test/review-r4-heartbeat-envelope.test.ts @@ -0,0 +1,134 @@ +// Round 4 regression test, ported from /home/tom/today/evals-2026-09-23/link/scratch-r4-2/heartbeat-envelope.repro.ts (assertions kept; additions marked). +// Schema compatibility: the round-3 fix made the Lease and Complete success envelopes lenient ("a changed stats shape +// would otherwise fail the decode"), but Heartbeat still decodes with the strict L1 schema. The same drift on the +// Heartbeat reply (here `stats: null`, which Lease tolerates) makes every heartbeat a transient error: held leases are +// never renewed, the floor requeues a healthy running attempt as a2, and the link fences a1 and runs the job again. +import { expect, it } from "vitest" +import { job, start, until, world, bodyOf, sleep } from "./review-r4-harness.ts" + +const drift = (rpc: string, floorFetch: typeof globalThis.fetch): typeof globalThis.fetch => async (input, init) => { + const body = await bodyOf(input, init) + const res = await floorFetch(input, init) + if (!body.includes(`"${rpc}"`)) return res + const text = await res.text() + let m: any + try { m = JSON.parse(text) } catch { return new Response(text, { status: res.status, headers: res.headers }) } + const walk = (x: any) => { if (x && typeof x === "object") { if ("stats" in x && "seq" in (x.stats ?? {})) x.stats = null; for (const v of Object.values(x)) walk(v) } } + walk(m) + return new Response(JSON.stringify(m), { status: res.status, headers: res.headers }) +} + +for (const rpc of ["Lease", "Heartbeat"]) { + it(`stats: null on every ${rpc} reply`, async () => { + const w = world() + w.floor.enqueue(job("1")) + const l = start(w, undefined, {}, drift(rpc, w.floor.fetch)) + try { + await until("a1 running", () => w.ax.tasks.get("wf-test-1-a1")?.phase === "Running") + await sleep(1500) // about 12 lease durations (6 s x 20 ms) with a1 healthy and running + const J = w.floor.jobs.get("wf-test-1")! + const hbErr = l.logs.filter((x) => x.ev === "heartbeat-error").length + const out = { rpc, floorState: J.state, attempt: J.attempt, history: J.history, heartbeatErrors: hbErr, + a1: w.ax.tasks.get("wf-test-1-a1")?.phase ?? "deleted", a2: w.ax.tasks.get("wf-test-1-a2")?.phase ?? "absent", + events: l.logs.map((x) => x.ev).filter((e, i, a) => a.indexOf(e) === i) } + console.log(JSON.stringify(out)) + // The expected behaviour, as for the Lease drift: a healthy a1 keeps its lease and nothing reruns. + expect(J.attempt).toBe(1) + expect(w.ax.tasks.get("wf-test-1-a2")).toBeUndefined() + } finally { await l.drain(); w.floor.close() } + }) +} + +// The consequence: the floor renews on its side, but the link never reads `cancelRequested`, `lost` or `renewed`. +for (const rpc of ["Lease", "Heartbeat"]) { + it(`cancel with stats: null on every ${rpc} reply`, async () => { + const w = world() + w.floor.enqueue(job("1")) + const l = start(w, undefined, {}, drift(rpc, w.floor.fetch)) + try { + await until("a1 running", () => w.ax.tasks.get("wf-test-1-a1")?.phase === "Running") + w.floor.cancel("wf-test-1") + await sleep(1500) + const J = w.floor.jobs.get("wf-test-1")! + const out = { rpc, floorState: J.state, result: J.result ?? null, cancel: J.cancel ?? null, history: J.history, + a1: w.ax.tasks.get("wf-test-1-a1")?.phase ?? "deleted", cancelLogged: l.has("cancel") } + console.log(JSON.stringify(out)) + expect(J.state).toBe("done") + expect(J.result).toBe("cancelled") + } finally { await l.drain(); w.floor.close() } + }) +} + +// B13: a grant journaled by a previous process and never created is resumed only once a Heartbeat reply renews it. +for (const rpc of ["Lease", "Heartbeat"]) { + it(`restart resume with stats: null on every ${rpc} reply`, async () => { + const { Effect } = await import("effect") + const w = world() + w.floor.enqueue(job("1")) + const orig = w.ax.createTask + ;(w.ax as any).createTask = () => Effect.never // first process: UpdateTask never answers, then the process stops + const l1 = start(w) + const dir = l1.dir + await until("a1 creating", () => l1.has("leased")) + await sleep(100) + await l1.drain() + ;(w.ax as any).createTask = orig + const J = w.floor.jobs.get("wf-test-1")! + const l2 = start(w, dir, {}, drift(rpc, w.floor.fetch)) + try { + await sleep(2000) + const out = { rpc, floorState: J.state, attempt: J.attempt, history: J.history, result: J.result ?? null, + a1: w.ax.tasks.get("wf-test-1-a1")?.phase ?? "absent", resumed: l2.has("resume"), + heartbeatErrors: l2.logs.filter((x) => x.ev === "heartbeat-error").length, + events: l2.logs.map((x) => x.ev).filter((e, i, a) => a.indexOf(e) === i) } + console.log(JSON.stringify(out)) + expect(J.state === "done" || w.ax.tasks.get("wf-test-1-a1") !== undefined).toBe(true) + } finally { await l2.drain(); w.floor.close() } + }) +} + +// Round 4 addition (finding 6): a Heartbeat reply that omits `cancelRequested` (or has it null) is read for the rest. +const omit = (key: string, value: "omit" | null, floorFetch: typeof globalThis.fetch): typeof globalThis.fetch => async (input, init) => { + const body = await bodyOf(input, init) + const res = await floorFetch(input, init) + if (!body.includes('"Heartbeat"')) return res + const text = await res.text() + let m: any + try { m = JSON.parse(text) } catch { return new Response(text, { status: res.status, headers: res.headers }) } + const walk = (x: any) => { if (x && typeof x === "object") { if (key in x && "renewed" in x) { if (value === "omit") delete x[key]; else x[key] = value } for (const v of Object.values(x)) walk(v) } } + walk(m) + return new Response(JSON.stringify(m), { status: res.status, headers: res.headers }) +} +for (const value of ["omit", null] as const) { + it(`R4-6b: restart resume with cancelRequested ${value === "omit" ? "omitted" : "null"} on every Heartbeat reply`, async () => { + const { Effect } = await import("effect") + const w = world() + w.floor.enqueue(job("1")) + const orig = w.ax.createTask + ;(w.ax as any).createTask = () => Effect.never + const l1 = start(w) + await until("a1 creating", () => l1.has("leased")) + await sleep(100) + await l1.drain() + ;(w.ax as any).createTask = orig + const l2 = start(w, l1.dir, {}, omit("cancelRequested", value, w.floor.fetch)) + try { + await until("a1 resumed and created", () => w.ax.tasks.get("wf-test-1-a1") !== undefined, 3000) + expect(l2.has("resume", "wf-test-1-a1")).toBe(true) + expect(l2.logs.filter((x) => x.ev === "heartbeat-error").length).toBe(0) + } finally { await l2.drain(); w.floor.close() } + }) +} +it("R4-6c: a lost lease is still acted on when the Heartbeat reply omits stats and cancelRequested", async () => { + const w = world() + w.floor.enqueue(job("1")) + const both: typeof globalThis.fetch = omit("cancelRequested", "omit", drift("Heartbeat", w.floor.fetch)) + const l = start(w, undefined, {}, both) + try { + await until("a1 running", () => w.ax.tasks.get("wf-test-1-a1")?.phase === "Running") + const J = w.floor.jobs.get("wf-test-1")! + J.holder = "someone-else" // the floor answers lost:[a1] from now on + await until("lost", () => l.has("lost", "wf-test-1-a1"), 2000) + await until("a1 deleted", () => w.ax.deletes.includes("wf-test-1-a1"), 2000) + } finally { await l.drain(); w.floor.close() } +}) diff --git a/pkgs/substrate-link/src/apps/link/test/review-r4-journal-key.test.ts b/pkgs/substrate-link/src/apps/link/test/review-r4-journal-key.test.ts new file mode 100644 index 000000000..4f39aed63 --- /dev/null +++ b/pkgs/substrate-link/src/apps/link/test/review-r4-journal-key.test.ts @@ -0,0 +1,48 @@ +// Round 4 regression test for finding 8 (repro: /home/tom/today/evals-2026-09-23/link/scratch-r4-2/journal-key.repro.ts). +// AX_CONWIP_JOURNAL_KEY must be the FIELD-MAP 5a journal key ("sha256 of canonical [prompt, opts minus label] plus the +// occurrence"), carried on the AgentJob as the `ultracode.mecattaf.dev/journal-key` annotation, never the prompt +// digest. The expected values are pinned from the ax-conwip taskspec (journalKey in +// /home/tom/mecattaf/ax-conwip-wt-integration/src/taskspec.ts at b42b310, run 2026-09-23), so this test does not +// import another worktree. +import { createHash } from "node:crypto" +import { expect, it } from "vitest" +import { axTaskFromGrant } from "../src/jobs.ts" +import { job } from "./review-r4-harness.ts" + +const TASKSPEC = { + halogen1: "29a9c6cb8956c07073345109161d0e7eaa08708f4c2039f4e4af7a09d212ee07:1", // journalKey("same prompt", {model: halogen}, 1) + opus1: "bd381a2d66f7a1d93c7df296954218de4c9749a0feb9d94719b9c484f8c06718:1", // journalKey("same prompt", {model: opus}, 1) + halogen2: "29a9c6cb8956c07073345109161d0e7eaa08708f4c2039f4e4af7a09d212ee07:2" // the same call, second occurrence +} +const sha = (s: string) => createHash("sha256").update(s).digest("hex") +const A = "ultracode.mecattaf.dev/" +const grant = (n: string, model: string, key: string | undefined) => { + const j: any = job(n) + j.spec.with.prompt = "same prompt" + j.spec.with.prompt_ref = { sha256: sha("same prompt"), bytes: 11, uri: `journal://wf_test/${n}/prompt.md` } + j.spec.with.model = model + if (key === undefined) delete j.metadata.annotations[A + "journal-key"] + else j.metadata.annotations[A + "journal-key"] = key + return { leaseId: `wf-test-${n}-a1`, attempt: 1, job: j, lease: { holderIdentity: "h", leaseDurationSeconds: 90, acquireTime: 1, renewTime: 1, leaseTransitions: 0 } } +} +const shape = { atespace: "fleet", image: "ax-agent", gateway: "halogen", command: () => ["ax-agent", "pi"] } +const env = (g: any) => { const b: any = axTaskFromGrant(g, shape); expect(b._tag).toBe("ok"); return Object.fromEntries(b.task.spec.env.map((e: any) => [e.name, e.value])) } + +it("R4-8a: AX_CONWIP_JOURNAL_KEY is the taskspec journal key for two calls that share a prompt", () => { + const a = env(grant("1", "halogen-qwen3.8-flash-next", TASKSPEC.halogen1)) + const b = env(grant("2", "claude-opus-5-5", TASKSPEC.opus1)) + const c = env(grant("3", "halogen-qwen3.8-flash-next", TASKSPEC.halogen2)) + expect(a.AX_CONWIP_JOURNAL_KEY).toBe(TASKSPEC.halogen1) + expect(b.AX_CONWIP_JOURNAL_KEY).toBe(TASKSPEC.opus1) + expect(c.AX_CONWIP_JOURNAL_KEY).toBe(TASKSPEC.halogen2) + expect(a.AX_CONWIP_JOURNAL_KEY).not.toBe(a.AX_CONWIP_PROMPT_SHA256) + expect(a.AX_CONWIP_PROMPT_SHA256).toBe(b.AX_CONWIP_PROMPT_SHA256) // the prompt digest is shared; the journal key is not +}) + +it("R4-8b: a grant without the annotation, or with a malformed one, is refused pre-start/invalid-spec", () => { + for (const key of [undefined, "", sha("same prompt"), `${sha("x")}:`, `${sha("x").toUpperCase()}:1`]) { + const b = axTaskFromGrant(grant("4", "halogen-qwen3.8-flash-next", key) as any, shape) + expect(b._tag).toBe("refused") + if (b._tag === "refused") expect(b.reason).toBe("pre-start/invalid-spec") + } +}) diff --git a/pkgs/substrate-link/src/apps/link/test/review-r4-overtake.test.ts b/pkgs/substrate-link/src/apps/link/test/review-r4-overtake.test.ts new file mode 100644 index 000000000..e1274ca32 --- /dev/null +++ b/pkgs/substrate-link/src/apps/link/test/review-r4-overtake.test.ts @@ -0,0 +1,78 @@ +// Round 4 regression test, ported from /home/tom/today/evals-2026-09-23/link/scratch-r4-0/overtake.repro.ts (assertions kept; additions marked). +// Reply ordering: the regrant of a2 (gen 2) overtakes the Complete reply that withdrew a2 (gen 1). Set-up is R2-5a +// (review-r2.test.ts): a1 fails retryable while a2 gen 1 is leased to this link and waits for a1's verdict. a1's first +// Complete answers 500 once (server-error), which by design stops gating Lease (round 2), so a Lease with capacity 1 is +// in flight while the retried Complete is applied. The floor withdraws gen 1, requeues gen 2 and grants it on that +// Lease; the Lease reply lands before the Complete reply. The link still holds gen 1 unreleased, so gen 2 is dropped as +// `duplicate-grant`; the Complete reply then releases gen 1. Nothing of a2 is ever created, although the floor has +// gen 2 leased to this link: the job waits for lease expiry and is released `omitted` (an infra release). +import { expect, it } from "vitest" +import { bodyOf, job, sleep, start, until, world } from "./review-r4-harness.ts" +import type { World } from "./review-r4-harness.ts" + +const rpcOf = async (input: unknown, init?: RequestInit) => { const b = await bodyOf(input, init); return b.includes('"Complete"') ? "Complete" : b.includes('"Lease"') ? "Lease" : b.includes('"Heartbeat"') ? "Heartbeat" : "?" } +const failRetryable = (w: World, name: string) => { + const t = w.ax.tasks.get(name)! + t.exited = true; t.phase = "Failed" + t.conditions = [{ type: "Ready", status: "False", reason: "ResourceExhausted", message: "ResourceExhausted: no free worker" }] +} + +// Round 4 addition: also against a floor that omits withdrewTransitions (the link then uses the generation of a2 it +// held when it SENT a1's verdict, not the regrant it holds when the reply lands). +for (const sendWithdrewGen of [true, false]) +it(`the regrant of a2 is created even when its Lease reply lands before the Complete reply that withdrew gen 1 (withdrewTransitions ${sendWithdrewGen ? "sent" : "omitted"})`, async () => { + const w = world({ sendWithdrewGen }); w.floor.enqueue(job("1")) + const J = () => w.floor.jobs.get("wf-test-1")! + let holdLease = false, failHeartbeat = false, failComplete = false, fiveHundredOnce = false + let l: ReturnType + let armed = false, leaseHeld = false + const fetch: typeof globalThis.fetch = async (input, init) => { + const body = await bodyOf(input, init) + const rpc = await rpcOf(input, init) + if (rpc === "Heartbeat" && (failHeartbeat || armed)) throw new TypeError("fetch failed") // no Heartbeat in the window + if (rpc === "Complete" && failComplete) return new Response("upstream timeout", { status: 503 }) + if (rpc === "Complete" && fiveHundredOnce) { fiveHundredOnce = false; armed = true; return new Response("internal error", { status: 500 }) } + if (rpc === "Lease") while (holdLease) await sleep(2) + if (rpc === "Lease" && armed && /"capacity":[1-9]/.test(body) && !leaseHeld) { + leaseHeld = true // a Lease sent before the retried Complete, delivered after it (network reordering) + await until("the retried Complete was applied", () => J().history.includes("withdrawn:a2"), 3000).catch(() => {}) + // Round 4 addition: longer than the dispatch wait period (at most 2 x SEC), so the gen-1 dispatch is inside, or + // waiting on, the outbox lock when the Complete reply lands, and reaches its withdrawn branch deterministically + await sleep(80) + } + const res = await w.floor.fetch(input as any, { ...init, body }) + if (rpc === "Complete" && armed) { + await until("the Lease reply reached the link", () => l.has("duplicate-grant", "wf-test-1-a2") || l.has("regrant", "wf-test-1-a2"), 1500).catch(() => {}) + armed = false + } + return res + } + l = start(w, undefined, {}, fetch) + await until("running", () => w.ax.tasks.get("wf-test-1-a1")?.phase === "Running") + w.floor.down = true + await until("floor requeued attempt 2", () => J().state === "queued" && J().attempt === 2, 4000) + failHeartbeat = true; holdLease = true; failComplete = true + w.floor.down = false + await sleep(80) + failRetryable(w, "wf-test-1-a1") + await until("verdict written", () => l.has("verdict", "wf-test-1-a1")) + holdLease = false + await until("a2 received", () => l.has("leased", "wf-test-1-a2"), 4000) + await until("dispatch waits for the verdict", () => l.has("supersede-waits-for-verdict", "wf-test-1-a2")) + fiveHundredOnce = true + failComplete = false; failHeartbeat = false + await until("a2 created, or 1.5 s", () => w.ax.updates.has("wf-test-1-a2"), 1500).catch(() => {}) + await sleep(300) + const trace = l.logs.filter((x) => x.leaseId === "wf-test-1-a2" || x.task === "wf-test-1-a2" || x.ev === "complete" || x.ev === "outbox-keep" || x.ev === "lease-gated-by-outbox").map((x) => `${x.ev}:${x.leaseId ?? x.task}${x.why ? ":" + x.why : ""}${x.withdrew ? ":withdrew=" + x.withdrew : ""}`) + const out = { leaseHeld, floor: J().history, state: J().state, attempt: J().attempt, infraSpent: J().infraSpent, a2Updates: w.ax.updates.get("wf-test-1-a2") ?? 0, trace } + console.log(JSON.stringify(out)) + if (out.a2Updates !== 1 || out.floor.includes("requeue:omitted")) console.log("FULL", JSON.stringify(l.logs.map((x) => ({ ...x, cause: undefined }))), JSON.stringify(w.floor.calls.slice(-40))) + await l.drain(); w.floor.close() + expect(out.a2Updates).toBe(1) + expect(out.floor).not.toContain("requeue:omitted") // round 4 addition: no infra release spent on the stall + expect(out.leaseHeld).toBe(true) + // Round 4 addition: the gen-1 dispatch never releases the regrant (the withdrawn branch is bound to its own grant) + const a2 = l.logs.filter((x) => x.leaseId === "wf-test-1-a2").map((x) => String(x.ev)) + expect(a2.slice(a2.indexOf("regrant"))).not.toContain("withdrawn") + expect(w.ax.deletes).not.toContain("wf-test-1-a2") +}) diff --git a/pkgs/substrate-link/src/apps/link/test/review-r4-p1stale.test.ts b/pkgs/substrate-link/src/apps/link/test/review-r4-p1stale.test.ts new file mode 100644 index 000000000..17e776007 --- /dev/null +++ b/pkgs/substrate-link/src/apps/link/test/review-r4-p1stale.test.ts @@ -0,0 +1,74 @@ +// Round 4 regression test, ported from /home/tom/today/evals-2026-09-23/link/scratch-r4-1/p1stale.repro.ts (assertions kept; additions marked). +// Round 4, lens capacity gate: canCreate() needs "P1 proven", but the probe runs only on an ax-up transition +// (link.ts resync: `if (wasDown) serverP1 = undefined`). ax-server is a replicas:1 Deployment with the default +// RollingUpdate strategy (upstream deploy/ax-server.yaml), so a rollout to a server without P1 happens with no failed +// ListTasks, and the link keeps creating Tasks it can never read a verdict from. +import { Effect } from "effect" +import { expect, it } from "vitest" +import { job, sleep, start, until, world } from "./review-r4-harness.ts" + +it("A: ax rolled to stock v0.3.0 (server and controller) with no ListTasks failure", async () => { + const w = world() + w.floor.enqueue(job("1")) + const l = start(w) + await until("a1 running", () => w.ax.tasks.get("wf-test-1-a1")?.phase === "Running") + const probesBefore = l.logs.filter((x) => x.ev === "completion-probe").length + w.ax.p1 = false // the rollout: GetTaskResult is now UNIMPLEMENTED and no terminal phase is ever written + await until("p1-missing logged", () => l.has("p1-missing"), 1000) // round 4 fix: the probe runs on every resync + const leasesAtFlip = w.floor.calls.filter((c) => c.rpc === "Lease").length + await sleep(100) // several resyncs (30 ms) and Lease polls pass; none fails + w.floor.enqueue(job("2")) + await sleep(1500) // round 4: job 2 is never created on the stock server (the repro waited for its create here) + const J2 = w.floor.jobs.get("wf-test-2")! + const out = { + probesBefore, probesAfter: l.logs.filter((x) => x.ev === "completion-probe").length, + p1MissingLogged: l.has("p1-missing"), + job2: { state: J2.state, attempt: J2.attempt, history: J2.history }, + task2Phase: w.ax.tasks.get("wf-test-2-a1")?.phase, task2Exited: (w.ax.tasks.get("wf-test-2-a1") as any)?.exited, + verdictJob2: l.has("verdict", "wf-test-2-a1"), + leaseCapacitiesAfterRollout: [...new Set(w.floor.calls.filter((c) => c.rpc === "Lease").slice(-10).map((c) => c.capacity))] + } + console.log("A", JSON.stringify(out)) + const leasesAfter = w.floor.calls.filter((c) => c.rpc === "Lease").slice(leasesAtFlip + 1) // round 4 addition + await l.drain(); w.floor.close() + // Fail-closed expectation: after the P1 route disappears the link leases and creates nothing new. + expect(w.ax.updates.get("wf-test-2-a1") ?? 0).toBe(0) + expect(out.p1MissingLogged).toBe(true) + expect(leasesAfter.length).toBeGreaterThan(0) + expect(leasesAfter.every((c) => c.capacity === 0)).toBe(true) + expect(J2.state).toBe("queued") +}) + +it("B: ax-server rolled to stock, the P1 controller still writes Completed", async () => { + const w = world() + w.floor.enqueue(job("1")) + const l = start(w) + await until("a1 running", () => w.ax.tasks.get("wf-test-1-a1")?.phase === "Running") + let stockServer = false + const orig = w.ax.getTaskResult + ;(w.ax as any).getTaskResult = (n: string) => stockServer ? Effect.succeed("unimplemented" as const) : orig(n) + stockServer = true + w.ax.finish("wf-test-1-a1", 0, { answer: 1 }) // Completed, result stored by the controller, unreadable via ax-server + for (let i = 0; i < 40; i++) { // every attempt the floor grants finishes with a success the link cannot read + for (const [name, t] of w.ax.tasks) if (t.phase === "Running" && !t.exited) w.ax.finish(name, 0, { answer: name }) + await sleep(50) + } + const J = w.floor.jobs.get("wf-test-1")! + const out = { + probes: l.logs.filter((x) => x.ev === "completion-probe").length, p1MissingLogged: l.has("p1-missing"), + creates: [...w.ax.updates.keys()], job1: { state: J.state, result: (J as any).result, output: (J as any).output, history: J.history }, + verdicts: l.logs.filter((x) => x.ev === "verdict").map((x) => `${x.leaseId}:${x.result}:${x.reason ?? ""}`) + } + console.log("B", JSON.stringify(out)) + // Round 4 addition: the Task is kept, uncharged, and read once P1 is back: the success is reported, not thrown away + stockServer = false + await until("job 1 done", () => J.state === "done", 3000) + const leasesUnimpl = w.floor.calls.filter((c) => c.rpc === "Lease") + await l.drain(); w.floor.close() + expect(out.creates.length).toBe(1) // fail closed: nothing created once the server answered UNIMPLEMENTED + expect(out.p1MissingLogged).toBe(true) + expect(out.verdicts.some((v) => v.includes("result-unreadable"))).toBe(false) + expect((J as any).result).toBe("success") + expect((J as any).output).toEqual({ answer: 1 }) + expect(leasesUnimpl.length).toBeGreaterThan(0) +}) diff --git a/pkgs/substrate-link/src/apps/link/test/review-r4-restart-replay.test.ts b/pkgs/substrate-link/src/apps/link/test/review-r4-restart-replay.test.ts new file mode 100644 index 000000000..9307a75a5 --- /dev/null +++ b/pkgs/substrate-link/src/apps/link/test/review-r4-restart-replay.test.ts @@ -0,0 +1,143 @@ +// Round 4 regression test, ported from /home/tom/today/evals-2026-09-23/link/scratch-r4-0/restart-replay.repro.ts (assertions kept; additions marked). +// Kill -9 mid-Lease (B8): the floor applied the Lease and granted a1, the link journaled the requestKey but died +// before the reply was journaled. The restart takes longer than the lease (orphaned) and less than lease + grace. +// B8 says the journaled key is replayed so the grant is recovered. At startup the link sends its first Heartbeat +// (held set without a1) BEFORE it replays the key, so the floor releases the omitted orphan (rule 7 / L3 a), and the +// replay then returns nothing: one infra release is spent and the job reruns as a2. +import { readFileSync } from "node:fs" +import { Effect, Fiber } from "effect" +import { expect, it } from "vitest" +import { job, SEC, sleep, start, until, world, bodyOf } from "./review-r4-harness.ts" + +it("a journaled requestKey whose grant outlived the lease is recovered on restart", async () => { + const w = world() + w.floor.enqueue(job("r1")) + // link #1: the first capacity>0 Lease reaches the floor, then the process "dies" before the reply lands + let leased = false + const dying: typeof globalThis.fetch = async (input, init) => { + const body = await bodyOf(input, init) + if (/"Lease"/.test(body) && /"capacity":[1-9]/.test(body)) { + await w.floor.fetch(input as any, { ...init, body }) + leased = true + return new Promise(() => {}) // the reply never reaches the journal + } + return w.floor.fetch(input as any, { ...init, body }) + } + const l1 = start(w, undefined, {}, dying) + await until("floor granted a1", () => leased) + Effect.runFork(Fiber.interrupt(l1.fiber)) // kill -9: nothing after this point runs in link #1 + await sleep(50) + const journal1 = readFileSync(l1.dir + "/journal.jsonl", "utf8") + const keyJournaled = /"ev":"lease-key"/.test(journal1) && !/"ev":"lease-replied"/.test(journal1) + // downtime: past the lease (6 s * SEC = 120 ms) and well inside lease + grace (21 s * SEC = 420 ms) + await sleep(8 * SEC) + const stateAtRestart = w.floor.jobs.get("wf-test-r1")!.state + const l2 = start(w, l1.dir) + await until("a Task exists", () => w.ax.tasks.size > 0, 3000).catch(() => {}) + await sleep(200) + const j = w.floor.jobs.get("wf-test-r1")! + const out = { keyJournaled, stateAtRestart, history: j.history, infraSpent: j.infraSpent, tasks: [...w.ax.tasks.keys()], + l2: l2.logs.filter((l) => ["leased", "duplicate-grant", "created", "resume"].includes(String(l.ev))).map((l) => `${l.ev}:${l.leaseId}`) } + console.log(JSON.stringify(out)) + await l2.drain() + w.floor.close() + expect(keyJournaled).toBe(true) + expect(stateAtRestart).toBe("orphaned") + expect(j.history).not.toContain("requeue:omitted") // the grant under the journaled key was released by the first heartbeat + expect(out.tasks).toContain("wf-test-r1-a1") + expect(j.infraSpent).toBe(0) // round 4 addition + expect(out.tasks).not.toContain("wf-test-r1-a2") +}) + +it("control: the same kill with a restart shorter than the lease recovers a1", async () => { + const w = world() + w.floor.enqueue(job("r2")) + let leased = false + const dying: typeof globalThis.fetch = async (input, init) => { + const body = await bodyOf(input, init) + if (/"Lease"/.test(body) && /"capacity":[1-9]/.test(body)) { await w.floor.fetch(input as any, { ...init, body }); leased = true; return new Promise(() => {}) } + return w.floor.fetch(input as any, { ...init, body }) + } + const l1 = start(w, undefined, {}, dying) + await until("floor granted a1", () => leased) + Effect.runFork(Fiber.interrupt(l1.fiber)) + await sleep(2 * SEC) + const l2 = start(w, l1.dir) + await until("a Task exists", () => w.ax.tasks.size > 0, 3000).catch(() => {}) + const j = w.floor.jobs.get("wf-test-r2")! + console.log(JSON.stringify({ control: true, history: j.history, tasks: [...w.ax.tasks.keys()] })) + await l2.drain(); w.floor.close() + expect([...w.ax.tasks.keys()]).toContain("wf-test-r2-a1") +}) + +it("running process: the reply is lost and Lease stays unanswered past the lease while Heartbeat works", async () => { + const w = world() + w.floor.enqueue(job("r3")) + let first = true, blockUntil = 0 + const f: typeof globalThis.fetch = async (input, init) => { + const body = await bodyOf(input, init) + if (/"Lease"/.test(body)) { + if (first && /"capacity":[1-9]/.test(body)) { first = false; await w.floor.fetch(input as any, { ...init, body }); blockUntil = Date.now() + 10 * SEC; throw new TypeError("fetch failed") } + if (Date.now() < blockUntil) throw new TypeError("fetch failed") + } + return w.floor.fetch(input as any, { ...init, body }) + } + const l = start(w, undefined, {}, f) + await until("a Task exists", () => w.ax.tasks.size > 0, 4000).catch(() => {}) + await sleep(100) + const j = w.floor.jobs.get("wf-test-r3")! + console.log(JSON.stringify({ running: true, history: j.history, infraSpent: j.infraSpent, tasks: [...w.ax.tasks.keys()] })) + await l.drain(); w.floor.close() + expect(j.history).not.toContain("requeue:omitted") + expect([...w.ax.tasks.keys()]).toEqual(["wf-test-r3-a1"]) // round 4 addition +}) + +// Round 4 addition: the running-process half needs the floor's rule 6b. Against an L1-only floor (vouchByKey false) +// the residual is still there; this pins that the link-side fix alone does not claim it. +it("control: an L1-only floor that ignores pendingRequestKey still releases the omitted orphan", async () => { + const w = world() + w.floor.vouchByKey = false + w.floor.enqueue(job("r4")) + let first = true, blockUntil = 0 + const f: typeof globalThis.fetch = async (input, init) => { + const body = await bodyOf(input, init) + if (/"Lease"/.test(body)) { + if (first && /"capacity":[1-9]/.test(body)) { first = false; await w.floor.fetch(input as any, { ...init, body }); blockUntil = Date.now() + 10 * SEC; throw new TypeError("fetch failed") } + if (Date.now() < blockUntil) throw new TypeError("fetch failed") + } + return w.floor.fetch(input as any, { ...init, body }) + } + const l = start(w, undefined, {}, f) + await until("a Task exists", () => w.ax.tasks.size > 0, 4000).catch(() => {}) + const j = w.floor.jobs.get("wf-test-r4")! + await l.drain(); w.floor.close() + expect(j.history).toContain("requeue:omitted") +}) + +// Round 4 addition: the startup half does not depend on the floor. Against an L1-only floor (no rule 6b) the journaled +// key is still answered before the first Heartbeat, so the orphan granted under it is listed and re-adopted. +it("restart past the lease against an L1-only floor: the startup replay recovers a1", async () => { + const w = world() + w.floor.vouchByKey = false + w.floor.enqueue(job("r5")) + let leased = false + const dying: typeof globalThis.fetch = async (input, init) => { + const body = await bodyOf(input, init) + if (/"Lease"/.test(body) && /"capacity":[1-9]/.test(body)) { await w.floor.fetch(input as any, { ...init, body }); leased = true; return new Promise(() => {}) } + return w.floor.fetch(input as any, { ...init, body }) + } + const l1 = start(w, undefined, {}, dying) + await until("floor granted a1", () => leased) + Effect.runFork(Fiber.interrupt(l1.fiber)) + await sleep(8 * SEC) + const stateAtRestart = w.floor.jobs.get("wf-test-r5")!.state + const l2 = start(w, l1.dir) + await until("a Task exists", () => w.ax.tasks.size > 0, 3000).catch(() => {}) + await sleep(100) + const j = w.floor.jobs.get("wf-test-r5")! + await l2.drain(); w.floor.close() + expect(stateAtRestart).toBe("orphaned") + expect(l2.has("lease-key-replayed")).toBe(true) + expect(j.history).not.toContain("requeue:omitted") + expect([...w.ax.tasks.keys()]).toEqual(["wf-test-r5-a1"]) +}) diff --git a/pkgs/substrate-link/src/apps/link/test/review-r4-stale-lost.test.ts b/pkgs/substrate-link/src/apps/link/test/review-r4-stale-lost.test.ts new file mode 100644 index 000000000..a999b9fd9 --- /dev/null +++ b/pkgs/substrate-link/src/apps/link/test/review-r4-stale-lost.test.ts @@ -0,0 +1,67 @@ +// Round 4 regression test, ported from /home/tom/today/evals-2026-09-23/link/scratch-r4-0/stale-lost.repro.ts (assertions kept; additions marked). +// Duplicate/stale delivery: a Heartbeat reply is applied to whatever record holds that leaseId when the reply lands, +// not to the generation the Heartbeat vouched for. Set-up is R2-5a (review-r2.test.ts): a1 fails retryable while a2 +// gen 1 is leased to this link and waits for a1's verdict. One Heartbeat carrying [a2] (gen 1) reaches the floor +// after rule 4b withdrew a2 gen 1 and requeued a2 gen 2, so the floor answers lost:[a2]. Its reply lands after the +// link took the regrant of a2 (gen 2). The link then releases gen 2 as `lost` and never creates it, although the floor +// has gen 2 leased to it: the job is stranded until the lease expires, then released `omitted` (an infra release). +import { Effect } from "effect" +import { expect, it } from "vitest" +import { bodyOf, job, sleep, start, until, world } from "./review-r4-harness.ts" +import type { World } from "./review-r4-harness.ts" + +const rpcOf = async (input: unknown, init?: RequestInit) => { const b = await bodyOf(input, init); return b.includes('"Complete"') ? "Complete" : b.includes('"Lease"') ? "Lease" : b.includes('"Heartbeat"') ? "Heartbeat" : "?" } +const failRetryable = (w: World, name: string) => { + const t = w.ax.tasks.get(name)! + t.exited = true; t.phase = "Failed" + t.conditions = [{ type: "Ready", status: "False", reason: "ResourceExhausted", message: "ResourceExhausted: no free worker" }] +} + +it("a Heartbeat reply computed for a2 gen 1 does not release a2 gen 2", async () => { + const w = world(); w.floor.enqueue(job("1")) + const J = () => w.floor.jobs.get("wf-test-1")! + let holdLease = false, failHeartbeat = false, failComplete = false + let armed = false, hbForwarded = false, hbIntercepted = false + let l: ReturnType + const fetch: typeof globalThis.fetch = async (input, init) => { + const body = await bodyOf(input, init) + const rpc = await rpcOf(input, init) + if (rpc === "Heartbeat" && failHeartbeat) throw new TypeError("fetch failed") + if (rpc === "Complete" && failComplete) return new Response("upstream timeout", { status: 503 }) + if (rpc === "Complete" && armed) await until("a Heartbeat vouching for a2 gen 1 is in flight", () => hbIntercepted, 4000) + if (rpc === "Lease") { while (holdLease) await sleep(2); while (armed && !hbForwarded) await sleep(2) } + if (rpc === "Heartbeat" && armed && !hbIntercepted && body.includes("wf-test-1-a2")) { + hbIntercepted = true + await until("floor withdrew a2 gen 1 and requeued a2 gen 2", () => J().history.includes("withdrawn:a2") && J().state === "queued", 4000) + const res = await w.floor.fetch(input as any, { ...init, body }) + hbForwarded = true // Lease may go now: the floor regrants a2 (gen 2) + await until("link took the regrant", () => l.has("regrant", "wf-test-1-a2"), 4000).catch(() => {}) + return res // the reply (lost:[a2]) lands after the regrant + } + return w.floor.fetch(input as any, { ...init, body }) + } + l = start(w, undefined, {}, fetch) + await until("running", () => w.ax.tasks.get("wf-test-1-a1")?.phase === "Running") + w.floor.down = true + await until("floor requeued attempt 2", () => J().state === "queued" && J().attempt === 2, 4000) + failHeartbeat = true; holdLease = true; failComplete = true + w.floor.down = false + await sleep(80) + failRetryable(w, "wf-test-1-a1") + await until("verdict written", () => l.has("verdict", "wf-test-1-a1")) + holdLease = false + await until("a2 received", () => l.has("leased", "wf-test-1-a2"), 4000) + await until("dispatch waits for the verdict", () => l.has("supersede-waits-for-verdict", "wf-test-1-a2")) + armed = true + failComplete = false; failHeartbeat = false + await until("a2 created, or 1 s", () => w.ax.updates.has("wf-test-1-a2"), 1000).catch(() => {}) + await sleep(300) + const trace = l.logs.filter((x) => x.leaseId === "wf-test-1-a2" || x.task === "wf-test-1-a2").map((x) => `${x.ev}${x.why ? ":" + x.why : ""}`) + const out = { hbIntercepted, floor: J().history, state: J().state, attempt: J().attempt, infraSpent: J().infraSpent, a2Updates: w.ax.updates.get("wf-test-1-a2") ?? 0, a2trace: trace } + console.log(JSON.stringify(out)) + await l.drain(); w.floor.close() + expect(hbIntercepted).toBe(true) + expect(out.a2Updates).toBe(1) + expect(out.floor).not.toContain("requeue:omitted") // round 4 addition + expect(out.a2trace).toContain("stale-heartbeat-reply") +}) diff --git a/pkgs/substrate-link/src/apps/link/test/workerd.integration.test.ts b/pkgs/substrate-link/src/apps/link/test/workerd.integration.test.ts new file mode 100644 index 000000000..866ba309e --- /dev/null +++ b/pkgs/substrate-link/src/apps/link/test/workerd.integration.test.ts @@ -0,0 +1,190 @@ +// Integration: the link against a REAL floor process and a REAL gRPC ax wire. +// floor the option-A prototype Durable Object (../arc/proto/floor, Effect's RpcServer.layerHttp inside a DO, DO +// SQLite, a storage alarm) served by `wrangler dev --local` on loopback (workerd). Copied to a temp dir first, +// so nothing is written into the prototype's own tree. It is NOT the floor port (FL1 to FL8 are not in it). +// ax a gRPC server on loopback built from the vendored ax.proto plus P1's GetTaskResult (fixtures/ax-p1.proto), +// with ax's own ListTasks paging (50 rows, newest first) and a Gateway. The link uses its production grpcAx. +// Opt-in: LINK_WORKERD=1, run inside ~/.local/bin/runtime-test (it starts workerd). Skipped otherwise. +import { spawn } from "node:child_process" +import type { ChildProcess } from "node:child_process" +import { createHash } from "node:crypto" +import { cpSync, mkdtempSync, rmSync, symlinkSync } from "node:fs" +import { tmpdir } from "node:os" +import { join } from "node:path" +import * as grpc from "@grpc/grpc-js" +import * as protoLoader from "@grpc/proto-loader" +import { Deferred, Effect, Exit, Fiber } from "effect" +import { afterAll, beforeAll, describe, expect, it } from "vitest" +import { grpcAx } from "../src/ax.ts" +import type { AgentJob } from "../src/contract.ts" +import { rpcFloor } from "../src/floor.ts" +import { Journal } from "../src/journal.ts" +import { runLink } from "../src/link.ts" +import type { LinkConfig } from "../src/link.ts" + +const RUN = process.env.LINK_WORKERD === "1" +const PROTO_DIR = "/home/tom/today/evals-2026-09-23/arc/proto" +const P1_PROTO = join(import.meta.dirname, "fixtures", "ax-p1.proto") +const TOKEN = "local-dev-only-not-a-secret" // the prototype's wrangler.jsonc var, a public test value +const PORT = 18000 + Math.floor(Math.random() * 900) +const F = `http://127.0.0.1:${PORT}` +const A = "ultracode.mecattaf.dev/" + +const job = (n: string): AgentJob => ({ + apiVersion: "ultracode.mecattaf.dev/v1alpha1", kind: "AgentJob", + metadata: { name: `wf-int-${n}`, labels: { [A + "workflow"]: "link-int", [A + "phase-index"]: "1" }, annotations: { [A + "run-id-raw"]: "wf_int", [A + "label"]: `int:${n}`, [A + "item-key"]: `wf_int#${n}`, [A + "journal-key"]: `${"cd".repeat(32)}:1`, [A + "phase-title"]: "Integration" } }, + spec: { "runs-on": ["seat:halogen", "runtime:gvisor"], with: { prompt: `say ${n}`, prompt_ref: { sha256: "cd".repeat(32), bytes: 5, uri: `journal://wf_int/${n}/prompt.md` }, model: "halogen-qwen3.8-flash-next" } } +}) + +// ---------------------------------------------------------------- the ax gRPC server (P1 wire) +const tasks = new Map() +const results = new Map() +let axUnavailable = false +let axServer: grpc.Server +let axPort = 0 +const unavailable = { code: grpc.status.UNAVAILABLE, details: "ax-server unavailable" } +const nf = (what: string) => ({ code: grpc.status.NOT_FOUND, details: `${what} not found` }) +const guard = (cb: grpc.sendUnaryData, f: () => void) => axUnavailable ? cb(unavailable) : f() + +async function startAx() { + const def = protoLoader.loadSync(P1_PROTO, { keepCase: false, longs: String, enums: String, defaults: true, oneofs: true }) + const svc = (grpc.loadPackageDefinition(def) as any).ax.v1alpha1.AX.service + axServer = new grpc.Server() + axServer.addService(svc, { + GetTask: (c: any, cb: any) => guard(cb, () => { const t = tasks.get(c.request.name); t ? cb(null, t) : cb(nf(c.request.name)) }), + UpdateTask: (c: any, cb: any) => guard(cb, () => { tasks.set(c.request.task.metadata.name, { ...c.request.task, status: { phase: "Pending" } }); cb(null, c.request.task) }), + ListTasks: (c: any, cb: any) => guard(cb, () => { const n = Number(c.request.limit) > 0 ? Number(c.request.limit) : 50, o = Number(c.request.offset) || 0; cb(null, { tasks: [...tasks.values()].reverse().slice(o, o + n) }) }), + DeleteTask: (c: any, cb: any) => guard(cb, () => { tasks.delete(c.request.name); results.delete(c.request.name); cb(null, {}) }), + GetGateway: (c: any, cb: any) => guard(cb, () => c.request.name === "halogen" + ? cb(null, { apiVersion: "ax.io/v1alpha1", kind: "Gateway", metadata: { name: "halogen", atespace: "fleet" }, spec: { egress: { allowlist: { hosts: [{ host: "worker", port: 8731 }] } } } }) + : cb(nf("gateway"))), + GetTaskResult: (c: any, cb: any) => guard(cb, () => { const b = results.get(c.request.name); b ? cb(null, { content: b, sha256: createHash("sha256").update(b).digest("hex") }) : cb(nf("result")) }) + }) + axPort = await new Promise((res, rej) => axServer.bindAsync("127.0.0.1:0", grpc.ServerCredentials.createInsecure(), (e, p) => e ? rej(e) : res(p))) +} +const run = (name: string) => { tasks.get(name).status = { phase: "Running" } } +const finish = (name: string, out: unknown) => { + results.set(name, Buffer.from(JSON.stringify(out))) + tasks.get(name).status = { phase: "Completed", usage: { promptTokens: 5, completionTokens: 3, toolCalls: 1 }, conditions: [{ type: "Ready", status: "False", reason: "CommandExited", message: "ExitCode=0" }] } +} + +// ---------------------------------------------------------------- the floor under wrangler dev --local +let work = "", wrangler: ChildProcess | undefined +const floorLog: Array = [] +async function floorUp() { + wrangler = spawn("wrangler", ["dev", "--local", "--ip", "127.0.0.1", "--port", String(PORT), "--persist-to", join(work, "state"), + "--var", "LEASE_SECONDS:30", "--show-interactive-dev-session=false"], + { cwd: join(work, "floor"), env: { ...process.env, WRANGLER_SEND_METRICS: "false", CI: "1" }, detached: true, stdio: ["ignore", "pipe", "pipe"] }) + wrangler.stdout!.on("data", (d) => floorLog.push(String(d))) + wrangler.stderr!.on("data", (d) => floorLog.push(String(d))) + await until("floor up", async () => (await admin("GET", "/admin/state").catch(() => undefined)) !== undefined, 60_000) +} +async function floorDown() { + if (wrangler?.pid) { try { process.kill(-wrangler.pid, "SIGTERM") } catch { /* gone */ } } + await until("floor down", async () => (await admin("GET", "/admin/state").catch(() => undefined)) === undefined, 20_000) +} +async function admin(method: string, path: string, body?: unknown): Promise { + const r = await fetch(F + path, { method, headers: { authorization: `Bearer ${TOKEN}`, "content-type": "application/json" }, ...(body ? { body: JSON.stringify(body) } : {}), signal: AbortSignal.timeout(3000) }) + if (!r.ok) throw new Error(`admin ${path}: ${r.status}`) + return r.json() +} +const jobRow = async (n: string) => ((await admin("GET", "/admin/state")).jobs as Array).find((j) => j.name === `wf-int-${n}`) + +async function until(what: string, pred: () => boolean | Promise, ms = 30_000) { + const t0 = Date.now() + while (!(await pred())) { if (Date.now() - t0 > ms) throw new Error(`timeout waiting for: ${what}`); await new Promise((r) => setTimeout(r, 100)) } +} +const sleep = (ms: number) => new Promise((r) => setTimeout(r, ms)) + +function startLink(token = TOKEN) { + const logs: Array> = [] + const dir = mkdtempSync(join(tmpdir(), "conwip-link-int-")) + const stop = Effect.runSync(Deferred.make()) + const ax = grpcAx(`127.0.0.1:${axPort}`, "fleet", 5_000, P1_PROTO) + const cfg: LinkConfig = { + holder: "nas-link-1", maxInFlight: 2, servedLabels: ["seat:halogen", "runtime:gvisor"], + shape: { atespace: "fleet", image: "ax-agent", gateway: "halogen", command: () => ["ax-agent", "pi"] }, + completion: "p1", secondMs: 1000, resyncMs: 500, pendingTimeoutMs: 60_000, deleteAfterMs: 0, deadlineBackstopMs: 60_000, + createAttempts: 3, outboxBackoffMs: [200, 1000], fenceTimeoutMs: 5_000, initialPollSeconds: 1, initialHeartbeatSeconds: 2 + } + const fiber = Effect.runFork(Effect.scoped(Effect.gen(function*() { + const floor = yield* rpcFloor({ url: F, token, sessionId: `s:${dir}`, timeoutsMs: { lease: 5_000, heartbeat: 5_000, complete: 10_000 } }) + return yield* runLink(cfg, { ax, floor, journal: Journal.open(dir), log: (ev, f) => logs.push({ ev, ...f }), stop, floorUrl: F }) + }))) + return { + logs, + has: (ev: string) => logs.some((l) => l.ev === ev), + exit: () => Effect.runPromise(Fiber.await(fiber)), + drain: async () => { Effect.runSync(Deferred.succeed(stop, undefined)); const e = await Effect.runPromise(Fiber.await(fiber)); ax.close(); rmSync(dir, { recursive: true, force: true }); return e } + } +} + +describe.skipIf(!RUN)("link against the prototype floor under wrangler dev --local and a P1 gRPC ax", () => { + beforeAll(async () => { + work = mkdtempSync(join(tmpdir(), "conwip-floor-int-")) + cpSync(join(PROTO_DIR, "floor"), join(work, "floor"), { recursive: true, filter: (s) => !s.includes(".wrangler") }) + cpSync(join(PROTO_DIR, "contract.ts"), join(work, "contract.ts")) + symlinkSync(join(PROTO_DIR, "node_modules"), join(work, "node_modules")) + await startAx() + await floorUp() + }, 90_000) + afterAll(async () => { + await floorDown().catch(() => undefined) + axServer?.forceShutdown() + if (process.env.LINK_DEBUG) console.log(floorLog.join("").slice(-4000)) + if (work) rmSync(work, { recursive: true, force: true }) + }, 30_000) + + it("I1 happy path over real HTTP RPC and real gRPC: lease, create-only, P1 result with sha256, Complete, janitor delete", async () => { + await admin("POST", "/admin/enqueue", job("1")) + const l = startLink() + await until("created", () => tasks.has("wf-int-1-a1")) + expect(tasks.get("wf-int-1-a1").spec.env.find((e: any) => e.name === "AX_CONWIP_PROMPT").value).toBe("say 1") + run("wf-int-1-a1"); finish("wf-int-1-a1", { answer: 42 }) + await until("floor done", async () => (await jobRow("1"))?.state === "done") + const r = await jobRow("1") + expect([r.result, JSON.parse(r.output), JSON.parse(r.usage), r.attempt]).toEqual(["success", { answer: 42 }, { prompt_tokens: 5, completion_tokens: 3, tool_calls: 1 }, 1]) + await until("janitor deleted the Task", () => !tasks.has("wf-int-1-a1")) + expect(l.has("completion-probe") && l.has("gateway-ok")).toBe(true) + expect(Exit.isSuccess(await l.drain())).toBe(true) + }, 60_000) + + it("I2 floor process restarted while a verdict is pending: the outbox keeps it and it lands once, attempt 1", async () => { + await admin("POST", "/admin/enqueue", job("2")) + const l = startLink() + await until("created", () => tasks.has("wf-int-2-a1")) + run("wf-int-2-a1") + await floorDown() + finish("wf-int-2-a1", { restarted: true }) + await until("outbox keeps", () => l.has("outbox-keep")) + await floorUp() // same --persist-to: the DO's SQLite and alarm survive + await until("floor done", async () => (await jobRow("2"))?.state === "done", 45_000) + const r = await jobRow("2") + expect([r.result, JSON.parse(r.output), r.attempt, r.transitions]).toEqual(["success", { restarted: true }, 1, 0]) + expect(Exit.isSuccess(await l.drain())).toBe(true) + }, 120_000) + + it("I3 ax answers UNAVAILABLE: every Lease asks for 0, the job stays queued; ax returns and the job runs", async () => { + axUnavailable = true + const l = startLink() + await sleep(1_500) // the start-up resync fails; nothing is probed, capacity stays 0 + expect(l.has("completion-probe")).toBe(false) + await admin("POST", "/admin/enqueue", job("3")) + await sleep(6_000) // three polls at the floor's 2 s + expect((await jobRow("3")).state).toBe("queued") + axUnavailable = false + await until("created", () => tasks.has("wf-int-3-a1")) + run("wf-int-3-a1"); finish("wf-int-3-a1", { back: true }) + await until("floor done", async () => (await jobRow("3"))?.state === "done") + expect((await jobRow("3")).result).toBe("success") + expect(Exit.isSuccess(await l.drain())).toBe(true) + }, 60_000) + + it("I4 the real Worker entry answers 401 to a wrong bearer: the link stops with kind auth (exit 78 in main)", async () => { + const l = startLink("not-the-token") + const exit = await l.exit() + expect(Exit.isFailure(exit)).toBe(true) + expect(JSON.stringify(exit)).toContain("\"kind\":\"auth\"") + await l.drain() + }, 30_000) +}) diff --git a/pkgs/substrate-link/src/package.json b/pkgs/substrate-link/src/package.json new file mode 100644 index 000000000..2d2b8d8b0 --- /dev/null +++ b/pkgs/substrate-link/src/package.json @@ -0,0 +1,22 @@ +{ + "name": "ax-conwip", + "version": "0.1.0", + "private": true, + "type": "module", + "description": "CONWIP scheduler that turns Claude ultracode workflow run records into ax Tasks under a fixed work-in-progress cap.", + "scripts": { + "test": "vitest run", + "conwip": "tsx src/cli.ts" + }, + "dependencies": { + "@grpc/grpc-js": "^1.12.6", + "@grpc/proto-loader": "^0.7.13", + "effect": "4.0.0-rc.112" + }, + "devDependencies": { + "@types/node": "^22.10.2", + "tsx": "^4.19.2", + "typescript": "^5.9.3", + "vitest": "^3.2.4" + } +} diff --git a/pkgs/substrate-link/src/pnpm-lock.yaml b/pkgs/substrate-link/src/pnpm-lock.yaml new file mode 100644 index 000000000..5fc4eb8b8 --- /dev/null +++ b/pkgs/substrate-link/src/pnpm-lock.yaml @@ -0,0 +1,1401 @@ +lockfileVersion: '9.0' + +settings: + autoInstallPeers: true + excludeLinksFromLockfile: false + +importers: + + .: + dependencies: + '@grpc/grpc-js': + specifier: ^1.12.6 + version: 1.14.5 + '@grpc/proto-loader': + specifier: ^0.7.13 + version: 0.7.15 + effect: + specifier: 4.0.0-rc.112 + version: 4.0.0-rc.112 + devDependencies: + '@types/node': + specifier: ^22.10.2 + version: 22.20.4 + tsx: + specifier: ^4.19.2 + version: 4.23.15 + typescript: + specifier: ^5.9.3 + version: 5.9.3 + vitest: + specifier: ^3.2.4 + version: 3.2.7(@types/node@22.20.4)(tsx@4.23.15) + +packages: + + '@esbuild/aix-ppc64@0.28.2': + resolution: {integrity: sha512-XExcO+dvLKvVtNTibSTBej1NCAbaGhWn9Ww1ZPx80qsahhPFe/8jgWP0IchNe0F3HwkU7n8ejhH8bjonqht8mQ==} + engines: {node: '>=18'} + cpu: [ppc64] + os: [aix] + + '@esbuild/android-arm64@0.28.2': + resolution: {integrity: sha512-5YfKeeI8qWfBZIX+u2xZC3Zlb3Os/gLS2sbEKM+I4ZOcsWmHS2WLysCcQZDAFRslDUU5Oiq44gf6PYN1vGwG5A==} + engines: {node: '>=18'} + cpu: [arm64] + os: [android] + + '@esbuild/android-arm@0.28.2': + resolution: {integrity: sha512-kXXoiPVVGQcnIYGOeaovwOURpniDBpSq4A03qkQ+BMQqtGG6HYap3xne9C1O1yo4TR3qxlCX5IqqmX6fFo2Lqg==} + engines: {node: '>=18'} + cpu: [arm] + os: [android] + + '@esbuild/android-x64@0.28.2': + resolution: {integrity: sha512-O387ite7SzUyCcy3JQX4P4bLtEA7bLLkx+esve5JHnyYfNTxcVpXZo9jhdB0lTKN44gztELTdU7nS8Nr16Fs1Q==} + engines: {node: '>=18'} + cpu: [x64] + os: [android] + + '@esbuild/darwin-arm64@0.28.2': + resolution: {integrity: sha512-n4KqkOQrraxHJcgjM1RvwbigfQKIKJVpM7xp+KsxiyUSrRdIXnt73VhrPAx0fV44hgfmIVKjxMN9J1t5jySVkw==} + engines: {node: '>=18'} + cpu: [arm64] + os: [darwin] + + '@esbuild/darwin-x64@0.28.2': + resolution: {integrity: sha512-uq6suIWYP37qzGddBKPw5QEQPi6HiLGsO7UmkpfyaYNQ3D+rN6w6WfwH+nuqcGXWvawGwxOEroO4YGnFh95azw==} + engines: {node: '>=18'} + cpu: [x64] + os: [darwin] + + '@esbuild/freebsd-arm64@0.28.2': + resolution: {integrity: sha512-n+I0BTSRIoy+d6RPKnEVwql5UwBJolytvY4mAOIEJorKlqgPII8ix6slVVrfZ5Tnj7glIZvloylbB/EJPMWEXw==} + engines: {node: '>=18'} + cpu: [arm64] + os: [freebsd] + + '@esbuild/freebsd-x64@0.28.2': + resolution: {integrity: sha512-78XJTJkvPs0kz2w61301PJjXl4g7q3JqiYMZ/M/yVI73EHBrCRTgkhu9oqG7vPqq+a/yadEW8aD+agKlk5xrmg==} + engines: {node: '>=18'} + cpu: [x64] + os: [freebsd] + + '@esbuild/linux-arm64@0.28.2': + resolution: {integrity: sha512-pW4AC0P3it8c7do9MVM4p51FzHzdM/TZrerurgRcHJ2WTa1VQ1CIq18xncfpBJw4ojkiZZrKW2yIBWBP92j6Ug==} + engines: {node: '>=18'} + cpu: [arm64] + os: [linux] + + '@esbuild/linux-arm@0.28.2': + resolution: {integrity: sha512-XlDnu2q5yoqems+xay6wSAcg9DDD7K9RLKZEBOMZm3ckNpJBvOX20tSfby8KfrrhINDyv9V2YVZKY/SpoGJI8w==} + engines: {node: '>=18'} + cpu: [arm] + os: [linux] + + '@esbuild/linux-ia32@0.28.2': + resolution: {integrity: sha512-CYbnj78HsIeA+DhgUKgFCfvNsTHFhMMrinUrMZpDXJXKN8T3XViTZ/+wtHeVxEWY8ewSzTFN+nRmSwO2tZaLUQ==} + engines: {node: '>=18'} + cpu: [ia32] + os: [linux] + + '@esbuild/linux-loong64@0.28.2': + resolution: {integrity: sha512-buwkd8nsph4R+ajRvw0qM5Hja/TXQow3ptzWO2EbG/cqcIkHloRrdlBtQlshyYGTNFvfkfJ5tpPLVkY4DtsPfQ==} + engines: {node: '>=18'} + cpu: [loong64] + os: [linux] + + '@esbuild/linux-mips64el@0.28.2': + resolution: {integrity: sha512-ZVykbDyk7519VwiNb9Lcj9m8XM6v5V9uKPvrEMkkEedVewf+0itkhahp4HDpgERXhwLRpWFypsGbG/J8s0QjJA==} + engines: {node: '>=18'} + cpu: [mips64el] + os: [linux] + + '@esbuild/linux-ppc64@0.28.2': + resolution: {integrity: sha512-CAXl+Dtd9UUuJd8pKKdwh6MLm3MUMiqMPmhZ3tTSXPqfyQ3vDl6R5hZdZ/kYojK4ofXtdfSv1tFq8XzWx3heNQ==} + engines: {node: '>=18'} + cpu: [ppc64] + os: [linux] + + '@esbuild/linux-riscv64@0.28.2': + resolution: {integrity: sha512-GeXCej4IQtU1B+QlDV8W/RRvbzI3O/Stss+/bCXv4lZls5WGRtu2a+3JkA3i4qIUlMXpcHebWpF8AkJhATowuA==} + engines: {node: '>=18'} + cpu: [riscv64] + os: [linux] + + '@esbuild/linux-s390x@0.28.2': + resolution: {integrity: sha512-3H1weTYZPxt/WOhByszQZybS9w5lKzUn1FDMsgEChbHWQwHYQQRfBxgCcZvPhjHfKyJjIievvMmEUawJrdY9Dg==} + engines: {node: '>=18'} + cpu: [s390x] + os: [linux] + + '@esbuild/linux-x64@0.28.2': + resolution: {integrity: sha512-4xTZr1FUmSoQW4XIWmit3tzQrUTZM+N3P0XV8xROKYF50XfI7xeO90+1bZvNwxIufQ9hDQVRJH5YhgPVF8A/HQ==} + engines: {node: '>=18'} + cpu: [x64] + os: [linux] + + '@esbuild/netbsd-arm64@0.28.2': + resolution: {integrity: sha512-sSATRjPeDBg3pdgHoQfoYBob11Kk1FGa9lui5RIHZCoCkJa9QKlvl3/vKz2usCmYYjs7ymJR/2Nnsqe+Hjt5nw==} + engines: {node: '>=18'} + cpu: [arm64] + os: [netbsd] + + '@esbuild/netbsd-x64@0.28.2': + resolution: {integrity: sha512-lqnzCV+mM0gIADaKihiCg6ifgfU2L3h5E33rNQBN1Y4MaVGnzryzmvvf7UHxprpQdE8hpqLolJ9Rl+SkIRDpyw==} + engines: {node: '>=18'} + cpu: [x64] + os: [netbsd] + + '@esbuild/openbsd-arm64@0.28.2': + resolution: {integrity: sha512-AL2qJILH7lNjrDmCQDvdxMfAUIv8KMNZOvrwAQ8i8//ntL9FflhOyMJ8OZSMBb8/AWXe3/5v5S20y3zCoZWKoQ==} + engines: {node: '>=18'} + cpu: [arm64] + os: [openbsd] + + '@esbuild/openbsd-x64@0.28.2': + resolution: {integrity: sha512-QtiuPytchRyC4rwUKhexJdQKvDuZ6hWloi3igqPQNUJCS1/v9EiO3UTOXR6A3FoMo4fnAKbWJdqaIwhOzh8qEw==} + engines: {node: '>=18'} + cpu: [x64] + os: [openbsd] + + '@esbuild/openharmony-arm64@0.28.2': + resolution: {integrity: sha512-WkhYDmpTjLvGlScA1rwjRUmhl4k8oXR3cIbtqWmELgU/dFeHHlEllxDvdWcNJV9rbzCexB5vz8gtNewWLgCT7Q==} + engines: {node: '>=18'} + cpu: [arm64] + os: [openharmony] + + '@esbuild/sunos-x64@0.28.2': + resolution: {integrity: sha512-GPMSkTOtMnv2U2F8gxe4Io6qmVs+YKyp832Etqqxr0hFngmXQ3rzwytelm3GIn7T4VviRUlf3sOgBOiTdvaf7g==} + engines: {node: '>=18'} + cpu: [x64] + os: [sunos] + + '@esbuild/win32-arm64@0.28.2': + resolution: {integrity: sha512-PIhhEkE9uPBleRBrQEJpUn7MBnibZzbGzYWPmY3x+YoVg/95zbjB4CxPPOQ8l5tYYM4mMaCthF8/1DIfBQQyWQ==} + engines: {node: '>=18'} + cpu: [arm64] + os: [win32] + + '@esbuild/win32-ia32@0.28.2': + resolution: {integrity: sha512-YmJbfTlvU7Sdn9BB+4PRES4oB6pxgS37MAONj+hBr/cpXS1aBPKXxNnDbu+QCWPj0o9dgyxeq79g6c5P8KeuYA==} + engines: {node: '>=18'} + cpu: [ia32] + os: [win32] + + '@esbuild/win32-x64@0.28.2': + resolution: {integrity: sha512-5ebpxr3nWMzrL/rnUI755Jkuee0bHL/Gq0WTF9lvcpv73wAp5eu8MfBUgWK9bhWvZjj7yX8etf/8tI8Ney695g==} + engines: {node: '>=18'} + cpu: [x64] + os: [win32] + + '@grpc/grpc-js@1.14.5': + resolution: {integrity: sha512-7VZM+SVdEcUUqSQeNI3zM8Qs/BhQKZndPo2h5VkYkAM8Iz0wJIa8mKV5ekQGqG8UUsnkQ0NMxIxwkIHYvj0qOw==} + engines: {node: '>=12.10.0'} + + '@grpc/proto-loader@0.7.15': + resolution: {integrity: sha512-tMXdRCfYVixjuFK+Hk0Q1s38gV9zDiDJfWL3h1rv4Qc39oILCu1TRTDt7+fGUI8K4G1Fj125Hx/ru3azECWTyQ==} + engines: {node: '>=6'} + hasBin: true + + '@grpc/proto-loader@0.8.1': + resolution: {integrity: sha512-wtF6h+DY6M3YaDBPAmvuuA6jV8Sif9MjtOI5euKFWRgCDl5PeDpPsHR9u2l6St5ceY8AZgoNDww5+HvEsXFsGg==} + engines: {node: '>=6'} + hasBin: true + + '@jridgewell/sourcemap-codec@1.6.0': + resolution: {integrity: sha512-T7jf+5zgsZHwNJ4lvQ7/aezbyk0nNX+zJVWpmHA7VYsEx7a7qr5Rg5IbtJFqkgze5Y2sruq1RUY8Q837Od7iFw==} + + '@js-sdsl/ordered-map@4.4.2': + resolution: {integrity: sha512-iUKgm52T8HOE/makSxjqoWhe95ZJA1/G1sYsGev2JDKUSS14KAgg1LHb+Ba+IPow0xflbnSkOsZcO08C7w1gYw==} + + '@msgpackr-extract/msgpackr-extract-darwin-arm64@3.0.4': + resolution: {integrity: sha512-LCkGo6JDfaBhgST7UpPWgNgLINpcpabaHfyz5OBx75nUYxBsaEPxjnyNjWpeb/xBup/682QnBfRBy2/LvPutZQ==} + cpu: [arm64] + os: [darwin] + + '@msgpackr-extract/msgpackr-extract-darwin-x64@3.0.4': + resolution: {integrity: sha512-zExlW9zUJKZH/tOtVMttwjKa4Xm/3KcNjnE3dPN92uCktwavMxpgCA3MoJK/DOnTWsQgo224OaST27/mPNAf+w==} + cpu: [x64] + os: [darwin] + + '@msgpackr-extract/msgpackr-extract-linux-arm64@3.0.4': + resolution: {integrity: sha512-dgX0P/9wGPJeHFBG+ZmhgE6bmtMt7NP5CRBGyyktpopdk/mW4POnrpQsSLtKI1dwpc+pPLuXHDh6vvskyQE/sw==} + cpu: [arm64] + os: [linux] + + '@msgpackr-extract/msgpackr-extract-linux-arm@3.0.4': + resolution: {integrity: sha512-Tg3yX65f5GbtXLkrYEHE5oibZG9epyYWas7FogTTEJeDEF9JlXJzKgXaNhT3UXlTOeA+AfZpYZYZ0uPj7Cfquw==} + cpu: [arm] + os: [linux] + + '@msgpackr-extract/msgpackr-extract-linux-x64@3.0.4': + resolution: {integrity: sha512-8TNXMEjJc3QEy7R/x1INhgiU+XakDAFUzBhaz7+Rbrs8NH5UQeHQxxmzsSBJGyV6I1jW79undiQm8tOI+D+8FQ==} + cpu: [x64] + os: [linux] + + '@msgpackr-extract/msgpackr-extract-win32-x64@3.0.4': + resolution: {integrity: sha512-CmCXPQrkbwExx3j946/PtHWHbYJiCRBRDl4BlkRQcJB/YOwQxJRTpoo7aTsortjgoJ1x7opzTSxn7C+ASSLVjQ==} + cpu: [x64] + os: [win32] + + '@napi-rs/lzma-linux-x64-gnu@1.5.1': + resolution: {integrity: sha512-oTXEIha4SsuXdTA4Iyskj0kpdx2yVXdhd75c2v3xGrHFfVMsbhTPZU/nMPL4sWKo4pBHm3aucLaqGlF696dTyQ==} + engines: {node: ^22.20 || ^24.12 || >=25} + cpu: [x64] + os: [linux] + libc: [glibc] + + '@protobufjs/aspromise@1.1.2': + resolution: {integrity: sha512-j+gKExEuLmKwvz3OgROXtrJ2UG2x8Ch2YZUxahh+s1F2HZ+wAceUNLkvy6zKCPVRkU++ZWQrdxsUeQXmcg4uoQ==} + + '@protobufjs/base64@1.1.2': + resolution: {integrity: sha512-AZkcAA5vnN/v4PDqKyMR5lx7hZttPDgClv83E//FMNhR2TMcLUhfRUBHCmSl0oi9zMgDDqRUJkSxO3wm85+XLg==} + + '@protobufjs/codegen@2.0.5': + resolution: {integrity: sha512-zgXFLzW3Ap33e6d0Wlj4MGIm6Ce8O89n/apUaGNB/jx+hw+ruWEp7EwGUshdLKVRCxZW12fp9r40E1mQrf/34g==} + + '@protobufjs/eventemitter@1.1.1': + resolution: {integrity: sha512-vW1GmwMZNnL+gMRaovlh9yZX74kc+TTU3FObkkurpMaRtBfLP3ldjS9KQWlwZgraRE0+dheEEoAxdzcJQ8eXZg==} + + '@protobufjs/fetch@1.1.1': + resolution: {integrity: sha512-GpptLrs57adMSuHi3VNj0mAF8dwh36LMaYF6XyJ6JMWlVsc+t42tm1HSEDmOs3A8fC9yyeisgLhsTVQokOZ0zw==} + + '@protobufjs/float@1.0.2': + resolution: {integrity: sha512-Ddb+kVXlXst9d+R9PfTIxh1EdNkgoRe5tOX6t01f1lYWOvJnSPDBlG241QLzcyPdoNTsblLUdujGSE4RzrTZGQ==} + + '@protobufjs/path@1.1.2': + resolution: {integrity: sha512-6JOcJ5Tm08dOHAbdR3GrvP+yUUfkjG5ePsHYczMFLq3ZmMkAD98cDgcT2iA1lJ9NVwFd4tH/iSSoe44YWkltEA==} + + '@protobufjs/pool@1.1.0': + resolution: {integrity: sha512-0kELaGSIDBKvcgS4zkjz1PeddatrjYcmMWOlAuAPwAeccUrPHdUqo/J6LiymHHEiJT5NrF1UVwxY14f+fy4WQw==} + + '@protobufjs/utf8@1.1.2': + resolution: {integrity: sha512-b1UQwcEZ4yCnMCD8DAL1VlbvBJE9/IX4FTIp7BG1xYpf29SLazLSrqUkj4w7Y5y7cCVP6E5tcqqcI0xemPkHug==} + + '@rollup/rollup-android-arm-eabi@4.63.4': + resolution: {integrity: sha512-I+BSHzTAhKN2n7ZwGZsegGcZjDpLqFOMAtJz/u6uFGe0pUFbq56dEHjqJV/ZUdRJtNXNxA+hREUatZBvMR3Oiw==} + cpu: [arm] + os: [android] + + '@rollup/rollup-android-arm64@4.63.4': + resolution: {integrity: sha512-pu3BdjS2LtEzRu2elmGzS3fIeWSZy4BMDIaLNwjorO76+k2d0LMluijhsDx3KQyQBQ/lLUZCQA9/s6csvUfuhw==} + cpu: [arm64] + os: [android] + + '@rollup/rollup-darwin-arm64@4.63.4': + resolution: {integrity: sha512-xfSrj9MHnWK9GaSqT9U0ImHtH/N8WZlHLx4cZHiuLcqs640hvZ3hLPd5UR2AZS57FaE8HrRUSpltbZdWRxHiDA==} + cpu: [arm64] + os: [darwin] + + '@rollup/rollup-darwin-x64@4.63.4': + resolution: {integrity: sha512-bqU99PLJb/dqb3S0GIMdeuyAEETSUgZBoqXYd3Sd+WCsV+MmPhnN6JrotWyir31+QgH7EvvE5/mwGJlEoci8Fw==} + cpu: [x64] + os: [darwin] + + '@rollup/rollup-freebsd-arm64@4.63.4': + resolution: {integrity: sha512-JinsFZ5G40oXQb+sUuiA5x689vhr6dDYK0H0NL+rwKdL6CqnmYN8PE4ZwfRSoIjrCxqTQG/SLfTtSvHeGxoVlw==} + cpu: [arm64] + os: [freebsd] + + '@rollup/rollup-freebsd-x64@4.63.4': + resolution: {integrity: sha512-GAdA4UxpiNm27cLHr2GqXBpAD0x9FqwYBY7/YSP0Ss0/PNi4k8gbviqpIpYbVSRBaS2ZcegXEzgTQMbRNCwxCw==} + cpu: [x64] + os: [freebsd] + + '@rollup/rollup-linux-arm-gnueabihf@4.63.4': + resolution: {integrity: sha512-qDd6NoA1znaLjp4jR5U/KWCdLAKDJNB8W9ChbbDaKbo0xA+Atln5HK6LFCZ4oJQpemtRZA288DCirFRjrspptw==} + cpu: [arm] + os: [linux] + libc: [glibc] + + '@rollup/rollup-linux-arm-musleabihf@4.63.4': + resolution: {integrity: sha512-WtB5Tz5KTNINb8ZA+8sQ7bmjuS1JrRT7YverYIhUGdWWDlpzVWmIwuZE+jidkEXUn1l0zrEkaIMa8dHF3NGcsA==} + cpu: [arm] + os: [linux] + libc: [musl] + + '@rollup/rollup-linux-arm64-gnu@4.63.4': + resolution: {integrity: sha512-VcQ3L1tjnkKzWjryAVaFhHEWcqOfICX9uxVVoDzm2t0DpgKRHd2zOpVrJc0xsWeBZcBFyYROCIBdyR/fS174pg==} + cpu: [arm64] + os: [linux] + libc: [glibc] + + '@rollup/rollup-linux-arm64-musl@4.63.4': + resolution: {integrity: sha512-6+ZQX6P5s0cMDN2Ypb8Lbm2+/sZYmZjdaYny992ujUU9UKi/4CWoJWsl1pNvjWJHNHGK51m+jKGLlh1ylb2ifQ==} + cpu: [arm64] + os: [linux] + libc: [musl] + + '@rollup/rollup-linux-loong64-gnu@4.63.4': + resolution: {integrity: sha512-D72ZnvkFkBXOfzMMQLcwfPLyGkKb7HZ9/mf97B7v6/P5Lbv4oFOtSY/uHbS8lH6uKUOxoKiuokdb50XZSzzbJw==} + cpu: [loong64] + os: [linux] + libc: [glibc] + + '@rollup/rollup-linux-loong64-musl@4.63.4': + resolution: {integrity: sha512-piU6BxeqA3O9KSu3kRCIQQtNqFFaTu21SEV4FwaRZowpnj3bLaWPZHw+xFqCs0XlJ+aOH3PTRWGoglH+mKA/OA==} + cpu: [loong64] + os: [linux] + libc: [musl] + + '@rollup/rollup-linux-ppc64-gnu@4.63.4': + resolution: {integrity: sha512-/5PGpHwqt2EEEOUs1XwzubE/ucr0dWDQ+to3zqi4Ds7EWpwtQ79wXc4JBoxqj/OwpawTsKWzJxHfSuBOq3DrWA==} + cpu: [ppc64] + os: [linux] + libc: [glibc] + + '@rollup/rollup-linux-ppc64-musl@4.63.4': + resolution: {integrity: sha512-cX3beZDLWt7G2oJF+nhChiT+qtaihs+S2xi7ziGmVB+2pwPng6D0Ed0HmElQOgv2UsUmSJJLGwpBao/3TDx3VA==} + cpu: [ppc64] + os: [linux] + libc: [musl] + + '@rollup/rollup-linux-riscv64-gnu@4.63.4': + resolution: {integrity: sha512-1uz2mGWHyptR7DgHHrlbdRAjXK7v7elGZ9lMja910/RP+ZYbX6xAmCiU9UZSX4hqmgtHMv6lr5l3kq1HIOpcag==} + cpu: [riscv64] + os: [linux] + libc: [glibc] + + '@rollup/rollup-linux-riscv64-musl@4.63.4': + resolution: {integrity: sha512-nLS8topojxyz7SRpKR2IODRpQ0XPZ+xaOXvT3+hqK/Uy8Lo5HFgkkIBiIrCu5tL5YqzTvgovGw55PwpahTAGig==} + cpu: [riscv64] + os: [linux] + libc: [musl] + + '@rollup/rollup-linux-s390x-gnu@4.63.4': + resolution: {integrity: sha512-gs7DRKotr3l3q+jGPQBjH0ng1FjlEDm5ueQrkw5JtQvtLyEIcLASqAEaor56BhkKRzk+IcQzrcanBdb/bBQn8g==} + cpu: [s390x] + os: [linux] + libc: [glibc] + + '@rollup/rollup-linux-x64-gnu@4.63.4': + resolution: {integrity: sha512-791ET7W17NnScOZM7h4dX5hYspxE28htPFsb1awY/NRR8+PRNkS53e475rDdxXXDrP+kwnCcNWg9CX5ztn/Aqw==} + cpu: [x64] + os: [linux] + libc: [glibc] + + '@rollup/rollup-linux-x64-musl@4.63.4': + resolution: {integrity: sha512-iwZQRcmj7g88g3tzefIrQY7qvmuA/cfYwhrDtTBhsmukO4U2huVO5W+86XacUMRvdSFVAc6kZUZy21JaRwiB9w==} + cpu: [x64] + os: [linux] + libc: [musl] + + '@rollup/rollup-openbsd-x64@4.63.4': + resolution: {integrity: sha512-dVHFp9gRWrdTpnqQuGfCwd7hOQDatK1VCP2iWhLY/cGrOQs/ucFzJ6A5SRqbXX12ZDI8EUuejSM5kwg+ja7Png==} + cpu: [x64] + os: [openbsd] + + '@rollup/rollup-openharmony-arm64@4.63.4': + resolution: {integrity: sha512-t3NlauOW6gxZVVFcBEnO62Cb4wbyDFL416gTg1uFI/2tgqYQlf69FbSE115Ajre9I+c26Lk4mcmdFUsS/DGifQ==} + cpu: [arm64] + os: [openharmony] + + '@rollup/rollup-win32-arm64-msvc@4.63.4': + resolution: {integrity: sha512-xWuIaSye5FWZF8+UYtVEcHtRJDN5kN9Kfgxx3Kq8XIov9KSKbc1fiqQCm90SKrgQbUXZelbnUhnlUJmfSE7P9A==} + cpu: [arm64] + os: [win32] + + '@rollup/rollup-win32-ia32-msvc@4.63.4': + resolution: {integrity: sha512-9ALJJUOg/ZflMJepVo2PlgsGxSaxN7SQ4Z8GoZfVlarWr6r3rkHUNsd/zAio7p4YMtChSMXPionxej4Hkf6CXQ==} + cpu: [ia32] + os: [win32] + + '@rollup/rollup-win32-x64-gnu@4.63.4': + resolution: {integrity: sha512-blj9z5qx/Pv4WU0W1NMFDB97e0JH5ed+aZGywW8WCvp/NhWX/4PFAq5uu6Q0AebNn+Vo6KzUYDT++JzTT5ojlQ==} + cpu: [x64] + os: [win32] + + '@rollup/rollup-win32-x64-msvc@4.63.4': + resolution: {integrity: sha512-Erx822VRBwLa124shbj+wNXe//BOgMEctDV0m1aqTQdNO1S69DgNUCFKC1RCeZfixs1J31l6igk1ziyXErbigQ==} + cpu: [x64] + os: [win32] + + '@types/chai@5.2.3': + resolution: {integrity: sha512-Mw558oeA9fFbv65/y4mHtXDs9bPnFMZAL/jxdPFUpOHHIXX91mcgEHbS5Lahr+pwZFR8A7GQleRWeI6cGFC2UA==} + + '@types/deep-eql@4.0.2': + resolution: {integrity: sha512-c9h9dVVMigMPc4bwTvC5dxqtqJZwQPePsWjPlpSOnojbor6pGqdk541lfA7AqFQr5pB1BRdq0juY9db81BwyFw==} + + '@types/estree@1.0.9': + resolution: {integrity: sha512-GhdPgy1el4/ImP05X05Uw4cw2/M93BCUmnEvWZNStlCzEKME4Fkk+YpoA5OiHNQmoS7Cafb8Xa3Pya8m1Qrzeg==} + + '@types/node@22.20.4': + resolution: {integrity: sha512-zJRE40jpHtKqE/C4fgHrAKQLJuSpzEnP9ff9Y7YtoR3Wd2pwqzlekDeEuUQXjRd+QCYnVnNwuJYmhdk9XV8gvA==} + + '@vitest/expect@3.2.7': + resolution: {integrity: sha512-E8eBXaKibuvH2pSZErOjdVb5vF4PbKYcrnluBTYxEk1l/VhhwZg1kZQsdtjq+CsF5CFydf2Rdkz7jDHKSisi3w==} + + '@vitest/mocker@3.2.7': + resolution: {integrity: sha512-Trr0hYO9CM3Wj6ksWHRhK9IZpIY6wTMO5u/MqXurMxT57sWBaOPEtP3Oq60ihZuh5JsiagKfz95OcxdEP6dBrA==} + peerDependencies: + msw: ^2.4.9 + vite: ^5.0.0 || ^6.0.0 || ^7.0.0-0 + peerDependenciesMeta: + msw: + optional: true + vite: + optional: true + + '@vitest/pretty-format@3.2.7': + resolution: {integrity: sha512-KUHlwqVu0sRlhCdyPdQ/wBoTfRahjUky1MubOmYw9fWfIZy1gNoHpuaaQBPAaMaVYdQYHJLurzj8ECCj5OwTqA==} + + '@vitest/runner@3.2.7': + resolution: {integrity: sha512-sB9y4ovltoQP+WaUPwmSxO9WIg9Ig694Di5PalVPsYHklAdE027mehpWF2SQSVq+k6sFgaivbTjTJwZLSHbedA==} + + '@vitest/snapshot@3.2.7': + resolution: {integrity: sha512-7C+MwShwtBSI5Buwoyg3s/iY1eHL9PKAf+O1wVh/TdnjXUtkoL/9YQtre90i4MtNXM6edP1wJ2zOBpfCyhIS7g==} + + '@vitest/spy@3.2.7': + resolution: {integrity: sha512-Q2eQGI6d2L/hBtZ0qNuKcAGid68XK6cv1xsoaIma6PaJhHPoqcEJhYpXZ/5myCMqkNgtP6UKuBhbc0nHKnrkuQ==} + + '@vitest/utils@3.2.7': + resolution: {integrity: sha512-x6BDOd7dyo3PFLY3I9/HJ25X/6OurhGXk2/B9gOZNPF7XDVjeBK4k01lQE5uvDpbuheErh91qYuE1E2OEjK3Rw==} + + ansi-regex@5.0.1: + resolution: {integrity: sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==} + engines: {node: '>=8'} + + ansi-styles@4.3.0: + resolution: {integrity: sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==} + engines: {node: '>=8'} + + assertion-error@2.0.1: + resolution: {integrity: sha512-Izi8RQcffqCeNVgFigKli1ssklIbpHnCYc6AknXGYoB6grJqyeby7jv12JUQgmTAnIDnbck1uxksT4dzN3PWBA==} + engines: {node: '>=12'} + + cac@6.7.14: + resolution: {integrity: sha512-b6Ilus+c3RrdDk+JhLKUAQfzzgLEPy6wcXqS7f/xe1EETvsDP6GORG7SFuOs6cID5YkqchW/LXZbX5bc8j7ZcQ==} + engines: {node: '>=8'} + + chai@5.3.3: + resolution: {integrity: sha512-4zNhdJD/iOjSH0A05ea+Ke6MU5mmpQcbQsSOkgdaUMJ9zTlDTD/GYlwohmIE2u0gaxHYiVHEn1Fw9mZ/ktJWgw==} + engines: {node: '>=18'} + + check-error@2.1.3: + resolution: {integrity: sha512-PAJdDJusoxnwm1VwW07VWwUN1sl7smmC3OKggvndJFadxxDRyFJBX/ggnu/KE4kQAB7a3Dp8f/YXC1FlUprWmA==} + engines: {node: '>= 16'} + + cliui@8.0.1: + resolution: {integrity: sha512-BSeNnyus75C4//NQ9gQt1/csTXyo/8Sb+afLAkzAptFuMsod9HFokGNudZpi/oQV73hnVK+sR+5PVRMd+Dr7YQ==} + engines: {node: '>=12'} + + color-convert@2.0.1: + resolution: {integrity: sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==} + engines: {node: '>=7.0.0'} + + color-name@1.1.4: + resolution: {integrity: sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==} + + debug@4.4.3: + resolution: {integrity: sha512-RGwwWnwQvkVfavKVt22FGLw+xYSdzARwm0ru6DhTVA3umU5hZc28V3kO4stgYryrTlLpuvgI9GiijltAjNbcqA==} + engines: {node: '>=6.0'} + peerDependencies: + supports-color: '*' + peerDependenciesMeta: + supports-color: + optional: true + + deep-eql@5.0.2: + resolution: {integrity: sha512-h5k/5U50IJJFpzfL6nO9jaaumfjO/f2NjK/oYB2Djzm4p9L+3T9qWpZqZ2hAbLPuuYq9wrU08WQyBTL5GbPk5Q==} + engines: {node: '>=6'} + + detect-libc@2.1.2: + resolution: {integrity: sha512-Btj2BOOO83o3WyH59e8MgXsxEQVcarkUOpEYrubB0urwnN10yQ364rsiByU11nZlqWYZm05i/of7io4mzihBtQ==} + engines: {node: '>=8'} + + effect@4.0.0-rc.112: + resolution: {integrity: sha512-wXxwuh1Ywnv4cPRM3Wfa0vDwuOHnZ1TsTgHJkG9XgzND6inhBH9n1vBxhg3iIXOia/OrpmvVmd3lrD4vq6bF3A==} + + emoji-regex@8.0.0: + resolution: {integrity: sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==} + + es-module-lexer@1.7.0: + resolution: {integrity: sha512-jEQoCwk8hyb2AZziIOLhDqpm5+2ww5uIE6lkO/6jcOCusfk6LhMHpXXfBLXTZ7Ydyt0j4VoUQv6uGNYbdW+kBA==} + + esbuild@0.28.2: + resolution: {integrity: sha512-HKVLS8dvII+xoKW9kmqxbRKrnWEXfJJr/FZhhJmiqIB0e053QNYFqOBouTMO/k5sID4MvCiUCvv8b9M4h32wIA==} + engines: {node: '>=18'} + hasBin: true + + escalade@3.2.0: + resolution: {integrity: sha512-WUj2qlxaQtO4g6Pq5c29GTcWGDyd8itL8zTlipgECz3JesAiiOKotd8JU6otB3PACgG6xkJUyVhboMS+bje/jA==} + engines: {node: '>=6'} + + estree-walker@3.0.3: + resolution: {integrity: sha512-7RUKfXgSMMkzt6ZuXmqapOurLGPPfgj6l9uRZ7lRGolvk0y2yocc35LdcxKC5PQZdn2DMqioAQ2NoWcrTKmm6g==} + + expect-type@1.4.0: + resolution: {integrity: sha512-KfYbmpRm0VbLjEvVa9yGwCi9GI34xvi7A/HXYWQO65CSD2u3MczUJSuwXKFIxlGsgBQizV9q5J9NHj4VG0n+pA==} + engines: {node: '>=12.0.0'} + + fast-check@4.10.2: + resolution: {integrity: sha512-iK2f+YrcmoeGqk6fA0ea2bptcu/itMIm4NfEozq6N25+aG6h7s5HZbB/k1aV7b5w5sFLMCbbtRUsTVR+BgC3xw==} + engines: {node: '>=12.17.0'} + + fdir@6.5.0: + resolution: {integrity: sha512-tIbYtZbucOs0BRGqPJkshJUYdL+SDH7dVM8gjy+ERp3WAUjLEFJE+02kanyHtwjWOnwrKYBiwAmM0p4kLJAnXg==} + engines: {node: '>=12.0.0'} + peerDependencies: + picomatch: ^3 || ^4 + peerDependenciesMeta: + picomatch: + optional: true + + fsevents@2.3.3: + resolution: {integrity: sha512-5xoDfX+fL7faATnagmWPpbFtwh/R77WmMMqqHGS65C3vvB0YHrgF+B1YmZ3441tMj5n63k0212XNoJwzlhffQw==} + engines: {node: ^8.16.0 || ^10.6.0 || >=11.0.0} + os: [darwin] + + get-caller-file@2.0.5: + resolution: {integrity: sha512-DyFP3BM/3YHTQOCUL/w0OZHR0lpKeGrxotcHWcqNEdnltqFwXVfhEBQ94eIo34AfQpo0rGki4cyIiftY06h2Fg==} + engines: {node: 6.* || 8.* || >= 10.*} + + is-fullwidth-code-point@3.0.0: + resolution: {integrity: sha512-zymm5+u+sCsSWyD9qNaejV3DFvhCKclKdizYaJUuHA83RLjb7nSuGnddCHGv0hk+KY7BMAlsWeK4Ueg6EV6XQg==} + engines: {node: '>=8'} + + js-tokens@9.0.1: + resolution: {integrity: sha512-mxa9E9ITFOt0ban3j6L5MpjwegGz6lBQmM1IJkWeBZGcMxto50+eWdjC/52xDbS2vy0k7vIMK0Fe2wfL9OQSpQ==} + + lodash.camelcase@4.3.0: + resolution: {integrity: sha512-TwuEnCnxbc3rAvhf/LbG7tJUDzhqXyFnv3dtzLOPgCG/hODL7WFnsbwktkD7yUV0RrreP/l1PALq/YSg6VvjlA==} + + long@5.3.2: + resolution: {integrity: sha512-mNAgZ1GmyNhD7AuqnTG3/VQ26o760+ZYBPKjPvugO8+nLbYfX6TVpJPseBvopbdY+qpZ/lKUnmEc1LeZYS3QAA==} + + loupe@3.2.1: + resolution: {integrity: sha512-CdzqowRJCeLU72bHvWqwRBBlLcMEtIvGrlvef74kMnV2AolS9Y8xUv1I0U/MNAWMhBlKIoyuEgoJ0t/bbwHbLQ==} + + magic-string@0.30.21: + resolution: {integrity: sha512-vd2F4YUyEXKGcLHoq+TEyCjxueSeHnFxyyjNp80yg0XV4vUhnDer/lvvlqM/arB5bXQN5K2/3oinyCRyx8T2CQ==} + + ms@2.1.3: + resolution: {integrity: sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==} + + msgpackr-extract@3.0.4: + resolution: {integrity: sha512-4kmO/MdyUIkLIvTPr8VHLil4AtoKIoniWPIEk5+CDy0xnWC84azhSFmuJ7PxZdsYtiP5kEeQsORAVIeMgxT+Hw==} + hasBin: true + + msgpackr@2.1.0: + resolution: {integrity: sha512-p/pBCVO63CsvvpkomUnNNag6+n38rULuDA6HHe70o2gtC8ODI52foF/4ko2qQcp6OiErJXTmrZeXmsGGHsIQNQ==} + + nanoid@3.3.19: + resolution: {integrity: sha512-Y2tUNy4ouw6tq5oDSKeQYGOyhkUBhNOcGV/02KC+6kd9eDGqdZd++mjMiIDilrBYvjEnCYvVtsuHCuP+okSfug==} + engines: {node: ^10 || ^12 || ^13.7 || ^14 || >=15.0.1} + hasBin: true + + node-gyp-build-optional-packages@5.2.2: + resolution: {integrity: sha512-s+w+rBWnpTMwSFbaE0UXsRlg7hU4FjekKU4eyAih5T8nJuNZT1nNsskXpxmeqSK9UzkBl6UgRlnKc8hz8IEqOw==} + hasBin: true + + pathe@2.0.3: + resolution: {integrity: sha512-WUjGcAqP1gQacoQe+OBJsFA7Ld4DyXuUIjZ5cc75cLHvJ7dtNsTugphxIADwspS+AraAUePCKrSVtPLFj/F88w==} + + pathval@2.0.1: + resolution: {integrity: sha512-//nshmD55c46FuFw26xV/xFAaB5HF9Xdap7HJBBnrKdAd6/GxDBaNA1870O79+9ueg61cZLSVc+OaFlfmObYVQ==} + engines: {node: '>= 14.16'} + + picocolors@1.1.1: + resolution: {integrity: sha512-xceH2snhtb5M9liqDsmEw56le376mTZkEX/jEb/RxNFyegNul7eNslCXP9FDj/Lcu0X8KEyMceP2ntpaHrDEVA==} + + picomatch@4.0.7: + resolution: {integrity: sha512-qcJu88Q2IWqJsDD529JKMdwGm/dvInW4HvQnRwiH9JtihJvzGOscDtHE3x1pBKeUOTysQ8kVmLnJ2kJu7yhcGA==} + engines: {node: '>=12'} + + postcss@8.5.28: + resolution: {integrity: sha512-RRuzqDtt5Y9h3quz5hWhK+TPnsmVs6WwSU6LkJMeY4HstUEDuYTG8UJSdawMRzmzAtV+KEoG8N3Qg2qLy5vM/A==} + engines: {node: ^10 || ^12 || >=14} + + protobufjs@7.6.6: + resolution: {integrity: sha512-dYDWdjSl5RNb7SgPxGQcRU+GtvP7s2fpkrY0r432PcOIaZ0/rBcxEZnQN67iJhFuQiVw754JDoPruPCNdGsbjg==} + engines: {node: '>=12.0.0'} + + pure-rand@8.4.2: + resolution: {integrity: sha512-vvuOGgcuPJAirlHvuQw1TrOiw7ptaIXXmIbNuiNOY6lNGJJH49PQ1Kj4nd783nPdQhQdicgOjVI2yI/9BD6/Ng==} + + require-directory@2.1.1: + resolution: {integrity: sha512-fGxEI7+wsG9xrvdjsrlmL22OMTTiHRwAMroiEeMgq8gzoLC/PQr7RsRDSTLUg/bZAZtF+TVIkHc6/4RIKrui+Q==} + engines: {node: '>=0.10.0'} + + rollup@4.63.4: + resolution: {integrity: sha512-4U0liVayNIoLp3GFl1FcI8561WepLnZ1rqfraGh7S9B3Ur5F9S283y8Futii7RUU2C/97tOBmBy7nYvhoiOpbQ==} + engines: {node: '>=18.0.0', npm: '>=8.0.0'} + hasBin: true + + siginfo@2.0.0: + resolution: {integrity: sha512-ybx0WO1/8bSBLEWXZvEd7gMW3Sn3JFlW3TvX1nREbDLRNQNaeNN8WK0meBwPdAaOI7TtRRRJn/Es1zhrrCHu7g==} + + source-map-js@1.2.1: + resolution: {integrity: sha512-UXWMKhLOwVKb728IUtQPXxfYU+usdybtUrK/8uGE8CQMvrhOpwvzDBwj0QhSL7MQc7vIsISBG8VQ8+IDQxpfQA==} + engines: {node: '>=0.10.0'} + + stackback@0.0.2: + resolution: {integrity: sha512-1XMJE5fQo1jGH6Y/7ebnwPOBEkIEnT4QF32d5R1+VXdXveM0IBMJt8zfaxX1P3QhVwrYe+576+jkANtSS2mBbw==} + + std-env@3.10.0: + resolution: {integrity: sha512-5GS12FdOZNliM5mAOxFRg7Ir0pWz8MdpYm6AY6VPkGpbA7ZzmbzNcBJQ0GPvvyWgcY7QAhCgf9Uy89I03faLkg==} + + string-width@4.2.3: + resolution: {integrity: sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==} + engines: {node: '>=8'} + + strip-ansi@6.0.1: + resolution: {integrity: sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==} + engines: {node: '>=8'} + + strip-literal@3.1.0: + resolution: {integrity: sha512-8r3mkIM/2+PpjHoOtiAW8Rg3jJLHaV7xPwG+YRGrv6FP0wwk/toTpATxWYOW0BKdWwl82VT2tFYi5DlROa0Mxg==} + + tinybench@2.9.0: + resolution: {integrity: sha512-0+DUvqWMValLmha6lr4kD8iAMK1HzV0/aKnCtWb9v9641TnP/MFb7Pc2bxoxQjTXAErryXVgUOfv2YqNllqGeg==} + + tinyexec@0.3.2: + resolution: {integrity: sha512-KQQR9yN7R5+OSwaK0XQoj22pwHoTlgYqmUscPYoknOoWCWfj/5/ABTMRi69FrKU5ffPVh5QcFikpWJI/P1ocHA==} + + tinyglobby@0.2.17: + resolution: {integrity: sha512-wXR/dYpcqKmfWpEdZjiKJOwCNFndD0DMnrW/cYjVGttEkBfVgcLFHoNrlj47mjOVic9yyNu65alsgF4NQyTa2g==} + engines: {node: '>=12.0.0'} + + tinypool@1.1.1: + resolution: {integrity: sha512-Zba82s87IFq9A9XmjiX5uZA/ARWDrB03OHlq+Vw1fSdt0I+4/Kutwy8BP4Y/y/aORMo61FQ0vIb5j44vSo5Pkg==} + engines: {node: ^18.0.0 || >=20.0.0} + + tinyrainbow@2.0.0: + resolution: {integrity: sha512-op4nsTR47R6p0vMUUoYl/a+ljLFVtlfaXkLQmqfLR1qHma1h/ysYk4hEXZ880bf2CYgTskvTa/e196Vd5dDQXw==} + engines: {node: '>=14.0.0'} + + tinyspy@4.0.6: + resolution: {integrity: sha512-u8KszXvGfU68hVcZpRHKG28T0krMuv2G5nDhiHaMLen/gIuFEgIJhaJuO69qjnXg5paSrbPMFfx3brNuN8eVSg==} + engines: {node: '>=14.0.0'} + + tsx@4.23.15: + resolution: {integrity: sha512-Yiex1Ovn8z2xPpOWckIiysV1SSyRMY9BkLF++q0yKiDxCqRhosKfMg3janKkiLBwZ5c/YryloKwGZcrEmtwxKw==} + engines: {node: '>=18.0.0'} + hasBin: true + + typescript@5.9.3: + resolution: {integrity: sha512-jl1vZzPDinLr9eUt3J/t7V6FgNEw9QjvBPdysz9KfQDD41fQrC2Y4vKQdiaUpFT4bXlb1RHhLpp8wtm6M5TgSw==} + engines: {node: '>=14.17'} + hasBin: true + + undici-types@6.21.0: + resolution: {integrity: sha512-iwDZqg0QAGrg9Rav5H4n0M64c3mkR59cJ6wQp+7C4nI0gsmExaedaYLNO44eT4AtBBwjbTiGPMlt2Md0T9H9JQ==} + + vite-node@3.2.4: + resolution: {integrity: sha512-EbKSKh+bh1E1IFxeO0pg1n4dvoOTt0UDiXMd/qn++r98+jPO1xtJilvXldeuQ8giIB5IkpjCgMleHMNEsGH6pg==} + engines: {node: ^18.0.0 || ^20.0.0 || >=22.0.0} + hasBin: true + + vite@7.3.6: + resolution: {integrity: sha512-4XP60spRGjSZFf1qYH+dJIkK2znL3zQfl9KkOV9MkkRR/3Dls0dxaBsQPTloEc5BLXWPL9vsOxopxyKoMmDueg==} + engines: {node: ^20.19.0 || >=22.12.0} + hasBin: true + peerDependencies: + '@types/node': ^20.19.0 || >=22.12.0 + jiti: '>=1.21.0' + less: ^4.0.0 + lightningcss: ^1.21.0 + sass: ^1.70.0 + sass-embedded: ^1.70.0 + stylus: '>=0.54.8' + sugarss: ^5.0.0 + terser: ^5.16.0 + tsx: ^4.8.1 + yaml: ^2.4.2 + peerDependenciesMeta: + '@types/node': + optional: true + jiti: + optional: true + less: + optional: true + lightningcss: + optional: true + sass: + optional: true + sass-embedded: + optional: true + stylus: + optional: true + sugarss: + optional: true + terser: + optional: true + tsx: + optional: true + yaml: + optional: true + + vitest@3.2.7: + resolution: {integrity: sha512-KrxIJ62Fd89gfysR4WotlgZABiz2dqFPgqGzX7s+CwsqLFomRH7777ZcrOD6+WVAh7khPQP41A+BKbpcJFrdEg==} + engines: {node: ^18.0.0 || ^20.0.0 || >=22.0.0} + hasBin: true + peerDependencies: + '@edge-runtime/vm': '*' + '@types/debug': ^4.1.12 + '@types/node': ^18.0.0 || ^20.0.0 || >=22.0.0 + '@vitest/browser': 3.2.7 + '@vitest/ui': 3.2.7 + happy-dom: '*' + jsdom: '*' + peerDependenciesMeta: + '@edge-runtime/vm': + optional: true + '@types/debug': + optional: true + '@types/node': + optional: true + '@vitest/browser': + optional: true + '@vitest/ui': + optional: true + happy-dom: + optional: true + jsdom: + optional: true + + why-is-node-running@2.3.0: + resolution: {integrity: sha512-hUrmaWBdVDcxvYqnyh09zunKzROWjbZTiNy8dBEjkS7ehEDQibXJ7XvlmtbwuTclUiIyN+CyXQD4Vmko8fNm8w==} + engines: {node: '>=8'} + hasBin: true + + wrap-ansi@7.0.0: + resolution: {integrity: sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q==} + engines: {node: '>=10'} + + y18n@5.0.8: + resolution: {integrity: sha512-0pfFzegeDWJHJIAmTLRP2DwHjdF5s7jo9tuztdQxAhINCdvS+3nGINqPd00AphqJR/0LhANUS6/+7SCb98YOfA==} + engines: {node: '>=10'} + + yargs-parser@21.1.1: + resolution: {integrity: sha512-tVpsJW7DdjecAiFpbIB1e3qxIQsE6NoPc5/eTdrbbIC4h0LVsWhnoa3g+m2HclBIujHzsxZ4VJVA+GUuc2/LBw==} + engines: {node: '>=12'} + + yargs@17.7.3: + resolution: {integrity: sha512-GZtjxm/J/4TSxuL3FNYjCmLktBTnIw/rVmKSIyKeYAZpmJB2ig9VauCC5xsa82GNKVKDAqpOn3KVzNt0zmrU0g==} + engines: {node: '>=12'} + +snapshots: + + '@esbuild/aix-ppc64@0.28.2': + optional: true + + '@esbuild/android-arm64@0.28.2': + optional: true + + '@esbuild/android-arm@0.28.2': + optional: true + + '@esbuild/android-x64@0.28.2': + optional: true + + '@esbuild/darwin-arm64@0.28.2': + optional: true + + '@esbuild/darwin-x64@0.28.2': + optional: true + + '@esbuild/freebsd-arm64@0.28.2': + optional: true + + '@esbuild/freebsd-x64@0.28.2': + optional: true + + '@esbuild/linux-arm64@0.28.2': + optional: true + + '@esbuild/linux-arm@0.28.2': + optional: true + + '@esbuild/linux-ia32@0.28.2': + optional: true + + '@esbuild/linux-loong64@0.28.2': + optional: true + + '@esbuild/linux-mips64el@0.28.2': + optional: true + + '@esbuild/linux-ppc64@0.28.2': + optional: true + + '@esbuild/linux-riscv64@0.28.2': + optional: true + + '@esbuild/linux-s390x@0.28.2': + optional: true + + '@esbuild/linux-x64@0.28.2': + optional: true + + '@esbuild/netbsd-arm64@0.28.2': + optional: true + + '@esbuild/netbsd-x64@0.28.2': + optional: true + + '@esbuild/openbsd-arm64@0.28.2': + optional: true + + '@esbuild/openbsd-x64@0.28.2': + optional: true + + '@esbuild/openharmony-arm64@0.28.2': + optional: true + + '@esbuild/sunos-x64@0.28.2': + optional: true + + '@esbuild/win32-arm64@0.28.2': + optional: true + + '@esbuild/win32-ia32@0.28.2': + optional: true + + '@esbuild/win32-x64@0.28.2': + optional: true + + '@grpc/grpc-js@1.14.5': + dependencies: + '@grpc/proto-loader': 0.8.1 + '@js-sdsl/ordered-map': 4.4.2 + + '@grpc/proto-loader@0.7.15': + dependencies: + lodash.camelcase: 4.3.0 + long: 5.3.2 + protobufjs: 7.6.6 + yargs: 17.7.3 + + '@grpc/proto-loader@0.8.1': + dependencies: + lodash.camelcase: 4.3.0 + long: 5.3.2 + protobufjs: 7.6.6 + yargs: 17.7.3 + + '@jridgewell/sourcemap-codec@1.6.0': {} + + '@js-sdsl/ordered-map@4.4.2': {} + + '@msgpackr-extract/msgpackr-extract-darwin-arm64@3.0.4': + optional: true + + '@msgpackr-extract/msgpackr-extract-darwin-x64@3.0.4': + optional: true + + '@msgpackr-extract/msgpackr-extract-linux-arm64@3.0.4': + optional: true + + '@msgpackr-extract/msgpackr-extract-linux-arm@3.0.4': + optional: true + + '@msgpackr-extract/msgpackr-extract-linux-x64@3.0.4': + optional: true + + '@msgpackr-extract/msgpackr-extract-win32-x64@3.0.4': + optional: true + + '@napi-rs/lzma-linux-x64-gnu@1.5.1': + optional: true + + '@protobufjs/aspromise@1.1.2': {} + + '@protobufjs/base64@1.1.2': {} + + '@protobufjs/codegen@2.0.5': {} + + '@protobufjs/eventemitter@1.1.1': {} + + '@protobufjs/fetch@1.1.1': + dependencies: + '@protobufjs/aspromise': 1.1.2 + + '@protobufjs/float@1.0.2': {} + + '@protobufjs/path@1.1.2': {} + + '@protobufjs/pool@1.1.0': {} + + '@protobufjs/utf8@1.1.2': {} + + '@rollup/rollup-android-arm-eabi@4.63.4': + optional: true + + '@rollup/rollup-android-arm64@4.63.4': + optional: true + + '@rollup/rollup-darwin-arm64@4.63.4': + optional: true + + '@rollup/rollup-darwin-x64@4.63.4': + optional: true + + '@rollup/rollup-freebsd-arm64@4.63.4': + optional: true + + '@rollup/rollup-freebsd-x64@4.63.4': + optional: true + + '@rollup/rollup-linux-arm-gnueabihf@4.63.4': + optional: true + + '@rollup/rollup-linux-arm-musleabihf@4.63.4': + optional: true + + '@rollup/rollup-linux-arm64-gnu@4.63.4': + optional: true + + '@rollup/rollup-linux-arm64-musl@4.63.4': + optional: true + + '@rollup/rollup-linux-loong64-gnu@4.63.4': + optional: true + + '@rollup/rollup-linux-loong64-musl@4.63.4': + optional: true + + '@rollup/rollup-linux-ppc64-gnu@4.63.4': + optional: true + + '@rollup/rollup-linux-ppc64-musl@4.63.4': + optional: true + + '@rollup/rollup-linux-riscv64-gnu@4.63.4': + optional: true + + '@rollup/rollup-linux-riscv64-musl@4.63.4': + optional: true + + '@rollup/rollup-linux-s390x-gnu@4.63.4': + optional: true + + '@rollup/rollup-linux-x64-gnu@4.63.4': + optional: true + + '@rollup/rollup-linux-x64-musl@4.63.4': + optional: true + + '@rollup/rollup-openbsd-x64@4.63.4': + optional: true + + '@rollup/rollup-openharmony-arm64@4.63.4': + optional: true + + '@rollup/rollup-win32-arm64-msvc@4.63.4': + optional: true + + '@rollup/rollup-win32-ia32-msvc@4.63.4': + optional: true + + '@rollup/rollup-win32-x64-gnu@4.63.4': + optional: true + + '@rollup/rollup-win32-x64-msvc@4.63.4': + optional: true + + '@types/chai@5.2.3': + dependencies: + '@types/deep-eql': 4.0.2 + assertion-error: 2.0.1 + + '@types/deep-eql@4.0.2': {} + + '@types/estree@1.0.9': {} + + '@types/node@22.20.4': + dependencies: + undici-types: 6.21.0 + + '@vitest/expect@3.2.7': + dependencies: + '@types/chai': 5.2.3 + '@vitest/spy': 3.2.7 + '@vitest/utils': 3.2.7 + chai: 5.3.3 + tinyrainbow: 2.0.0 + + '@vitest/mocker@3.2.7(vite@7.3.6(@types/node@22.20.4)(tsx@4.23.15))': + dependencies: + '@vitest/spy': 3.2.7 + estree-walker: 3.0.3 + magic-string: 0.30.21 + optionalDependencies: + vite: 7.3.6(@types/node@22.20.4)(tsx@4.23.15) + + '@vitest/pretty-format@3.2.7': + dependencies: + tinyrainbow: 2.0.0 + + '@vitest/runner@3.2.7': + dependencies: + '@vitest/utils': 3.2.7 + pathe: 2.0.3 + strip-literal: 3.1.0 + + '@vitest/snapshot@3.2.7': + dependencies: + '@vitest/pretty-format': 3.2.7 + magic-string: 0.30.21 + pathe: 2.0.3 + + '@vitest/spy@3.2.7': + dependencies: + tinyspy: 4.0.6 + + '@vitest/utils@3.2.7': + dependencies: + '@vitest/pretty-format': 3.2.7 + loupe: 3.2.1 + tinyrainbow: 2.0.0 + + ansi-regex@5.0.1: {} + + ansi-styles@4.3.0: + dependencies: + color-convert: 2.0.1 + + assertion-error@2.0.1: {} + + cac@6.7.14: {} + + chai@5.3.3: + dependencies: + assertion-error: 2.0.1 + check-error: 2.1.3 + deep-eql: 5.0.2 + loupe: 3.2.1 + pathval: 2.0.1 + + check-error@2.1.3: {} + + cliui@8.0.1: + dependencies: + string-width: 4.2.3 + strip-ansi: 6.0.1 + wrap-ansi: 7.0.0 + + color-convert@2.0.1: + dependencies: + color-name: 1.1.4 + + color-name@1.1.4: {} + + debug@4.4.3: + dependencies: + ms: 2.1.3 + + deep-eql@5.0.2: {} + + detect-libc@2.1.2: + optional: true + + effect@4.0.0-rc.112: + dependencies: + fast-check: 4.10.2 + msgpackr: 2.1.0 + + emoji-regex@8.0.0: {} + + es-module-lexer@1.7.0: {} + + esbuild@0.28.2: + optionalDependencies: + '@esbuild/aix-ppc64': 0.28.2 + '@esbuild/android-arm': 0.28.2 + '@esbuild/android-arm64': 0.28.2 + '@esbuild/android-x64': 0.28.2 + '@esbuild/darwin-arm64': 0.28.2 + '@esbuild/darwin-x64': 0.28.2 + '@esbuild/freebsd-arm64': 0.28.2 + '@esbuild/freebsd-x64': 0.28.2 + '@esbuild/linux-arm': 0.28.2 + '@esbuild/linux-arm64': 0.28.2 + '@esbuild/linux-ia32': 0.28.2 + '@esbuild/linux-loong64': 0.28.2 + '@esbuild/linux-mips64el': 0.28.2 + '@esbuild/linux-ppc64': 0.28.2 + '@esbuild/linux-riscv64': 0.28.2 + '@esbuild/linux-s390x': 0.28.2 + '@esbuild/linux-x64': 0.28.2 + '@esbuild/netbsd-arm64': 0.28.2 + '@esbuild/netbsd-x64': 0.28.2 + '@esbuild/openbsd-arm64': 0.28.2 + '@esbuild/openbsd-x64': 0.28.2 + '@esbuild/openharmony-arm64': 0.28.2 + '@esbuild/sunos-x64': 0.28.2 + '@esbuild/win32-arm64': 0.28.2 + '@esbuild/win32-ia32': 0.28.2 + '@esbuild/win32-x64': 0.28.2 + + escalade@3.2.0: {} + + estree-walker@3.0.3: + dependencies: + '@types/estree': 1.0.9 + + expect-type@1.4.0: {} + + fast-check@4.10.2: + dependencies: + pure-rand: 8.4.2 + + fdir@6.5.0(picomatch@4.0.7): + optionalDependencies: + picomatch: 4.0.7 + + fsevents@2.3.3: + optional: true + + get-caller-file@2.0.5: {} + + is-fullwidth-code-point@3.0.0: {} + + js-tokens@9.0.1: {} + + lodash.camelcase@4.3.0: {} + + long@5.3.2: {} + + loupe@3.2.1: {} + + magic-string@0.30.21: + dependencies: + '@jridgewell/sourcemap-codec': 1.6.0 + + ms@2.1.3: {} + + msgpackr-extract@3.0.4: + dependencies: + node-gyp-build-optional-packages: 5.2.2 + optionalDependencies: + '@msgpackr-extract/msgpackr-extract-darwin-arm64': 3.0.4 + '@msgpackr-extract/msgpackr-extract-darwin-x64': 3.0.4 + '@msgpackr-extract/msgpackr-extract-linux-arm': 3.0.4 + '@msgpackr-extract/msgpackr-extract-linux-arm64': 3.0.4 + '@msgpackr-extract/msgpackr-extract-linux-x64': 3.0.4 + '@msgpackr-extract/msgpackr-extract-win32-x64': 3.0.4 + optional: true + + msgpackr@2.1.0: + optionalDependencies: + msgpackr-extract: 3.0.4 + + nanoid@3.3.19: {} + + node-gyp-build-optional-packages@5.2.2: + dependencies: + detect-libc: 2.1.2 + optional: true + + pathe@2.0.3: {} + + pathval@2.0.1: {} + + picocolors@1.1.1: {} + + picomatch@4.0.7: {} + + postcss@8.5.28: + dependencies: + nanoid: 3.3.19 + picocolors: 1.1.1 + source-map-js: 1.2.1 + + protobufjs@7.6.6: + dependencies: + '@protobufjs/aspromise': 1.1.2 + '@protobufjs/base64': 1.1.2 + '@protobufjs/codegen': 2.0.5 + '@protobufjs/eventemitter': 1.1.1 + '@protobufjs/fetch': 1.1.1 + '@protobufjs/float': 1.0.2 + '@protobufjs/path': 1.1.2 + '@protobufjs/pool': 1.1.0 + '@protobufjs/utf8': 1.1.2 + '@types/node': 22.20.4 + long: 5.3.2 + + pure-rand@8.4.2: {} + + require-directory@2.1.1: {} + + rollup@4.63.4: + dependencies: + '@types/estree': 1.0.9 + optionalDependencies: + '@napi-rs/lzma-linux-x64-gnu': 1.5.1 + '@rollup/rollup-android-arm-eabi': 4.63.4 + '@rollup/rollup-android-arm64': 4.63.4 + '@rollup/rollup-darwin-arm64': 4.63.4 + '@rollup/rollup-darwin-x64': 4.63.4 + '@rollup/rollup-freebsd-arm64': 4.63.4 + '@rollup/rollup-freebsd-x64': 4.63.4 + '@rollup/rollup-linux-arm-gnueabihf': 4.63.4 + '@rollup/rollup-linux-arm-musleabihf': 4.63.4 + '@rollup/rollup-linux-arm64-gnu': 4.63.4 + '@rollup/rollup-linux-arm64-musl': 4.63.4 + '@rollup/rollup-linux-loong64-gnu': 4.63.4 + '@rollup/rollup-linux-loong64-musl': 4.63.4 + '@rollup/rollup-linux-ppc64-gnu': 4.63.4 + '@rollup/rollup-linux-ppc64-musl': 4.63.4 + '@rollup/rollup-linux-riscv64-gnu': 4.63.4 + '@rollup/rollup-linux-riscv64-musl': 4.63.4 + '@rollup/rollup-linux-s390x-gnu': 4.63.4 + '@rollup/rollup-linux-x64-gnu': 4.63.4 + '@rollup/rollup-linux-x64-musl': 4.63.4 + '@rollup/rollup-openbsd-x64': 4.63.4 + '@rollup/rollup-openharmony-arm64': 4.63.4 + '@rollup/rollup-win32-arm64-msvc': 4.63.4 + '@rollup/rollup-win32-ia32-msvc': 4.63.4 + '@rollup/rollup-win32-x64-gnu': 4.63.4 + '@rollup/rollup-win32-x64-msvc': 4.63.4 + fsevents: 2.3.3 + + siginfo@2.0.0: {} + + source-map-js@1.2.1: {} + + stackback@0.0.2: {} + + std-env@3.10.0: {} + + string-width@4.2.3: + dependencies: + emoji-regex: 8.0.0 + is-fullwidth-code-point: 3.0.0 + strip-ansi: 6.0.1 + + strip-ansi@6.0.1: + dependencies: + ansi-regex: 5.0.1 + + strip-literal@3.1.0: + dependencies: + js-tokens: 9.0.1 + + tinybench@2.9.0: {} + + tinyexec@0.3.2: {} + + tinyglobby@0.2.17: + dependencies: + fdir: 6.5.0(picomatch@4.0.7) + picomatch: 4.0.7 + + tinypool@1.1.1: {} + + tinyrainbow@2.0.0: {} + + tinyspy@4.0.6: {} + + tsx@4.23.15: + dependencies: + esbuild: 0.28.2 + optionalDependencies: + fsevents: 2.3.3 + + typescript@5.9.3: {} + + undici-types@6.21.0: {} + + vite-node@3.2.4(@types/node@22.20.4)(tsx@4.23.15): + dependencies: + cac: 6.7.14 + debug: 4.4.3 + es-module-lexer: 1.7.0 + pathe: 2.0.3 + vite: 7.3.6(@types/node@22.20.4)(tsx@4.23.15) + transitivePeerDependencies: + - '@types/node' + - jiti + - less + - lightningcss + - sass + - sass-embedded + - stylus + - sugarss + - supports-color + - terser + - tsx + - yaml + + vite@7.3.6(@types/node@22.20.4)(tsx@4.23.15): + dependencies: + esbuild: 0.28.2 + fdir: 6.5.0(picomatch@4.0.7) + picomatch: 4.0.7 + postcss: 8.5.28 + rollup: 4.63.4 + tinyglobby: 0.2.17 + optionalDependencies: + '@types/node': 22.20.4 + fsevents: 2.3.3 + tsx: 4.23.15 + + vitest@3.2.7(@types/node@22.20.4)(tsx@4.23.15): + dependencies: + '@types/chai': 5.2.3 + '@vitest/expect': 3.2.7 + '@vitest/mocker': 3.2.7(vite@7.3.6(@types/node@22.20.4)(tsx@4.23.15)) + '@vitest/pretty-format': 3.2.7 + '@vitest/runner': 3.2.7 + '@vitest/snapshot': 3.2.7 + '@vitest/spy': 3.2.7 + '@vitest/utils': 3.2.7 + chai: 5.3.3 + debug: 4.4.3 + expect-type: 1.4.0 + magic-string: 0.30.21 + pathe: 2.0.3 + picomatch: 4.0.7 + std-env: 3.10.0 + tinybench: 2.9.0 + tinyexec: 0.3.2 + tinyglobby: 0.2.17 + tinypool: 1.1.1 + tinyrainbow: 2.0.0 + vite: 7.3.6(@types/node@22.20.4)(tsx@4.23.15) + vite-node: 3.2.4(@types/node@22.20.4)(tsx@4.23.15) + why-is-node-running: 2.3.0 + optionalDependencies: + '@types/node': 22.20.4 + transitivePeerDependencies: + - jiti + - less + - lightningcss + - msw + - sass + - sass-embedded + - stylus + - sugarss + - supports-color + - terser + - tsx + - yaml + + why-is-node-running@2.3.0: + dependencies: + siginfo: 2.0.0 + stackback: 0.0.2 + + wrap-ansi@7.0.0: + dependencies: + ansi-styles: 4.3.0 + string-width: 4.2.3 + strip-ansi: 6.0.1 + + y18n@5.0.8: {} + + yargs-parser@21.1.1: {} + + yargs@17.7.3: + dependencies: + cliui: 8.0.1 + escalade: 3.2.0 + get-caller-file: 2.0.5 + require-directory: 2.1.1 + string-width: 4.2.3 + y18n: 5.0.8 + yargs-parser: 21.1.1 diff --git a/pkgs/substrate-link/src/pnpm-workspace.yaml b/pkgs/substrate-link/src/pnpm-workspace.yaml new file mode 100644 index 000000000..269d4cb25 --- /dev/null +++ b/pkgs/substrate-link/src/pnpm-workspace.yaml @@ -0,0 +1,4 @@ +allowBuilds: + esbuild: true + msgpackr-extract: false + protobufjs: false diff --git a/pkgs/substrate-link/src/tsconfig.json b/pkgs/substrate-link/src/tsconfig.json new file mode 100644 index 000000000..ef31c9521 --- /dev/null +++ b/pkgs/substrate-link/src/tsconfig.json @@ -0,0 +1,16 @@ +{ + "compilerOptions": { + "target": "ES2022", + "module": "NodeNext", + "moduleResolution": "NodeNext", + "lib": ["ES2022"], + "strict": true, + "exactOptionalPropertyTypes": false, + "noUncheckedIndexedAccess": true, + "allowImportingTsExtensions": true, + "noEmit": true, + "skipLibCheck": true, + "types": ["node"] + }, + "include": ["src", "test", "apps"] +} diff --git a/pkgs/substrate/default.nix b/pkgs/substrate/default.nix new file mode 100644 index 000000000..2c186d350 --- /dev/null +++ b/pkgs/substrate/default.nix @@ -0,0 +1,146 @@ +{ + lib, + buildGoModule, + fetchFromGitHub, + applyPatches, + runCommand, + # Passed in by the caller from the nixpkgs-go input, exactly as pkgs/ax gets + # it (see overlays/default.nix and the nixpkgs-go comment in flake.nix). + # Substrate's go.mod says `go 1.27.0`; the probe built it with 1.27.1 + # (MEASURED, evals-2026-09-23/substrate/probe-build.md section 2), and one + # toolchain for ax and Substrate keeps the fleet on one Go. + go_1_27, +}: +# Agent Substrate, the sandbox control plane ax delegates to. Upstream +# github.com/agent-substrate/substrate, pinned by commit to d277088b: the +# SERVER version the 2026-09-23 version rule names. ax keeps its own vendored +# CLIENT pin (672533541dbf); wire compatibility between the two is MEASURED +# (probe-build.md section 4). +# +# Not in nixpkgs. Upstream publishes no release images the fleet may use (the +# kagent-dev fork's ghcr images are ruled out), so every component image is +# built here and seeded into the NAS registry by pkgs/substrate/images.nix. +# +# The four patches touch only manifests/, never Go code: +# 0001 pauseImage -> localhost:5000/pause (atelet pulls it itself; the fleet +# must not depend on registry.k8s.io at sandbox start). +# 0002 third-party images -> their linux/amd64 child digests, so the NAS +# seeds ~0.7 GB instead of every platform (see the patch header). +# 0003 the kind overlay's literal RustFS credential (public in the upstream +# repo) -> secretKeyRef to Secret ate-system/ax-fleet-rustfs, which the +# bootstrap step 25-rustfs-secret generates once on the NAS. +# 0004 the kind overlay's tracing stack, bounded (fix round 3): jaeger +# keeps at most 10000 traces in memory, the collector gets a +# memory_limiter and debug at basic, and both get limits.memory +# 512Mi. Upstream leaves them unbounded (MEASURED at d277088b), and +# they run on the NAS next to Immich, Paperless and NFS. +# ate-setup reads the manifests from its working directory's repository root +# (it walks up to go.mod). What it reads for `deploy ate-system` is go.mod, +# manifests/ and hack/ (kustomize overlays, CSI manifests; MEASURED grep of +# cmd/ate-setup/internal/steps), so passthru.installTree is those three, +# about 1.3 MB instead of the 172 MB tree, and passthru.ate-setup is the +# installer alone (55 MB instead of all seven binaries). Only those two sit in +# the NAS closure at run time; the component binaries reach the NAS inside +# the images. +let + version = "d277088b"; + rev = "d277088bc1d081ef716d81dd7986d05d0a36ad3a"; + + source = applyPatches { + name = "substrate-source-${version}"; + src = fetchFromGitHub { + owner = "agent-substrate"; + repo = "substrate"; + inherit rev; + # nix-prefetch-url --unpack of the GitHub archive; the unpacked tree is + # byte-identical to /home/tom/Downloads/substrate at this rev (MEASURED + # diff -rq, 2026-09-23). + hash = "sha256-/KY4vYgHbeiVnpRUsVFOVfoN3SVEtWJrXTj0Pb4Zeqw="; + }; + patches = [ + ./patches/0001-sandboxconfig-pause-localhost.patch + ./patches/0002-images-linux-amd64-digests.patch + ./patches/0003-kind-rustfs-credential-secret.patch + ./patches/0004-kind-otel-memory-bounds.patch + ]; + }; + + buildGo127Module = buildGoModule.override { go = go_1_27; }; + + common = { + inherit version; + src = source; + + # The tree is vendored (vendor/modules.txt), so no module download. + vendorHash = null; + + env.CGO_ENABLED = "0"; + + # The Makefile's LDFLAGS (Makefile:44-45), with the version the node label, + # the image tag and ate-setup's VERSION all share. + ldflags = [ + "-s" + "-w" + "-X=github.com/agent-substrate/substrate/internal/version.Version=${version}" + ]; + + # Upstream's suite needs Docker (225 PostgreSQL testcontainer tests), root + # (19 tests) and an FHS /bin/sleep; the probe ran it outside nix (1422 + # pass, 2 environment-only failures, MEASURED probe-build.md section 3). + # Inside the nix sandbox it can only fail for the same reasons. + doCheck = false; + }; + + ate-setup = buildGo127Module ( + common + // { + pname = "ate-setup"; + subPackages = [ "cmd/ate-setup" ]; + meta.mainProgram = "ate-setup"; + } + ); + + installTree = runCommand "substrate-install-tree-${version}" { } '' + mkdir -p $out + cp ${source}/go.mod $out/ + cp -r ${source}/manifests ${source}/hack $out/ + ''; + + # The six components the kind install and the WorkerPool run, plus the + # installer. Image names are the import paths' last element, which is what + # ate-setup's --image-repo mode looks up (cmd/ate-setup/internal/images). + components = [ + "ateapi" + "atecontroller" + "atelet" + "atenet" + "podcertcontroller" + "ateom-gvisor" + ]; +in +buildGo127Module ( + common + // { + pname = "substrate"; + + subPackages = [ "cmd/ate-setup" ] ++ map (c: "cmd/${c}") components; + + passthru = { + inherit + source + installTree + ate-setup + rev + components + ; + }; + + meta = { + description = "Agent Substrate (upstream main ${version}): ate-setup and the gVisor control plane"; + homepage = "https://github.com/agent-substrate/substrate"; + license = lib.licenses.asl20; + platforms = [ "x86_64-linux" ]; + mainProgram = "ate-setup"; + }; + } +) diff --git a/pkgs/substrate/images.nix b/pkgs/substrate/images.nix new file mode 100644 index 000000000..83da1c65c --- /dev/null +++ b/pkgs/substrate/images.nix @@ -0,0 +1,298 @@ +{ + lib, + dockerTools, + runCommand, + linkFarm, + fetchurl, + skopeo, + jq, + cacert, + tzdata, + # pkgs/substrate/default.nix, built with the fleet's Go 1.27.1. + substrate, +}: +# Every image the Substrate install and the gVisor WorkerPool pull, as OCI +# layouts the NAS seeds into its registry (modules/ax-fleet/substrate.nix, +# myAxFleet.registrySeed). Each layout carries a `digest` file holding the +# manifest digest, so references are written by digest without IFD: whatever +# needs the digest reads the file at build or run time. +# +# components the six Substrate components, nix-built from d277088b, tagged +# with the version (ate-setup --image-repo/--image-tag resolves +# each tag to a digest with a HEAD request, then pins it). +# thirdParty the images the kind install names by digest, fetched as the +# linux/amd64 child manifest of the upstream-pinned index +# (patch 0002), bytes and digest preserved. Fixed-output. +# gvisor the runsc tarball SandboxConfig gvisor-default names, fetched +# with the sha256 that manifest carries. +# +# The whole set is one derivation (a linkFarm) so `nix build` of the flake's +# substrate-images output proves every image; the parts are in passthru. +let + inherit (substrate) version; + + # ko's default base is gcr.io/distroless/static-debian13 (.ko.yaml). What a + # static Go binary needs from it: CA roots, zoneinfo, a passwd naming root and + # nonroot 65532 (atenet runs as 65532, atelet as 0; MEASURED + # manifests/ate-install), and a world-writable /tmp. + staticBase = '' + mkdir -p etc tmp home/nonroot + chmod 1777 tmp + cat > etc/passwd <<'EOF' + root:x:0:0:root:/root:/sbin/nologin + nonroot:x:65532:65532:nonroot:/home/nonroot:/sbin/nologin + nobody:x:65534:65534:nobody:/nonexistent:/sbin/nologin + EOF + cat > etc/group <<'EOF' + root:x:0: + nonroot:x:65532: + nobody:x:65534: + EOF + sed -i 's/^ //' etc/passwd etc/group + ''; + + # docker-archive -> OCI layout plus its manifest digest. Layers are gzipped + # so the registry and containerd hold compressed blobs, as they would for a + # ko push. + toOci = + { + image, + tag, + repo, + }: + runCommand "${lib.replaceStrings [ "/" ] [ "-" ] repo}-${tag}-oci" + { + nativeBuildInputs = [ + skopeo + jq + ]; + passthru = { + inherit repo tag image; + }; + } + '' + export HOME=$TMPDIR + skopeo --insecure-policy --tmpdir "$TMPDIR" copy \ + --dest-compress --dest-compress-format gzip \ + docker-archive:${image} oci:$out:${tag} + jq -r '.manifests | if length == 1 then .[0].digest else error("expected one manifest") end' \ + $out/index.json > $out/digest + grep -Eq '^sha256:[0-9a-f]{64}$' $out/digest + ''; + + component = + name: + let + image = dockerTools.buildLayeredImage { + name = "substrate/${name}"; + tag = version; + contents = [ + cacert + tzdata + ]; + extraCommands = staticBase + '' + mkdir -p ko-app + cp ${substrate}/bin/${name} ko-app/${name} + ''; + config = { + # ko's layout: the binary at /ko-app/ is the entrypoint. + Entrypoint = [ "/ko-app/${name}" ]; + Env = [ + "SSL_CERT_FILE=${cacert}/etc/ssl/certs/ca-bundle.crt" + "ZONEINFO=${tzdata}/share/zoneinfo" + "PATH=/ko-app" + ]; + WorkingDir = "/"; + Labels = { + "org.opencontainers.image.source" = "https://github.com/agent-substrate/substrate"; + "org.opencontainers.image.revision" = substrate.rev; + "org.opencontainers.image.version" = version; + }; + }; + }; + in + toOci { + inherit image; + tag = version; + repo = "substrate/${name}"; + }; + + components = lib.genAttrs substrate.components component; + + # A digest-preserving copy of one upstream manifest. The build fetches the + # upstream-pinned index, checks that it lists `digest` for linux/amd64, then + # copies that manifest and its blobs byte for byte. outputHash pins the result. + thirdPartyImage = + { + name, + upstream, + index, + digest, + repo, + tag, + hash, + }: + runCommand "${name}-${tag}-oci" + { + nativeBuildInputs = [ + skopeo + jq + ]; + SSL_CERT_FILE = "${cacert}/etc/ssl/certs/ca-bundle.crt"; + impureEnvVars = lib.fetchers.proxyImpureEnvVars; + outputHashMode = "recursive"; + outputHashAlgo = "sha256"; + outputHash = hash; + passthru = { + inherit + repo + tag + digest + index + upstream + ; + }; + } + '' + export HOME=$TMPDIR + skopeo --insecure-policy --tmpdir "$TMPDIR" inspect --raw docker://${upstream}@${index} > index.json + echo "${lib.removePrefix "sha256:" index} index.json" | sha256sum -c - + jq -e --arg d ${digest} \ + '[.manifests[] | select(.digest == $d and .platform.os == "linux" and .platform.architecture == "amd64")] | length == 1' \ + index.json + skopeo --insecure-policy --tmpdir "$TMPDIR" copy --preserve-digests \ + docker://${upstream}@${digest} oci:$out:${tag} + jq -e --arg d ${digest} '.manifests[0].digest == $d' $out/index.json + echo ${digest} > $out/digest + ''; + + # upstream index digests: manifests/ate-install at d277088b (MEASURED grep). + # linux/amd64 children: `skopeo inspect --raw` of each index (MEASURED + # 2026-09-23); the same values are in patches/0002. + thirdParty = lib.mapAttrs (name: a: thirdPartyImage (a // { inherit name; })) { + envoy = { + upstream = "docker.io/envoyproxy/envoy"; + repo = "envoyproxy/envoy"; + tag = "v1.39-latest"; + index = "sha256:57e14a549d7bd43c8d3f6d03e8cfa653e037d4b38e133acd9b54f38c524401b4"; + digest = "sha256:be87c8b52663c1164a5bdf3c5419017a269cb3d8c74be1ec93638a71f1ffbd4b"; + hash = "sha256-8eny55hV9lIrOBFpRt9Zw563sJHEf5IsJsAgvyYrSNs="; + }; + postgres = { + upstream = "docker.io/library/postgres"; + repo = "library/postgres"; + tag = "18-alpine"; + index = "sha256:9a8afca54e7861fd90fab5fdf4c42477a6b1cb7d293595148e674e0a3181de15"; + digest = "sha256:b6a16ed0eb96e2c362811f7eeb951eac8b459e7b40be4149ea5444aa7c65569b"; + hash = "sha256-s0f5LZJr0644wB7RDApe03GXwwNIvvUZD3JXta5qcQQ="; + }; + rustfs = { + upstream = "docker.io/rustfs/rustfs"; + repo = "rustfs/rustfs"; + tag = "1.0.0-beta.3"; + index = "sha256:378642b05b7dcb4849fb77ebe6aca4ced1c3f66e7e504247df95a5c9018d3358"; + digest = "sha256:d441111efe3af5bbd6e29eba7cd124f96bb7d49a0612390907f9216d5f970382"; + hash = "sha256-TKKJLC0H9huZDQQNgx8s9xGvpre2bSa2GGArAFfv53s="; + }; + aws-cli = { + upstream = "docker.io/amazon/aws-cli"; + repo = "amazon/aws-cli"; + tag = "2.17.0"; + index = "sha256:643507c10ada7964ca6157b3d799f030b90577643da9955d319a77399ed80d73"; + digest = "sha256:7b7edf789765c22d75e61ad6f307c06950e15357b79ea1749104641ce3a11fec"; + hash = "sha256-kZF9g+VfTk5gmtNsZhJNBaX9KceOvDl2s/eWD/ucY00="; + }; + otel-collector = { + upstream = "docker.io/otel/opentelemetry-collector-contrib"; + repo = "otel/opentelemetry-collector-contrib"; + tag = "0.157.0"; + index = "sha256:f2f01157055a9b2aab9df7118e1f1c9abf345e99b23bc7a2bc791db374a7d0f6"; + digest = "sha256:4eb842091c796156d4d3c994eb22ba793590f5723719dbf6b8436cb4dfc17f48"; + hash = "sha256-/btIX9XM2O9wkPwAiN6mLJw8JZ2qoB1yzChWcYiEaGY="; + }; + jaeger = { + upstream = "docker.io/jaegertracing/all-in-one"; + repo = "jaegertracing/all-in-one"; + tag = "1.55"; + index = "sha256:f6b5d09073f14f76873d300f565a6691d815e81bea8e07e1dc3ff67e0596dd4e"; + digest = "sha256:d5bbf80eb37e3a0d1b1644f17d1c3a7b88abd74177ec06198d4a289b58b41798"; + hash = "sha256-Ag47jLKCMmMczbW0muNZuaOYkMtEHxVYEt9+BkgYWQ0="; + }; + prometheus = { + upstream = "docker.io/prom/prometheus"; + repo = "prom/prometheus"; + tag = "v3.5.3"; + index = "sha256:ddc2493835a1509976d5e4e0c94199c4f843ce1f42dd6bcfc8231ba734a93ff7"; + digest = "sha256:442634af681c5988a3ccbd4c6e6ab57e077dc89eead4a31e2abf410501874a92"; + hash = "sha256-yf0l7RsqNY+X6muqRkaK+PlfuUP8w2+2UMaM8I1oUxs="; + }; + # Named by the patched SandboxConfig as localhost:5000/pause:3.10.2@...; + # atelet rewrites localhost:5000 to kind-registry:5000, so repo `pause`. + pause = { + upstream = "registry.k8s.io/pause"; + repo = "pause"; + tag = "3.10.2"; + index = "sha256:f548e0e8e3dc1896ca956272154dde3314e8cc4fde0a57577ee9fa1c63f5baf4"; + digest = "sha256:412c4a7219cb8a299a37337f3d87810c5340095322e15594a1637785adad0f17"; + hash = "sha256-wQV0nZBm2siOhrsElV7IS1dyk3ppsRzC0TtugyGGxA0="; + }; + }; + + # SandboxConfig gvisor-default's amd64 asset (sandboxconfig-gvisor.yaml:34-35 + # at d277088b). atelet tries anonymous GCS, then its S3 client with the same + # bucket and key against the in-cluster RustFS; bootstrap step 40 puts this + # file there, so a node without internet still gets runsc. + gvisor = + let + bucket = "gvisor"; + key = "releases/nightly/2026-09-02/x86_64/gvisor.tar.zstd"; + sha256 = "d547d81401461fd1c679c5c4fa0a6c2b8ef7dc3c22ce23c9e25dcc4c69cfd06f"; + in + fetchurl { + name = "gvisor-nightly-2026-09-02-x86_64.tar.zstd"; + url = "https://storage.googleapis.com/${bucket}/${key}"; + inherit sha256; + passthru = { + inherit bucket key sha256; + }; + }; + + # The shape myAxFleet.registrySeed takes: name -> { oci, repo, tag }. + seed = + lib.mapAttrs' ( + n: v: + lib.nameValuePair "substrate-${n}" { + oci = v; + inherit (v) repo tag; + } + ) components + // lib.mapAttrs (_: v: { + oci = v; + inherit (v) repo tag; + }) thirdParty; +in +linkFarm "substrate-images-${version}" ( + lib.mapAttrsToList (n: v: { + name = "components/${n}"; + path = v; + }) components + ++ lib.mapAttrsToList (n: v: { + name = "third-party/${n}"; + path = v; + }) thirdParty + ++ [ + { + name = "gvisor/gvisor.tar.zstd"; + path = gvisor; + } + ] +) +// { + inherit + components + thirdParty + gvisor + seed + version + ; +} diff --git a/pkgs/substrate/patches/0001-sandboxconfig-pause-localhost.patch b/pkgs/substrate/patches/0001-sandboxconfig-pause-localhost.patch new file mode 100644 index 000000000..94eab02b6 --- /dev/null +++ b/pkgs/substrate/patches/0001-sandboxconfig-pause-localhost.patch @@ -0,0 +1,15 @@ +--- a/manifests/ate-install/sandboxconfig-gvisor.yaml ++++ b/manifests/ate-install/sandboxconfig-gvisor.yaml +@@ -27,7 +27,11 @@ + sandboxClass: gvisor + # The root sandbox container's image. On GCP, prefer the in-project mirror + # gcr.io/gke-release/pause@sha256:bcbd57ba5653580ec647b16d8163cdd1112df3609129b01f912a8032e48265da. +- pauseImage: "registry.k8s.io/pause:3.10.2@sha256:f548e0e8e3dc1896ca956272154dde3314e8cc4fde0a57577ee9fa1c63f5baf4" ++ # mecattaf fleet: the same upstream image, seeded digest-preserved into the ++ # NAS registry as its linux/amd64 manifest (a child of the upstream index ++ # f548e0e8...). atelet pulls it itself and rewrites localhost:5000 through ++ # --localhost-registry-replacement, so no node needs registry.k8s.io. ++ pauseImage: "localhost:5000/pause:3.10.2@sha256:412c4a7219cb8a299a37337f3d87810c5340095322e15594a1637785adad0f17" + assets: + amd64: + gvisor: diff --git a/pkgs/substrate/patches/0002-images-linux-amd64-digests.patch b/pkgs/substrate/patches/0002-images-linux-amd64-digests.patch new file mode 100644 index 000000000..ec5c4ea82 --- /dev/null +++ b/pkgs/substrate/patches/0002-images-linux-amd64-digests.patch @@ -0,0 +1,122 @@ +Pin every third-party image the kind install pulls to its linux/amd64 manifest. + +Upstream pins each image by its multi-arch index digest. Seeding an index +digest-preserved means copying every platform it lists (postgres:18-alpine +lists 16 manifests), several GB that would sit in the NAS's /nix/store on its +57 GB eMMC root. Each digest below is the linux/amd64 child listed in the +upstream-pinned index (MEASURED 2026-09-23 with `skopeo inspect --raw`), so +the content pin is the same Merkle chain, one level down. The fleet is +x86_64-only. pkgs/substrate/images.nix fetches each child by this digest and +checks at fetch time that the upstream index lists it for linux/amd64. + +diff -ru a/manifests/ate-install/atenet-egress-with-sdsmint.yaml b/manifests/ate-install/atenet-egress-with-sdsmint.yaml +--- a/manifests/ate-install/atenet-egress-with-sdsmint.yaml ++++ b/manifests/ate-install/atenet-egress-with-sdsmint.yaml +@@ -1111,7 +1111,7 @@ + readOnly: true + containers: + - name: envoy +- image: envoyproxy/envoy:v1.39-latest@sha256:57e14a549d7bd43c8d3f6d03e8cfa653e037d4b38e133acd9b54f38c524401b4 ++ image: envoyproxy/envoy:v1.39-latest@sha256:be87c8b52663c1164a5bdf3c5419017a269cb3d8c74be1ec93638a71f1ffbd4b + securityContext: + allowPrivilegeEscalation: false + capabilities: +diff -ru a/manifests/ate-install/atenet-egress.yaml b/manifests/ate-install/atenet-egress.yaml +--- a/manifests/ate-install/atenet-egress.yaml ++++ b/manifests/ate-install/atenet-egress.yaml +@@ -601,7 +601,7 @@ + terminationGracePeriodSeconds: 60 + containers: + - name: envoy +- image: envoyproxy/envoy:v1.39-latest@sha256:57e14a549d7bd43c8d3f6d03e8cfa653e037d4b38e133acd9b54f38c524401b4 ++ image: envoyproxy/envoy:v1.39-latest@sha256:be87c8b52663c1164a5bdf3c5419017a269cb3d8c74be1ec93638a71f1ffbd4b + securityContext: + allowPrivilegeEscalation: false + capabilities: +diff -ru a/manifests/ate-install/atenet-router.yaml b/manifests/ate-install/atenet-router.yaml +--- a/manifests/ate-install/atenet-router.yaml ++++ b/manifests/ate-install/atenet-router.yaml +@@ -243,7 +243,7 @@ + - name: "drain-signal" + mountPath: "/var/run/atenet" + - name: envoy +- image: envoyproxy/envoy:v1.39-latest@sha256:57e14a549d7bd43c8d3f6d03e8cfa653e037d4b38e133acd9b54f38c524401b4 ++ image: envoyproxy/envoy:v1.39-latest@sha256:be87c8b52663c1164a5bdf3c5419017a269cb3d8c74be1ec93638a71f1ffbd4b + command: + - "/usr/local/bin/envoy" + - "-c" +diff -ru a/manifests/ate-install/kind/otel-collector.yaml b/manifests/ate-install/kind/otel-collector.yaml +--- a/manifests/ate-install/kind/otel-collector.yaml ++++ b/manifests/ate-install/kind/otel-collector.yaml +@@ -177,7 +177,7 @@ + spec: + containers: + - name: otel-collector +- image: otel/opentelemetry-collector-contrib:0.157.0@sha256:f2f01157055a9b2aab9df7118e1f1c9abf345e99b23bc7a2bc791db374a7d0f6 ++ image: otel/opentelemetry-collector-contrib:0.157.0@sha256:4eb842091c796156d4d3c994eb22ba793590f5723719dbf6b8436cb4dfc17f48 + args: + - --config=/conf/otel-collector-config.yaml + volumeMounts: +@@ -242,7 +242,7 @@ + spec: + containers: + - name: jaeger +- image: jaegertracing/all-in-one:1.55@sha256:f6b5d09073f14f76873d300f565a6691d815e81bea8e07e1dc3ff67e0596dd4e ++ image: jaegertracing/all-in-one:1.55@sha256:d5bbf80eb37e3a0d1b1644f17d1c3a7b88abd74177ec06198d4a289b58b41798 + ports: + - name: otlp-grpc + containerPort: 4317 +diff -ru a/manifests/ate-install/kind/prometheus.yaml b/manifests/ate-install/kind/prometheus.yaml +--- a/manifests/ate-install/kind/prometheus.yaml ++++ b/manifests/ate-install/kind/prometheus.yaml +@@ -172,7 +172,7 @@ + type: RuntimeDefault + containers: + - name: prometheus +- image: prom/prometheus:v3.5.3@sha256:ddc2493835a1509976d5e4e0c94199c4f843ce1f42dd6bcfc8231ba734a93ff7 ++ image: prom/prometheus:v3.5.3@sha256:442634af681c5988a3ccbd4c6e6ab57e077dc89eead4a31e2abf410501874a92 + args: + - --config.file=/etc/prometheus/prometheus.yml + - --storage.tsdb.path=/prometheus +diff -ru a/manifests/ate-install/kind/rustfs.yaml b/manifests/ate-install/kind/rustfs.yaml +--- a/manifests/ate-install/kind/rustfs.yaml ++++ b/manifests/ate-install/kind/rustfs.yaml +@@ -62,7 +62,7 @@ + fsGroup: 10001 + containers: + - name: rustfs +- image: rustfs/rustfs:1.0.0-beta.3@sha256:378642b05b7dcb4849fb77ebe6aca4ced1c3f66e7e504247df95a5c9018d3358 ++ image: rustfs/rustfs:1.0.0-beta.3@sha256:d441111efe3af5bbd6e29eba7cd124f96bb7d49a0612390907f9216d5f970382 + imagePullPolicy: IfNotPresent + ports: + - containerPort: 9000 +@@ -102,7 +102,7 @@ + restartPolicy: OnFailure + containers: + - name: create-bucket +- image: amazon/aws-cli:2.17.0@sha256:643507c10ada7964ca6157b3d799f030b90577643da9955d319a77399ed80d73 ++ image: amazon/aws-cli:2.17.0@sha256:7b7edf789765c22d75e61ad6f307c06950e15357b79ea1749104641ce3a11fec + env: + - name: AWS_ACCESS_KEY_ID + value: rustfsadmin +diff -ru a/manifests/ate-install/postgres/postgres.yaml b/manifests/ate-install/postgres/postgres.yaml +--- a/manifests/ate-install/postgres/postgres.yaml ++++ b/manifests/ate-install/postgres/postgres.yaml +@@ -120,7 +120,7 @@ + initContainers: + - name: tls-reloader + restartPolicy: Always +- image: postgres:18-alpine@sha256:9a8afca54e7861fd90fab5fdf4c42477a6b1cb7d293595148e674e0a3181de15 ++ image: postgres:18-alpine@sha256:b6a16ed0eb96e2c362811f7eeb951eac8b459e7b40be4149ea5444aa7c65569b + securityContext: + runAsUser: 70 + command: +@@ -143,7 +143,7 @@ + memory: 32Mi + containers: + - name: postgres +- image: postgres:18-alpine@sha256:9a8afca54e7861fd90fab5fdf4c42477a6b1cb7d293595148e674e0a3181de15 ++ image: postgres:18-alpine@sha256:b6a16ed0eb96e2c362811f7eeb951eac8b459e7b40be4149ea5444aa7c65569b + lifecycle: + postStart: + exec: diff --git a/pkgs/substrate/patches/0003-kind-rustfs-credential-secret.patch b/pkgs/substrate/patches/0003-kind-rustfs-credential-secret.patch new file mode 100644 index 000000000..8ebef8952 --- /dev/null +++ b/pkgs/substrate/patches/0003-kind-rustfs-credential-secret.patch @@ -0,0 +1,85 @@ +--- a/manifests/ate-install/kind/atelet/kustomization.yaml ++++ b/manifests/ate-install/kind/atelet/kustomization.yaml +@@ -52,7 +52,15 @@ + - name: AWS_S3_USE_PATH_STYLE + value: "true" + # TODO: use a secret / identity management ++ # mecattaf fleet: the kind default credential is replaced by a ++ # per-cluster one the ax-fleet bootstrap generates on the NAS. + - name: AWS_ACCESS_KEY_ID +- value: rustfsadmin ++ valueFrom: ++ secretKeyRef: ++ name: ax-fleet-rustfs ++ key: access-key + - name: AWS_SECRET_ACCESS_KEY +- value: rustfsadmin ++ valueFrom: ++ secretKeyRef: ++ name: ax-fleet-rustfs ++ key: secret-key +--- a/manifests/ate-install/kind/kustomization.yaml ++++ b/manifests/ate-install/kind/kustomization.yaml +@@ -66,8 +66,16 @@ + - name: AWS_S3_USE_PATH_STYLE + value: "true" + # TODO: use a secret / identity management ++ # mecattaf fleet: the kind default credential is replaced by a ++ # per-cluster one the ax-fleet bootstrap generates on the NAS. + - name: AWS_ACCESS_KEY_ID +- value: rustfsadmin ++ valueFrom: ++ secretKeyRef: ++ name: ax-fleet-rustfs ++ key: access-key + - name: AWS_SECRET_ACCESS_KEY +- value: rustfsadmin ++ valueFrom: ++ secretKeyRef: ++ name: ax-fleet-rustfs ++ key: secret-key + +--- a/manifests/ate-install/kind/rustfs.yaml ++++ b/manifests/ate-install/kind/rustfs.yaml +@@ -78,10 +78,18 @@ + value: "true" + - name: RUSTFS_VOLUMES + value: "/data" ++ # mecattaf fleet: the kind default credential is replaced by a ++ # per-cluster one the ax-fleet bootstrap generates on the NAS. + - name: RUSTFS_ACCESS_KEY +- value: "rustfsadmin" ++ valueFrom: ++ secretKeyRef: ++ name: ax-fleet-rustfs ++ key: access-key + - name: RUSTFS_SECRET_KEY +- value: "rustfsadmin" ++ valueFrom: ++ secretKeyRef: ++ name: ax-fleet-rustfs ++ key: secret-key + volumeMounts: + - name: data + mountPath: /data +@@ -104,10 +112,18 @@ + - name: create-bucket + image: amazon/aws-cli:2.17.0@sha256:643507c10ada7964ca6157b3d799f030b90577643da9955d319a77399ed80d73 + env: ++ # mecattaf fleet: the kind default credential is replaced by a ++ # per-cluster one the ax-fleet bootstrap generates on the NAS. + - name: AWS_ACCESS_KEY_ID +- value: rustfsadmin ++ valueFrom: ++ secretKeyRef: ++ name: ax-fleet-rustfs ++ key: access-key + - name: AWS_SECRET_ACCESS_KEY +- value: rustfsadmin ++ valueFrom: ++ secretKeyRef: ++ name: ax-fleet-rustfs ++ key: secret-key + - name: AWS_REGION + value: us-east-1 + - name: AWS_ENDPOINT_URL diff --git a/pkgs/substrate/patches/0004-kind-otel-memory-bounds.patch b/pkgs/substrate/patches/0004-kind-otel-memory-bounds.patch new file mode 100644 index 000000000..c097991d9 --- /dev/null +++ b/pkgs/substrate/patches/0004-kind-otel-memory-bounds.patch @@ -0,0 +1,77 @@ +--- a/manifests/ate-install/kind/otel-collector.yaml 2026-09-23 13:06:34.261419536 +0200 ++++ b/manifests/ate-install/kind/otel-collector.yaml 2026-09-23 13:06:45.024736968 +0200 +@@ -32,6 +32,13 @@ + http: + endpoint: 0.0.0.0:4318 + processors: ++ # mecattaf fleet: the collector runs on the NAS (22 GB RAM, shared with ++ # Immich, Paperless and NFS). memory_limiter refuses data before the ++ # container limit below is reached, instead of being OOM-killed. ++ memory_limiter: ++ check_interval: 1s ++ limit_mib: 400 ++ spike_limit_mib: 100 + batch: + + # The count connector below sends deltas, and the Prometheus exporter +@@ -95,17 +102,19 @@ + insecure: true + prometheus: + endpoint: 0.0.0.0:8889 ++ # mecattaf fleet: basic, not detailed; detailed writes every span and ++ # log record to the pod log. + debug: +- verbosity: detailed ++ verbosity: basic + service: + pipelines: + traces: + receivers: [otlp] +- processors: [batch] ++ processors: [memory_limiter, batch] + exporters: [otlp/jaeger, debug] + metrics: + receivers: [otlp] +- processors: [batch] ++ processors: [memory_limiter, batch] + exporters: [prometheus, debug] + + # Every ate event, plus any workload that sends OTLP logs. debug is the +@@ -115,7 +124,7 @@ + # record reports a stale state with no sign that anything is missing. + logs: + receivers: [otlp] +- processors: [batch] ++ processors: [memory_limiter, batch] + exporters: [debug] + + # The count pipelines read the same receiver as the pipelines above, +@@ -180,6 +189,12 @@ + image: otel/opentelemetry-collector-contrib:0.157.0@sha256:4eb842091c796156d4d3c994eb22ba793590f5723719dbf6b8436cb4dfc17f48 + args: + - --config=/conf/otel-collector-config.yaml ++ # mecattaf fleet: bounded (see memory_limiter above). ++ resources: ++ requests: ++ memory: 128Mi ++ limits: ++ memory: 512Mi + volumeMounts: + - name: config + mountPath: /conf +@@ -243,6 +258,15 @@ + containers: + - name: jaeger + image: jaegertracing/all-in-one:1.55@sha256:d5bbf80eb37e3a0d1b1644f17d1c3a7b88abd74177ec06198d4a289b58b41798 ++ # mecattaf fleet: in-memory storage keeps every trace with no bound ++ # by default. Cap the trace count and the container. ++ args: ++ - --memory.max-traces=10000 ++ resources: ++ requests: ++ memory: 128Mi ++ limits: ++ memory: 512Mi + ports: + - name: otlp-grpc + containerPort: 4317 diff --git a/secrets.nix b/secrets.nix index 5d94f6b0b..974c837ac 100644 --- a/secrets.nix +++ b/secrets.nix @@ -92,6 +92,20 @@ in # NAS private media HTTPS: zone-limited DNS-01 token, no broad Wrangler OAuth # authority. Ciphertext is provisioned before enabling personal-https.nix. "secrets/nas-cloudflare-dns.age".publicKeys = editors ++ nasOnly; + # The k3s credentials (ax on the fleet, modules/ax-fleet/k3s.nix), split in + # fix round 2 (2026-09-23). k3s-token.age is the SERVER token: read by the + # NAS only; with it comes the k3s:server role (/v1-k3s/token, /cacerts, + # /encrypt/config). k3s-agent-token.age is the agent join credential: the + # NAS passes it as --agent-token-file, the coordinator joins with it. Both + # were minted fresh from /dev/urandom with `age -R` over these lists, never + # displayed; the 643a4196 ciphertext (also decryptable by the worker and the + # client) was replaced, not re-encrypted, before any cluster used it. + # Rotate with: nix develop -c agenix -e secrets/.age + "secrets/k3s-token.age".publicKeys = editors ++ nasOnly; + "secrets/k3s-agent-token.age".publicKeys = editors ++ coordinatorOnly ++ nasOnly; + # substrate-link bearer (hosts/nas/substrate-link.nix): one token per link identity, minted by Tom as a + # Worker secret on the floor and sealed here. NAS only: the link runs on the NAS host. + "secrets/floor-link-token.age".publicKeys = editors ++ nasOnly; # --- wifi PSK tier: the coordinator, whose Freebox uplink # (wlp192s0) is now declarative too (migrated from an imperative profile on # flash night — refs #37). Rekey after this change: nix develop -c agenix -r diff --git a/secrets/k3s-agent-token.age b/secrets/k3s-agent-token.age new file mode 100644 index 000000000..091edc6bd --- /dev/null +++ b/secrets/k3s-agent-token.age @@ -0,0 +1,9 @@ +age-encryption.org/v1 +-> X25519 RYDnXPmLP7LRAQUOkbt8OfXKlRrAkxw3ifwxFPxO1U0 +lPdZbsFZ0Q+tIikHyjq4zuiSRO6D/BmWQw4ilwGcrG8 +-> ssh-ed25519 b4VnIg 1Ew3degzdeyANd3OUkRJ4zkCFgTMpeZUaPcvqaNiQXs +b4tGOKtANgMigmVK9kDTjxVRGFaooM0/X6BfFMAai5w +-> ssh-ed25519 sKUETw Yir5k5hDZ2wzoziKlhicGETCYMpKURtD6bv+0A4ykjI +Wb8h0aOXwq3VpycN8k9WjRitw8ooiz8hHirbcb1un20 +--- W0F7q+18w3EXzMQBF7OT5o5kLcHo30Pkarp3RMTJxPQ +) x:{Tzeh~5lLʚvd7ox:=̲/r9m? \ No newline at end of file diff --git a/secrets/k3s-token.age b/secrets/k3s-token.age new file mode 100644 index 000000000..35d03103a --- /dev/null +++ b/secrets/k3s-token.age @@ -0,0 +1,7 @@ +age-encryption.org/v1 +-> X25519 dtiPpi1BLiotWQN+ZoSMCY16V39onbZjA2tXCedlGhw +O8vycEoqUfgd4RjUn2TC2CBweycUR9YHaus2/b3IuQg +-> ssh-ed25519 sKUETw zN+hojttTBmbuORJJbvGBJo+feofYkS9R8s7eHnllWw ++wLRUCZuZv7n1sDj4Djnuu2mTDi2sUfNaPTQ0MuVeD4 +--- hvbZ54nuJjxpSPeHNTegaKSMV92TALVThkKhTcK9pbw +4|HxmSWfHme;uҭp>aHM`l~큿  0ϝ{eݮH[6$%ίF"­WH2$: \ No newline at end of file diff --git a/tests/ax-fleet-boot/default.nix b/tests/ax-fleet-boot/default.nix new file mode 100644 index 000000000..27600d673 --- /dev/null +++ b/tests/ax-fleet-boot/default.nix @@ -0,0 +1,112 @@ +{ + pkgs, + lib, + inputs, +}: +# checks.x86_64-linux.ax-fleet-boot (DESIGN.md 12.2): a NAS node with ax ON +# FROM BOOT. The 4-VM test proves the live-switch ordering; this proves the +# boot ordering: the bind mounts come up before k3s, the tmpfiles links land +# on the /mnt/fast side, the registry seed and the bootstrap succeed, and the +# node is Ready and untainted. The substrate track adds its Available checks +# here through the same bootstrap. +# +# Built from nixpkgs-STABLE, as the real NAS is (fix round 2): the round-2 +# review MEASURED that every other ax-fleet VM runs NixOS 26.11 and systemd +# 261 while the NAS runs 26.05 and systemd 260, so the NAS's own module set +# (k3s, docker-registry, nftables, systemd) had never run. The NAS boots with +# the systemd initrd, as the real one does (boot.initrd.systemd.enable is +# true on it, MEASURED nix eval), because that is where the boot-time +# activation runs. +let + stablePkgs = inputs.nixpkgs-stable.legacyPackages.x86_64-linux; + nodes = import ../ax-fleet/nodes.nix { + pkgs = stablePkgs; + inherit lib inputs; + }; +in +stablePkgs.testers.runNixOSTest { + name = "ax-fleet-boot"; + node.specialArgs = { inherit inputs; }; + nodes.nas = { + # The pre-ax NAS plus its fleet module, ON from boot (fix round 4: the + # ax-fleet test's base no longer imports the module). + imports = [ + nodes.nasBase + nodes.nasFleet + ]; + myAxFleet.enable = true; + boot.initrd.systemd.enable = true; + # The NAS's kernel (hosts/nas/kernel.nix: freshPkgs.linuxPackages_7_2); + # ax-fleet-topology asserts it is the same derivation. + boot.kernelPackages = + (import inputs.nixpkgs-fresh { + system = "x86_64-linux"; + config.allowUnfree = true; + }).linuxPackages_7_2; + myAxFleet.kubelet.systemReserved = "cpu=1,memory=1Gi"; + # The real NAS gets 10.42.0.1 from NetworkManager seconds AFTER + # network(-online).target (MEASURED 2026-09-23: target at 11.78 s, address + # at 20.15 s). Reproduce that: no static address, then a unit nothing + # waits for adds it late. The delay is a test parameter longer than the + # units' 30 s address wait, so both the wait and Restart= are exercised. + networking.interfaces.eth1.ipv4.addresses = lib.mkOverride 10 [ ]; + systemd.services.late-lan-addr = { + wantedBy = [ "multi-user.target" ]; + after = [ "network.target" ]; + path = [ + pkgs.iproute2 + pkgs.coreutils + ]; + script = '' + sleep 40 + ip link set eth1 up + ip addr add 10.42.0.1/24 dev eth1 + date +%s > /run/late-lan-addr.done + ''; + }; + }; + globalTimeout = 3600; + testScript = '' + nas.start() + nas.wait_for_file("/run/late-lan-addr.done", timeout=600) + nas.wait_for_unit("k3s.service", timeout=900) + nas.wait_for_unit("ax-fleet-bootstrap.service", timeout=1800) + for p in ("/var/lib/rancher/k3s", "/var/lib/kubelet", "/var/log/pods"): + src = nas.succeed(f"findmnt -n -o SOURCE -T {p}").strip() + assert src.startswith("/dev/vdb"), f"{p} is on {src}, not /mnt/fast" + nas.succeed("ls /mnt/fast/k3s/rancher/k3s/agent/images/ | grep -q airgap") + nas.succeed("systemctl show ax-fleet-registry-seed.service -p Result --value | grep -x success") + # The registry came up on the late address and stays up. + nas.succeed("systemctl is-active docker-registry.service") + nas.succeed("curl -sf --max-time 10 http://10.42.0.1:5000/v2/") + addr_at = int(nas.succeed("cat /run/late-lan-addr.done").strip()) + reg_at = int(nas.succeed( + "date -d \"$(systemctl show docker-registry.service -p ActiveEnterTimestamp --value)\" +%s" + ).strip()) + assert reg_at >= addr_at, f"registry active at {reg_at}, before the address at {addr_at}" + print("AXFLEET-BOOT registry NRestarts=" + nas.succeed("systemctl show docker-registry.service -p NRestarts --value").strip()) + nas.succeed("systemctl show ax-fleet-bootstrap.service -p Result --value | grep -x success") + nas.wait_until_succeeds( + "k3s kubectl get node nas -o jsonpath='{.status.conditions[?(@.type==\"Ready\")].status}' | grep -x True", + timeout=600, + ) + # k3s's own transient taints (uninitialized, not-ready) clear on their own; + # the NAS carries no taint of ours. + nas.wait_until_succeeds("test -z \"$(k3s kubectl get node nas -o jsonpath='{.spec.taints}')\"", timeout=300) + nas.succeed("k3s kubectl -n kube-system wait --for=condition=Available deploy/coredns deploy/local-path-provisioner --timeout=600s") + nas.succeed("test -s /etc/ax-fleet/admin.kubeconfig") + # The snapshot was taken at first activation, before k3s ever ran, and at + # boot it carries the host's declared values, not the kernel defaults the + # boot activation sees (fix round 2: it had recorded ip_forward = 0 on + # the router, which the teardown would have applied). + print("AXFLEET-BOOT snapshot:\n" + nas.succeed("cat /var/lib/ax-fleet/sysctl-before.conf")) + print("AXFLEET-BOOT release=" + nas.succeed("nixos-version").strip() + " systemd=" + nas.succeed("systemctl --version | head -1").strip()) + nas.succeed("test \"$(sysctl -n net.ipv4.ip_forward)\" = 1") + nas.succeed("grep -x 'net.ipv4.ip_forward = 1' /var/lib/ax-fleet/sysctl-before.conf") + nas.succeed("grep -q '^kernel.panic = ' /var/lib/ax-fleet/sysctl-before.conf") + # kubelet ran; its panic values are not what the host is left with. + snap_panic = nas.succeed("sed -n 's/^kernel.panic = //p' /var/lib/ax-fleet/sysctl-before.conf").strip() + nas.wait_until_succeeds(f"test \"$(sysctl -n kernel.panic)\" = '{snap_panic}'", timeout=300) + nas.fail("findmnt -n -T /var/lib/rancher/k3s -o SOURCE | grep -q vda") + ''; +} diff --git a/tests/ax-fleet-topology/default.nix b/tests/ax-fleet-topology/default.nix new file mode 100644 index 000000000..f64ad81d0 --- /dev/null +++ b/tests/ax-fleet-topology/default.nix @@ -0,0 +1,289 @@ +{ + pkgs, + lib, + self, +}: +# checks.x86_64-linux.ax-fleet-topology (DESIGN.md 12.3): evaluation-only +# assertions over the REAL host configurations. Every assert is eval-time and +# sits in front of the runCommand, so each runs under --no-build. +# +# Not asserted here, on purpose: "no 8731 on the coordinator's wlp192s0". That +# is #461's change; nas-topology already asserts exactly it and fails on +# origin/main until #461 is merged (inherited, not introduced). Duplicating it +# here would make this check red for a reason outside this PR. +let + hostCfg = host: self.nixosConfigurations.${host}.config; + offCfg = + host: + (self.nixosConfigurations.${host}.extendModules { + modules = [ { myAxFleet.enable = lib.mkForce false; } ]; + }).config; + + flagsOf = + cfg: + let + raw = cfg.systemd.services.k3s.serviceConfig.ExecStart; + s = if builtins.isList raw then lib.concatStringsSep " " raw else raw; + in + lib.filter (w: w != "" && w != "\\") (lib.splitString " " (lib.replaceStrings [ "\n" ] [ " " ] s)); + + has = cfg: flag: builtins.elem flag (flagsOf cfg); + hasPrefix = cfg: p: builtins.any (lib.hasPrefix p) (flagsOf cfg); + + nas = hostCfg "nas"; + coord = hostCfg "coordinator"; + worker = hostCfg "worker"; + client = hostCfg "client"; + ax = cfg: cfg.myAxFleet; + + axLines = text: lib.filter (l: lib.hasInfix "ax-fleet:" l) (lib.splitString "\n" text); + + # ── the kill switch: with enable = mkForce false nothing from this PR renders ── + killed = + host: + let + c = offCfg host; + units = lib.attrNames c.systemd.services ++ lib.attrNames c.systemd.sockets; + in + !c.services.k3s.enable + && !c.services.dockerRegistry.enable + && !(builtins.any (lib.hasPrefix "ax-fleet") units) + && !(builtins.any (lib.hasPrefix "ax-server-proxy") units) + && !(builtins.any (m: m.where == "/var/lib/rancher") c.systemd.mounts) + && !(c.environment.etc ? "NetworkManager/conf.d/90-ax-fleet.conf") + && !(c.environment.etc ? "rancher/k3s/registries.yaml") + && !(c.system.activationScripts ? ax-fleet-sysctl-snapshot) + && !(lib.hasInfix "ax-fleet:" (c.networking.firewall.extraInputRules or "")) + && !(c.boot.kernel.sysctl ? "net.ipv4.conf.veth*.proxy_arp"); + + # ...except the guards, which stay for the role whatever `enable` says (fix + # round 3): the kill switch leaves pods running until the teardown. + guardsKept = + host: + let + c = offCfg host; + in + if host == "coordinator" then + lib.hasInfix "ax-fleet-guard" c.networking.firewall.extraCommands + && lib.hasInfix "ax-fleet-pod-input" c.networking.firewall.extraCommands + && lib.hasInfix "ax-fleet-api" c.networking.firewall.extraCommands + else if host == "nas" then + lib.hasInfix "hook forward" c.networking.nftables.tables.ax-fleet-guard.content + else + !(lib.hasInfix "ax-fleet-guard" c.networking.firewall.extraCommands) + && !(c.networking.nftables.tables ? ax-fleet-guard); + + # ── parity with the VM test: flags and firewall text, interfaces substituted ── + testNodes = self.checks.x86_64-linux.ax-fleet.nodes; + testOn = name: testNodes.${name}.specialisation.ax-on.configuration; + # Flags that legitimately differ: the token source and the VM's kubelet + # reservations (a VM has 8 GiB, the desk 128). + volatile = + f: + f == "--token-file" + || f == "--agent-token-file" + || lib.hasSuffix "/k3s-token" f + || lib.hasSuffix "/k3s-agent-token" f + || lib.hasSuffix "-ax-fleet-vm-token" f + || lib.hasSuffix "-ax-fleet-vm-agent-token" f + || lib.hasInfix "reserved=" f + || lib.hasInfix "eviction-hard=" f; + normFlags = + subst: cfg: + lib.sort (a: b: a < b) ( + map (lib.replaceStrings (lib.attrNames subst) (lib.attrValues subst)) ( + lib.filter (f: !(volatile f)) (flagsOf cfg) + ) + ); + nasSubst = { + "enp1s0" = "eth1"; + }; + coordSubst = { + "wlp192s0" = "eth1"; + "tailscale0" = "eth2"; + "enp191s0" = "eth3"; + }; + guardText = + subst: cfg: + map (lib.replaceStrings (lib.attrNames subst) (lib.attrValues subst)) ( + lib.filter ( + l: + lib.hasInfix "ax-fleet-guard" l + || lib.hasInfix "8472" l + || lib.hasInfix "ax-fleet-pod-input" l + || lib.hasInfix "ax-fleet-api" l + ) (lib.splitString "\n" cfg.networking.firewall.extraCommands) + ); + pkgNames = cfg: map (p: p.pname or p.name or "") cfg.environment.systemPackages; + hasTeardown = cfg: builtins.elem "ax-fleet-teardown" (pkgNames cfg); + + # every image the real NAS seeds is the store path the VM NAS seeds (fix round 2) + seedPaths = cfg: lib.mapAttrs (_: s: s.oci.outPath) cfg.myAxFleet.registrySeed; + nasSeeds = seedPaths nas; + vmSeeds = seedPaths (testOn "nas"); +in +# nas: the control node +assert (ax nas).enable && (ax nas).role == "control"; +assert has nas "--flannel-iface=enp1s0"; +assert has nas "--node-ip=10.42.0.1"; +assert has nas "--flannel-backend=vxlan"; +assert !(hasPrefix nas "--node-taint"); +assert has nas "--node-label=ate.dev/substrate-version=none"; +assert has nas "--node-label=ax.mecattaf.dev/role=control"; +assert has nas "--default-local-storage-path=/mnt/nas/services/ax-fleet/local-path"; +assert !(builtins.any (lib.hasInfix "local-storage") nas.services.k3s.disable); +assert has nas "--kube-apiserver-arg=runtime-config=certificates.k8s.io/v1beta1=true"; +assert has nas "--service-node-port-range=30000-30999"; +assert !(hasPrefix nas "--data-dir"); +assert !(hasPrefix nas "--cluster-init"); +assert nas.services.k3s.containerdConfigTemplate == null; +assert nas.services.k3s.package == (ax nas).k3sPackage; +assert (ax nas).k3sPackage.version == "1.36.2+k3s1"; +# every ax-fleet NAS rule is scoped to a source and to an interface, none to the tailnet +assert builtins.length (axLines nas.networking.firewall.extraInputRules) == 4; +assert builtins.all (l: lib.hasInfix "ip saddr" l && lib.hasInfix "iifname" l) ( + axLines nas.networking.firewall.extraInputRules +); +assert + !(builtins.any (lib.hasInfix "tailscale0") (axLines nas.networking.firewall.extraInputRules)); +assert nas.services.dockerRegistry.listenAddress == "10.42.0.1"; +assert !nas.services.dockerRegistry.openFirewall; +assert lib.hasPrefix "/mnt/nas/" nas.services.dockerRegistry.storagePath; +assert builtins.all (m: lib.hasPrefix "/mnt/fast/" m.what) ( + lib.filter ( + m: m.where == "/var/lib/rancher" || m.where == "/var/lib/kubelet" || m.where == "/var/log/pods" + ) nas.systemd.mounts +); +assert + builtins.length (lib.filter (m: lib.hasPrefix "/mnt/fast/k3s" m.what) nas.systemd.mounts) == 3; +# the shared PostgreSQL is not touched: identical settings with the switch off +assert nas.services.postgresql.settings == (offCfg "nas").services.postgresql.settings; +assert nas.services.postgresql.authentication == (offCfg "nas").services.postgresql.authentication; + +# coordinator: the harness node +assert (ax coord).enable && (ax coord).role == "harness"; +assert has coord "--flannel-iface=wlp192s0"; +assert has coord "--node-ip=10.42.0.2"; +assert has coord "--server"; +assert has coord "https://10.42.0.1:6443"; +assert has coord "--node-taint=${(ax coord).harnessTaint}"; +assert (ax coord).harnessTaint == "ate.dev/sandboxClass=gvisor:NoSchedule"; +assert has coord "--node-label=ate.dev/substrate-version=${(ax coord).substrateVersion}"; +assert (ax coord).substrateVersion == (ax nas).substrateVersion; +assert coord.services.k3s.containerdConfigTemplate == null; +assert coord.services.k3s.package == nas.services.k3s.package; +assert lib.hasInfix "ax-fleet-guard" coord.networking.firewall.extraCommands; +assert lib.hasInfix "-s 10.42.0.1 -p udp --dport 8472" coord.networking.firewall.extraCommands; +assert !(coord.networking.firewall.interfaces ? cni0); +assert coord.environment.etc ? "NetworkManager/conf.d/90-ax-fleet.conf"; +# NO NetworkManager restart trigger: NetworkManager.conf renders byte-identical with the switch off +assert + coord.environment.etc."NetworkManager/NetworkManager.conf".source == (offCfg "coordinator") + .environment.etc."NetworkManager/NetworkManager.conf".source; +assert + coord.networking.networkmanager.unmanaged == (offCfg "coordinator") + .networking.networkmanager.unmanaged; +# proxy ARP only on the pod interfaces, never through all/default (fix round 3) +assert coord.boot.kernel.sysctl."net.ipv4.conf.veth*.proxy_arp" == 1; +assert coord.boot.kernel.sysctl."net.ipv4.conf.cni0.proxy_arp" == 1; +assert coord.boot.kernel.sysctl."net.ipv4.conf.flannel/1.proxy_arp" == 1; +assert builtins.all ( + k: !(coord.boot.kernel.sysctl ? ${k}) || coord.boot.kernel.sysctl.${k} == null +) [ "net.ipv4.conf.all.proxy_arp" "net.ipv4.conf.default.proxy_arp" ]; +# the wired port is a guarded LAN leg with a route metric above the wifi's +assert (ax coord).lan.extraInterfaces == [ "enp191s0" ]; +assert lib.hasInfix "-i cni0 -o enp191s0 -j RETURN" coord.networking.firewall.extraCommands; +assert lib.hasInfix "-i cni0 -d 10.0.0.0/8 -j DROP" coord.networking.firewall.extraCommands; +assert lib.hasInfix "-o cni0 -j DROP" coord.networking.firewall.extraCommands; +assert lib.hasInfix "match-device=interface-name:enp191s0" coord.environment.etc."NetworkManager/conf.d/90-ax-fleet.conf".text; +assert lib.hasInfix "ipv4.route-metric=700" coord.environment.etc."NetworkManager/conf.d/90-ax-fleet.conf".text; +# the ax API: only root, tom and the proxy (fix round 3) +assert lib.hasInfix "--uid-owner tom -j RETURN" coord.networking.firewall.extraCommands; +assert coord.systemd.services.ax-server-proxy.serviceConfig.User == "ax-server-proxy"; +assert (coord.boot.kernel.sysctl."net.ipv6.conf.all.forwarding" or 0) == 0; +assert coord.services.tailscale.useRoutingFeatures == "none"; +assert coord.systemd.sockets.ax-server-proxy.listenStreams == [ "127.0.0.1:8099" ]; +assert coord.environment.sessionVariables.AX_SERVER == "http://127.0.0.1:8099"; +# never the address ax-conwip's default (and the mock stack) points at +assert !(builtins.elem coord.home-manager.users.tom.myAxConwip.serverUrl coord.systemd.sockets.ax-server-proxy.listenStreams); +assert coord.myAxClient.enable; + +# worker: inference, nothing at runtime +assert (ax worker).enable && (ax worker).role == "inference"; +assert !worker.services.k3s.enable; +assert builtins.elem 8731 worker.networking.firewall.interfaces.enp191s0.allowedTCPPorts; +# client: untouched +assert !(client ? myAxFleet); +assert !client.services.k3s.enable; + +# the registry is read-only to the network; the seed writes on loopback only +assert nas.services.dockerRegistry.extraConfig.storage.maintenance.readonly.enabled; +assert !nas.services.dockerRegistry.enableDelete; +# two k3s credentials: the server token stays on the NAS +assert lib.hasSuffix "/k3s-token" nas.services.k3s.tokenFile; +assert lib.hasSuffix "/k3s-agent-token" nas.services.k3s.agentTokenFile; +assert lib.hasSuffix "/k3s-agent-token" coord.services.k3s.tokenFile; +assert !(coord.age.secrets ? k3s-token); +# pods never open a connection to the desk itself; the desk outweighs kubepods +assert lib.hasInfix + "-i cni0 -m conntrack --ctstate NEW -m comment --comment ax-fleet-pod-input -j nixos-fw-refuse" + coord.networking.firewall.extraCommands; +assert coord.systemd.slices.user.sliceConfig.CPUWeight == 10000; +assert coord.systemd.slices.system.sliceConfig.CPUWeight == 10000; +# kube-proxy leaves the host's conntrack table as it is +assert builtins.all + ( + h: + has h "--kube-proxy-arg=conntrack-tcp-timeout-established=0s" + && has h "--kube-proxy-arg=conntrack-max-per-core=0" + ) + [ + nas + coord + ]; +# the VM coordinator runs the desk's kernel +assert + coord.boot.kernelPackages.kernel.outPath == (testOn "coordinator") + .boot.kernelPackages.kernel.outPath; +# the NAS-from-boot VM runs the NAS's release line and kernel +assert + self.checks.x86_64-linux.ax-fleet-boot.nodes.nas.system.nixos.release == nas.system.nixos.release; +assert + self.checks.x86_64-linux.ax-fleet-boot.nodes.nas.boot.kernelPackages.kernel.outPath + == nas.boot.kernelPackages.kernel.outPath; +# image parity with the VM +assert builtins.all (n: vmSeeds ? ${n} && vmSeeds.${n} == nasSeeds.${n}) (lib.attrNames nasSeeds); + +# the kill switch; the teardown stays on the host's PATH with the switch off +assert builtins.all (h: hasTeardown (offCfg h)) [ + "nas" + "coordinator" +]; +assert !(hasTeardown worker); +assert builtins.all killed [ + "nas" + "coordinator" + "worker" +]; +assert builtins.all guardsKept [ + "nas" + "coordinator" + "worker" +]; +# the NAS's pods: Halogen on its port, no other private address (fix round 3) +assert lib.hasInfix "ip daddr 10.42.0.5 tcp dport 8731 return" nas.networking.nftables.tables.ax-fleet-guard.content; +assert + lib.replaceStrings [ "tailscale0" ] [ "eth2" ] nas.networking.nftables.tables.ax-fleet-guard.content + == (testOn "nas").networking.nftables.tables.ax-fleet-guard.content; + +# parity with the VM test +assert normFlags nasSubst nas == normFlags { } (testOn "nas"); +assert normFlags coordSubst coord == normFlags { } (testOn "coordinator"); +assert + map (lib.replaceStrings [ "enp1s0" ] [ "eth1" ]) (axLines nas.networking.firewall.extraInputRules) + == axLines (testOn "nas").networking.firewall.extraInputRules; +assert guardText coordSubst coord == guardText { } (testOn "coordinator"); + +pkgs.runCommand "ax-fleet-topology" { } '' + touch "$out" +'' diff --git a/tests/ax-fleet/default.nix b/tests/ax-fleet/default.nix new file mode 100644 index 000000000..ff7b622af --- /dev/null +++ b/tests/ax-fleet/default.nix @@ -0,0 +1,175 @@ +{ + pkgs, + lib, + inputs, +}: +# checks.x86_64-linux.ax-fleet: the 4-VM proof before any switch (DESIGN.md +# 12.1). The script mirrors the real motion: baseline, switch the NAS, switch +# the coordinator (the worker is not switched), Tasks, resilience, rollback. +# +# The test script is phases/*.py concatenated in name order, after the +# prelude below: 10-cluster and 90-rollback (cluster track), 20-substrate +# (substrate track), 30-nop1 and 32-fleet (ax track). Every subtest a phase runs through +# `step(...)` is named in $out/receipt.json with the values it recorded. +let + nodes = import ./nodes.nix { inherit pkgs lib inputs; }; + teardown = pkgs.callPackage ../../pkgs/ax-fleet-teardown { + k3s = inputs.nixpkgs.legacyPackages.x86_64-linux.k3s_1_36; + }; + phaseDir = ./phases; + phaseFiles = lib.sort (a: b: a < b) ( + lib.filter (n: lib.hasSuffix ".py" n) (lib.attrNames (builtins.readDir phaseDir)) + ); + prelude = '' + import json + import os + import re + import time + from contextlib import contextmanager + + # 90-rollback runs the teardown from the host's PATH, as documented; this + # store path is only compared against it. + TEARDOWN = "${teardown}/bin/ax-fleet-teardown" + PROBE_IMAGE = "ax-fleet-probe:test" + PROBE_TARBALL = "${nodes.probeImage}" + # Test-only task image variant with claude-code (nodes.nix); its OCI layout + # carries the manifest digest the Task names. + CLAUDE_PROBE_OCI = "${nodes.claudeProbeImage}" + LOCAL_PATH_ROOT = "/mnt/nas/services/ax-fleet/local-path" + SYSCTLS = [ + "net.ipv4.ip_forward", + "net.ipv6.conf.all.forwarding", + "kernel.panic", + "kernel.panic_on_oops", + "vm.overcommit_memory", + # kube-proxy's conntrack keys (fix round 2): the house router's NAT + # table must keep the host's timeouts. + "net.netfilter.nf_conntrack_max", + "net.netfilter.nf_conntrack_tcp_timeout_established", + "net.netfilter.nf_conntrack_tcp_timeout_close_wait", + ] + + from typing import Any + + receipt_subtests: list[dict[str, Any]] = [] + receipt_values: dict[str, Any] = {} + receipt: dict[str, Any] = {"test": "ax-fleet", "subtests": receipt_subtests, "values": receipt_values} + + + def save_receipt(): + out = os.environ.get("out", ".") + os.makedirs(out, exist_ok=True) + with open(os.path.join(out, "receipt.json"), "w") as f: + json.dump(receipt, f, indent=2, sort_keys=True) + + + def record(key, value): + receipt_values[key] = value + # Also into the build log: a failed build keeps no $out. + print(f"AXFLEET-RECORD {key} = {json.dumps(value, sort_keys=True)}") + save_receipt() + + + @contextmanager + def step(name): + t0 = time.monotonic() + with subtest(name): + yield + receipt_subtests.append({"name": name, "result": "pass", "seconds": round(time.monotonic() - t0, 1)}) + save_receipt() + + + def kubectl(args): + """kubectl on the NAS, as root, with the k3s admin kubeconfig.""" + return nas.succeed(f"k3s kubectl {args}") + + + def jsonpath(obj, path): + return kubectl(f"get {obj} -o jsonpath='{path}'").strip() + + + def sysctls(machine): + return {k: machine.succeed(f"sysctl -n {k}").strip() for k in SYSCTLS} + + + def node_ready(name): + nas.wait_until_succeeds( + f"k3s kubectl get node {name} -o jsonpath='{{.status.conditions[?(@.type==\"Ready\")].status}}' | grep -x True", + timeout=600, + ) + + + KERNEL_KEYS = ("kernel.panic", "kernel.panic_on_oops", "vm.overcommit_memory") + + + def kernel_keys_back(machine, baseline): + """myAxFleet.kubelet.keepHostKernelTunables: kubelet's values are put back.""" + for k in KERNEL_KEYS: + machine.wait_until_succeeds(f"test \"$(sysctl -n {k})\" = '{baseline[k]}'", timeout=300) + + + def flap_until_unreachable(tag): + """Take the coordinator's LAN leg down until the control plane has + reacted (Ready=Unknown and the unreachable taint), then bring it back. + A test parameter, not an estimate: it waits for the transition.""" + t0 = time.monotonic() + coordinator.succeed("ip link set eth1 down") + try: + nas.wait_until_succeeds( + "k3s kubectl get node coordinator -o jsonpath='{.status.conditions[?(@.type==\"Ready\")].status}' | grep -qx Unknown", + timeout=900, + ) + nas.wait_until_succeeds( + "k3s kubectl get node coordinator -o jsonpath='{.spec.taints[*].key}' | grep -qw node.kubernetes.io/unreachable", + timeout=600, + ) + record(f"flap_{tag}_taints_while_down", nas.succeed("k3s kubectl get node coordinator -o jsonpath='{.spec.taints}'").strip()) + finally: + coordinator.succeed("ip link set eth1 up") + outage = round(time.monotonic() - t0, 1) + record(f"flap_{tag}_outage_seconds", outage) + return outage + + + def user_unit_pid(unit): + return coordinator.succeed( + "runuser -u alice -- env XDG_RUNTIME_DIR=/run/user/$(id -u alice) " + f"systemctl --user show {unit} -p MainPID --value" + ).strip() + + + def nm_invocation(): + return coordinator.succeed("systemctl show NetworkManager -p InvocationID --value").strip() + + + def unit_invocation(machine, unit): + return machine.succeed(f"systemctl show {unit} -p InvocationID --value").strip() + + ''; +in +pkgs.testers.runNixOSTest { + name = "ax-fleet"; + node.specialArgs = { inherit inputs; }; + nodes = { + inherit (nodes) + nas + coordinator + worker + peer + ; + }; + # Long waits are test parameters (k3s start, image import), not estimates. + globalTimeout = 3 * 3600; + testScript = + prelude + + lib.concatMapStrings (f: '' + + # ───────────── phases/${f} ───────────── + ${builtins.readFile (phaseDir + "/${f}")} + '') phaseFiles + + '' + + save_receipt() + ''; + passthru.axFleet = { inherit nodes teardown; }; +} diff --git a/tests/ax-fleet/halogen_stub.py b/tests/ax-fleet/halogen_stub.py new file mode 100644 index 000000000..190c9b84c --- /dev/null +++ b/tests/ax-fleet/halogen_stub.py @@ -0,0 +1,165 @@ +#!/usr/bin/env python3 +"""Halogen stand-in for the ax-fleet VM test (DESIGN.md 12.1, node `worker`). + +OpenAI-compatible enough for the ax task image's two model paths: + - `halogen-smoke`: one non-streaming POST /v1/chat/completions (curl); + - `pi`: streaming POST /v1/chat/completions (SSE, `data: {...}` chunks, then + `data: [DONE]`). +Also GET /v1/models and GET /health, like the real server. + +Every request is appended to --log as one JSON line with the source address +and a per-request id, so the test can prove that sandbox traffic arrived +SNAT'd from the NAS (10.42.0.1) and count requests per Task. +""" + +import argparse +import json +import threading +import time +import uuid +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer + +MODEL = "halogen-qwen3.8-flash-next" +REPLY = "halogen-stub-ok" +# The pi mode asks for one JSON object against a JSON Schema in its system +# prompt; the adapter's default schema wants {"answer": }. +JSON_REPLY = '{"answer": 42}' + + +def reply_for(req): + """REPLY, or JSON_REPLY when any message mentions a JSON Schema.""" + for msg in req.get("messages") or []: + content = msg.get("content") if isinstance(msg, dict) else None + if isinstance(content, list): + content = " ".join(str(p.get("text", "")) for p in content if isinstance(p, dict)) + if isinstance(content, str) and "JSON Schema" in content: + return JSON_REPLY + return REPLY +LOCK = threading.Lock() + +# probe/ax-fleet-nop1: a stand-in floor. A Task's command POSTs its own +# completion to /floor/complete (the in-sandbox adapter reporting straight to +# the floor, LINK-DESIGN L7); the test driver, standing in for the link, reads +# GET /floor/results and then deletes the Task. Every POST is kept, so a +# golden-snapshot pre-run of the command would show as a second report. +FLOOR = [] + + +def make_handler(log_path): + class Handler(BaseHTTPRequestHandler): + protocol_version = "HTTP/1.1" + + def log_message(self, fmt, *args): # quiet stderr; the jsonl is the log + pass + + def _record(self, body): + entry = { + "id": uuid.uuid4().hex, + "ts": time.time(), + "src": self.client_address[0], + "method": self.command, + "path": self.path, + "bytes": len(body), + } + with LOCK, open(log_path, "a", encoding="utf-8") as f: + f.write(json.dumps(entry) + "\n") + return entry["id"] + + def _json(self, code, obj): + data = json.dumps(obj).encode() + self.send_response(code) + self.send_header("Content-Type", "application/json") + self.send_header("Content-Length", str(len(data))) + self.end_headers() + self.wfile.write(data) + + def do_GET(self): + rid = self._record(b"") + if self.path.rstrip("/") == "/floor/results": + with LOCK: + self._json(200, {"reports": list(FLOOR)}) + return + if self.path.rstrip("/") == "/health": + self._json(200, {"status": "ok", "in_flight": 0, "queued": 0, "request_id": rid}) + elif self.path.rstrip("/") == "/v1/models": + self._json(200, {"object": "list", "data": [{"id": MODEL, "object": "model", "owned_by": "halogen"}]}) + else: + self._json(404, {"error": "not found", "request_id": rid}) + + def do_POST(self): + length = int(self.headers.get("Content-Length") or 0) + body = self.rfile.read(length) if length else b"" + rid = self._record(body) + if self.path.rstrip("/") == "/floor/complete": + try: + report = json.loads(body or b"{}") + except json.JSONDecodeError: + self._json(400, {"error": "bad json", "request_id": rid}) + return + with LOCK: + FLOOR.append({"received": time.time(), "src": self.client_address[0], "request_id": rid, "report": report}) + n = len(FLOOR) + self._json(200, {"ok": True, "request_id": rid, "n": n}) + return + if self.path.rstrip("/") != "/v1/chat/completions": + self._json(404, {"error": "not found", "request_id": rid}) + return + try: + req = json.loads(body or b"{}") + except json.JSONDecodeError: + self._json(400, {"error": "bad json", "request_id": rid}) + return + created = int(time.time()) + cid = "chatcmpl-" + rid + if not req.get("stream"): + self._json( + 200, + { + "id": cid, + "object": "chat.completion", + "created": created, + "model": MODEL, + "choices": [ + { + "index": 0, + "message": {"role": "assistant", "content": reply_for(req)}, + "finish_reason": "stop", + } + ], + "usage": {"prompt_tokens": 1, "completion_tokens": 1, "total_tokens": 2}, + }, + ) + return + self.send_response(200) + self.send_header("Content-Type", "text/event-stream") + self.send_header("Cache-Control", "no-cache") + self.send_header("Connection", "close") + self.end_headers() + chunks = [ + {"role": "assistant", "content": ""}, + {"content": reply_for(req)}, + ] + for i, delta in enumerate(chunks + [None]): + choice = {"index": 0, "delta": delta or {}, "finish_reason": None if delta else "stop"} + obj = {"id": cid, "object": "chat.completion.chunk", "created": created, "model": MODEL, "choices": [choice]} + if delta is None: + obj["usage"] = {"prompt_tokens": 1, "completion_tokens": 1, "total_tokens": 2} + self.wfile.write(b"data: " + json.dumps(obj).encode() + b"\n\n") + self.wfile.flush() + self.wfile.write(b"data: [DONE]\n\n") + self.wfile.flush() + self.close_connection = True + + return Handler + + +def main(): + ap = argparse.ArgumentParser() + ap.add_argument("--port", type=int, default=8731) + ap.add_argument("--log", default="/tmp/halogen-stub.jsonl") + args = ap.parse_args() + ThreadingHTTPServer(("0.0.0.0", args.port), make_handler(args.log)).serve_forever() + + +if __name__ == "__main__": + main() diff --git a/tests/ax-fleet/nodes.nix b/tests/ax-fleet/nodes.nix new file mode 100644 index 000000000..9134a448e --- /dev/null +++ b/tests/ax-fleet/nodes.nix @@ -0,0 +1,462 @@ +{ + pkgs, + lib, + inputs, +}: +# The four VMs of checks.x86_64-linux.ax-fleet (DESIGN.md 12.1), shared with +# checks.x86_64-linux.ax-fleet-boot. They carry the fleet's real hostnames and +# LAN addresses, so every manifest and firewall rule the modules render is the +# production one; only interface names (eth1, eth2), the token and the kubelet +# reservations differ, and ax-fleet-topology pins that parity. +# +# Each base config is "today": it does NOT import modules/ax-fleet at all, as +# origin/main's hosts/{nas,coordinator} do not (fix round 4: the round-3 base +# carried the role, so every role-scoped guard was already in place before the +# switch). Two specialisations import the fleet module: +# ax-on myAxFleet.enable = true; the test switches to it live, NAS first, +# exactly as Tom will. +# ax-off the role declared, enable = false: the kill switch (DESIGN 13). +# 90-rollback runs the kill switch on the coordinator, and the generation +# rollback (back to this base) on both hosts. The worker is not switched in the +# motion and keeps its role in the base. +let + # A plain file, test-only, not a secret: the fleet reads agenix instead. + token = pkgs.writeText "ax-fleet-vm-token" "ax-fleet-vm-test-token-0123456789abcdef"; + # The agent credential, distinct from the server token as on the fleet. + agentToken = pkgs.writeText "ax-fleet-vm-agent-token" "ax-fleet-vm-agent-token-fedcba9876543210"; + + # busybox (httpd, nslookup) plus curl, imported by k3s from the images + # directory on both nodes, so probe pods need no registry and no network. + probeImage = pkgs.dockerTools.buildImage { + name = "ax-fleet-probe"; + tag = "test"; + copyToRoot = pkgs.buildEnv { + name = "ax-fleet-probe-root"; + paths = [ + pkgs.busybox + pkgs.curl + pkgs.cacert + ]; + pathsToLink = [ + "/bin" + "/etc" + ]; + }; + config.Cmd = [ + "/bin/sh" + "-c" + "sleep 1000000" + ]; + }; + + # Test-only: the fleet task image (pkgs/ax-agent-image, the same ax and pi + # the fleet seeds) plus claude-code, for 25-harness-probe's `claude + # --version`. No credential exists in it or in any Task. The fleet image + # itself stays pi-only on day one (DESIGN.md 10.2 and 11); only the NAS test + # node seeds this variant, from its ax-on specialisation. + claudeProbeImage = + inputs.nixpkgs.legacyPackages.x86_64-linux.callPackage ../../pkgs/ax-agent-image + { + ax = inputs.self.packages.x86_64-linux.ax; + pi = inputs.llm-agents.packages.x86_64-linux.pi; + extraPaths = [ inputs.llm-agents.packages.x86_64-linux.claude-code ]; + variant = "-claude-probe"; + }; + + # probe/ax-fleet-nop1, test-only: Substrate's own kubectl plugin from the + # same pinned source, so the no-P1 phase can list actors, templates and + # workers (leak and worker-freed checks). Not in any host closure. + kubectlAte = + (pkgs.callPackage ../../pkgs/substrate { + go_1_27 = inputs.nixpkgs-go.legacyPackages.x86_64-linux.go_1_27; + }).ate-setup.overrideAttrs + (o: { + pname = "kubectl-ate"; + subPackages = [ "cmd/kubectl-ate" ]; + meta = o.meta // { + mainProgram = "kubectl-ate"; + }; + }); + + lanAddr = address: { + interface = "eth1"; + inherit address; + }; + + setAddr = iface: address: prefixLength: { + networking.interfaces.${iface}.ipv4.addresses = lib.mkForce [ { inherit address prefixLength; } ]; + }; + + # Everything that makes a node a fleet node in the test (the ax-on and + # ax-off specialisations import it; the base does not). + fleetNode = + { role, address }: + { + imports = [ + ../../modules/ax-fleet + # As on the real hosts (hosts/{coordinator,worker,client}): the harness + # role turns myAxClient on by mkDefault, which puts `ax` and `kubectl` + # on the coordinator's PATH for the Task phases. + ../../modules/ax-client.nix + inputs.agenix.nixosModules.default + ]; + myAxFleet = { + inherit role; + lan = lanAddr address; + k3sTokenFile = "${token}"; + k3sAgentTokenFile = "${agentToken}"; + guardInterfaces = [ "eth2" ]; + }; + }; + + # The test tools, in the base: identical in every state. + testBase = { + system.switch.enable = true; + environment.systemPackages = [ + pkgs.jq + pkgs.curl + pkgs.iptables + pkgs.nftables + pkgs.iproute2 + pkgs.dnsutils + ]; + }; + + # fleet: the host's fleet module (fleetNode plus host-specific settings). + # extra: ax-on only (test images, reservations). + axStates = fleet: extra: { + specialisation.ax-on.configuration = { + imports = [ + fleet + extra + ]; + myAxFleet.enable = true; + services.k3s.images = [ probeImage ]; + }; + specialisation.ax-off.configuration = { + imports = [ fleet ]; + }; + }; + + nasFleet = fleetNode { + role = "control"; + address = "10.42.0.1"; + }; + + coordinatorFleet = { + imports = [ + (fleetNode { + role = "harness"; + address = "10.42.0.2"; + }) + ]; + # The desk's wired port (fix round 3): eth3, NetworkManager-managed, + # DHCP from the worker's second leg, as enp191s0's "Wired connection 1". + myAxFleet.lan.extraInterfaces = [ "eth3" ]; + }; + + # The NAS as it is today, before ax (shared with ax-fleet-boot, which adds + # nasFleet in its base and boots it). + nasBase = + { ... }: + { + imports = [ + testBase + (setAddr "eth1" "10.42.0.1" 24) + ]; + networking.hostName = "nas"; + # The internet stand-in (fix round 4): TEST-NET-2 lives on the worker. + networking.interfaces.eth1.ipv4.routes = [ + { + address = "198.51.100.0"; + prefixLength = 24; + via = "10.42.0.5"; + } + ]; + virtualisation = { + vlans = [ 1 ]; + memorySize = 10240; + cores = 6; + diskSize = 8192; + emptyDiskImages = [ + 8192 + 16384 + ]; + fileSystems = { + "/mnt/fast" = { + device = "/dev/vdb"; + fsType = "ext4"; + autoFormat = true; + }; + "/mnt/nas" = { + device = "/dev/vdc"; + fsType = "btrfs"; + autoFormat = true; + }; + }; + }; + boot.supportedFilesystems = [ "btrfs" ]; + # The house router, as hosts/nas/router.nix declares it (fix round 2): + # the snapshot and the teardown must keep it. + boot.kernel.sysctl."net.ipv4.ip_forward" = 1; + + # The NAS firewall as the real one is shaped: nftables, interface-scoped + # extraInputRules, filterForward off, strict rpfilter, and a reload that + # keeps other tables (flushRuleset = false). + networking.nftables.enable = true; + networking.nftables.flushRuleset = false; + networking.firewall = { + enable = true; + filterForward = false; + checkReversePath = "strict"; + extraInputRules = '' + iifname "eth1" udp dport 53 accept comment "stand-in AdGuard" + iifname "eth1" tcp dport 53 accept comment "stand-in AdGuard" + ''; + }; + + # AdGuard stand-in: answers one record nobody else serves. + services.dnsmasq = { + enable = true; + resolveLocalQueries = false; + settings = { + listen-address = [ "10.42.0.1" ]; + bind-interfaces = true; + no-resolv = true; + address = [ "/only-nas.test/10.42.0.77" ]; + }; + }; + + # A recording `tailscale` on the system PATH, as the real NAS has one: + # k3s-killall.sh's remove_interfaces runs `tailscale set + # --advertise-routes=` whenever the binary is reachable, which would + # withdraw the house subnet route. 90-rollback asserts the teardown + # never calls it. + environment.systemPackages = [ + kubectlAte + (pkgs.writeShellScriptBin "tailscale" '' + echo "$*" >> /var/log/tailscale-stub.log + '') + ]; + + # The bystander: the NAS's shared PostgreSQL (Paperless, Immich) must + # not restart and must not change. + services.postgresql = { + enable = true; + ensureDatabases = [ "paperless" ]; + }; + }; + + + vmReservations = { + systemReserved = "cpu=500m,memory=512Mi"; + kubeReserved = "cpu=250m,memory=256Mi"; + # The full signal set, as harness.nix renders it (fix round 4): a set flag + # replaces every kubelet default. + evictionHard = "memory.available<256Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<15%,imagefs.inodesFree<5%"; + }; +in +{ + inherit + probeImage + token + agentToken + claudeProbeImage + kubectlAte + nasBase + nasFleet + ; + + nas = + { ... }: + { + imports = [ + nasBase + (axStates nasFleet { + myAxFleet.kubelet = { + systemReserved = "cpu=1,memory=1Gi"; + }; + myAxFleet.registrySeed.ax-agent-claude-probe = { + oci = claudeProbeImage; + repo = "ax/ax-agent-claude-probe"; + inherit (claudeProbeImage.passthru) tag; + }; + }) + ]; + }; + + coordinator = + { ... }: + { + imports = [ + testBase + (axStates coordinatorFleet { myAxFleet.kubelet = vmReservations; }) + (setAddr "eth1" "10.42.0.2" 24) + (setAddr "eth2" "100.105.121.73" 10) + ]; + networking.hostName = "coordinator"; + # myAxFleet.apiUsers defaults to [ "tom" ]; alice is the other local user. + users.users.tom.isNormalUser = true; + virtualisation = { + vlans = [ + 1 + 2 + 3 + ]; + memorySize = 8192; + cores = 4; + diskSize = 8192; + podman.enable = true; + }; + # The desk's shape: iptables firewall, strict rpfilter, NetworkManager + # running (its InvocationID must not change at switch), zram swap. + networking.nftables.enable = false; + networking.firewall = { + enable = true; + checkReversePath = "strict"; + allowedTCPPorts = [ 80 ]; + }; + networking.networkmanager.enable = true; + # eth1/eth2 are the test driver's static addresses; identical in base + # and specialisation, so NetworkManager.conf does not change at switch. + networking.networkmanager.unmanaged = [ + "eth1" + "eth2" + ]; + zramSwap.enable = true; + # The desk's kernel, where runsc runs (fix round 2: the VM had run the + # pin's default kernel). The same expression as modules/strix.nix, so + # the same derivation; ax-fleet-topology asserts it. + boot.kernelPackages = + (import inputs.nixpkgs-fresh { + system = "x86_64-linux"; + config.allowUnfree = true; + }).linuxPackages_7_2; + # The desk's sshd as modules/common.nix renders it: port 22 open on every + # interface, passwords accepted (fix round 2). Pods must not reach it. + services.openssh = { + enable = true; + openFirewall = true; + settings.PasswordAuthentication = true; + settings.KbdInteractiveAuthentication = true; + }; + + services.caddy = { + enable = true; + virtualHosts.":80".extraConfig = '' + respond "caddy-ok" + ''; + }; + + # herdr stand-in: a lingering user's long-lived user unit with the same + # X-SwitchMethod herdr has. Its MainPID must survive switch and rollback. + users.users.alice = { + isNormalUser = true; + linger = true; + }; + systemd.user.services.herdr-standin = { + wantedBy = [ "default.target" ]; + unitConfig."X-SwitchMethod" = "keep-old"; + serviceConfig.ExecStart = "${pkgs.coreutils}/bin/sleep 1000000"; + }; + }; + + worker = + { ... }: + { + imports = [ + testBase + (fleetNode { + role = "inference"; + address = "10.42.0.5"; + }) + # 198.51.100.5 (TEST-NET-2): a public address for the egress tests + # (fix round 4), routed to the worker by the NAS. + { + networking.interfaces.eth1.ipv4.addresses = lib.mkForce [ + { + address = "10.42.0.5"; + prefixLength = 24; + } + { + address = "198.51.100.5"; + prefixLength = 32; + } + ]; + } + (setAddr "eth2" "192.168.43.5" 24) + ]; + networking.hostName = "worker"; + virtualisation.vlans = [ + 1 + 3 + ]; + # vlan 3: a second LAN segment for the coordinator's wired leg (fix + # round 3). DHCP with no router option, so no default route moves. + services.dnsmasq = { + enable = true; + resolveLocalQueries = false; + settings = { + port = 0; + interface = [ "eth2" ]; + bind-interfaces = true; + dhcp-range = [ "192.168.43.100,192.168.43.150,1h" ]; + dhcp-option = [ "3" ]; + }; + }; + # Another worker port (fix round 3): the real worker opens 22 with + # passwords on every interface; pods and the egress gateway must reach + # 8731 and nothing else. + # The public target (fix round 4): what a Task with no Gateway must not + # reach, while the NAS host does. + systemd.services.public-8000 = { + wantedBy = [ "multi-user.target" ]; + # All addresses: binding 198.51.100.5 raced its assignment (MEASURED, + # fix-round-4 run 1: the unit exited 1 before network-addresses-eth1 + # added the address). + serviceConfig = { + ExecStart = "${pkgs.busybox}/bin/httpd -f -p 8000 -h ${pkgs.writeTextDir "index.html" "public-reached\n"}"; + Restart = "always"; + }; + }; + systemd.services.worker-2222 = { + wantedBy = [ "multi-user.target" ]; + serviceConfig.ExecStart = "${pkgs.busybox}/bin/httpd -f -p 2222 -h ${pkgs.writeTextDir "index.html" "worker-port-2222-reached\n"}"; + }; + # The worker is not switched in the motion; its fleet role is ON from + # boot and renders only the 8731 assertion, as on the real host. + myAxFleet.enable = true; + networking.firewall = { + enable = true; + interfaces.eth1.allowedTCPPorts = [ + 8731 + 2222 + 8000 + ]; + interfaces.eth2.allowedTCPPorts = [ 8731 ]; + interfaces.eth2.allowedUDPPorts = [ 67 ]; + }; + systemd.services.halogen-stub = { + wantedBy = [ "multi-user.target" ]; + serviceConfig = { + ExecStart = "${pkgs.python3}/bin/python3 ${./halogen_stub.py} --port 8731 --log /var/lib/halogen-stub/requests.jsonl"; + StateDirectory = "halogen-stub"; + }; + }; + }; + + peer = + { ... }: + { + imports = [ (setAddr "eth1" "100.64.0.9" 10) ]; + networking.hostName = "peer"; + virtualisation.vlans = [ 2 ]; + networking.firewall.enable = false; + environment.systemPackages = [ pkgs.curl ]; + # Something to reach: the guard must stop pods from getting here. + systemd.services.peer-http = { + wantedBy = [ "multi-user.target" ]; + serviceConfig.ExecStart = "${pkgs.busybox}/bin/httpd -f -p 8000 -h /etc"; + }; + }; +} diff --git a/tests/ax-fleet/phases/10-cluster.py b/tests/ax-fleet/phases/10-cluster.py new file mode 100644 index 000000000..312c2cefa --- /dev/null +++ b/tests/ax-fleet/phases/10-cluster.py @@ -0,0 +1,404 @@ +# Phase 1 (baseline), 2 and 3 (switch nas, then coordinator): the cluster +# assertions. Track cluster. DESIGN.md 12.1 and 6.6. The substrate and ax +# phases (20, 30) run after this one, on the cluster it leaves behind. + +AX_ON = "/run/current-system/specialisation/ax-on/bin/switch-to-configuration test" + +PROBE_POD = """ +apiVersion: v1 +kind: Pod +metadata: + name: {name} + namespace: default + labels: {{app: ax-fleet-probe}} +spec: + nodeSelector: {{ax.mecattaf.dev/role: {role}}} + tolerations: + - {{key: ate.dev/sandboxClass, operator: Exists, effect: NoSchedule}} + containers: + - name: probe + image: ax-fleet-probe:test + imagePullPolicy: Never + command: ["/bin/sh", "-c", "echo probe-started-{name}; mkdir -p /www; echo pod-ok > /www/index.html; exec httpd -f -p 8000 -h /www"] + ports: + - {{containerPort: 8000, hostPort: {host_port}}} + {volume_mounts} + {volumes} +""" + + +def apply_probe(name, role, host_port, pvc=None): + vm = "volumeMounts: [{name: data, mountPath: /data}]" if pvc else "" + vols = f"volumes: [{{name: data, persistentVolumeClaim: {{claimName: {pvc}}}}}]" if pvc else "" + doc = PROBE_POD.format(name=name, role=role, host_port=host_port, volume_mounts=vm, volumes=vols) + nas.succeed(f"cat > /tmp/{name}.yaml <<'EOF'\n{doc}\nEOF") + kubectl(f"apply -f /tmp/{name}.yaml") + kubectl(f"wait --for=condition=Ready pod/{name} --timeout=300s") + + +with step("baseline"): + start_all() + for m in (nas, coordinator, worker, peer): + m.wait_for_unit("multi-user.target") + coordinator.wait_for_unit("caddy.service") + coordinator.wait_for_unit("NetworkManager.service") + nas.wait_for_unit("postgresql.service") + nas.wait_for_unit("dnsmasq.service") + worker.wait_for_unit("halogen-stub.service") + worker.wait_for_open_port(8731) + coordinator.wait_until_succeeds( + "runuser -u alice -- env XDG_RUNTIME_DIR=/run/user/$(id -u alice) systemctl --user is-active herdr-standin" + ) + worker.succeed("curl -sf --max-time 10 http://10.42.0.2/ | grep -x caddy-ok") + peer.succeed("curl -sf --max-time 10 http://100.105.121.73/ | grep -x caddy-ok") + # Not a k3s node, never switched: nothing from the fleet module runs here. + worker.fail("systemctl cat k3s.service") + nas.fail("systemctl cat k3s.service") + coordinator.fail("systemctl cat k3s.service") + + base = { + "herdr_pid": user_unit_pid("herdr-standin"), + "nm_invocation": nm_invocation(), + "postgres_invocation": unit_invocation(nas, "postgresql.service"), + "nas_databases": sorted(nas.succeed("runuser -u postgres -- psql -Atc 'select datname from pg_database'").split()), + "nas_nft": nas.succeed("nft -s list ruleset"), + "nas_nixos_fw": nas.succeed("nft -s list table inet nixos-fw"), + "sysctl_nas": sysctls(nas), + "sysctl_coordinator": sysctls(coordinator), + } + record("baseline", {k: v for k, v in base.items() if k not in ("nas_nft", "nas_nixos_fw")}) + + +with step("switch nas"): + nas.succeed(f"{AX_ON} >&2") + nas.succeed("systemctl is-active k3s.service docker-registry.service") + for u in ("ax-fleet-registry-seed.service", "ax-fleet-bootstrap.service"): + nas.succeed(f"systemctl is-active {u}") + nas.succeed(f"systemctl show {u} -p Result --value | grep -x success") + + +with step("nas: k3s state on the fast tier, links on that disk"): + src = nas.succeed("findmnt -n -o SOURCE -T /var/lib/rancher/k3s").strip() + record("nas_rancher_source", src) + assert src.startswith("/dev/vdb"), f"/var/lib/rancher/k3s is on {src}, not the /mnt/fast disk" + for p in ("/var/lib/kubelet", "/var/log/pods"): + s = nas.succeed(f"findmnt -n -o SOURCE -T {p}").strip() + assert s.startswith("/dev/vdb"), f"{p} is on {s}" + # The airgap images link lives under the bind mount, i.e. on /mnt/fast. + nas.succeed("ls -l /var/lib/rancher/k3s/agent/images/ | grep -q airgap") + nas.succeed("ls -l /mnt/fast/k3s/rancher/k3s/agent/images/ | grep -q airgap") + nas.succeed("test -s /etc/ax-fleet/admin.kubeconfig") + nas.succeed("stat -c '%a %U %G' /etc/ax-fleet/admin.kubeconfig | grep -x '640 root wheel'") + nas.succeed("grep -q 'server: https://10.42.0.1:6443' /etc/ax-fleet/admin.kubeconfig") + nas.succeed("test -s /var/lib/ax-fleet/sysctl-before.conf") + # The house router's forwarding is what the snapshot says (fix round 2). + nas.succeed("grep -x 'net.ipv4.ip_forward = 1' /var/lib/ax-fleet/sysctl-before.conf") + + +with step("nas: node Ready, untainted, control labels"): + node_ready("nas") + # k3s's own transient taints (uninitialized, not-ready) clear on their own. + nas.wait_until_succeeds("test -z \"$(k3s kubectl get node nas -o jsonpath='{.spec.taints}')\"", timeout=300) + labels = json.loads(kubectl("get node nas -o jsonpath='{.metadata.labels}'")) + assert labels.get("ax.mecattaf.dev/role") == "control", labels + assert labels.get("ate.dev/substrate-version") == "none", labels + ip = jsonpath("node nas", "{.status.addresses[?(@.type==\"InternalIP\")].address}") + assert ip == "10.42.0.1", ip + kubectl("-n kube-system wait --for=condition=Available deploy/coredns deploy/local-path-provisioner --timeout=600s") + + +with step("nas: certificates.k8s.io/v1beta1 serves the Substrate resources"): + res = json.loads(kubectl("get --raw /apis/certificates.k8s.io/v1beta1")) + names = sorted(r["name"] for r in res["resources"]) + record("certificates_v1beta1", names) + assert "clustertrustbundles" in names and "podcertificaterequests" in names, names + + +with step("nas: PersistentVolumes land on the data pool"): + nas.succeed( + "cat > /tmp/pvc.yaml <<'EOF'\n" + "apiVersion: v1\nkind: PersistentVolumeClaim\nmetadata: {name: ax-fleet-probe-pvc, namespace: default}\n" + "spec: {accessModes: [ReadWriteOnce], storageClassName: local-path, resources: {requests: {storage: 16Mi}}}\n" + "EOF" + ) + kubectl("apply -f /tmp/pvc.yaml") + apply_probe("probe-nas", "control", 18086, pvc="ax-fleet-probe-pvc") + kubectl("exec probe-nas -- sh -c 'echo data-pool > /data/marker'") + paths = kubectl( + "get pv -o jsonpath='{range .items[*]}{.spec.hostPath.path}{.spec.local.path}{\"\\n\"}{end}'" + ).split() + record("pv_paths", paths) + assert paths and all(p.startswith(LOCAL_PATH_ROOT) for p in paths), paths + nas.succeed(f"grep -rqx data-pool {LOCAL_PATH_ROOT}") + src = nas.succeed(f"findmnt -n -o SOURCE -T {LOCAL_PATH_ROOT}").strip() + assert src.startswith("/dev/vdc"), f"local-path root is on {src}, not the data pool" + + +with step("nas: bystanders untouched"): + assert unit_invocation(nas, "postgresql.service") == base["postgres_invocation"], "the shared PostgreSQL restarted" + dbs = sorted(nas.succeed("runuser -u postgres -- psql -Atc 'select datname from pg_database'").split()) + assert dbs == base["nas_databases"], (dbs, base["nas_databases"]) + nas.succeed("dig +short @10.42.0.1 only-nas.test | grep -x 10.42.0.77") + ruleset = nas.succeed("nft list ruleset") + record("nas_kube_services_in_nft", "KUBE-SERVICES" in ruleset) + nas.succeed("nft list chain inet ax-fleet-guard prerouting | grep -q 'ax-fleet: cluster ranges'") + record("sysctl_nas_after_switch", sysctls(nas)) + kernel_keys_back(nas, base["sysctl_nas"]) + + +with step("switch coordinator"): + coordinator.succeed(f"{AX_ON} >&2") + coordinator.succeed("systemctl is-active k3s.service ax-fleet-nm-unmanaged.service ax-server-proxy.socket") + node_ready("coordinator") + + +with step("coordinator: Ready with the harness taint and labels"): + taints = json.loads(kubectl("get node coordinator -o jsonpath='{.spec.taints}'")) + assert {"key": "ate.dev/sandboxClass", "value": "gvisor", "effect": "NoSchedule"} in taints, taints + labels = json.loads(kubectl("get node coordinator -o jsonpath='{.metadata.labels}'")) + assert labels.get("ax.mecattaf.dev/role") == "harness", labels + assert labels.get("ate.dev/substrate-version") == "d277088b", labels + ip = jsonpath("node coordinator", "{.status.addresses[?(@.type==\"InternalIP\")].address}") + assert ip == "10.42.0.2", ip + # flannel.1 appears shortly after the node registers; wait for it. + coordinator.wait_until_succeeds("ip -d link show flannel.1 | grep -q 'dev eth1'", timeout=180) + record("sysctl_coordinator_after_switch", sysctls(coordinator)) + + +with step("coordinator: only tolerating pods land here; control stays on the NAS"): + apply_probe("probe-coord", "harness", 18085) + placed = kubectl( + "get pods -A -o jsonpath='{range .items[*]}{.metadata.namespace}/{.metadata.name} {.spec.nodeName}{\"\\n\"}{end}'" + ) + record("pod_placement", placed.strip().splitlines()) + for line in placed.strip().splitlines(): + name, node = line.split() + if node == "coordinator": + assert ( + name == "default/probe-coord" or "/atelet" in name or "/ateom-gvisor" in name + ), f"{name} landed on the coordinator" + # kubectl logs of a coordinator pod rides the agent tunnel (egress-selector agent). + kubectl("logs probe-coord | grep -q probe-started-probe-coord") + + +with step("coordinator: nothing changed for Tom"): + assert user_unit_pid("herdr-standin") == base["herdr_pid"], "herdr stand-in restarted" + assert nm_invocation() == base["nm_invocation"], "NetworkManager restarted" + coordinator.wait_until_succeeds("nmcli -t -f DEVICE,STATE d | grep -x 'cni0:unmanaged'", timeout=60) + coordinator.succeed("nmcli -t -f DEVICE,STATE d | grep -x 'flannel.1:unmanaged'") + record("nmcli_devices", coordinator.succeed("nmcli -t -f DEVICE,STATE d").strip().splitlines()) + worker.succeed("curl -sf --max-time 10 http://10.42.0.2/ | grep -x caddy-ok") + peer.succeed("curl -sf --max-time 10 http://100.105.121.73/ | grep -x caddy-ok") + # A podman container on the default bridge still reaches the host's Caddy + # (br_netfilter is loaded now, rpfilter is strict). + loaded = coordinator.succeed(f"podman load -i {PROBE_TARBALL}") + ref = [l.split("Loaded image:")[1].strip() for l in loaded.splitlines() if "Loaded image" in l][0] + coordinator.succeed(f"podman run --rm --network bridge {ref} curl -sf --max-time 10 http://10.88.0.1/ | grep -x caddy-ok") + coordinator.succeed("lsmod | grep -q br_netfilter") + # kubelet's panic tunables are put back: an oops does not reboot the desk. + kernel_keys_back(coordinator, base["sysctl_coordinator"]) + record("sysctl_coordinator_after_restore", sysctls(coordinator)) + + +with step("coordinator: kubelet's tunables are put back whenever they appear, not only after a k3s start"): + # Fix round 2. kubelet applies them when its container manager starts, + # which on an agent is when the server first answers, possibly long after + # k3s.service started (MEASURED by the review). Write kubelet's values with + # no k3s restart, well after the switch: the watcher must restore them. + nrestarts = coordinator.succeed("systemctl show k3s.service -p NRestarts --value").strip() + coordinator.succeed("sysctl -w kernel.panic=10 kernel.panic_on_oops=1 vm.overcommit_memory=1") + kernel_keys_back(coordinator, base["sysctl_coordinator"]) + assert coordinator.succeed("systemctl show k3s.service -p NRestarts --value").strip() == nrestarts + coordinator.succeed("systemctl is-active ax-fleet-kernel-tunables.service") + + +with step("coordinator: the desk outweighs kubepods for CPU"): + w = {s: int(coordinator.succeed(f"cat /sys/fs/cgroup/{s}/cpu.weight").strip()) for s in ("user.slice", "system.slice", "kubepods.slice")} + record("cpu_weights", w) + assert w["user.slice"] > w["kubepods.slice"] and w["system.slice"] > w["kubepods.slice"], w + + +with step("coordinator: kubelet evicts on disk pressure, not only memory"): + # Fix round 4: a set --eviction-hard replaces kubelet's defaults, so the + # harness renders every signal. The kubelet logs the map it runs with. + line = coordinator.succeed( + "journalctl -u k3s --no-pager -o cat | grep -o 'HardEvictionThresholds.*' | tail -n1 | cut -c1-2000" + ) + signals = sorted(set(re.findall(r'"Signal":"([a-z.A-Z]+)"', line))) + record("coordinator_hard_eviction_signals", signals) + for s in ("memory.available", "nodefs.available", "nodefs.inodesFree", "imagefs.available"): + assert s in signals, (s, signals) + + +with step("coordinator: the guards hold"): + pod_ip = jsonpath("pod probe-coord", "{.status.podIP}") + record("probe_coord_ip", pod_ip) + # Make the negative tests discriminating: the peer routes the LAN and the + # pod network through the coordinator, and the worker routes the tailnet + # back through it. Without the guard chain, ip_forward=1 would carry these. + peer.succeed("ip route replace 10.42.0.0/24 via 100.105.121.73") + peer.succeed("ip route replace 10.200.0.0/16 via 100.105.121.73") + worker.succeed("ip route replace 100.64.0.0/10 via 10.42.0.2") + peer.fail("curl -sf --max-time 5 http://10.42.0.5:8731/health") + peer.fail(f"curl -sf --max-time 5 http://{pod_ip}:8000/") + for port in (8085, 9090, 10250, 6443, 18085): + peer.fail(f"curl -sk --max-time 5 -o /dev/null https://100.105.121.73:{port}/") + peer.fail(f"curl -s --max-time 5 -o /dev/null http://100.105.121.73:{port}/") + # hostPorts: from the NAS yes (atelet's peers), from the worker no. + nas.succeed("curl -sf --max-time 10 http://10.42.0.2:18085/ | grep -x pod-ok") + worker.fail("curl -sf --max-time 5 http://10.42.0.2:18085/") + # Pods never reach the tailnet; the coordinator host itself still does. + coordinator.succeed("curl -sf --max-time 5 http://100.64.0.9:8000/hostname") + kubectl("exec probe-coord -- sh -c '! curl -sf --max-time 5 http://100.64.0.9:8000/hostname'") + # Pods reach the apiserver Service (any HTTP answer is reachability). + code = kubectl("exec probe-coord -- curl -sk -o /dev/null -w '%{http_code}' --max-time 10 https://10.201.0.1/readyz").strip() + assert code != "000", "a coordinator pod cannot reach the apiserver Service" + # Pods never reach the LAN except the NAS's apiserver and registry. + kubectl("exec probe-coord -- sh -c '! curl -sf --max-time 5 http://10.42.0.5:8731/health'") + worker.succeed("ip route del 100.64.0.0/10 via 10.42.0.2") + assert coordinator.succeed("sysctl -n net.ipv6.conf.all.forwarding").strip() == "0" + coordinator.succeed("iptables -t mangle -S nixos-fw-rpfilter | grep -q rpfilter") + coordinator.succeed("iptables -t mangle -S FORWARD 1 | grep -q ax-fleet-guard") + coordinator.fail("ip -br link | grep -qi cilium") + record("guard_chain", coordinator.succeed("iptables -t mangle -S ax-fleet-guard").strip().splitlines()) + + +with step("coordinator: pods never reach the coordinator host (sshd accepts passwords)"): + # Fix round 2. Discriminating: sshd answers the worker on the LAN, and + # port 22 is open on every interface (openFirewall), so only the pod-input + # refusal stops a pod. + worker.succeed("timeout 10 bash -c 'exec 3<>/dev/tcp/10.42.0.2/22; head -c 7 <&3' | grep -x SSH-2.0") + gw = coordinator.succeed("ip -4 -o addr show dev cni0 | awk '{print $4}' | cut -d/ -f1").strip() + fl = coordinator.succeed("ip -4 -o addr show dev flannel.1 | awk '{print $4}' | cut -d/ -f1").strip() + record("pod_input_targets", {"cni0": gw, "flannel.1": fl}) + for ip in ("10.42.0.2", gw, fl, "100.105.121.73"): + kubectl(f"exec probe-coord -- sh -c '! (nc -w 5 {ip} 22 /dev/null | grep -q SSH)'") + # The NAS's pods, over VXLAN, neither. + kubectl(f"exec probe-nas -- sh -c '! (nc -w 5 {fl} 22 /dev/null | grep -q SSH)'") + # Pod reachability of the host's own hostPort and of Services is unchanged. + code = kubectl("exec probe-coord -- curl -sk -o /dev/null -w '%{http_code}' --max-time 10 https://10.201.0.1/readyz").strip() + assert code != "000", "a coordinator pod lost the apiserver Service" + refused = coordinator.succeed("iptables -S nixos-fw | grep -c ax-fleet-pod-input").strip() + assert refused == "2", refused + coordinator.succeed("ip6tables -S nixos-fw | grep -q 'cni0.*ax-fleet-pod-input'") + + +with step("coordinator: pods reach no private range on the LAN leg (the Freebox fallback case)"): + # A private subnet this module does not know, on the same leg, as when + # NetworkManager falls back to the Freebox profile. Discriminating: the + # coordinator host reaches it; a pod must not. + worker.succeed("ip addr add 192.168.77.5/24 dev eth1") + coordinator.succeed("ip route replace 192.168.77.0/24 dev eth1") + try: + coordinator.succeed("curl -sf --max-time 10 http://192.168.77.5:8731/health") + kubectl("exec probe-coord -- sh -c '! curl -s --max-time 5 -o /dev/null http://192.168.77.5:8731/health'") + finally: + coordinator.succeed("ip route del 192.168.77.0/24 dev eth1") + worker.succeed("ip addr del 192.168.77.5/24 dev eth1") + + +with step("coordinator: a second LAN leg (the desk's wired port) is guarded and never takes the routes"): + # Fix round 3. eth3 stands in for enp191s0: NetworkManager-managed, DHCP + # from the worker's second leg (vlan 3). Round 2's guard named only eth1, + # so pod egress and inbound hostPorts over eth3 matched no DROP. + rc, _ = coordinator.execute("timeout 120 sh -c 'until ip -4 -o addr show dev eth3 | grep -q \"inet 192.168.43.\"; do sleep 2; done'") + if rc != 0: + record("eth3_profile_added", True) + coordinator.succeed("nmcli con add type ethernet ifname eth3 con-name wired-eth3 ipv4.method auto ipv6.method ignore") + record("eth3_routes_before_reactivation", coordinator.succeed("ip -4 route show dev eth3").strip().splitlines()) + # The drop-in's metric applies at the next activation, as on the desk, + # where the wired port is down today. + coordinator.succeed("nmcli device disconnect eth3 && nmcli device connect eth3") + coordinator.wait_until_succeeds("ip -4 -o addr show dev eth3 | grep -q 'inet 192.168.43.'", timeout=120) + routes = coordinator.succeed("ip -4 route show dev eth3").strip().splitlines() + record("eth3_routes", routes) + assert routes and all("metric 700" in r for r in routes), routes + addr3 = coordinator.succeed("ip -4 -o addr show dev eth3 | awk '{print $4}' | cut -d/ -f1").strip() + pod_ip = jsonpath("pod probe-coord", "{.status.podIP}") + # Out: the host reaches the worker's second leg; a pod does not. + coordinator.succeed("curl -sf --max-time 10 http://192.168.43.5:8731/health") + kubectl("exec probe-coord -- sh -c '! curl -s --max-time 5 -o /dev/null http://192.168.43.5:8731/health'") + # In: Caddy answers on the second leg; the pod hostPort and a routed + # path into the pod network do not. + worker.succeed(f"curl -sf --max-time 10 http://{addr3}/ | grep -x caddy-ok") + worker.fail(f"curl -s --max-time 5 -o /dev/null http://{addr3}:18085/") + worker.succeed(f"ip route replace 10.200.0.0/16 via {addr3}") + try: + worker.fail(f"curl -s --max-time 5 -o /dev/null http://{pod_ip}:8000/") + finally: + worker.succeed(f"ip route del 10.200.0.0/16 via {addr3}") + + +with step("coordinator: proxy ARP on the pod interfaces only, never on a LAN leg or a NIC that appears later"): + # Fix round 3. Round 2 set conf.default, which every later NIC copies + # (the desk's NICs appear after systemd-sysctl at boot). + veth = coordinator.succeed("ip -o link show type veth | awk -F': ' '{print $2}' | cut -d@ -f1 | head -1").strip() + coordinator.succeed("ip link add axprobe0 type dummy && udevadm settle") + try: + parp = { + i: coordinator.succeed(f"cat /proc/sys/net/ipv4/conf/{i}/proxy_arp").strip() + for i in ("default", "all", "eth1", "eth2", "eth3", "axprobe0", "cni0", "flannel.1", veth) + } + finally: + coordinator.succeed("ip link del axprobe0") + record("proxy_arp", parp) + assert parp["cni0"] == "1" and parp["flannel.1"] == "1" and parp[veth] == "1", parp + assert all(parp[i] == "0" for i in ("default", "all", "eth1", "eth2", "eth3", "axprobe0")), parp + + +with step("coordinator: LAN traffic routed through the NAS never reaches a harness pod"): + # The house default gateway is the NAS. Before fix round 1 this path + # (worker -> nas -> flannel.1 -> harness pod) answered (MEASURED). + pod_ip = jsonpath("pod probe-coord", "{.status.podIP}") + nas.succeed(f"curl -sf --max-time 10 http://{pod_ip}:8000/ | grep -x pod-ok") # the path itself works + worker.succeed("ip route replace 10.200.0.0/16 via 10.42.0.1") + try: + worker.fail(f"curl -s --max-time 5 -o /dev/null http://{pod_ip}:8000/") + finally: + worker.succeed("ip route del 10.200.0.0/16 via 10.42.0.1") + + +def diag(cmds): + out = {} + for name, (machine, cmd) in cmds.items(): + _, text = machine.execute(cmd + " 2>&1") + out[name] = text[-3000:] + return out + + +with step("cluster plumbing: pod to pod across nodes (flannel VXLAN)"): + nas_pod_ip = jsonpath("pod probe-nas", "{.status.podIP}") + status, _ = nas.execute(f"k3s kubectl exec probe-coord -- curl -sf --max-time 10 http://{nas_pod_ip}:8000/") + if status != 0: + record("diag_flannel", diag({ + "coord_routes": (coordinator, "ip route; ip -d link show flannel.1"), + "nas_routes": (nas, "ip route; ip -d link show flannel.1"), + "coord_fw": (coordinator, "iptables -S nixos-fw; iptables -t mangle -S ax-fleet-guard"), + "nas_nft_input": (nas, "nft list chain inet nixos-fw input-allow"), + })) + nas.succeed(f"k3s kubectl exec probe-coord -- curl -sf --max-time 10 http://{nas_pod_ip}:8000/ | grep -x pod-ok") + + +with step("cluster plumbing: DNS through the NAS stand-in"): + # A records only: the stand-in has no upstream, so AAAA answers REFUSED. + status, _ = nas.execute("k3s kubectl exec probe-coord -- nslookup -type=a only-nas.test 2>&1 | grep -q 10.42.0.77") + if status != 0: + record("diag_dns", diag({ + "coord_nslookup": (nas, "k3s kubectl exec probe-coord -- nslookup only-nas.test"), + "coord_nslookup_svc": (nas, "k3s kubectl exec probe-coord -- nslookup kubernetes.default.svc.cluster.local"), + "nas_pod_direct": (nas, "k3s kubectl exec probe-nas -- nslookup only-nas.test 10.42.0.1"), + "nas_pod_coredns": (nas, "k3s kubectl exec probe-nas -- nslookup only-nas.test"), + "coredns_logs": (nas, "k3s kubectl -n kube-system logs deploy/coredns --tail=50"), + "coredns_pod_resolv": (nas, "k3s kubectl -n kube-system exec deploy/coredns -- cat /etc/resolv.conf"), + "dnsmasq": (nas, "journalctl -u dnsmasq --no-pager | tail -30; ss -lunp | grep :53"), + "nas_nft": (nas, "nft list table inet nixos-fw"), + })) + kubectl("exec probe-coord -- nslookup -type=a only-nas.test | grep -q 10.42.0.77") + kubectl("exec probe-nas -- nslookup -type=a only-nas.test | grep -q 10.42.0.77") + + +# The coordinator link flap runs at the END of 32-fleet (fix round 1), together +# with the Substrate/ax flap: a flap long enough to take the node NotReady +# can leave connections to the NAS stale (INFERRED), and the Task phases before it +# must run on a cluster that has not seen an outage. probe-coord stays up +# until then; that subtest asserts its uid survives. diff --git a/tests/ax-fleet/phases/20-substrate.py b/tests/ax-fleet/phases/20-substrate.py new file mode 100644 index 000000000..01a6e8633 --- /dev/null +++ b/tests/ax-fleet/phases/20-substrate.py @@ -0,0 +1,164 @@ +# Phase 20: Substrate (track substrate; DESIGN.md sections 12.1 phases 2 and 3, +# and section 14 S4). Concatenated after 10-cluster.py, so both switches have +# happened: the NAS runs k3s, the registry seed and the bootstrap, and the +# coordinator has joined as the tainted harness node. +# +# Uses only the machine objects `nas` and `coordinator`. Every kubectl runs on +# the NAS as root against /etc/rancher/k3s/k3s.yaml. +import json + +SUB_NS = "ate-system" +SUB_LOCAL_PATH = "/mnt/nas/services/ax-fleet/local-path/" +SUB_CONTROL_DEPLOYMENTS = [ + "ate-api-server", + "ate-controller", + "atenet-router", +] +# ate-setup installs the pod-certificate controller in its own namespace +# (manifests/ate-install/pod-certificate-controller.yaml), not in ate-system. +SUB_PODCERT_NS = "podcertificate-controller-system" + + +def sub_k(args, timeout=120): + return nas.succeed(f"KUBECONFIG=/etc/rancher/k3s/k3s.yaml kubectl {args}", timeout=timeout) + + +def sub_json(args): + return json.loads(sub_k(f"{args} -o json")) + + +with step("substrate: bootstrap steps 20-50 ran"): + nas.wait_for_unit("ax-fleet-bootstrap.service", timeout=3600) + boot_log = nas.succeed("journalctl -b -u ax-fleet-bootstrap.service --no-pager") + # Not `step`: that name is the prelude's receipt context manager, which + # 90-rollback still needs. + for step_name in ["20-registry-svc", "30-substrate", "40-gvisor-asset", "50-workerpool"]: + assert f"step {step_name}" in boot_log, f"bootstrap step {step_name} did not run" + assert "gvisor asset verified" in boot_log, "40-gvisor-asset did not verify the tarball" + stamp = sub_k( + "-n kube-system get configmap ax-fleet-substrate -o jsonpath='{.data.version}'" + ).strip() + assert stamp == "d277088b", f"install stamp version {stamp!r}" + +with step("substrate: every control workload Available, on the NAS"): + nas.wait_until_succeeds( + f"KUBECONFIG=/etc/rancher/k3s/k3s.yaml kubectl -n {SUB_NS} wait " + "--for=condition=Available deploy --all --timeout=10s", + timeout=900, + ) + names = {d["metadata"]["name"] for d in sub_json(f"-n {SUB_NS} get deploy")["items"]} + for want in SUB_CONTROL_DEPLOYMENTS: + assert want in names, f"deployment {want} missing: {sorted(names)}" + # atenet-egress is the actor egress gateway; its kind differs across + # dataplanes, so it is found by name among Deployments and StatefulSets. + sts = {s["metadata"]["name"] for s in sub_json(f"-n {SUB_NS} get statefulset")["items"]} + assert any(n.startswith("atenet-egress") for n in names | sts), "no atenet-egress" + assert "postgres" in sts, f"postgres StatefulSet missing: {sorted(sts)}" + sub_k(f"-n {SUB_NS} rollout status statefulset/postgres --timeout=600s", timeout=660) + sub_k( + f"-n {SUB_PODCERT_NS} wait --for=condition=Available deploy/podcertificate-controller --timeout=600s", + timeout=660, + ) + for pod in sub_json(f"-n {SUB_PODCERT_NS} get pods")["items"]: + assert pod["spec"].get("nodeName") == "nas", f"{pod['metadata']['name']} on {pod['spec'].get('nodeName')!r}" + for pod in sub_json(f"-n {SUB_NS} get pods")["items"]: + owner = (pod["metadata"].get("ownerReferences") or [{}])[0].get("kind", "") + name = pod["metadata"]["name"] + node = pod["spec"].get("nodeName", "") + if name.startswith("atelet") or pod["metadata"].get("labels", {}).get( + "ax.mecattaf.dev/pool" + ): + continue + if owner == "Job" and pod["status"].get("phase") == "Succeeded": + continue + assert node == "nas", f"control pod {name} landed on {node!r}" + +with step("substrate: nas keeps substrate-version=none, coordinator carries the version"): + nodes = {n["metadata"]["name"]: n for n in sub_json("get nodes")["items"]} + assert nodes["nas"]["metadata"]["labels"].get("ate.dev/substrate-version") == "none" + assert ( + nodes["coordinator"]["metadata"]["labels"].get("ate.dev/substrate-version") + == "d277088b" + ) + +with step("substrate: atelet runs on the coordinator only"): + # The atelet DaemonSet is version-keyed by ate-setup; find it by label. + ds = sub_k(f"-n {SUB_NS} get ds -l app=atelet -o jsonpath='{{.items[0].metadata.name}}'").strip() + sub_k(f"-n {SUB_NS} rollout status ds/{ds} --timeout=600s", timeout=660) + atelet = [ + p + for p in sub_json(f"-n {SUB_NS} get pods")["items"] + if p["metadata"]["name"].startswith("atelet") + ] + assert atelet, "no atelet pod" + for p in atelet: + assert p["spec"]["nodeName"] == "coordinator", ( + f"atelet on {p['spec']['nodeName']}" + ) + +with step("substrate: ClusterTrustBundles and pod certificates served"): + res = sub_k("get --raw /apis/certificates.k8s.io/v1beta1") + assert "clustertrustbundles" in res and "podcertificaterequests" in res, res + assert sub_json("get clustertrustbundles")["items"], "no ClusterTrustBundle objects" + +with step("substrate: WorkerPool ateom-gvisor Ready 2 on the coordinator"): + nas.wait_until_succeeds( + "test \"$(KUBECONFIG=/etc/rancher/k3s/k3s.yaml kubectl -n ate-system get " + "workerpool ateom-gvisor -o jsonpath='{.status.readyReplicas}')\" = 2", + timeout=900, + ) + wp = sub_json(f"-n {SUB_NS} get workerpool ateom-gvisor") + assert "@sha256:" in wp["spec"]["workerImage"], wp["spec"]["workerImage"] + workers = [ + p + for p in sub_json(f"-n {SUB_NS} get pods -l ax.mecattaf.dev/pool=ateom-gvisor")["items"] + ] + assert len(workers) == 2, f"{len(workers)} worker pods" + for p in workers: + assert p["spec"]["nodeName"] == "coordinator", p["spec"]["nodeName"] + lims = [c.get("resources", {}).get("limits", {}).get("memory") for c in p["spec"]["containers"]] + assert any(lims), f"worker pod has no memory limit: {lims}" + # Fix round 4: the writable layer is bounded per pod too. + eph = [c.get("resources", {}).get("limits", {}).get("ephemeral-storage") for c in p["spec"]["containers"]] + assert any(eph), f"worker pod has no ephemeral-storage limit: {eph}" + record("worker_ephemeral_storage_limit", eph) + +with step("substrate: gVisor fetched through the RustFS fallback"): + # No internet in the VM: atelet's anonymous GCS open of gs://gvisor/... + # fails, then its S3 client reads the same bucket and key from the + # in-cluster RustFS (40-gvisor-asset). The prewarmer logs "Sandbox assets + # prewarmed" for gvisor-default only once the pause image and the gVisor + # tarball (sha256-checked by atelet) are both on the node + # (cmd/atelet/sandbox_prewarm.go). + nas.wait_until_succeeds( + "KUBECONFIG=/etc/rancher/k3s/k3s.yaml kubectl -n ate-system logs -l app=atelet " + "--all-containers --tail=-1 | grep 'Sandbox assets prewarmed' | grep -q gvisor-default", + timeout=900, + ) + logs = sub_k(f"-n {SUB_NS} logs -l app=atelet --all-containers --tail=-1", timeout=300) + print("\n".join(l for l in logs.splitlines() if "gvisor" in l.lower())[-4000:]) + assert "gVisor release download complete" in logs or "gvisor" in logs.lower() + +with step("substrate: every PersistentVolume on the data pool"): + pvs = sub_json("get pv")["items"] + assert pvs, "no PersistentVolumes" + for pv in pvs: + spec = pv["spec"] + path = (spec.get("hostPath") or spec.get("local") or {}).get("path", "") + assert path.startswith(SUB_LOCAL_PATH), (pv["metadata"]["name"], path) + node_terms = ( + spec.get("nodeAffinity", {}).get("required", {}).get("nodeSelectorTerms", []) + ) + values = [ + v + for t in node_terms + for e in t.get("matchExpressions", []) + for v in e.get("values", []) + ] + assert "coordinator" not in values, f"PV {pv['metadata']['name']} on the coordinator" + +with step("substrate: every image came from the NAS registry by digest"): + pods = sub_json(f"-n {SUB_NS} get pods")["items"] + for p in pods: + for c in p["spec"].get("containers", []) + p["spec"].get("initContainers", []): + assert "@sha256:" in c["image"], f"{p['metadata']['name']}: {c['image']}" diff --git a/tests/ax-fleet/phases/30-nop1.py b/tests/ax-fleet/phases/30-nop1.py new file mode 100644 index 000000000..b8f08594d --- /dev/null +++ b/tests/ax-fleet/phases/30-nop1.py @@ -0,0 +1,496 @@ +# Phase 30: Tasks on stock ax v0.3.0, zero carried patches (ported from +# probe/ax-fleet-nop1 and probe/ax-fleet-zeropatch; the controller runs +# without --running-resync). Instead of the controller learning +# that a command exited, each Task's command reports its own completion to a +# stand-in floor (POST /floor/complete on the worker's stub, the one allowlisted +# egress target), and this driver, standing in for the link, reads the floor +# and then DELETES the Task through the stock ax client. Measured here: +# - the Task's phase just before the delete (stock ax: expected Running); +# - T5 shape without P1: 6 Tasks in a row, then 2 rounds of 2 at once, on the +# 2-worker pool; none ResourceExhausted, every result intact; +# - the delete frees the worker (the next Task gets one) and is idempotent; +# - leaks after delete: ax Tasks, Substrate actors and templates, PVs, the +# RustFS volume; +# - control: without the delete the 3rd Task is refused (the P1 failure); +# - side: SuspendTask instead of delete also frees a worker. +# Everything is recorded with record(); assertions are only the acceptance +# items above. +import base64 +import hashlib +import json +import re +from typing import Any + +AX = "AX_SERVER=http://127.0.0.1:8099 ax -a fleet" # ax-server-proxy.socket (fix round 3 moved it off :8080) +ATE = "KUBECONFIG=/etc/rancher/k3s/k3s.yaml kubectl-ate" +STUB = "http://10.42.0.5:8731" +NS = "fleet" +TASK_TIMEOUT = 900 # test parameter: per-Task wait for a floor report +FAILED_FINAL = 20 # test parameter: stock ax does not requeue a failed reconcile (MEASURED run 3: Failed held 5 min) +MAX_ATTEMPTS = 4 # the link's requeue: a Task whose resume failed is deleted and created again as attempt n+1 + +TASK_SCRIPT = r""" +set -u +nonce="$(date +%s%N)-$$-$RANDOM" +started="$(date +%s.%N)" +ax-agent halogen-smoke >/tmp/agent.out 2>&1 +rc=$? +out="${AX_RESULT_PATH:-.ax/result.json}" +b64="" +sha="" +if [ -f "$out" ]; then + b64="$(base64 -w0 "$out")" + sha="$(sha256sum "$out" | cut -d' ' -f1)" +fi +body="$(jq -nc --arg task "$FLOOR_TASK" --arg nonce "$nonce" --arg started "$started" \ + --arg pwd "$PWD" --argjson rc "$rc" --arg b64 "$b64" --arg sha "$sha" \ + --arg kernel "$(cat /proc/version 2>/dev/null)" \ + '{task:$task, nonce:$nonce, started:$started, pwd:$pwd, rc:$rc, result_b64:$b64, result_sha256:$sha, kernel:$kernel}')" +for i in 1 2 3 4 5; do + curl -sf --max-time 20 -H 'content-type: application/json' -d "$body" "$FLOOR_URL/floor/complete" >/dev/null && break + sleep 2 +done +exit "$rc" +""" +TASK_SCRIPT_B64 = base64.b64encode(TASK_SCRIPT.encode()).decode() + + +def ax(args: str) -> tuple[int, str]: + rc, out = coordinator.execute(f"{AX} {args} 2>&1") + return rc, out.strip() + + +def apply_task(name: str) -> None: + task: dict[str, Any] = { + "apiVersion": "ax.io/v1alpha1", + "kind": "Task", + "metadata": {"name": name, "atespace": NS}, + "spec": { + "image": IMAGE, + # One line: stock v0.3.0's runner refuses AX_TASK_YAML when the + # command carries a multi-line string (MEASURED run 2: "yaml: line + # 30: mapping values are not allowed in this context"). + "command": ["bash", "-c", f"echo {TASK_SCRIPT_B64} | base64 -d | bash"], + "env": [ + {"name": "HALOGEN_URL", "value": STUB}, + {"name": "FLOOR_URL", "value": STUB}, + {"name": "FLOOR_TASK", "value": name}, + ], + "gateway": {"name": "halogen"}, + }, + } + b = base64.b64encode(json.dumps(task).encode()).decode() + coordinator.succeed(f"echo {b} | base64 -d | {AX} apply -f -") + + +def task_state(name: str) -> Any: + """(exists, phase, ready_reason, ready_message, raw) from `ax get task`.""" + rc, out = ax(f"get task {name}") + if rc != 0: + return {"exists": False, "rc": rc, "out": out[-300:]} + m = re.search(r"^\s+phase:\s*(\S+)", out, re.M) + phase = m.group(1).strip("\"'") if m else None + ready: Any = None + for block in re.split(r"\n\s*- ", out): + if re.search(r"type:\s*\"?Ready\"?", block): + r = re.search(r"reason:\s*(.+)", block) + msg = re.search(r"message:\s*(.+)", block) + ready = { + "reason": r.group(1).strip().strip("\"'") if r else None, + "message": msg.group(1).strip().strip("\"'")[:300] if msg else None, + } + return {"exists": True, "phase": phase, "ready": ready} + + +def floor_reports(name: Any = None) -> Any: + out = worker.succeed("curl -sf http://127.0.0.1:8731/floor/results") + reps = json.loads(out)["reports"] + if name is None: + return reps + return [r for r in reps if r["report"].get("task") == name] + + +def ate_json(args: str) -> Any: + rc, out = nas.execute(f"{ATE} {args} -o json 2>&1") + if rc != 0: + return {"rc": rc, "error": out.strip()[-400:]} + try: + return json.loads(out) + except json.JSONDecodeError: + return {"rc": rc, "unparsed": out.strip()[-400:]} + + +def actors() -> Any: + """Actors in the fleet atespace: [{name, state, worker}], or the error.""" + j = ate_json(f"get actors --atespace {NS}") + if isinstance(j, dict) and ("error" in j or "unparsed" in j): + return j + items = j if isinstance(j, list) else (j.get("actors") or j.get("items") or []) + res: list[Any] = [] + for a in items: + st = a.get("status") or {} + res.append( + { + "name": (a.get("metadata") or {}).get("name"), + "state": st.get("state") or st.get("phase"), + "worker": ((st.get("workerAssignment") or {}).get("worker") or {}).get("name"), + } + ) + return res + + +def templates() -> Any: + j = ate_json(f"get actor-template --atespace {NS}") + if isinstance(j, dict) and ("error" in j or "unparsed" in j): + return j + items = j if isinstance(j, list) else (j.get("actorTemplates") or j.get("items") or []) + return sorted((t.get("metadata") or {}).get("name") for t in items) + + +def rustfs_volume() -> Any: + pvs = json.loads(kubectl("get pv -o json"))["items"] + paths: list[str] = [] + for pv in pvs: + claim = (pv["spec"].get("claimRef") or {}).get("name", "") + path = (pv["spec"].get("local") or {}).get("path") or (pv["spec"].get("hostPath") or {}).get("path") + if "rustfs" in claim and path: + paths.append(path) + res: dict[str, Any] = {"pv_count": len(pvs), "rustfs_paths": paths} + for p in paths: + res[p] = nas.succeed(f"echo $(find {p} -type f | wc -l) $(du -sb {p} | cut -f1)").strip() + return res + + +def leak_snapshot(tag: str) -> Any: + snap: dict[str, Any] = { + "ax_tasks": ax("get tasks")[1], + "actors": actors(), + "templates": templates(), + "volumes": rustfs_volume(), + } + record(f"nop1_leaks_{tag}", snap) + return snap + + +DIAGNOSED: list[str] = [] + + +def diagnose(name: str) -> None: + """Once per Task: the logs that say why a resume failed.""" + if name in DIAGNOSED: + return + DIAGNOSED.append(name) + _, wp = nas.execute("k3s kubectl -n ate-system logs -l ax.mecattaf.dev/pool=ateom-gvisor --all-containers --tail=120 2>&1") + _, atelet = nas.execute("k3s kubectl -n ate-system logs -l app=atelet --all-containers --tail=80 2>&1") + _, ctl = nas.execute("k3s kubectl -n ax-system logs deploy/ax-controller --tail=60 2>&1") + _, pods = nas.execute("k3s kubectl -n ate-system get pods -o wide 2>&1; k3s kubectl -n ate-system logs deploy/ate-api-server --all-containers --tail=40 2>&1") + _, stub = worker.execute("tail -n 20 /var/lib/halogen-stub/requests.jsonl 2>&1") + record(f"nop1_diag_{name}", {"workers": wp[-8000:], "atelet": atelet[-6000:], "controller": ctl[-5000:], "stub": stub[-3000:], "ate_pods_api": pods[-6000:]}) + + +def wait_report(name: str) -> Any: + """Poll the floor for NAME's report. Stock ax marks a Task Failed on a + reconcile error and requeues it, so Failed is final only when it says + ResourceExhausted or has lasted FAILED_FINAL seconds.""" + t0 = time.monotonic() + failed_since: Any = None + states: list[Any] = [] + while time.monotonic() - t0 < TASK_TIMEOUT: + reps = floor_reports(name) + if reps: + return reps, task_state(name), round(time.monotonic() - t0, 1) + st = task_state(name) + key = (st.get("phase"), (st.get("ready") or {}).get("reason")) + if not states or states[-1]["key"] != list(key): + states.append({"t": round(time.monotonic() - t0, 1), "key": list(key)}) + record(f"nop1_states_{name}", states) + if st.get("phase") == "Failed": + diagnose(name) + failed_since = failed_since or time.monotonic() + msg = json.dumps(st) + if "ResourceExhausted" in msg or "no free workers" in msg or time.monotonic() - failed_since > FAILED_FINAL: + return [], st, round(time.monotonic() - t0, 1) + else: + failed_since = None + time.sleep(2) + diagnose(name) + return [], task_state(name), round(time.monotonic() - t0, 1) + + +def check_report(name: str, reps: Any) -> Any: + rep = reps[0]["report"] + raw = base64.b64decode(rep["result_b64"]) if rep.get("result_b64") else b"" + try: + result: Any = json.loads(raw) if raw else {} + except json.JSONDecodeError: + result = {} + intact = ( + rep.get("rc") == 0 + and raw != b"" + and hashlib.sha256(raw).hexdigest() == rep.get("result_sha256") + and result.get("ok") is True + and result.get("content") == "halogen-stub-ok" + ) + return { + "reports": len(reps), + "distinct_nonces": len({r["report"]["nonce"] for r in reps}), + "rc": rep["rc"], + "sha256": rep["result_sha256"], + "intact": intact, + "src": reps[0]["src"], + "gvisor": "gvisor" in rep.get("kernel", ""), + "kernel": rep.get("kernel", ""), + } + + +def delete_hang_diag(name: str, calls: Any) -> None: + """A delete that did not finish: what ax, the controller, ateapi and atelet + say, so a stuck Terminating Task is diagnosable (run 2 lost all of it).""" + diag: dict[str, Any] = {"calls": calls} + diag["task"] = ax(f"get task {name}")[1][-1500:] + diag["actors"] = actors() + diag["templates"] = templates() + _, diag["pods"] = nas.execute("k3s kubectl get pods -A -o wide 2>&1") + _, diag["controller"] = nas.execute("k3s kubectl -n ax-system logs deploy/ax-controller --since=30m 2>&1 | tail -n 150") + _, diag["controller_prev"] = nas.execute("k3s kubectl -n ax-system logs deploy/ax-controller --previous --tail=60 2>&1") + _, diag["ateapi"] = nas.execute( + "for p in $(k3s kubectl -n ate-system get pods -o name | grep ate-api-server); do echo \"== $p\"; " + f"k3s kubectl -n ate-system logs $p --all-containers --since=30m 2>&1 | grep -a -i -E '{name}|error|warn|terminate' | tail -n 80; done" + ) + _, diag["ate_controller"] = nas.execute("k3s kubectl -n ate-system logs deploy/ate-controller --since=30m 2>&1 | tail -n 60") + _, diag["atelet"] = nas.execute( + f"k3s kubectl -n ate-system logs -l app=atelet --all-containers --since=30m 2>&1 | grep -a -i -E '{name}|Terminate|error' | tail -n 80" + ) + _, diag["redis"] = nas.execute( + "k3s kubectl -n ax-system exec deploy/ax-redis -- sh -c " + "'redis-cli XINFO GROUPS ax:stream:tasks; redis-cli XINFO CONSUMERS ax:stream:tasks ax-controllers; " + "redis-cli XPENDING ax:stream:tasks ax-controllers; redis-cli XREVRANGE ax:stream:tasks + - COUNT 8' 2>&1" + ) + _, diag["host_load"] = coordinator.execute("cat /proc/loadavg; cat /proc/pressure/cpu 2>/dev/null") + diag = {k: (v[-8000:] if isinstance(v, str) else v) for k, v in diag.items()} + record(f"nop1_delete_hang_{name}", diag) + + +def delete_task(name: str) -> Any: + """Delete as the link would; measure idempotence and the Task's removal.""" + t0 = time.monotonic() + first = ax(f"delete task {name}") + first_s = round(time.monotonic() - t0, 1) + calls: dict[str, Any] = {"first": {"rc": first[0], "out": first[1][-300:], "done_at_s": first_s}} + if first[0] != 0 or first_s > 30: + # The client's own wait (5 min) did not see NotFound: capture now, while stuck. + delete_hang_diag(name, calls) + again = ax(f"delete task {name}") # while Terminating + calls["while_terminating"] = {"rc": again[0], "out": again[1][-300:], "done_at_s": round(time.monotonic() - t0, 1)} + try: + coordinator.wait_until_succeeds(f"! {AX} get task {name} >/dev/null 2>&1", timeout=300) + except Exception: + delete_hang_diag(name, calls) + raise + gone_s = round(time.monotonic() - t0, 1) + after = ax(f"delete task {name}") # once gone + return { + **calls, + "after_gone": {"rc": after[0], "out": after[1][-200:]}, + "gone_seconds": gone_s, + } + + +def run_one(name: str) -> Any: + """One logical job: attempts name-a1.. until the floor has a report.""" + failed: list[Any] = [] + entry: dict[str, Any] = {} + for attempt in range(1, MAX_ATTEMPTS + 1): + n = f"{name}-a{attempt}" + apply_task(n) + reps, before, secs = wait_report(n) + entry = {"task": n, "attempt": attempt, "report_seconds": secs, "before_delete": before} + if reps: + entry["floor"] = check_report(n, reps) + assert entry["floor"]["gvisor"], reps + entry["actors_before_delete"] = actors() + entry["delete"] = delete_task(n) + entry["actors_after_delete"] = actors() + if reps or resource_exhausted(entry): + break + failed.append({"task": n, "state": before, "delete_rc": entry["delete"]["first"]["rc"]}) + entry["failed_attempts"] = failed + return entry + + +def resource_exhausted(entry: Any) -> bool: + msg = json.dumps(entry.get("before_delete", {})) + return "ResourceExhausted" in msg or "no free workers" in msg + + +with step("nop1: stock ax control plane up (no P1, no --running-resync)"): + for d in ["ax-redis", "ax-server", "ax-controller"]: + kubectl(f"-n ax-system rollout status deploy/{d} --timeout=600s") + coordinator.wait_until_succeeds("curl -sf http://127.0.0.1:8099/healthz", timeout=300) + args = kubectl("-n ax-system get deploy ax-controller -o jsonpath='{.spec.template.spec.containers[0].args}'") + record("nop1_controller_args", args) + assert "running-resync" not in args, args + rc, out = ax("result task nothing") + record("nop1_ax_result_verb", {"rc": rc, "out": out[-200:]}) + IMAGE = coordinator.succeed("ax-fleet-image-ref").strip() + record("nop1_image", IMAGE) + gw: dict[str, Any] = { + "apiVersion": "ax.io/v1alpha1", + "kind": "Gateway", + "metadata": {"name": "halogen", "atespace": NS}, + "spec": {"egress": {"allowlist": {"hosts": [{"host": "10.42.0.5/32", "port": 8731}]}}}, + } + coordinator.succeed(f"echo {base64.b64encode(json.dumps(gw).encode()).decode()} | base64 -d | {AX} apply -f -") + worker.succeed("curl -sf http://127.0.0.1:8731/floor/results") + record("nop1_workers_baseline", ate_json("get workers")) + leak_snapshot("baseline") + +with step("zeropatch: the running ax is stock, it refuses spec.sandboxClass"): + # Negative control ported from probe/ax-fleet-nosc: with sandbox-class.patch + # removed, strict protojson decode must reject the unknown field. + ctl_task: dict[str, Any] = { + "apiVersion": "ax.io/v1alpha1", + "kind": "Task", + "metadata": {"name": "nosc-control", "atespace": NS}, + "spec": {"image": IMAGE, "sandboxClass": "gvisor", "command": ["true"]}, + } + nosc_rc, nosc_out = coordinator.execute( + f"echo {base64.b64encode(json.dumps(ctl_task).encode()).decode()} | base64 -d | {AX} apply -f - 2>&1" + ) + record("nosc_sandboxclass_refused", {"rc": nosc_rc, "out": nosc_out[-2000:]}) + coordinator.execute(f"{AX} delete task nosc-control 2>&1 || true") + assert nosc_rc != 0, nosc_out + + +with step("nop1 shape check: the P1 runs' command form, [ax-agent, halogen-smoke] (recorded, not asserted)"): + shape: dict[str, Any] = {"name": "nop1-shape"} + shape_task: dict[str, Any] = { + "apiVersion": "ax.io/v1alpha1", + "kind": "Task", + "metadata": {"name": "nop1-shape", "atespace": NS}, + "spec": { + "image": IMAGE, + "command": ["ax-agent", "halogen-smoke"], + "env": [{"name": "HALOGEN_URL", "value": STUB}], + "gateway": {"name": "halogen"}, + }, + } + coordinator.succeed(f"echo {base64.b64encode(json.dumps(shape_task).encode()).decode()} | base64 -d | {AX} apply -f -") + shape_states: list[Any] = [] + t0 = time.monotonic() + while time.monotonic() - t0 < 300: + st = task_state("nop1-shape") + key = [st.get("phase"), (st.get("ready") or {}).get("reason")] + if not shape_states or shape_states[-1]["key"] != key: + shape_states.append({"t": round(time.monotonic() - t0, 1), "key": key, "ready": st.get("ready")}) + if key == ["Running", "TaskRunning"]: + break + time.sleep(2) + shape["states"] = shape_states + shape["actors"] = actors() + if shape_states[-1]["key"] != ["Running", "TaskRunning"]: + diagnose("nop1-shape") + shape["delete"] = delete_task("nop1-shape") + record("nop1_shape", shape) + +with step("nop1 T5 sequential: 6 Tasks in a row on the 2-worker pool, reported to the floor, deleted by the driver"): + seq: list[Any] = [] + for i in range(1, 7): + e = run_one(f"nop1-seq-{i}") + record(f"nop1_seq_{i}", e) + seq.append(e) + for e in seq: + assert not resource_exhausted(e), e + assert e.get("floor", {}).get("intact") is True, e + assert e["before_delete"].get("exists") is True, e + assert e["delete"]["first"]["rc"] == 0, e + record("nop1_seq_phase_before_delete", [e["before_delete"].get("phase") for e in seq]) + +with step("nop1 T5 concurrent: 2 rounds of 2 Tasks at once"): + conc: list[Any] = [] + for rnd in (1, 2): + names = [f"nop1-conc-{rnd}-{j}" for j in (1, 2)] + for n in names: + apply_task(n) + entries: list[Any] = [] + for n in names: + reps, before, secs = wait_report(n) + e: dict[str, Any] = {"task": n, "report_seconds": secs, "before_delete": before} + if reps: + e["floor"] = check_report(n, reps) + entries.append(e) + acts = actors() + for e in entries: + e["actors_before_delete"] = acts + e["delete"] = delete_task(e["task"]) + for i, e in enumerate(entries): + if "floor" not in e and not resource_exhausted(e): + # resume failed (not capacity): the link requeues it as a new attempt + retry = run_one(e["task"] + "-retry") + retry["concurrent_first_attempt"] = e + entries[i] = retry + record(f"nop1_conc_round_{rnd}", entries) + conc.extend(entries) + for e in conc: + assert not resource_exhausted(e), e + assert e.get("floor", {}).get("intact") is True, e + leak_snapshot("after_t5") + +with step("nop1 control: without the delete, the 3rd Task on 2 workers"): + ctl: dict[str, Any] = {} + for n in ("nop1-ctl-1", "nop1-ctl-2"): + apply_task(n) + for n in ("nop1-ctl-1", "nop1-ctl-2"): + reps, before, secs = wait_report(n) + ctl[n] = {"reported": bool(reps), "state": before, "seconds": secs} + # Both commands have exited and reported; stock ax still holds their workers. + time.sleep(30) + ctl["held_after_30s"] = {n: task_state(n) for n in ("nop1-ctl-1", "nop1-ctl-2")} + ctl["actors_held"] = actors() + apply_task("nop1-ctl-3") + reps, st, secs = wait_report("nop1-ctl-3") + ctl["nop1-ctl-3"] = {"reported": bool(reps), "state": st, "seconds": secs} + record("nop1_control", ctl) + for n in ("nop1-ctl-1", "nop1-ctl-2", "nop1-ctl-3"): + ctl[f"delete_{n}"] = delete_task(n) + # After deleting the holders, the pool serves again. + ctl["after"] = run_one("nop1-ctl-after") + record("nop1_control", ctl) + assert ctl["after"].get("floor", {}).get("intact") is True, ctl["after"] + +with step("nop1 side: SuspendTask instead of delete frees a worker"): + side: dict[str, Any] = {} + for n in ("nop1-sus-1", "nop1-sus-2"): + apply_task(n) + for n in ("nop1-sus-1", "nop1-sus-2"): + reps, before, secs = wait_report(n) + side[n] = {"reported": bool(reps), "state": before} + side["suspend"] = ax("suspend task nop1-sus-1") + t0 = time.monotonic() + while time.monotonic() - t0 < 300 and task_state("nop1-sus-1").get("phase") != "Suspended": + time.sleep(2) + side["suspended_state"] = task_state("nop1-sus-1") + side["suspended_seconds"] = round(time.monotonic() - t0, 1) + side["actors_after_suspend"] = actors() + apply_task("nop1-sus-3") + reps, st, secs = wait_report("nop1-sus-3") + side["nop1-sus-3"] = {"floor": check_report("nop1-sus-3", reps) if reps else None, "state": st, "seconds": secs} + side["sus1_after_60s"] = task_state("nop1-sus-1") + side["reports_sus1"] = len(floor_reports("nop1-sus-1")) + record("nop1_suspend", side) + leak_snapshot("suspend_before_delete") + for n in ("nop1-sus-1", "nop1-sus-2", "nop1-sus-3"): + side[f"delete_{n}"] = delete_task(n) + record("nop1_suspend", side) + +with step("nop1: leaks after every Task is deleted"): + time.sleep(30) + final = leak_snapshot("final") + all_reports = floor_reports() + per_task: dict[str, list[Any]] = {} + for r in all_reports: + per_task.setdefault(r["report"]["task"], []).append(r["report"]["nonce"]) + record("nop1_reports_per_task", {k: len(v) for k, v in sorted(per_task.items())}) + record("nop1_workers_final", ate_json("get workers")) + rc, out = ax("get tasks") + assert "nop1-" not in out, out diff --git a/tests/ax-fleet/phases/32-fleet.py b/tests/ax-fleet/phases/32-fleet.py new file mode 100644 index 000000000..9785239ef --- /dev/null +++ b/tests/ax-fleet/phases/32-fleet.py @@ -0,0 +1,187 @@ +# Phase 32: the bring-up's non-P1 cases on stock ax, in the floor-report shape. +# +# 25-harness-probe and 30-ax asserted P1 behaviour (a Task reaching +# Completed, `ax result`). Stock ax v0.3.0 keeps a finished Task Running, so +# every case here reports its own outcome to the stand-in floor (the worker's +# stub, as in 30-nop1) and the driver deletes the Task, as the link will. The +# checks themselves are the bring-up's: the harness probe (claude --version, +# Halogen through the Gateway, gVisor), T4 and T4b egress refusals, the API +# owner match (fix round 3), and the resilience cases with the LAN flap (fix +# round 1). +import base64 +import json +import re +import time +from typing import Any + +FLEET_REPORT_TAIL = r""" +rc=$? +b64="" +[ -f /tmp/result.json ] && b64="$(base64 -w0 /tmp/result.json)" +body="$(jq -nc --arg task "$FLOOR_TASK" --arg nonce "$(date +%s%N)-$$" --argjson rc "$rc" \ + --arg b64 "$b64" --arg kernel "$(cat /proc/version 2>/dev/null)" \ + '{task:$task, nonce:$nonce, rc:$rc, result_b64:$b64, result_sha256:"", kernel:$kernel}')" +for i in 1 2 3 4 5; do + curl -sf --max-time 20 -H 'content-type: application/json' -d "$body" "$FLOOR_URL/floor/complete" >/dev/null && break + sleep 2 +done +exit "$rc" +""" + + +def fleet_task(name: str, body: str, gateway: Any = "halogen", image: Any = None) -> None: + """Apply one Task whose BODY writes /tmp/result.json; the tail reports it.""" + script = "set -u\n(\n" + body + "\n)\n" + FLEET_REPORT_TAIL + spec: dict[str, Any] = { + "image": image or IMAGE, + "command": ["bash", "-c", f"echo {base64.b64encode(script.encode()).decode()} | base64 -d | bash"], + "env": [ + {"name": "HALOGEN_URL", "value": STUB}, + {"name": "FLOOR_URL", "value": STUB}, + {"name": "FLOOR_TASK", "value": name}, + ], + } + if gateway is not None: + spec["gateway"] = {"name": gateway} + task = {"apiVersion": "ax.io/v1alpha1", "kind": "Task", "metadata": {"name": name, "atespace": NS}, "spec": spec} + b = base64.b64encode(json.dumps(task).encode()).decode() + coordinator.succeed(f"echo {b} | base64 -d | {AX} apply -f -") + + +REAPPLY_RESUME = 3 # test parameter: re-applies of a Task whose resume failed transiently + + +def resume_failed_transient(st: Any) -> bool: + """Failed with ActorResumeFailed for a reason other than a full pool.""" + msg = json.dumps(st) + return ( + st.get("phase") == "Failed" + and (st.get("ready") or {}).get("reason") == "ActorResumeFailed" + and "ResourceExhausted" not in msg + and "no free workers" not in msg + ) + + +def fleet_run(name: str, body: str, gateway: Any = "halogen", image: Any = None, attempts: int = MAX_ATTEMPTS) -> Any: + """Attempts NAME-a1.. until the floor has a report; returns the decoded result.""" + tried: list[Any] = [] + for attempt in range(1, attempts + 1): + n = f"{name}-a{attempt}" + fleet_task(n, body, gateway, image) + reps, before, secs = wait_report(n) + reapplied: list[Any] = [] + # A resume that failed on a stale gRPC connection (ActorResumeFailed, + # Unavailable) left the actor placed but never restored: atelet wrote + # no sandbox record, and Substrate's Terminate then fails on every + # delete (MEASURED r1 run 2, cmd/atelet/main.go:1281). Deleting that + # attempt strands it in ACTOR_STATE_DELETING. Stock ax reconciles on + # every save and re-runs ResumeActor with no phase guard, so the + # recovery is to apply the same Task again, not to delete it. + while not reps and len(reapplied) < REAPPLY_RESUME and resume_failed_transient(before): + reapplied.append({"state": before, "after_s": secs}) + fleet_task(n, body, gateway, image) + reps, before, secs = wait_report(n) + entry: dict[str, Any] = {"task": n, "report_seconds": secs, "before_delete": before, "reapplied": reapplied} + if reps: + rep = reps[0]["report"] + raw = base64.b64decode(rep["result_b64"]) if rep.get("result_b64") else b"{}" + entry.update(rc=rep["rc"], kernel=rep.get("kernel", ""), result=json.loads(raw or b"{}")) + assert "gvisor" in entry["kernel"], entry + entry["delete"] = delete_task(n) + tried.append(entry) + if reps: + break + record(f"fleet_{name}", tried) + assert "result" in tried[-1], f"{name}: no floor report after {len(tried)} attempt(s)" + return tried[-1] + + +def curl_body(url: str) -> str: + return ( + f"code=$(curl -s -o /dev/null -w '%{{http_code}}' --max-time 10 {url}); crc=$?\n" + "jq -nc --arg code \"$code\" --argjson crc \"$crc\" '{http_code:$code, curl_rc:$crc}' >/tmp/result.json" + ) + + +with step("probe: a gVisor ax Task runs claude --version and reaches Halogen through the Gateway (floor report)"): + digest = coordinator.succeed(f"cat {CLAUDE_PROBE_OCI}/digest").strip() + assert digest.startswith("sha256:"), digest + stub_before = len(worker.succeed("cat /var/lib/halogen-stub/requests.jsonl 2>/dev/null || true").splitlines()) + host_kernel = coordinator.succeed("cat /proc/version").strip() + body = ( + "v=$(claude --version 2>&1); vrc=$?\n" + "m=$(curl -s --max-time 20 \"$HALOGEN_URL/v1/models\"); mrc=$?\n" + "jq -nc --arg v \"$v\" --argjson vrc \"$vrc\" --arg m \"$m\" --argjson mrc \"$mrc\" " + "'{claude_version:$v, claude_rc:$vrc, models:$m, curl_rc:$mrc}' >/tmp/result.json" + ) + p = fleet_run("probe-claude", body, image=f"localhost:5000/ax/ax-agent-claude-probe@{digest}") + res = p["result"] + assert res["claude_rc"] == 0 and re.search(r"\d+\.\d+\.\d+", res["claude_version"]), res + assert res["curl_rc"] == 0 and "halogen-qwen3.8-flash-next" in res["models"], res + assert p["kernel"] != host_kernel, (p["kernel"], host_kernel) + new = worker.succeed("cat /var/lib/halogen-stub/requests.jsonl 2>/dev/null || true").splitlines()[stub_before:] + assert any("/v1/models" in l for l in new), new + nas.fail("pgrep -f '[r]unsc'") + +with step("T4 egress-deny: a non-allowlisted target is refused by the Gateway"): + worker.succeed("curl -s -o /dev/null --max-time 10 http://10.42.0.2/") + t4 = fleet_run("t4-deny", curl_body("http://10.42.0.2/")) + assert t4["result"]["http_code"] != "200", t4 + +with step("T4b egress-deny: the Gateway's host on another port is refused (the NAS enforces the port)"): + nas.succeed("curl -sf --max-time 10 http://10.42.0.5:2222/ | grep -q worker-port-2222-reached") + t4b = fleet_run("t4b-port", curl_body("http://10.42.0.5:2222/")) + assert t4b["result"]["http_code"] != "200", t4b + +with step("security: only root and apiUsers reach the ax API and the cluster ranges from the desk"): + ax_pod = kubectl( + "-n ax-system get pods -l app.kubernetes.io/name=ax-server -o jsonpath='{.items[0].status.podIP}'" + ).strip() + coordinator.succeed("runuser -u tom -- curl -sf --max-time 10 http://127.0.0.1:8099/healthz") + coordinator.succeed("curl -sf --max-time 10 http://10.201.0.80:8080/healthz") + for url in ("http://127.0.0.1:8099/healthz", "http://10.201.0.80:8080/healthz", f"http://{ax_pod}:8080/healthz"): + coordinator.fail(f"runuser -u alice -- curl -s --max-time 5 -o /dev/null {url}") + coordinator.fail("ss -ltn | grep -q '127.0.0.1:8080 '") + record("api_owner_chain", coordinator.succeed("iptables -S ax-fleet-api").strip().splitlines()) + +with step("resilience: a restarted ax-redis keeps a held Task (AOF on the volume)"): + fleet_task("held-a1", "sleep 600; echo '{}' >/tmp/result.json") + coordinator.wait_until_succeeds(f"{AX} get task held-a1 >/dev/null", timeout=60) + before = coordinator.succeed(f"{AX} get tasks | tail -n +2 | wc -l").strip() + kubectl("-n ax-system delete pod -l app.kubernetes.io/name=ax-redis --wait=true") + kubectl("-n ax-system rollout status deploy/ax-redis --timeout=300s") + coordinator.wait_until_succeeds(f"{AX} get tasks >/dev/null", timeout=120) + after = coordinator.succeed(f"{AX} get tasks | tail -n +2 | wc -l").strip() + assert before == after and int(after) >= 1, f"tasks before={before} after={after}" + record("resilience_redis_held_delete", delete_task("held-a1")) + +with step("resilience: a restarted ax-controller, then a Task reports and is deleted"): + kubectl("-n ax-system delete pod -l app.kubernetes.io/name=ax-controller --wait=true") + kubectl("-n ax-system rollout status deploy/ax-controller --timeout=300s") + r = fleet_run("after-ctl", curl_body(f"{STUB}/v1/models")) + assert r["result"]["http_code"] == "200", r + +with step("resilience: the LAN leg is down until the coordinator is NotReady; pods keep their names and uid, a new Task reports"): + pods = lambda: kubectl( + "-n ate-system get pods --field-selector spec.nodeName=coordinator -o name | grep ateom | sort" + ) + before = pods() + probe_uid = kubectl("get pod probe-coord -o jsonpath='{.metadata.uid}'").strip() + flap_until_unreachable("ax") + nas.wait_until_succeeds( + "k3s kubectl get node coordinator -o jsonpath='{.status.conditions[?(@.type==\"Ready\")].status}' | grep -qx True", + timeout=300, + ) + nas.wait_until_succeeds( + "! k3s kubectl get node coordinator -o jsonpath='{.spec.taints[*].key}' | grep -qw node.kubernetes.io/unreachable", + timeout=300, + ) + assert pods() == before, f"worker pods changed: {before!r}" + kubectl("wait --for=condition=Ready pod/probe-coord --timeout=300s") + assert kubectl("get pod probe-coord -o jsonpath='{.metadata.uid}'").strip() == probe_uid, "the probe pod was replaced by the flap" + nas.wait_until_succeeds("curl -sf --max-time 5 http://10.42.0.2:18085/ | grep -x pod-ok", timeout=120) + # Fix round 1 MEASURED that each stale gRPC connection left by an outage + # costs one Task (ActorResumeFailed, Unavailable). The bound is a test + # parameter: the fleet must recover within it without a restart. + r = fleet_run("after-flap", curl_body(f"{STUB}/v1/models"), attempts=6) + assert r["result"]["http_code"] == "200", r diff --git a/tests/ax-fleet/phases/33-gateway-default.py b/tests/ax-fleet/phases/33-gateway-default.py new file mode 100644 index 000000000..e26eb69d1 --- /dev/null +++ b/tests/ax-fleet/phases/33-gateway-default.py @@ -0,0 +1,75 @@ +# Phase 33: egress without a gateway lands behind the atespace's default +# Gateway (modules/ax-fleet/gateways.nix). Stock ax v0.3.0 gives such a Task +# allow-all; the fleet closes it without an ax patch: the bootstrap declared a +# default Gateway (Halogen and the floor only) and ax-fleet-gateway-default +# points the Task at it. Discriminating: the NAS host reaches the public +# stand-in (TEST-NET-2 on the worker, fix round 4), and a Gateway that +# allowlists it gets its 200 through the same egress path. +import re + +PUBLIC = "http://198.51.100.5:8000/" + + +def task_gateway(name: str) -> Any: + rc, out = ax(f"get task {name}") + if rc != 0: + return None + m = re.search(r"gateway:\s*\n\s+name:\s*(\S+)", out) + return m.group(1).strip("\"'") if m else "" + + +def late_curl_body(url: str, wait: int) -> str: + # Waits WAIT seconds first: the repoint is asynchronous by design. + return f"sleep {wait}\n" + curl_body(url) + + +with step("gateways: the bootstrap declared a default Gateway in every declared atespace"): + for ns in ("fleet", "default"): + coordinator.succeed(f"AX_SERVER=http://127.0.0.1:8099 ax -a {ns} get gateway default") + nas.succeed("systemctl is-active ax-fleet-gateway-default.service") + record("gateway_default_unit", nas.succeed("systemctl show ax-fleet-gateway-default -p ExecStart --value").strip()) + +with step("gateways: a Task without a gateway, or naming a missing one, is pointed at the default and cannot reach the public stand-in"): + worker.wait_for_unit("public-8000.service") + nas.wait_until_succeeds(f"curl -sf --max-time 10 {PUBLIC} | grep -q public-reached", timeout=120) + results = {} + for label, missing_gw in (("none", None), ("missing", "no-such-gateway")): + name = f"gwdef-{label}" + n = f"{name}-a1" + t0 = time.monotonic() + fleet_task(n, late_curl_body(PUBLIC, 30), gateway=missing_gw) + coordinator.wait_until_succeeds( + f"{AX} get task {n} | grep -A1 -E '^\\s+gateway:' | grep -qE 'name:\\s*\"?default\"?'", timeout=120 + ) + repoint_s = round(time.monotonic() - t0, 1) + reps, before, secs = wait_report(n) + tries = 0 + while not reps and tries < REAPPLY_RESUME and resume_failed_transient(before): + # Same recovery as fleet_run (32-fleet): apply again, never delete a + # Task whose resume failed on a stale connection. + tries += 1 + fleet_task(n, late_curl_body(PUBLIC, 30), gateway=missing_gw) + reps, before, secs = wait_report(n) + assert reps, f"{n}: no floor report (the default Gateway allows the floor): {before}" + rep = reps[0]["report"] + res = json.loads(base64.b64decode(rep["result_b64"]) or b"{}") + results[label] = {"repoint_seconds": repoint_s, "report_seconds": secs, "result": res, "state": before} + record("gateway_default_repoint", results) + assert task_gateway(n) == "default", n + assert res.get("http_code") != "200", (label, res) + delete_task(n) + +with step("gateways: positive control, a Gateway allowlisting the public stand-in reaches it"): + coordinator.succeed( + "AX_SERVER=http://127.0.0.1:8099 ax -a fleet apply -f - <<'EOF'\n" + "apiVersion: ax.io/v1alpha1\nkind: Gateway\nmetadata:\n name: public-test\n atespace: fleet\n" + "spec:\n egress:\n allowlist:\n hosts:\n" + " - host: \"198.51.100.5/32\"\n port: 8000\n" + " - host: \"10.42.0.5/32\"\n port: 8731\n" + "EOF" + ) + ok = fleet_run("gw-public", curl_body(PUBLIC), gateway="public-test") + record("gateway_public_positive_control", ok["result"]) + assert ok["result"]["http_code"] == "200", ok + assert task_gateway("gw-public-a1") in (None, "public-test") + coordinator.execute("AX_SERVER=http://127.0.0.1:8099 ax -a fleet delete gateway public-test") diff --git a/tests/ax-fleet/phases/35-lan-guard.py b/tests/ax-fleet/phases/35-lan-guard.py new file mode 100644 index 000000000..e05010b83 --- /dev/null +++ b/tests/ax-fleet/phases/35-lan-guard.py @@ -0,0 +1,124 @@ +# Phase: the house LAN cannot reach the cluster ranges through the NAS, and +# the credential and RBAC surface is what fix round 1 left (2026-09-23). Runs +# after 30-nop1 and 32-fleet, so ax-server, ax-redis and RustFS all exist. The worker is a +# plain LAN host; the NAS is its default gateway on the real LAN. + + +def tcp_open(ip, port): + return f"timeout 5 bash -c 'exec 3<>/dev/tcp/{ip}/{port}'" + + +with step("lan: routed LAN traffic never reaches a ClusterIP or pod IP"): + def svc_ip(ns, name): + return jsonpath(f"-n {ns} svc {name}", "{.spec.clusterIP}") + + ax_ip = svc_ip("ax-system", "ax-server") + redis_ip = svc_ip("ax-system", "ax-redis") + rustfs_ip = svc_ip("ate-system", "rustfs") + ax_pod = kubectl( + "-n ax-system get pods -l app.kubernetes.io/name=ax-server -o jsonpath='{.items[0].status.podIP}'" + ).strip() + record("lan_probe_targets", {"ax": ax_ip, "redis": redis_ip, "rustfs": rustfs_ip, "ax_pod": ax_pod}) + + # Positive controls from the NAS host (OUTPUT path, not prerouting): the + # endpoints are up, so a failure from the worker is the guard. + nas.succeed(f"curl -sf --max-time 10 http://{ax_ip}:8080/healthz") + nas.succeed(tcp_open(redis_ip, 6379)) + nas.succeed(f"curl -s --max-time 10 -o /dev/null http://{rustfs_ip}:9000/") + + worker.succeed("ip route replace 10.200.0.0/16 via 10.42.0.1") + worker.succeed("ip route replace 10.201.0.0/16 via 10.42.0.1") + try: + # curl without -f: rc 0 on ANY HTTP answer, so fail() means no answer. + worker.fail(f"curl -s --max-time 5 -o /dev/null http://{ax_ip}:8080/healthz") + worker.fail(f"curl -s --max-time 5 -o /dev/null http://{ax_pod}:8080/healthz") + worker.fail(f"curl -s --max-time 5 -o /dev/null http://{rustfs_ip}:9000/") + worker.fail(tcp_open(redis_ip, 6379)) + worker.fail("dig +time=2 +tries=1 @10.201.0.10 ax-server.ax-system.svc.cluster.local") + finally: + worker.succeed("ip route del 10.200.0.0/16 via 10.42.0.1") + worker.succeed("ip route del 10.201.0.0/16 via 10.42.0.1") + record("nas_range_guard", nas.succeed("nft list chain inet ax-fleet-guard prerouting").strip().splitlines()) + + # Fix round 4: the NAS's own non-root processes (paperless, immich, ...) + # get none of it; root, above, still does. `nobody` and the dnsmasq + # stand-in's user stand in for them. + for u in ("nobody", "dnsmasq"): + nas.fail(f"runuser -u {u} -- curl -s --max-time 5 -o /dev/null http://{ax_ip}:8080/healthz") + nas.fail(f"runuser -u {u} -- curl -s --max-time 5 -o /dev/null http://{ax_pod}:8080/healthz") + nas.fail(f"runuser -u {u} -- {tcp_open(redis_ip, 6379)}") + nas.succeed(f"curl -sf --max-time 10 http://{ax_ip}:8080/healthz") + # Pods still resolve through the NAS's AdGuard stand-in (replies are not NEW). + kubectl("exec probe-nas -- sh -c 'nslookup only-nas.test 10.42.0.1 | grep -q 10.42.0.77'") + record("nas_output_guard", nas.succeed("nft list chain inet ax-fleet-guard output").strip().splitlines()) + + +with step("nas: pods reach Halogen on its port and no other private address"): + # Fix round 3. The round-3 review MEASURED postgres-0 reaching worker:2222; + # the NAS had no pod egress guard. Discriminating: the NAS host reaches + # both targets. + nas.succeed("curl -sf --max-time 10 http://10.42.0.5:2222/ | grep -q worker-port-2222-reached") + nas.succeed("curl -sf --max-time 10 http://10.42.0.2/ | grep -x caddy-ok") + kubectl("exec probe-nas -- curl -sf --max-time 10 http://10.42.0.5:8731/health") + kubectl("exec probe-nas -- sh -c '! curl -s --max-time 5 -o /dev/null http://10.42.0.5:2222/'") + kubectl("exec probe-nas -- sh -c '! curl -s --max-time 5 -o /dev/null http://10.42.0.2/'") + record("nas_pod_egress", nas.succeed("nft list chain inet ax-fleet-guard forward").strip().splitlines()) + + +with step("security: the registry is read-only to the coordinator; the seed wrote everything"): + # Fix round 2. The round-2 review MEASURED 202 for an upload and 201 for a + # tag overwrite from an unprivileged coordinator user. + reg = "http://10.42.0.1:5000" + alice = "runuser -u alice -- curl -s -o /dev/null -w '%{http_code}' --max-time 10" + assert coordinator.succeed(f"{alice} {reg}/v2/_catalog").strip() == "200" + codes = { + "upload": coordinator.succeed(f"{alice} -X POST {reg}/v2/secprobe/blobs/uploads/").strip(), + "mount": coordinator.succeed(f"{alice} -X POST '{reg}/v2/secprobe/blobs/uploads/?mount=sha256:0000000000000000000000000000000000000000000000000000000000000000&from=ax/ax-redis'").strip(), + "delete": coordinator.succeed(f"{alice} -X DELETE {reg}/v2/ax/ax-redis/manifests/sha256:0000000000000000000000000000000000000000000000000000000000000000").strip(), + } + record("registry_write_codes", codes) + assert all(c not in ("201", "202") for c in codes.values()), codes + nas.fail("ss -ltn | grep -q '127.0.0.1:5001'") # the seed's writer is gone + nas.succeed("ss -ltn | grep -q '10.42.0.1:5000'") + + +with step("security: ax-controller holds no Secret grant and no API token"): + nas.fail("k3s kubectl get clusterrole ax-controller") + nas.fail("k3s kubectl get clusterrolebinding ax-controller") + auto = jsonpath("-n ax-system deploy ax-controller", "{.spec.template.spec.automountServiceAccountToken}") + assert auto == "false", auto + vols = kubectl( + "-n ax-system get pods -l app.kubernetes.io/name=ax-controller -o jsonpath='{.items[0].spec.volumes[*].name}'" + ).split() + record("ax_controller_volumes", vols) + assert not any(v.startswith("kube-api-access") for v in vols), vols + can = nas.succeed( + "k3s kubectl auth can-i list secrets --all-namespaces --as=system:serviceaccount:ax-system:ax-controller || true" + ).strip() + assert can == "no", can + + +with step("security: RustFS and its clients read a generated credential, not the kind default"): + nas.succeed("stat -c '%a %U' /var/lib/ax-fleet/rustfs.env | grep -x '600 root'") + nas.succeed("k3s kubectl -n ate-system get secret ax-fleet-rustfs -o name") + wanted = {"RUSTFS_ACCESS_KEY", "RUSTFS_SECRET_KEY", "AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY"} + seen = [] + # The bucket-init Job uses the credential too; fix round 1 first missed it + # and it crash-looped (MEASURED), so it must complete here. + kubectl("-n ate-system wait --for=condition=complete job/rustfs-bucket-init --timeout=300s") + for obj in ("deploy rustfs", "deploy ate-api-server", "ds atelet", "job rustfs-bucket-init"): + kind, name = obj.split() + names = kubectl(f"-n ate-system get {kind} -o name").split() + target = [n for n in names if n.split("/")[-1].startswith(name)] + for t in target: + doc = json.loads(kubectl(f"-n ate-system get {t} -o json")) + for c in doc["spec"]["template"]["spec"]["containers"]: + for e in c.get("env", []): + if e["name"] in wanted: + assert "value" not in e, f"{t} {e['name']} carries a literal value" + assert e["valueFrom"]["secretKeyRef"]["name"] == "ax-fleet-rustfs", (t, e["name"]) + seen.append(f"{t}:{e['name']}") + record("rustfs_credential_refs", sorted(seen)) + assert any(s.endswith("RUSTFS_SECRET_KEY") for s in seen), seen + assert any(s.endswith("AWS_SECRET_ACCESS_KEY") for s in seen), seen + assert any(s.startswith("job") and s.endswith("AWS_SECRET_ACCESS_KEY") for s in seen), seen diff --git a/tests/ax-fleet/phases/90-rollback.py b/tests/ax-fleet/phases/90-rollback.py new file mode 100644 index 000000000..55a045755 --- /dev/null +++ b/tests/ax-fleet/phases/90-rollback.py @@ -0,0 +1,187 @@ +# Phase 6: rollback, the kill switch and the generation rollback proven +# (DESIGN.md 12.1, 13). Track cluster. Fix round 4: the base is "today" +# (pre-ax, no modules/ax-fleet), so every path below lands where Tom would. +# coordinator (a) ax-fleet-teardown inside the ax-on generation: the guards +# are re-applied after k3s-killall.sh strips every flannel rule, +# and a k3s that starts again runs behind them; +# (b) the kill switch (ax-off: role declared, enable false), then +# the teardown from the host's PATH; +# (c) the generation rollback to the pre-ax base, then the +# flake's teardown (the pre-ax PATH has none). +# nas the generation rollback straight from ax-on, pods still +# running, then the flake's teardown. + +AX_OFF = "/run/booted-system/specialisation/ax-off/bin/switch-to-configuration test" +PRE_AX = "/run/booted-system/bin/switch-to-configuration test" +LEFTOVER_RULES = "iptables-save 2>/dev/null | grep -E 'KUBE-|FLANNEL|CNI-'" +POD_NETNS_PID = ( + "host=$(readlink /proc/1/ns/net); for p in $(pgrep -x pause); do " + "[ \"$(readlink /proc/$p/ns/net)\" != \"$host\" ] && { echo $p; break; }; done" +) +EXPECTED_GUARDS = {"pod_input": 2, "pod_input_flannel_v4": 1, "pod_input_flannel_v6": 1, "vxlan_source": 1, "guard_flannel": 5} + + +def pod_netns_pid(machine): + # A pod sandbox's pause process in a network namespace other than the host's. + return machine.succeed(POD_NETNS_PID).strip() + + +def rule_count(machine, cmd): + return int(machine.succeed(f"{cmd} || true").strip() or "0") + + +def guards(machine): + return { + "pod_input": rule_count(machine, "iptables -S nixos-fw | grep -c ax-fleet-pod-input"), + "pod_input_flannel_v4": rule_count(machine, "iptables -S nixos-fw | grep -c 'flannel.1.*ax-fleet-pod-input'"), + "pod_input_flannel_v6": rule_count(machine, "ip6tables -S nixos-fw | grep -c 'flannel.1.*ax-fleet-pod-input'"), + # iptables -S prints the source before the interface (MEASURED run 2). + "vxlan_source": rule_count(machine, "iptables -t mangle -S ax-fleet-guard | grep -cE -- '! -s [0-9./]+ -i flannel.1 -j DROP'"), + "guard_flannel": rule_count(machine, "iptables -t mangle -S ax-fleet-guard | grep -c flannel.1"), + } + + +with step("rollback coordinator (a): the teardown inside ax-on re-applies the guards; k3s restarts behind them"): + # Fix round 4. k3s-killall.sh runs `iptables-save | grep -iv flannel | + # iptables-restore`, which deleted every guard rule naming flannel.1 while + # the teardown reported them "left in place" (MEASURED by the review). + coordinator.succeed("ax-fleet-teardown >&2") + coordinator.fail("ip link show flannel.1") + g = guards(coordinator) + record("guards_after_teardown_in_ax_on", g) + assert g == EXPECTED_GUARDS, g + # "It starts again at the next boot or switch": no firewall reload first. + coordinator.succeed("systemctl start k3s.service") + node_ready("coordinator") + coordinator.wait_until_succeeds("ip link show flannel.1", timeout=300) + kubectl("wait --for=condition=Ready pod/probe-coord --timeout=600s") + g = guards(coordinator) + record("guards_after_k3s_restart", g) + assert g == EXPECTED_GUARDS, g # re-applied once, never duplicated + # Discriminating: the NAS's pod reaches a coordinator pod over VXLAN, and + # the coordinator's sshd listens on its flannel.1 address, yet the pod + # gets no SSH banner from it. + # The teardown killed the sandboxes; `kubectl wait` above can pass on the + # pre-teardown status (MEASURED r1 run 3: 0.24 s), so podIP may still name + # the old sandbox for a while. Read it on every try, never once. + record("probe_coord_ip_stale_read", jsonpath("pod probe-coord", "{.status.podIP}")) + try: + nas.wait_until_succeeds( + "ip=$(k3s kubectl get pod probe-coord -o jsonpath='{.status.podIP}') && " + "k3s kubectl exec probe-nas -- curl -sf --max-time 5 http://$ip:8000/ | grep -x pod-ok", + timeout=300, + ) + except Exception: + record("rollback_a_pod_path_diag", { + "pod": nas.execute("k3s kubectl get pod probe-coord -o wide 2>&1")[1], + "curl": nas.execute("ip=$(k3s kubectl get pod probe-coord -o jsonpath='{.status.podIP}'); " + "k3s kubectl exec probe-nas -- curl -sv --max-time 5 http://$ip:8000/ 2>&1 | tail -5")[1], + "coord_cni0": coordinator.execute("ip -4 -o addr show dev cni0; ip neigh show dev cni0 2>&1")[1], + "coord_forward": coordinator.execute("sysctl -n net.ipv4.ip_forward; iptables -S FORWARD 2>&1 | head -30")[1], + "coord_flannel": coordinator.execute("ip -d link show flannel.1 2>&1 | head -3; ip route 2>&1")[1], + "nas_fdb": nas.execute("bridge fdb show dev flannel.1 2>&1; ip neigh show dev flannel.1 2>&1")[1], + "coord_listen": coordinator.execute("ss -ltnp 2>&1 | grep -w 8000 || true")[1], + }) + raise + record("probe_coord_ip_after_k3s_restart", jsonpath("pod probe-coord", "{.status.podIP}")) + fl = coordinator.succeed("ip -4 -o addr show dev flannel.1 | awk '{print $4}' | cut -d/ -f1").strip() + coordinator.succeed(f"timeout 10 bash -c 'exec 3<>/dev/tcp/{fl}/22'") + kubectl(f"exec probe-nas -- sh -c '! (nc -w 5 {fl} 22 /dev/null | grep -q SSH)'") + record("ip_forward_after_k3s_restart", coordinator.succeed("sysctl -n net.ipv4.ip_forward").strip()) + + +with step("rollback coordinator (b): the kill switch, then the teardown from PATH"): + coordinator.succeed(f"{AX_OFF} >&2") + coordinator.fail("systemctl is-active k3s.service") + # Fix round 3: between the kill switch and the teardown the pods still + # run (KillMode=process). The role-scoped guards keep them off the host. + pid = pod_netns_pid(coordinator) + assert pid, "no pod network namespace survived the kill switch" + coordinator.succeed("timeout 10 bash -c 'exec 3<>/dev/tcp/10.42.0.2/22'") # sshd is up + coordinator.fail(f"nsenter -t {pid} -n timeout 5 bash -c 'exec 3<>/dev/tcp/10.42.0.2/22'") + coordinator.fail(f"nsenter -t {pid} -n timeout 5 bash -c 'exec 3<>/dev/tcp/100.105.121.73/22'") + coordinator.fail(f"nsenter -t {pid} -n timeout 5 bash -c 'exec 3<>/dev/tcp/10.42.0.5/8731'") + assert guards(coordinator) == EXPECTED_GUARDS, guards(coordinator) + coordinator.succeed("iptables -t mangle -S FORWARD 1 | grep -q ax-fleet-guard") + # As documented (fix round 2): the teardown from the host's own PATH. + coordinator.succeed("test -x /run/current-system/sw/bin/ax-fleet-teardown") + coordinator.succeed("ax-fleet-teardown >&2") + coordinator.fail("ip link show cni0") + coordinator.fail("ip link show flannel.1") + coordinator.fail(LEFTOVER_RULES) + coordinator.fail("pgrep -f containerd-shim") + # The guards belong to the harness role's every generation (fix round 3), + # re-applied whole after the killall (fix round 4). + g = guards(coordinator) + record("guards_after_kill_switch_teardown", g) + assert g == EXPECTED_GUARDS, g + coordinator.succeed("iptables -S OUTPUT 1 | grep -q ax-fleet-api") + + +with step("rollback coordinator (c): the generation rollback to the pre-ax base, then the flake's teardown"): + coordinator.succeed(f"{PRE_AX} >&2") + coordinator.fail("test -e /etc/ax-fleet/guard-declared") + # The pre-ax PATH has no teardown; Tom runs the flake's (DESIGN 13). + coordinator.fail("command -v ax-fleet-teardown") + record("coordinator_pre_ax_stale", { + "mangle_guard": rule_count(coordinator, "iptables -t mangle -S | grep -c ax-fleet-guard"), + "api_chain": rule_count(coordinator, "iptables -S | grep -c ax-fleet-api"), + "pod_input": rule_count(coordinator, "iptables -S nixos-fw | grep -c ax-fleet-pod-input"), + }) + coordinator.succeed(f"{TEARDOWN} >&2") + coordinator.fail("iptables -t mangle -S ax-fleet-guard") + coordinator.fail("iptables -S ax-fleet-api") + coordinator.fail("iptables-save | grep -q ax-fleet") + coordinator.fail("ip6tables-save | grep -q ax-fleet") + after = sysctls(coordinator) + record("sysctl_coordinator_after_rollback", after) + assert after == base["sysctl_coordinator"], (after, base["sysctl_coordinator"]) + assert user_unit_pid("herdr-standin") == base["herdr_pid"], "herdr stand-in restarted" + assert nm_invocation() == base["nm_invocation"], "NetworkManager restarted" + coordinator.fail("test -e /etc/NetworkManager/conf.d/90-ax-fleet.conf") + worker.succeed("curl -sf --max-time 10 http://10.42.0.2/ | grep -x caddy-ok") + peer.succeed("curl -sf --max-time 10 http://100.105.121.73/ | grep -x caddy-ok") + + +with step("rollback nas: the generation rollback straight from ax-on, then the flake's teardown"): + # Diagnostic only: which processes hold the kubelet bind before the switch. + _, holders = nas.execute("ls -l /proc/[0-9]*/cwd /proc/[0-9]*/root 2>/dev/null | grep -c /var/lib/kubelet") + record("nas_kubelet_holders_before_rollback", holders.strip()) + nas.succeed(f"{PRE_AX} >&2") + nas.fail("systemctl is-active k3s.service") + nas.fail("systemctl is-active docker-registry.service") + nas.fail("test -e /etc/ax-fleet/guard-declared") + nas.fail("command -v ax-fleet-teardown") + # The window DESIGN 13 documents: pods outlive the switch until the + # teardown. Recorded, not asserted: which guard (if any) still holds them. + pid = pod_netns_pid(nas) + window = {"pod_survived": bool(pid)} + if pid: + rc, _ = nas.execute(f"nsenter -t {pid} -n timeout 5 bash -c 'exec 3<>/dev/tcp/10.42.0.5/2222'") + window["pod_reaches_worker_2222"] = rc == 0 + window["guard_table_present"] = nas.execute("nft list table inet ax-fleet-guard")[0] == 0 + record("nas_generation_rollback_window", window) + nas.succeed("command -v tailscale") # the stub is reachable from a root shell + nas.succeed(f"{TEARDOWN} >&2") + # The teardown's pinned PATH keeps k3s-killall.sh away from tailscale. + nas.fail("test -e /var/log/tailscale-stub.log") + nas.fail("ip link show cni0") + nas.fail("ip link show flannel.1") + nas.fail("pgrep -f containerd-shim") + ruleset = nas.succeed("nft -s list ruleset") + for marker in ("KUBE-", "FLANNEL", "CNI-", "ax-fleet"): + assert marker not in ruleset, f"{marker} left in the NAS ruleset" + nixos_fw = nas.succeed("nft -s list table inet nixos-fw") + assert nixos_fw == base["nas_nixos_fw"], "the NAS firewall table differs from the baseline" + record("nas_ruleset_equal_baseline", ruleset == base["nas_nft"]) + if ruleset != base["nas_nft"]: + record("nas_ruleset_after_rollback", ruleset) + after = sysctls(nas) + record("sysctl_nas_after_rollback", after) + assert after == base["sysctl_nas"], (after, base["sysctl_nas"]) + assert unit_invocation(nas, "postgresql.service") == base["postgres_invocation"], "the shared PostgreSQL restarted" + dbs = sorted(nas.succeed("runuser -u postgres -- psql -Atc 'select datname from pg_database'").split()) + assert dbs == base["nas_databases"], dbs + # Left on disk on purpose; deleting it is Tom's call. + nas.succeed("test -d /mnt/fast/k3s/rancher/k3s") + nas.fail("findmnt /var/lib/rancher")