diff --git a/docs/ax-conwip.md b/docs/ax-conwip.md index e4be4e41..7e57e3ad 100644 --- a/docs/ax-conwip.md +++ b/docs/ax-conwip.md @@ -7,6 +7,13 @@ Module: `home/ax-conwip.nix`. Option namespace: `myAxConwip`. Check: this module defines no unit on any host today.** The flake check asserts exactly that, and it is written so that it goes red on the flip. Flipping it is Tom's. +**2026-09-24 note.** The repository this page describes was renamed substrate +(E10). Its box-side programs are not run through this module: the capacity +pusher and the interpreter-host puller are the user units `substrate-pusher` +and `substrate-puller` from `modules/substrate.nix`, declared ON on the +coordinator as of 2026-09-24. They replace the hand-started nohup processes of +`~/today/wednesday-prep-2026-09-23/substrate` (RUN.md), with the same configs. + ## What the CONWIP is `/home/tom/mecattaf/ax-conwip` is a CONWIP scheduler. CONWIP is constant diff --git a/flake.nix b/flake.nix index b385a43c..991a093e 100644 --- a/flake.nix +++ b/flake.nix @@ -659,6 +659,9 @@ let amdAi = inputs.nix-amd-ai.packages.${system}; strixAi = inputs.nix-strix-halo.packages.${system}; + # The Cloudflare Substrate's box-side programs, vendored from + # agency-agency/substrate at the sha in pkgs/substrate-apps/SYNC.md. + substrateApps = pkgs.callPackage ./pkgs/substrate-apps { }; in { inherit (pkgs) @@ -725,6 +728,14 @@ go_1_27 = inputs.nixpkgs-go.legacyPackages.${system}.go_1_27; }; }; + # `nix build .#substrate-pusher`, `.#substrate-puller`, `.#substrate-apps-link`: + # the coordinator's gentle capacity pusher, its interpreter-host puller and + # the floor-to-ax link from the same pinned + # substrate sha (pkgs/substrate-apps). Installed by modules/substrate.nix + # (gates OFF) and hosts/nas/substrate-link.nix (still on pkgs/substrate-link). + substrate-pusher = substrateApps.pusher; + substrate-puller = substrateApps.puller; + substrate-apps-link = substrateApps.link; }; # The same teardown the control and harness roles keep on their PATH @@ -778,6 +789,12 @@ inherit pkgs self; inherit (nixpkgs) lib; }; + # modules/substrate.nix rendered OFF on the coordinator and armed with + # fixtures (tests/substrate-modules); every assertion is eval-time. + substrate-modules = import ./tests/substrate-modules { + inherit pkgs self; + inherit (nixpkgs) lib; + }; # DF-5: the vendored Cargo.lock must equal upstream's at the pinned rev. # A build-time comparison, not an evaluation-time read, so evaluation diff --git a/hosts/coordinator/default.nix b/hosts/coordinator/default.nix index f37e13a2..165841df 100644 --- a/hosts/coordinator/default.nix +++ b/hosts/coordinator/default.nix @@ -86,10 +86,37 @@ # all three interactive hosts, OFF on all three; read that module's header # for the runbook and for what it deliberately does not declare. ../../modules/ax-client.nix + # The Cloudflare Substrate's coordinator side (2026-09-23, E1/A2): the + # gentle capacity pusher and the interpreter-host puller, both declared + # ON below (2026-09-24). The NAS side is hosts/nas/substrate-link.nix. + ../../modules/substrate.nix ]; networking.hostName = "coordinator"; + # ── Substrate on this box: declared ON (2026-09-24) ──────────────────── + # modules/substrate.nix. Two user units on tom's manager, replacing the + # hand-started nohup processes of + # ~/today/wednesday-prep-2026-09-23/substrate (RUN.md) with the same + # configuration (runtimes opus/halogen/codex/codex-rw, pusher peerCacheDir + # "inherit"). Tokens: secrets/substrate-floor-token.age (FLOOR_TOKEN) and + # secrets/substrate-link-token-coordinator.age (this holder's LINK_TOKENS + # entry), delivered to tom 0400. No runtime-test wrapper: the puller runs + # with the real /run/user so herdr and ssh stay reachable. Cutover, in this + # order: (1) SIGTERM both nohup processes by their HOST pids and confirm + # they exited (a unit pusher treats the namespace pid 2 in pusher.pid as + # stale and would run beside a live nohup pusher); (2) rm -f + # ~/.local/state/substrate/{puller.pid,pusher.pid,puller.host.pid,puller.supervisor.pid}; + # (3) only then switch. Rollback: both `false` and switch. + services.substrate = { + floorUrl = "https://substrate.mecattaf.dev"; + pusher = { + enable = true; + owners.codex = "tom"; # E6 (2026-09-23): the codex login is Tom's + }; + puller.enable = true; + }; + # ── ax on the fleet: THE kill switch for this host ───────────────────── # The HARNESS node (modules/ax-fleet/harness.nix): a k3s agent tainted # ate.dev/sandboxClass=gvisor:NoSchedule, so only atelet and the gVisor diff --git a/modules/substrate.nix b/modules/substrate.nix new file mode 100644 index 00000000..1ccd30af --- /dev/null +++ b/modules/substrate.nix @@ -0,0 +1,667 @@ +{ + config, + lib, + pkgs, + ... +}: +# services.substrate: the coordinator's two box-side programs of the Cloudflare Substrate, declared ON on the +# coordinator since 2026-09-24. +# +# CODE: pkgs/substrate-apps (vendored from agency-agency/substrate at the sha in its SYNC.md). ARCHITECTURE: +# ~/today/wednesday-prep-2026-09-23/03-substrate-ax-conwip.md section 2. RULING: E1 (2026-09-23), the code that +# wraps ax lives in dotfiles; A2, the NAS link, the coordinator puller and the capacity pusher are dotfiles +# packages and modules. The NAS side is hosts/nas/substrate-link.nix (Lane A); this file is the coordinator side. +# +# pusher reads `seats --json --no-spend` (the box's one capacity oracle, ~/.local/bin/seats) under the gentle +# policy (120 s floor, 900 s idle, 300 s active, 180 s above 85 percent, one read at reset + 90 s) and +# posts seat-capacity/2 snapshots to the floor's POST /capacity/snapshots. A dead pusher fails closed +# at the floor: readings grade STALE after 1200 s and admission stops. +# puller the interpreter host: leases runs (runtime:interpreter) from the floor with its per-link token, runs +# each workflow's agent() calls on the runtimes this module renders into runtimes.toml (opus, halogen, +# codex, codex-rw: all host runtimes, the shape proven live on 2026-09-24), each claude seat spending +# its own config dir by path ([credentials.seats], RG-1), heartbeats, completes, resumes from the run's +# journal. Its config is the [puller] table of a client config.toml (packages/api/src/config.ts), +# rendered here with placeholders for the two credential paths and filled at unit start. +# +# WHY USER UNITS. Both programs speak for Tom's logins: `seats` reads each Claude seat's OAuth usage from its +# config dir, and the puller runs claude, herdr and ssh as Tom. A system unit with User=tom would carry none of +# the session (herdr socket, ssh agent, XDG dirs). ConditionUser pins each unit to that one user's manager. +# +# NO runtime-test WRAPPER. The hand-started proof (2026-09-23/24) ran both programs inside ~/.local/bin/runtime-test, +# which is a test harness: a private /run/user and PID/IPC namespaces. A unit is not a test. The puller runs on the +# user manager with the real /run/user so herdr's socket and the ssh agent stay reachable, and the host runtimes +# see the same session a login shell does (runtimeTestWrapper stays available, off, for the microvm runtime). +# +# THE PULLER'S PIDFILE LIVES IN ITS RuntimeDirectory ($RUNTIME_DIRECTORY/puller.pid, placeholder @RUNTIME_DIRECTORY@ +# filled at start), not on persistent disk. apps/puller/src/pidfile.ts counts any pid it cannot signal (EPERM) as a +# live puller, so a pidfile left on disk by a crash, SIGKILL, OOM kill or power loss, whose pid is later taken by a +# root or other-user process, would make every start exit 3. systemd clears the RuntimeDirectory on unit stop and +# at boot, and the unit already guarantees one instance, so that trap cannot arise; 3 is also no longer in +# RestartPreventExitStatus, so a transient hold is retried every RestartSec. +# +# CUTOVER from the hand-started processes: they wrote ~/.local/state/substrate/{puller,pusher}.pid from inside the +# runtime-test namespace, so both hold namespace pid 2 (a kernel thread on the host). The unit puller no longer +# reads that path; the unit pusher treats pid 2 as stale and REPLACES it, so a unit pusher started beside a live +# nohup pusher runs as a second pusher on one gentle-state.json. Order: (1) SIGTERM both nohup processes by their +# HOST pids (RUN.md) and confirm they exited; (2) rm -f ~/.local/state/substrate/{puller.pid,pusher.pid, +# puller.host.pid,puller.supervisor.pid}; (3) only then switch / start the units. +# +# THE TOKENS ARE PATHS, NEVER VALUES. The floor's operator bearer (FLOOR_TOKEN) and the puller's per-link token +# (one entry of the floor's LINK_TOKENS, bound to `holder`) are agenix secrets, decrypted at /run/agenix/ +# for the user (owner = user, 0400), and handed to each unit as private copies by LoadCredential; the pusher gets +# `--token-file %d/floor-token`, the puller a config.toml and a substrate.json written into its RuntimeDirectory +# at start with `$CREDENTIALS_DIRECTORY/floor-token` and `.../link-token` as token_file, link_token_file and +# capacityFloorTokenFile. Nothing here reads, prints or interpolates a value; every rendered file carries paths. +# Minting (wrangler secret put FLOOR_TOKEN and LINK_TOKENS, the two age files) is Tom's hand. +# +# THE GATE. Both `enable`s default to false. hosts/coordinator sets both true as of 2026-09-24, replacing the +# hand-started processes of ~/today/wednesday-prep-2026-09-23/substrate (RUN.md), with the floor live at floorUrl +# and the two tokens sealed as secrets/substrate-floor-token.age and secrets/substrate-link-token-coordinator.age +# (secrets.nix, editors ++ coordinatorOnly). The rendered files match that proven deployment: prove/runtimes.toml, +# prove/client.config.toml, tom.pusher.config.json and tom.substrate.config.json (axProtoFallbackPath excepted: +# null here, the vendored ax.proto). tests/substrate-modules renders both units armed with fixtures, so the shape is +# proven at evaluation time without a token on disk. +let + cfg = config.services.substrate; + inherit (lib) + mkEnableOption + mkIf + mkOption + types + ; + substrateApps = pkgs.callPackage ../pkgs/substrate-apps { }; + home = config.users.users.${cfg.user}.home; + tomlFormat = pkgs.formats.toml { }; + anyEnabled = cfg.pusher.enable || cfg.puller.enable; + credentialPath = config.age.secrets.${cfg.tokenSecret}.path; + + # NixOS renders a unit's `path` as Environment=PATH=..., which REPLACES the user manager's PATH rather than + # extending it. The runners spawn bare `claude`, `pi` and `codex` (packages/runners/src/harness.ts) and `seats` + # spawns bare `codex` for the codex seat, so both units carry the user's own profile explicitly (each entry + # gets /bin appended), in the manager's order: ~/.local/bin (seats), the per-user profile (claude, codex, pi, + # herdr), then the system profile. + userProfilePath = [ + "${home}/.local" + "/etc/profiles/per-user/${cfg.user}" + "/run/current-system/sw" + ]; + + # The pusher's config file: apps/pusher/pusher.example.json shape, minus tokenFile (given as --token-file). + pusherConfig = pkgs.writeText "substrate-pusher.json" ( + builtins.toJSON ( + { + floorUrl = cfg.floorUrl; + seatsBin = cfg.pusher.seatsBin; + host = cfg.pusher.host; + seats = cfg.pusher.seats; + seatIds = cfg.pusher.seatIds; + owners = cfg.pusher.owners; + not_dispatchable = cfg.pusher.notDispatchable; + plans = cfg.pusher.plans; + slots = cfg.pusher.slots; + estimatedProviders = cfg.pusher.estimatedProviders; + peerCacheDir = cfg.pusher.peerCacheDir; + } + // cfg.pusher.extraConfig + ) + ); + + runtimesToml = tomlFormat.generate "substrate-runtimes.toml" cfg.puller.runtimes; + + # The deploy config (src/deploy-config.ts) with the bearer path left as a placeholder; the unit fills it from + # $CREDENTIALS_DIRECTORY at start, into its RuntimeDirectory, so no path to a credential is guessed at eval. + deployConfigTemplate = pkgs.writeText "substrate.json.in" ( + builtins.toJSON ( + cfg.puller.deployConfig + // { + capacityFloorUrl = cfg.floorUrl; + capacityFloorTokenFile = "@FLOOR_TOKEN_FILE@"; + } + ) + ); + + # The client config (packages/api/src/config.ts, deploy/client.config.example.toml): the floor, WHERE each + # credential is, and the [puller] table. Both token paths are placeholders until unit start. + clientConfigTemplate = tomlFormat.generate "substrate-client.toml.in" ({ + floor_url = cfg.floorUrl; + token_file = "@FLOOR_TOKEN_FILE@"; + puller = { + holder = cfg.puller.holder; + link_token_file = "@LINK_TOKEN_FILE@"; + state_dir = cfg.puller.stateDir; + pidfile = if cfg.puller.pidfile == null then "@RUNTIME_DIRECTORY@/puller.pid" else cfg.puller.pidfile; + max_runs = cfg.puller.maxRuns; + node_dispatch = cfg.puller.nodeDispatch; + runtimes = "${runtimesToml}"; + seat = cfg.puller.claudeSeat; + cap = cfg.puller.cap; + default_model = cfg.puller.defaultModel; + node_runs_on = cfg.puller.nodeRunsOn; + demand_dir = cfg.puller.demandDir; + drain_timeout_s = cfg.puller.drainTimeoutS; + capacity_wait_s = cfg.puller.capacityWaitS; + } + // lib.optionalAttrs (cfg.puller.axServer != null) { ax_server = cfg.puller.axServer; } + // lib.optionalAttrs (cfg.puller.healthAddr != null) { health_addr = cfg.puller.healthAddr; }; + }); + + pullerExec = lib.escapeShellArgs ( + lib.optionals (cfg.puller.runtimeTestWrapper != null) [ + cfg.puller.runtimeTestWrapper + "--" + ] + ++ [ (lib.getExe cfg.puller.package) ] + ); + + pullerStart = pkgs.writeShellScript "substrate-puller-start" '' + set -euo pipefail + : "''${CREDENTIALS_DIRECTORY:?substrate-puller needs LoadCredential (no CREDENTIALS_DIRECTORY)}" + : "''${RUNTIME_DIRECTORY:?substrate-puller needs RuntimeDirectory}" + token="$CREDENTIALS_DIRECTORY/floor-token" + link="$CREDENTIALS_DIRECTORY/link-token" + test -r "$token" || { echo "substrate-puller: no floor-token credential" >&2; exit 78; } + test -r "$link" || { echo "substrate-puller: no link-token credential" >&2; exit 78; } + umask 077 + ${pkgs.gnused}/bin/sed "s|@FLOOR_TOKEN_FILE@|$token|" ${deployConfigTemplate} > "$RUNTIME_DIRECTORY/substrate.json" + ${pkgs.gnused}/bin/sed -e "s|@FLOOR_TOKEN_FILE@|$token|" -e "s|@LINK_TOKEN_FILE@|$link|" -e "s|@RUNTIME_DIRECTORY@|$RUNTIME_DIRECTORY|" ${clientConfigTemplate} > "$RUNTIME_DIRECTORY/config.toml" + export SUBSTRATE_CONFIG="$RUNTIME_DIRECTORY/substrate.json" + export SUBSTRATE_CLIENT_CONFIG="$RUNTIME_DIRECTORY/config.toml" + # [puller].runtimes is the primary; loadRuntimes (packages/runners/src/config.ts) still falls back to this. + export AX_CONWIP_RUNTIMES=${runtimesToml} + # Review 2026-09-24 (low): hostHarnessEnv passes CREDENTIALS_DIRECTORY through to every host agent. The token + # paths are already in the rendered files, so the variable is dropped here; the directory stays readable. + unset CREDENTIALS_DIRECTORY + exec ${pullerExec} + ''; + + commonService = { + Restart = "on-failure"; + RestartSec = "30s"; + TimeoutStopSec = "30s"; + UMask = "0077"; + NoNewPrivileges = true; + LoadCredential = [ "floor-token:${credentialPath}" ]; + }; +in +{ + options.services.substrate = { + user = mkOption { + type = types.str; + default = "tom"; + description = "The login whose user manager runs both units and whose seats they speak for."; + }; + + floorUrl = mkOption { + type = types.str; + default = ""; + example = "https://substrate.example.dev"; + description = "The floor Worker's base URL. HTTPS only; the bearer never travels in clear."; + }; + + tokenSecret = mkOption { + type = types.str; + default = "substrate-floor-token"; + description = "The agenix secret NAME holding the floor's operator bearer (FLOOR_TOKEN). Declared by this module when a unit is enabled, decrypted for `user`, handed to each unit as the credential `floor-token`. A name, never a value."; + }; + + tokenAgeFile = mkOption { + type = types.path; + default = ../secrets/substrate-floor-token.age; + defaultText = lib.literalExpression "../secrets/substrate-floor-token.age"; + description = "The sealed bearer (secrets.nix: editors ++ the coordinator key). Evaluated only when a unit is enabled."; + }; + + pusher = { + enable = mkEnableOption "the gentle capacity pusher (seats --json to the floor). ON on the coordinator; see this file's header"; + + package = mkOption { + type = types.package; + default = substrateApps.pusher; + defaultText = lib.literalExpression "(pkgs.callPackage ../pkgs/substrate-apps { }).pusher"; + description = "The pusher, built from pkgs/substrate-apps/src/apps/pusher."; + }; + + configFile = mkOption { + type = types.path; + readOnly = true; + default = pusherConfig; + description = "The rendered pusher.json (read-only; for tests)."; + }; + + seatsBin = mkOption { + type = types.str; + default = "${home}/.local/bin/seats"; + defaultText = lib.literalExpression ''"''${home}/.local/bin/seats"''; + description = "The capacity oracle. A chezmoi-managed script (home/dot_local/bin/seats), stdlib python."; + }; + + host = mkOption { + type = types.str; + default = config.networking.hostName; + defaultText = lib.literalExpression "config.networking.hostName"; + description = "The host name the snapshot carries."; + }; + + seats = mkOption { + type = types.listOf types.str; + default = [ + "cc" + "cc2" + "codex" + "pi-qwencloud" + "halogen" + ]; + description = "Seats published to the floor (runs-on seat ids)."; + }; + + seatIds = mkOption { + type = types.attrsOf types.str; + default = { + gpu-worker = "halogen"; + }; + description = "Oracle seat id to runs-on seat id."; + }; + + owners = mkOption { + type = types.attrsOf types.str; + default = { }; + example = { + codex = "tom"; + }; + description = "Overrides the oracle's third-party marks (E6: codex is Tom's login tonight)."; + }; + + notDispatchable = mkOption { + type = types.attrsOf types.str; + default = { + cc3 = "evicted"; + gpu-coordinator = "halogen is declared but not resident on the coordinator"; + }; + description = "Seats never published, with the reason."; + }; + + plans = mkOption { + type = types.attrsOf types.str; + default = { }; + description = "Plan expiry per seat (ISO instant)."; + }; + + slots = mkOption { + type = types.attrsOf types.int; + default = { + halogen = 1; + }; + description = "Slot capacity for seats the oracle reports no holders for (rule 15: the floor subtracts its own WIP)."; + }; + + estimatedProviders = mkOption { + type = types.listOf types.str; + default = [ "qwen" ]; + description = "Providers whose readings grade ESTIMATED (never admitted unless the floor ungates them)."; + }; + + peerCacheDir = mkOption { + type = types.str; + default = "inherit"; + description = "pusher.json peerCacheDir: the seats oracle's peer cache. `inherit` (the proven setting) keeps the oracle's own default, so `seats` reuses the tally seat-feeder cache and adds no usage-endpoint calls; a path gives the pusher its own cache (never a tally-rewrite path; apps/pusher/src/seatsOracle.mjs refuses one)."; + }; + + extraConfig = mkOption { + type = types.attrs; + default = { }; + description = "Merged last into pusher.json. Never a token."; + }; + }; + + puller = { + enable = mkEnableOption "the coordinator puller (the interpreter host). ON on the coordinator; see this file's header"; + + package = mkOption { + type = types.package; + default = substrateApps.puller; + defaultText = lib.literalExpression "(pkgs.callPackage ../pkgs/substrate-apps { }).puller"; + description = "The puller, built from pkgs/substrate-apps/src/apps/puller with its workspace."; + }; + + holder = mkOption { + type = types.str; + default = "coordinator"; + description = "The holder its link token is bound to in the floor's LINK_TOKENS (which also binds the runs-on labels it may lease, runtime:interpreter and seat:coordinator in docs/DEPLOY.md). One session per holder (L5, exit 75)."; + }; + + linkTokenSecret = mkOption { + type = types.str; + default = "substrate-link-token-coordinator"; + description = "The agenix secret NAME holding this puller's per-link token (the LINK_TOKENS entry for `holder`). Handed to the unit as the credential `link-token`. A name, never a value."; + }; + + linkTokenAgeFile = mkOption { + type = types.path; + default = ../secrets/substrate-link-token-coordinator.age; + defaultText = lib.literalExpression "../secrets/substrate-link-token-coordinator.age"; + description = "The sealed per-link token. Evaluated only when the puller is enabled."; + }; + + maxRuns = mkOption { + type = types.ints.positive; + default = 1; + description = "[puller].max_runs: runs held at once (the Lease capacity)."; + }; + + cap = mkOption { + type = types.ints.positive; + default = 2; + description = "[puller].cap: the WIP cap across this puller's agent() calls (node_dispatch = local)."; + }; + + nodeDispatch = mkOption { + type = types.enum [ + "local" + "floor" + ]; + default = "local"; + description = "[puller].node_dispatch: local runs agent() nodes on this box's runtimes; floor sends every node back to the floor as an AgentJob for whichever link serves its labels."; + }; + + nodeRunsOn = mkOption { + type = types.listOf types.str; + default = [ + "seat:halogen" + "runtime:gvisor" + ]; + description = "[puller].node_runs_on: runs-on for a node that names none (floor dispatch)."; + }; + + defaultModel = mkOption { + type = types.str; + default = "claude-opus-5-5"; + description = "[puller].default_model."; + }; + + axServer = mkOption { + type = types.nullOr types.str; + default = null; + example = "nas:7070"; + description = "[puller].ax_server: probed only to say why an ax node is refused here (the NAS link dispatches ax)."; + }; + + demandDir = mkOption { + type = types.str; + default = "${home}/.local/state/substrate/demand"; + defaultText = lib.literalExpression ''"''${home}/.local/state/substrate/demand"''; + description = "[puller].demand_dir: marked while a run is held, so the pusher (same dir by default) reads at its active cadence."; + }; + + drainTimeoutS = mkOption { + type = types.ints.unsigned; + default = 60; + description = "[puller].drain_timeout_s: on the first SIGTERM, how long in-flight runs may finish before they are aborted. Keep TimeoutStopSec above it."; + }; + + capacityWaitS = mkOption { + type = types.ints.unsigned; + default = 600; + description = "[puller].capacity_wait_s: how long a node waits for capacity (a stale or refused reading) before it fails."; + }; + + healthAddr = mkOption { + type = types.nullOr types.str; + default = null; + example = "127.0.0.1:9311"; + description = "[puller].health_addr: loopback host:port for GET /status.json and /metrics. null: no endpoint (key omitted)."; + }; + + stateDir = mkOption { + type = types.str; + default = "${home}/.local/state/substrate/puller"; + defaultText = lib.literalExpression ''"''${home}/.local/state/substrate/puller"''; + description = "[puller].state_dir: held leases, verdict outbox, runs// journals."; + }; + + pidfile = mkOption { + type = types.nullOr types.str; + default = null; + description = "[puller].pidfile: one puller per box (a second exits 3). null (the default) renders $RUNTIME_DIRECTORY/puller.pid, which systemd clears on unit stop and at boot, so a pidfile left by an unclean exit cannot block the next start. A path on persistent disk brings that stale-pidfile trap back."; + }; + + clientConfigFile = mkOption { + type = types.path; + readOnly = true; + default = clientConfigTemplate; + description = "The rendered config.toml template, credential paths as placeholders (read-only; for tests)."; + }; + + claudeSeat = mkOption { + type = types.str; + default = "cc2"; + description = "The Claude seat this host's claude harness spends ([seats].claude and TALLY_SEAT)."; + }; + + claudeConfigDir = mkOption { + type = types.str; + default = "${home}/.claude-work"; + defaultText = lib.literalExpression ''"''${home}/.claude-work"''; + description = "CLAUDE_CONFIG_DIR for that seat, by path (cc2 = ~/.claude-work, the seats oracle's table). Mounted, never copied, into sandboxed runtimes."; + }; + + runtimeTestWrapper = mkOption { + type = types.nullOr types.str; + default = null; + example = "/home/tom/.local/bin/runtime-test"; + description = "When set, ExecStart runs under this wrapper (` -- ...`): a private /run/user tree and PID/IPC namespaces, which the microvm runtime requires (packages/runners/src/microvm.ts). Off by default: herdr and ssh want the live session."; + }; + + runtimes = mkOption { + type = tomlFormat.type; + default = { + default = "opus"; + allow = [ + "opus" + "halogen" + "codex" + "codex-rw" + ]; + seats = { + claude = cfg.puller.claudeSeat; + pi = "halogen"; + codex = "codex"; + }; + credentials = { + claude = cfg.puller.claudeConfigDir; + mode = "rw"; + scope = "credential"; + # RG-1 (critique pass 2026-09-24): a claude job mounts the dir of the seat it is gated on, never another. + # cc is ~/.claude; claudeSeat (cc2) is claudeConfigDir. `//` so claudeSeat = "cc" overrides, not clashes. + seats = { + cc = "${home}/.claude"; + } + // { + ${cfg.puller.claudeSeat} = cfg.puller.claudeConfigDir; + }; + }; + runtime = { + opus = { + type = "host"; + harness = "claude"; + seat = cfg.puller.claudeSeat; + timeoutMs = 900000; + }; + # pi on the coordinator against the Halogen server on the worker (the proven pattern). ssh:worker was + # refused in the proof and never exercised, so it is not declared. + halogen = { + type = "host"; + harness = "pi"; + seat = "halogen"; + timeoutMs = 1800000; + }; + codex = { + type = "host"; + harness = "codex"; + seat = "codex"; + timeoutMs = 900000; + codexSandbox = "read-only"; + }; + # The build-node runtime: codex may write in its job dir. + codex-rw = { + type = "host"; + harness = "codex"; + seat = "codex"; + timeoutMs = 900000; + codexSandbox = "workspace-write"; + }; + }; + }; + description = "The runtimes file (packages/runners/src/config.ts), rendered to TOML and named by [puller].runtimes (and AX_CONWIP_RUNTIMES). Default: the shape proven live on 2026-09-24. opus (host, claude on claudeSeat), halogen (host, pi against the worker's Halogen server), codex (read-only) and codex-rw (workspace-write), default opus, each claude seat bound to its config dir in [credentials.seats]. herdr, gvisor and ssh:worker tables are not declared by default (never exercised); override this option to add them."; + }; + + runtimesFile = mkOption { + type = types.path; + readOnly = true; + default = runtimesToml; + description = "The rendered runtimes.toml (read-only; for tests)."; + }; + + deployConfig = mkOption { + type = types.attrs; + default = { + forbiddenReadPrefixes = [ + "~/rawa" + "~/.local/state/secrets-holdout-2026-09-18" + ]; + forbiddenLedgerPrefixes = [ + "/run/user" + "~/.local/state/tally-rewrite" + ]; + metersDir = "~/.local/state/tally-rewrite/meters"; + axProtoFallbackPath = null; + realSeats = [ + "cc" + "cc2" + "cc3" + "halogen" + "codex" + ]; + codexWorkdir = "~/mecattaf/substrate"; + }; + description = "src/deploy-config.ts keys (SUBSTRATE_CONFIG, read by the capacity gate). capacityFloorUrl and capacityFloorTokenFile are set by the unit from floorUrl and the credential."; + }; + + workingDirectory = mkOption { + type = types.str; + default = "${home}/mecattaf/substrate"; + defaultText = lib.literalExpression ''"''${home}/mecattaf/substrate"''; + description = "The unit's WorkingDirectory. The live puller ran from the substrate checkout: the runners derive a worktree from process.cwd() for agent({isolation = \"worktree\"}) calls and the interpreter's fallback resolver looks in cwd/.claude/workflows, so a git checkout keeps parity with the proof (review 2026-09-24)."; + }; + extraPath = mkOption { + type = types.listOf types.package; + default = [ ]; + description = "Packages on the unit's PATH besides openssh, coreutils and the user's profile directories (~/.local/bin, /etc/profiles/per-user//bin, /run/current-system/sw/bin), which supply claude, pi, codex and herdr."; + }; + }; + }; + + config = mkIf anyEnabled { + assertions = [ + { + assertion = lib.hasPrefix "https://" cfg.floorUrl; + message = "services.substrate.floorUrl must be an https:// URL (the bearer never travels in clear)."; + } + { + assertion = !cfg.puller.enable || cfg.puller.linkTokenSecret != cfg.tokenSecret; + message = "services.substrate.puller.linkTokenSecret must differ from tokenSecret: the per-link token is not the operator bearer."; + } + { + assertion = !cfg.pusher.enable || !(cfg.pusher.extraConfig ? tokenFile); + message = "services.substrate.pusher.extraConfig must not name a tokenFile: the bearer arrives as the credential floor-token."; + } + ]; + + age.secrets = lib.mkMerge [ + { + ${cfg.tokenSecret} = { + file = cfg.tokenAgeFile; + owner = cfg.user; + mode = "0400"; + }; + } + (mkIf cfg.puller.enable { + ${cfg.puller.linkTokenSecret} = { + file = cfg.puller.linkTokenAgeFile; + owner = cfg.user; + mode = "0400"; + }; + }) + ]; + + systemd.user.services.substrate-pusher = mkIf cfg.pusher.enable { + description = "substrate-pusher: seats --json to the Substrate floor, gently"; + wantedBy = [ "default.target" ]; + unitConfig.ConditionUser = cfg.user; + path = [ + pkgs.python3 + pkgs.coreutils + ] + ++ userProfilePath; + serviceConfig = commonService // { + ExecStart = "${lib.getExe cfg.pusher.package} --config ${pusherConfig} --token-file %d/floor-token"; + # 2 usage, 3 another pusher holds the pidfile: neither is cured by a restart. + RestartPreventExitStatus = [ + 2 + 3 + ]; + MemoryMax = "256M"; + }; + }; + + systemd.user.services.substrate-puller = mkIf cfg.puller.enable { + description = "substrate-puller: the Substrate interpreter host on this box"; + wantedBy = [ "default.target" ]; + unitConfig.ConditionUser = cfg.user; + path = [ + pkgs.openssh + pkgs.coreutils + ] + ++ cfg.puller.extraPath + ++ userProfilePath; + environment = { + CLAUDE_CONFIG_DIR = cfg.puller.claudeConfigDir; + TALLY_SEAT = cfg.puller.claudeSeat; + }; + serviceConfig = commonService // { + ExecStart = pullerStart; + LoadCredential = commonService.LoadCredential ++ [ + "link-token:${config.age.secrets.${cfg.puller.linkTokenSecret}.path}" + ]; + RuntimeDirectory = "substrate-puller"; + RuntimeDirectoryMode = "0700"; + # 75 another session holds this holder (L5), 78 bad config or token: neither is cured by a restart. 3 (another + # puller holds the pidfile) is retried every RestartSec: the pidfile is in the RuntimeDirectory, so a hold is + # transient and a restart is the cure. + RestartPreventExitStatus = [ + 75 + 78 + ]; + # Above drain_timeout_s, so the first SIGTERM's drain can finish before systemd escalates. + TimeoutStopSec = "${toString (cfg.puller.drainTimeoutS + 30)}s"; + # Review 2026-09-24 (medium): the default KillMode=control-group would SIGTERM every claude, codex and pi + # child at once and defeat the puller's own drain ladder; mixed signals only the puller, which drains its + # nodes (drain_timeout_s) and kills its live groups itself. One OOM-killed node must not stop the puller. + KillMode = "mixed"; + OOMPolicy = "continue"; + WorkingDirectory = cfg.puller.workingDirectory; + # Two claude agents at ~400 MB each plus the puller and tool builds approach 2G; 2G is the reclaim line, + # 4G the hard limit (review 2026-09-24, low). + MemoryHigh = "2G"; + MemoryMax = "4G"; + # The proof ran inside runtime-test with a private /run/user. The unit keeps the real one (herdr and ssh + # stay reachable) but the desktop session's sockets are not something a headless agent should inherit. + UnsetEnvironment = [ + "DBUS_SESSION_BUS_ADDRESS" + "WAYLAND_DISPLAY" + "DISPLAY" + "NIRI_SOCKET" + ]; + }; + }; + }; +} diff --git a/pkgs/substrate-apps/SYNC.md b/pkgs/substrate-apps/SYNC.md new file mode 100644 index 00000000..f0919d9d --- /dev/null +++ b/pkgs/substrate-apps/SYNC.md @@ -0,0 +1,102 @@ +# pkgs/substrate-apps/src: vendored source + +- Upstream: agency-agency/substrate (private; the repo formerly named ax-conwip, Tom 2026-09-23 E10), local + checkout `/home/tom/mecattaf/substrate`, branch `main`. +- Source sha: `fc2f8bd5d1492343585914b4ed343d001a192f33` (synced 2026-09-24T12:01:48Z, apps/puller present) + "DECISIONS: D-S13, queued jobs never expire (ruling M2)", committed 2026-09-24 09:04 CEST. It is 6f681d3 (the + checkout the live coordinator puller and pusher were hand-started from on 2026-09-24) plus two floor-only commits + (78b285d, fc2f8bd), so the box-side programs here are the ones proven live. History: first vendored at dc7cd1d + (link and pusher only), then b105179 (2026-09-23, puller added), then this resync (63 commits later). +- Taken with `git archive package.json pnpm-lock.yaml pnpm-workspace.yaml tsconfig.base.json tsconfig.json + src proto packages deploy apps/link apps/pusher apps/puller`, unchanged (`./sync.sh /home/tom/mecattaf/substrate + ` does exactly this; 332 files, 3.6 MB at fc2f8bd). The puller imports the workspace root (`substrate/src/...`), + `@substrate/api`, `@substrate/link`, `@substrate/interpreter` and `@substrate/runners`, hence `src/`, `proto/` and + `packages/*`. Not vendored: `test/`, `tools/`, `docs/`, `apps/floor` (the Cloudflare Worker, deployed from the + substrate repo), `apps/cli` and `apps/mcp` (operator interfaces, not services of a box), `apps/evaluator` (new + at fc2f8bd, the mechanical receipt evaluator; nothing on the box imports it). pnpm tolerates the lockfile + importers of the apps not vendored. +- Not vendored on purpose: the pusher's own vitest suite depends on `@substrate/planning` and `@substrate/factory` + (workspace packages); it runs upstream (1530 passed at b105179, REPORTED substrate/RECEIPT.md; not re-counted at fc2f8bd). The nix + checkPhase runs one dry tick against `apps/pusher/test/fixtures/seats-v1.json` instead. +- `apps/link` at this sha differs from the copy in `pkgs/substrate-link/src` (a021003) only in tests, + `tsconfig.json` and `vitest.config.ts` (MEASURED `git diff --stat a021003 dc7cd1d -- apps/link`: 7 files, no + `src/` change). The NAS module `hosts/nas/substrate-link.nix` keeps `pkgs/substrate-link` as its default until a + follow-up re-points it here; `.#substrate-apps-link` is the same program from the newer sha. + +## What changed between b105179 and fc2f8bd that matters to the box + +From `git log --oneline b105179..fc2f8bd -- apps/puller apps/pusher packages/runners packages/api deploy` (24 +commits) and the config diffs. The pnpm dependency tree did not change (the lockfile only gains the +`apps/evaluator` importer), so `pnpmDeps.hash` is the same value as at b105179 (MEASURED with lib.fakeHash). + +- Seat routing by `agent()` option (be42f18, merging eb956de and 22c4ae8): `agent(p, { seat: "codex" })` or + `runsOn: ["seat:codex"]` runs on the one runtime `runtimes.toml` permits for that seat (the phase or default + runtime first), else it is refused with the runtime=seat list. A runtime table's `seat` wins over `[seats]`; the + floor refuses an unserved seat (d112022: runs-on carries the seat the runtime spends). +- `[credentials.seats]` (RG-1) and its alias `seat_dirs` (TX5), one map seat id -> Claude config dir, merged at + load; a seat both spell with different dirs, or one dir claimed by two seats, is a load error; entries must be + absolute after `~` expansion. With the map present every claude runtime with a local credential must name a seat + in it and a call whose seat has no entry is refused. Without it, `[credentials].claude` serves only when set + explicitly and at most one claude seat is bound. Also new in `[credentials]`: `context` (list of absolute paths + bound read-only into gVisor jobs). +- `codexSandbox` (runtime tables, codex harness only): `"read-only"` (default) or `"workspace-write"`; set on any + other harness it is a load error. +- `runtimes.toml` top level also gains `cancel_grace_ms` (default 10000; SIGTERM then SIGKILL, G-BK1) and + `[hooks]` `pre_start` / `pre_exit` argv with `timeout_ms` (G-BK8). A runtime named `locked` is reserved for the + untrusted shape (gvisor, `credential = false`, network not host). +- `peerCacheDir` (pusher.json; `--peer-cache-dir`): the seats oracle's `SEATS_PEER_CACHE_DIR`. Default now + `~/.local/state/substrate/seats-peer-cache` (D-S09, a substrate-owned cache); `"inherit"` keeps the oracle's own + default; a `tally-rewrite` path is refused (exit 2). The live pusher config sets `"inherit"`. +- New `[puller]` keys (`packages/api/src/config.ts`, `deploy/client.config.example.toml`), all optional: + - `demand_dir`: marker dir touched while a run is held so the pusher reads at its active cadence (example + `~/.local/state/substrate/demand`, the pusher's own `--demand-dir` default). + - `capacity_wait_s`: how long a node waits on a stale or refused capacity reading before it fails. Default 600. + - `drain_timeout_s`: first SIGTERM lets in-flight runs finish this long, then aborts them with the cancel + grace; a second signal aborts, a third kills. Default 60; 0 aborts at once (G-BK4). + - `health_addr`: loopback `host:port` serving `GET /status.json` and `/metrics`. Absent: no endpoint (G-BK7). + - Top level (not `[puller]`): `read_token_file` (or `SUBSTRATE_READ_TOKEN_FILE`), the read-only bearer for + `substrate watch` (TX4); the puller does not read it. +- Puller behaviour: streams the run's `events.jsonl` to the floor's live log while it holds the lease (G-BK3), + uploads every local node's transcript before the verdict (TX2), run-scoped writes under the run's interpreter + lease (G-BK5), self-fences before `reassignSeconds`, graceful drain ladder (G-BK2/G-BK4), marks pusher demand + while a run is held and waits on capacity instead of nulling a node (0ba7550). Runners archive each job's + harness transcript outside the run dir before cleanup and record finished attempts by stable id for resume (C3-4). +- Unchanged contract: the puller with no config exits 78 with a `config-invalid` line (`apps/puller/src/main.ts`; + exit 3 pidfile held, 75 holder session conflict). The pusher still imports only `node:` builtins and its own + files. Upstream `packages/runners/package.json` names a bin `bin/substrate-runners.mjs` that does not exist + (the file is `bin/ax-conwip-runners.mjs`); pnpm warns and skips it, harmless for the puller. + +## Programs and where they run + +| attr | upstream | host | dotfiles module | +|---|---|---|---| +| `link` | `apps/link` | NAS (system unit) | `hosts/nas/substrate-link.nix` (still on pkgs/substrate-link a021003) | +| `pusher` | `apps/pusher/bin/substrate-pusher.mjs` | coordinator (user unit) | `modules/substrate.nix` `services.substrate.pusher` | +| `puller` | `apps/puller/bin/substrate-puller.mjs` | coordinator (user unit) | `modules/substrate.nix` `services.substrate.puller` | + +## How the puller is installed + +Upstream starts it as `node apps/puller/bin/substrate-puller.mjs`, which registers tsx's ESM loader and imports +`src/main.ts`; nothing is bundled upstream. The derivation keeps that: `pnpm install --prod --offline +--frozen-lockfile` from the shared `pnpmDeps`, then the whole tree with its production `node_modules` is copied to +`$out/lib/substrate-apps` and `bin/substrate-puller` wraps node over it. Its checkPhase starts it with no config +and requires exit 78 with a `config-invalid` line: every import resolved through tsx and the workspace links, and +`main()` ran to its first config check. Its configuration is the `[puller]` table of a client `config.toml` +(`packages/api/src/config.ts`, `deploy/client.config.example.toml`), which the module renders. + +## To resync + +1. `./sync.sh /home/tom/mecattaf/substrate ` (rewrites `src/`, the sha line above and `sourceSha`), then + `git add pkgs/substrate-apps`: the flake only sees tracked files, and a new upstream file left untracked fails + the puller check with ERR_MODULE_NOT_FOUND (it did at fc2f8bd: `packages/api/src/seats.ts`). +2. In `default.nix` set `pnpmDeps.hash = lib.fakeHash`, `nix build .#substrate-apps-link`, paste the `got:` value. +3. `nix build .#substrate-pusher .#substrate-puller .#substrate-apps-link --no-link` and `nix flake check --no-build`. +4. Commit with the sha in the message. + +## Unknowns and proposed defaults + +- Whether the vendored copy should become a flake input once the repo is public-shaped. Default: keep vendoring by + sha (E1: the code that wraps ax lives in dotfiles; a private input would also need a token on every build host). +- Whether `pkgs/substrate-link` (a021003) is retired in favour of this set's `link`. Default: yes, in the follow-up + that re-points `hosts/nas/substrate-link.nix`, after Lane A's 4-VM test is green on ax/fleet-zero; not in this lane + (hosts/ is Lane A's). diff --git a/pkgs/substrate-apps/default.nix b/pkgs/substrate-apps/default.nix new file mode 100644 index 00000000..ff6539a1 --- /dev/null +++ b/pkgs/substrate-apps/default.nix @@ -0,0 +1,172 @@ +{ + lib, + stdenvNoCC, + nodejs_24, + pnpm_10, + esbuild, + makeWrapper, +}: +# substrate-apps: the box-side programs of the Cloudflare Substrate (agency-agency/substrate), +# vendored in ./src at the sha SYNC.md names and built here as one package set: +# +# link apps/link the NAS side: leases AgentJobs from the floor over /rpc and runs each as one +# ax Task. Bundled with esbuild exactly as pkgs/substrate-link does (that older +# copy, a021003, stays the NAS module's default until hosts/nas is re-pointed). +# pusher apps/pusher the coordinator's gentle capacity pusher: `seats --json --no-spend` under the +# refresh policy, converted to seat-capacity/2 and posted to the floor. +# puller apps/puller the coordinator's interpreter host: leases runs (runtime:interpreter) from the +# floor, runs them with the interpreter and the runners, heartbeats, completes, +# resumes from the run's journal after a kill. Upstream runs it through tsx's ESM +# loader over the workspace packages (substrate, @substrate/{api,link,interpreter, +# runners}), so it is installed as the whole tree with its production node_modules. +# +# One pnpm workspace, one fixed-output dependency tree (pnpmDeps) shared by every program that needs +# node_modules. The pusher needs none: its bin and src import only node: builtins and each other +# (MEASURED grep, 2026-09-23, again at fc2f8bd 2026-09-24), so it is installed as files and smoke-run in its checkPhase. +# +# Re-vendor: ./sync.sh , then refresh pnpmDeps.hash (lib.fakeHash, build, +# paste the got: value). E1: the code that wraps ax lives in dotfiles, pinned by sha, never a flake input. +let + version = "0.1.0-unstable-2026-09-24"; + src = ./src; + # The upstream commit ./src was taken from. Keep in step with SYNC.md (sync.sh rewrites both). + sourceSha = "fc2f8bd5d1492343585914b4ed343d001a192f33"; + + pnpmDeps = pnpm_10.fetchDeps { + pname = "substrate-apps"; + inherit version src; + # nix build .#substrate-apps-link with lib.fakeHash, then the "got:" value (MEASURED 2026-09-23; unchanged at + # fc2f8bd, 2026-09-24: its lockfile only adds the apps/evaluator importer, no new packages). + hash = "sha256-/10ykNk5FGChNhjBxfP8lfrUJI4G5O4kE+phfewLODc="; + fetcherVersion = 3; + }; + + link = stdenvNoCC.mkDerivation { + pname = "substrate-link"; + inherit version src pnpmDeps; + nativeBuildInputs = [ + nodejs_24 + pnpm_10.configHook + esbuild + makeWrapper + ]; + buildPhase = '' + runHook preBuild + esbuild apps/link/src/main.ts --bundle --platform=node --format=esm --target=node22 \ + --outfile=dist/substrate-link.mjs \ + "--banner:js=import{createRequire as __cr}from'node:module';const require=__cr(import.meta.url);" + runHook postBuild + ''; + installPhase = '' + runHook preInstall + install -Dm444 dist/substrate-link.mjs $out/lib/substrate-link/substrate-link.mjs + install -Dm444 apps/link/proto/ax-p1.proto $out/share/substrate-link/ax-p1.proto + makeWrapper ${lib.getExe nodejs_24} $out/bin/substrate-link \ + --add-flags $out/lib/substrate-link/substrate-link.mjs \ + --set LINK_AX_PROTO_PATH $out/share/substrate-link/ax-p1.proto + runHook postInstall + ''; + passthru = { inherit sourceSha; }; + meta = { + description = "Outbound-only link from the Cloudflare Substrate floor to ax on the NAS"; + mainProgram = "substrate-link"; + platforms = lib.platforms.linux; + }; + }; + + pusher = stdenvNoCC.mkDerivation { + pname = "substrate-pusher"; + inherit version src; + nativeBuildInputs = [ + nodejs_24 + makeWrapper + ]; + dontBuild = true; + doCheck = true; + # One dry tick against the vendored seat-capacity/1 fixture: reads, converts, prints, posts nothing. + # HOME and every state path point into the build directory, so nothing outside the sandbox is touched. + checkPhase = '' + runHook preCheck + export HOME=$PWD/home + mkdir -p $HOME + node --check apps/pusher/bin/substrate-pusher.mjs + node apps/pusher/bin/substrate-pusher.mjs --dry-run --once \ + --seats-json apps/pusher/test/fixtures/seats-v1.json \ + --pidfile $PWD/pusher.pid --state $PWD/state.json --demand-dir $PWD/demand > tick.jsonl + grep -q '"event"' tick.jsonl + if grep -q '"read-failed"' tick.jsonl; then echo "the dry tick failed to read the fixture"; cat tick.jsonl; exit 1; fi + test ! -e $PWD/pusher.pid + runHook postCheck + ''; + installPhase = '' + runHook preInstall + mkdir -p $out/lib/substrate-pusher + cp -r apps/pusher/bin apps/pusher/src apps/pusher/package.json $out/lib/substrate-pusher/ + chmod -R a-w $out/lib/substrate-pusher + makeWrapper ${lib.getExe nodejs_24} $out/bin/substrate-pusher \ + --add-flags $out/lib/substrate-pusher/bin/substrate-pusher.mjs + runHook postInstall + ''; + passthru = { inherit sourceSha; }; + meta = { + description = "The gentle capacity pusher: seats --json to the Substrate floor's POST /capacity/snapshots"; + mainProgram = "substrate-pusher"; + platforms = lib.platforms.linux; + }; + }; + + puller = stdenvNoCC.mkDerivation { + pname = "substrate-puller"; + inherit version src pnpmDeps; + nativeBuildInputs = [ + nodejs_24 + pnpm_10.configHook + makeWrapper + ]; + # Production dependencies only: tsx, effect, smol-toml, grpc and the workspace links. vitest, + # typescript and fast-check stay out of the closure. + pnpmInstallFlags = [ "--prod" ]; + dontBuild = true; + doCheck = true; + # Start it with no config: every import resolves through tsx and the workspace links, main() runs, + # and it exits 78 with a `config-invalid` line ("[puller] holder is required"). Anything else, a + # missing module above all, is a different exit. + checkPhase = '' + runHook preCheck + export HOME=$PWD/home + mkdir -p $HOME + set +e + log=$(node apps/puller/bin/substrate-puller.mjs 2>&1) + rc=$? + set -e + printf '%s\n' "$log" | tail -n 40 + if [ "$rc" != 78 ]; then echo "expected exit 78 (config-invalid) with no config, got $rc"; exit 1; fi + printf '%s\n' "$log" | grep -q '"config-invalid"' + runHook postCheck + ''; + installPhase = '' + runHook preInstall + rm -rf home + mkdir -p $out/lib + cp -a . $out/lib/substrate-apps + makeWrapper ${lib.getExe nodejs_24} $out/bin/substrate-puller \ + --add-flags $out/lib/substrate-apps/apps/puller/bin/substrate-puller.mjs + runHook postInstall + ''; + passthru = { inherit sourceSha; }; + meta = { + description = "The Substrate interpreter host: leases runs from the floor and runs them on this box's runtimes"; + mainProgram = "substrate-puller"; + platforms = lib.platforms.linux; + }; + }; +in +{ + inherit + link + pusher + puller + pnpmDeps + sourceSha + ; +} diff --git a/pkgs/substrate-apps/src/apps/link/package.json b/pkgs/substrate-apps/src/apps/link/package.json new file mode 100644 index 00000000..5e63fcc6 --- /dev/null +++ b/pkgs/substrate-apps/src/apps/link/package.json @@ -0,0 +1,21 @@ +{ + "name": "@substrate/link", + "private": true, + "type": "module", + "version": "0.1.0", + "description": "The outbound link on the hypervisor host: leases AgentJobs from the floor over /rpc (Lease, Heartbeat, Complete) and runs each as one ax Task (create-only, durable verdict outbox).", + "license": "UNLICENSED", + "exports": { ".": "./src/link.ts", "./*": "./src/*" }, + "scripts": { "check": "tsc --noEmit -p .", "test": "vitest run" }, + "dependencies": { + "@grpc/grpc-js": "^1.12.6", + "@grpc/proto-loader": "^0.7.13" + }, + "peerDependencies": { "effect": "4.0.0-rc.112" }, + "devDependencies": { + "@types/node": "^22.10.2", + "effect": "4.0.0-rc.112", + "typescript": "^5.9.3", + "vitest": "^3.2.4" + } +} diff --git a/pkgs/substrate-apps/src/apps/link/proto/ax-p1.proto b/pkgs/substrate-apps/src/apps/link/proto/ax-p1.proto new file mode 100644 index 00000000..134a0d24 --- /dev/null +++ b/pkgs/substrate-apps/src/apps/link/proto/ax-p1.proto @@ -0,0 +1,406 @@ +// RUNTIME PROTO of conwip-link (round 2): the vendored ax.proto (v0.3.0 d8ed0fe) plus the three hunks of carried patch +// P1 that the link reads: rpc GetTaskResult, messages GetTaskResultRequest and TaskResult, UsageStats.tool_calls. +// Copied from dotfiles branch ax/fleet-bringup pkgs/ax/patches/p1-completion.patch (lines 3240-3300) on 2026-09-23. +// A superset of the stock proto: against a stock server GetTaskResult reaches the server, which answers UNIMPLEMENTED, +// so the capability probe (B6) is decided by the server, never by the client's proto. Keep in sync with P1. +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +syntax = "proto3"; + +package ax.v1alpha1; + +import "google/protobuf/struct.proto"; +import "google/protobuf/timestamp.proto"; + +option go_package = "github.com/google/ax/pkg/apis/v1alpha1"; + +// AX defines the core control plane gRPC service for autonomous agent orchestration. +service AX { + // Manifests are parsed by the client (see `ax apply`) and submitted through the + // typed Update* RPCs below; the server never receives raw YAML. + + // Tasks + rpc GetTask(GetTaskRequest) returns (Task); + rpc ListTasks(ListTasksRequest) returns (ListTasksResponse); + rpc UpdateTask(UpdateTaskRequest) returns (Task); + rpc DeleteTask(DeleteTaskRequest) returns (DeleteTaskResponse); + rpc SuspendTask(SuspendTaskRequest) returns (Task); + rpc ResumeTask(ResumeTaskRequest) returns (Task); + rpc WatchTask(WatchTaskRequest) returns (stream WatchTaskResponse); + // GetTaskResult returns the result file the task's command wrote, as copied + // by the controller when the command exited. NotFound until then. + rpc GetTaskResult(GetTaskResultRequest) returns (TaskResult); + + // Gateways + rpc GetGateway(GetGatewayRequest) returns (Gateway); + rpc ListGateways(ListGatewaysRequest) returns (ListGatewaysResponse); + rpc UpdateGateway(UpdateGatewayRequest) returns (Gateway); + rpc DeleteGateway(DeleteGatewayRequest) returns (DeleteGatewayResponse); + + // Workspaces + rpc GetWorkspace(GetWorkspaceRequest) returns (Workspace); + rpc ListWorkspaces(ListWorkspacesRequest) returns (ListWorkspacesResponse); + rpc UpdateWorkspace(UpdateWorkspaceRequest) returns (Workspace); + rpc DeleteWorkspace(DeleteWorkspaceRequest) returns (DeleteWorkspaceResponse); + + // Models + rpc GetModel(GetModelRequest) returns (Model); + rpc ListModels(ListModelsRequest) returns (ListModelsResponse); + rpc UpdateModel(UpdateModelRequest) returns (Model); + rpc DeleteModel(DeleteModelRequest) returns (DeleteModelResponse); + +} + +// ObjectMeta is metadata for AX resources. +message ObjectMeta { + string name = 1; + string atespace = 2; + google.protobuf.Timestamp creation_timestamp = 3; +} + +// --- Task --- + +message Task { + string api_version = 1; + string kind = 2; + ObjectMeta metadata = 3; + TaskSpec spec = 4; + TaskStatus status = 5; +} + +message TaskSpec { + // Field 1 was `goal`, removed; the workspace goal lives on WorkspaceRef. + reserved 1; + reserved "goal"; + bool suspend = 2; + string image = 3; + repeated string command = 4; + repeated EnvVar env = 5; + ResourceReqs resources = 6; + // workspaces binds one or more Workspaces, each mounted at its own path + // under /workspace. The first entry is the task command's working directory. + repeated WorkspaceRef workspaces = 7; + GatewayRef gateway = 8; + // Field 9 was `policies` (budget and approval config), removed for now. + reserved 9; + reserved "policies"; + // debug enables the in-container guest services (process execution and file + // access) that back `ax ssh`. Off by default. + bool debug = 10; +} + +message EnvVar { + string name = 1; + string value = 2; +} + +message ResourceReqs { + ResourceList requests = 1; + ResourceList limits = 2; +} + +message ResourceList { + string cpu = 1; + string memory = 2; +} + +message WorkspaceRef { + string name = 1; + string path = 2; + string goal = 3; +} + +message GatewayRef { + string name = 1; +} + +message TaskStatus { + string phase = 1; + string id = 2; + string actor = 3; + // json_name keeps the established `workerIP` spelling in YAML and JSON. + string worker_ip = 4 [json_name = "workerIP"]; + PendingApproval pending_approval = 5; + UsageStats usage = 6; + repeated Condition conditions = 7; +} + +message PendingApproval { + string id = 1; + string action = 2; + google.protobuf.Timestamp requested_at = 3; +} + +message UsageStats { + int32 prompt_tokens = 1; + int32 completion_tokens = 2; + int32 tool_calls = 3; +} + +message Condition { + string type = 1; + string status = 2; + google.protobuf.Timestamp last_transition_time = 3; + string reason = 4; + string message = 5; +} + +// --- Gateway --- + +message Gateway { + string api_version = 1; + string kind = 2; + ObjectMeta metadata = 3; + GatewaySpec spec = 4; +} + +message GatewaySpec { + repeated Listener listeners = 1; + EgressConfig egress = 2; +} + +message Listener { + string name = 1; + int32 port = 2; + string protocol = 3; +} + +message EgressConfig { + EgressAllowlist allowlist = 1; +} + +message EgressAllowlist { + repeated HostRule hosts = 1; +} + +message HostRule { + string host = 1; + int32 port = 2; +} + +// --- Workspace --- + +message Workspace { + string api_version = 1; + string kind = 2; + ObjectMeta metadata = 3; + WorkspaceSpec spec = 4; +} + +message WorkspaceSpec { + repeated GitRepo git = 1; + MCPConfig mcp = 2; + SkillsConfig skills = 3; +} + +message GitRepo { + string name = 1; + string repo = 2; + string branch = 3; + string dir = 4; + int32 depth = 5; +} + +message MCPConfig { + repeated MCPRegistry registries = 1; + repeated MCPServer servers = 2; +} + +message MCPRegistry { + string provider = 1; + string project = 2; + string query = 3; + repeated MCPServer servers = 4; +} + +message MCPServer { + string name = 1; + string endpoint = 2; + string command = 3; + repeated string args = 4; +} + +message SkillsConfig { + repeated SkillRegistry registries = 1; + string path = 2; +} + +message SkillRegistry { + string provider = 1; + string project = 2; + string query = 3; +} + +// --- Model --- + +message Model { + string api_version = 1; + string kind = 2; + ObjectMeta metadata = 3; + ModelSpec spec = 4; +} + +message ModelSpec { + string provider = 1; + string model = 2; + // Fields 3-5 were typed temperature, max_tokens, and system_instruction; + // provider settings now live in the free-form `parameters` map. + reserved 3, 4, 5; + reserved "temperature", "max_tokens", "system_instruction"; + SecretKeyRef secret_key = 6; + // parameters are provider-specific generation settings passed through to the + // model API as-is, for example temperature or maxOutputTokens for Gemini. + google.protobuf.Struct parameters = 7; +} + +message SecretKeyRef { + string name = 1; + string key = 2; +} + +// --- RPC Request & Response Messages --- + +// Tasks +message GetTaskRequest { + string atespace = 1; + string name = 2; +} + +message GetTaskResultRequest { + string atespace = 1; + string name = 2; +} + +message TaskResult { + bytes content = 1; + string sha256 = 2; +} + +message ListTasksRequest { + string atespace = 1; + int64 limit = 2; + int64 offset = 3; +} + +message ListTasksResponse { + repeated Task tasks = 1; +} + +message UpdateTaskRequest { + Task task = 1; +} + +message DeleteTaskRequest { + string atespace = 1; + string name = 2; +} + +message DeleteTaskResponse {} + +message SuspendTaskRequest { + string atespace = 1; + string name = 2; +} + +message ResumeTaskRequest { + string atespace = 1; + string name = 2; +} + +message WatchTaskRequest { + string atespace = 1; + string name = 2; +} + +message WatchTaskResponse { + Task task = 1; + string action = 2; +} + +// Gateways +message GetGatewayRequest { + string atespace = 1; + string name = 2; +} + +message ListGatewaysRequest { + string atespace = 1; +} + +message ListGatewaysResponse { + repeated Gateway gateways = 1; +} + +message UpdateGatewayRequest { + Gateway gateway = 1; +} + +message DeleteGatewayRequest { + string atespace = 1; + string name = 2; +} + +message DeleteGatewayResponse {} + +// Workspaces +message GetWorkspaceRequest { + string atespace = 1; + string name = 2; +} + +message ListWorkspacesRequest { + string atespace = 1; +} + +message ListWorkspacesResponse { + repeated Workspace workspaces = 1; +} + +message UpdateWorkspaceRequest { + Workspace workspace = 1; +} + +message DeleteWorkspaceRequest { + string atespace = 1; + string name = 2; +} + +message DeleteWorkspaceResponse {} + +// Models +message GetModelRequest { + string atespace = 1; + string name = 2; +} + +message ListModelsRequest { + string atespace = 1; +} + +message ListModelsResponse { + repeated Model models = 1; +} + +message UpdateModelRequest { + Model model = 1; +} + +message DeleteModelRequest { + string atespace = 1; + string name = 2; +} + +message DeleteModelResponse {} + diff --git a/pkgs/substrate-apps/src/apps/link/src/ax.ts b/pkgs/substrate-apps/src/apps/link/src/ax.ts new file mode 100644 index 00000000..cdff0d5d --- /dev/null +++ b/pkgs/substrate-apps/src/apps/link/src/ax.ts @@ -0,0 +1,121 @@ +// The link's view of ax: five calls over plain gRPC (h2c), no CreateTask (ax has none; UpdateTask is an upsert, so +// the link only ever calls it after GetTask answered NotFound: decision L4, create-only). +// ListTasks is paged: ax-server answers limit 0 with 50 rows, newest first (MEASURED upstream +// internal/server/server.go:91-104), so `listAll` walks pages until a short one (B1, critique C1). +import { createHash } from "node:crypto" +import { isIPv4, isIPv6 } from "node:net" +import { dirname, resolve } from "node:path" +import { fileURLToPath } from "node:url" +import * as grpc from "@grpc/grpc-js" +import * as protoLoader from "@grpc/proto-loader" +import { Effect } from "effect" +import type { AxTask } from "./contract.ts" + +export class AxError { + readonly _tag = "AxError" + constructor(readonly code: number, readonly message: string) {} + get unavailable() { return this.code === grpc.status.UNAVAILABLE || this.code === grpc.status.DEADLINE_EXCEEDED || this.code === grpc.status.INTERNAL } + get resourceExhausted() { return this.code === grpc.status.RESOURCE_EXHAUSTED } +} +export const AX_PAGE = 50 // ax-server's own default page size + +export interface Condition { readonly type?: string; readonly status?: string; readonly reason?: string; readonly message?: string } +export interface AxObserved { + readonly name: string + readonly phase: string // "" and Pending before the controller acts; Running; Completed/Failed (P1); Terminating + readonly spec: { image?: string; command?: ReadonlyArray; env?: ReadonlyArray<{ name?: string; value?: string }>; gateway?: { name?: string } | null } + readonly conditions: ReadonlyArray + readonly usage?: { readonly promptTokens?: number; readonly completionTokens?: number; readonly toolCalls?: number } | null +} +/** `digestOk` is false when the server's sha256 does not match the bytes received, undefined when it sent none (B4). */ +export interface AxResult { readonly content: string; readonly sha256: string; readonly digestOk?: boolean } +/** A Gateway's egress, as ax applies it (MEASURED upstream internal/controller/reconciler.go:188-198 and + * internal/substrate/client.go:450-462): no egress or no allowlist means `*:443`, an empty host list applies no + * policy, and a `*` or `0.0.0.0/0` host allows everything. */ +export interface AxGateway { readonly hosts: ReadonlyArray<{ readonly host: string; readonly port: number }>; readonly hasAllowlist: boolean } +/** Round 2: ax sends every host containing "/" to Substrate as a CIDR rule (MEASURED upstream client.go:456-491), so an + * allowlist can be open by arithmetic (`::/0`, `0.0.0.0/1` + `128.0.0.0/1`). Fail closed: a CIDR that does not parse, + * or one at or wider than /8 (IPv4) or /16 (IPv6), counts as open. */ +export const cidrIsOpen = (host: string): boolean => { + const [addr = "", len, ...rest] = host.split("/") + if (rest.length > 0 || len === undefined || !/^\d{1,3}$/.test(len)) return true + const n = Number(len) + if (isIPv4(addr)) return n > 32 || n <= 8 + if (isIPv6(addr)) return n > 128 || n <= 16 + return true +} +export const gatewayAllowsAll = (g: AxGateway) => + !g.hasAllowlist || g.hosts.length === 0 || g.hosts.some((x) => { + const h = x.host.trim().toLowerCase() + return h === "*" || h === "" || (h.includes("/") && cidrIsOpen(h)) + }) + +/** Round 2: the link's own runtime proto, P1 included (see its header). `LINK_AX_PROTO_PATH` overrides it. */ +export const LINK_PROTO_PATH = resolve(dirname(fileURLToPath(import.meta.url)), "..", "proto", "ax-p1.proto") + +export interface AxApi { + readonly getTask: (name: string) => Effect.Effect // NotFound -> undefined + readonly createTask: (task: AxTask) => Effect.Effect // UpdateTask, called only after NotFound + /** One page of the executor's list (Buildkite informer); `limit` 0 means the server's 50. */ + readonly listTasks: (limit: number, offset: number) => Effect.Effect, AxError> + readonly getGateway: (name: string) => Effect.Effect // NotFound -> undefined + readonly deleteTask: (name: string) => Effect.Effect // NotFound counts as done + /** P1's GetTaskResult. `unimplemented` means the server predates P1 (stock v0.3.0): completion is the guest's (L7). */ + readonly getTaskResult: (name: string) => Effect.Effect +} + +/** Every page, until a short one. Rows can shift between pages while ax inserts or deletes; the link therefore + * confirms any "absent" Task with GetTask before acting on it (B1). */ +export const listAll = (ax: AxApi, page = AX_PAGE, maxPages = 400): Effect.Effect, AxError> => Effect.gen(function*() { + const out = new Map() + for (let i = 0; i < maxPages; i++) { + const rows = yield* ax.listTasks(page, i * page) + for (const t of rows) out.set(t.name, t) + if (rows.length < page) return [...out.values()] + } + return yield* Effect.fail(new AxError(grpc.status.OUT_OF_RANGE, `more than ${maxPages * page} Tasks in the atespace`)) +}) + +const observe = (t: any): AxObserved => ({ + name: t?.metadata?.name ?? "", + phase: t?.status?.phase ?? "", + spec: t?.spec ?? {}, + conditions: t?.status?.conditions ?? [], + usage: t?.status?.usage ?? null +}) + +/** The production AxApi: `address` is ax-server's `host:port` (the ClusterIP 10.201.0.80:8080 on the NAS). */ +export function grpcAx(address: string, atespace: string, deadlineMs = 10_000, protoPath: string = LINK_PROTO_PATH): AxApi & { close: () => void } { + const def = protoLoader.loadSync(protoPath, { keepCase: false, longs: String, enums: String, defaults: true, oneofs: true }) + const Ctor = (grpc.loadPackageDefinition(def) as any).ax?.v1alpha1?.AX + if (typeof Ctor !== "function") throw new Error(`ax.v1alpha1.AX not found in ${protoPath}`) + // Round 2: a proto without GetTaskResult would answer the P1 probe "unimplemented" locally, never asking the server + if (typeof Ctor.prototype?.GetTaskResult !== "function") throw new Error(`${protoPath} declares no GetTaskResult; the link needs its P1 proto`) + const client = new Ctor(address, grpc.credentials.createInsecure()) + const unary = (method: string, req: unknown) => Effect.callback((resume) => { + if (typeof client[method] !== "function") return resume(Effect.fail(new AxError(grpc.status.UNIMPLEMENTED, `${method} not in proto`))) + client[method](req, { deadline: new Date(Date.now() + deadlineMs) }, (err: grpc.ServiceError | null, res: T) => + resume(err ? Effect.fail(new AxError(err.code ?? grpc.status.UNKNOWN, err.details ?? err.message)) : Effect.succeed(res))) + }) + const notFound = (e: Effect.Effect, dflt: A) => + e.pipe(Effect.catchIf((x: AxError) => x.code === grpc.status.NOT_FOUND, () => Effect.succeed(dflt))) + return { + getTask: (name) => notFound(unary("GetTask", { atespace, name }).pipe(Effect.map(observe)), undefined), + createTask: (task) => unary("UpdateTask", { task }).pipe(Effect.asVoid), + listTasks: (limit, offset) => unary("ListTasks", { atespace, limit, offset }).pipe(Effect.map((r) => (r?.tasks ?? []).map(observe))), + getGateway: (name) => notFound(unary("GetGateway", { atespace, name }).pipe(Effect.map((g): AxGateway => { + const al = g?.spec?.egress?.allowlist + return { hasAllowlist: al != null, hosts: (al?.hosts ?? []).map((h: any) => ({ host: String(h?.host ?? ""), port: Number(h?.port ?? 0) })) } + })), undefined), + deleteTask: (name) => notFound(unary("DeleteTask", { atespace, name }).pipe(Effect.asVoid), undefined), + getTaskResult: (name) => notFound(unary("GetTaskResult", { atespace, name }).pipe( + Effect.map((r): AxResult => { + const bytes = Buffer.from(r?.content ?? "") + const sha256 = String(r?.sha256 ?? "") + return { content: bytes.toString("utf8"), sha256, ...(sha256 ? { digestOk: createHash("sha256").update(bytes).digest("hex") === sha256.toLowerCase() } : {}) } + }), + Effect.catchIf((x: AxError) => x.code === grpc.status.UNIMPLEMENTED, () => Effect.succeed("unimplemented" as const)) + ), undefined), + close: () => client.close() + } +} diff --git a/pkgs/substrate-apps/src/apps/link/src/config.ts b/pkgs/substrate-apps/src/apps/link/src/config.ts new file mode 100644 index 00000000..bb2bcc8e --- /dev/null +++ b/pkgs/substrate-apps/src/apps/link/src/config.ts @@ -0,0 +1,152 @@ +// conwip-link configuration from the environment (round 1, finding "config"). Every numeric key must be a plain +// non-negative integer inside its range and every enum key one of its literals; anything else is ConfigInvalid, which +// main.ts turns into exit 78 so systemd's RestartPreventExitStatus holds the unit down instead of letting it spin +// (Effect.sleep(NaN) is a zero sleep) or crash-loop. Errors name the key and the rule, never the value. +import { seatOf } from "./jobs.ts" +import type { LinkConfig } from "./link.ts" + +export class ConfigInvalid extends Error { + readonly _tag = "ConfigInvalid" + constructor(readonly key: string, readonly why: string) { super(`${key}: ${why}`) } +} + +type Env = Readonly> + +const str = (env: Env, key: string, d?: string): string => { + const v = env[key] + if (v !== undefined && v !== "") return v + if (d !== undefined) return d + throw new ConfigInvalid(key, "missing") +} + +export const int = (env: Env, key: string, d: number, min: number, max = 1_000_000): number => { + const v = str(env, key, String(d)).trim() + if (!/^\d+$/.test(v)) throw new ConfigInvalid(key, "not a non-negative integer") + const n = Number(v) + if (n < min || n > max) throw new ConfigInvalid(key, `outside ${min}..${max}`) + return n +} + +export const literal = (env: Env, key: string, d: L, allowed: ReadonlyArray): L => { + const v = str(env, key, d) + if (!(allowed as ReadonlyArray).includes(v)) throw new ConfigInvalid(key, `not one of ${allowed.join(", ")}`) + return v as L +} + +export const seatCommands = (env: Env, key: string, d: string): Record> => { + let parsed: unknown + try { parsed = JSON.parse(str(env, key, d)) } catch { throw new ConfigInvalid(key, "not JSON") } + if (parsed === null || typeof parsed !== "object" || Array.isArray(parsed)) throw new ConfigInvalid(key, "not a JSON object") + const out: Record> = {} + for (const [seat, cmd] of Object.entries(parsed)) { + if (!Array.isArray(cmd) || cmd.length === 0 || !cmd.every((c) => typeof c === "string" && c !== "")) + throw new ConfigInvalid(key, `seat ${JSON.stringify(seat)} is not a non-empty array of non-empty strings`) + out[seat] = cmd as Array + } + return out +} + +export interface LinkEnv { + readonly holder: string + readonly maxInFlight: number + readonly servedLabels: Array + readonly atespace: string + readonly image: string + readonly gateway: string + readonly axServer: string + readonly guestCompleteUrl: string | undefined + readonly seatCommands: Record> + readonly completion: "auto" | "p1" | "guest" + readonly resyncSeconds: number + readonly pendingTimeoutSeconds: number + readonly deleteAfterSeconds: number + readonly deadlineBackstopSeconds: number + readonly createAttempts: number + readonly outboxBackoffSeconds: readonly [number, number] + readonly fenceTimeoutSeconds: number + readonly resultReadTries: number + readonly verdictAttempts: number // round 2 + readonly maxOutputBytes: number // round 2: below the floor's 2 MB DO row limit + readonly internalHosts: Array // round 2: single-label names a guest Complete URL may use + readonly axProtoPath: string | undefined // round 2: the link's ax proto (must declare GetTaskResult) + readonly leaseKeyAttempts: number // critique D.2: was a LinkConfig field no environment key reached + readonly terminatingRetrySeconds: number // critique D.2 ladder: re-delete period + readonly terminatingEscalateSeconds: number // critique D.2 ladder: escalation by log (and Substrate read, if wired) + readonly terminatingGiveUpSeconds: number // critique D.2 ladder: delete-stuck +} + +/** Throws ConfigInvalid on the first bad key. The defaults here are the only defaults: the NixOS module + * (dotfiles hosts/nas/substrate-link.nix) sets none of the numeric keys yet (critique D.2 item 4). */ +export const readLinkEnv = (env: Env): LinkEnv => { + const outMin = int(env, "LINK_OUTBOX_BACKOFF_MIN_SECONDS", 60, 1, 86_400) + const outMax = int(env, "LINK_OUTBOX_BACKOFF_MAX_SECONDS", 900, 1, 86_400) + if (outMax < outMin) throw new ConfigInvalid("LINK_OUTBOX_BACKOFF_MAX_SECONDS", "below LINK_OUTBOX_BACKOFF_MIN_SECONDS") + const servedLabels = str(env, "LINK_SERVED_LABELS", "seat:halogen,runtime:gvisor").split(",").map((l) => l.trim()).filter((l) => l !== "") + if (servedLabels.length === 0) throw new ConfigInvalid("LINK_SERVED_LABELS", "empty") + const termRetry = int(env, "LINK_TERMINATING_RETRY_SECONDS", 60, 1, 3600) + const termEscalate = int(env, "LINK_TERMINATING_ESCALATE_SECONDS", 300, 1, 86_400) + const termGiveUp = int(env, "LINK_TERMINATING_GIVE_UP_SECONDS", 900, 1, 86_400) + if (termEscalate <= termRetry) throw new ConfigInvalid("LINK_TERMINATING_ESCALATE_SECONDS", "not above LINK_TERMINATING_RETRY_SECONDS") + if (termGiveUp <= termEscalate) throw new ConfigInvalid("LINK_TERMINATING_GIVE_UP_SECONDS", "not above LINK_TERMINATING_ESCALATE_SECONDS") + return { + holder: str(env, "LINK_HOLDER", "nas-link-1"), + maxInFlight: int(env, "LINK_MAX_IN_FLIGHT", 2, 1, 1000), + servedLabels, + atespace: str(env, "AX_ATESPACE", "fleet"), + image: str(env, "LINK_IMAGE", "ax-agent"), + gateway: str(env, "LINK_GATEWAY", "halogen"), + axServer: str(env, "AX_SERVER", "10.201.0.80:8080"), + guestCompleteUrl: env.LINK_GUEST_COMPLETE_URL ? env.LINK_GUEST_COMPLETE_URL : undefined, + seatCommands: seatCommands(env, "LINK_SEAT_COMMANDS", '{"halogen":["ax-agent","pi"]}'), + completion: literal(env, "LINK_COMPLETION", "auto", ["auto", "p1", "guest"]), + resyncSeconds: int(env, "LINK_RESYNC_SECONDS", 15, 1, 3600), + pendingTimeoutSeconds: int(env, "LINK_PENDING_TIMEOUT_SECONDS", 900, 1, 86_400), + deleteAfterSeconds: int(env, "LINK_DELETE_AFTER_SECONDS", 600, 0, 604_800), + deadlineBackstopSeconds: int(env, "LINK_DEADLINE_BACKSTOP_SECONDS", 300, 0, 86_400), + createAttempts: int(env, "LINK_CREATE_ATTEMPTS", 7, 1, 100), + outboxBackoffSeconds: [outMin, outMax], + fenceTimeoutSeconds: int(env, "LINK_FENCE_TIMEOUT_SECONDS", 120, 1, 86_400), + resultReadTries: int(env, "LINK_RESULT_READ_TRIES", 5, 1, 1000), + verdictAttempts: int(env, "LINK_VERDICT_ATTEMPTS", 8, 1, 1000), + maxOutputBytes: int(env, "LINK_MAX_OUTPUT_BYTES", 1_000_000, 1024, 2_000_000), + internalHosts: (env.LINK_INTERNAL_HOSTS ?? "").split(",").map((h) => h.trim()).filter((h) => h !== ""), + axProtoPath: env.LINK_AX_PROTO_PATH ? env.LINK_AX_PROTO_PATH : undefined, + leaseKeyAttempts: int(env, "LINK_LEASE_KEY_ATTEMPTS", 5, 1, 1000), + terminatingRetrySeconds: termRetry, + terminatingEscalateSeconds: termEscalate, + terminatingGiveUpSeconds: termGiveUp + } +} + +/** HF link-ladder (VERIFY LL-5, M11): the LinkEnv to LinkConfig mapping main.ts uses, pure so a test pins that every + * key reaches runLink (seconds become milliseconds; a real process runs at secondMs 1000). */ +export const linkConfigOf = (c: LinkEnv, floorUrls: ReadonlyArray): LinkConfig => ({ + holder: c.holder, + maxInFlight: c.maxInFlight, + servedLabels: c.servedLabels, + shape: { + atespace: c.atespace, + image: c.image, + gateway: c.gateway, + command: (job) => { const seat = seatOf(job); return seat === undefined ? undefined : c.seatCommands[seat] }, // B7: no default seat + ...(c.guestCompleteUrl !== undefined ? { completeUrl: c.guestCompleteUrl } : {}) + }, + completion: c.completion, + secondMs: 1000, + resyncMs: c.resyncSeconds * 1000, + pendingTimeoutMs: c.pendingTimeoutSeconds * 1000, + deleteAfterMs: c.deleteAfterSeconds * 1000, + deadlineBackstopMs: c.deadlineBackstopSeconds * 1000, + createAttempts: c.createAttempts, + outboxBackoffMs: [c.outboxBackoffSeconds[0] * 1000, c.outboxBackoffSeconds[1] * 1000], + fenceTimeoutMs: c.fenceTimeoutSeconds * 1000, + resultReadTries: c.resultReadTries, + verdictAttempts: c.verdictAttempts, + maxOutputBytes: c.maxOutputBytes, + internalHosts: c.internalHosts, + leaseKeyAttempts: c.leaseKeyAttempts, // critique D.2 item 4 + terminatingRetryMs: c.terminatingRetrySeconds * 1000, + terminatingEscalateMs: c.terminatingEscalateSeconds * 1000, + terminatingGiveUpMs: c.terminatingGiveUpSeconds * 1000, + floorUrls +}) diff --git a/pkgs/substrate-apps/src/apps/link/src/contract.ts b/pkgs/substrate-apps/src/apps/link/src/contract.ts new file mode 100644 index 00000000..383d8cc8 --- /dev/null +++ b/pkgs/substrate-apps/src/apps/link/src/contract.ts @@ -0,0 +1,188 @@ +// The floor <-> link wire contract (LINK-DESIGN.md section 3). Starting point: the option-A prototype +// (/home/tom/today/evals-2026-09-23/arc/proto/contract.ts, decision L1). Every L1 name is kept as it was. +// Five keys are ADDED, each optional so an L1-only peer ignores them, each named after its source: +// Grant.supersedes L3 (two-stage expiry): the old attempt the link must DeleteTask first +// Grant.leaseToken L7 (completion before ax P1): a per-lease token for the guest (Buildkite job token) +// Grant.deadline Temporal start-to-close, as an absolute epoch ms the link can hold across a restart +// Lease.endpoint Buildkite ping `endpoint`: move the link to a new floor URL after a test call +// Complete.withdrew round 2: the leaseId of attempt n+1 that rule 4b withdrew when it accepted attempt n's verdict. +// The link releases n+1 only when it is named here; it never infers a withdrawal (a duplicate +// answer or a delivered retryable failure withdrew nothing) +// Grant.reassignSeconds red team double-run-r1-1: when another holder may get n+1, so the link fences itself first +// Complete.withdrewTransitions round 3: the leaseTransitions of the generation rule 4b withdrew. A floor that +// requeues the withdrawn attempt under the same leaseId grants it again with a higher value; the +// link treats a grant as withdrawn only when its generation matches +// Nothing here is GitHub-hosted at runtime. +import { Schema } from "effect" +import { Rpc, RpcGroup } from "effect/unstable/rpc" + +// Actions job result values (GitHub/Gitea job `result`). +export const Result = Schema.Literals(["success", "failure", "cancelled", "skipped"]) +export type Result = typeof Result.Type + +// ---- AgentJob: an ultracode agent() node as an Actions-style job in a Kubernetes object envelope (unchanged) ---- +const Labels = Schema.Record(Schema.String, Schema.String) +export const PromptRef = Schema.Struct({ + sha256: Schema.String, // content address; the guest refuses on mismatch (FIELD-MAP 5a AX_CONWIP_PROMPT_SHA256) + bytes: Schema.Int, + uri: Schema.String // journal:////prompt.md (metadata: not part of a job's enqueue identity) +}) +export const AgentJob = Schema.Struct({ + apiVersion: Schema.Literal("ultracode.mecattaf.dev/v1alpha1"), + kind: Schema.Literal("AgentJob"), + metadata: Schema.Struct({ name: Schema.String, labels: Schema.optionalKey(Labels), annotations: Schema.optionalKey(Labels) }), + spec: Schema.Struct({ + // CONWIP admission key, e.g. ["seat:halogen","runtime:gvisor"]. B7 asked for Schema.NonEmptyArray here; it stays + // Array on the wire because one malformed job would then fail the decode of a whole Lease reply and strand every + // grant in it (INFERRED). The link refuses it per job instead (`validRunsOn`, jobs.ts), and the floor at enqueue. + "runs-on": Schema.Array(Schema.String), + "timeout-minutes": Schema.optionalKey(Schema.Int), // Temporal start-to-close, enforced by the floor + "continue-on-error": Schema.optionalKey(Schema.Boolean), + environment: Schema.optionalKey(Schema.NullOr(Schema.String)), // Tom's ack gate + with: Schema.Struct({ + prompt: Schema.optionalKey(Schema.String), + prompt_ref: PromptRef, + model: Schema.String, + effort: Schema.optionalKey(Schema.String), + schema: Schema.optionalKey(Schema.Unknown), + isolation: Schema.optionalKey(Schema.String), + "agent-type": Schema.optionalKey(Schema.String) + }), + needs: Schema.optionalKey(Schema.Record(Schema.String, Schema.Struct({ result: Result, outputs_ref: Schema.String }))), + outputs: Schema.optionalKey(Schema.Array(Schema.String)) + }) +}) +export type AgentJob = typeof AgentJob.Type + +// ---- ax Task JSON (protojson camelCase of ax.proto v0.3.0 d8ed0fe). The link emits it; it never rides the wire ---- +// apiVersion is ax's own constant `ax.io/v1alpha1` (ax pkg/apis/v1alpha1/types.go:31); the prototype's +// `ax/v1alpha1` passed only because ax stores what it is given. +export const AxTask = Schema.Struct({ + apiVersion: Schema.Literal("ax.io/v1alpha1"), + kind: Schema.Literal("Task"), + metadata: Schema.Struct({ name: Schema.String, atespace: Schema.String }), + spec: Schema.Struct({ + image: Schema.String, + command: Schema.Array(Schema.String), + env: Schema.Array(Schema.Struct({ name: Schema.String, value: Schema.String })), + gateway: Schema.optionalKey(Schema.Struct({ name: Schema.String })) + }) +}) +export type AxTask = typeof AxTask.Type + +// ---- the link contract ---- +// Kubernetes coordination.k8s.io/v1 LeaseSpec field names. +export const LeaseSpec = Schema.Struct({ + holderIdentity: Schema.String, leaseDurationSeconds: Schema.Int, + acquireTime: Schema.Number, renewTime: Schema.Number, leaseTransitions: Schema.Int +}) +// Absolute counts on every response plus an increasing seq (ARC scaleset statistics). +export const Stats = Schema.Struct({ seq: Schema.Int, cap: Schema.Int, wip: Schema.Int, queued: Schema.Int, done: Schema.Int }) +export const Grant = Schema.Struct({ + leaseId: Schema.String, // `-a`; it is also the ax Task name (the idempotency key on ax) + attempt: Schema.Int, + job: AgentJob, + lease: LeaseSpec, + supersedes: Schema.optionalKey(Schema.Array(Schema.String)), // L3 + leaseToken: Schema.optionalKey(Schema.String), // L7, only while the guest completes (pre-P1) + deadline: Schema.optionalKey(Schema.Number), // epoch ms; acquireTime + timeout-minutes + // Red team double-run-r1-1: seconds after the last renewal past which the floor may grant attempt n+1 to ANOTHER + // holder (lease + grace + the rule 7b pin). A link that cannot renew deletes its Task before then, since that holder's + // supersedes fence cannot reach this executor. + reassignSeconds: Schema.optionalKey(Schema.Number) +}) +export type Grant = typeof Grant.Type +export const Usage = Schema.Struct({ prompt_tokens: Schema.Int, completion_tokens: Schema.Int, tool_calls: Schema.Int }) +export type Usage = typeof Usage.Type + +// Temporal PollActivityTaskQueue + ARC free capacity on every poll + Forgejo request key + Buildkite intervals. +export const Lease = Rpc.make("Lease", { + payload: { holderIdentity: Schema.String, capacity: Schema.Int, requestKey: Schema.String }, + success: Schema.Struct({ + grants: Schema.Array(Grant), stats: Stats, nextPollSeconds: Schema.Int, heartbeatSeconds: Schema.Int, + endpoint: Schema.optionalKey(Schema.String) // Buildkite ping endpoint switch + }) +}) +// Temporal RecordActivityTaskHeartbeat with cancel_requested, batched per session. `leaseIds` is the COMPLETE set +// the holder believes it holds (level-triggered, ARC): an orphaned lease it omits is requeued (L3 a). +// Round 4: `pendingRequestKey` (Forgejo request key, B8) is the Lease requestKey the holder journaled and has not +// seen answered. The holder cannot list leaseIds it never received, so it vouches for the key instead: the floor +// renews every lease granted under that key and never releases one as an omitted orphan (L3 a) while it is named. +export const Heartbeat = Rpc.make("Heartbeat", { + payload: { holderIdentity: Schema.String, leaseIds: Schema.Array(Schema.String), pendingRequestKey: Schema.optionalKey(Schema.String) }, + success: Schema.Struct({ + renewed: Schema.Array(Schema.String), lost: Schema.Array(Schema.String), + cancelRequested: Schema.Array(Schema.String), stats: Stats + }) +}) +// Temporal RespondActivityTask{Completed,Failed,Canceled}. The verdict travels here, never as an exit code (ARC). +// The two errors are terminal: stop retrying (Buildkite 422). +export const CompleteError = Schema.Struct({ code: Schema.Literals(["unknown-lease", "stale-attempt"]) }) +export const Complete = Rpc.make("Complete", { + payload: { + leaseId: Schema.String, attempt: Schema.Int, result: Result, + output: Schema.optionalKey(Schema.Unknown), usage: Schema.optionalKey(Usage) + }, + success: Schema.Struct({ duplicate: Schema.Boolean, stats: Stats, withdrew: Schema.optionalKey(Schema.String), + withdrewTransitions: Schema.optionalKey(Schema.Int) }), + error: CompleteError +}) +export class FloorLink extends RpcGroup.make(Lease, Heartbeat, Complete) {} + +// The link's own view of the same wire (round 1, finding "poison grant"): identical tags, payloads and JSON, but each +// grant is decoded by the link one at a time, so one malformed job fails only its own lease (pre-start/invalid-spec) +// instead of turning the whole Lease reply into a decode defect that the journaled requestKey would replay for ever. +// Round 3: the whole envelope is lenient too. A drifted interval (the prototype floor sends leaseMs / 3000, e.g. +// 3.333), a null endpoint or a changed stats shape would otherwise fail the decode of every replay of the journaled +// requestKey and strand the grants in it. The link bounds the intervals itself (link.ts `boundedSeconds`). +export const LeaseLenient = Rpc.make("Lease", { + payload: { holderIdentity: Schema.String, capacity: Schema.Int, requestKey: Schema.String }, + success: Schema.Struct({ + grants: Schema.Array(Schema.Unknown), stats: Schema.optionalKey(Schema.Unknown), + nextPollSeconds: Schema.optionalKey(Schema.Unknown), heartbeatSeconds: Schema.optionalKey(Schema.Unknown), + endpoint: Schema.optionalKey(Schema.Unknown) + }) +}) +// Round 2: the link decodes Complete's error code as any string, so a code the floor adds later is a permanent refusal +// (dropped, logged with the code), never a decode defect retried for ever at the head of the outbox. +// Round 3: its success is lenient as well. Any 2xx Exit Success means the floor applied the verdict; a drifted key +// (`withdrew: null`, a missing stats) must not keep an applied verdict at the head of the outbox. +export const CompleteLenient = Rpc.make("Complete", { + payload: Complete.payloadSchema.fields, + success: Schema.Struct({ duplicate: Schema.optionalKey(Schema.Unknown), stats: Schema.optionalKey(Schema.Unknown), + withdrew: Schema.optionalKey(Schema.Unknown), withdrewTransitions: Schema.optionalKey(Schema.Unknown) }), + error: Schema.Struct({ code: Schema.String }) +}) +// Round 4: Heartbeat is lenient as well. The strict reply schema turned a drifted `stats` into a heartbeat-error on +// every call, so `cancelRequested`, `lost` and `renewed` were never read while the floor kept renewing (a cancel was +// never carried out, a journaled grant never resumed). Every key is optional Unknown; floor.ts keeps the strings. +export const HeartbeatLenient = Rpc.make("Heartbeat", { + payload: Heartbeat.payloadSchema.fields, + success: Schema.Struct({ renewed: Schema.optionalKey(Schema.Unknown), lost: Schema.optionalKey(Schema.Unknown), + cancelRequested: Schema.optionalKey(Schema.Unknown), stats: Schema.optionalKey(Schema.Unknown) }) +}) +export class FloorLinkClient extends RpcGroup.make(LeaseLenient, HeartbeatLenient, CompleteLenient) {} + +// On result "failure" the link's `output` is this shape (Buildkite finish signal_reason, ARC failure reason). +// The reasons in RETRYABLE are the floor's to retry as attempt+1 (Temporal retry policy, Buildkite finish -1 on a +// pre-start failure); every other reason is final: a deterministic refusal, or the agent's own failure, which goes +// to the interpreter (null-on-failure). +export const FailureOutput = Schema.Struct({ + reason: Schema.String, // pre-start/env-over-budget, pre-start/name-conflict, infra/task-lost, deadline-exceeded, ... + message: Schema.optionalKey(Schema.String), + phase: Schema.optionalKey(Schema.String), + exitCode: Schema.optionalKey(Schema.Int) +}) +export type FailureOutput = typeof FailureOutput.Type +export const RETRYABLE = new Set([ + "pre-start/ax-unavailable", "pre-start/pending-timeout", "pre-start/resource-exhausted", + "infra/task-lost", "infra/actor-crashed", "infra/result-unreadable", + "pre-start/superseded-verdict-pending", // round 1: n+1 refused while attempt n's verdict could not be delivered + "infra/link-defect" // round 4: a dispatch fiber died on a defect before the Task was created; the floor requeues +]) +// Round 2, final (not in RETRYABLE): `infra/verdict-undeliverable` (the floor answered every delivery of the real +// verdict with a server error; a rerun would most likely produce the same bytes) and `agent/output-too-large` (the +// result is over the link's cap, below the floor's 2 MB row limit). Critique D.2: `infra/delete-stuck` (a superseded +// attempt stayed Terminating through the fence timeout; retried, the floor would place attempt n+1 on the same holder, +// which fences the same stuck name again, so it is final until the floor can avoid a holder). +export const isRetryableReason = (reason: string) => RETRYABLE.has(reason) diff --git a/pkgs/substrate-apps/src/apps/link/src/floor.ts b/pkgs/substrate-apps/src/apps/link/src/floor.ts new file mode 100644 index 00000000..caef5f23 --- /dev/null +++ b/pkgs/substrate-apps/src/apps/link/src/floor.ts @@ -0,0 +1,169 @@ +// The link's view of the floor: the three L1 RPCs over Effect's stock HTTP RPC client (decision L2), outbound only. +// HTTP statuses the floor's Worker entry answers before any RPC runs are classified here (Buildkite's retry +// classifier, api/retryable.go): 401/403 fatal (token), 409 fatal (a second session for this identity, L5), +// 429 back off honouring Retry-After. Round 2: any 3xx is fatal (`redirect`: redirects are never followed, so the only +// way to another floor URL is the Lease `endpoint` checked against the Nix list, B18); 404 and 405 are fatal +// (`misrouted`: a wrong route answers every call so, and dropping verdicts for it would lose them); any other 4xx except 408 is +// `rejected` (permanent for these bytes: Complete drops the verdict); a 500, 501 or 505+ and an RPC Defect in the reply +// are `server-error` (the floor answered and refused these bytes; the outbox counts them per verdict); 502, 503, 504, +// 408, network errors and timeouts stay `transient`. Every call has a ceiling (B9). +import { Effect, Layer, Schema, Scope } from "effect" +import { FetchHttpClient, HttpClient, HttpClientRequest } from "effect/unstable/http" +import { RpcClient, RpcSerialization } from "effect/unstable/rpc" +import { Complete, FloorLinkClient, Grant } from "./contract.ts" +import type { Usage, Result } from "./contract.ts" + +/** A grant the link could not decode. `leaseId` and `attempt` are set when they alone decode, so the lease can be + * failed back to the floor as pre-start/invalid-spec; otherwise the floor's own expiry is the only way out. */ +export interface InvalidGrant { readonly leaseId?: string; readonly attempt?: number; readonly message: string } +/** Round 3: the reply as the link uses it, after the lenient envelope decode. Intervals and endpoint are kept only + * when they have the right JSON type; the link bounds the intervals (link.ts `boundedSeconds`). */ +export type LeaseReply = { + readonly grants: ReadonlyArray; readonly invalid?: ReadonlyArray + readonly stats?: unknown; readonly nextPollSeconds?: number; readonly heartbeatSeconds?: number; readonly endpoint?: string +} +/** Round 4: the Heartbeat reply after the lenient decode: each list keeps only its strings, a missing one is []. */ +type HeartbeatReply = { readonly renewed: ReadonlyArray; readonly lost: ReadonlyArray; readonly cancelRequested: ReadonlyArray; readonly stats?: unknown } +const strings = (v: unknown): Array => Array.isArray(v) ? v.filter((x): x is string => typeof x === "string") : [] +const heartbeatReply = (r: { readonly renewed?: unknown; readonly lost?: unknown; readonly cancelRequested?: unknown; readonly stats?: unknown }): HeartbeatReply => + ({ renewed: strings(r.renewed), lost: strings(r.lost), cancelRequested: strings(r.cancelRequested), stats: r.stats }) +type CompleteReply = { readonly duplicate: boolean; readonly withdrew?: string; readonly withdrewTransitions?: number; readonly stats?: unknown } +const num = (v: unknown) => typeof v === "number" ? v : undefined +const leaseReply = (r: { readonly grants: ReadonlyArray; readonly nextPollSeconds?: unknown; readonly heartbeatSeconds?: unknown; readonly endpoint?: unknown; readonly stats?: unknown }): LeaseReply => { + const s = splitGrants(r) + const poll = num(r.nextPollSeconds), hb = num(r.heartbeatSeconds) + return { grants: s.grants, invalid: s.invalid, stats: r.stats, ...(poll !== undefined ? { nextPollSeconds: poll } : {}), + ...(hb !== undefined ? { heartbeatSeconds: hb } : {}), ...(typeof r.endpoint === "string" ? { endpoint: r.endpoint } : {}) } +} +const completeReply = (r: { readonly duplicate?: unknown; readonly withdrew?: unknown; readonly withdrewTransitions?: unknown; readonly stats?: unknown }): CompleteReply => { + const wt = num(r.withdrewTransitions) + return { duplicate: r.duplicate === true, stats: r.stats, ...(typeof r.withdrew === "string" ? { withdrew: r.withdrew } : {}), + ...(wt !== undefined && Number.isInteger(wt) ? { withdrewTransitions: wt } : {}) } +} +const encodeComplete = Schema.encodeUnknownExit(Complete.payloadSchema) + +export type FloorErrorKind = "transient" | "auth" | "session-conflict" | "rate-limited" | "redirect" | "misrouted" | "rejected" | "server-error" +export class FloorError { + readonly _tag = "FloorError" + constructor(readonly kind: FloorErrorKind, readonly message: string, readonly retryAfterMs?: number, readonly status?: number) {} + get fatal() { return this.kind === "auth" || this.kind === "session-conflict" || this.kind === "redirect" || this.kind === "misrouted" } +} +/** The floor refused these bytes for good: one of its CompleteError codes (any string, round 2), or an HTTP 4xx. */ +export class CompleteRefused { + readonly _tag = "CompleteRefused" + constructor(readonly code: string) {} +} + +export interface CompletePayload { leaseId: string; attempt: number; result: Result; output?: unknown; usage?: Usage } +export interface FloorApi { + readonly lease: (p: { holderIdentity: string; capacity: number; requestKey: string }) => Effect.Effect + readonly heartbeat: (p: { holderIdentity: string; leaseIds: ReadonlyArray; pendingRequestKey?: string }) => Effect.Effect + readonly complete: (p: CompletePayload) => Effect.Effect +} + +export interface FloorOptions { + readonly url: string // e.g. https://substrate..workers.dev (no trailing /rpc) + readonly token: string // the per-link bearer, read from $CREDENTIALS_DIRECTORY; never logged + readonly sessionId: string // stable per state directory: a restart is the same session, a second replica is not + readonly fetch?: typeof globalThis.fetch // tests hand in an in-process handler; production uses globalThis.fetch + /** B9: a ceiling on every call (Buildkite's 330 s on acquire, shortened): Lease and Heartbeat 20 s, Complete 60 s. */ + readonly timeoutsMs?: { readonly lease: number; readonly heartbeat: number; readonly complete: number } +} +export const DEFAULT_FLOOR_TIMEOUTS_MS = { lease: 20_000, heartbeat: 20_000, complete: 60_000 } as const + +/** B9 (critique R9): a non-2xx answer is thrown by the fetch wrapper of the call that received it, so its status + * travels inside that call's own error chain; nothing is shared between concurrent calls. */ +class HttpStatus extends Error { + readonly linkHttpStatus = true + constructor(readonly status: number, readonly retryAfterMs: number | undefined) { super(`floor answered ${status}`) } +} +const statusIn = (e: unknown): HttpStatus | undefined => { + let x: any = e + for (let i = 0; i < 10 && x != null && typeof x === "object"; i++) { + if (x instanceof HttpStatus || x.linkHttpStatus === true) return x as HttpStatus + x = x.reason ?? x.cause ?? x.error + } + return undefined +} +const decodeGrant = Schema.decodeUnknownExit(Grant) +const Envelope = Schema.decodeUnknownExit(Schema.Struct({ leaseId: Schema.String, attempt: Schema.Int })) +/** Decode each grant on its own: the good ones are dispatched, the bad ones are refused one by one. */ +export const splitGrants = }>(r: R): Omit & { grants: Array; invalid: Array } => { + const grants: Array = [], invalid: Array = [] + for (const raw of r.grants) { + const g = decodeGrant(raw) + if (g._tag === "Success") { grants.push(g.value); continue } + const env = Envelope(raw) + const message = String(g.cause).slice(0, 500) + invalid.push(env._tag === "Success" ? { leaseId: env.value.leaseId, attempt: env.value.attempt, message } : { message }) + } + return { ...r, grants, invalid } +} + +export const classifyFloorError = (e: unknown): FloorError => { + const st = statusIn(e) + const s = st?.status ?? 0 + const msg = e instanceof Error ? e.message : String((e as any)?.message ?? e) + if (s >= 300 && s < 400) return new FloorError("redirect", `floor answered ${s}: redirects are never followed; fix LINK_FLOOR_URL`, undefined, s) + if (s === 401 || s === 403) return new FloorError("auth", `floor answered ${s}`, undefined, s) + if (s === 409) return new FloorError("session-conflict", "floor answered 409: another session holds this link identity", undefined, s) + if (s === 429) return new FloorError("rate-limited", "floor answered 429", st?.retryAfterMs, s) + // 404/405: the route is wrong for every call, not these bytes; fatal (exit 78), so no verdict is dropped for it + if (s === 404 || s === 405) return new FloorError("misrouted", `floor answered ${s}: fix LINK_FLOOR_URL or the Worker route`, undefined, s) + if (s >= 400 && s < 500 && s !== 408) return new FloorError("rejected", `floor answered ${s}`, undefined, s) + if (s >= 500 && s !== 502 && s !== 503 && s !== 504) return new FloorError("server-error", `floor answered ${s}`, undefined, s) + return new FloorError("transient", s ? `floor answered ${s}` : msg, undefined, s || undefined) +} +/** Round 3: a defect raised by a call is always the reply's: the link encodes the Complete payload itself before it + * sends (a failure there is `transient`, the link's own bug), and the envelopes it decodes are lenient. A defect is + * then the floor's RPC `Defect` reply (e.g. SQLITE_TOOBIG) or a reply that is not the protocol at all: the server + * refusing these bytes (`server-error`), counted per verdict on Complete and per requestKey on Lease. */ +export const rpcFloor = (o: FloorOptions): Effect.Effect => Effect.gen(function*() { + const base = o.fetch ?? globalThis.fetch + const t = o.timeoutsMs ?? DEFAULT_FLOOR_TIMEOUTS_MS + const observing: typeof globalThis.fetch = async (input, init) => { + // Round 2: never follow a redirect. A followed 307 lands on an origin the Nix list never declared, and the link + // would act on its replies (B18 bypass); an Access 302 to a login page would read as a transient decode error. + const res = await base(input, { ...init, redirect: "manual" }) + if (res.type === "opaqueredirect") throw new HttpStatus(307, undefined) + if (!res.ok) { + const ra = Number(res.headers.get("retry-after")) + throw new HttpStatus(res.status, Number.isFinite(ra) && ra > 0 ? ra * 1000 : undefined) + } + return res + } + const protocol = RpcClient.layerProtocolHttp({ + url: `${o.url.replace(/\/$/, "")}/rpc`, + transformClient: HttpClient.mapRequest((r) => r.pipe( + HttpClientRequest.bearerToken(o.token), + HttpClientRequest.setHeader("x-link-session", o.sessionId))) + }).pipe(Layer.provide([FetchHttpClient.layer, RpcSerialization.layerJson]), + Layer.provide(Layer.succeed(FetchHttpClient.Fetch, observing))) + const client = yield* RpcClient.make(FloorLinkClient).pipe(Effect.provide(protocol)) + // The timeout interrupts the call, which aborts its fetch (FetchHttpClient passes an AbortSignal). + const within = (ms: number, what: string) => (e: Effect.Effect) => + e.pipe(Effect.timeoutOrElse({ duration: ms, orElse: () => Effect.fail(new FloorError("transient", `${what} timed out after ${ms} ms`)) })) + return { + // Round 1: a reply that does not decode is a typed transient failure, never a defect that kills the link. + lease: (p) => client.Lease(p).pipe(Effect.catchDefect((d) => Effect.fail(new FloorError("server-error", `Lease reply undecodable: ${String(d).slice(0, 300)}`))), + Effect.mapError((e) => e instanceof FloorError ? e : classifyFloorError(e)), Effect.map(leaseReply), within(t.lease, "Lease")), + // Round 4: lenient like Lease and Complete; a reply that still does not decode is the server's (server-error). + heartbeat: (p) => client.Heartbeat({ holderIdentity: p.holderIdentity, leaseIds: p.leaseIds, ...(p.pendingRequestKey !== undefined ? { pendingRequestKey: p.pendingRequestKey } : {}) }).pipe( + Effect.catchDefect((d) => Effect.fail(new FloorError("server-error", `Heartbeat reply undecodable: ${String(d).slice(0, 300)}`))), + Effect.mapError((e) => e instanceof FloorError ? e : classifyFloorError(e)), Effect.map(heartbeatReply), within(t.heartbeat, "Heartbeat")), + // L9: omit absent optional keys; `usage: undefined` in an optionalKey field is an encode defect, not a failure. + complete: (p) => { + const payload = { leaseId: p.leaseId, attempt: p.attempt, result: p.result, + ...(p.output !== undefined ? { output: p.output } : {}), ...(p.usage !== undefined ? { usage: p.usage } : {}) } + const enc = encodeComplete(payload) + if (enc._tag === "Failure") return Effect.fail(new FloorError("transient", `encode defect: ${String(enc.cause).slice(0, 300)}`)) + return client.Complete(payload).pipe(Effect.map(completeReply), + Effect.catchDefect((d) => Effect.fail(new FloorError("server-error", `server defect: ${String((d as any)?.message ?? d).slice(0, 300)}`))), + Effect.mapError((e: any) => e instanceof FloorError ? e + : e && typeof e === "object" && "code" in e && !("_tag" in e) ? new CompleteRefused(String(e.code)) : classifyFloorError(e)), + // Round 2: a 4xx other than 401/403/408/409/429 refuses these bytes for good (Buildkite: not retryable) + Effect.mapError((e) => e instanceof FloorError && e.kind === "rejected" ? new CompleteRefused(`http-${e.status}`) : e), + within(t.complete, "Complete")) + } + } +}) diff --git a/pkgs/substrate-apps/src/apps/link/src/jobs.ts b/pkgs/substrate-apps/src/apps/link/src/jobs.ts new file mode 100644 index 00000000..830bfa8d --- /dev/null +++ b/pkgs/substrate-apps/src/apps/link/src/jobs.ts @@ -0,0 +1,120 @@ +// Pure mapping AgentJob -> ax Task JSON (FIELD-MAP-2026-09-23.md 5a keys). Ported from the option-A prototype +// (arc/proto/jobs.ts) with ARC-DECISION X1 applied: atespace `fleet`, image `ax-agent`, Gateway `halogen`, the names +// ax-fleet DESIGN.md 10.2 uses. No I/O: the link and any converter call the same function. +import { createHash } from "node:crypto" +import type { AgentJob, AxTask, Grant } from "./contract.ts" + +export const ENV_BUDGET = 16384 // FIELD-MAP 5a hard pre-dispatch check (stock v0.3.0, before P5) +const A = "ultracode.mecattaf.dev/" +const utf8Len = (s: string) => new TextEncoder().encode(s).length +/** FIELD-MAP 5a journal key: `:` (substrate taskspec.ts journalKey). */ +export const JOURNAL_KEY = /^[0-9a-f]{64}:\d+$/ + +export interface TaskShape { + readonly atespace: string // `fleet` + readonly image: string // `localhost:5000/ax-agent@sha256:...` (ax-fleet-image-ref) + readonly gateway: string // `halogen` + readonly command: (job: AgentJob) => ReadonlyArray | undefined // e.g. ["ax-agent", "pi"] for seat:halogen + readonly completeUrl?: string // set only in guest-completion mode (L7, pre-P1) + readonly holder?: string // round 1: AX_CONWIP_HOLDER, the ownership mark a journal-less link checks before a fence +} + +/** B7: no default seat. `undefined` unless `runs-on` names exactly one `seat:` label. */ +export const seatOf = (job: AgentJob): string | undefined => { + const seats = job.spec["runs-on"].filter((l) => l.startsWith("seat:")) + return seats.length === 1 ? seats[0]!.slice(5) : undefined +} + +/** B7: `runs-on` must be non-empty, name exactly one `seat:` label, and be wholly served by this link. */ +export const validRunsOn = (job: AgentJob, served: ReadonlyArray): { reason: string; message: string } | undefined => { + const on = job.spec["runs-on"] + if (on.length === 0) return { reason: "pre-start/runs-on-invalid", message: "runs-on is empty" } + const seats = on.filter((l) => l.startsWith("seat:")) + if (seats.length !== 1) return { reason: "pre-start/runs-on-invalid", message: `runs-on names ${seats.length} seat: labels, not 1` } + const unserved = on.filter((l) => !served.includes(l)) + if (unserved.length > 0) return { reason: "pre-start/label-not-served", message: unserved.join(",") } + return undefined +} + +export type Built = { readonly _tag: "ok"; readonly task: AxTask } | { readonly _tag: "refused"; readonly reason: string; readonly message: string } + +export function axTaskFromGrant(g: Grant, shape: TaskShape): Built { + const job = g.job + // Round 1 (fail closed): a guest Task without its per-lease token has no completion route; never create it. + if (shape.completeUrl !== undefined && (g.leaseToken === undefined || g.leaseToken === "")) + return { _tag: "refused", reason: "pre-start/lease-token-missing", message: "guest completion needs a leaseToken; the floor's token binding for this holder is not guest" } + const ann = job.metadata.annotations ?? {} + const lab = job.metadata.labels ?? {} + if (ann[A + "prompt-unresolved"] !== undefined) // L8: never dispatch an item whose prompt was not resolved + return { _tag: "refused", reason: "pre-start/prompt-unresolved", message: ann[A + "prompt-unresolved"]! } + // Round 4 (FIELD-MAP 5a): the journal key is sha256 of canonical [prompt, opts minus label] plus the occurrence, and + // only the floor-side converter can compute it (it holds opts and the occurrence). It rides the AgentJob as an + // annotation; without it the grant is refused, never replaced by the prompt digest (two calls sharing one prompt). + const journalKey = ann[A + "journal-key"] + if (journalKey === undefined || !JOURNAL_KEY.test(journalKey)) + return { _tag: "refused", reason: "pre-start/invalid-spec", message: journalKey === undefined ? `annotation ${A}journal-key is missing` : `annotation ${A}journal-key is not :` } + const w = job.spec.with + const seat = seatOf(job) + if (seat === undefined) return { _tag: "refused", reason: "pre-start/runs-on-invalid", message: "runs-on does not name exactly one seat: label" } + const command = shape.command(job) + if (command === undefined || command.length === 0) return { _tag: "refused", reason: "pre-start/seat-command-missing", message: `no command for seat ${seat}` } + const env: Array<{ name: string; value: string }> = [ + { name: "AX_CONWIP_RUN_ID", value: ann[A + "run-id-raw"] ?? "" }, + { name: "AX_CONWIP_LABEL", value: ann[A + "label"] ?? "" }, + { name: "AX_CONWIP_MODEL", value: w.model }, + { name: "AX_CONWIP_ITEM_KEY", value: ann[A + "item-key"] ?? "" }, + { name: "AX_CONWIP_JOURNAL_KEY", value: journalKey }, + { name: "AX_CONWIP_WORKFLOW", value: lab[A + "workflow"] ?? "" }, + { name: "AX_CONWIP_PHASE_INDEX", value: lab[A + "phase-index"] ?? "" }, + { name: "AX_CONWIP_PHASE_TITLE", value: ann[A + "phase-title"] ?? "" }, + { name: "AX_CONWIP_SEAT", value: seat }, + { name: "AX_CONWIP_ATTEMPT", value: String(g.attempt) }, + { name: "AX_CONWIP_LEASE_ID", value: g.leaseId }, + ...(shape.holder !== undefined ? [{ name: "AX_CONWIP_HOLDER", value: shape.holder }] : []), + { name: "AX_CONWIP_PROMPT_SHA256", value: w.prompt_ref.sha256 }, + { name: "AX_CONWIP_RESULT_PATH", value: "/workspace/.ax/result.json" }, + { name: "AX_CONWIP_USAGE_PATH", value: "/workspace/.ax/usage.json" } + ] + if (w.effort !== undefined) env.push({ name: "AX_CONWIP_EFFORT", value: w.effort }) + if (w.schema !== undefined) env.push({ name: "AX_CONWIP_SCHEMA_JSON", value: JSON.stringify(w.schema) }) + if (w.isolation !== undefined) env.push({ name: "AX_CONWIP_ISOLATION", value: w.isolation }) + if (w["agent-type"] !== undefined) env.push({ name: "AX_CONWIP_AGENT_TYPE", value: w["agent-type"] }) + if (job.spec["timeout-minutes"] !== undefined) + env.push({ name: "AX_CONWIP_TIMEOUT_MINUTES", value: String(job.spec["timeout-minutes"]) }) + if (shape.completeUrl !== undefined && g.leaseToken !== undefined) { // L7 (a missing token was refused above): a per-lease token, never the fleet bearer + env.push({ name: "AX_CONWIP_COMPLETE_URL", value: shape.completeUrl }) + env.push({ name: "AX_CONWIP_LEASE_TOKEN", value: g.leaseToken }) + } + const used = env.reduce((n, e) => n + utf8Len(e.value), 0) + // B12 (critique R2): the file route (AX_CONWIP_PROMPT_FILE) needs a Workspace that writes the file, and the Task + // carries none yet. Until that route exists a prompt that does not fit inline is refused, never sent to a dead path. + if (used > ENV_BUDGET) // FIELD-MAP D4 and probe test 4: over budget ax falls back silently, so refuse here + return { _tag: "refused", reason: "pre-start/env-over-budget", message: `env values ${used} B > ${ENV_BUDGET} B before the prompt` } + if (w.prompt === undefined || used + utf8Len(w.prompt) > ENV_BUDGET) + return { _tag: "refused", reason: "pre-start/prompt-file-route-missing", + message: w.prompt === undefined ? "no inline prompt and no Workspace route" : `prompt ${utf8Len(w.prompt)} B does not fit the ${ENV_BUDGET - used} B left inline` } + env.push({ name: "AX_CONWIP_PROMPT", value: w.prompt }) + const total = env.reduce((n, e) => n + utf8Len(e.value), 0) + if (total > ENV_BUDGET) // FIELD-MAP D4 and probe test 4: over budget ax falls back silently, so refuse here + return { _tag: "refused", reason: "pre-start/env-over-budget", message: `env values ${total} B > ${ENV_BUDGET} B` } + return { + _tag: "ok", + task: { + apiVersion: "ax.io/v1alpha1", + kind: "Task", + metadata: { name: g.leaseId, atespace: shape.atespace }, + spec: { image: shape.image, command: [...command], env, gateway: { name: shape.gateway } } + } + } +} + +// The spec digest used by create-only adoption (L4): found with the same digest = adopt, another = conflict. +// Only the fields the link writes are hashed, in a fixed order, so ax defaults added on the server do not count. +export const specDigest = (spec: { image?: string; command?: ReadonlyArray; env?: ReadonlyArray<{ name?: string; value?: string }>; gateway?: { name?: string } | null }) => + createHash("sha256").update(JSON.stringify([ + spec.image ?? "", [...(spec.command ?? [])], + [...(spec.env ?? [])].map((e) => [e.name ?? "", e.value ?? ""]), + spec.gateway?.name ?? "" + ])).digest("hex") + +export const envBytes = (t: AxTask) => t.spec.env.reduce((n, e) => n + utf8Len(e.value), 0) diff --git a/pkgs/substrate-apps/src/apps/link/src/journal.ts b/pkgs/substrate-apps/src/apps/link/src/journal.ts new file mode 100644 index 00000000..83b04be4 --- /dev/null +++ b/pkgs/substrate-apps/src/apps/link/src/journal.ts @@ -0,0 +1,204 @@ +// The link's crash-safe local state: one append-only JSONL file, fsynced per line, written BEFORE the action it +// records (the tally uplink's "offline is a queue" outbox, kept verbatim in spirit: done / keep / drop). The ax +// store is the ledger of Tasks (Buildkite: the cluster is the ledger); this file only covers what ax cannot know: +// grants not yet created, and Complete reports not yet acknowledged by the floor. +import { appendFileSync, closeSync, existsSync, fsyncSync, mkdirSync, openSync, readFileSync, renameSync, writeSync } from "node:fs" +import { join } from "node:path" +import type { Grant, Result, Usage } from "./contract.ts" + +export type Entry = + | { ev: "grant"; leaseId: string; grant: Grant; regrant?: true } // round 2: `regrant` replaces a released record + | { ev: "creating"; leaseId: string; digest?: string } // round 1: written BEFORE the first UpdateTask; the Task may exist from here on. Round 3: the spec digest the link will write, so a cleanup can tell its own Task from a foreign one + | { ev: "not-mine"; leaseId: string } // round 3: a Task of this name exists and is not the link's; `creating` no longer holds + | { ev: "created"; leaseId: string; digest: string } + | { ev: "grant-invalid"; leaseId: string; attempt: number; message: string } // round 1: a grant that did not decode + | { ev: "report"; leaseId: string; attempt: number; result: Result; output?: unknown; usage?: Usage; replace?: true } // outbox; round 2: `replace` swaps an undeliverable verdict + | { ev: "reported"; leaseId: string; duplicate: boolean; withdrew?: string; withdrewGen?: Gen } // done: 2xx from the floor; round 2: what rule 4b withdrew; round 3: which generation of it + | { ev: "dropped"; leaseId: string; code: string } // drop: the floor refused these bytes for good + | { ev: "released"; leaseId: string; why: string } // no longer held (lost, cancelled, superseded) + | { ev: "deleting"; leaseId: string; why: string } // DeleteTask accepted; waiting for NotFound (ax deletes in two phases) + | { ev: "deleted"; leaseId: string } + | { ev: "lease-key"; requestKey: string } // B8: a Lease is in flight with this key; reuse it until its reply lands + | { ev: "lease-replied"; requestKey: string; abandoned?: true } // the reply's grants are journaled; the next Lease draws a new key + | { ev: "tomb"; leaseId: string; gen: Gen } // round 3: written by compaction for a finished lease; refuses its redelivery + | { ev: "renewed"; leaseId: string } // red team double-run-r1-1: the floor renewed this grant at `t` (the send time); the self-fence clock survives a restart + | { ev: "delete-retry"; leaseId: string; n: number } // critique D.2 ladder: DeleteTask re-sent for the n-th time while the Task stays Terminating + | { ev: "delete-stuck"; leaseId: string; freeProven: boolean } // critique D.2 ladder: the delete is given up on; `freeProven` only from a Substrate read + +/** Round 3: a grant generation. `acquireTime` is absent when only the floor's `withdrewTransitions` is known. */ +export interface Gen { readonly leaseTransitions: number; readonly acquireTime?: number } +export const genOf = (g: Grant): Gen => ({ leaseTransitions: g.lease.leaseTransitions, acquireTime: g.lease.acquireTime }) +/** `g` is a strictly later generation than `than` (leaseTransitions first, then acquireTime). */ +export const laterGen = (g: Gen, than: Gen) => g.leaseTransitions > than.leaseTransitions || + (g.leaseTransitions === than.leaseTransitions && g.acquireTime !== undefined && than.acquireTime !== undefined && g.acquireTime > than.acquireTime) +/** `g` is the generation `w` names: equal leaseTransitions, and equal acquireTime when both are known. */ +export const sameGen = (g: Gen, w: Gen) => g.leaseTransitions === w.leaseTransitions && + (g.acquireTime === undefined || w.acquireTime === undefined || g.acquireTime === w.acquireTime) + +export interface LeaseRec { + grant: Grant + created?: string // spec digest + creating?: boolean // an UpdateTask may have landed (write-ahead); cleanup must DeleteTask by name and see NotFound + creatingDigest?: string // round 3: the spec digest the link was about to write + notMine?: boolean // round 3: the Task under this name is someone else's; never read, delete or report it + createdAt?: number + report?: Extract + reported?: boolean + duplicate?: boolean // round 2: the floor answered this verdict as a duplicate (rule 4c) + withdrew?: string // round 2: the leaseId the floor withdrew when it accepted this verdict (rule 4b), never inferred + withdrewGen?: Gen // round 3: the generation withdrawn; undefined when this link never held it (it then matches nothing) + dropped?: string + released?: string + deleting?: string // why: cancel, lost, dropped, janitor, pre-start, deadline, superseded + deleted?: boolean + renewedAt?: number // red team double-run-r1-1: the latest journaled renewal of this grant generation (local ms) + deletingAt?: number // critique D.2: when the FIRST DeleteTask intent was journaled; the Terminating ladder's clock (survives compaction) + deleteRetries?: number // critique D.2: DeleteTask re-sends journaled by the ladder + deleteStuck?: boolean // critique D.2: the ladder gave up on this name (delete-stuck) + freeProven?: boolean // critique D.2: a Substrate read proved the stuck Task holds no worker; only then is its slot released + at: number // when this record was last touched (ms) +} + +export class Journal { + readonly path: string + readonly recs = new Map() + /** Grants that did not decode: refused to the floor as pre-start/invalid-spec through the same outbox. */ + readonly invalid = new Map() + pendingLeaseKey: string | undefined + /** Round 3: finished leases, kept through compaction for TOMB_MS so a redelivered grant of one is never run again. */ + readonly tombs = new Map() + static TOMB_MS = 24 * 3600_000 // well past the floor's replay window (rule 2b: lease + grace, 390 s in production) + private readonly dir: string + private constructor(dir: string) { this.dir = dir; this.path = join(dir, "journal.jsonl") } + + static open(dir: string, now = Date.now()): Journal { + mkdirSync(dir, { recursive: true, mode: 0o700 }) + const j = new Journal(dir) + if (existsSync(j.path)) { + for (const line of readFileSync(j.path, "utf8").split("\n")) { + if (line.trim() === "") continue + let e: Entry & { t?: number } + try { e = JSON.parse(line) } catch { continue } // a torn last line from a crash mid-write is skipped + j.apply(e, e.t ?? now) + } + j.compact() + } + return j + } + + private apply(e: Entry, t: number) { + if (e.ev === "lease-key") { this.pendingLeaseKey = e.requestKey; return } + if (e.ev === "lease-replied") { if (this.pendingLeaseKey === e.requestKey) this.pendingLeaseKey = undefined; return } + if (e.ev === "tomb") { this.tombs.set(e.leaseId, { gen: e.gen, at: t }); return } + if (e.ev === "grant") { + if (this.recs.has(e.leaseId) && !e.regrant) return + this.recs.set(e.leaseId, { grant: e.grant, at: t }) + // round 2: the withdrawal recorded against the previous generation of this leaseId does not apply to the new one + if (e.regrant) for (const x of this.recs.values()) if (x.withdrew === e.leaseId) { delete x.withdrew; delete x.withdrewGen } + return + } + if (e.ev === "grant-invalid") { if (!this.recs.has(e.leaseId) && !this.invalid.has(e.leaseId)) this.invalid.set(e.leaseId, { attempt: e.attempt, message: e.message, at: t }); return } + const r = this.recs.get(e.leaseId) + if (e.ev === "renewed") { if (r) r.renewedAt = Math.max(r.renewedAt ?? 0, t); return } // not a touch: `at` stays + // Critique D.2: ladder bookkeeping is not a touch either, so `at` (the janitor's and the tomb's clock) stays + if (e.ev === "delete-retry") { if (r) r.deleteRetries = Math.max(r.deleteRetries ?? 0, e.n); return } + if (e.ev === "delete-stuck") { if (r) { r.deleteStuck = true; r.freeProven = e.freeProven } return } + if (!r) { + const x = this.invalid.get(e.leaseId) + if (x && e.ev === "reported") x.reported = true + if (x && e.ev === "dropped") x.dropped = e.code + if (x && e.ev === "released") x.released ??= e.why + return + } + // Critique D.2: compaction rewrites lines with their own times (the deleting line carries deletingAt, which is + // earlier than the last touch), so `at` is the latest time seen, not the time of the last line replayed. + r.at = Math.max(r.at, t) + switch (e.ev) { + case "creating": r.creating = true; delete r.notMine; if (e.digest !== undefined) r.creatingDigest = e.digest; break + case "not-mine": r.notMine = true; delete r.creating; break + case "created": r.created = e.digest; r.createdAt ??= t; break + case "report": if (!r.report || e.replace) r.report = e; break // first verdict wins (the floor dedupes as well), unless replaced + case "reported": r.reported = true; r.duplicate = e.duplicate; if (e.withdrew !== undefined) { r.withdrew = e.withdrew; if (e.withdrewGen !== undefined) r.withdrewGen = e.withdrewGen } break + case "dropped": r.dropped = e.code; break + case "released": r.released ??= e.why; break + case "deleting": r.deleting ??= e.why; r.deletingAt ??= t; break + case "deleted": r.deleted = true; break + } + } + + append(e: Entry, t = Date.now()) { + appendFileSync(this.path, JSON.stringify({ t, ...e }) + "\n", { mode: 0o600 }) + const fd = openSync(this.path, "r") + try { fsyncSync(fd) } finally { closeSync(fd) } + this.apply(e, t) + if (++this.appends >= Journal.COMPACT_EVERY) this.compact() // idle Lease keys must not grow the file for ever + } + static COMPACT_EVERY = 2000 + private appends = 0 + + /** Finished = nothing left to tell the floor and nothing left in ax. */ + static finished(r: LeaseRec) { + // Round 2: a verdict read after the lease was released (lost, superseded) is still owed to the floor + if (Journal.pending(r)) return false + const toldFloor = r.reported === true || r.dropped !== undefined || r.released !== undefined + const axClean = !Journal.maybeCreated(r) || r.deleted === true + return toldFloor && axClean + } + /** Round 1: a Task may exist in ax once `creating` is journaled, even if `created` never was (a crash, or a + * DEADLINE_EXCEEDED after the write landed). Every cleanup path treats such a record as created. */ + static maybeCreated(r: LeaseRec) { return r.created !== undefined || r.creating === true } + /** A verdict is journaled and neither acknowledged nor dropped. */ + static pending(r: LeaseRec) { return r.report !== undefined && !r.reported && r.dropped === undefined } + + /** Rewrite the file with only unfinished leases (atomic rename), so it never grows without bound. */ + compact() { + const tmp = this.path + ".tmp" + const fd = openSync(tmp, "w", 0o600) + try { + if (this.pendingLeaseKey !== undefined) writeSync(fd, JSON.stringify({ t: Date.now(), ev: "lease-key", requestKey: this.pendingLeaseKey }) + "\n") + for (const [id, r] of this.recs) { + if (Journal.finished(r)) { this.recs.delete(id); this.tombs.set(id, { gen: genOf(r.grant), at: r.at }); continue } + const lines: Array = [{ ev: "grant", leaseId: id, grant: r.grant }] + if (r.creating) lines.push({ ev: "creating", leaseId: id, ...(r.creatingDigest !== undefined ? { digest: r.creatingDigest } : {}) }) + if (r.notMine) lines.push({ ev: "not-mine", leaseId: id }) + if (r.created !== undefined) lines.push({ ev: "created", leaseId: id, digest: r.created }) + if (r.report) lines.push(r.report) + if (r.reported) lines.push({ ev: "reported", leaseId: id, duplicate: r.duplicate ?? false, ...(r.withdrew !== undefined ? { withdrew: r.withdrew } : {}), ...(r.withdrewGen !== undefined ? { withdrewGen: r.withdrewGen } : {}) }) + if (r.dropped !== undefined) lines.push({ ev: "dropped", leaseId: id, code: r.dropped }) + if (r.released !== undefined) lines.push({ ev: "released", leaseId: id, why: r.released }) + if (r.deleting !== undefined) lines.push({ ev: "deleting", leaseId: id, why: r.deleting }) + for (const l of lines) writeSync(fd, JSON.stringify({ t: l.ev === "created" || l.ev === "grant" ? (r.createdAt ?? r.at) : l.ev === "deleting" ? (r.deletingAt ?? r.at) : r.at, ...l }) + "\n") + if (r.deleteRetries !== undefined) writeSync(fd, JSON.stringify({ t: r.at, ev: "delete-retry", leaseId: id, n: r.deleteRetries }) + "\n") + if (r.deleteStuck) writeSync(fd, JSON.stringify({ t: r.at, ev: "delete-stuck", leaseId: id, freeProven: r.freeProven === true }) + "\n") + if (r.renewedAt !== undefined) writeSync(fd, JSON.stringify({ t: r.renewedAt, ev: "renewed", leaseId: id }) + "\n") + } + const now = Date.now() + for (const [id, x] of this.tombs) { + if (now - x.at > Journal.TOMB_MS) { this.tombs.delete(id); continue } + writeSync(fd, JSON.stringify({ t: x.at, ev: "tomb", leaseId: id, gen: x.gen }) + "\n") + } + for (const [id, x] of this.invalid) { // never created, so finished once the floor is told + if (x.reported || x.dropped !== undefined || x.released !== undefined) { this.invalid.delete(id); continue } + writeSync(fd, JSON.stringify({ t: x.at, ev: "grant-invalid", leaseId: id, attempt: x.attempt, message: x.message }) + "\n") + } + fsyncSync(fd) + } finally { closeSync(fd) } + renameSync(tmp, this.path) + this.appends = 0 + const d = openSync(this.dir, "r") // B20: make the rename itself durable + try { fsyncSync(d) } finally { closeSync(d) } + } + + /** The complete set this holder believes it holds: sent in every Heartbeat (level-triggered). A lease whose + * verdict is still in the outbox stays held, so a heartbeat that races the outbox drain cannot orphan it. */ + held(): Array { + return [...[...this.recs].filter(([, r]) => !r.reported && r.dropped === undefined && r.released === undefined).map(([id]) => id), + ...[...this.invalid].filter(([, x]) => !x.reported && x.dropped === undefined && x.released === undefined).map(([id]) => id)] + } + /** Leases whose verdict is written but not yet acknowledged; they are still held until the floor says done. */ + outbox(): Array> { + return [...[...this.recs.values()].filter(Journal.pending).map((r) => r.report!), + ...[...this.invalid].filter(([, x]) => !x.reported && x.dropped === undefined && x.released === undefined) + .map(([leaseId, x]) => ({ ev: "report" as const, leaseId, attempt: x.attempt, result: "failure" as const, output: { reason: "pre-start/invalid-spec", message: x.message } }))] + } +} diff --git a/pkgs/substrate-apps/src/apps/link/src/link.ts b/pkgs/substrate-apps/src/apps/link/src/link.ts new file mode 100644 index 00000000..2c001967 --- /dev/null +++ b/pkgs/substrate-apps/src/apps/link/src/link.ts @@ -0,0 +1,1123 @@ +// The NAS link (LINK-DESIGN.md section 5): one outbound-only process between the Cloudflare floor and ax. +// Lease (free capacity on every poll, ARC; the outbox drains first, B15; the requestKey is journaled, B8) +// -> journal the grant (write-ahead) -> fence every superseded attempt: DeleteTask and wait for NotFound (L3, B2) +// -> wait for a sandbox slot (B11) -> GetTask, UpdateTask only on NotFound (L4, create-only), read it back (B12) +// -> resync by paging ListTasks (Buildkite: the executor's list is the ledger; B1) and confirming every absence with +// GetTask -> read the outcome (P1: terminal phase + GetTaskResult, retried and digest-checked, B4; guest mode only +// when configured, L7, B6) -> Complete through the journal's outbox (L6) -> DeleteTask after the floor acknowledged. +// One batched Heartbeat per session carries the COMPLETE held set; its reply carries cancel and lost (Temporal). +// Nothing is leased while ax is unreachable, the completion mode is unproven (B6) or the egress Gateway is missing or +// open (B10). +import { createHash, randomUUID } from "node:crypto" +import { isIPv4, isIPv6 } from "node:net" +import { Cause, Deferred, Effect, Schedule, Semaphore } from "effect" +import { gatewayAllowsAll, listAll } from "./ax.ts" +import type { AxApi, AxError, AxObserved } from "./ax.ts" +import type { FailureOutput, Grant, Result, Usage } from "./contract.ts" +import type { FloorApi, FloorError, LeaseReply } from "./floor.ts" +import { axTaskFromGrant, specDigest, validRunsOn } from "./jobs.ts" +import type { TaskShape } from "./jobs.ts" +import { genOf, Journal, laterGen, sameGen } from "./journal.ts" +import type { Gen, LeaseRec } from "./journal.ts" + +export interface LinkConfig { + readonly holder: string // holderIdentity; bound to this link's token on the floor + readonly maxInFlight: number // sandbox cap: the WorkerPool replicas this link may fill + readonly servedLabels: ReadonlyArray // defence in depth: the floor already routes by holder + readonly shape: TaskShape + /** `p1` and `auto` both require P1 on the server, proven by a probe (B6); `auto` never selects guest. `guest` is + * the L7 reserve and needs a fleet-internal `shape.completeUrl` (critique A5). */ + readonly completion: "auto" | "p1" | "guest" + readonly secondMs: number // 1000 in production; tests shrink every server-set interval with it + readonly resyncMs: number // 15 s, the P1 --running-resync period + readonly pendingTimeoutMs: number // 15 min, agent-stack-k8s: a Pod stuck Pending fails the job + readonly deleteAfterMs: number // 10 min, agent-stack-k8s TTLSecondsAfterFinished + readonly deadlineBackstopMs: number // local stop past the floor's deadline, if the floor cannot say cancel + readonly createAttempts: number // UpdateTask tries while ax is unavailable before pre-start failure + readonly outboxBackoffMs: readonly [number, number] // uplink shut-door backoff: min, max + readonly fenceTimeoutMs?: number // B2: how long a superseded attempt may take to reach NotFound (default 120 s) + readonly resultReadTries?: number // B4: resyncs a Completed Task's result may fail to read (default 5) + readonly floorUrls?: ReadonlyArray // B18: the only URLs a Lease `endpoint` may move this link to + readonly initialPollSeconds?: number // until the first Lease reply sets them (defaults 15 and 30, section 3.7) + readonly initialHeartbeatSeconds?: number + /** Round 2: a result larger than this is refused `agent/output-too-large` before it is reported (the floor's DO row + * limit is 2 MB, MEASURED as SQLITE_TOOBIG at 2.2 MB; default 1 000 000 B). */ + readonly maxOutputBytes?: number + /** Round 2: server errors one verdict may collect before it is replaced by `infra/verdict-undeliverable`, and that + * replacement before it is dead-lettered (journaled `dropped`). Default 8. */ + readonly verdictAttempts?: number + /** Round 2: single-label hosts a guest Complete URL may name (fleet DNS names without a dot). Default none. */ + readonly internalHosts?: ReadonlyArray + /** Round 3: consecutive undecodable or server-error Lease replies for one journaled requestKey before the key is + * abandoned (its grants are then released by the floor's own expiry and requeued). Default 5. */ + readonly leaseKeyAttempts?: number + /** Critique D.2 (DELETE-HANG ladder): a Task that stays Terminating after its journaled DeleteTask is asked again every + * `terminatingRetryMs` (default 60 s) up to `terminatingRetries` times (default 5), escalated by log (and a Substrate + * read, when one is wired) at `terminatingEscalateMs` (default 5 min) and journaled `delete-stuck` at + * `terminatingGiveUpMs` (default 15 min). All measured from `deletingAt`, which survives restarts and compaction. */ + readonly terminatingRetryMs?: number + readonly terminatingEscalateMs?: number + readonly terminatingGiveUpMs?: number + readonly terminatingRetries?: number +} + +/** Critique D.2: the optional read port the ladder uses to PROVE a stuck Task holds no Substrate worker (DELETE-HANG + * step 3). Absent in production until a Substrate read credential is granted; without it no slot is ever released. */ +export interface SubstrateRead { + readonly actor: (task: string, atespace: string) => Effect.Effect<"absent" | "deleting" | "present", unknown> +} + +export type Log = (ev: string, fields?: Record) => void +export interface LinkDeps { + readonly ax: AxApi; readonly floor: FloorApi; readonly journal: Journal; readonly log: Log; readonly stop: Deferred.Deferred + /** G-BK4, Buildkite's graceful stop: once done, the link leases nothing more and stops by itself when no Task it + * holds is live and no verdict is undelivered, or after drainTimeoutMs; `stop` still stops it at once. */ + readonly drain?: Deferred.Deferred + readonly drainTimeoutMs?: number + readonly floorUrl?: string // the URL in use, to tell a real endpoint switch from an echo + readonly onEndpoint?: (url: string) => void // B18: called once for an allowed switch; main persists it and restarts + readonly substrate?: SubstrateRead // critique D.2: optional; see SubstrateRead +} + +const TERMINAL = new Set(["Completed", "Failed"]) +const jitter = (ms: number) => Math.round(ms * (0.8 + 0.4 * Math.random())) +const NOT_FOUND = 5 // gRPC status: DeleteTask of a name that is already gone +const MAX_POLL_SECONDS = 300, MAX_HEARTBEAT_SECONDS = 300 +/** Round 3: a server-driven interval (Buildkite: the agent falls back to its defaults). Missing, non-finite or not + * positive gives `def`; anything else is clamped to [1, hi]. `Effect.sleep(0)` would otherwise be a tight loop. */ +export const boundedSeconds = (v: number | undefined, def: number, hi: number) => + v === undefined || !Number.isFinite(v) || v <= 0 ? Math.min(def, hi) : Math.max(1, Math.min(v, hi)) + +/** Critique A5 and N3: a guest's Complete URL must stay inside the fleet, never a public Workers host. + * Round 2: private ranges are matched only on IP literals (after WHATWG normalisation, so `10.evil.example` and + * `127.0.0.1.nip.io` are names, not addresses); a name must end in a non-public suffix; a single-label name only when + * it is listed. */ +export const isFleetInternalUrl = (raw: string | undefined, internalHosts: ReadonlyArray = []): boolean => { + if (raw === undefined) return false + let u: URL + try { u = new URL(raw) } catch { return false } + let h = u.hostname.toLowerCase() + if (h.endsWith(".workers.dev") || h.endsWith(".pages.dev")) return false + if (h.startsWith("[") && h.endsWith("]")) h = h.slice(1, -1) + if (isIPv4(h)) { + const [a, b] = h.split(".").map(Number) as [number, number] + return a === 127 || a === 10 || (a === 192 && b === 168) || (a === 172 && b >= 16 && b <= 31) || (a === 100 && b >= 64 && b <= 127) + } + if (isIPv6(h)) return h === "::1" || /^f[cd][0-9a-f]{2}:/.test(h) // loopback, ULA fc00::/7 (the tailnet's fd7a:115c:a1e0::/48) + if (h === "localhost" || internalHosts.map((x) => x.toLowerCase()).includes(h)) return true + return /^([a-z0-9-]+\.)+(internal|lan|local|home\.arpa)$/.test(h) +} + +export const runLink = (cfg: LinkConfig, deps: LinkDeps) => Effect.gen(function*() { + const { ax, journal: j, log } = deps + // Red team loss-and-wip r3-3 (critique pass KEEP-4): every floor reply that is not an error proves the floor serves + // this link, so an HTTP 5xx that follows one is about the bytes of that call, not the whole floor. + let floorAnswered = 0 + const answered = (e: Effect.Effect) => e.pipe(Effect.tap(() => Effect.sync(() => { floorAnswered++ }))) + const floor: FloorApi = { lease: (p) => answered(deps.floor.lease(p)), heartbeat: (p) => answered(deps.floor.heartbeat(p)), complete: (p) => answered(deps.floor.complete(p)) } + const fatal = yield* Deferred.make() + const fenceTimeoutMs = cfg.fenceTimeoutMs ?? 120_000 + const resultReadTries = cfg.resultReadTries ?? 5 + const maxOutputBytes = cfg.maxOutputBytes ?? 1_000_000 + const verdictAttempts = cfg.verdictAttempts ?? 8 + const defaultPoll = cfg.initialPollSeconds ?? 15, defaultHb = cfg.initialHeartbeatSeconds ?? 30 + let pollSeconds = defaultPoll, hbAsked: number | undefined // round 3: the floor's raw heartbeatSeconds, bounded at use + let clampLogged = "" + const leaseKeyAttempts = cfg.leaseKeyAttempts ?? 5 + const termRetryMs = cfg.terminatingRetryMs ?? 60_000 + const termEscalateMs = cfg.terminatingEscalateMs ?? 300_000 + const termGiveUpMs = cfg.terminatingGiveUpMs ?? 900_000 + const termRetries = cfg.terminatingRetries ?? 5 + const escalated = new Set() // critique D.2: ax-delete-escalate is logged once per lease per process + let leaseKeyFailures = 0 + let axUp = false, resynced = false + let serverP1: boolean | undefined // the probe's answer; cleared on every ax-up transition (B6) + let gatewayOk: boolean | undefined // B10 + let gatewayState: string | undefined // critique D.2 item 5: ok, open, missing (with the code); logged on change + const guestUrlOk = cfg.completion !== "guest" || isFleetInternalUrl(cfg.shape.completeUrl, cfg.internalHosts) + if (!guestUrlOk) log("guest-url-invalid", { completion: "guest" }) + /** Critique D.2 item 6: `auto` falls back to guest when a Complete URL is declared and fleet-internal. */ + const autoGuestOk = cfg.completion === "auto" && cfg.shape.completeUrl !== undefined && isFleetInternalUrl(cfg.shape.completeUrl, cfg.internalHosts) + if (cfg.completion === "auto" && cfg.shape.completeUrl !== undefined && !autoGuestOk) log("guest-url-invalid", { completion: "auto" }) + let last = new Map() + let outboxWaitUntil = 0, outboxBackoff = cfg.outboxBackoffMs[0] + let outboxNotBefore = 0 // round 3: the floor's own Retry-After on Complete; Lease does not cut it short (B15 does not) + const dispatching = new Map() // round 2: keyed by leaseId, valued by the grant generation it runs + const admitted = new Set() // B11: grants that passed the slot gate in this process + const abandoned = new Map() // B3: set by the Heartbeat reply, read by dispatch + const absent = new Map() // B1: consecutive resyncs a held Task answered NotFound + const resultFailures = new Map() // B4 + const verdictFailures = new Map() // round 2: server errors per verdict (a restart grants a fresh budget) + const verdictNotBefore = new Map() // round 2: per-verdict backoff after a server error + const verdictAnsweredMark = new Map() // KEEP-4: floorAnswered when this verdict last failed with an HTTP 5xx + // Critique pass 2026-09-24 (red team durability-r2-8): a server error is charged to a verdict only when the floor + // answered a Lease or Heartbeat between two tries of it; a floor-wide outage charges nothing, so a success is never + // replaced by infra/verdict-undeliverable because the whole floor was down. Since the 2026-09-24 integrate this + // stands beside KEEP-4 (the same finding, fixed on two branches): a 5xx is charged only when both say the floor served. + const verdictTriedAt = new Map() + let floorOkAt = 0 + const markFloorOk = () => Effect.sync(() => { floorOkAt = Date.now() }) + const pendingResume = new Set() // B13: journaled, never created; resumed only once the floor renews them + /** Round 4 (Buildkite reserve-with-expiry): the local time of the last call the floor answered by renewing this + * grant generation (the Lease that granted it, then every Heartbeat listing it in `renewed`). The send time is taken, + * so it is never later than the floor's own renewTime. Keyed by the grant object: a regrant starts afresh. */ + const renewedAt = new WeakMap() + /** Round 4: a create may start only while the floor still holds the lease: the last renewal is less than 3/4 of a + * lease old. A grant never renewed in this process (journaled by a previous one) is not live until a Heartbeat says so. */ + const leaseLive = (g: Grant) => { + const at = renewedAt.get(g) + if (at === undefined) return false + const ms = g.lease.leaseDurationSeconds * cfg.secondMs + return Date.now() < at + ms - ms / 4 + } + const deadlinePassed = (g: Grant) => g.deadline !== undefined && Date.now() >= g.deadline + /** Red team double-run-r1-1 (Kubernetes Lease, Temporal heartbeat timeout: the holder stops before the server may + * reassign). Past lease + grace the floor may requeue attempt n+1 to a holder whose own executor cannot see this + * one's Task, so its supersedes fence finds nothing and both attempts run. The floor sends that bound as the grant's + * reassignSeconds (lease + grace + the rule 7b pin); the link deletes a still-running Task it has not seen renewed + * for reassignSeconds less half a lease (a floor that sends no bound promises no grace: one lease), measured from the + * send time of the last renewing call, which is never later than the floor's own renewTime. */ + const renewedLocal = (id: string, g: Grant, at: number) => { + renewedAt.set(g, Math.max(renewedAt.get(g) ?? 0, at)) + const r = j.recs.get(id) + if (r?.grant !== g) return + // journaled at most every half lease, so a restart measures from a recent renewal without one fsync per beat + if ((r.renewedAt ?? 0) + (g.lease.leaseDurationSeconds * cfg.secondMs) / 2 <= at) j.append({ ev: "renewed", leaseId: id }, at) + } + const selfFenceAt = (r: LeaseRec) => { + const g = r.grant + const base = Math.max(renewedAt.get(g) ?? 0, r.renewedAt ?? 0) || (r.createdAt ?? r.at) + const lease = g.lease.leaseDurationSeconds * cfg.secondMs + const bound = g.reassignSeconds !== undefined && Number.isFinite(g.reassignSeconds) ? g.reassignSeconds * cfg.secondMs : lease + return base + Math.max(lease / 2, bound - lease / 2) + } + /** A Task this link may have created, runs (no verdict read), and is still held: the ones a fence must stop. */ + const fenceable = (r: LeaseRec) => r.released === undefined && r.report === undefined && !r.reported && r.dropped === undefined + && !r.notMine && r.deleting === undefined && !r.deleted && Journal.maybeCreated(r) + /** Only a Task that still RUNS is fenced. A finished one (P1: suspended, holding no worker) has its verdict read + * and journaled by salvage and stays held with it in the outbox (rule 4b may still accept it); a Task ax cannot + * answer for is tried again at the next resync. */ + const fenceSelf = (id: string, why: string, fields: Record = {}) => Effect.gen(function*() { + const r = j.recs.get(id) + if (!r || !fenceable(r)) return + if ((yield* salvage(id)) !== "live") return + j.append({ ev: "released", leaseId: id, why }) + abandoned.set(id, "lost") + pendingResume.delete(id) + log(why, { leaseId: id, ...fields }) + yield* del(id, why) + }) + const selfFence = Effect.gen(function*() { + const now = Date.now() + for (const [id, r] of [...j.recs]) if (fenceable(r) && now >= selfFenceAt(r)) + yield* fenceSelf(id, "lease-expired", { lastRenewal: Math.max(renewedAt.get(r.grant) ?? 0, r.renewedAt ?? 0) || null }) + }) + let wake = yield* Deferred.make() // B17: a verdict that frees a slot leases at once + let hbWake = yield* Deferred.make() // final pass (R3-1): a Lease reply re-times a heartbeat sleep already begun + /** Red team double-run-r1-2: a 409 means another session holds this identity and may already run attempt n+1 of + * every lease held here; this process can never hear `lost` again. Every running Task is fenced before the exit. */ + const failFatal = (e: FloorError): Effect.Effect => Effect.gen(function*() { + if (e.kind === "session-conflict") for (const [id, r] of [...j.recs]) if (fenceable(r)) yield* fenceSelf(id, "session-lost") + yield* Deferred.fail(fatal, e) + }) + + /** The mode the link may create in, or undefined while it is unproven (capacity 0). Critique D.2 item 6: `auto` + * is p1 when the probe proved P1, guest when the probe proved its absence and a fleet-internal Complete URL is + * declared, and undefined while the probe has not answered (an outage never flips the mode). */ + const mode = (): "p1" | "guest" | undefined => { + if (cfg.completion === "guest") return guestUrlOk ? "guest" : undefined + if (serverP1 === true) return "p1" + if (serverP1 === false && autoGuestOk) return "guest" + return undefined + } + const canCreate = () => axUp && resynced && gatewayOk === true && mode() !== undefined + + // ---------------------------------------------------------------- verdicts (write-ahead, then the outbox drains) + /** `salvage` (round 2): the verdict was read from a Task whose lease is already released (lost, superseded). It is + * still owed to the floor, which accepts it by rule 4b or answers stale-attempt (then it is dropped). */ + /** Round 4: `g`, when given, is the grant generation the verdict is about; a record that holds another generation + * by now (a regrant of the same leaseId) is not given it. */ + const report = (id: string, result: Result, output?: unknown, usage?: Usage, salvage = false, g?: Grant) => Effect.gen(function*() { + const r = j.recs.get(id) + if (g !== undefined && r?.grant !== g) return log("stale-generation-verdict", { leaseId: id, result }) + if (!r || r.report || (r.released !== undefined && !salvage)) return + j.append({ ev: "report", leaseId: id, attempt: r.grant.attempt, result, ...(output !== undefined ? { output } : {}), ...(usage !== undefined ? { usage } : {}) }) + log("verdict", { leaseId: id, result, ...(result === "failure" ? { reason: (output as FailureOutput | undefined)?.reason } : {}), ...(salvage ? { salvaged: r.released } : {}) }) + yield* Deferred.succeed(wake, undefined) + }) + const fail = (id: string, o: FailureOutput) => report(id, "failure", o) + const failG = (g: Grant, o: FailureOutput) => report(g.leaseId, "failure", o, undefined, false, g) // round 4 + + // Round 1: single-flight. The main loop, the resync fiber and a dispatch waiting on a superseded verdict all drain; + // two concurrent drains read the same outbox before either journals `reported` and send every verdict twice. + const outboxLock = Semaphore.makeUnsafe(1) + const drainOutbox = outboxLock.withPermit(Effect.gen(function*() { + if (Date.now() < outboxWaitUntil || Date.now() < outboxNotBefore) return + for (const p of j.outbox()) { + if ((verdictNotBefore.get(p.leaseId) ?? 0) > Date.now()) continue + // Round 4: the generation of attempt n+1 this link held when the verdict was SENT. A regrant of n+1 taken while + // the reply is in flight is later than anything this Complete can have withdrawn. + const succId = `${p.leaseId.replace(/-a\d+$/, "")}-a${p.attempt + 1}` + const succAtSend = j.recs.get(succId)?.grant + const r = yield* floor.complete({ leaseId: p.leaseId, attempt: p.attempt, result: p.result, ...(p.output !== undefined ? { output: p.output } : {}), ...(p.usage !== undefined ? { usage: p.usage } : {}) }).pipe(Effect.result) + if (r._tag === "Success") { + const withdrew = r.success.withdrew + // Round 3: a withdrawal names a GENERATION, not a leaseId. The floor may requeue the withdrawn attempt under the + // same leaseId (a retryable verdict of n), and that later grant is live. The generation is the floor's + // `withdrewTransitions` when it sends one, else the one this link holds; a generation this link never received + // (n+1 was still queued) matches no later grant. + const w = withdrew !== undefined ? j.recs.get(withdrew) : undefined + const withdrewGen: Gen | undefined = withdrew === undefined ? undefined + : r.success.withdrewTransitions !== undefined ? { leaseTransitions: r.success.withdrewTransitions } + : succAtSend !== undefined && withdrew === succId ? genOf(succAtSend) + : w !== undefined ? genOf(w.grant) : undefined + j.append({ ev: "reported", leaseId: p.leaseId, duplicate: r.success.duplicate, ...(withdrew !== undefined ? { withdrew } : {}), ...(withdrewGen !== undefined ? { withdrewGen } : {}) }) + log("complete", { leaseId: p.leaseId, result: p.result, duplicate: r.success.duplicate, ...(withdrew !== undefined ? { withdrew } : {}) }) + if (withdrew !== undefined && withdrewGen === undefined) log("withdrawn-unheld", { leaseId: withdrew, by: p.leaseId }) + if (w !== undefined && withdrewGen !== undefined && sameGen(genOf(w.grant), withdrewGen) && w.released === undefined && !Journal.maybeCreated(w)) { + j.append({ ev: "released", leaseId: withdrew!, why: "withdrawn" }) // round 2: released at once, so a regrant + log("withdrawn", { leaseId: withdrew, by: p.leaseId }) // of it is not taken for a duplicate + } + outboxBackoff = cfg.outboxBackoffMs[0] + verdictFailures.delete(p.leaseId); verdictNotBefore.delete(p.leaseId); verdictAnsweredMark.delete(p.leaseId); verdictTriedAt.delete(p.leaseId) + } else if (r.failure._tag === "CompleteRefused") { // done/keep/drop: the floor refused these bytes for good + j.append({ ev: "dropped", leaseId: p.leaseId, code: r.failure.code }) + log("complete-dropped", { leaseId: p.leaseId, code: r.failure.code }) + } else if (r.failure.kind === "server-error" && r.failure.status !== undefined && (verdictAnsweredMark.get(p.leaseId) ?? -1) >= floorAnswered) { + // KEEP-4: an HTTP 5xx with no floor reply since this verdict last failed is not evidence against these bytes + // (a floor-wide 500 answers every call alike): the verdict waits with its own backoff and is not charged. + const wait = jitter(cfg.outboxBackoffMs[1]) + verdictAnsweredMark.set(p.leaseId, floorAnswered); verdictNotBefore.set(p.leaseId, Date.now() + wait) + log("outbox-keep", { leaseId: p.leaseId, kind: r.failure.kind, charged: false, waitMs: wait }) + continue + } else if (r.failure.kind === "server-error") { + if (r.failure.status !== undefined) verdictAnsweredMark.set(p.leaseId, floorAnswered) + // Round 2: the floor answered and refused THESE bytes (a 500, an RPC Defect such as SQLITE_TOOBIG). Counted per + // verdict; the rest of the outbox still goes out. At the ceiling the verdict is replaced by a small final + // failure the floor can store; if that fails as well it is dead-lettered, so one verdict never gates Lease. + const prevTry = verdictTriedAt.get(p.leaseId) + verdictTriedAt.set(p.leaseId, Date.now()) + const charged = prevTry !== undefined && floorOkAt > prevTry // durability-r2-8 + const n = (verdictFailures.get(p.leaseId) ?? 0) + (charged ? 1 : 0) + verdictFailures.set(p.leaseId, n) + if (n < verdictAttempts) { + const wait = jitter(Math.min(cfg.outboxBackoffMs[0] * 2 ** Math.max(0, n - 1), cfg.outboxBackoffMs[1])) + verdictNotBefore.set(p.leaseId, Date.now() + wait) + log("outbox-keep", { leaseId: p.leaseId, kind: r.failure.kind, try: n, charged, waitMs: wait }) + continue + } + verdictFailures.delete(p.leaseId); verdictNotBefore.delete(p.leaseId); verdictTriedAt.delete(p.leaseId) + const rec = j.recs.get(p.leaseId) + if (rec !== undefined && !p.replace) { + const digest = createHash("sha256").update(JSON.stringify(p.output ?? null)).digest("hex") + j.append({ ev: "report", leaseId: p.leaseId, attempt: p.attempt, result: "failure", replace: true, + output: { reason: "infra/verdict-undeliverable", message: `${p.result} verdict (output sha256 ${digest}) refused ${n} times: ${r.failure.message.slice(0, 200)}` } }) + log("verdict-replaced", { leaseId: p.leaseId, result: p.result, tries: n, outputSha256: digest }) + } else { + j.append({ ev: "dropped", leaseId: p.leaseId, code: "undeliverable" }) + log("verdict-dead-letter", { leaseId: p.leaseId, tries: n, message: r.failure.message.slice(0, 200) }) + } + } else { + if (r.failure.fatal) return yield* failFatal(r.failure) + outboxWaitUntil = Date.now() + Math.min(r.failure.retryAfterMs ?? jitter(outboxBackoff), MAX_POLL_SECONDS * cfg.secondMs) + if (r.failure.retryAfterMs !== undefined) outboxNotBefore = outboxWaitUntil + outboxBackoff = Math.min(outboxBackoff * 2, cfg.outboxBackoffMs[1]) + log("outbox-keep", { leaseId: p.leaseId, kind: r.failure.kind, waitMs: outboxWaitUntil - Date.now() }) + return // keep: oldest first, stop at the first transient failure + } + } + })) + + // ---------------------------------------------------------------- ax calls + // Round 3 (token scope): a record that is only `creating` names a Task the link MAY have written. The floor chose + // that name, so before anything reads or deletes it the Task must carry this link's write: the spec digest the + // link built for this grant, or this lease id and this holder in its env. Anything else is someone else's Task. + const envMarks = (id: string, t: AxObserved) => { + const env = new Map((t.spec.env ?? []).map((e) => [e.name ?? "", e.value ?? ""])) + return env.get("AX_CONWIP_LEASE_ID") === id && env.get("AX_CONWIP_HOLDER") === cfg.holder + } + const owns = (id: string, r: LeaseRec, t: AxObserved) => + r.created !== undefined || (r.creatingDigest !== undefined && specDigest(t.spec) === r.creatingDigest) || envMarks(id, t) + /** Journal that the Task under this name is not the link's: nothing reads, deletes or reports it any more. A cancel + * already asked for is answered `cancelled`, since nothing of this lease runs in ax. */ + const markNotMine = (id: string) => Effect.gen(function*() { + const r = j.recs.get(id) + if (!r || r.notMine) return + j.append({ ev: "not-mine", leaseId: id }) + log("not-mine", { leaseId: id }) + if (!r.report && r.released === undefined && (abandoned.get(id) === "cancel" || r.deleting === "cancel")) yield* report(id, "cancelled") + }) + /** GetTask a `creating`-only record's name: owned, absent, foreign (journaled not-mine) or unknown (ax did not answer). */ + const ownership = (id: string, r: LeaseRec) => Effect.gen(function*() { + const got = yield* ax.getTask(id).pipe(Effect.result) + if (got._tag === "Failure") return "unknown" as const + if (got.success === undefined) return "absent" as const + if (owns(id, r, got.success)) return "owned" as const + yield* markNotMine(id) + return "foreign" as const + }) + + // Round 3: the intent is journaled BEFORE DeleteTask (`deleting`), so a DeleteTask that lands and whose reply is lost + // (DEADLINE_EXCEEDED) still reads as a delete on the next resync (a cancel is reported `cancelled`, not task-lost), + // and one that did not land is sent again by reconcile. NotFound means already gone. + const del = (id: string, why: string): Effect.Effect => Effect.gen(function*() { + const rec = j.recs.get(id) + if (rec?.notMine) return true + if (rec !== undefined && rec.created === undefined && rec.creating) { + const o = yield* ownership(id, rec) + if (o === "unknown") { log("delete-deferred", { task: id, why, until: "ownership" }); return false } + if (o === "foreign") return true + } + if (rec !== undefined && rec.deleting === undefined) j.append({ ev: "deleting", leaseId: id, why }) + return yield* ax.deleteTask(id).pipe( + Effect.tap(() => Effect.sync(() => log("delete", { task: id, why }))), + Effect.as(true), + Effect.catch((e: AxError) => Effect.sync(() => { + if (e.code === NOT_FOUND) { log("delete", { task: id, why, absent: true }); return true } + log("delete-deferred", { task: id, why, code: e.code }); return false + }))) + }) + + /** B2: an old attempt is gone only when GetTask answers NotFound; until then attempt n+1 is not created. + * Critique D.2 (P2): a Task that stays Terminating is asked again every terminatingRetryMs, and a fence that times + * out on a Task ax still shows is `stuck` (infra/delete-stuck, final), not an ax outage: a retryable reason sent the + * job straight back to this holder, which fenced the same stuck name again until the attempts were spent. */ + const fence = (old: string) => Effect.gen(function*() { + const until = Date.now() + fenceTimeoutMs + let askedAt: number | undefined + let seen: "terminating" | "other" = "other" // only a Task last SEEN Terminating is a stuck delete; a refused DeleteTask is an ax outage + for (;;) { + if (j.recs.get(old)?.notMine) return "gone" as const // round 3: not this link's Task; nothing of attempt n runs under it + const r = yield* ax.getTask(old).pipe(Effect.result) + if (r._tag === "Success" && r.success === undefined) { + if (j.recs.has(old) && !j.recs.get(old)!.deleted) j.append({ ev: "deleted", leaseId: old }) + return "gone" as const + } + seen = r._tag === "Success" && r.success!.phase === "Terminating" ? "terminating" : "other" + if (r._tag === "Success") { + const terminating = r.success!.phase === "Terminating" + if (askedAt === undefined || !terminating) { if (yield* del(old, "superseded")) askedAt ??= Date.now() } + else if (Date.now() - askedAt >= termRetryMs) { + askedAt = Date.now() + const again = yield* ax.deleteTask(old).pipe(Effect.result) + log("delete-retry", { task: old, where: "fence", ...(again._tag === "Failure" ? { code: again.failure.code } : {}) }) + } + } + if (Date.now() >= until) return seen === "terminating" ? "stuck" as const : "unanswered" as const + yield* Effect.sleep(Math.min(cfg.resyncMs, 2 * cfg.secondMs)) + } + }) + const fenceFailure = (old: string, f: "stuck" | "unanswered"): FailureOutput => f === "stuck" + ? { reason: "infra/delete-stuck", message: `superseded attempt ${old} still Terminating in ax after the fence timeout` } // final + : { reason: "pre-start/ax-unavailable", message: `superseded attempt ${old} not confirmed deleted` } + + /** B11: never more live Tasks than maxInFlight. Busy = live Tasks in the atespace (anyone's) U admitted leases + * not known finished. Synchronous, so two dispatch fibers cannot both take the last slot. */ + /** Critique D.2: a Task the ladder gave up on (delete-stuck) whose worker a Substrate read proved gone. Only such a + * name stops counting against maxInFlight; without the read port it counts until an operator clears it. */ + const provenFree = (name: string) => { const r = j.recs.get(name); return r?.deleteStuck === true && r.freeProven === true } + const busy = () => { + // HF link-ladder (VERIFY LL-4 M1 root cause): a finished admitted lease leaves `admitted`, but its name is not + // removed from the set: while ax still lists its Task live (Running, or stuck Terminating) it holds a worker, and + // only the provably-free exclusion above may stop it counting. Before, `b.delete(id)` dropped it for one call. + const b = new Set() + for (const [name, t] of last) if (!TERMINAL.has(t.phase) && !provenFree(name)) b.add(name) + for (const id of admitted) { + const r = j.recs.get(id), t = last.get(id) + if (!r || r.report || r.released !== undefined || r.dropped !== undefined || r.deleted || (t && TERMINAL.has(t.phase))) { admitted.delete(id); continue } // HF link-ladder: a live Task in the snapshot still counts + b.add(id) + } + return b + } + const admit = (id: string) => Effect.sync(() => { + if (!canCreate()) return false + const b = busy() + if (b.has(id) || b.size < cfg.maxInFlight) { admitted.add(id); return true } + return false + }) + + /** A grant this link will not create any more (B3): a cancel is reported once any Task that may exist is gone. */ + const abandon = (id: string, why: "cancel" | "lost", where: string) => Effect.gen(function*() { + log(where, { leaseId: id, why }) + const rec = j.recs.get(id) + if (rec && Journal.maybeCreated(rec)) return yield* del(id, why) // round 1: reconcile reports cancelled on NotFound + if (why === "cancel") yield* report(id, "cancelled") + }) + + let guestTokenWarned = false + const createOnly = (g: Grant) => Effect.gen(function*() { + const id = g.leaseId + let waited = false + let unconfirmed = false + for (;;) { // round 1: the capacity gate is re-checked before every UpdateTask; a closed gate goes back to waiting + // Round 4: and the lease must still be live (renewed within the lease) before the slot is taken + while (!leaseLive(g) || !(yield* admit(id))) { // B11: journaled and heartbeated while it waits + if (abandoned.has(id) || deadlinePassed(g)) break + if (!leaseLive(g)) { if (!unconfirmed) { log("lease-unconfirmed", { leaseId: id }); unconfirmed = true } } + else if (!waited) { log("waiting-for-slot", { leaseId: id }); waited = true } + yield* Effect.sleep(cfg.resyncMs) + } + const early = abandoned.get(id) + if (early !== undefined) return yield* abandon(id, early, "abandoned-before-create") + const now = j.recs.get(id) // round 2: withdrawn, or replaced by a newer grant of the same leaseId: never create + if (now?.grant !== g || now.released !== undefined) return log("grant-superseded-before-create", { leaseId: id }) + if (deadlinePassed(g)) return yield* failG(g, { reason: "deadline-exceeded", message: "the grant's deadline passed before its Task was created" }) // round 4 + const m = mode() + const built = axTaskFromGrant(g, { ...cfg.shape, holder: cfg.holder, ...(m === "guest" ? {} : { completeUrl: undefined }) }) + if (built._tag === "refused") { + if (built.reason === "pre-start/lease-token-missing" && !guestTokenWarned) { + guestTokenWarned = true + log("guest-token-missing", { leaseId: id, hint: "floor and link disagree: the link completes as guest, the floor mints no lease token for this holder" }) + } + return yield* failG(g, { reason: built.reason, message: built.message }) + } + const digest = specDigest(built.task.spec) + for (let i = 1; ; i++) { + const why = abandoned.get(id) // B3: checked before every UpdateTask + if (why !== undefined) return yield* abandon(id, why, "abandoned-before-create") + const cur = j.recs.get(id) + if (cur?.grant !== g || cur.released !== undefined) return log("grant-superseded-before-create", { leaseId: id }) + if (!canCreate() || mode() !== m) { log("gate-closed-before-create", { leaseId: id, try: i }); break } // round 1 + if (deadlinePassed(g)) return yield* failG(g, { reason: "deadline-exceeded", message: "the grant's deadline passed before its Task was created" }) + if (!leaseLive(g)) { log("lease-unconfirmed-before-create", { leaseId: id, try: i }); break } // round 4: back to waiting + if (!j.recs.get(id)?.creating) j.append({ ev: "creating", leaseId: id, digest }) // round 1: write-ahead; round 3: with the digest + const r = yield* Effect.gen(function*() { + const found = yield* ax.getTask(id) + if (found === undefined) { yield* ax.createTask(built.task); return "created" as const } + if (specDigest(found.spec) === digest) return "adopted" as const + return envMarks(id, found) ? "conflict" as const : "foreign" as const + }).pipe(Effect.result) + if (r._tag === "Success") { + // L4: never overwrite a Task this link did not write. Round 3: a Task without this link's marks is journaled + // not-mine first, so no cleanup path (janitor, lost, cancel, salvage) ever reads or deletes it. + if (r.success === "foreign") yield* markNotMine(id) + if (r.success === "conflict" || r.success === "foreign") + return yield* failG(g, { reason: "pre-start/name-conflict", message: "a Task with this name and another spec exists" }) + j.append({ ev: "created", leaseId: id, digest }) + log(r.success, { leaseId: id, attempt: g.attempt }) + const after = abandoned.get(id) // B3: and again once UpdateTask returned + if (after !== undefined) { yield* del(id, after); return } + const back = yield* ax.getTask(id).pipe(Effect.result) // B12: FIELD-MAP 5a read-back + if (back._tag === "Success" && back.success !== undefined && specDigest(back.success.spec) !== digest) { + yield* del(id, "pre-start") + return yield* failG(g, { reason: "pre-start/readback-mismatch", message: "ax stored another spec than the link wrote" }) + } + return + } + const e = r.failure + if (e.resourceExhausted) return yield* failG(g, { reason: "pre-start/resource-exhausted", message: e.message }) // B5 + if (!e.unavailable) return yield* failG(g, { reason: "pre-start/invalid-spec", message: e.message }) + if (i >= cfg.createAttempts) { + // Critique pass 2026-09-24 (red team double-run-r3-1): an UpdateTask whose reply was lost may have landed. A + // retryable pre-start verdict lets the floor hand attempt n+1 to another holder at once, so it is sent only + // once GetTask answers NotFound; otherwise the grant goes back to waiting, and the next try adopts a landed Task. + const seen = yield* ax.getTask(id).pipe(Effect.result) + if (seen._tag === "Success" && seen.success === undefined) return yield* failG(g, { reason: "pre-start/ax-unavailable", message: e.message }) + log("create-unconfirmed", { leaseId: id, try: i, code: e.code }) + yield* Effect.sleep(cfg.resyncMs) + break + } + yield* Effect.sleep(jitter(2 ** i * 100)) + } + } + }) + + /** Round 1: an unjournaled name may be fenced only if it is `-a` with k < this attempt and the Task's own + * env says this lease id and this holder (AX_CONWIP_HOLDER, written by this link at create). */ + const ownEarlierAttempt = (old: string, g: Grant, t: AxObserved) => { + const base = g.leaseId.replace(/-a\d+$/, ""), m = /^(.*)-a(\d+)$/.exec(old) + if (m === null || m[1] !== base || Number(m[2]) >= g.attempt) return false + return envMarks(old, t) + } + + const dispatch = (g: Grant) => Effect.gen(function*() { + if (j.recs.get(g.leaseId)?.grant === g) dispatching.set(g.leaseId, g) // round 2: only the current generation + const bad = validRunsOn(g.job, cfg.servedLabels) // B7 + if (bad) return yield* failG(g, bad) + // Final pass (VERIFY-LINK V1): the fence set is `supersedes` plus every earlier attempt of the same job this link + // journaled, that may exist in ax and is not confirmed deleted. A floor that omits `supersedes` (or a Heartbeat + // whose `lost` arrives after this grant) no longer lets attempt n+1 start beside a running attempt n. + const olds = [...new Set([...(g.supersedes ?? []), ...[...j.recs.values()] + .filter((r) => r.grant.leaseId !== g.leaseId && r.grant.job.metadata.name === g.job.metadata.name && r.grant.attempt < g.attempt + && Journal.maybeCreated(r) && r.deleted !== true && !r.notMine) + .map((r) => r.grant.leaseId)])] + if (olds.length > (g.supersedes ?? []).length) log("fence-set-extended", { leaseId: g.leaseId, supersedes: g.supersedes ?? [], fence: olds }) + // Round 2: an earlier attempt that may exist and has no verdict yet is READ before anything fences it. A Task that + // finished while its lease expired carries a result the floor can still accept (rule 4b); deleting it first loses + // the success and runs the job again. + for (const o of olds) { + const until = Date.now() + fenceTimeoutMs + for (;;) { + const s = yield* salvage(o) + if (s !== "unknown") break + const why = abandoned.get(g.leaseId) + if (why !== undefined) return yield* abandon(g.leaseId, why, "abandoned-before-create") + if (Date.now() >= until) + return yield* failG(g, { reason: "pre-start/ax-unavailable", message: `superseded attempt ${o} could not be read before its fence` }) + yield* Effect.sleep(Math.min(cfg.resyncMs, 2 * cfg.secondMs)) + } + } + // Round 1 (B15 at dispatch): an earlier attempt whose verdict is still in the outbox is delivered BEFORE it is + // fenced. The floor accepts it (rule 4b: n+1 is queued, or leased to this holder and not created) and withdraws + // this grant; deleting the finished Task first would throw its result away and rerun the job. + const undelivered = () => olds.filter((o) => { const r = j.recs.get(o); return r?.report !== undefined && !r.reported && r.dropped === undefined }) + if (undelivered().length > 0) { + log("supersede-waits-for-verdict", { leaseId: g.leaseId, old: undelivered() }) + const until = Date.now() + fenceTimeoutMs + for (;;) { + if (undelivered().length === 0) break // delivered (maybe by another fiber): decided below + const mine = j.recs.get(g.leaseId) + if (mine?.grant !== g || mine.released !== undefined) return log("grant-superseded-before-create", { leaseId: g.leaseId }) + const why = abandoned.get(g.leaseId) + if (why !== undefined) return yield* abandon(g.leaseId, why, "abandoned-before-create") + yield* drainOutbox + if (undelivered().length === 0) break + if (Date.now() >= until) + return yield* failG(g, { reason: "pre-start/superseded-verdict-pending", message: `the verdict of ${undelivered().join(",")} is not delivered yet` }) + yield* Effect.sleep(Math.min(cfg.resyncMs, 2 * cfg.secondMs)) + } + } + // Round 2: withdrawn only when the floor SAID so (Complete.withdrew names this grant). A duplicate answer (rule 4c) + // or a delivered retryable failure withdrew nothing: this grant is the retry, so fence the old attempt and create. + // Round 3: and only for the generation it withdrew (see drainOutbox). + const by = olds.find((o) => { const x = j.recs.get(o); return x?.withdrew === g.leaseId && x.withdrewGen !== undefined && sameGen(genOf(g), x.withdrewGen) }) + if (by !== undefined) { + // Round 4: only this generation's record; a regrant taken meanwhile is the floor's newer word and stays live + const cur = j.recs.get(g.leaseId) + if (cur?.grant !== g) return log("grant-superseded-before-create", { leaseId: g.leaseId }) + if (cur.released === undefined) j.append({ ev: "released", leaseId: g.leaseId, why: "withdrawn" }) + return log("withdrawn", { leaseId: g.leaseId, by }) + } + const self = j.recs.get(g.leaseId) + if (self?.grant !== g || self.released !== undefined) return log("grant-superseded-before-create", { leaseId: g.leaseId }) + for (const old of olds) { // L3 and B2: the old attempt is gone before attempt n+1 is created + const rec = j.recs.get(old) + if (rec && rec.released === undefined) j.append({ ev: "released", leaseId: old, why: "superseded" }) + if (rec && Journal.maybeCreated(rec)) { + const f = yield* fence(old) + if (f !== "gone") { + log("fence-failed", { leaseId: g.leaseId, old, why: f }) + return yield* failG(g, fenceFailure(old, f)) + } + continue + } + // Round 1 (token scope): a name this link has no journal record for is never deleted on the floor's word alone. + // Absent is fine. Present is fenced only when it is an earlier attempt of THIS job and the Task itself carries + // this link's marks (the journal-lost case, F8); anything else is someone else's Task and this grant is refused. + const r = yield* ax.getTask(old).pipe(Effect.result) + if (r._tag === "Failure") + return yield* failG(g, { reason: "pre-start/ax-unavailable", message: `superseded attempt ${old} not confirmed absent` }) + if (r.success !== undefined && ownEarlierAttempt(old, g, r.success)) { + log("supersedes-unjournaled-own", { leaseId: g.leaseId, old }) + const f = yield* fence(old) + if (f !== "gone") { + log("fence-failed", { leaseId: g.leaseId, old, why: f }) + return yield* failG(g, fenceFailure(old, f)) + } + continue + } + if (r.success !== undefined) { + log("supersedes-foreign", { leaseId: g.leaseId, old }) + return yield* failG(g, { reason: "pre-start/supersedes-foreign", message: `supersedes names ${old}, a Task this link did not create` }) + } + } + yield* createOnly(g) + }).pipe( + Effect.ensuring(Effect.sync(() => { if (dispatching.get(g.leaseId) === g) dispatching.delete(g.leaseId) })), + // L9. Round 4: a defect is not swallowed while the lease is held. The grant is failed back (infra/link-defect, + // retryable: the floor requeues it and the next attempt fences anything this one may have created); if even that + // cannot be journaled, the process dies (exit 1) and a restart resumes from the journal (B13). + Effect.catchCause((c) => Cause.hasInterruptsOnly(c) ? Effect.interrupt : Effect.gen(function*() { + log("job-fiber-died", { leaseId: g.leaseId, cause: Cause.pretty(c) }) + const rec = j.recs.get(g.leaseId) + if (rec?.grant !== g || rec.report !== undefined || rec.released !== undefined || rec.notMine) return + yield* failG(g, { reason: "infra/link-defect", message: Cause.pretty(c).slice(0, 300) }) + }).pipe(Effect.catchCause((c2) => Cause.hasInterruptsOnly(c2) ? Effect.interrupt + : Effect.sync(() => log("link-defect-fatal", { leaseId: g.leaseId, cause: Cause.pretty(c2).slice(0, 300) })).pipe( + Effect.andThen(Deferred.die(fatal, Cause.squash(c2))), Effect.asVoid))))) + + // ---------------------------------------------------------------- resync: the executor's list is the ledger + const outcomeOf = (id: string, t: AxObserved, salvage = false) => Effect.gen(function*() { + const fail = (id: string, o: FailureOutput) => report(id, "failure", o, undefined, salvage) + if (t.phase === "Completed") { + const res = yield* ax.getTaskResult(id).pipe(Effect.result) + // Round 4: UNIMPLEMENTED is the server losing P1 (a rollout), not this Task's result: close the gate at once and + // keep the Task, uncharged, to read it once P1 is back. Guest mode does not read results through ax. + if (res._tag === "Success" && res.success === "unimplemented" && cfg.completion !== "guest") { + setServerP1(false) + return log("result-read-deferred", { leaseId: id, why: "server without P1" }) + } + const unreadable = res._tag === "Failure" ? `GetTaskResult: ${res.failure.message}` + : res.success === undefined ? "no result stored" + : res.success === "unimplemented" ? "server without P1" + : res.success.digestOk === false ? "sha256 mismatch" : undefined + if (unreadable !== undefined) { // B4: a transient read is retried at the next resync, bounded + const n = (resultFailures.get(id) ?? 0) + 1 + resultFailures.set(id, n) + const permanent = res._tag === "Failure" && !res.failure.unavailable + if (!permanent && n < resultReadTries) return log("result-read-retry", { leaseId: id, try: n, why: unreadable }) + return yield* fail(id, { reason: "infra/result-unreadable", phase: t.phase, message: unreadable }) + } + const ok = (res as { success: { content: string; sha256: string } }).success + const bytes = Buffer.byteLength(ok.content, "utf8") + if (bytes > maxOutputBytes) // round 2: refused here, typed, before the floor answers SQLITE_TOOBIG for ever + return yield* fail(id, { reason: "agent/output-too-large", phase: t.phase, + message: `result ${bytes} B > ${maxOutputBytes} B (sha256 ${ok.sha256 || createHash("sha256").update(ok.content).digest("hex")})` }) + let output: unknown + try { output = JSON.parse(ok.content) } catch { return yield* fail(id, { reason: "agent/result-not-json", phase: t.phase }) } + const u = t.usage + return yield* report(id, "success", output, u ? { prompt_tokens: u.promptTokens ?? 0, completion_tokens: u.completionTokens ?? 0, tool_calls: u.toolCalls ?? 0 } : undefined, salvage) + } + const ready = t.conditions.find((c) => c.type === "Ready") + const why = `${ready?.reason ?? ""} ${ready?.message ?? ""}` + const code = /ExitCode=(\d+)/.exec(why) + const reason = /ResourceExhausted/i.test(why) ? "pre-start/resource-exhausted" // B5: a capacity race, retryable (#367) + : /ActorTemplateRejected/.test(why) ? "pre-start/actor-template-rejected" + : /CRASH/i.test(why) ? "infra/actor-crashed" : code ? "agent/exit-code" : "agent/failed" + yield* fail(id, { reason, phase: t.phase, ...(ready?.message ? { message: ready.message } : {}), ...(code ? { exitCode: Number(code[1]) } : {}) }) + }) + + /** B1: a Task missing from the list is confirmed with GetTask. `gone` is true only on NotFound. */ + const confirm = (id: string) => ax.getTask(id).pipe(Effect.map((t) => ({ t, gone: t === undefined })), Effect.orElseSucceed(() => undefined)) + + /** Round 2: before any DeleteTask of a Task that may exist and whose verdict is not journaled (heartbeat `lost`, the + * janitor, a supersede fence), GetTask it; a terminal Task has its outcome read and journaled first. `unknown` means + * ax did not answer or the result read is being retried: delete nothing yet. */ + const salvage = (id: string) => Effect.gen(function*() { + const r = j.recs.get(id) + if (!r || r.report !== undefined || !Journal.maybeCreated(r) || r.deleting !== undefined || r.deleted) return "none" as const + const t = yield* ax.getTask(id).pipe(Effect.result) + if (t._tag === "Failure") return "unknown" as const + if (t.success === undefined) return "gone" as const + if (!owns(id, r, t.success)) { yield* markNotMine(id); return "foreign" as const } // round 3: never read another's result + if (!TERMINAL.has(t.success.phase)) return "live" as const + yield* outcomeOf(id, t.success, true) + return j.recs.get(id)?.report !== undefined ? "read" as const : "unknown" as const + }) + + /** Critique D.2 (DELETE-HANG ladder). The Task's DeleteTask is journaled and ax still shows it Terminating (stock ax + * ACKs a delete event and may drop it). Every step is timed from `deletingAt`, so a restart or a compaction does not + * reset the clock: re-delete k is due at deletingAt + k * retry (k <= retries), the escalation at escalate, the + * give-up at giveUp. The slot is released only when a Substrate read proves the actor absent. */ + const classify = (id: string) => deps.substrate === undefined ? Effect.succeed(undefined) + : deps.substrate.actor(id, cfg.shape.atespace).pipe(Effect.result, Effect.map((x) => x._tag === "Success" ? x.success : "unknown" as const)) + const terminatingStep = (id: string, now: number) => Effect.gen(function*() { + const r = j.recs.get(id) + if (!r || r.deleting === undefined || r.deleted) return + const since = r.deletingAt ?? r.at + const age = now - since + if (r.deleteStuck) { // given up: only a newly proven-free actor changes anything + if (!r.freeProven && deps.substrate !== undefined && (yield* classify(id)) === "absent") { + j.append({ ev: "delete-stuck", leaseId: id, freeProven: true }) + log("delete-stuck", { task: id, freeProven: true, since }) + yield* Deferred.succeed(wake, undefined) + } + return + } + const n = r.deleteRetries ?? 0 + if (n < termRetries && age >= (n + 1) * termRetryMs) { + j.append({ ev: "delete-retry", leaseId: id, n: n + 1 }) // write-ahead, like `deleting` + const again = yield* ax.deleteTask(id).pipe(Effect.result) + log("delete-retry", { task: id, n: n + 1, since, ...(again._tag === "Failure" ? { code: again.failure.code } : {}) }) + } + if (age >= termEscalateMs && !escalated.has(id)) { + escalated.add(id) + const actor = yield* classify(id) + log("ax-delete-escalate", { task: id, atespace: cfg.shape.atespace, since, retries: j.recs.get(id)?.deleteRetries ?? 0, actor: actor ?? "no-reader" }) + } + if (age >= termGiveUpMs) { + const freeProven = (yield* classify(id)) === "absent" + j.append({ ev: "delete-stuck", leaseId: id, freeProven }) + log("delete-stuck", { task: id, why: r.deleting, since, retries: j.recs.get(id)?.deleteRetries ?? 0, freeProven }) + if (freeProven) yield* Deferred.succeed(wake, undefined) + } + }) + + const reconcile = (id: string, r: LeaseRec, listed: AxObserved | undefined, now: number) => Effect.gen(function*() { + if (!Journal.maybeCreated(r)) return // not created yet: dispatch owns it (or a restart resumes it) + let t = listed + let gone = false + if (t === undefined && !r.deleted) { + const c = yield* confirm(id) + if (c === undefined) return // ax did not answer; decide nothing this round + t = c.t; gone = c.gone + } + if (!gone) absent.delete(id) + if (t !== undefined && !gone && !owns(id, r, t)) return yield* markNotMine(id) // round 3: token scope + if (r.deleting !== undefined) { + if (gone) { + j.append({ ev: "deleted", leaseId: id }) + if (r.deleting === "cancel" && !r.report && r.released === undefined) yield* report(id, "cancelled") + // double-run-r3-1b: a pending-timeout whose DeleteTask was deferred is reported once the Task is gone + else if (r.deleting === "pre-start" && !r.report && !r.reported && r.released === undefined && r.dropped === undefined) + yield* fail(id, { reason: "pre-start/pending-timeout", message: "deleted after the pending timeout" }) + } else if (t !== undefined && t.phase === "Terminating") { + yield* terminatingStep(id, now) + } else if (t !== undefined && t.phase !== "Terminating") { // round 3: a journaled DeleteTask that did not land + if (r.released !== undefined && r.report === undefined && !r.reported && r.dropped === undefined && TERMINAL.has(t.phase)) { + yield* outcomeOf(id, t, true) // a released lease's finished Task is read before it is deleted + if (j.recs.get(id)?.report === undefined) return + } + yield* del(id, r.deleting) + } + return + } + if (r.reported || r.dropped !== undefined || r.released !== undefined) { // the floor is told: janitor only + if (gone) return j.append({ ev: "deleted", leaseId: id }) + if (t === undefined) return + if (r.released !== undefined && r.report === undefined && !r.reported && r.dropped === undefined && TERMINAL.has(t.phase)) { + yield* outcomeOf(id, t, true) // round 2: a released lease's finished Task is read before the janitor deletes it + if (j.recs.get(id)?.report === undefined) return // read retried at the next resync; nothing deleted before + } + const live = !TERMINAL.has(t.phase) // B3: a told lease's sandbox that still runs goes at once + if (live || r.released !== undefined || r.dropped !== undefined || now - r.at >= cfg.deleteAfterMs) + yield* del(id, r.released ? "released" : r.dropped ? "dropped" : live ? "told-still-running" : "janitor") + return + } + if (r.created === undefined) return // round 1: `creating` only: cleanup above, otherwise dispatch or a resume owns it + if (r.report) return // verdict written, waiting for the outbox + if (gone) { + const n = (absent.get(id) ?? 0) + 1 + absent.set(id, n) + if (n < 2) return log("task-absent", { leaseId: id, resyncs: n }) // B1: two NotFound resyncs in a row + return yield* fail(id, { reason: "infra/task-lost", message: "created by this link, NotFound on two resyncs" }) + } + if (t === undefined) return + if (TERMINAL.has(t.phase)) return yield* outcomeOf(id, t) + if ((t.phase === "" || t.phase === "Pending") && now - (r.createdAt ?? r.at) > cfg.pendingTimeoutMs) { + // Critique pass 2026-09-24 (red team double-run-r3-1b): a DeleteTask that did not land leaves a Task that may + // still start; the retryable verdict waits until the delete lands (the `deleting` branch above reports it). + if (!(yield* del(id, "pre-start"))) return log("pre-start-deferred", { leaseId: id, why: "delete-unconfirmed" }) + return yield* fail(id, { reason: "pre-start/pending-timeout", phase: t.phase || "Pending" }) + } + const deadline = r.grant.deadline + if (t.phase === "Running" && deadline !== undefined && now > deadline + cfg.deadlineBackstopMs) { + yield* del(id, "deadline") + return yield* fail(id, { reason: "deadline-exceeded", phase: t.phase }) + } + }) + + /** Round 4: "P1 proven" is a live condition. Logged on every change; p1-missing closes the gate (capacity 0). */ + const setServerP1 = (v: boolean) => { + if (v === serverP1) return + serverP1 = v + log("completion-probe", { serverP1, completion: cfg.completion, mode: mode() ?? "none" }) + if (!v && cfg.completion !== "guest") log("p1-missing", { completion: cfg.completion }) + } + const probe = Effect.gen(function*() { // P1 adds GetTaskResult; stock v0.3.0 answers UNIMPLEMENTED + const p = yield* ax.getTaskResult("conwip-link-capability-probe").pipe(Effect.result) + if (p._tag === "Failure") return // not answered: keep what is known (undefined after an outage: capacity 0) + setServerP1(p.success !== "unimplemented") + }) + + const checkGateway = Effect.gen(function*() { // B10 + const g = yield* ax.getGateway(cfg.shape.gateway).pipe(Effect.result) + const open = g._tag === "Success" && g.success !== undefined && gatewayAllowsAll(g.success) + const ok = g._tag === "Success" && g.success !== undefined && !open + // Critique D.2 item 5 (V2): three events. `gateway-ok` carries the allowlist size, an existing Gateway that + // allows everything is `gateway-open`, and only an absent Gateway or a failed lookup is `gateway-missing`. + const state = g._tag === "Failure" ? `missing:${g.failure.code}` : g.success === undefined ? "missing" : open ? "open" : "ok" + if (ok !== gatewayOk || state !== gatewayState) { + if (g._tag === "Failure") log("gateway-missing", { gateway: cfg.shape.gateway, code: g.failure.code }) + else if (g.success === undefined) log("gateway-missing", { gateway: cfg.shape.gateway, found: false }) + else if (open) log("gateway-open", { gateway: cfg.shape.gateway, allowsAll: true }) + else log("gateway-ok", { gateway: cfg.shape.gateway, hosts: g.success.hosts.length }) + } + gatewayOk = ok; gatewayState = state + }) + + const resync = Effect.gen(function*() { + const listed = yield* listAll(ax).pipe(Effect.result) + if (listed._tag === "Failure") { + if (axUp) log("ax-down", { code: listed.failure.code }) + axUp = false + // Round 2: nothing read before the outage opens the gate after it. Guest mode does not depend on serverP1, so + // the Gateway verdict and the resync flag are cleared too; canCreate() opens only after a full resync. + resynced = false + gatewayOk = undefined; gatewayState = undefined + return + } + const wasDown = !axUp + if (wasDown) serverP1 = undefined // B6: nothing proven before the outage holds after it + yield* probe // round 4: on every resync, since a rollout replaces ax-server with no failed ListTasks + yield* checkGateway + last = new Map(listed.success.map((t) => [t.name, t])) + if (wasDown) log("ax-up", { tasks: listed.success.length }) + axUp = true // round 2: only once probe, Gateway and the occupancy snapshot are all from this side of the outage + const now = Date.now() + for (const [id, r] of [...j.recs]) { + if (dispatching.has(id)) continue + yield* reconcile(id, r, last.get(id), now) + } + resynced = true + }) + + // Occupancy = |live Tasks in the atespace (ours or not) U leases held and not known terminal| (limiter rebuilt from + // the executor's list, agent-stack-k8s limiter.go). A P1-terminal Task is suspended and holds no worker. + const occupancy = () => { + const b = new Set() + for (const [name, t] of last) if (!TERMINAL.has(t.phase) && !provenFree(name)) b.add(name) + for (const id of j.held()) { const t = last.get(id); if (!(t && TERMINAL.has(t.phase))) b.add(id) } + return b.size + } + + // ---------------------------------------------------------------- heartbeat: level-triggered, complete held set + const heartbeat = Effect.gen(function*() { + const ids = j.held() + // Round 4: the reply is about the generations this call vouched for. A reply that lands after a regrant of the + // same leaseId (gen n+1) says nothing about it: `lost`, `cancelRequested` and `renewed` apply only while the record + // still holds the grant object that was sent. + const sent = new Map(ids.map((id) => [id, j.recs.get(id)?.grant])) + const current = (id: string) => sent.has(id) && j.recs.get(id)?.grant === sent.get(id) + const sentAt = Date.now() + const pendingRequestKey = j.pendingLeaseKey // round 4: vouch for grants under a key whose reply never landed (B8) + const r = yield* floor.heartbeat({ holderIdentity: cfg.holder, leaseIds: ids, ...(pendingRequestKey !== undefined ? { pendingRequestKey } : {}) }).pipe(Effect.tap(markFloorOk), Effect.result) + if (r._tag === "Failure") { + if (r.failure.fatal) { yield* failFatal(r.failure); return undefined } + log("heartbeat-error", { kind: r.failure.kind }) + return undefined + } + for (const id of r.success.renewed) { const g = sent.get(id); if (g !== undefined && current(id)) renewedLocal(id, g, sentAt) } + for (const id of r.success.lost) { // fencing: the lease is gone, so the Task goes; no report + if (!current(id)) { log("stale-heartbeat-reply", { leaseId: id, about: "lost" }); continue } + const rec = j.recs.get(id) + pendingResume.delete(id) + if (!rec) { const x = j.invalid.get(id); if (x && x.released === undefined) j.append({ ev: "released", leaseId: id, why: "lost" }); continue } + if (rec.released !== undefined) continue + j.append({ ev: "released", leaseId: id, why: "lost" }) + abandoned.set(id, "lost") + log("lost", { leaseId: id }) + if (Journal.maybeCreated(rec)) { // round 1: `creating` counts; the janitor confirms NotFound + const s = yield* salvage(id) // round 2: a finished Task's verdict is read and journaled before the delete + if (s === "unknown") log("delete-deferred", { task: id, why: "lost", until: "verdict-read" }) // the janitor reads, then deletes + else if (s !== "gone" && s !== "foreign") yield* del(id, "lost") + } + } + for (const id of r.success.cancelRequested) { // Temporal cancel_requested; ax cancel is DeleteTask, two-phase + if (!current(id)) { log("stale-heartbeat-reply", { leaseId: id, about: "cancel" }); continue } + const rec = j.recs.get(id) + if (!rec || rec.report || rec.deleting !== undefined) continue + log("cancel", { leaseId: id }) + pendingResume.delete(id) + abandoned.set(id, "cancel") // B3: a running dispatch reports or deletes; no later resume creates it + if (dispatching.has(id)) continue + // Round 1: a Task that may exist (`creating`) is deleted and seen NotFound before `cancelled` is reported + if (Journal.maybeCreated(rec)) yield* del(id, "cancel") + else yield* report(id, "cancelled") + } + const renewed = new Set(r.success.renewed) + for (const id of [...pendingResume]) { // B13: resume only what the floor still says is ours + if (!renewed.has(id) || !current(id)) continue + pendingResume.delete(id) + const rec = j.recs.get(id) + if (rec && rec.created === undefined && !rec.report && rec.released === undefined && rec.dropped === undefined) { + log("resume", { leaseId: id }) + yield* Effect.forkScoped(dispatch(rec.grant)) + } + } + return renewed + }) + + // ---------------------------------------------------------------- the loops + const every = (name: string, ms: () => number, body: Effect.Effect) => Effect.forever( + Effect.suspend(() => Effect.sleep(ms())).pipe(Effect.andThen(body), // re-read: the floor may change the interval + Effect.catchCause((c) => Cause.hasInterruptsOnly(c) ? Effect.interrupt : Effect.sync(() => log(`${name}-died`, { cause: Cause.pretty(c) }))))) + + /** Round 3: the heartbeat interval, at most a third of the shortest lease held, so a large floor value can never let + * a held lease expire between two heartbeats. */ + const hbSeconds = () => { + let hi = MAX_HEARTBEAT_SECONDS + for (const id of j.held()) { + const d = j.recs.get(id)?.grant.lease.leaseDurationSeconds + if (d !== undefined && Number.isFinite(d) && d > 0) hi = Math.min(hi, d / 3) + } + return boundedSeconds(hbAsked, defaultHb, Math.max(1, hi)) + } + + /** Final pass (VERIFY-LINK R3-1). The heartbeat sleep is measured from the last heartbeat and is cut short when a + * Lease reply shrinks the interval or journals a grant with a shorter lease: a sleep begun at the 30 s default while + * nothing was held no longer outlives a 12 s lease granted during it. */ + let lastBeatAt = Date.now() + const heartbeatLoop = Effect.forever(Effect.gen(function*() { + let target = jitter(hbSeconds() * cfg.secondMs) + for (;;) { + const left = lastBeatAt + target - Date.now() + if (left <= 0) break + const w = hbWake + yield* Effect.raceFirst(Effect.sleep(left), Deferred.await(w)) + if (yield* Deferred.isDone(w)) { + hbWake = yield* Deferred.make() + target = Math.min(target, jitter(hbSeconds() * cfg.secondMs)) + } + } + lastBeatAt = Date.now() + yield* heartbeat + }).pipe(Effect.catchCause((c) => Cause.hasInterruptsOnly(c) ? Effect.interrupt : Effect.sync(() => log("heartbeat-died", { cause: Cause.pretty(c) }))))) + + let endpointSeen: string | undefined + const leaseOnce = Effect.gen(function*() { + outboxWaitUntil = Math.min(outboxWaitUntil, Date.now()) // B15: a verdict goes out before new work comes in + yield* drainOutbox + // Round 1 (B15): while any verdict is still undelivered, ask for nothing. A grant that arrived now could supersede + // that verdict's attempt, and once n+1 is leased the floor answers n's verdict stale-attempt. + // Round 2: a verdict the floor has answered with a server error is not an outage; it no longer gates Lease (its + // own budget replaces or dead-letters it, and a grant that supersedes it waits at dispatch, bounded). + const undelivered = j.outbox().filter((p) => !verdictFailures.has(p.leaseId)).length + if (undelivered > 0) log("lease-gated-by-outbox", { undelivered }) + const draining = deps.drain !== undefined && (yield* Deferred.isDone(deps.drain)) + const free = !draining && undelivered === 0 && canCreate() ? Math.max(0, cfg.maxInFlight - occupancy()) : 0 + // B8: a key is journaled before the call and reused until a reply is journaled, so a lost reply is replayed, + // not stranded. A capacity-0 call can grant nothing, so it is not journaled. + let requestKey = j.pendingLeaseKey + if (requestKey === undefined) { + requestKey = randomUUID() + if (free > 0) j.append({ ev: "lease-key", requestKey }) + } + const sentAt = Date.now() + const r = yield* callLease(free, requestKey) + if (r._tag === "Failure") { + if (r.failure.fatal) return yield* failFatal(r.failure) + leaseFailed(r.failure, requestKey) + // Round 3: bounded, and only this loop waits; the heartbeat and resync fibers run on their own intervals + return yield* Effect.sleep(Math.min(r.failure.retryAfterMs ?? jitter(pollSeconds * cfg.secondMs), MAX_POLL_SECONDS * cfg.secondMs)) + } + yield* applyLease(r.success, requestKey, sentAt) + }) + + const callLease = (capacity: number, requestKey: string) => floor.lease({ holderIdentity: cfg.holder, capacity, requestKey }).pipe( + Effect.tap(markFloorOk), Effect.retry({ times: 2, while: (e: FloorError) => e.kind === "transient", schedule: Schedule.exponential(Math.max(1, cfg.secondMs / 5)).pipe(Schedule.jittered) }), Effect.result) + const leaseFailed = (e: FloorError, requestKey: string) => { + log("lease-error", { kind: e.kind, pendingKey: j.pendingLeaseKey !== undefined, message: e.message.slice(0, 200) }) + // Round 3: a journaled key is replayed through outages (B8), but not through replies the floor keeps answering + // with a server error or that do not decode: those would strand the key's grants for ever. After + // `leaseKeyAttempts` in a row the key is abandoned; the floor's expiry requeues what it granted under it. + if (e.kind === "server-error" && j.pendingLeaseKey === requestKey) { + if (++leaseKeyFailures >= leaseKeyAttempts) { + j.append({ ev: "lease-replied", requestKey, abandoned: true }) + log("lease-key-abandoned", { tries: leaseKeyFailures, message: e.message.slice(0, 200) }) + leaseKeyFailures = 0 + } + } + } + /** Round 4 (B8 at startup): a key journaled by a previous process whose reply never landed is replayed BEFORE the + * first Heartbeat, so its grants are journaled and in the held set when the Heartbeat re-adopts orphans; replayed + * after it, an orphan granted under the key is omitted and released (L3 a). Capacity 0: a key the floor never saw + * grants nothing new. A failed replay leaves the key pending; the Heartbeat then vouches for it. */ + const replayPendingLease = Effect.gen(function*() { + const requestKey = j.pendingLeaseKey + if (requestKey === undefined) return + const sentAt = Date.now() + const r = yield* callLease(0, requestKey) + if (r._tag === "Failure") { + if (r.failure.fatal) return yield* failFatal(r.failure) + return leaseFailed(r.failure, requestKey) + } + log("lease-key-replayed", { grants: r.success.grants.length }) + yield* applyLease(r.success, requestKey, sentAt) + }) + + const applyLease = (reply: LeaseReply, requestKey: string, sentAt: number) => Effect.gen(function*() { + leaseKeyFailures = 0 + // Round 3: the server-driven intervals are bounded (Buildkite falls back to its defaults): 0, a negative or a + // missing value would make both loops tight loops of billed requests. + pollSeconds = boundedSeconds(reply.nextPollSeconds, defaultPoll, MAX_POLL_SECONDS) + hbAsked = reply.heartbeatSeconds + const hbNow = hbSeconds() + const clamp = `${reply.nextPollSeconds}->${pollSeconds} ${reply.heartbeatSeconds}->${hbNow}` + if ((pollSeconds !== reply.nextPollSeconds || hbNow !== reply.heartbeatSeconds) && clamp !== clampLogged) { + clampLogged = clamp + log("interval-clamped", { nextPollSeconds: reply.nextPollSeconds ?? null, usedPoll: pollSeconds, heartbeatSeconds: reply.heartbeatSeconds ?? null, usedHeartbeat: hbNow }) + } + for (const g of reply.grants) { + const prior = j.recs.get(g.leaseId) + // Round 3: a grant of a lease this link finished (compacted away since) is at-least-once redelivery, not work. + const tomb = j.tombs.get(g.leaseId) + if (prior === undefined && tomb !== undefined && !laterGen(genOf(g), tomb.gen)) { log("duplicate-grant", { leaseId: g.leaseId, finished: true }); continue } + if (prior !== undefined) { + // The floor already said it withdrew this generation (Complete.withdrew journaled on an earlier attempt) but its + // dispatch has not run yet: release it now, so its regrant is not taken for a duplicate. + const pg = genOf(prior.grant) + if (prior.released === undefined && !Journal.maybeCreated(prior) && [...j.recs.values()].some((x) => x.withdrew === g.leaseId && x.withdrewGen !== undefined && sameGen(pg, x.withdrewGen))) { + j.append({ ev: "released", leaseId: g.leaseId, why: "withdrawn" }) + log("withdrawn", { leaseId: g.leaseId, by: "journal" }) + } + // Round 2: dedupe on the grant's identity, not its leaseId. A floor that withdrew n+1 (rule 4b) and then requeued + // it for n's retryable verdict grants the same leaseId again with a new acquireTime and leaseTransitions; that + // is a new grant generation, dispatched, when the old record is released and holds nothing in ax or the outbox. + const newer = laterGen(genOf(g), pg) // round 3: strictly later, so a stale redelivery is never a regrant + // Round 4: the floor's generation is authoritative. A strictly later grant means the floor already withdrew + // the one held (its Complete reply may still be in flight, or lost). If the held one never reached ax and owes + // no verdict, it is released now and the regrant taken; the floor does not redeliver it. Its dispatch sees the + // record change and stops (grant-superseded-before-create). + if (newer && prior.released === undefined && !Journal.maybeCreated(prior) && !Journal.pending(prior)) { + j.append({ ev: "released", leaseId: g.leaseId, why: "superseded-by-regrant" }) + log("superseded-by-regrant", { leaseId: g.leaseId, was: pg.leaseTransitions, now: g.lease.leaseTransitions }) + } + const reusable = prior.released !== undefined && !Journal.pending(prior) && (!Journal.maybeCreated(prior) || prior.deleted === true) + if (!(newer && reusable)) { log("duplicate-grant", { leaseId: g.leaseId }); continue } // at-least-once delivery + for (const m of [abandoned, absent, resultFailures, verdictFailures, verdictNotBefore, verdictTriedAt]) m.delete(g.leaseId) + admitted.delete(g.leaseId); pendingResume.delete(g.leaseId) + j.append({ ev: "grant", leaseId: g.leaseId, grant: g, regrant: true }) + log("regrant", { leaseId: g.leaseId, was: prior.released }) + } else j.append({ ev: "grant", leaseId: g.leaseId, grant: g }) + renewedLocal(g.leaseId, g, sentAt) // round 4: the floor acquired it no earlier than this call was sent + log("leased", { leaseId: g.leaseId, attempt: g.attempt, supersedes: g.supersedes ?? [] }) + yield* Effect.forkScoped(dispatch(g)) + } + for (const b of reply.invalid ?? []) { // round 1: refuse one bad grant, keep the rest of the reply + if (b.leaseId === undefined || b.attempt === undefined) { log("grant-undecodable", { message: b.message }); continue } // the floor's expiry frees it + if (j.recs.has(b.leaseId) || j.invalid.has(b.leaseId)) continue + j.append({ ev: "grant-invalid", leaseId: b.leaseId, attempt: b.attempt, message: b.message }) + log("grant-invalid", { leaseId: b.leaseId, attempt: b.attempt }) + yield* Deferred.succeed(wake, undefined) + } + if (j.pendingLeaseKey === requestKey) j.append({ ev: "lease-replied", requestKey }) + yield* Deferred.succeed(hbWake, undefined) // final pass (R3-1): the interval or the held set may have shrunk + const ep = reply.endpoint // B18: only an exact match in the Nix-declared list, and only once + if (ep !== undefined && ep !== deps.floorUrl && ep !== endpointSeen) { + endpointSeen = ep + if ((cfg.floorUrls ?? []).includes(ep) && ep.startsWith("https://")) { log("endpoint-accepted", { endpoint: ep }); deps.onEndpoint?.(ep) } + else log("endpoint-ignored", { endpoint: ep }) + } + }) + + const body = Effect.gen(function*() { + yield* resync // limiter and outcomes rebuilt before the first heartbeat or lease + yield* selfFence // red team double-run-r1-1: a Task left unrenewed past lease + grace by a down link goes first + for (const [id, r] of j.recs) // grants journaled by a previous process but never created (B13: resumed on renewal) + if (r.created === undefined && !r.report && r.released === undefined && r.dropped === undefined && !r.notMine) pendingResume.add(id) + yield* replayPendingLease // round 4: a key left pending by a kill is answered before the first Heartbeat + yield* heartbeat // vouch first: re-adopt or release orphans (L3), then resume only renewed grants + lastBeatAt = Date.now() + // Round 3: renewal and resync start now, before any Complete or Lease can stall (a 429 Retry-After, a timeout + // chain): held leases keep being renewed whatever the first Lease does. Both re-read their interval every time. + yield* Effect.forkScoped(heartbeatLoop) + yield* Effect.forkScoped(every("resync", () => cfg.resyncMs, resync.pipe(Effect.andThen(selfFence), Effect.andThen(drainOutbox)))) + yield* drainOutbox // replay before new work (uplink) + yield* Effect.raceFirst(leaseOnce, Deferred.await(deps.stop)) // its reply sets the server-driven intervals + let drainDeadline: number | undefined + while (!(yield* Deferred.isDone(deps.stop))) { + const w = wake + // G-BK4: while draining, look every resync (not every poll) for the moment nothing is held any more + const draining = deps.drain !== undefined && (yield* Deferred.isDone(deps.drain)) + const nap = draining ? Math.min(cfg.resyncMs, pollSeconds * cfg.secondMs) : jitter(pollSeconds * cfg.secondMs) + const drainWake = deps.drain !== undefined && !draining ? Deferred.await(deps.drain) : Effect.never + yield* Effect.raceFirst(Effect.raceFirst(Effect.raceFirst(Effect.suspend(() => Effect.sleep(nap)), Deferred.await(w)), Deferred.await(deps.stop)), drainWake) + if (yield* Deferred.isDone(deps.stop)) break + if (deps.drain !== undefined && (yield* Deferred.isDone(deps.drain))) { + drainDeadline ??= Date.now() + (deps.drainTimeoutMs ?? Infinity) + const outbox = j.outbox().length + if (occupancy() === 0 && outbox === 0) { log("drain-complete", {}); break } + if (Date.now() >= drainDeadline) { log("drain-timeout", { occupancy: occupancy(), outbox }); break } + } + if (yield* Deferred.isDone(w)) wake = yield* Deferred.make() + yield* Effect.raceFirst(leaseOnce, Deferred.await(deps.stop)) + } + // drain (Buildkite stop before disconnect): no more leases; ax Tasks keep running and are re-adopted on restart + outboxWaitUntil = 0 + yield* drainOutbox + log("drained", { held: j.held().length }) + }) + + return yield* Effect.raceFirst(Effect.scoped(body), Deferred.await(fatal)) +}) diff --git a/pkgs/substrate-apps/src/apps/link/src/main.ts b/pkgs/substrate-apps/src/apps/link/src/main.ts new file mode 100644 index 00000000..ea7f0dfa --- /dev/null +++ b/pkgs/substrate-apps/src/apps/link/src/main.ts @@ -0,0 +1,89 @@ +// conwip-link: the NAS-side process. Configuration is environment only (the NixOS module sets it); the bearer is a +// file under $CREDENTIALS_DIRECTORY (systemd LoadCredential from the agenix secret), read once and never logged. +// Exit codes: 0 drained on SIGTERM; 75 another session holds this identity (L5), or an allowed endpoint switch (B18) +// was persisted and the unit should restart onto it; 78 the floor refused the token, answered a redirect (round 2), or +// the configuration is invalid. +import { randomUUID } from "node:crypto" +import { existsSync, readFileSync, writeFileSync } from "node:fs" +import { join } from "node:path" +import { Deferred, Effect } from "effect" +import { grpcAx } from "./ax.ts" +import { ConfigInvalid, linkConfigOf, readLinkEnv } from "./config.ts" +import type { LinkEnv } from "./config.ts" +import { rpcFloor } from "./floor.ts" +import { Journal } from "./journal.ts" +import { runLink } from "./link.ts" + +const env = (k: string, d?: string) => { + const v = process.env[k] + if (v !== undefined && v !== "") return v + if (d !== undefined) return d + console.error(JSON.stringify({ ev: "config-missing", key: k })) + process.exit(78) +} +const log = (ev: string, f: Record = {}) => console.log(JSON.stringify({ t: new Date().toISOString(), ev, ...f })) +// Round 1: every numeric and enum key is validated up front; a bad one is exit 78 naming the key, never the value. +let c: LinkEnv +try { c = readLinkEnv(process.env) } catch (e) { + if (e instanceof ConfigInvalid) { console.error(JSON.stringify({ ev: "config-invalid", key: e.key, why: e.why })); process.exit(78) } + throw e +} + +const stateDir = env("LINK_STATE_DIR", process.env.STATE_DIRECTORY ?? "/var/lib/conwip-link") +const tokenPath = env("LINK_TOKEN_FILE", join(process.env.CREDENTIALS_DIRECTORY ?? "/nonexistent", "floor-link-token")) +if (!existsSync(tokenPath)) { log("token-missing", { path: tokenPath }); process.exit(78) } +const token = readFileSync(tokenPath, "utf8").trim() +const sessionPath = join(stateDir, "session-id") // stable across restarts of this unit, distinct for a second replica +const journal = Journal.open(stateDir) +if (!existsSync(sessionPath)) writeFileSync(sessionPath, randomUUID() + "\n", { mode: 0o600 }) +const sessionId = readFileSync(sessionPath, "utf8").trim() + +// B18: the floor URLs this link may use, declared in Nix. A switch the floor asked for is persisted here and used on +// the next start only while it is still in the declared list. +const declaredUrl = env("LINK_FLOOR_URL") +const floorUrls = (process.env.LINK_FLOOR_URLS ?? declaredUrl).split(",").map((u) => u.trim()).filter((u) => u !== "") +const endpointPath = join(stateDir, "floor-endpoint") +const persisted = existsSync(endpointPath) ? readFileSync(endpointPath, "utf8").trim() : "" +const floorUrl = persisted !== "" && floorUrls.includes(persisted) ? persisted : declaredUrl +let restartForEndpoint = false + +// Round 2: the link loads its own P1 proto (apps/link/proto/ax-p1.proto), never the stock one: with the stock proto +// GetTaskResult is answered "unimplemented" locally and completion auto/p1 would lease nothing for ever. +let ax: ReturnType +try { ax = grpcAx(c.axServer, c.atespace, undefined, c.axProtoPath) } catch (e) { + console.error(JSON.stringify({ ev: "config-invalid", key: "LINK_AX_PROTO_PATH", why: String((e as Error)?.message ?? e).slice(0, 200) })) + process.exit(78) +} +const program = Effect.gen(function*() { + const stop = yield* Deferred.make() + // G-BK4: with LINK_DRAIN_TIMEOUT_SECONDS > 0 the first signal drains (no new leases; the link exits once every Task + // it holds has ended and its verdict is delivered, or at the timeout), the second stops at once. Default 0: stop at + // once, as before (ax Tasks keep running and are re-adopted by the next start). + const drain = yield* Deferred.make() + const drainS = Number(process.env.LINK_DRAIN_TIMEOUT_SECONDS ?? "0") + let signals = 0 + for (const sig of ["SIGTERM", "SIGINT"] as const) + process.on(sig, () => { + signals++ + if (signals === 1 && Number.isFinite(drainS) && drainS > 0) { log("drain-requested", { signal: sig, drainTimeoutS: drainS }); Effect.runFork(Deferred.succeed(drain, undefined)); return } + log("stop-requested", { signal: sig }); Effect.runFork(Deferred.succeed(stop, undefined)) + }) + const floor = yield* rpcFloor({ url: floorUrl, token, sessionId }) + yield* runLink(linkConfigOf(c, floorUrls), { + ax, floor, journal, log, stop, floorUrl, drain, ...(drainS > 0 ? { drainTimeoutMs: drainS * 1000 } : {}), + onEndpoint: (url) => { // B18: persist, drain, exit 75 so systemd restarts onto the new URL + writeFileSync(endpointPath, url + "\n", { mode: 0o600 }) + restartForEndpoint = true + Effect.runFork(Deferred.succeed(stop, undefined)) + } + }) +}) + +Effect.runPromise(Effect.scoped(program)).then( + () => { ax.close(); process.exit(restartForEndpoint ? 75 : 0) }, + (e) => { + ax.close() + const kind = (e as { kind?: string })?.kind + log("fatal", { kind: kind ?? "defect", message: String((e as { message?: string })?.message ?? e) }) + process.exit(kind === "session-conflict" ? 75 : kind === "auth" || kind === "redirect" || kind === "misrouted" ? 78 : 1) + }) diff --git a/pkgs/substrate-apps/src/apps/link/test/ax-grpc.test.ts b/pkgs/substrate-apps/src/apps/link/test/ax-grpc.test.ts new file mode 100644 index 00000000..32a137f3 --- /dev/null +++ b/pkgs/substrate-apps/src/apps/link/test/ax-grpc.test.ts @@ -0,0 +1,91 @@ +// The production AxApi against a real gRPC server loaded from the vendored ax.proto (v0.3.0 d8ed0fe), on loopback. +// Proves the wire mapping the doubles skip: protojson field names survive UpdateTask, NotFound maps to undefined, +// DeleteTask NotFound is success. Round 2: the link loads its own P1 proto (apps/link/proto/ax-p1.proto), so the +// capability probe is decided by the SERVER: a stock server answers UNIMPLEMENTED over the wire (pre-P1), a P1 server +// answers; a proto without GetTaskResult is refused at construction instead of answering "unimplemented" locally. +import { createHash } from "node:crypto" +import * as grpc from "@grpc/grpc-js" +import * as protoLoader from "@grpc/proto-loader" +import { Effect } from "effect" +import { afterAll, beforeAll, expect, it } from "vitest" +import { resolveProtoPath, VENDORED_PROTO_PATH } from "../../../src/axclient.ts" +import { gatewayAllowsAll, grpcAx, LINK_PROTO_PATH, listAll } from "../src/ax.ts" +import { axTaskFromGrant } from "../src/jobs.ts" + +const stored = new Map() +const gateways = new Map([ + ["halogen", { apiVersion: "ax.io/v1alpha1", kind: "Gateway", metadata: { name: "halogen", atespace: "fleet" }, spec: { egress: { allowlist: { hosts: [{ host: "worker", port: 8731 }] } } } }], + ["open", { apiVersion: "ax.io/v1alpha1", kind: "Gateway", metadata: { name: "open", atespace: "fleet" }, spec: {} }] +]) +let server: grpc.Server, port = 0 +const nf = (name: string) => ({ code: grpc.status.NOT_FOUND, details: `task "${name}" not found` }) + +beforeAll(async () => { + const def = protoLoader.loadSync(resolveProtoPath(), { keepCase: false, longs: String, enums: String, defaults: true, oneofs: true }) + const svc = (grpc.loadPackageDefinition(def) as any).ax.v1alpha1.AX.service + server = new grpc.Server() + const impl: Record> = { + GetTask: (c, cb) => { const t = stored.get(c.request.name); t ? cb(null, t) : cb(nf(c.request.name)) }, + UpdateTask: (c, cb) => { stored.set(c.request.task.metadata.name, { ...c.request.task, status: { phase: "Pending" } }); cb(null, c.request.task) }, + // upstream server.go:91-104: limit <= 0 means 50, newest first (store.go ZRevRange) + ListTasks: (c, cb) => { const n = Number(c.request.limit) > 0 ? Number(c.request.limit) : 50, o = Number(c.request.offset) || 0; cb(null, { tasks: [...stored.values()].reverse().slice(o, o + n) }) }, + GetGateway: (c, cb) => { const g = gateways.get(c.request.name); g ? cb(null, g) : cb({ code: grpc.status.NOT_FOUND, details: "gateway not found" }) }, + DeleteTask: (c, cb) => stored.delete(c.request.name) ? cb(null, {}) : cb(nf(c.request.name)) + } + server.addService(svc, impl) + port = await new Promise((res, rej) => server.bindAsync("127.0.0.1:0", grpc.ServerCredentials.createInsecure(), (e, p) => e ? rej(e) : res(p))) +}) +afterAll(() => { server.forceShutdown() }) + +it("G1 grpcAx speaks ax.proto: create-only round trip, NotFound as undefined, a stock server answers GetTaskResult UNIMPLEMENTED", async () => { + const ax = grpcAx(`127.0.0.1:${port}`, "fleet", 3000) + const job = { + apiVersion: "ultracode.mecattaf.dev/v1alpha1" as const, kind: "AgentJob" as const, + metadata: { name: "wf-g-1-abababab", annotations: { "ultracode.mecattaf.dev/item-key": "wf_g#1", "ultracode.mecattaf.dev/journal-key": `${"cd".repeat(32)}:1` } }, + spec: { "runs-on": ["seat:halogen"], with: { prompt: "hi", prompt_ref: { sha256: "ab".repeat(32), bytes: 2, uri: "journal://wf_g/1/prompt.md" }, model: "halogen-qwen3.8-flash-next" } } + } + const built = axTaskFromGrant({ leaseId: "wf-g-1-abababab-a1", attempt: 1, job, lease: { holderIdentity: "nas-link-1", leaseDurationSeconds: 90, acquireTime: 0, renewTime: 0, leaseTransitions: 0 } }, + { atespace: "fleet", image: "localhost:5000/ax-agent@sha256:00", gateway: "halogen", command: () => ["ax-agent", "pi"] }) + if (built._tag !== "ok") throw new Error(built.reason) + const run = (e: Effect.Effect) => Effect.runPromise(e) + expect(await run(ax.getTask("wf-g-1-abababab-a1"))).toBeUndefined() + await run(ax.createTask(built.task)) + const back = stored.get("wf-g-1-abababab-a1") + expect([back.apiVersion, back.metadata.atespace, back.spec.gateway.name, back.spec.command]).toEqual(["ax.io/v1alpha1", "fleet", "halogen", ["ax-agent", "pi"]]) + expect(back.spec.env.find((e: any) => e.name === "AX_CONWIP_ITEM_KEY").value).toBe("wf_g#1") + const seen = await run(ax.getTask("wf-g-1-abababab-a1")) + expect(seen?.phase).toBe("Pending") + expect((await run(ax.listTasks(0, 0))).map((t) => t.name)).toEqual(["wf-g-1-abababab-a1"]) + for (let i = 0; i < 120; i++) stored.set(`bulk-${i}`, { apiVersion: "ax.io/v1alpha1", kind: "Task", metadata: { name: `bulk-${i}`, atespace: "fleet" }, spec: {}, status: { phase: "Completed" } }) + expect((await run(ax.listTasks(0, 0))).length).toBe(50) // B1: one call is one 50-row page + const all = await run(listAll(ax)) + expect([all.length, all.some((t) => t.name === "wf-g-1-abababab-a1")]).toEqual([121, true]) + for (let i = 0; i < 120; i++) stored.delete(`bulk-${i}`) + const gw = await run(ax.getGateway("halogen")) // B10 wire mapping + expect([gw?.hasAllowlist, gw?.hosts, gatewayAllowsAll(gw!)]).toEqual([true, [{ host: "worker", port: 8731 }], false]) + const open = await run(ax.getGateway("open")) + expect([open?.hasAllowlist, gatewayAllowsAll(open!)]).toEqual([false, true]) // no egress: ax applies *:443 + expect(await run(ax.getGateway("missing"))).toBeUndefined() + await run(ax.deleteTask("wf-g-1-abababab-a1")) + await run(ax.deleteTask("wf-g-1-abababab-a1")) // NotFound is success: delete is idempotent for the janitor + expect(await run(ax.getTaskResult("anything"))).toBe("unimplemented") // v0.3.0 server has no GetTaskResult: pre-P1 mode + ax.close() +}) + +it("R2-10 the packaged link (default proto) proves P1 against a P1 server and reads a result; a stock proto is refused", async () => { + let calls = 0 + const def = protoLoader.loadSync(LINK_PROTO_PATH, { keepCase: false, longs: String, enums: String, defaults: true, oneofs: true }) + const svc = (grpc.loadPackageDefinition(def) as any).ax.v1alpha1.AX.service + const p1 = new grpc.Server() + const body = Buffer.from('{"ok":true}') + p1.addService(svc, { + GetTaskResult: (c: any, cb: any) => { calls++; c.request.name === "t1" ? cb(null, { content: body, sha256: createHash("sha256").update(body).digest("hex") }) : cb({ code: grpc.status.NOT_FOUND, details: "nf" }) } + }) + const p1Port = await new Promise((res, rej) => p1.bindAsync("127.0.0.1:0", grpc.ServerCredentials.createInsecure(), (e, p) => e ? rej(e) : res(p))) + const ax = grpcAx(`127.0.0.1:${p1Port}`, "fleet", 3000) // exactly what main.ts builds when LINK_AX_PROTO_PATH is unset + expect(await Effect.runPromise(ax.getTaskResult("conwip-link-capability-probe"))).toBeUndefined() // NotFound: P1 present + expect(await Effect.runPromise(ax.getTaskResult("t1"))).toEqual({ content: '{"ok":true}', sha256: createHash("sha256").update(body).digest("hex"), digestOk: true }) + expect(calls).toBe(2) + ax.close(); p1.forceShutdown() + expect(() => grpcAx("127.0.0.1:1", "fleet", 1000, resolveProtoPath({ AX_CONWIP_PROTO_PATH: VENDORED_PROTO_PATH }))).toThrow(/GetTaskResult/) +}) diff --git a/pkgs/substrate-apps/src/apps/link/test/buildkite-drain.test.ts b/pkgs/substrate-apps/src/apps/link/test/buildkite-drain.test.ts new file mode 100644 index 00000000..2b304b27 --- /dev/null +++ b/pkgs/substrate-apps/src/apps/link/test/buildkite-drain.test.ts @@ -0,0 +1,96 @@ +// Buildkite-family critique pass (2026-09-24), G-BK4 on the link: a drain leases nothing more, lets the Tasks it +// holds finish and deliver their verdicts, and then ends by itself; a drain past its timeout ends anyway. +import { mkdtempSync, rmSync } from "node:fs" +import { tmpdir } from "node:os" +import { join } from "node:path" +import { Deferred, Effect, Fiber } from "effect" +import { afterEach, describe, expect, it } from "vitest" +import type { AgentJob } from "../src/contract.ts" +import { rpcFloor } from "../src/floor.ts" +import { Journal } from "../src/journal.ts" +import { runLink } from "../src/link.ts" +import type { LinkConfig } from "../src/link.ts" +import { FakeAx } from "./fake-ax.ts" +import { FakeFloor } from "./fake-floor.ts" + +const SEC = 20 +const A = "ultracode.mecattaf.dev/" +const JK = `${"cd".repeat(32)}:1` +const job = (n: string): AgentJob => ({ + apiVersion: "ultracode.mecattaf.dev/v1alpha1", kind: "AgentJob", + metadata: { + name: `wf-test-${n}`, + labels: { [A + "run-id"]: "wf-test", [A + "workflow"]: "link-test", [A + "phase-index"]: "1" }, + annotations: { [A + "run-id-raw"]: "wf_test", [A + "label"]: `probe:${n}`, [A + "item-key"]: `wf_test#${n}`, [A + "journal-key"]: JK, [A + "phase-title"]: "Probe" } + }, + spec: { + "runs-on": ["seat:halogen", "runtime:gvisor"], + with: { prompt: `say ${n}`, prompt_ref: { sha256: "ab".repeat(32), bytes: 5, uri: `journal://wf_test/${n}/prompt.md` }, model: "halogen-qwen3.8-flash-next" } + } +}) + +function start(floor: FakeFloor, ax: FakeAx, dir: string, drainTimeoutMs?: number) { + const logs: Array> = [] + const stop = Effect.runSync(Deferred.make()) + const drain = Effect.runSync(Deferred.make()) + const cfg: LinkConfig = { + holder: "nas-link-1", maxInFlight: 2, servedLabels: ["seat:halogen", "runtime:gvisor"], + shape: { atespace: "fleet", image: "ax-agent", gateway: "halogen", command: () => ["ax-agent", "pi"] }, + completion: "auto", secondMs: SEC, resyncMs: 30, pendingTimeoutMs: 1500, deleteAfterMs: 0, deadlineBackstopMs: 300, + createAttempts: 3, outboxBackoffMs: [20, 100], initialPollSeconds: 1, initialHeartbeatSeconds: 2 + } + const fiber = Effect.runFork(Effect.scoped(Effect.gen(function*() { + const f = yield* rpcFloor({ url: "http://floor.test", token: "tok-nas", sessionId: "s1", fetch: (i, init) => floor.fetch(i, init) }) + return yield* runLink(cfg, { ax, floor: f, journal: Journal.open(dir), log: (ev, fl) => logs.push({ ev, ...fl }), stop, drain, ...(drainTimeoutMs !== undefined ? { drainTimeoutMs } : {}) }) + }))) + let ended = false + void Effect.runPromise(Fiber.await(fiber)).then(() => { ended = true }) + return { + logs, ended: () => ended, + drain: () => Effect.runSync(Deferred.succeed(drain, undefined)), + stop: () => { Effect.runSync(Deferred.succeed(stop, undefined)); return Effect.runPromise(Fiber.await(fiber)) } + } +} +const until = async (what: string, pred: () => boolean, ms = 5000) => { + const t0 = Date.now() + while (!pred()) { if (Date.now() - t0 > ms) throw new Error(`timeout waiting for: ${what}`); await new Promise((r) => setTimeout(r, 5)) } +} +const sleep = (ms: number) => new Promise((r) => setTimeout(r, ms)) + +describe("G-BK4 link drain", () => { + const dirs: Array = [] + const mk = () => { const d = mkdtempSync(join(tmpdir(), "bk-drain-")); dirs.push(d); return d } + afterEach(() => { for (const d of dirs) rmSync(d, { recursive: true, force: true }); dirs.length = 0 }) + + it("leases nothing after the drain, delivers the held Task's verdict, then ends by itself", async () => { + const floor = new FakeFloor({ cap: 4, leaseSeconds: 60, graceSeconds: 60, pollSeconds: 1, heartbeatSeconds: 2, maxAttempts: 3, secondMs: SEC, tokens: { "tok-nas": "nas-link-1" } }) + const ax = new FakeAx() + floor.enqueue(job("1")) + const l = start(floor, ax, mk()) + await until("a1 created", () => ax.tasks.has("wf-test-1-a1")) + ax.tick() + l.drain() + floor.enqueue(job("2")) + await sleep(6 * SEC) + expect(ax.tasks.has("wf-test-2-a1")).toBe(false) // a draining link takes no new work + expect(l.ended()).toBe(false) // it still holds a live Task + ax.finish("wf-test-1-a1", 0, { text: "done" }) + await until("the link ends by itself", () => l.ended(), 8000) + expect(l.logs.some((x) => x.ev === "drain-complete")).toBe(true) + expect(floor.jobs.get("wf-test-1")!.state).toBe("done") + expect(floor.jobs.get("wf-test-2")!.state).toBe("queued") + }, 20000) + + it("a drain past its timeout ends with the Task still held (re-adopted by the next start)", async () => { + const floor = new FakeFloor({ cap: 4, leaseSeconds: 60, graceSeconds: 60, pollSeconds: 1, heartbeatSeconds: 2, maxAttempts: 3, secondMs: SEC, tokens: { "tok-nas": "nas-link-1" } }) + const ax = new FakeAx() + floor.enqueue(job("3")) + const l = start(floor, ax, mk(), 200) + await until("a1 created", () => ax.tasks.has("wf-test-3-a1")) + ax.tick() + l.drain() + await until("the link ends at the drain timeout", () => l.ended(), 8000) + expect(l.logs.some((x) => x.ev === "drain-timeout")).toBe(true) + expect(ax.live("wf-test-3-a1")).toBe(true) // never deleted: a restart re-adopts it + }, 20000) +}) diff --git a/pkgs/substrate-apps/src/apps/link/test/critique-double-run-r3.test.ts b/pkgs/substrate-apps/src/apps/link/test/critique-double-run-r3.test.ts new file mode 100644 index 00000000..75db8c1c --- /dev/null +++ b/pkgs/substrate-apps/src/apps/link/test/critique-double-run-r3.test.ts @@ -0,0 +1,119 @@ +// Critique pass 2026-09-24, red team double-run-r3-1: promoted from the audit probe audit-dr-r3.test.ts and inverted. Two links (two holders, two executors) serve the same labels, the topology round 1's r1-1 names. +// Unlike r1-1 there is no partition: link A is in contact with the floor throughout and itself tells the floor that +// attempt 1 never started (a pre-start/* verdict), while attempt 1's Task exists in A's ax. +import { mkdtempSync } from "node:fs" +import { tmpdir } from "node:os" +import { join } from "node:path" +import { Deferred, Effect, Fiber } from "effect" +import { afterEach, describe, expect, it } from "vitest" +import { AxError } from "../src/ax.ts" +import { rpcFloor } from "../src/floor.ts" +import { Journal } from "../src/journal.ts" +import { runLink } from "../src/link.ts" +import type { LinkConfig } from "../src/link.ts" +import { FakeAx } from "./fake-ax.ts" +import { FakeFloor } from "./fake-floor.ts" +import { job, sleep, SEC, until } from "./review-r4-harness.ts" + +const UNAVAILABLE = 14, DEADLINE_EXCEEDED = 4 + +/** ax whose UpdateTask fails without landing on the first tries and, on the last try, lands and then answers + * DEADLINE_EXCEEDED (the write committed, the reply was lost). After that the API is down (`up = false`) for a while: + * the realistic reason a create's reply is lost is an ax-server that is going away. The controller keeps running. */ +class LandsOnLastTry extends FakeAx { + creates = 0 + constructor(readonly lastTry: number) { + super() + const base = this.createTask // the parent's field, wrapped + this.createTask = (task): ReturnType => { + this.creates++ + if (this.creates < this.lastTry) return Effect.fail(new AxError(UNAVAILABLE, "connection reset")) + if (this.creates === this.lastTry) { + this.tasks.set(task.metadata.name, { task, phase: "Running", conditions: [] }) + this.events.push(`update:${task.metadata.name}`) + this.up = false + return Effect.fail(new AxError(DEADLINE_EXCEEDED, "deadline exceeded")) + } + return base(task) + } + } +} +/** ax whose DeleteTask fails (UNAVAILABLE) until `deleteOk`; everything else answers. */ +class DeleteFails extends FakeAx { + deleteOk = false + deleteTries = 0 + override deleteTask = (name: string) => { + this.deleteTries++ + if (!this.deleteOk) return Effect.fail(new AxError(UNAVAILABLE, "delete: connection reset")) + const t = this.tasks.get(name) + if (t) { t.phase = "Terminating"; this.deletes.push(name); this.events.push(`delete:${name}`) } + return Effect.void + } +} + +const running: Array<{ drain: () => Promise }> = [] +afterEach(async () => { for (const l of running.splice(0)) await l.drain() }) + +function startLink(floor: FakeFloor, ax: FakeAx, token: string, holder: string, over: Partial = {}) { + const dir = mkdtempSync(join(tmpdir(), `r3-dbl-${holder}-`)) + const logs: Array> = [] + const stop = Effect.runSync(Deferred.make()) + const cfg: LinkConfig = { + holder, maxInFlight: 1, servedLabels: ["seat:halogen", "runtime:gvisor"], + shape: { atespace: "fleet", image: "ax-agent", gateway: "halogen", command: () => ["ax-agent", "pi"] }, + completion: "auto", secondMs: SEC, resyncMs: 30, pendingTimeoutMs: 1500, deleteAfterMs: 0, deadlineBackstopMs: 300, + createAttempts: 3, outboxBackoffMs: [20, 100], initialPollSeconds: 1, initialHeartbeatSeconds: 2, fenceTimeoutMs: 300, resultReadTries: 3, ...over + } + const fiber = Effect.runFork(Effect.scoped(Effect.gen(function*() { + const f = yield* rpcFloor({ url: "http://floor.test", token, sessionId: `s:${dir}`, fetch: floor.fetch }) + return yield* runLink(cfg, { ax, floor: f, journal: Journal.open(dir), log: (ev, x) => logs.push({ t: Date.now(), ev, ...x }), stop, floorUrl: "http://floor.test" }) + }))) + const l = { + logs, + has: (ev: string, leaseId?: string) => logs.some((x) => x.ev === ev && (leaseId === undefined || x.leaseId === leaseId)), + drain: () => { Effect.runSync(Deferred.succeed(stop, undefined)); return Promise.race([Effect.runPromise(Fiber.await(fiber)), sleep(1500)]) } + } + running.push(l) + return l +} + +describe("double-run r3-1 (fixed): no retryable pre-start verdict for a Task that may exist", () => { + it("R3-1 create lands on the last try: A waits (create-unconfirmed), B never gets an attempt, A adopts a1 once ax answers", async () => { + const floor = new FakeFloor({ cap: 2, leaseSeconds: 6, graceSeconds: 15, pollSeconds: 1, heartbeatSeconds: 2, maxAttempts: 3, secondMs: SEC, + tokens: { "tok-a": "nas-link-a", "tok-b": "nas-link-b" } }) + const axA = new LandsOnLastTry(3), axB = new FakeAx() + floor.enqueue(job("1")) + const a = startLink(floor, axA, "tok-a", "nas-link-a") + await until("A cannot confirm its create", () => a.has("create-unconfirmed", "wf-test-1-a1"), 8000) + expect(axA.live("wf-test-1-a1")).toBe(true) + const b = startLink(floor, axB, "tok-b", "nas-link-b") + await sleep(40 * SEC) + expect(floor.jobs.get("wf-test-1")!.history.some((h) => h.startsWith("requeue:pre-start"))).toBe(false) + expect([...axB.tasks.keys()].some((n) => n.startsWith("wf-test-1-a"))).toBe(false) + expect(b.has("leased")).toBe(false) + axA.up = true + await until("A adopts a1 once ax answers", () => a.has("adopted", "wf-test-1-a1"), 5000) + expect(axA.deletes).not.toContain("wf-test-1-a1") + floor.close() + }, 30_000) + + it("R3-1b pending timeout with a DeleteTask that did not land: no verdict until the delete lands; B never runs a2 beside a1", async () => { + const floor = new FakeFloor({ cap: 2, leaseSeconds: 6, graceSeconds: 15, pollSeconds: 1, heartbeatSeconds: 2, maxAttempts: 3, secondMs: SEC, + tokens: { "tok-a": "nas-link-a", "tok-b": "nas-link-b" } }) + const axA = new DeleteFails(), axB = new FakeAx() + axA.holdPending.add("wf-test-1-a1") + floor.enqueue(job("1")) + const a = startLink(floor, axA, "tok-a", "nas-link-a", { pendingTimeoutMs: 200 }) + await until("A defers its pre-start verdict", () => a.has("pre-start-deferred", "wf-test-1-a1"), 8000) + expect(floor.jobs.get("wf-test-1")!.history.includes("requeue:pre-start/pending-timeout")).toBe(false) + axA.holdPending.delete("wf-test-1-a1"); axA.tick() // a1 starts after all + const b = startLink(floor, axB, "tok-b", "nas-link-b") + await sleep(20 * SEC) + expect(axB.live("wf-test-1-a2")).toBe(false) + // the delete lands: a1 goes, and only then is the retryable verdict sent + axA.deleteOk = true + await until("the floor hears pre-start/pending-timeout after a1 is gone", () => floor.jobs.get("wf-test-1")!.history.includes("requeue:pre-start/pending-timeout"), 8000) + expect(axA.live("wf-test-1-a1")).toBe(false) + floor.close() + }, 30_000) +}) diff --git a/pkgs/substrate-apps/src/apps/link/test/critique-durability-outage.test.ts b/pkgs/substrate-apps/src/apps/link/test/critique-durability-outage.test.ts new file mode 100644 index 00000000..b984f0d3 --- /dev/null +++ b/pkgs/substrate-apps/src/apps/link/test/critique-durability-outage.test.ts @@ -0,0 +1,151 @@ +// Critique pass 2026-09-24, red team durability-r2-8: a floor-wide 500 outage never turns a success into the final +// infra/verdict-undeliverable. Promoted from the audit probe audit-d-r2b-outage.test.ts and inverted; the harness is +// the review-r2 one. +import { mkdtempSync, rmSync } from "node:fs" +import { tmpdir } from "node:os" +import { join } from "node:path" +import { Deferred, Effect, Fiber } from "effect" +import { afterEach, beforeEach, describe, expect, it } from "vitest" +import { cidrIsOpen, gatewayAllowsAll } from "../src/ax.ts" +import type { AgentJob } from "../src/contract.ts" +import { rpcFloor } from "../src/floor.ts" +import { Journal } from "../src/journal.ts" +import { isFleetInternalUrl, runLink } from "../src/link.ts" +import type { LinkConfig } from "../src/link.ts" +import { FakeAx } from "./fake-ax.ts" +import { FakeFloor } from "./fake-floor.ts" +import type { FloorConfig } from "./fake-floor.ts" + +const SEC = 20 +const A = "ultracode.mecattaf.dev/" +const JK = `${"cd".repeat(32)}:1` // round 4: FIELD-MAP 5a journal key (jobs.ts refuses a grant without it) +const job = (n: string): AgentJob => ({ + apiVersion: "ultracode.mecattaf.dev/v1alpha1", kind: "AgentJob", + metadata: { name: `wf-test-${n}`, labels: { [A + "run-id"]: "wf-test", [A + "workflow"]: "link-test", [A + "phase-index"]: "1" }, + annotations: { [A + "run-id-raw"]: "wf_test", [A + "label"]: `probe:${n}`, [A + "item-key"]: `wf_test#${n}`, [A + "journal-key"]: JK, [A + "phase-title"]: "Probe" } }, + spec: { "runs-on": ["seat:halogen", "runtime:gvisor"], + with: { prompt: `say ${n}`, prompt_ref: { sha256: "ab".repeat(32), bytes: 5, uri: `journal://wf_test/${n}/prompt.md` }, model: "halogen-qwen3.8-flash-next" } } +}) +const world = (o: Partial = {}) => ({ + floor: new FakeFloor({ cap: 2, leaseSeconds: 6, graceSeconds: 15, pollSeconds: 1, heartbeatSeconds: 2, maxAttempts: 3, secondMs: SEC, tokens: { "tok-nas": "nas-link-1" }, ...o }), + ax: new FakeAx() +}) +type World = ReturnType +function start(w: World, dir: string, over: Partial = {}, fetch?: typeof globalThis.fetch) { + const logs: Array> = [] + const stop = Effect.runSync(Deferred.make()) + const cfg: LinkConfig = { + holder: "nas-link-1", maxInFlight: 2, servedLabels: ["seat:halogen", "runtime:gvisor"], + shape: { atespace: "fleet", image: "ax-agent", gateway: "halogen", command: () => ["ax-agent", "pi"] }, + completion: "auto", secondMs: SEC, resyncMs: 30, pendingTimeoutMs: 1500, deleteAfterMs: 0, deadlineBackstopMs: 300, + createAttempts: 3, outboxBackoffMs: [20, 100], initialPollSeconds: 1, initialHeartbeatSeconds: 2, fenceTimeoutMs: 300, resultReadTries: 3, ...over + } + const fiber = Effect.runFork(Effect.scoped(Effect.gen(function*() { + const floor = yield* rpcFloor({ url: "http://floor.test", token: "tok-nas", sessionId: `s:${dir}`, fetch: fetch ?? w.floor.fetch }) + return yield* runLink(cfg, { ax: w.ax, floor, journal: Journal.open(dir), log: (ev, f) => logs.push({ t: Date.now(), ev, ...f }), stop, floorUrl: "http://floor.test" }) + }))) + return { + logs, fiber, + has: (ev: string, leaseId?: string) => logs.some((l) => l.ev === ev && (leaseId === undefined || l.leaseId === leaseId)), + trace: () => logs.filter((x) => typeof x.leaseId === "string" || typeof x.task === "string").map((x) => `${x.ev}:${x.leaseId ?? x.task}${x.why ? ":" + x.why : ""}`), + crash: () => Effect.runPromise(Fiber.interrupt(fiber)), + drain: () => { Effect.runSync(Deferred.succeed(stop, undefined)); return Promise.race([Effect.runPromise(Fiber.await(fiber)), sleep(1500)]) } + } +} +const until = async (what: string, pred: () => boolean, ms = 5000) => { + const t0 = Date.now() + while (!pred()) { if (Date.now() - t0 > ms) throw new Error(`timeout waiting for: ${what}`); await new Promise((r) => setTimeout(r, 5)) } +} +const sleep = (ms: number) => new Promise((r) => setTimeout(r, ms)) +const J = (w: World, n = "1") => w.floor.jobs.get(`wf-test-${n}`)! +const bodyOf = async (input: unknown, init?: RequestInit) => { + const b = init?.body + if (typeof b === "string") return b + if (b instanceof Uint8Array) return new TextDecoder().decode(b) + if (b) return await new Response(b as ConstructorParameters[0]).text() + return (input instanceof Request) ? await input.clone().text() : "" +} +const rpcOf = async (input: unknown, init?: RequestInit) => { const b = await bodyOf(input, init); return b.includes('"Complete"') ? "Complete" : b.includes('"Lease"') ? "Lease" : b.includes('"Heartbeat"') ? "Heartbeat" : "?" } +/** A capacity race: Failed with ResourceExhausted, which the link maps to retryable pre-start/resource-exhausted. */ +const failRetryable = (w: World, name: string) => { + const t = w.ax.tasks.get(name)! + t.exited = true; t.phase = "Failed" + t.conditions = [{ type: "Ready", status: "False", reason: "ResourceExhausted", message: "ResourceExhausted: no free worker" }] +} +const why = (w: World, l: ReturnType) => JSON.stringify({ floor: J(w).history, link: l.trace() }) + +// A floor stub for the fail-closed tests (scratch-r2-1/repro.mts): hands out its grants once, renews everything. +const stats = { seq: 1, cap: 2, wip: 0, queued: 0, done: 0 } +const grantOf = (n: string, extra: Record = {}) => ({ leaseId: `wf-test-${n}-a1`, attempt: 1, job: job(n), + lease: { holderIdentity: "nas-link-1", leaseDurationSeconds: 30, acquireTime: Date.now(), renewTime: Date.now(), leaseTransitions: 0 }, ...extra }) +const stubFloor = (grants: Array, gate: () => boolean = () => true) => { + let sent = false + return { + lease: (p: any) => Effect.sync(() => { const g = !sent && p.capacity > 0 && gate() ? grants : []; if (g.length) sent = true; return { grants: g, invalid: [], stats, nextPollSeconds: 1, heartbeatSeconds: 2 } }), + heartbeat: (p: any) => Effect.succeed({ renewed: [...p.leaseIds], lost: [], cancelRequested: [], stats }), + complete: (_p: any) => Effect.succeed({ duplicate: false, stats }) + } +} +const runStub = async (over: Partial, ax: FakeAx, floor: any, ms: number) => { + const dir = mkdtempSync(join(tmpdir(), "conwip-link-r2s-")) + const logs: Array> = [] + const stop = Effect.runSync(Deferred.make()) + const cfg: LinkConfig = { holder: "nas-link-1", maxInFlight: 2, servedLabels: ["seat:halogen", "runtime:gvisor"], + shape: { atespace: "fleet", image: "ax-agent", gateway: "halogen", command: () => ["ax-agent", "pi"] }, + completion: "auto", secondMs: SEC, resyncMs: 30, pendingTimeoutMs: 60_000, deleteAfterMs: 60_000, deadlineBackstopMs: 300, + createAttempts: 3, outboxBackoffMs: [20, 100], initialPollSeconds: 1, initialHeartbeatSeconds: 2, ...over } + const f = Effect.runFork(runLink(cfg, { ax, floor, journal: Journal.open(dir), log: (ev, x) => logs.push({ t: Date.now(), ev, ...x }), stop })) + await sleep(ms) + Effect.runSync(Deferred.succeed(stop, undefined)); await Effect.runPromise(Fiber.await(f)); rmSync(dir, { recursive: true, force: true }) + return logs +} + +describe("durability-r2-8 (fixed): a floor-wide 500 outage", () => { + let dir: string + let worlds: Array = [] + const W = (o: Partial = {}) => { const w = world(o); worlds.push(w); return w } + beforeEach(() => { dir = mkdtempSync(join(tmpdir(), "rt-r2b-")) }) + afterEach(() => { for (const w of worlds) w.floor.close(); worlds = []; rmSync(dir, { recursive: true, force: true }) }) + + it("every RPC answers 500 for a while: the success verdict waits it out and is delivered with its output", async () => { + const w = W({ cap: 4 }); w.floor.enqueue(job("1")) + let outage = false, refusedComplete = 0, refusedOther = 0 + const fetch: typeof globalThis.fetch = async (input, init) => { + if (outage) { + const r = await rpcOf(input, init) + if (r === "Complete") refusedComplete++; else refusedOther++ + return new Response("error code: 1101", { status: 500 }) + } + return w.floor.fetch(input, init) + } + const l = start(w, dir, { outboxBackoffMs: [20, 60], verdictAttempts: 4 }, fetch) + await until("a1 running", () => w.ax.tasks.get("wf-test-1-a1")?.phase === "Running") + outage = true + w.ax.finish("wf-test-1-a1", 0, { answer: 42 }) + await until("verdict written", () => l.has("verdict", "wf-test-1-a1")) + await until("Complete refused well past verdictAttempts", () => refusedComplete >= 12, 8000) + expect(l.has("verdict-replaced", "wf-test-1-a1")).toBe(false) + expect(refusedOther).toBeGreaterThan(0) // Lease and Heartbeat were refused too: the outage was floor-wide + outage = false + await until("job 1 closed", () => J(w).state === "done", 4000) + expect(J(w).result).toBe("success") + expect(JSON.stringify(J(w).output)).toContain("42") + await l.drain() + }) + + it("a verdict the floor alone refuses while Lease and Heartbeat succeed is still replaced at its ceiling", async () => { + const w = W({ cap: 4 }); w.floor.enqueue(job("1")) + let refuse = false + const fetch: typeof globalThis.fetch = async (input, init) => + refuse && (await rpcOf(input, init)) === "Complete" ? new Response("SQLITE_TOOBIG", { status: 500 }) : w.floor.fetch(input, init) + const l = start(w, dir, { outboxBackoffMs: [20, 60], verdictAttempts: 3 }, fetch) + await until("a1 running", () => w.ax.tasks.get("wf-test-1-a1")?.phase === "Running") + refuse = true + w.ax.finish("wf-test-1-a1", 0, { answer: 42 }) + await until("replaced", () => l.has("verdict-replaced", "wf-test-1-a1"), 8000) + refuse = false + await until("job 1 closed", () => J(w).state === "done", 4000) + expect([J(w).result, (J(w).output as any)?.reason]).toEqual(["failure", "infra/verdict-undeliverable"]) + await l.drain() + }) +}) diff --git a/pkgs/substrate-apps/src/apps/link/test/critique-fixed.test.ts b/pkgs/substrate-apps/src/apps/link/test/critique-fixed.test.ts new file mode 100644 index 00000000..7abf733e --- /dev/null +++ b/pkgs/substrate-apps/src/apps/link/test/critique-fixed.test.ts @@ -0,0 +1,406 @@ +// The nine defects the 2026-09-23 critique reproduced (LINK-DESIGN.md "Critique applied", section B), each INVERTED +// into the fixed behaviour now that its build item is in (commit history: the characterization versions are in +// 6342476 and b900624). Plus the new cases the build list names for B1, B6, B9 to B13, B15, B17 and B18. +// Same doubles as link.test.ts; the floor double implements section H's amended rules (B16). +import { mkdtempSync, rmSync } from "node:fs" +import { tmpdir } from "node:os" +import { join } from "node:path" +import { Deferred, Effect, Fiber } from "effect" +import { afterEach, beforeEach, describe, expect, it } from "vitest" +import { AxError } from "../src/ax.ts" +import type { AgentJob, AxTask } from "../src/contract.ts" +import { rpcFloor } from "../src/floor.ts" +import type { FloorOptions } from "../src/floor.ts" +import { Journal } from "../src/journal.ts" +import { isFleetInternalUrl, runLink } from "../src/link.ts" +import type { LinkConfig } from "../src/link.ts" +import { FakeAx } from "./fake-ax.ts" +import { FakeFloor } from "./fake-floor.ts" +import type { FloorConfig } from "./fake-floor.ts" + +const SEC = 20 +const UNAVAILABLE = 14 +const A = "ultracode.mecattaf.dev/" +const JK = `${"cd".repeat(32)}:1` // round 4: FIELD-MAP 5a journal key (jobs.ts refuses a grant without it) +const job = (n: string, spec: Partial = {}): AgentJob => ({ + apiVersion: "ultracode.mecattaf.dev/v1alpha1", kind: "AgentJob", + metadata: { + name: `wf-test-${n}`, + labels: { [A + "run-id"]: "wf-test", [A + "workflow"]: "link-test", [A + "phase-index"]: "1" }, + annotations: { [A + "run-id-raw"]: "wf_test", [A + "label"]: `probe:${n}`, [A + "item-key"]: `wf_test#${n}`, [A + "journal-key"]: JK, [A + "phase-title"]: "Probe" } + }, + spec: { + "runs-on": ["seat:halogen", "runtime:gvisor"], + with: { prompt: `say ${n}`, prompt_ref: { sha256: "ab".repeat(32), bytes: 5, uri: `journal://wf_test/${n}/prompt.md` }, model: "halogen-qwen3.8-flash-next" }, + ...spec + } +}) +const world = (o: Partial = {}) => ({ + floor: new FakeFloor({ cap: 2, leaseSeconds: 6, graceSeconds: 15, pollSeconds: 1, heartbeatSeconds: 2, maxAttempts: 3, secondMs: SEC, tokens: { "tok-nas": "nas-link-1" }, ...o }), + ax: new FakeAx() +}) +type World = ReturnType +const foreign = (name: string): AxTask => ({ apiVersion: "ax.io/v1alpha1", kind: "Task", metadata: { name, atespace: "fleet" }, spec: { image: "someone-else", command: ["true"], env: [] } }) + +interface Extra { fetch?: typeof globalThis.fetch; timeoutsMs?: FloorOptions["timeoutsMs"]; onEndpoint?: (u: string) => void } +function start(w: World, dir: string, over: Partial = {}, x: Extra = {}) { + const logs: Array> = [] + const stop = Effect.runSync(Deferred.make()) + const cfg: LinkConfig = { + holder: "nas-link-1", maxInFlight: 2, servedLabels: ["seat:halogen", "runtime:gvisor"], + shape: { atespace: "fleet", image: "ax-agent", gateway: "halogen", command: () => ["ax-agent", "pi"] }, + completion: "auto", secondMs: SEC, resyncMs: 30, pendingTimeoutMs: 1500, deleteAfterMs: 0, deadlineBackstopMs: 300, + createAttempts: 3, outboxBackoffMs: [20, 100], initialPollSeconds: 1, initialHeartbeatSeconds: 2, fenceTimeoutMs: 300, resultReadTries: 3, ...over + } + const fiber = Effect.runFork(Effect.scoped(Effect.gen(function*() { + const floor = yield* rpcFloor({ url: "http://floor.test", token: "tok-nas", sessionId: `s:${dir}`, fetch: x.fetch ?? w.floor.fetch, ...(x.timeoutsMs ? { timeoutsMs: x.timeoutsMs } : {}) }) + return yield* runLink(cfg, { ax: w.ax, floor, journal: Journal.open(dir), log: (ev, f) => logs.push({ ev, ...f }), stop, floorUrl: "http://floor.test", ...(x.onEndpoint ? { onEndpoint: x.onEndpoint } : {}) }) + }))) + return { + logs, + has: (ev: string, leaseId?: string) => logs.some((l) => l.ev === ev && (leaseId === undefined || l.leaseId === leaseId)), + verdict: (reason: string) => logs.some((l) => l.ev === "verdict" && l.reason === reason), + crash: () => Effect.runPromise(Fiber.interrupt(fiber)), + drain: () => { Effect.runSync(Deferred.succeed(stop, undefined)); return Promise.race([Effect.runPromise(Fiber.await(fiber)), sleep(1500)]) } + } +} +const until = async (what: string, pred: () => boolean, ms = 5000) => { + const t0 = Date.now() + while (!pred()) { if (Date.now() - t0 > ms) throw new Error(`timeout waiting for: ${what}`); await new Promise((r) => setTimeout(r, 5)) } +} +const sleep = (ms: number) => new Promise((r) => setTimeout(r, ms)) +const J = (w: World, n = "1") => w.floor.jobs.get(`wf-test-${n}`)! +const liveOwn = (w: World) => [...w.ax.tasks.keys()].filter((k) => k.startsWith("wf-test-") && w.ax.live(k) && !["Completed", "Failed"].includes(w.ax.tasks.get(k)!.phase)).length + +describe("critique 2026-09-23: the defects, fixed", () => { + let dir: string + let worlds: Array = [] + const W = (o: Partial = {}) => { const w = world(o); worlds.push(w); return w } + beforeEach(() => { dir = mkdtempSync(join(tmpdir(), "conwip-link-critique-")) }) + afterEach(() => { for (const w of worlds) w.floor.close(); worlds = []; rmSync(dir, { recursive: true, force: true }) }) + + it("C1 (B1) a live Task off ListTasks' first 50-row page is neither lost nor rerun", async () => { + const w = W(); w.floor.enqueue(job("1")) + const l = start(w, dir) + await until("running", () => w.ax.tasks.get("wf-test-1-a1")?.phase === "Running") + for (let i = 0; i < 50; i++) w.ax.put(foreign(`foreign-${i}`), "Completed") // newer than ours: ours is on page 2 + const calls = w.ax.listCalls + await until("ten resyncs", () => w.ax.listCalls >= calls + 10) + expect([l.verdict("infra/task-lost"), l.has("task-absent"), w.ax.tasks.has("wf-test-1-a2"), w.ax.deletes]).toEqual([false, false, false, []]) + w.ax.finish("wf-test-1-a1", 0, { answer: 42 }) + await until("floor done", () => J(w).state === "done") + expect([J(w).result, J(w).output, J(w).attempt]).toEqual(["success", { answer: 42 }, 1]) + await l.drain() + }) + + it("C1b (B1) 120 Tasks: occupancy counts a live foreign Task on page 3, and a truly gone Task needs two NotFound resyncs", async () => { + const w = W() + for (let i = 0; i < 2; i++) w.ax.put(foreign(`busy-${i}`), "Running") // oldest: on the last page + for (let i = 0; i < 118; i++) w.ax.put(foreign(`done-${i}`), "Completed") + w.floor.enqueue(job("1")) + const l = start(w, dir) + await until("a Lease was sent", () => w.floor.calls.some((c) => c.rpc === "Lease")) + await sleep(10 * SEC) + expect(w.floor.calls.filter((c) => c.rpc === "Lease").every((c) => c.capacity === 0)).toBe(true) // 2 busy of 2 + w.ax.tasks.delete("busy-0"); w.ax.tasks.delete("busy-1") + await until("created once the sandboxes free", () => w.ax.tasks.get("wf-test-1-a1")?.phase === "Running") + w.ax.tasks.delete("wf-test-1-a1") // the store forgets it (F19) + await until("task-lost after two NotFound resyncs", () => l.verdict("infra/task-lost")) + const absent = l.logs.filter((x) => x.ev === "task-absent" && x.leaseId === "wf-test-1-a1") + expect(absent.length).toBe(1) + await until("attempt 2 created", () => w.ax.tasks.has("wf-test-1-a2")) + await l.drain() + }) + + it("C2 (B2) a superseded attempt that will not delete blocks attempt n+1: retryable pre-start/ax-unavailable, never two at once", async () => { + const w = W(); w.floor.enqueue(job("1")); w.ax.holdPending.add("wf-test-1-a1") + const del = w.ax.deleteTask + w.ax.deleteTask = (name) => name === "wf-test-1-a1" ? Effect.fail(new AxError(UNAVAILABLE, "unavailable")) : del(name) + const l = start(w, dir, { pendingTimeoutMs: 200 }) + // Critique pass 2026-09-24 (red team double-run-r3-1b): a1's pending-timeout verdict now waits for its delete to + // land, so the floor never makes attempt 2 while a1 may still start. Never two at once still holds, one step earlier. + await until("a1's verdict deferred", () => l.has("pre-start-deferred", "wf-test-1-a1"), 8000) + await sleep(1000) + expect(J(w).attempt).toBe(1) + expect(w.ax.updates.has("wf-test-1-a2")).toBe(false) + expect([...w.ax.updates.keys()]).toEqual(["wf-test-1-a1"]) // only ever one attempt in ax + await l.drain() + }) + + it("C3 (B3) a cancel that lands while UpdateTask is retrying stops the create; nothing runs on", async () => { + const w = W(); w.floor.enqueue(job("1")) + let refusals = 2 + const create = w.ax.createTask + w.ax.createTask = (t) => refusals-- > 0 ? Effect.fail(new AxError(UNAVAILABLE, "unavailable")) : create(t) + const l = start(w, dir, { deleteAfterMs: 60_000, createAttempts: 5 }) + await until("leased", () => l.has("leased", "wf-test-1-a1")) + w.floor.cancel("wf-test-1", "tom") + await until("floor done", () => J(w).state === "done") + expect(J(w).result).toBe("cancelled") + await sleep(600) + expect([w.ax.live("wf-test-1-a1"), l.has("abandoned-before-create", "wf-test-1-a1")]).toEqual([false, true]) + await l.drain() + }) + + it("C3b (B3) a cancel that lands while UpdateTask is in flight deletes the Task as soon as it returns, then reports cancelled", async () => { + const w = W(); w.floor.enqueue(job("1")) + let gate!: () => void + const create = w.ax.createTask + w.ax.createTask = (t) => create(t).pipe(Effect.andThen(Effect.callback((resume) => { gate = () => resume(Effect.void) }))) + const l = start(w, dir, { deleteAfterMs: 60_000 }) + await until("create in flight", () => w.ax.tasks.has("wf-test-1-a1") && gate !== undefined) + w.floor.cancel("wf-test-1", "tom") + await until("cancel seen", () => l.has("cancel", "wf-test-1-a1")) + gate() + await until("floor done", () => J(w).state === "done") + expect([J(w).result, w.ax.live("wf-test-1-a1")]).toEqual(["cancelled", false]) + await l.drain() + }) + + it("C4 (B4) one transient GetTaskResult error is retried at the next resync: the success lands, nothing reruns", async () => { + const w = W(); w.floor.enqueue(job("1")) + let blips = 1 + const res = w.ax.getTaskResult + w.ax.getTaskResult = (n) => n === "wf-test-1-a1" && blips-- > 0 ? Effect.fail(new AxError(UNAVAILABLE, "unavailable")) : res(n) + const l = start(w, dir) + await until("running", () => w.ax.tasks.get("wf-test-1-a1")?.phase === "Running") + w.ax.finish("wf-test-1-a1", 0, { answer: 42 }) + await until("floor done", () => J(w).state === "done") + expect([J(w).result, J(w).output, J(w).attempt, l.has("result-read-retry", "wf-test-1-a1")]).toEqual(["success", { answer: 42 }, 1, true]) + expect(J(w).history.some((h) => h.startsWith("requeue"))).toBe(false) + await l.drain() + }) + + it("C4b (B4) a result whose sha256 never matches is infra/result-unreadable after the bounded tries", async () => { + const w = W(); w.floor.enqueue(job("1")) + const res = w.ax.getTaskResult + w.ax.getTaskResult = (n) => res(n).pipe(Effect.map((r) => r && r !== "unimplemented" && n === "wf-test-1-a1" ? { ...r, digestOk: false } : r)) + const l = start(w, dir) + await until("running", () => w.ax.tasks.get("wf-test-1-a1")?.phase === "Running") + w.ax.finish("wf-test-1-a1", 0, { answer: 42 }) + await until("attempt 2 created", () => w.ax.tasks.has("wf-test-1-a2")) + expect(J(w).history).toContain("requeue:infra/result-unreadable") + expect(l.logs.filter((x) => x.ev === "result-read-retry" && x.leaseId === "wf-test-1-a1").length).toBe(2) + await l.drain() + }) + + it("C5 (B5) Failed with ResourceExhausted is the retryable pre-start/resource-exhausted", async () => { + const w = W(); w.floor.enqueue(job("1")) + const l = start(w, dir) + await until("running", () => w.ax.tasks.get("wf-test-1-a1")?.phase === "Running") + const t = w.ax.tasks.get("wf-test-1-a1")! + t.phase = "Failed" + t.conditions = [{ type: "Ready", status: "False", reason: "ActorCreateFailed", message: "ResourceExhausted: no free workers available" }] + await until("attempt 2 created", () => w.ax.tasks.has("wf-test-1-a2")) + expect(J(w).history).toContain("requeue:pre-start/resource-exhausted") + await l.drain() + }) + + it("C6 (B6) configured p1 on a server without P1 leases nothing and says p1-missing", async () => { + const w = W(); w.floor.enqueue(job("1")); w.ax.p1 = false + const l = start(w, dir, { completion: "p1" }) + await until("probe", () => l.has("p1-missing")) + await sleep(10 * SEC) + expect(w.floor.calls.filter((c) => c.rpc === "Lease").every((c) => c.capacity === 0)).toBe(true) + expect([w.ax.updates.size, J(w).state]).toEqual([0, "queued"]) + await l.drain() + }) + + it("C6b (B6) auto never selects guest, and re-probes when ax comes back: P1 appears, work starts", async () => { + const w = W(); w.floor.enqueue(job("1")); w.ax.p1 = false + const l = start(w, dir, { completion: "auto" }) + await until("probe", () => l.has("p1-missing")) + await sleep(5 * SEC) + expect(w.ax.updates.size).toBe(0) + w.ax.up = false + await until("ax down", () => l.has("ax-down")) + w.ax.p1 = true; w.ax.up = true // the operator rolled out P1 + await until("created in p1 mode", () => w.ax.tasks.has("wf-test-1-a1")) + const env = Object.fromEntries(w.ax.tasks.get("wf-test-1-a1")!.task.spec.env.map((e) => [e.name, e.value])) + expect(env.AX_CONWIP_COMPLETE_URL).toBeUndefined() + await l.drain() + }) + + it("C6c (B6, A5) guest mode with a workers.dev Complete URL leases nothing", async () => { + expect([isFleetInternalUrl("https://conwip-floor.x.workers.dev/guest"), isFleetInternalUrl("http://link.fleet.internal/guest"), isFleetInternalUrl("http://10.201.0.9:8732/g"), isFleetInternalUrl("https://example.com/")]).toEqual([false, true, true, false]) + const w = W(); w.floor.enqueue(job("1")); w.ax.p1 = false + const l = start(w, dir, { completion: "guest", shape: { atespace: "fleet", image: "ax-agent", gateway: "halogen", command: () => ["ax-agent", "pi"], completeUrl: "https://conwip-floor.x.workers.dev/guest" } }) + await until("refused", () => l.has("guest-url-invalid")) + await sleep(5 * SEC) + expect(w.floor.calls.filter((c) => c.rpc === "Lease").every((c) => c.capacity === 0)).toBe(true) + await l.drain() + }) + + it("C7 (B7) an empty runs-on, or two seats, is refused by the link (and by the floor's enqueue)", async () => { + const w = W({ cap: 3 }) + expect(w.floor.enqueue(job("1", { "runs-on": [] }))).toBe(false) + w.floor.enqueueUnchecked(job("1", { "runs-on": [] })) + w.floor.o.tokens["tok-nas"] = { holder: "nas-link-1", labels: ["seat:halogen", "seat:cc", "runtime:gvisor"] } + w.floor.enqueueUnchecked(job("2", { "runs-on": ["seat:halogen", "seat:cc"] })) + const l = start(w, dir, { maxInFlight: 3, servedLabels: ["seat:halogen", "seat:cc", "runtime:gvisor"] }) + await until("both done", () => J(w).state === "done" && J(w, "2").state === "done") + expect([(J(w).output as { reason: string }).reason, (J(w, "2").output as { reason: string }).reason]).toEqual(["pre-start/runs-on-invalid", "pre-start/runs-on-invalid"]) + expect(w.ax.updates.size).toBe(0) + await l.drain() + }) + + it("C8 (B8) a Lease reply lost on all three tries is replayed by the journaled requestKey: attempt 1 runs", async () => { + const w = W(); w.floor.enqueue(job("1")) + let lose = 3 + const lossy: typeof globalThis.fetch = async (input, init) => { + const res = await w.floor.fetch(input, init) + if (lose > 0 && (await res.clone().text()).includes("\"leaseId\":\"wf-test-1-a1\"")) { lose--; throw new TypeError("fetch failed: reply lost") } + return res + } + const l = start(w, dir, {}, { fetch: lossy }) + await until("attempt 1 created", () => w.ax.tasks.has("wf-test-1-a1"), 8000) + expect(J(w).history.some((h) => h.startsWith("requeue"))).toBe(false) + expect(w.ax.tasks.has("wf-test-1-a2")).toBe(false) + await l.drain() + }) + + it("C9 (B9) a hung Heartbeat times out; the next one renews; the running attempt survives", async () => { + const w = W(); w.floor.enqueue(job("1")) + let hang = false + const hanging: typeof globalThis.fetch = async (input, init) => { + const req = new Request(input as string | URL | Request, init) + if (hang && (await req.clone().text()).includes("\"Heartbeat\"")) { hang = false; return new Promise(() => {}) } + return w.floor.fetch(req) + } + const l = start(w, dir, {}, { fetch: hanging, timeoutsMs: { lease: 150, heartbeat: 150, complete: 300 } }) + await until("running", () => w.ax.tasks.get("wf-test-1-a1")?.phase === "Running") + hang = true + await until("the hang timed out", () => l.logs.some((x) => x.ev === "heartbeat-error")) + await sleep(25 * SEC) // past lease plus grace + expect([J(w).history.includes("requeue:grace"), w.ax.deletes.includes("wf-test-1-a1"), J(w).attempt]).toEqual([false, false, 1]) + w.ax.finish("wf-test-1-a1", 0, { ok: 1 }) + await until("floor done", () => J(w).state === "done") + expect([J(w).result, J(w).attempt]).toEqual(["success", 1]) + await l.drain() + }) + + it("C9b (B9) concurrent 401 and 503 are each charged to their own call", async () => { + const w = W() + const f: typeof globalThis.fetch = async (input, init) => { + const body = await new Request(input as string | URL | Request, init).text() + if (body.includes("\"Lease\"")) { await sleep(30); return new Response("busy", { status: 503 }) } + return new Response("no", { status: 401 }) + } + const out = await Effect.runPromise(Effect.scoped(Effect.gen(function*() { + const floor = yield* rpcFloor({ url: "http://floor.test", token: "tok-nas", sessionId: "s", fetch: f }) + return yield* Effect.all([ + floor.lease({ holderIdentity: "nas-link-1", capacity: 1, requestKey: "k" }).pipe(Effect.flip), + floor.heartbeat({ holderIdentity: "nas-link-1", leaseIds: [] }).pipe(Effect.flip) + ], { concurrency: "unbounded" }) + }))) + expect(out.map((e) => e.kind)).toEqual(["transient", "auth"]) + void w + }) + + it("B10 a missing Gateway, or one that allows everything, leases nothing; a restricted one does", async () => { + const w = W(); w.floor.enqueue(job("1")) + const halogen = w.ax.gateways.get("halogen")! + w.ax.gateways.delete("halogen") + const l = start(w, dir) + await until("gateway-missing", () => l.has("gateway-missing")) + w.ax.gateways.set("halogen", { hasAllowlist: true, hosts: [{ host: "*", port: 443 }] }) + await sleep(5 * SEC) + expect([w.ax.updates.size, w.floor.calls.filter((c) => c.rpc === "Lease").every((c) => c.capacity === 0)]).toEqual([0, true]) + w.ax.gateways.set("halogen", halogen) + await until("created", () => w.ax.tasks.has("wf-test-1-a1")) + expect(l.has("gateway-ok")).toBe(true) + await l.drain() + }) + + it("B11 a floor that over-grants never makes the link over-create: extra grants wait, journaled and heartbeated", async () => { + const w = W({ cap: 10, overGrant: 2 }) + for (const n of ["1", "2", "3"]) w.floor.enqueue(job(n)) + const l = start(w, dir, { maxInFlight: 1 }) + let peak = 0 + const sample = setInterval(() => { peak = Math.max(peak, liveOwn(w)) }, 2) + await until("three grants", () => ["1", "2", "3"].every((n) => J(w, n).state === "leased")) + await until("waiting", () => l.has("waiting-for-slot")) + for (const n of ["1", "2", "3"]) { + await until(`job ${n} running`, () => w.ax.tasks.get(`wf-test-${n}-a1`)?.phase === "Running") + w.ax.finish(`wf-test-${n}-a1`, 0, { n }) + await until(`job ${n} done`, () => J(w, n).state === "done") + } + clearInterval(sample) + expect(peak).toBe(1) + expect(["1", "2", "3"].map((n) => [J(w, n).result, J(w, n).attempt])).toEqual([["success", 1], ["success", 1], ["success", 1]]) + await l.drain() + }) + + it("B12 a Task that reads back with another spec is deleted and refused", async () => { + const w = W(); w.floor.enqueue(job("1")) + const create = w.ax.createTask + w.ax.createTask = (t) => create({ ...t, spec: { ...t.spec, env: t.spec.env.slice(0, 3) } }) // the store dropped env rows + const l = start(w, dir) + await until("floor done", () => J(w).state === "done") + expect([(J(w).output as { reason: string }).reason, w.ax.live("wf-test-1-a1")]).toEqual(["pre-start/readback-mismatch", false]) + await l.drain() + }) + + it("B13 a journaled grant the floor released while the link was dead is never created on restart", async () => { + const w = W(); w.floor.enqueue(job("1")); w.ax.blockCreate = true + const l1 = start(w, dir) + await until("grant journaled", () => l1.has("leased", "wf-test-1-a1")) + await l1.crash() + w.ax.blockCreate = false + await until("requeued after the grace", () => J(w).history.includes("requeue:grace"), 3000) + const l2 = start(w, dir) + await until("attempt 2 created", () => w.ax.tasks.has("wf-test-1-a2")) + expect([w.ax.updates.has("wf-test-1-a1"), l2.has("resume", "wf-test-1-a1")]).toEqual([false, false]) + w.ax.tick(); w.ax.finish("wf-test-1-a2", 0, { ok: 2 }) + await until("floor done", () => J(w).state === "done") + expect([J(w).result, J(w).attempt]).toEqual(["success", 2]) + await l2.drain() + }) + + it("B15 floor down past lease plus grace: the queued success goes out before the next Lease and is accepted (rule 4b)", async () => { + const w = W(); w.floor.enqueue(job("1")) + const l = start(w, dir) + await until("running", () => w.ax.tasks.get("wf-test-1-a1")?.phase === "Running") + w.floor.down = true + w.ax.finish("wf-test-1-a1", 0, { v: 1 }) + await until("outbox keeps", () => l.has("outbox-keep", "wf-test-1-a1")) + await until("the alarm requeued it while the floor was unreachable", () => J(w).history.includes("requeue:grace"), 3000) + w.floor.down = false + await until("floor done", () => J(w).state === "done") + expect([J(w).result, J(w).output, J(w).attempt]).toEqual(["success", { v: 1 }, 1]) + expect(J(w).history).toContain("withdrawn:a2") + expect(w.ax.tasks.has("wf-test-1-a2")).toBe(false) + await l.drain() + }) + + it("B17 a verdict that frees a slot leases at once, not at the next poll", async () => { + const w = W({ pollSeconds: 100 }) // 2 s between polls here + w.floor.enqueue(job("1")); w.floor.enqueue(job("2")) + const l = start(w, dir, { maxInFlight: 1 }) + await until("running", () => w.ax.tasks.get("wf-test-1-a1")?.phase === "Running", 4000) + await sleep(200) + const t0 = Date.now() + w.ax.finish("wf-test-1-a1", 0, { ok: 1 }) + await until("job 2 created", () => w.ax.tasks.has("wf-test-2-a1"), 1500) + expect(Date.now() - t0).toBeLessThan(1000) + await l.drain() + }) + + it("B18 a Lease endpoint is followed only when it is in the declared list", async () => { + const w = W() + let ep = "https://evil.example/floor" + const f: typeof globalThis.fetch = async (input, init) => { + const res = await w.floor.fetch(input, init) + const text = await res.text() + const h = new Headers(res.headers); h.delete("content-length") + return new Response(text.replace("\"nextPollSeconds\":", `"endpoint":"${ep}","nextPollSeconds":`), { status: res.status, headers: h }) + } + const seen: Array = [] + const l = start(w, dir, { floorUrls: ["https://conwip-floor-2.example/"] }, { fetch: f, onEndpoint: (u) => seen.push(u) }) + await until("ignored", () => l.has("endpoint-ignored")) + ep = "https://conwip-floor-2.example/" + await until("accepted", () => l.has("endpoint-accepted")) + expect(seen).toEqual(["https://conwip-floor-2.example/"]) + await l.drain() + }) +}) diff --git a/pkgs/substrate-apps/src/apps/link/test/fake-ax.ts b/pkgs/substrate-apps/src/apps/link/test/fake-ax.ts new file mode 100644 index 00000000..5cc096e8 --- /dev/null +++ b/pkgs/substrate-apps/src/apps/link/test/fake-ax.ts @@ -0,0 +1,84 @@ +// Test double for ax-server, with stock v0.3.0 semantics where they matter to the link (MEASURED in ax d8ed0fe: +// internal/server/server.go UpdateTask is a blind upsert that also overwrites status; DeleteTask is two-phase; +// FIELD-MAP D3: a save re-runs the reconcile and resumes the actor). `p1` switches on the carried patch P1: +// a terminal phase is written by the controller and GetTaskResult exists. ListTasks pages like ax's Redis store +// (ZRevRange: newest first, limit 0 means 50; MEASURED upstream server.go:91-104, store.go:209-212). GetGateway +// answers from `gateways`; the default `halogen` Gateway allows one host, as ax-fleet's does. +import { createHash } from "node:crypto" +import { Effect } from "effect" +import { AxError } from "../src/ax.ts" +import type { AxApi, AxGateway, AxObserved, AxResult } from "../src/ax.ts" +import type { AxTask } from "../src/contract.ts" + +const NOT_FOUND = 5, UNAVAILABLE = 14 +type T = { task: AxTask; phase: string; conditions: Array<{ type: string; status: string; reason: string; message: string }>; result?: string; usage?: { promptTokens: number; completionTokens: number; toolCalls: number }; exited?: boolean } + +export class FakeAx implements AxApi { + readonly tasks = new Map() + readonly updates = new Map() // UpdateTask calls per name + upsertsOnExisting = 0 // the hazard: an UpdateTask on a name that already existed (re-runs the actor on v0.3.0) + resumedAfterExit = 0 // how many times that hazard restarted a finished agent + deletes: Array = [] + events: Array = [] // update: and delete:, in call order + up = true + p1 = true + blockCreate = false // createTask never returns (a crash window before the Task exists) + hangAfterCreate = false // createTask writes the Task, then never returns (a crash window after it exists) + holdPending = new Set() // names the controller never starts + gateways = new Map([["halogen", { hasAllowlist: true, hosts: [{ host: "worker", port: 8731 }] }]]) + listCalls = 0 + + private guard = (): Effect.Effect => this.up ? Effect.void : Effect.fail(new AxError(UNAVAILABLE, "connection refused")) + private obs = (name: string, t: T): AxObserved => ({ name, phase: t.phase, spec: t.task.spec, conditions: t.conditions, usage: t.usage ?? null }) + + /** The controller's work between two resyncs: start Pending Tasks, finish two-phase deletes. */ + tick() { + for (const [name, t] of this.tasks) { + if (t.phase === "Terminating") this.tasks.delete(name) + else if (t.phase === "Pending" && !this.holdPending.has(name)) t.phase = "Running" + } + } + /** The agent exits. With P1 the controller writes the terminal phase; without it the Task stays Running. */ + finish(name: string, exitCode: number, result?: unknown, usage = { promptTokens: 11, completionTokens: 7, toolCalls: 3 }) { + const t = this.tasks.get(name) + if (!t) throw new Error(`no task ${name}`) + t.exited = true + if (!this.p1) return + t.phase = exitCode === 0 ? "Completed" : "Failed" + t.conditions = [{ type: "Ready", status: "False", reason: "CommandExited", message: `ExitCode=${exitCode}` }] + if (result !== undefined) t.result = JSON.stringify(result) + t.usage = usage + } + put(task: AxTask, phase = "Running") { this.tasks.set(task.metadata.name, { task, phase, conditions: [] }) } + live(name: string) { const t = this.tasks.get(name); return t !== undefined && t.phase !== "Terminating" } + + getTask = (name: string) => this.guard().pipe(Effect.map(() => { const t = this.tasks.get(name); return t ? this.obs(name, t) : undefined })) + createTask = (task: AxTask) => this.guard().pipe(Effect.andThen(() => { + if (this.blockCreate) return Effect.never + const name = task.metadata.name + this.updates.set(name, (this.updates.get(name) ?? 0) + 1) + this.events.push(`update:${name}`) + const prior = this.tasks.get(name) + if (prior) { this.upsertsOnExisting++; if (prior.exited) this.resumedAfterExit++ } // v0.3.0: status reset, actor resumed + this.tasks.set(name, { task, phase: "Pending", conditions: [] }) + return this.hangAfterCreate ? Effect.never : Effect.void + })) + listTasks = (limit: number, offset: number) => this.guard().pipe(Effect.map(() => { + if (offset === 0) { this.tick(); this.listCalls++ } + const n = limit > 0 ? limit : 50 + return [...this.tasks].reverse().slice(offset, offset + n).map(([name, t]) => this.obs(name, t)) + })) + getGateway = (name: string) => this.guard().pipe(Effect.map(() => this.gateways.get(name))) + deleteTask = (name: string) => this.guard().pipe(Effect.map(() => { + const t = this.tasks.get(name) + if (t) { t.phase = "Terminating"; this.deletes.push(name); this.events.push(`delete:${name}`) } + })) + getTaskResult = (name: string) => this.guard().pipe(Effect.andThen((): Effect.Effect => { + if (!this.p1) return Effect.succeed("unimplemented" as const) + const t = this.tasks.get(name) + if (!t) return Effect.succeed(undefined) + const content = t.result ?? "null" + return Effect.succeed({ content, sha256: createHash("sha256").update(content).digest("hex"), digestOk: true }) + })) + static notFound = NOT_FOUND +} diff --git a/pkgs/substrate-apps/src/apps/link/test/fake-floor.ts b/pkgs/substrate-apps/src/apps/link/test/fake-floor.ts new file mode 100644 index 00000000..91d3f827 --- /dev/null +++ b/pkgs/substrate-apps/src/apps/link/test/fake-floor.ts @@ -0,0 +1,311 @@ +// Test double for the Cloudflare floor: the FloorLink group served by Effect's stock HTTP RPC server through a web +// handler (the same `RpcServer.layerHttp` + `HttpRouter.toWebHandler` pair the prototype DO runs), called in-process +// through the link's `fetch`. State is in memory. It implements the floor rules of LINK-DESIGN.md section 4 as amended +// by "Critique applied", section H (B16): +// rule 1 states queued, leased, orphaned, done; WIP = leased + orphaned +// rule 2 route by runs-on against the holder's labels, cap per label, an admission stub, refuse at enqueue a job +// whose runs-on is empty or does not name exactly one seat: label +// rule 2b a replayed requestKey returns the same grants (kept for the life of the double, at least lease + grace) +// rule 4 stale-attempt, unknown-lease, duplicate; rule 4b: a verdict for attempt n while n+1 is still queued is +// accepted and n+1 withdrawn +// round 1 rule 4c: the verdict applied is recorded per leaseId, and a repeat for the same (leaseId, attempt) answers +// duplicate: true before rule 4b is considered (a lost reply or a second sender never spends a second +// release); rule 4b also accepts n's verdict while n+1 is leased to the SAME holder, which does not create +// n+1 while n's verdict is undelivered (link.ts dispatch) +// round 2 rule 4b names the grant it withdrew in the answer (`withdrew`); the link never infers a withdrawal. Rule +// 4c's duplicate answer replays the original answer's `withdrew`, so a lost reply is answered the same way +// round 3 rule 4b also names the generation withdrawn (`withdrewTransitions`: the leaseTransitions a grant of it +// carried or would have carried); `sendWithdrewGen: false` is a floor that omits it +// rule 5 a timer (the DO alarm): leased past renew + lease becomes orphaned, even with a cancel pending +// rule 6 the holder's heartbeat renews and re-adopts; lost for anything else; cancelRequested +// rule 7 an omitted orphan is released, so is one past the grace; supersedes lists EVERY earlier attempt +// round 4 rule 6b: a Heartbeat's `pendingRequestKey` vouches for every lease granted under that key (renewed and +// re-adopted, never released as omitted) while the holder has not seen that Lease answered +// rule 8 two budgets: infrastructure releases and agent retries, each final reason naming its budget +// rule 9 a per-lease token, minted only for guest-mode holders (A3), single use +// rule 10 at the deadline the floor sets cancel with reason deadline +// rule 11 holder = f(bearer): a payload holderIdentity that differs answers 403; Complete only for that holder's +// leases; one live session per holder (409); pause per holder +// It is NOT the production floor. +import { randomUUID } from "node:crypto" +import { Effect, Layer } from "effect" +import { HttpRouter } from "effect/unstable/http" +import { RpcSerialization, RpcServer } from "effect/unstable/rpc" +import { FloorLink, FloorLinkClient } from "../src/contract.ts" +import type { AgentJob, FailureOutput } from "../src/contract.ts" + +type Job = { + name: string; job: AgentJob; state: "queued" | "leased" | "orphaned" | "done"; attempt: number + leaseId?: string; holder?: string; acquire?: number; renew?: number; orphanedAt?: number; transitions: number + cancel?: string; result?: string; output?: unknown; usage?: unknown; supersedes: Array; token?: string; deadline?: number + history: Array; infraSpent: number; agentSpent: number; holders: Record + pin?: { holder: string; until: number } // rule 7b (red team double-run-r1-1) +} +export type TokenBinding = string | { holder: string; labels?: Array; guest?: boolean } +export interface FloorConfig { + cap: number; leaseSeconds: number; graceSeconds: number; pollSeconds: number; heartbeatSeconds: number + maxAttempts: number // rule 8: agent retries (actor-crashed, result-unreadable) + infraAttempts?: number // rule 8: infrastructure releases (default 5) + secondMs: number + tokens: Record + labelCaps?: Record // rule 2: cap per label + timer?: boolean // rule 5: the DO alarm as a real timer (default true) + overGrant?: number // a broken floor that grants this many more than asked (B11's test) + skewedEncode?: boolean // round 1: a floor whose schema drifted from the link's; grants go out without validation + sendWithdrewGen?: boolean // round 3: Complete names the generation withdrawn (default true) + pinSeconds?: number // rule 7b: a grace release keeps n+1 for n's holder this long, or until it heartbeats (default graceSeconds) +} +const DEFAULT_LABELS = ["seat:halogen", "runtime:gvisor"] +const INFRA = new Set(["omitted", "grace", "infra/task-lost", "pre-start/pending-timeout", "pre-start/ax-unavailable", "pre-start/resource-exhausted", "pre-start/superseded-verdict-pending", "infra/link-defect"]) +const AGENT = new Set(["infra/actor-crashed", "infra/result-unreadable"]) + +export class FakeFloor { + readonly jobs = new Map() + readonly grants = new Map>() + readonly sessions = new Map() + readonly calls: Array<{ rpc: string; holder?: string; capacity?: number; granted?: number; leaseIds?: ReadonlyArray; status?: number }> = [] + readonly paused = new Set() + readonly verdicts = new Map() // rule 4c + admit: (job: AgentJob) => boolean = () => true // rule 3 stub (seat readings) + seq = 0 + down = false + redeliver: Array = [] // at-least-once: the next Lease hands these leases out again + private readonly timer: ReturnType | undefined + constructor(readonly o: FloorConfig) { + if (o.timer !== false) { this.timer = setInterval(() => this.sweep(), o.secondMs); this.timer.unref?.() } + } + close() { if (this.timer) clearInterval(this.timer) } + private ms = (s: number) => s * this.o.secondMs + private binding(token: string) { + const b = this.o.tokens[token] + if (b === undefined) return undefined + return typeof b === "string" ? { holder: b, labels: DEFAULT_LABELS, guest: false } : { holder: b.holder, labels: b.labels ?? DEFAULT_LABELS, guest: b.guest ?? false } + } + private bindingOf(holder: string) { + for (const t of Object.keys(this.o.tokens)) { const b = this.binding(t)!; if (b.holder === holder) return b } + return undefined + } + + /** Rule 2 (amended): refuses a job whose runs-on is empty or does not name exactly one seat: label. */ + enqueue(job: AgentJob): boolean { + const seats = job.spec["runs-on"].filter((l) => l.startsWith("seat:")) + if (job.spec["runs-on"].length === 0 || seats.length !== 1) return false + this.enqueueUnchecked(job) + return true + } + /** A floor without rule 2's enqueue check, to test the link's own defence (B7). */ + enqueueUnchecked(job: AgentJob) { + if (!this.jobs.has(job.metadata.name)) this.jobs.set(job.metadata.name, { name: job.metadata.name, job, state: "queued", attempt: 1, transitions: 0, supersedes: [], history: [], infraSpent: 0, agentSpent: 0, holders: {} }) + } + cancel(name: string, why = "tom") { + const j = this.jobs.get(name)! + if (j.state === "queued") this.finish(j, "cancelled", { reason: why }) + else if (j.state !== "done") j.cancel = why + } + wip(label?: string) { return [...this.jobs.values()].filter((j) => (j.state === "leased" || j.state === "orphaned") && (label === undefined || j.job.spec["runs-on"].includes(label))).length } + stats() { return { seq: ++this.seq, cap: this.o.cap, wip: this.wip(), queued: [...this.jobs.values()].filter((j) => j.state === "queued").length, done: [...this.jobs.values()].filter((j) => j.state === "done").length } } + private finish(j: Job, result: string, output?: unknown, usage?: unknown) { + Object.assign(j, { state: "done", result, output, usage, token: undefined }); j.history.push(`done:${result}`) + } + /** An orphan the fleet side no longer vouches for: a cancelled job ends cancelled, any other goes back to the queue. */ + private release(j: Job, why: string) { + if (j.cancel) { j.history.push(`released:${why}`); return this.finish(j, "cancelled", { reason: j.cancel }) } + const h = j.holder + this.requeue(j, why) + const pin = this.o.pinSeconds ?? this.o.graceSeconds + if (why === "grace" && h !== undefined && j.state === "queued" && pin > 0) j.pin = { holder: h, until: Date.now() + this.ms(pin) } // rule 7b + } + private requeue(j: Job, why: string) { + j.history.push(`requeue:${why}`) + const infra = INFRA.has(why) || !AGENT.has(why) + if (infra) j.infraSpent++; else j.agentSpent++ + if (infra && j.infraSpent >= (this.o.infraAttempts ?? 5)) return this.finish(j, "failure", { reason: "infra/retry-budget-spent", message: why }) + if (!infra && j.agentSpent >= this.o.maxAttempts) return this.finish(j, "failure", { reason: "agent/retry-budget-spent", message: why }) + const old = j.leaseId! + Object.assign(j, { state: "queued", attempt: j.attempt + 1, transitions: j.transitions + 1, supersedes: [...j.supersedes, old], leaseId: undefined, holder: undefined, token: undefined, orphanedAt: undefined }) + } + /** The DO storage alarm's work (rules 5, 7 b and 10). Runs on the timer and on every request. */ + sweep(now = Date.now()) { + for (const j of this.jobs.values()) { + if (j.state === "leased" && j.deadline !== undefined && now >= j.deadline && !j.cancel) j.cancel = "deadline" + if (j.state === "leased" && j.renew! + this.ms(this.o.leaseSeconds) <= now) { // L3: expiry never frees a live slot, + j.state = "orphaned"; j.orphanedAt = now; j.history.push("orphaned") // even when a cancel is pending + } + if (j.state === "orphaned" && j.orphanedAt! + this.ms(this.o.graceSeconds) <= now) this.release(j, "grace") // L3 b + } + } + private grantable(j: Job, labels: Array, extra: Map) { + if (j.state !== "queued") return false + if (!j.job.spec["runs-on"].every((l) => labels.includes(l))) return false // rule 2: routed by label + for (const l of j.job.spec["runs-on"]) { + const cap = this.o.labelCaps?.[l] + if (cap !== undefined && this.wip(l) + (extra.get(l) ?? 0) >= cap) return false + } + return this.admit(j.job) + } + lease(p: { holderIdentity: string; capacity: number; requestKey: string }) { + this.sweep() + const b = this.bindingOf(p.holderIdentity) + let rows: Array + const prior = this.grants.get(p.requestKey) + if (prior) rows = prior.map((id) => [...this.jobs.values()].find((j) => j.leaseId === id && j.state !== "done")).filter((j): j is Job => j !== undefined) + else if (this.paused.has(p.holderIdentity) || b === undefined) rows = [] + else { + const free = Math.max(0, Math.min(p.capacity, this.o.cap - this.wip())) + (this.o.overGrant ?? 0) + const extra = new Map() + rows = [] + for (const j of this.jobs.values()) { + if (rows.length >= free) break + if (!this.grantable(j, b.labels, extra)) continue + if (j.pin !== undefined && j.pin.holder !== p.holderIdentity && j.pin.until > Date.now()) continue // rule 7b + rows.push(j) + for (const l of j.job.spec["runs-on"]) extra.set(l, (extra.get(l) ?? 0) + 1) + } + const now = Date.now() + for (const j of rows) { + const tm = j.job.spec["timeout-minutes"] + const leaseId = `${j.name}-a${j.attempt}` + Object.assign(j, { pin: undefined, state: "leased", leaseId, holder: p.holderIdentity, acquire: now, renew: now, token: b.guest ? randomUUID() : undefined, + deadline: tm === undefined ? undefined : now + this.ms(tm * 60) }) + j.holders[leaseId] = p.holderIdentity + j.history.push(`leased:a${j.attempt}`) + } + this.grants.set(p.requestKey, rows.map((j) => j.leaseId!)) + } + if (this.redeliver.length) { + rows = [...rows, ...this.redeliver.map((id) => [...this.jobs.values()].find((j) => j.leaseId === id)).filter((j): j is Job => j !== undefined)] + this.redeliver = [] + } + this.calls.push({ rpc: "Lease", holder: p.holderIdentity, capacity: p.capacity, granted: rows.length }) + return { + grants: rows.map((j) => ({ + leaseId: j.leaseId!, attempt: j.attempt, job: j.job, + lease: { holderIdentity: j.holder!, leaseDurationSeconds: this.o.leaseSeconds, acquireTime: j.acquire!, renewTime: j.renew!, leaseTransitions: j.transitions }, + ...(j.supersedes.length ? { supersedes: [...j.supersedes] } : {}), ...(j.token ? { leaseToken: j.token } : {}), ...(j.deadline !== undefined ? { deadline: j.deadline } : {}), reassignSeconds: this.o.leaseSeconds + this.o.graceSeconds + (this.o.pinSeconds ?? this.o.graceSeconds) + })), + stats: this.stats(), nextPollSeconds: this.paused.has(p.holderIdentity) ? this.o.pollSeconds * 10 : this.o.pollSeconds, heartbeatSeconds: this.o.heartbeatSeconds + } + } + vouchByKey = true // round 4 rule 6b; false is an L1-only floor that ignores pendingRequestKey + heartbeat(p: { holderIdentity: string; leaseIds: ReadonlyArray; pendingRequestKey?: string }) { + this.sweep() + this.calls.push({ rpc: "Heartbeat", holder: p.holderIdentity, leaseIds: p.leaseIds }) + for (const j of this.jobs.values()) if (j.pin?.holder === p.holderIdentity) j.pin = undefined // rule 7b: it hears lost now + const byKey = new Set(this.vouchByKey && p.pendingRequestKey !== undefined ? this.grants.get(p.pendingRequestKey) ?? [] : []) + for (const id of byKey) { // rule 6b: vouched for by key, not listed (the holder never saw these ids) + if (p.leaseIds.includes(id)) continue + const j = [...this.jobs.values()].find((x) => x.leaseId === id && (x.state === "leased" || x.state === "orphaned") && x.holder === p.holderIdentity) + if (!j) continue + if (j.state === "orphaned") { j.state = "leased"; j.history.push("re-adopted") } + j.renew = Date.now() + } + const renewed: Array = [], lost: Array = [], cancelRequested: Array = [] + for (const id of p.leaseIds) { + const j = [...this.jobs.values()].find((x) => x.leaseId === id && (x.state === "leased" || x.state === "orphaned") && x.holder === p.holderIdentity) + if (!j) { lost.push(id); continue } + if (j.state === "orphaned") { j.state = "leased"; j.history.push("re-adopted") } // the holder vouches again + j.renew = Date.now(); renewed.push(id) + if (j.cancel) cancelRequested.push(id) + } + for (const j of this.jobs.values()) // L3 a: an orphaned lease its holder no longer lists is gone on the fleet side + if (j.state === "orphaned" && j.holder === p.holderIdentity && !p.leaseIds.includes(j.leaseId!) && !byKey.has(j.leaseId!)) this.release(j, "omitted") + return { renewed, lost, cancelRequested, stats: this.stats() } + } + /** `holder` is the bearer's holder (rule 11); undefined only for the guest path, which its lease token authorises. */ + complete(p: { leaseId: string; attempt: number; result: string; output?: unknown; usage?: unknown }, holder?: string) { + this.sweep() + this.calls.push({ rpc: "Complete", leaseIds: [p.leaseId] }) + const seen = this.verdicts.get(p.leaseId) // rule 4c: at most one verdict per (leaseId, attempt) + if (seen !== undefined && seen.attempt === p.attempt) return { duplicate: true, stats: this.stats(), ...this.w(seen.withdrew, seen.withdrewTransitions) } + const name = p.leaseId.replace(/-a\d+$/, "") + let j = [...this.jobs.values()].find((x) => x.leaseId === p.leaseId) + let withdrew: string | undefined, withdrewTransitions: number | undefined + if (!j) { + const q = this.jobs.get(name) + if (!q) return { code: "unknown-lease" as const } + // rule 4b: attempt n's verdict while n+1 is still queued: accept it and withdraw n+1 + // rule 4b (round 1): n+1 queued, or leased to the holder of n (that holder has not created it) + const nHolder = q.holders[p.leaseId] + const withdrawable = q.state === "queued" || (q.state === "leased" && q.holder === nHolder) + if (withdrawable && q.attempt === p.attempt + 1 && q.supersedes.includes(p.leaseId) && (holder === undefined || nHolder === holder)) { + q.history.push(`withdrawn:a${q.attempt}`) + withdrew = `${q.name}-a${q.attempt}`; withdrewTransitions = q.transitions + Object.assign(q, { state: "leased", attempt: p.attempt, leaseId: p.leaseId, holder: nHolder, token: undefined, renew: Date.now(), supersedes: q.supersedes.filter((x) => x !== p.leaseId) }) + j = q + } else return { code: "stale-attempt" as const } + } + if (holder !== undefined && j.holders[p.leaseId] !== holder) return { code: "unknown-lease" as const } // rule 11 (red team auth r2-7: no oracle for a stranger) + if (j.state === "done") return { duplicate: true, stats: this.stats() } + if ((j.state !== "leased" && j.state !== "orphaned") || j.attempt !== p.attempt) return { code: "stale-attempt" as const } + const reason = (p.output as FailureOutput | undefined)?.reason + this.verdicts.set(p.leaseId, { attempt: p.attempt, result: p.result, ...(reason !== undefined ? { reason } : {}), ...(withdrew !== undefined ? { withdrew, withdrewTransitions } : {}) }) + const w = this.w(withdrew, withdrewTransitions) + if (p.result === "failure" && reason !== undefined && (INFRA.has(reason) || AGENT.has(reason))) { this.requeue(j, reason); return { duplicate: false, stats: this.stats(), ...w } } + this.finish(j, p.result, p.output ?? (j.cancel ? { reason: j.cancel } : null), p.usage ?? null) + return { duplicate: false, stats: this.stats(), ...w } + } + private w(withdrew: string | undefined, t: number | undefined) { + if (withdrew === undefined) return {} + return this.o.sendWithdrewGen === false || t === undefined ? { withdrew } : { withdrew, withdrewTransitions: t } + } + /** L7: the guest's own Complete, authorised by its per-lease token alone (single use, bound to one lease). */ + guestComplete(leaseId: string, token: string, result: string, output?: unknown) { + const j = [...this.jobs.values()].find((x) => x.leaseId === leaseId) + if (!j || j.token === undefined || j.token !== token) return { refused: "token" as const } + j.token = undefined // single use + return this.complete({ leaseId, attempt: j.attempt, result, output }) + } + + private handlerCache: ((r: Request) => Promise) | undefined + get handler(): (r: Request) => Promise { return this.handlerCache ??= this.skewed() ? this.lenientHandler() : this.strictHandler() } + private skewed() { return this.o.skewedEncode === true } + /** Round 1: the same handlers served through the link's lenient view of Lease, so a grant is sent as the floor + * holds it, without the strict encode (a floor whose contract drifted). */ + private lenientHandler() { + return HttpRouter.toWebHandler( + RpcServer.layerHttp({ group: FloorLinkClient, path: "/rpc", protocol: "http" }).pipe( + Layer.provide(FloorLinkClient.toLayer({ + Lease: (p) => Effect.sync(() => this.lease(p)), + Heartbeat: (p) => Effect.sync(() => this.heartbeat(p)), + Complete: (p, o) => Effect.suspend(() => { + const r = this.complete(p, String((o.headers as Record)["x-link-holder"] ?? "")) + return r.code !== undefined ? Effect.fail({ code: r.code }) : Effect.succeed({ duplicate: r.duplicate!, stats: r.stats!, ...(r.withdrew !== undefined ? { withdrew: r.withdrew } : {}), ...(r.withdrewTransitions !== undefined ? { withdrewTransitions: r.withdrewTransitions } : {}) }) + }) + })), + Layer.provide(RpcSerialization.layerJson)), { disableLogger: true }).handler as (r: Request) => Promise + } + private strictHandler() { return HttpRouter.toWebHandler( + RpcServer.layerHttp({ group: FloorLink, path: "/rpc", protocol: "http" }).pipe( + Layer.provide(FloorLink.toLayer({ + Lease: (p) => Effect.sync(() => this.lease(p)), + Heartbeat: (p) => Effect.sync(() => this.heartbeat(p)), + Complete: (p, o) => Effect.suspend(() => { + const r = this.complete(p, String((o.headers as Record)["x-link-holder"] ?? "")) + return r.code !== undefined ? Effect.fail({ code: r.code }) : Effect.succeed({ duplicate: r.duplicate!, stats: r.stats!, ...(r.withdrew !== undefined ? { withdrew: r.withdrew } : {}), ...(r.withdrewTransitions !== undefined ? { withdrewTransitions: r.withdrewTransitions } : {}) }) + }) + })), + Layer.provide(RpcSerialization.layerJson)), { disableLogger: true }).handler as (r: Request) => Promise } + + /** The Worker entry: network, bearer (per-link token bound to a holder), holder binding (403), then one session per + * holder (409, L5). The resolved holder reaches the handlers as `x-link-holder`; a client's own copy is dropped. */ + readonly fetch: typeof globalThis.fetch = async (input, init) => { + if (this.down) throw new TypeError("fetch failed") + const req = new Request(input as string | URL | Request, init) + const b = this.binding((req.headers.get("authorization") ?? "").replace(/^Bearer /, "")) + const answer = (status: number, body: string) => { this.calls.push({ rpc: "entry", status }); return new Response(body, { status }) } + if (b === undefined) return answer(401, "unauthorized") + const body = await req.clone().text() + let msgs: Array = [] + try { const m = JSON.parse(body); msgs = Array.isArray(m) ? m : [m] } catch { /* not JSON: the RPC server answers */ } + if (msgs.some((m) => m?.payload && typeof m.payload.holderIdentity === "string" && m.payload.holderIdentity !== b.holder)) return answer(403, "holder mismatch") + const sid = req.headers.get("x-link-session") ?? "" + const s = this.sessions.get(b.holder), now = Date.now() + if (s && s.id !== sid && s.until > now) return answer(409, "session held") + this.sessions.set(b.holder, { id: sid, until: now + this.ms(3 * this.o.heartbeatSeconds) }) + const u = new URL(req.url) + if (u.pathname === "/rpc/") u.pathname = "/rpc" // Effect's client posts to "/" (PROTO.md section 4) + const h = new Headers(req.headers) + h.set("x-link-holder", b.holder) + return this.handler(new Request(u, { method: req.method, headers: h, body })) + } +} diff --git a/pkgs/substrate-apps/src/apps/link/test/final-pass.test.ts b/pkgs/substrate-apps/src/apps/link/test/final-pass.test.ts new file mode 100644 index 00000000..8a468018 --- /dev/null +++ b/pkgs/substrate-apps/src/apps/link/test/final-pass.test.ts @@ -0,0 +1,72 @@ +// Final pass (2026-09-23): regression tests for the two link-side findings of wave/VERIFY-LINK.md that were still open at +// 3b036da, both reproduced there by the chaos harness against the prototype floor under wrangler dev --local +// (link/final-verify/chaos-fresh-3b036da: f1 ran as a1 and a2, OVERLAP 8 s). +// FP-1 (R3-1): a heartbeat sleep begun at the 30 s default while nothing was held outlived a short lease granted +// during it; the floor released the orphan and the job ran twice. +// FP-2 (V1): the link fenced only what Grant.supersedes named; a floor that omits it got attempt n+1 created beside +// a still Running attempt n. +import { expect, it } from "vitest" +import { bodyOf, job, sleep, start, until, world } from "./review-r4-harness.ts" + +it("FP-1 a short lease granted during the first long heartbeat sleep is renewed in time", async () => { + // Lease 6 s and grace 2 s at SEC = 20 ms; the link starts with the production default heartbeat (30 s = 600 ms). + const w = world({ leaseSeconds: 6, graceSeconds: 2, heartbeatSeconds: 2 }) + // Every Lease reply is held 30 ms, as a real HTTPS round trip is (the chaos run: wrangler dev --local), so the heartbeat + // fiber sizes its first sleep before the first reply has set the server's interval. With an in-process reply the fiber + // happened to start after it, which hid the defect from every earlier test. + const slowLease: typeof globalThis.fetch = async (input, init) => { + const body = await bodyOf(input, init) + if (body.includes('"Lease"')) await sleep(30) + return w.floor.fetch(input as any, { ...init, body }) + } + const l = start(w, undefined, { initialHeartbeatSeconds: 30 }, slowLease) + await until("first Lease answered", () => w.floor.jobs.size === 0 && l.logs.some((x) => x.ev === "ax-up"), 2000).catch(() => {}) + await sleep(60) // the first heartbeat sleep has begun, sized from the 30 s default + w.floor.enqueue(job("1")) + const J = () => w.floor.jobs.get("wf-test-1")! + await until("a1 running", () => w.ax.tasks.get("wf-test-1-a1") !== undefined, 3000) + await sleep(1200) // twice the default heartbeat: without the fix the lease expires and the grace runs out first + const out = { history: J().history, attempt: J().attempt, a2: w.ax.updates.get("wf-test-1-a2") ?? 0 } + console.log(JSON.stringify(out)) + await l.drain(); w.floor.close() + expect(out.attempt).toBe(1) + expect(out.a2).toBe(0) + expect(out.history).not.toContain("requeue:omitted") + expect(out.history.filter((h: string) => h.startsWith("requeue")).length).toBe(0) +}) + +it("FP-2 attempt n+1 is never created beside a live attempt n, even when the floor omits supersedes", async () => { + // red team double-run-r1-1: a pin (rule 7b) long enough that the link's own lease-expired fence does not delete a1 + // before a2's dispatch; this case is about the dispatch fence set, which must still catch a1 + const w = world({ leaseSeconds: 6, graceSeconds: 2, heartbeatSeconds: 2, pinSeconds: 60 }) + let failHeartbeat = false + const strip = (s: string) => s.replace(/,"supersedes":\[[^\]]*\]/g, "").replace(/"supersedes":\[[^\]]*\],/g, "") + const fetch: typeof globalThis.fetch = async (input, init) => { + const body = await bodyOf(input, init) + if (failHeartbeat && body.includes('"Heartbeat"')) throw new TypeError("fetch failed") + const res = await w.floor.fetch(input as any, { ...init, body }) + const text = await res.text() + return new Response(strip(text), { status: res.status, headers: res.headers }) + } + let a1LiveAtA2Create: boolean | undefined + const create = w.ax.createTask + ;(w.ax as any).createTask = (task: any) => { + if (task.metadata.name === "wf-test-1-a2" && a1LiveAtA2Create === undefined) a1LiveAtA2Create = w.ax.live("wf-test-1-a1") + return create(task) + } + w.floor.enqueue(job("1")) + const l = start(w, undefined, {}, fetch) + const J = () => w.floor.jobs.get("wf-test-1")! + await until("a1 running", () => w.ax.live("wf-test-1-a1") && l.has("created", "wf-test-1-a1"), 3000) + failHeartbeat = true // a1's lease expires and the grace runs out; the floor requeues attempt 2 and grants it on Lease + await until("floor requeued attempt 2", () => J().attempt === 2, 4000) + await until("a2 leased", () => l.has("leased", "wf-test-1-a2"), 4000) + failHeartbeat = false + await until("a2 created", () => (w.ax.updates.get("wf-test-1-a2") ?? 0) > 0, 4000) + const out = { a1LiveAtA2Create, extended: l.has("fence-set-extended", "wf-test-1-a2"), events: (w.ax as any).events?.filter((e: string) => e.includes("wf-test-1")) } + console.log(JSON.stringify(out)) + await l.drain(); w.floor.close() + expect(out.extended).toBe(true) + expect(out.a1LiveAtA2Create).toBe(false) + expect(w.ax.upsertsOnExisting).toBe(0) +}) diff --git a/pkgs/substrate-apps/src/apps/link/test/fixtures/ax-p1.proto b/pkgs/substrate-apps/src/apps/link/test/fixtures/ax-p1.proto new file mode 100644 index 00000000..a18e4a43 --- /dev/null +++ b/pkgs/substrate-apps/src/apps/link/test/fixtures/ax-p1.proto @@ -0,0 +1,405 @@ +// TEST FIXTURE: the vendored ax.proto (v0.3.0 d8ed0fe) plus the three hunks of carried patch P1 that the link reads: +// rpc GetTaskResult, messages GetTaskResultRequest and TaskResult, UsageStats.tool_calls. Copied from dotfiles branch +// ax/fleet-bringup pkgs/ax/patches/p1-completion.patch (lines 3240-3300) on 2026-09-23. Used only by the workerd +// integration test; production vendors the P1 proto when ax-fleet ships it (LINK-DESIGN section 9). +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +syntax = "proto3"; + +package ax.v1alpha1; + +import "google/protobuf/struct.proto"; +import "google/protobuf/timestamp.proto"; + +option go_package = "github.com/google/ax/pkg/apis/v1alpha1"; + +// AX defines the core control plane gRPC service for autonomous agent orchestration. +service AX { + // Manifests are parsed by the client (see `ax apply`) and submitted through the + // typed Update* RPCs below; the server never receives raw YAML. + + // Tasks + rpc GetTask(GetTaskRequest) returns (Task); + rpc ListTasks(ListTasksRequest) returns (ListTasksResponse); + rpc UpdateTask(UpdateTaskRequest) returns (Task); + rpc DeleteTask(DeleteTaskRequest) returns (DeleteTaskResponse); + rpc SuspendTask(SuspendTaskRequest) returns (Task); + rpc ResumeTask(ResumeTaskRequest) returns (Task); + rpc WatchTask(WatchTaskRequest) returns (stream WatchTaskResponse); + // GetTaskResult returns the result file the task's command wrote, as copied + // by the controller when the command exited. NotFound until then. + rpc GetTaskResult(GetTaskResultRequest) returns (TaskResult); + + // Gateways + rpc GetGateway(GetGatewayRequest) returns (Gateway); + rpc ListGateways(ListGatewaysRequest) returns (ListGatewaysResponse); + rpc UpdateGateway(UpdateGatewayRequest) returns (Gateway); + rpc DeleteGateway(DeleteGatewayRequest) returns (DeleteGatewayResponse); + + // Workspaces + rpc GetWorkspace(GetWorkspaceRequest) returns (Workspace); + rpc ListWorkspaces(ListWorkspacesRequest) returns (ListWorkspacesResponse); + rpc UpdateWorkspace(UpdateWorkspaceRequest) returns (Workspace); + rpc DeleteWorkspace(DeleteWorkspaceRequest) returns (DeleteWorkspaceResponse); + + // Models + rpc GetModel(GetModelRequest) returns (Model); + rpc ListModels(ListModelsRequest) returns (ListModelsResponse); + rpc UpdateModel(UpdateModelRequest) returns (Model); + rpc DeleteModel(DeleteModelRequest) returns (DeleteModelResponse); + +} + +// ObjectMeta is metadata for AX resources. +message ObjectMeta { + string name = 1; + string atespace = 2; + google.protobuf.Timestamp creation_timestamp = 3; +} + +// --- Task --- + +message Task { + string api_version = 1; + string kind = 2; + ObjectMeta metadata = 3; + TaskSpec spec = 4; + TaskStatus status = 5; +} + +message TaskSpec { + // Field 1 was `goal`, removed; the workspace goal lives on WorkspaceRef. + reserved 1; + reserved "goal"; + bool suspend = 2; + string image = 3; + repeated string command = 4; + repeated EnvVar env = 5; + ResourceReqs resources = 6; + // workspaces binds one or more Workspaces, each mounted at its own path + // under /workspace. The first entry is the task command's working directory. + repeated WorkspaceRef workspaces = 7; + GatewayRef gateway = 8; + // Field 9 was `policies` (budget and approval config), removed for now. + reserved 9; + reserved "policies"; + // debug enables the in-container guest services (process execution and file + // access) that back `ax ssh`. Off by default. + bool debug = 10; +} + +message EnvVar { + string name = 1; + string value = 2; +} + +message ResourceReqs { + ResourceList requests = 1; + ResourceList limits = 2; +} + +message ResourceList { + string cpu = 1; + string memory = 2; +} + +message WorkspaceRef { + string name = 1; + string path = 2; + string goal = 3; +} + +message GatewayRef { + string name = 1; +} + +message TaskStatus { + string phase = 1; + string id = 2; + string actor = 3; + // json_name keeps the established `workerIP` spelling in YAML and JSON. + string worker_ip = 4 [json_name = "workerIP"]; + PendingApproval pending_approval = 5; + UsageStats usage = 6; + repeated Condition conditions = 7; +} + +message PendingApproval { + string id = 1; + string action = 2; + google.protobuf.Timestamp requested_at = 3; +} + +message UsageStats { + int32 prompt_tokens = 1; + int32 completion_tokens = 2; + int32 tool_calls = 3; +} + +message Condition { + string type = 1; + string status = 2; + google.protobuf.Timestamp last_transition_time = 3; + string reason = 4; + string message = 5; +} + +// --- Gateway --- + +message Gateway { + string api_version = 1; + string kind = 2; + ObjectMeta metadata = 3; + GatewaySpec spec = 4; +} + +message GatewaySpec { + repeated Listener listeners = 1; + EgressConfig egress = 2; +} + +message Listener { + string name = 1; + int32 port = 2; + string protocol = 3; +} + +message EgressConfig { + EgressAllowlist allowlist = 1; +} + +message EgressAllowlist { + repeated HostRule hosts = 1; +} + +message HostRule { + string host = 1; + int32 port = 2; +} + +// --- Workspace --- + +message Workspace { + string api_version = 1; + string kind = 2; + ObjectMeta metadata = 3; + WorkspaceSpec spec = 4; +} + +message WorkspaceSpec { + repeated GitRepo git = 1; + MCPConfig mcp = 2; + SkillsConfig skills = 3; +} + +message GitRepo { + string name = 1; + string repo = 2; + string branch = 3; + string dir = 4; + int32 depth = 5; +} + +message MCPConfig { + repeated MCPRegistry registries = 1; + repeated MCPServer servers = 2; +} + +message MCPRegistry { + string provider = 1; + string project = 2; + string query = 3; + repeated MCPServer servers = 4; +} + +message MCPServer { + string name = 1; + string endpoint = 2; + string command = 3; + repeated string args = 4; +} + +message SkillsConfig { + repeated SkillRegistry registries = 1; + string path = 2; +} + +message SkillRegistry { + string provider = 1; + string project = 2; + string query = 3; +} + +// --- Model --- + +message Model { + string api_version = 1; + string kind = 2; + ObjectMeta metadata = 3; + ModelSpec spec = 4; +} + +message ModelSpec { + string provider = 1; + string model = 2; + // Fields 3-5 were typed temperature, max_tokens, and system_instruction; + // provider settings now live in the free-form `parameters` map. + reserved 3, 4, 5; + reserved "temperature", "max_tokens", "system_instruction"; + SecretKeyRef secret_key = 6; + // parameters are provider-specific generation settings passed through to the + // model API as-is, for example temperature or maxOutputTokens for Gemini. + google.protobuf.Struct parameters = 7; +} + +message SecretKeyRef { + string name = 1; + string key = 2; +} + +// --- RPC Request & Response Messages --- + +// Tasks +message GetTaskRequest { + string atespace = 1; + string name = 2; +} + +message GetTaskResultRequest { + string atespace = 1; + string name = 2; +} + +message TaskResult { + bytes content = 1; + string sha256 = 2; +} + +message ListTasksRequest { + string atespace = 1; + int64 limit = 2; + int64 offset = 3; +} + +message ListTasksResponse { + repeated Task tasks = 1; +} + +message UpdateTaskRequest { + Task task = 1; +} + +message DeleteTaskRequest { + string atespace = 1; + string name = 2; +} + +message DeleteTaskResponse {} + +message SuspendTaskRequest { + string atespace = 1; + string name = 2; +} + +message ResumeTaskRequest { + string atespace = 1; + string name = 2; +} + +message WatchTaskRequest { + string atespace = 1; + string name = 2; +} + +message WatchTaskResponse { + Task task = 1; + string action = 2; +} + +// Gateways +message GetGatewayRequest { + string atespace = 1; + string name = 2; +} + +message ListGatewaysRequest { + string atespace = 1; +} + +message ListGatewaysResponse { + repeated Gateway gateways = 1; +} + +message UpdateGatewayRequest { + Gateway gateway = 1; +} + +message DeleteGatewayRequest { + string atespace = 1; + string name = 2; +} + +message DeleteGatewayResponse {} + +// Workspaces +message GetWorkspaceRequest { + string atespace = 1; + string name = 2; +} + +message ListWorkspacesRequest { + string atespace = 1; +} + +message ListWorkspacesResponse { + repeated Workspace workspaces = 1; +} + +message UpdateWorkspaceRequest { + Workspace workspace = 1; +} + +message DeleteWorkspaceRequest { + string atespace = 1; + string name = 2; +} + +message DeleteWorkspaceResponse {} + +// Models +message GetModelRequest { + string atespace = 1; + string name = 2; +} + +message ListModelsRequest { + string atespace = 1; +} + +message ListModelsResponse { + repeated Model models = 1; +} + +message UpdateModelRequest { + Model model = 1; +} + +message DeleteModelRequest { + string atespace = 1; + string name = 2; +} + +message DeleteModelResponse {} + diff --git a/pkgs/substrate-apps/src/apps/link/test/link-ladder-hf.test.ts b/pkgs/substrate-apps/src/apps/link/test/link-ladder-hf.test.ts new file mode 100644 index 00000000..0d9d738e --- /dev/null +++ b/pkgs/substrate-apps/src/apps/link/test/link-ladder-hf.test.ts @@ -0,0 +1,131 @@ +// HF link-ladder (critique-pass VERIFY-link-ladder.md): the four sub-claims no test pinned. Each test here fails when +// the hunk behind its claim is reverted (the mutation named in its title, from the VERIFY mutation table): +// M1 busy() counts a delete-stuck, proven-free name -> L8 (and L2, now that busy() is fixed) +// M10 `auto` picks guest while the probe is unanswered -> L9 +// M11 main.ts drops the four new keys on the way to runLink -> L10 +// M15 the ladder is timed from the last touch, not deletingAt -> L11 +// FakeAx and FakeFloor only; nothing spawns workerd or wrangler. +import { appendFileSync, readFileSync } from "node:fs" +import { join } from "node:path" +import { Effect, Fiber } from "effect" +import { describe, expect, it } from "vitest" +import { AxError } from "../src/ax.ts" +import { linkConfigOf, readLinkEnv } from "../src/config.ts" +import type { SubstrateRead } from "../src/link.ts" +import { job, sleep, start, until, world } from "./review-r4-harness.ts" +import type { World } from "./review-r4-harness.ts" + +const stickTerminating = (w: World) => { + w.ax.tick = () => { for (const t of w.ax.tasks.values()) if (t.phase === "Pending") t.phase = "Running" } +} +const LADDER = { terminatingRetryMs: 150, terminatingEscalateMs: 600, terminatingGiveUpMs: 1200, terminatingRetries: 3 } +const journalLines = (dir: string) => readFileSync(join(dir, "journal.jsonl"), "utf8").trim().split("\n").map((x) => JSON.parse(x) as Record) +const tOf = (logs: ReadonlyArray>, ev: string, leaseId?: string) => + logs.find((x) => x.ev === ev && (leaseId === undefined || x.leaseId === leaseId))?.t as number | undefined + +const stuckAfterJob1 = async (w: World, l: ReturnType) => { + await until("a1 running", () => w.ax.tasks.get("wf-test-1-a1")?.phase === "Running") + w.ax.finish("wf-test-1-a1", 0, { ok: true }) + await until("floor done", () => w.floor.jobs.get("wf-test-1")!.state === "done", 5000) + await until("a1 Terminating", () => w.ax.tasks.get("wf-test-1-a1")?.phase === "Terminating", 5000) + await until("deleting journaled", () => l.has("delete"), 2000) + await sleep(300) // the snapshot catches up (AUDIT section 2 side observation) +} + +describe("HF link-ladder: the sub-claims VERIFY found untested", () => { + it("L8 (M1, LL-4 busy half) with a floor that over-grants, the admission gate alone holds a stuck slot until a read proves it free", async () => { + // overGrant makes the floor hand out job 2 whatever capacity the link asks for, so occupancy() is not a gate + // here and busy() in admit() is the binding one. Root cause of M1 surviving in VERIFY: busy() dropped a + // finished admitted lease's name from the set even while ax listed its Task live, so a1 never counted. + const w = world({ overGrant: 1 }); stickTerminating(w) + let answer: "absent" | "present" = "present" + const substrate: SubstrateRead = { actor: () => Effect.sync(() => answer) } + w.floor.enqueue(job("1")) + const l = start(w, undefined, { maxInFlight: 1, ...LADDER }, undefined, { substrate }) + await stuckAfterJob1(w, l) + w.floor.enqueue(job("2")) + await until("job 2 leased", () => l.has("leased", "wf-test-2-a1"), 3000) + await until("job 2 waits", () => l.has("waiting-for-slot", "wf-test-2-a1"), 3000) + await until("delete-stuck, actor present", () => l.has("delete-stuck"), 4000) + expect(l.logs.find((x) => x.ev === "delete-stuck")).toMatchObject({ task: "wf-test-1-a1", freeProven: false }) + await sleep(300) + expect(w.ax.updates.has("wf-test-2-a1")).toBe(false) // a stuck, not-proven-free Task keeps its slot in admit() + answer = "absent" + await until("job 2 created", () => w.ax.updates.has("wf-test-2-a1"), 4000) + expect(l.logs.filter((x) => x.ev === "delete-stuck").at(-1)).toMatchObject({ task: "wf-test-1-a1", freeProven: true }) + expect(tOf(l.logs, "created", "wf-test-2-a1")!).toBeGreaterThanOrEqual(l.logs.filter((x) => x.ev === "delete-stuck").at(-1)!.t as number) + expect(w.ax.tasks.get("wf-test-1-a1")?.phase).toBe("Terminating") + await l.drain() + }, 20000) + + it("L8b (the busy() fix) a reported lease whose Task ax still lists Running keeps its admission slot against an over-granting floor", async () => { + // Stock ax (no P1): the agent's verdict arrives by relay while the Task stays Running until deleted. Here the + // controller also never finishes the delete, so the Task is live for the whole test. + const w = world({ overGrant: 1 }); stickTerminating(w) + w.floor.enqueue(job("1")) + const l = start(w, undefined, { maxInFlight: 1, terminatingRetryMs: 60_000, terminatingEscalateMs: 120_000, terminatingGiveUpMs: 180_000 }) + await stuckAfterJob1(w, l) + w.floor.enqueue(job("2")) + await until("job 2 waits", () => l.has("waiting-for-slot", "wf-test-2-a1"), 3000) + await sleep(600) // twenty resyncs and many admit() calls + expect(w.ax.updates.has("wf-test-2-a1")).toBe(false) + expect([...w.ax.tasks.values()].filter((t) => t.phase !== "Completed" && t.phase !== "Failed").length).toBe(1) + await l.drain() + }, 20000) + + it("L9 (M10, LL-7) auto creates nothing while the P1 probe is unanswered, even with ListTasks and GetGateway up; guest once it answers", async () => { + const w = world({ tokens: { "tok-nas": { holder: "nas-link-1", guest: true } } }); w.ax.p1 = false + const real = w.ax.getTaskResult + let probeDown = true + w.ax.getTaskResult = (name: string) => probeDown && name === "conwip-link-capability-probe" + ? Effect.fail(new AxError(14, "probe timed out")) : real(name) + w.floor.enqueue(job("1")) + const RELAY = { atespace: "fleet", image: "ax-agent", gateway: "halogen", command: () => ["ax-agent", "pi"], completeUrl: "http://relay.internal:8787/complete" } + const l = start(w, undefined, { completion: "auto", shape: RELAY }) + await until("ax-up", () => l.has("ax-up")) + await until("gateway-ok", () => l.has("gateway-ok")) + await sleep(800) + expect(l.has("completion-probe")).toBe(false) // never answered, so never decided + expect(w.ax.updates.size).toBe(0) + expect(w.floor.calls.filter((c) => c.rpc === "Lease").every((c) => c.capacity === 0)).toBe(true) + probeDown = false + await until("created", () => w.ax.tasks.has("wf-test-1-a1"), 4000) + expect(l.logs.find((x) => x.ev === "completion-probe")).toMatchObject({ serverP1: false, mode: "guest" }) + await l.drain() + }) + + it("L10 (M11, LL-5) main.ts's mapping carries the four new keys, in milliseconds, to runLink", () => { + const c = linkConfigOf(readLinkEnv({ LINK_LEASE_KEY_ATTEMPTS: "2", LINK_TERMINATING_RETRY_SECONDS: "30", LINK_TERMINATING_ESCALATE_SECONDS: "120", LINK_TERMINATING_GIVE_UP_SECONDS: "600" }), ["https://a.example"]) + expect([c.leaseKeyAttempts, c.terminatingRetryMs, c.terminatingEscalateMs, c.terminatingGiveUpMs]).toEqual([2, 30_000, 120_000, 600_000]) + const d = linkConfigOf(readLinkEnv({}), []) + expect([d.leaseKeyAttempts, d.terminatingRetryMs, d.terminatingEscalateMs, d.terminatingGiveUpMs]).toEqual([5, 60_000, 300_000, 900_000]) + expect(d.secondMs).toBe(1000) + expect(c.floorUrls).toEqual(["https://a.example"]) + const src = readFileSync(new URL("../src/main.ts", import.meta.url), "utf8") + expect(src).toMatch(/runLink\(linkConfigOf\(c, floorUrls\)/) // main.ts uses this mapping, not a copy of it + }) + + it("L11 (M15, LL-1/LL-3) a touch after `deleting` does not restart the ladder clock: steps are due from deletingAt", async () => { + const w = world(); stickTerminating(w) + w.floor.enqueue(job("1")) + const quiet = { terminatingRetryMs: 60_000, terminatingEscalateMs: 120_000, terminatingGiveUpMs: 180_000 } + const l1 = start(w, undefined, { maxInFlight: 1, ...quiet }) + await stuckAfterJob1(w, l1) + Effect.runFork(Fiber.interrupt(l1.fiber)) // no drain: the journal is kept as it is + await sleep(100) + const deletingAt = journalLines(l1.dir).find((x) => x.ev === "deleting" && x.leaseId === "wf-test-1-a1")!.t as number + const ladder = { terminatingRetryMs: 300, terminatingEscalateMs: 900, terminatingGiveUpMs: 60_000, terminatingRetries: 5 } + await sleep(Math.max(0, deletingAt + ladder.terminatingEscalateMs + 100 - Date.now())) + const touch = Date.now() + appendFileSync(join(l1.dir, "journal.jsonl"), JSON.stringify({ t: touch, ev: "deleting", leaseId: "wf-test-1-a1", why: "janitor" }) + "\n") + const l2 = start(w, l1.dir, { maxInFlight: 1, ...ladder }) + await until("escalated", () => l2.has("ax-delete-escalate"), 3000) + const esc = l2.logs.find((x) => x.ev === "ax-delete-escalate")! + expect(esc.since).toBe(deletingAt) + // from the last touch it would be due at touch + 900 ms; from deletingAt it is already due at the first resync + expect((esc.t as number) - touch).toBeLessThan(ladder.terminatingEscalateMs / 2) + const r1 = l2.logs.find((x) => x.ev === "delete-retry")! + expect(r1).toMatchObject({ n: 1, since: deletingAt }) + await l2.drain() + }, 20000) +}) diff --git a/pkgs/substrate-apps/src/apps/link/test/link-ladder.test.ts b/pkgs/substrate-apps/src/apps/link/test/link-ladder.test.ts new file mode 100644 index 00000000..2a06e916 --- /dev/null +++ b/pkgs/substrate-apps/src/apps/link/test/link-ladder.test.ts @@ -0,0 +1,182 @@ +// Critique D.2 (cp/link-ladder): the Terminating ladder, the provably-free slot, deletingAt through compaction, the +// fence on a stuck superseded attempt, the tunables through config.ts, the V2 Gateway log and `auto` falling back to +// guest. Each test is an audit probe (critique-pass AUDIT-link-ladder.md, P1 to P5) with its assertion inverted. +// FakeAx and FakeFloor only; ladder times are shrunk the way the rest of the suite shrinks seconds. +import { mkdtempSync, readFileSync } from "node:fs" +import { tmpdir } from "node:os" +import { join } from "node:path" +import { Effect, Fiber } from "effect" +import { describe, expect, it } from "vitest" +import { ConfigInvalid, readLinkEnv } from "../src/config.ts" +import { Journal } from "../src/journal.ts" +import type { SubstrateRead } from "../src/link.ts" +import { job, sleep, start, until, world } from "./review-r4-harness.ts" +import type { World } from "./review-r4-harness.ts" + +/** DELETE-HANG run 2: the controller ACKs a delete event and drops it; a Terminating Task is never finished. */ +const stickTerminating = (w: World) => { + w.ax.tick = () => { for (const t of w.ax.tasks.values()) if (t.phase === "Pending") t.phase = "Running" } +} +const LADDER = { terminatingRetryMs: 150, terminatingEscalateMs: 600, terminatingGiveUpMs: 1200, terminatingRetries: 3 } +const journalLines = (dir: string) => readFileSync(join(dir, "journal.jsonl"), "utf8").trim().split("\n").map((x) => JSON.parse(x) as Record) + +/** Job 1 runs, completes, is acknowledged and deleted by the janitor into a Terminating that never ends. */ +const stuckAfterJob1 = async (w: World, l: ReturnType) => { + await until("a1 running", () => w.ax.tasks.get("wf-test-1-a1")?.phase === "Running") + w.ax.finish("wf-test-1-a1", 0, { ok: true }) + await until("floor done", () => w.floor.jobs.get("wf-test-1")!.state === "done", 5000) + await until("a1 Terminating", () => w.ax.tasks.get("wf-test-1-a1")?.phase === "Terminating", 5000) + await until("deleting journaled", () => l.has("delete"), 2000) + // AUDIT section 2 side observation: until the next resync the snapshot still shows a1 Completed (free by design); + // the ladder is about the steady state, so the snapshot is let catch up first (about 10 resyncs) + await sleep(300) +} + +describe("critique D.2: the Terminating ladder", () => { + it("L1 (P1 inverted) a stuck delete is re-sent, escalated and journaled delete-stuck; with no Substrate read the slot stays taken", async () => { + const w = world(); stickTerminating(w) + w.floor.enqueue(job("1")) + const l = start(w, undefined, { maxInFlight: 1, ...LADDER }) + await stuckAfterJob1(w, l) + w.floor.enqueue(job("2")) + await until("delete-stuck", () => l.has("delete-stuck"), 4000) + await sleep(600) // many more resyncs past the give-up: nothing further is sent + const deletesOfA1 = w.ax.deletes.filter((n) => n === "wf-test-1-a1").length + expect(deletesOfA1).toBe(1 + LADDER.terminatingRetries) // the first DeleteTask, then exactly `retries` re-sends + expect(l.logs.filter((x) => x.ev === "delete-retry").map((x) => x.n)).toEqual([1, 2, 3]) + const esc = l.logs.filter((x) => x.ev === "ax-delete-escalate") + expect(esc.length).toBe(1) + expect(esc[0]).toMatchObject({ task: "wf-test-1-a1", atespace: "fleet", actor: "no-reader" }) + const stuck = l.logs.filter((x) => x.ev === "delete-stuck") + expect(stuck.length).toBe(1) + expect(stuck[0]).toMatchObject({ task: "wf-test-1-a1", freeProven: false }) + // ladder order and timing measured from deletingAt + const at = (ev: string) => l.logs.find((x) => x.ev === ev)!.t as number + expect(at("delete-retry")).toBeLessThan(at("ax-delete-escalate")) + expect(at("ax-delete-escalate")).toBeLessThan(at("delete-stuck")) + const lines = journalLines(l.dir) + expect(lines.some((x) => x.ev === "delete-stuck" && x.freeProven === false)).toBe(true) + // DELETE-HANG default: capacity lowered by one until an operator clears it; job 2 is never created on top of it + expect(w.ax.updates.has("wf-test-2-a1")).toBe(false) + await l.drain() + }, 20000) + + it("L2 a Substrate read that proves the actor absent releases the slot after the give-up, and not before", async () => { + const w = world(); stickTerminating(w) + const asked: Array = [] + const substrate: SubstrateRead = { actor: (task) => Effect.sync(() => { asked.push(task); return "absent" as const }) } + w.floor.enqueue(job("1")) + const l = start(w, undefined, { maxInFlight: 1, ...LADDER }, undefined, { substrate }) + await stuckAfterJob1(w, l) + const stuckFrom = Date.now() + w.floor.enqueue(job("2")) + await until("escalated", () => l.has("ax-delete-escalate"), 3000) + expect(l.logs.find((x) => x.ev === "ax-delete-escalate")).toMatchObject({ actor: "absent" }) + expect(w.ax.updates.has("wf-test-2-a1")).toBe(false) // escalation alone releases nothing + await until("job 2 created", () => w.ax.updates.has("wf-test-2-a1"), 5000) + const stuck = l.logs.find((x) => x.ev === "delete-stuck")! + expect(stuck).toMatchObject({ task: "wf-test-1-a1", freeProven: true }) + expect(Date.now() - stuckFrom).toBeGreaterThan(LADDER.terminatingGiveUpMs / 2) // the slot stayed taken until the give-up + expect(w.ax.tasks.get("wf-test-1-a1")?.phase).toBe("Terminating") // ax still shows it; the read decided + expect(asked.every((t) => t === "wf-test-1-a1")).toBe(true) + await l.drain() + }, 20000) + + it("L3 the ladder clock (deletingAt), retry count and delete-stuck survive compaction and reopen", () => { + const dir = mkdtempSync(join(tmpdir(), "ladder-j-")) + const grant = { leaseId: "wf-x-a1", attempt: 1, lease: { leaseTransitions: 1, acquireTime: 1, leaseDurationSeconds: 90 }, job: { metadata: { name: "wf-x" } } } as never + const j = Journal.open(dir) + j.append({ ev: "grant", leaseId: "wf-x-a1", grant }, 1000) + j.append({ ev: "created", leaseId: "wf-x-a1", digest: "d" }, 2000) + j.append({ ev: "report", leaseId: "wf-x-a1", attempt: 1, result: "success" }, 3000) + j.append({ ev: "reported", leaseId: "wf-x-a1", duplicate: false }, 3500) + j.append({ ev: "deleting", leaseId: "wf-x-a1", why: "janitor" }, 4000) + j.append({ ev: "delete-retry", leaseId: "wf-x-a1", n: 1 }, 5000) + j.append({ ev: "deleting", leaseId: "wf-x-a1", why: "again" }, 8000) // a later intent never moves the clock + j.append({ ev: "delete-stuck", leaseId: "wf-x-a1", freeProven: false }, 9500) + const before = j.recs.get("wf-x-a1")! + expect([before.deletingAt, before.deleteRetries, before.deleteStuck, before.freeProven, before.at]).toEqual([4000, 1, true, false, 8000]) + for (let i = 0; i < 2; i++) { // open compacts; twice, so a compacted file compacts to itself + const r = Journal.open(dir).recs.get("wf-x-a1")! + expect([r.deletingAt, r.deleteRetries, r.deleteStuck, r.freeProven, r.deleting, r.at]).toEqual([4000, 1, true, false, "janitor", 8000]) + } + }) + + it("L4 (P2 inverted) a superseded attempt stuck Terminating: the fence re-sends DeleteTask and fails n+1 infra/delete-stuck, final; no attempt loop", async () => { + const w = world(); w.floor.enqueue(job("1")) + const l1 = start(w, undefined, {}) + await until("a1 running", () => w.ax.tasks.get("wf-test-1-a1")?.phase === "Running") + Effect.runFork(Fiber.interrupt(l1.fiber)) // l1 dies without a drain: no more heartbeats, the journal is kept + await sleep(1000); w.floor.sweep(); await sleep(7 * 20); w.floor.sweep() + stickTerminating(w) + const l2 = start(w, l1.dir, { fenceTimeoutMs: 1500, terminatingRetryMs: 300, terminatingEscalateMs: 60_000, terminatingGiveUpMs: 120_000 }) + await until("job done", () => w.floor.jobs.get("wf-test-1")!.state === "done", 10000) + const hist = w.floor.jobs.get("wf-test-1")!.history + expect(l2.logs.find((x) => x.ev === "fence-failed")).toMatchObject({ old: "wf-test-1-a1", why: "stuck" }) + expect(w.ax.updates.has("wf-test-1-a2")).toBe(false) // attempt n+1 never beside a possibly running n + expect(hist).not.toContain("leased:a3") + expect(hist.at(-1)).toBe("done:failure") + expect(w.floor.jobs.get("wf-test-1")!.output).toMatchObject({ reason: "infra/delete-stuck" }) + expect(w.ax.deletes.filter((n) => n === "wf-test-1-a1").length).toBeGreaterThanOrEqual(2) // re-asked inside the fence + expect(l2.logs.some((x) => x.ev === "delete-retry" && x.where === "fence")).toBe(true) + await l2.drain() + }, 20000) +}) + +describe("critique D.2: the remaining wiring", () => { + it("L5 (P3 inverted) gateway-ok carries the allowlist size and no allowsAll; an open Gateway is gateway-open", async () => { + const w = world(); const l = start(w) + await until("gateway-ok", () => l.has("gateway-ok")) + const ok = l.logs.find((x) => x.ev === "gateway-ok")! + expect(ok).toMatchObject({ gateway: "halogen", hosts: 1 }) + expect("allowsAll" in ok).toBe(false) + w.ax.gateways.set("halogen", { hasAllowlist: true, hosts: [{ host: "*", port: 443 }] }) + await until("gateway-open", () => l.has("gateway-open")) + expect(l.logs.find((x) => x.ev === "gateway-open")).toMatchObject({ gateway: "halogen", allowsAll: true }) + expect(l.logs.filter((x) => x.ev === "gateway-missing").length).toBe(0) + w.ax.gateways.delete("halogen") + await until("gateway-missing", () => l.has("gateway-missing")) + expect(l.logs.find((x) => x.ev === "gateway-missing")).toMatchObject({ found: false }) + await l.drain() + }) + + const RELAY = { atespace: "fleet", image: "ax-agent", gateway: "halogen", command: () => ["ax-agent", "pi"], completeUrl: "http://relay.internal:8787/complete" } + it("L6 (P4 inverted) auto with a fleet-internal Complete URL on stock ax creates in guest mode with a per-lease token", async () => { + const w = world({ tokens: { "tok-nas": { holder: "nas-link-1", guest: true } } }); w.ax.p1 = false; w.floor.enqueue(job("1")) + const l = start(w, undefined, { completion: "auto", shape: RELAY }) + await until("created", () => w.ax.tasks.has("wf-test-1-a1"), 4000) + expect(l.logs.find((x) => x.ev === "completion-probe")).toMatchObject({ serverP1: false, mode: "guest" }) + const env = new Map((w.ax.tasks.get("wf-test-1-a1")!.task.spec.env ?? []).map((e) => [e.name, e.value])) + expect(env.get("AX_CONWIP_COMPLETE_URL")).toBe(RELAY.completeUrl) + await l.drain() + }) + + it("L6b auto with a declared Complete URL still picks p1 when the server has P1; a public URL never enables guest", async () => { + const w = world(); w.floor.enqueue(job("1")) + const l = start(w, undefined, { completion: "auto", shape: RELAY }) + await until("created", () => w.ax.tasks.has("wf-test-1-a1"), 4000) + expect(l.logs.find((x) => x.ev === "completion-probe")).toMatchObject({ serverP1: true, mode: "p1" }) + const env = new Map((w.ax.tasks.get("wf-test-1-a1")!.task.spec.env ?? []).map((e) => [e.name, e.value])) + expect(env.has("AX_CONWIP_COMPLETE_URL")).toBe(false) + await l.drain() + const w2 = world(); w2.ax.p1 = false; w2.floor.enqueue(job("1")) + const l2 = start(w2, undefined, { completion: "auto", shape: { ...RELAY, completeUrl: "https://conwip-floor.x.workers.dev/guest" } }) + await sleep(800) + expect(l2.logs.find((x) => x.ev === "completion-probe")).toMatchObject({ mode: "none" }) + expect(l2.has("guest-url-invalid")).toBe(true) + expect(w2.ax.updates.size).toBe(0) + await l2.drain() + }) + + it("L7 (P5 inverted) LINK_LEASE_KEY_ATTEMPTS and the ladder keys are read, defaulted and ordered", () => { + const d = readLinkEnv({}) + expect([d.leaseKeyAttempts, d.terminatingRetrySeconds, d.terminatingEscalateSeconds, d.terminatingGiveUpSeconds]).toEqual([5, 60, 300, 900]) + const c = readLinkEnv({ LINK_LEASE_KEY_ATTEMPTS: "2", LINK_TERMINATING_RETRY_SECONDS: "30", LINK_TERMINATING_ESCALATE_SECONDS: "120", LINK_TERMINATING_GIVE_UP_SECONDS: "600" }) + expect([c.leaseKeyAttempts, c.terminatingRetrySeconds, c.terminatingEscalateSeconds, c.terminatingGiveUpSeconds]).toEqual([2, 30, 120, 600]) + const bad = (env: Record) => { try { readLinkEnv(env); return undefined } catch (e) { return e instanceof ConfigInvalid ? e.key : "other" } } + expect(bad({ LINK_LEASE_KEY_ATTEMPTS: "0" })).toBe("LINK_LEASE_KEY_ATTEMPTS") + expect(bad({ LINK_TERMINATING_RETRY_SECONDS: "300" })).toBe("LINK_TERMINATING_ESCALATE_SECONDS") // 300 is not above 300 + expect(bad({ LINK_TERMINATING_GIVE_UP_SECONDS: "200" })).toBe("LINK_TERMINATING_GIVE_UP_SECONDS") + expect(bad({ LINK_TERMINATING_RETRY_SECONDS: "x" })).toBe("LINK_TERMINATING_RETRY_SECONDS") + }) +}) diff --git a/pkgs/substrate-apps/src/apps/link/test/link.test.ts b/pkgs/substrate-apps/src/apps/link/test/link.test.ts new file mode 100644 index 00000000..ba0bdbd0 --- /dev/null +++ b/pkgs/substrate-apps/src/apps/link/test/link.test.ts @@ -0,0 +1,360 @@ +// The failure matrix of LINK-DESIGN.md section 8, one test per row, against the two doubles (fake ax with v0.3.0 +// upsert semantics and an optional P1; fake floor served through Effect's HTTP RPC server). Every server-set +// interval is scaled so one "second" is SEC ms. A crash is a fiber interrupt: nothing after it runs, and only what the +// journal fsynced survives, which is what kill -9 leaves. +import { mkdtempSync, rmSync } from "node:fs" +import { tmpdir } from "node:os" +import { join } from "node:path" +import { Deferred, Effect, Exit, Fiber } from "effect" +import { afterEach, beforeEach, describe, expect, it } from "vitest" +import type { AgentJob } from "../src/contract.ts" +import { rpcFloor } from "../src/floor.ts" +import { Journal } from "../src/journal.ts" +import { runLink } from "../src/link.ts" +import type { LinkConfig } from "../src/link.ts" +import { FakeAx } from "./fake-ax.ts" +import { FakeFloor } from "./fake-floor.ts" +import type { FloorConfig } from "./fake-floor.ts" + +const SEC = 20 +const A = "ultracode.mecattaf.dev/" +const JK = `${"cd".repeat(32)}:1` // round 4: FIELD-MAP 5a journal key (jobs.ts refuses a grant without it) +const job = (n: string, spec: Partial = {}, ann: Record = {}, w: Partial = {}): AgentJob => ({ + apiVersion: "ultracode.mecattaf.dev/v1alpha1", kind: "AgentJob", + metadata: { + name: `wf-test-${n}`, + labels: { [A + "run-id"]: "wf-test", [A + "workflow"]: "link-test", [A + "phase-index"]: "1" }, + annotations: { [A + "run-id-raw"]: "wf_test", [A + "label"]: `probe:${n}`, [A + "item-key"]: `wf_test#${n}`, [A + "journal-key"]: JK, [A + "phase-title"]: "Probe", ...ann } + }, + spec: { + "runs-on": ["seat:halogen", "runtime:gvisor"], + with: { prompt: `say ${n}`, prompt_ref: { sha256: "ab".repeat(32), bytes: 5, uri: `journal://wf_test/${n}/prompt.md` }, model: "halogen-qwen3.8-flash-next", ...w }, + ...spec + } +}) +const world = (o: Partial = {}) => ({ + floor: new FakeFloor({ cap: 2, leaseSeconds: 6, graceSeconds: 15, pollSeconds: 1, heartbeatSeconds: 2, maxAttempts: 3, secondMs: SEC, tokens: { "tok-nas": "nas-link-1" }, ...o }), + ax: new FakeAx() +}) +type World = ReturnType + +const GUEST = { completion: "guest" as const, shape: { atespace: "fleet", image: "ax-agent", gateway: "halogen", command: () => ["ax-agent", "pi"], completeUrl: "http://link.fleet.internal/guest" } } +const guestWorld = () => { const w = world({ tokens: { "tok-nas": { holder: "nas-link-1", guest: true } } }); w.ax.p1 = false; return w } + +function start(w: World, dir: string, over: Partial = {}, session = `s:${dir}`) { + const logs: Array> = [] + const stop = Effect.runSync(Deferred.make()) + const cfg: LinkConfig = { + holder: "nas-link-1", maxInFlight: 2, servedLabels: ["seat:halogen", "runtime:gvisor"], + shape: { atespace: "fleet", image: "ax-agent", gateway: "halogen", command: () => ["ax-agent", "pi"] }, + completion: "auto", secondMs: SEC, resyncMs: 30, pendingTimeoutMs: 1500, deleteAfterMs: 0, deadlineBackstopMs: 300, + createAttempts: 3, outboxBackoffMs: [20, 100], initialPollSeconds: 1, initialHeartbeatSeconds: 2, ...over + } + const fiber = Effect.runFork(Effect.scoped(Effect.gen(function*() { + const floor = yield* rpcFloor({ url: "http://floor.test", token: "tok-nas", sessionId: session, fetch: w.floor.fetch }) + return yield* runLink(cfg, { ax: w.ax, floor, journal: Journal.open(dir), log: (ev, f) => logs.push({ ev, ...f }), stop }) + }))) + return { + logs, + has: (ev: string, leaseId?: string) => logs.some((l) => l.ev === ev && (leaseId === undefined || l.leaseId === leaseId)), + crash: () => Effect.runPromise(Fiber.interrupt(fiber)), + drain: () => { Effect.runSync(Deferred.succeed(stop, undefined)); return Effect.runPromise(Fiber.await(fiber)) }, + exit: () => Effect.runPromise(Fiber.await(fiber)) + } +} +const until = async (what: string, pred: () => boolean, ms = 5000) => { + const t0 = Date.now() + while (!pred()) { if (Date.now() - t0 > ms) throw new Error(`timeout waiting for: ${what}`); await new Promise((r) => setTimeout(r, 5)) } +} +const sleep = (ms: number) => new Promise((r) => setTimeout(r, ms)) +const J = (w: World, n = "1") => w.floor.jobs.get(`wf-test-${n}`)! + +describe("conwip link failure matrix", () => { + let dir: string + beforeEach(() => { dir = mkdtempSync(join(tmpdir(), "conwip-link-")) }) + afterEach(() => rmSync(dir, { recursive: true, force: true })) + + it("F0 happy path on P1: one create, typed verdict with usage, Task deleted after the floor acknowledged", async () => { + const w = world(); w.floor.enqueue(job("1")) + const l = start(w, dir) + await until("running", () => w.ax.tasks.get("wf-test-1-a1")?.phase === "Running") + const env = Object.fromEntries(w.ax.tasks.get("wf-test-1-a1")!.task.spec.env.map((e) => [e.name, e.value])) + expect(env.AX_CONWIP_ITEM_KEY).toBe("wf_test#1"); expect(env.AX_CONWIP_ATTEMPT).toBe("1"); expect(env.AX_CONWIP_PROMPT).toBe("say 1") + expect(env.AX_CONWIP_LEASE_TOKEN).toBeUndefined() // P1 present: the guest carries no token at all + expect(w.ax.tasks.get("wf-test-1-a1")!.task.apiVersion).toBe("ax.io/v1alpha1") + w.ax.finish("wf-test-1-a1", 0, { answer: 42 }) + await until("floor done", () => J(w).state === "done") + expect([J(w).result, J(w).output, J(w).usage]).toEqual(["success", { answer: 42 }, { prompt_tokens: 11, completion_tokens: 7, tool_calls: 3 }]) + await until("janitor deleted the Task", () => !w.ax.tasks.has("wf-test-1-a1")) + expect(w.ax.updates.get("wf-test-1-a1")).toBe(1); expect(w.ax.upsertsOnExisting).toBe(0) + await l.drain() + }) + + it("F1 duplicate delivery: a redelivered grant creates nothing twice", async () => { + const w = world(); w.floor.enqueue(job("1")) + const l = start(w, dir) + await until("running", () => w.ax.tasks.get("wf-test-1-a1")?.phase === "Running") + w.floor.redeliver = ["wf-test-1-a1"] + await until("duplicate seen", () => l.has("duplicate-grant", "wf-test-1-a1")) + w.ax.finish("wf-test-1-a1", 0, { ok: true }) + await until("floor done", () => J(w).state === "done") + expect(w.ax.updates.get("wf-test-1-a1")).toBe(1); expect(w.ax.upsertsOnExisting).toBe(0) + await l.drain() + }) + + it("F2 crash after Lease, before UpdateTask: the restart creates exactly once", async () => { + const w = world(); w.floor.enqueue(job("1")); w.ax.blockCreate = true + const l1 = start(w, dir) + await until("grant journaled", () => l1.has("leased", "wf-test-1-a1")) + await l1.crash() + w.ax.blockCreate = false + const l2 = start(w, dir) + await until("running", () => w.ax.tasks.get("wf-test-1-a1")?.phase === "Running") + w.ax.finish("wf-test-1-a1", 0, { ok: 1 }) + await until("floor done", () => J(w).state === "done") + expect([J(w).result, J(w).attempt, w.ax.updates.get("wf-test-1-a1")]).toEqual(["success", 1, 1]) + await l2.drain() + }) + + it("F3 crash after UpdateTask, before the journal said created: the restart adopts, no second UpdateTask", async () => { + const w = world(); w.floor.enqueue(job("1")); w.ax.hangAfterCreate = true + const l1 = start(w, dir) + await until("task exists", () => w.ax.tasks.has("wf-test-1-a1")) + await l1.crash() + w.ax.hangAfterCreate = false + const l2 = start(w, dir) + await until("adopted", () => l2.has("adopted", "wf-test-1-a1")) + w.ax.finish("wf-test-1-a1", 0, { ok: 1 }) + await until("floor done", () => J(w).state === "done") + expect([J(w).result, w.ax.updates.get("wf-test-1-a1"), w.ax.upsertsOnExisting]).toEqual(["success", 1, 0]) + await l2.drain() + }) + + it("F4 crash while running; the agent finishes while the link is down; the restart reports once", async () => { + const w = world(); w.floor.enqueue(job("1")) + const l1 = start(w, dir) + await until("running", () => w.ax.tasks.get("wf-test-1-a1")?.phase === "Running") + await l1.crash() + w.ax.finish("wf-test-1-a1", 0, { late: true }) + const l2 = start(w, dir) + await until("floor done", () => J(w).state === "done") + expect([J(w).result, J(w).output, J(w).attempt]).toEqual(["success", { late: true }, 1]) + expect(w.floor.calls.filter((c) => c.rpc === "Complete").length).toBe(1) + await l2.drain() + }) + + it("F5 floor unreachable: the verdict waits in the outbox across a crash, the lease is orphaned not requeued, then lands", async () => { + const w = world(); w.floor.enqueue(job("1")) + const l1 = start(w, dir) + await until("running", () => w.ax.tasks.get("wf-test-1-a1")?.phase === "Running") + w.floor.down = true + w.ax.finish("wf-test-1-a1", 0, { v: 1 }) + await until("outbox keeps", () => l1.has("outbox-keep", "wf-test-1-a1")) + await l1.crash() + await sleep(7 * SEC) // past the 6 s lease, well inside the 15 s grace + const l2 = start(w, dir) + await sleep(3 * SEC) + w.floor.down = false + w.floor.sweep() // the DO alarm fires + await until("floor done", () => J(w).state === "done") + expect([J(w).result, J(w).attempt]).toEqual(["success", 1]) + expect(J(w).history.some((h) => h.startsWith("requeue"))).toBe(false) + await l2.drain() + }) + + it("F6 ax unreachable: capacity 0 on every poll, held leases still renewed, work resumes when ax returns", async () => { + const w = world(); w.floor.enqueue(job("1")) + const l = start(w, dir) + await until("running", () => w.ax.tasks.get("wf-test-1-a1")?.phase === "Running") + w.ax.up = false + await until("link saw ax down", () => l.has("ax-down")) + const mark = w.floor.calls.length + w.floor.enqueue(job("2")) + await sleep(10 * SEC) // longer than the 6 s lease + if (process.env.LINK_DEBUG) console.log(JSON.stringify({ history: J(w).history, calls: w.floor.calls.slice(mark).map((c) => [c.rpc, c.capacity, c.leaseIds]), logs: l.logs.slice(-15) })) + const during = w.floor.calls.slice(mark).filter((c) => c.rpc === "Lease") + expect(during.length).toBeGreaterThan(0) + expect(during.every((c) => c.capacity === 0 && c.granted === 0)).toBe(true) + expect([J(w).state, J(w, "2").state]).toEqual(["leased", "queued"]) + w.ax.up = true + await until("second job created", () => w.ax.tasks.has("wf-test-2-a1")) + await l.drain() + }) + + it("F7 lease expiry, link back inside the grace: the orphaned lease is re-adopted, same attempt, no second Task", async () => { + const w = world(); w.floor.enqueue(job("1")) + const l1 = start(w, dir) + await until("running", () => w.ax.tasks.get("wf-test-1-a1")?.phase === "Running") + await l1.crash() + await sleep(7 * SEC); w.floor.sweep() + expect([J(w).state, w.floor.wip()]).toEqual(["orphaned", 1]) // L3: never reclaim a live slot + const l2 = start(w, dir) + await until("re-adopted", () => J(w).history.includes("re-adopted")) + w.ax.finish("wf-test-1-a1", 0, { ok: 1 }) + await until("floor done", () => J(w).state === "done") + expect([J(w).result, J(w).attempt, w.ax.updates.get("wf-test-1-a1")]).toEqual(["success", 1, 1]) + await l2.drain() + }) + + it("F8 journal lost: the holder's complete heartbeat omits the orphan, attempt 2 supersedes it, old Task deleted first", async () => { + const w = world(); w.floor.enqueue(job("1")) + const l1 = start(w, dir) + await until("running", () => w.ax.tasks.get("wf-test-1-a1")?.phase === "Running") + await l1.crash() + rmSync(dir, { recursive: true, force: true }) + await sleep(7 * SEC); w.floor.sweep() + const l2 = start(w, dir, {}, "s:fresh-state") + await until("attempt 2 created", () => w.ax.tasks.has("wf-test-1-a2")) + expect(J(w).history).toContain("requeue:omitted") + expect(w.ax.events.indexOf("delete:wf-test-1-a1")).toBeGreaterThanOrEqual(0) + expect(w.ax.events.indexOf("delete:wf-test-1-a1")).toBeLessThan(w.ax.events.indexOf("update:wf-test-1-a2")) + w.ax.tick(); w.ax.finish("wf-test-1-a2", 0, { ok: 2 }) + await until("floor done", () => J(w).state === "done") + expect([J(w).result, J(w).attempt]).toEqual(["success", 2]) + await l2.drain() + }) + + it("F9 link dead past the grace: the alarm requeues; the returning link drops attempt 1 (lost) and runs attempt 2", async () => { + const w = world(); w.floor.enqueue(job("1")) + const l1 = start(w, dir) + await until("running", () => w.ax.tasks.get("wf-test-1-a1")?.phase === "Running") + await l1.crash() + await sleep(7 * SEC); w.floor.sweep() + await sleep(16 * SEC); w.floor.sweep() + expect(J(w).history).toContain("requeue:grace") + const l2 = start(w, dir) + await until("attempt 2 created", () => w.ax.tasks.has("wf-test-1-a2")) + expect(w.ax.deletes).toContain("wf-test-1-a1") + w.ax.tick(); w.ax.finish("wf-test-1-a2", 0, { ok: 2 }) + await until("floor done", () => J(w).state === "done") + expect([J(w).result, J(w).attempt]).toEqual(["success", 2]) + // the late verdict of attempt 1 can never be recorded: the floor fences on the attempt + expect(w.floor.complete({ leaseId: "wf-test-1-a1", attempt: 1, result: "success" })).toEqual({ code: "stale-attempt" }) + await l2.drain() + }) + + it("F10 cancel: carried in the heartbeat reply, DeleteTask, then Complete(cancelled) once the Task is gone", async () => { + const w = world(); w.floor.enqueue(job("1")) + const l = start(w, dir) + await until("running", () => w.ax.tasks.get("wf-test-1-a1")?.phase === "Running") + w.floor.cancel("wf-test-1") + await until("floor done", () => J(w).state === "done") + if (process.env.LINK_DEBUG) console.log(JSON.stringify({ history: J(w).history, events: w.ax.events, phase: w.ax.tasks.get("wf-test-1-a1")?.phase, logs: l.logs })) + expect([J(w).result, J(w).attempt, w.ax.tasks.has("wf-test-1-a1")]).toEqual(["cancelled", 1, false]) + await l.drain() + }) + + it("F11 stock v0.3.0 (no P1), guest mode configured: the Task stays Running after exit; the guest completes with its lease token; the link deletes on lost", async () => { + const w = guestWorld(); w.floor.enqueue(job("1")) + const l = start(w, dir, GUEST) + await until("running", () => w.ax.tasks.get("wf-test-1-a1")?.phase === "Running") + const env = Object.fromEntries(w.ax.tasks.get("wf-test-1-a1")!.task.spec.env.map((e) => [e.name, e.value])) + expect(env.AX_CONWIP_COMPLETE_URL).toBe("http://link.fleet.internal/guest") + expect(Object.values(env).some((v) => v.includes("tok-nas"))).toBe(false) // the fleet bearer never enters the sandbox + w.ax.finish("wf-test-1-a1", 0) + await sleep(5 * SEC) + expect(w.ax.tasks.get("wf-test-1-a1")!.phase).toBe("Running") // FIELD-MAP D3: no terminal phase without P1 + expect(w.floor.guestComplete("wf-test-1-a1", env.AX_CONWIP_LEASE_TOKEN!, "success", { guest: true })).toMatchObject({ duplicate: false }) + expect(w.floor.guestComplete("wf-test-1-a1", env.AX_CONWIP_LEASE_TOKEN!, "success", { guest: true })).toEqual({ refused: "token" }) + await until("link deleted the still-Running Task", () => !w.ax.tasks.has("wf-test-1-a1")) + expect([J(w).result, J(w).output]).toEqual(["success", { guest: true }]) + await l.drain() + }) + + it("F12 stuck Running (no P1, the guest never reports): the floor's deadline cancels it and ax frees the worker", async () => { + const w = guestWorld(); w.floor.enqueue(job("1", { "timeout-minutes": 1 })) // 60 s = 1200 ms here + const l = start(w, dir, GUEST) + await until("running", () => w.ax.tasks.get("wf-test-1-a1")?.phase === "Running") + w.ax.finish("wf-test-1-a1", 0) // exited, invisible to ax without P1 + await until("floor done", () => J(w).state === "done", 8000) + expect([J(w).result, J(w).output]).toEqual(["cancelled", { reason: "deadline" }]) + await until("Task gone", () => !w.ax.tasks.has("wf-test-1-a1")) + await l.drain() + }) + + it("F13 deadline passes while the floor is unreachable: the local backstop deletes the Task and queues the verdict", async () => { + // red team double-run-r1-1: a pin (rule 7b) long enough that the grant's reassignSeconds outlasts the deadline, so the + // deadline backstop acts before the self-fence would (lease 6 + grace 15 + pin 300 s against timeout 60 + 15 s) + const w = world({ tokens: { "tok-nas": { holder: "nas-link-1", guest: true } }, pinSeconds: 300 }); w.ax.p1 = false + w.floor.enqueue(job("1", { "timeout-minutes": 1 })) + const l = start(w, dir, GUEST) + await until("running", () => w.ax.tasks.get("wf-test-1-a1")?.phase === "Running") + w.floor.down = true + await until("backstop verdict", () => l.logs.some((x) => x.ev === "verdict" && x.reason === "deadline-exceeded"), 8000) + await until("Task deleted", () => !w.ax.tasks.has("wf-test-1-a1") || w.ax.tasks.get("wf-test-1-a1")!.phase === "Terminating") + await l.crash() + }) + + it("F14 stuck Pending: pre-start failure at the timeout, retried by the floor as attempt 2 superseding attempt 1", async () => { + const w = world(); w.floor.enqueue(job("1")); w.ax.holdPending.add("wf-test-1-a1") + const l = start(w, dir, { pendingTimeoutMs: 200 }) + await until("attempt 2 created", () => w.ax.tasks.has("wf-test-1-a2"), 8000) + expect(J(w).history).toContain("requeue:pre-start/pending-timeout") + expect(w.ax.deletes).toContain("wf-test-1-a1") + w.ax.tick(); w.ax.finish("wf-test-1-a2", 0, { ok: 2 }) + await until("floor done", () => J(w).state === "done") + expect([J(w).result, J(w).attempt]).toEqual(["success", 2]) + await l.drain() + }) + + it("F15 name conflict: a foreign Task with the lease's name is never overwritten (create-only, L4)", async () => { + const w = world(); w.floor.enqueue(job("1")) + w.ax.put({ apiVersion: "ax.io/v1alpha1", kind: "Task", metadata: { name: "wf-test-1-a1", atespace: "fleet" }, spec: { image: "someone-else", command: ["sleep"], env: [] } }) + const l = start(w, dir) + await until("floor done", () => J(w).state === "done") + expect([J(w).result, (J(w).output as { reason: string }).reason]).toEqual(["failure", "pre-start/name-conflict"]) + expect([w.ax.updates.get("wf-test-1-a1"), w.ax.tasks.get("wf-test-1-a1")?.task.spec.image]).toEqual([undefined, "someone-else"]) + await l.drain() + }) + + it("F16 deterministic refusals are final and touch nothing in ax: env over budget, unresolved prompt, prompt too big to inline (B12)", async () => { + const w = world({ cap: 3 }) + w.floor.enqueue(job("1", {}, {}, { schema: { description: "x".repeat(20000) } })) + w.floor.enqueue(job("2", {}, { [A + "prompt-unresolved"]: "no agent() call carries label" })) + w.floor.enqueue(job("3", {}, {}, { prompt: "p".repeat(17000) })) + const l = start(w, dir, { maxInFlight: 3 }) + await until("all done", () => J(w).state === "done" && J(w, "2").state === "done" && J(w, "3").state === "done") + expect([(J(w, "3").output as { reason: string }).reason, J(w, "3").attempt]).toEqual(["pre-start/prompt-file-route-missing", 1]) + expect([(J(w).output as { reason: string }).reason, J(w).attempt]).toEqual(["pre-start/env-over-budget", 1]) + expect([(J(w, "2").output as { reason: string }).reason, J(w, "2").attempt]).toEqual(["pre-start/prompt-unresolved", 1]) + expect(w.ax.updates.size).toBe(0) + await l.drain() + }) + + it("F17 a second replica with the same identity is refused (409) and exits; the first keeps its session", async () => { + const w = world(); w.floor.enqueue(job("1")) + const l1 = start(w, dir) + await until("running", () => w.ax.tasks.get("wf-test-1-a1")?.phase === "Running") + const dir2 = mkdtempSync(join(tmpdir(), "conwip-link-b-")) + const l2 = start(w, dir2, {}, "s:second-replica") + const exit = await l2.exit() + rmSync(dir2, { recursive: true, force: true }) + expect(Exit.isFailure(exit)).toBe(true) + expect(JSON.stringify(exit)).toContain("session-conflict") + w.ax.finish("wf-test-1-a1", 0, { ok: 1 }) + await until("floor done", () => J(w).state === "done") + await l1.drain() + }) + + it("F19 ax lost a Task it had created (Redis loss): infra/task-lost, the floor retries as attempt 2", async () => { + const w = world(); w.floor.enqueue(job("1")) + const l = start(w, dir) + await until("running", () => w.ax.tasks.get("wf-test-1-a1")?.phase === "Running") + w.ax.tasks.delete("wf-test-1-a1") // gone without a DeleteTask: the store forgot it + await until("attempt 2 created", () => w.ax.tasks.has("wf-test-1-a2")) + expect(J(w).history).toContain("requeue:infra/task-lost") + w.ax.tick(); w.ax.finish("wf-test-1-a2", 0, { ok: 2 }) + await until("floor done", () => J(w).state === "done") + expect([J(w).result, J(w).attempt]).toEqual(["success", 2]) + await l.drain() + }) + + it("F18 drain on SIGTERM: stop leasing, flush the outbox, leave running Tasks to be re-adopted", async () => { + const w = world(); w.floor.enqueue(job("1")) + const l = start(w, dir) + await until("running", () => w.ax.tasks.get("wf-test-1-a1")?.phase === "Running") + const exit = await l.drain() + expect(Exit.isSuccess(exit)).toBe(true) + expect([w.ax.tasks.get("wf-test-1-a1")?.phase, J(w).state]).toEqual(["Running", "leased"]) + }) +}) diff --git a/pkgs/substrate-apps/src/apps/link/test/redteam-double-run-r1-1.test.ts b/pkgs/substrate-apps/src/apps/link/test/redteam-double-run-r1-1.test.ts new file mode 100644 index 00000000..7bf4d9b4 --- /dev/null +++ b/pkgs/substrate-apps/src/apps/link/test/redteam-double-run-r1-1.test.ts @@ -0,0 +1,93 @@ +// Red team verify, double-run-r1-1 (2026-09-24). Scratch clone only. Runs the REAL link (runLink) twice, as two +// Place in apps/link/test/ of a substrate clone (imports ./fake-ax.ts, ./fake-floor.ts, ./review-r4-harness.ts). Verified failing at substrate 0ba7550. +// holders serving the same runs-on labels, each with its own executor (FakeAx), against the in-process floor double. +// link-a leases attempt 1, then loses the floor (partition) while its executor keeps running the Task. The floor +// orphans, releases by grace, requeues attempt 2 with supersedes [a1] and grants it to link-b, whose fence finds a1 +// absent on its own executor and creates a2. Asserts invariant I2 (at most one attempt executing); FAILS while the +// defect exists. Fixed 2026-09-24 (link.ts selfFence): it passes, and link-a logs lease-expired for a1. +import { mkdtempSync } from "node:fs" +import { tmpdir } from "node:os" +import { join } from "node:path" +import { Deferred, Effect, Fiber } from "effect" +import { expect, it } from "vitest" +import { rpcFloor } from "../src/floor.ts" +import { Journal } from "../src/journal.ts" +import { runLink } from "../src/link.ts" +import type { LinkConfig } from "../src/link.ts" +import { FakeAx } from "./fake-ax.ts" +import { FakeFloor } from "./fake-floor.ts" +import { job, SEC, sleep, until } from "./review-r4-harness.ts" + +const labels = ["seat:halogen", "runtime:gvisor"] +function startLink(holder: string, token: string, ax: FakeAx, fetch: typeof globalThis.fetch, dir = mkdtempSync(join(tmpdir(), `rt-dbl-${holder}-`))) { + const logs: Array> = [] + const stop = Effect.runSync(Deferred.make()) + const cfg: LinkConfig = { + holder, maxInFlight: 2, servedLabels: labels, + shape: { atespace: "fleet", image: "ax-agent", gateway: "halogen", command: () => ["ax-agent", "pi"] }, + completion: "auto", secondMs: SEC, resyncMs: 30, pendingTimeoutMs: 1500, deleteAfterMs: 0, deadlineBackstopMs: 300, + createAttempts: 3, outboxBackoffMs: [20, 100], initialPollSeconds: 1, initialHeartbeatSeconds: 2, fenceTimeoutMs: 300, resultReadTries: 3 + } + const fiber = Effect.runFork(Effect.scoped(Effect.gen(function*() { + const floor = yield* rpcFloor({ url: "http://floor.test", token, sessionId: `s:${dir}`, fetch }) + return yield* runLink(cfg, { ax, floor, journal: Journal.open(dir), log: (ev, f) => logs.push({ t: Date.now(), ev, ...f }), stop, floorUrl: "http://floor.test" }) + }))) + return { dir, logs, has: (ev: string, id?: string) => logs.some((l) => l.ev === ev && (id === undefined || l.leaseId === id)), + drain: () => { Effect.runSync(Deferred.succeed(stop, undefined)); return Promise.race([Effect.runPromise(Fiber.await(fiber)), sleep(1500)]) } } +} + +it("double-run-r1-1: a partitioned holder's attempt 1 keeps running on its executor while attempt 2 runs on the other holder's", async () => { + // lease 6 s, grace 15 s, heartbeat 2 s at SEC=20 ms per second + const floor = new FakeFloor({ cap: 2, leaseSeconds: 6, graceSeconds: 15, pollSeconds: 1, heartbeatSeconds: 2, maxAttempts: 3, secondMs: SEC, + tokens: { "tok-a": { holder: "link-a", labels }, "tok-b": { holder: "link-b", labels } } }) + const axA = new FakeAx(), axB = new FakeAx() + let partitionA = false + const fetchA: typeof globalThis.fetch = async (i, init) => { if (partitionA) throw new TypeError("fetch failed"); return floor.fetch(i, init) } + floor.enqueue(job("dbl")) + const A = startLink("link-a", "tok-a", axA, fetchA) + await until("a1 created on link-a's executor", () => axA.live("wf-test-dbl-a1"), 3000) + axA.tick() // Pending -> Running + partitionA = true // link-a loses the floor; its ax keeps the Task running (the agent has not finished) + const B = startLink("link-b", "tok-b", axB, floor.fetch) + const J = floor.jobs.get("wf-test-dbl")! + let overlapSeen = false, overlapMs = 0 + const t0 = Date.now() + while (Date.now() - t0 < 2500) { + const both = axA.live("wf-test-dbl-a1") && axB.live("wf-test-dbl-a2") + if (both) { overlapSeen = true; overlapMs += 5 } + await sleep(5) + } + const snapshot = { floorHistory: [...J.history], floorState: J.state, floorAttempt: J.attempt, floorHolder: J.holder, + a1LiveOnA: axA.live("wf-test-dbl-a1"), a2LiveOnB: axB.live("wf-test-dbl-a2"), aDeletes: [...axA.deletes], + aHbErrors: A.logs.filter((l) => l.ev === "heartbeat-error").length, bLeased: B.logs.filter((l) => l.ev === "leased").map((l) => [l.leaseId, l.supersedes]) } + console.log("R1-1 real-link snapshot", JSON.stringify(snapshot), "overlapMs~", overlapMs) + // heal the partition: link-a's next Heartbeat is answered lost and it deletes a1 (the only fence) + partitionA = false + // after the fix link-a fenced a1 itself (lease-expired) before the floor released it; before, `lost` was the only fence + await until("link-a fenced a1", () => A.has("lost", "wf-test-dbl-a1") || A.has("lease-expired", "wf-test-dbl-a1"), 3000).catch(() => {}) + console.log("after heal: a lost logged", A.has("lost", "wf-test-dbl-a1"), "a deletes", axA.deletes) + await A.drain(); await B.drain(); floor.close() + expect(A.has("lease-expired", "wf-test-dbl-a1")).toBe(true) + expect({ overlapSeen, a1LiveWhileA2Live: snapshot.a1LiveOnA && snapshot.a2LiveOnB }).toEqual({ overlapSeen: false, a1LiveWhileA2Live: false }) +}, 20000) + +it("double-run-r1-1 (fix): a link that was DOWN past reassignSeconds fences its Task at restart, before any floor call answers", async () => { + const floor = new FakeFloor({ cap: 2, leaseSeconds: 6, graceSeconds: 15, pollSeconds: 1, heartbeatSeconds: 2, maxAttempts: 3, secondMs: SEC, + tokens: { "tok-a": { holder: "link-a", labels } } }) + const axA = new FakeAx() + let partitionA = false + const fetchA: typeof globalThis.fetch = async (i, init) => { if (partitionA) throw new TypeError("fetch failed"); return floor.fetch(i, init) } + floor.enqueue(job("down")) + const A = startLink("link-a", "tok-a", axA, fetchA) + await until("a1 created", () => axA.live("wf-test-down-a1"), 3000) + axA.tick() + await sleep(10 * SEC) // a few renewals, journaled + await A.drain() // the unit stops; its ax Task keeps running (F18) + partitionA = true // and the floor is unreachable when it comes back + await sleep(40 * SEC) // past reassignSeconds (6 + 15 + 15 s) since the last renewal + expect(axA.live("wf-test-down-a1")).toBe(true) + const A2 = startLink("link-a", "tok-a", axA, fetchA, A.dir) + await until("fenced at restart", () => A2.has("lease-expired", "wf-test-down-a1"), 3000) + expect(axA.deletes).toContain("wf-test-down-a1") + await A2.drain(); floor.close() +}, 20000) diff --git a/pkgs/substrate-apps/src/apps/link/test/redteam-double-run-r1-2.test.ts b/pkgs/substrate-apps/src/apps/link/test/redteam-double-run-r1-2.test.ts new file mode 100644 index 00000000..2a3d0044 --- /dev/null +++ b/pkgs/substrate-apps/src/apps/link/test/redteam-double-run-r1-2.test.ts @@ -0,0 +1,102 @@ +// Red team verify, double-run-r1-2 (2026-09-24). Scratch clone only, never merged as is. +// Two replicas of ONE link identity (same token, same holder), each with its OWN executor (its own FakeAx) and its +// own state directory (so its own session id). The asserted property is the one the finding says is missing: a link +// told 409 (another session holds its identity) fences the Tasks it created before it exits 75. It FAILS today. +// Place in apps/link/test/ and run from apps/link: vitest run test/double-run-r1-2.test.ts (no workerd needed). +// Corollary (rule 4b withdraws an n+1 the other replica created) is reproduced by the engine test R1-2 in +// redteam/scratch/double-run-r1/substrate/apps/floor/test/redteam-double-run.test.ts (its lines 104-105 pass; line 107 fails). +import { mkdtempSync, rmSync } from "node:fs" +import { tmpdir } from "node:os" +import { join } from "node:path" +import { Deferred, Effect, Exit, Fiber } from "effect" +import { afterEach, beforeEach, describe, expect, it } from "vitest" +import type { AgentJob } from "../src/contract.ts" +import { rpcFloor } from "../src/floor.ts" +import { Journal } from "../src/journal.ts" +import { runLink } from "../src/link.ts" +import type { LinkConfig } from "../src/link.ts" +import { FakeAx } from "./fake-ax.ts" +import { FakeFloor } from "./fake-floor.ts" + +const SEC = 20 +const A = "ultracode.mecattaf.dev/" +const JK = `${"cd".repeat(32)}:1` +const job = (n: string): AgentJob => ({ + apiVersion: "ultracode.mecattaf.dev/v1alpha1", kind: "AgentJob", + metadata: { + name: `wf-test-${n}`, + labels: { [A + "run-id"]: "wf-test", [A + "workflow"]: "link-test", [A + "phase-index"]: "1" }, + annotations: { [A + "run-id-raw"]: "wf_test", [A + "label"]: `probe:${n}`, [A + "item-key"]: `wf_test#${n}`, [A + "journal-key"]: JK, [A + "phase-title"]: "Probe" } + }, + spec: { + "runs-on": ["seat:halogen", "runtime:gvisor"], + with: { prompt: `say ${n}`, prompt_ref: { sha256: "ab".repeat(32), bytes: 5, uri: `journal://wf_test/${n}/prompt.md` }, model: "halogen-qwen3.8-flash-next" } + } +}) + +function start(floor: FakeFloor, ax: FakeAx, dir: string, session: string, net: { down: boolean }) { + const logs: Array> = [] + const stop = Effect.runSync(Deferred.make()) + const cfg: LinkConfig = { + holder: "nas-link-1", maxInFlight: 2, servedLabels: ["seat:halogen", "runtime:gvisor"], + shape: { atespace: "fleet", image: "ax-agent", gateway: "halogen", command: () => ["ax-agent", "pi"] }, + completion: "auto", secondMs: SEC, resyncMs: 30, pendingTimeoutMs: 1500, deleteAfterMs: 0, deadlineBackstopMs: 300, + createAttempts: 3, outboxBackoffMs: [20, 100], initialPollSeconds: 1, initialHeartbeatSeconds: 2 + } + const fetch: typeof globalThis.fetch = (i, init) => net.down ? Promise.reject(new TypeError("fetch failed: partitioned")) : floor.fetch(i, init) + const fiber = Effect.runFork(Effect.scoped(Effect.gen(function*() { + const f = yield* rpcFloor({ url: "http://floor.test", token: "tok-nas", sessionId: session, fetch }) + return yield* runLink(cfg, { ax, floor: f, journal: Journal.open(dir), log: (ev, fl) => logs.push({ ev, ...fl }), stop }) + }))) + return { + logs, + exit: () => Effect.runPromise(Fiber.await(fiber)), + drain: () => { Effect.runSync(Deferred.succeed(stop, undefined)); return Effect.runPromise(Fiber.await(fiber)) } + } +} +const until = async (what: string, pred: () => boolean, ms = 5000) => { + const t0 = Date.now() + while (!pred()) { if (Date.now() - t0 > ms) throw new Error(`timeout waiting for: ${what}`); await new Promise((r) => setTimeout(r, 5)) } +} +const sleep = (ms: number) => new Promise((r) => setTimeout(r, ms)) + +describe("double-run-r1-2 verify: a superseded session cannot hear `lost` and exits without fencing", () => { + const dirs: Array = [] + const mk = () => { const d = mkdtempSync(join(tmpdir(), "r1-2-")); dirs.push(d); return d } + beforeEach(() => { dirs.length = 0 }) + afterEach(() => { for (const d of dirs) rmSync(d, { recursive: true, force: true }) }) + + it("replica s1 comes back to a 409 while its attempt 1 still runs beside s2's attempt 2", async () => { + const floor = new FakeFloor({ cap: 2, leaseSeconds: 6, graceSeconds: 15, pollSeconds: 1, heartbeatSeconds: 2, maxAttempts: 3, secondMs: SEC, tokens: { "tok-nas": "nas-link-1" } }) + const ax1 = new FakeAx(), ax2 = new FakeAx() + const net1 = { down: false }, net2 = { down: false } + floor.enqueue(job("1")) + const l1 = start(floor, ax1, mk(), "s1", net1) + await until("a1 created on ax1", () => ax1.tasks.has("wf-test-1-a1")) + ax1.tick() + expect(ax1.live("wf-test-1-a1")).toBe(true) + // s1 is cut off from the floor (partition, stall); its executor keeps running a1. + net1.down = true + await sleep(9 * SEC) // past the session (3 heartbeats = 6 s) and past the lease (6 s) + // s2: a second replica of the same identity with a fresh state directory and its own executor. + const l2 = start(floor, ax2, mk(), "s2", net2) + await until("a2 created on ax2", () => ax2.tasks.has("wf-test-1-a2")) + ax2.tick() + // s1 comes back. Every call it makes is answered 409 before the engine runs. + net1.down = false + const exit1 = await Promise.race([l1.exit(), sleep(4000).then(() => "still-running" as const)]) + const s1Refused = exit1 !== "still-running" && Exit.isFailure(exit1) && JSON.stringify(exit1).includes("session-conflict") + const observed = { + s1Refused409: s1Refused, + a1LiveOnS1Executor: ax1.live("wf-test-1-a1"), + a2LiveOnS2Executor: ax2.live("wf-test-1-a2"), + s1DeletesIssued: ax1.deletes.slice() + } + console.log("R1-2 observed", JSON.stringify(observed)) + await l2.drain() + expect(s1Refused).toBe(true) + expect(observed.a2LiveOnS2Executor).toBe(true) + // The property: a link that loses its identity must fence what it created before it exits. + expect(observed.a1LiveOnS1Executor, "attempt 1 still runs on s1's executor after s1 exited on 409 while attempt 2 runs on s2's").toBe(false) + }, 20000) +}) diff --git a/pkgs/substrate-apps/src/apps/link/test/redteam-double-run-r1-journal.test.ts b/pkgs/substrate-apps/src/apps/link/test/redteam-double-run-r1-journal.test.ts new file mode 100644 index 00000000..2860c3b6 --- /dev/null +++ b/pkgs/substrate-apps/src/apps/link/test/redteam-double-run-r1-journal.test.ts @@ -0,0 +1,24 @@ +// Red team double-run-r1-1 (2026-09-24): the self-fence clock survives a restart. A `renewed` entry keeps the send +// time of the last renewal without touching the record's `at`, and compaction carries it. +import { mkdtempSync, rmSync } from "node:fs" +import { tmpdir } from "node:os" +import { join } from "node:path" +import { expect, it } from "vitest" +import { Journal } from "../src/journal.ts" +import { job } from "./review-r4-harness.ts" + +it("a journaled renewal is read back after a reopen and after compaction", () => { + const dir = mkdtempSync(join(tmpdir(), "rt-renewed-")) + try { + const j = Journal.open(dir) + const grant = { leaseId: "wf-test-r-a1", attempt: 1, job: job("r"), lease: { holderIdentity: "h", leaseDurationSeconds: 90, acquireTime: 1, renewTime: 1, leaseTransitions: 0 }, reassignSeconds: 690 } + j.append({ ev: "grant", leaseId: grant.leaseId, grant } as never, 1_000) + j.append({ ev: "created", leaseId: grant.leaseId, digest: "d" }, 2_000) + j.append({ ev: "renewed", leaseId: grant.leaseId }, 50_000) + j.append({ ev: "renewed", leaseId: grant.leaseId }, 40_000) // an older send time never moves it back + expect(j.recs.get(grant.leaseId)).toMatchObject({ renewedAt: 50_000, at: 2_000 }) + const k = Journal.open(dir) // reopen compacts + expect(k.recs.get(grant.leaseId)).toMatchObject({ renewedAt: 50_000 }) + expect(Journal.open(dir).recs.get(grant.leaseId)!.renewedAt).toBe(50_000) + } finally { rmSync(dir, { recursive: true, force: true }) } +}) diff --git a/pkgs/substrate-apps/src/apps/link/test/redteam-loss.test.ts b/pkgs/substrate-apps/src/apps/link/test/redteam-loss.test.ts new file mode 100644 index 00000000..577359a8 --- /dev/null +++ b/pkgs/substrate-apps/src/apps/link/test/redteam-loss.test.ts @@ -0,0 +1,99 @@ +// Red team loss-and-wip r3-3 (critique pass KEEP-4): a floor-wide HTTP 500 is not charged to each verdict. +// Both cases failed on 2882545: the Task success ended failure infra/verdict-undeliverable. +import { mkdtempSync, rmSync } from "node:fs" +import { tmpdir } from "node:os" +import { join } from "node:path" +import { Deferred, Effect, Fiber } from "effect" +import { afterEach, beforeEach, describe, expect, it } from "vitest" +import { cidrIsOpen, gatewayAllowsAll } from "../src/ax.ts" +import type { AgentJob } from "../src/contract.ts" +import { rpcFloor } from "../src/floor.ts" +import { Journal } from "../src/journal.ts" +import { isFleetInternalUrl, runLink } from "../src/link.ts" +import type { LinkConfig } from "../src/link.ts" +import { FakeAx } from "./fake-ax.ts" +import { FakeFloor } from "./fake-floor.ts" +import type { FloorConfig } from "./fake-floor.ts" + +const SEC = 20 +const A = "ultracode.mecattaf.dev/" +const JK = `${"cd".repeat(32)}:1` // round 4: FIELD-MAP 5a journal key (jobs.ts refuses a grant without it) +const job = (n: string): AgentJob => ({ + apiVersion: "ultracode.mecattaf.dev/v1alpha1", kind: "AgentJob", + metadata: { name: `wf-test-${n}`, labels: { [A + "run-id"]: "wf-test", [A + "workflow"]: "link-test", [A + "phase-index"]: "1" }, + annotations: { [A + "run-id-raw"]: "wf_test", [A + "label"]: `probe:${n}`, [A + "item-key"]: `wf_test#${n}`, [A + "journal-key"]: JK, [A + "phase-title"]: "Probe" } }, + spec: { "runs-on": ["seat:halogen", "runtime:gvisor"], + with: { prompt: `say ${n}`, prompt_ref: { sha256: "ab".repeat(32), bytes: 5, uri: `journal://wf_test/${n}/prompt.md` }, model: "halogen-qwen3.8-flash-next" } } +}) +const world = (o: Partial = {}) => ({ + floor: new FakeFloor({ cap: 2, leaseSeconds: 6, graceSeconds: 15, pollSeconds: 1, heartbeatSeconds: 2, maxAttempts: 3, secondMs: SEC, tokens: { "tok-nas": "nas-link-1" }, ...o }), + ax: new FakeAx() +}) +type World = ReturnType +function start(w: World, dir: string, over: Partial = {}, fetch?: typeof globalThis.fetch) { + const logs: Array> = [] + const stop = Effect.runSync(Deferred.make()) + const cfg: LinkConfig = { + holder: "nas-link-1", maxInFlight: 2, servedLabels: ["seat:halogen", "runtime:gvisor"], + shape: { atespace: "fleet", image: "ax-agent", gateway: "halogen", command: () => ["ax-agent", "pi"] }, + completion: "auto", secondMs: SEC, resyncMs: 30, pendingTimeoutMs: 1500, deleteAfterMs: 0, deadlineBackstopMs: 300, + createAttempts: 3, outboxBackoffMs: [20, 100], initialPollSeconds: 1, initialHeartbeatSeconds: 2, fenceTimeoutMs: 300, resultReadTries: 3, ...over + } + const fiber = Effect.runFork(Effect.scoped(Effect.gen(function*() { + const floor = yield* rpcFloor({ url: "http://floor.test", token: "tok-nas", sessionId: `s:${dir}`, fetch: fetch ?? w.floor.fetch }) + return yield* runLink(cfg, { ax: w.ax, floor, journal: Journal.open(dir), log: (ev, f) => logs.push({ t: Date.now(), ev, ...f }), stop, floorUrl: "http://floor.test" }) + }))) + return { + logs, fiber, + has: (ev: string, leaseId?: string) => logs.some((l) => l.ev === ev && (leaseId === undefined || l.leaseId === leaseId)), + trace: () => logs.filter((x) => typeof x.leaseId === "string" || typeof x.task === "string").map((x) => `${x.ev}:${x.leaseId ?? x.task}${x.why ? ":" + x.why : ""}`), + crash: () => Effect.runPromise(Fiber.interrupt(fiber)), + drain: () => { Effect.runSync(Deferred.succeed(stop, undefined)); return Promise.race([Effect.runPromise(Fiber.await(fiber)), sleep(1500)]) } + } +} +const until = async (what: string, pred: () => boolean, ms = 5000) => { + const t0 = Date.now() + while (!pred()) { if (Date.now() - t0 > ms) throw new Error(`timeout waiting for: ${what}`); await new Promise((r) => setTimeout(r, 5)) } +} +const sleep = (ms: number) => new Promise((r) => setTimeout(r, ms)) +const J = (w: World, n = "1") => w.floor.jobs.get(`wf-test-${n}`)! +const bodyOf = async (input: unknown, init?: RequestInit) => { + const b = init?.body + if (typeof b === "string") return b + if (b instanceof Uint8Array) return new TextDecoder().decode(b) + if (b) return await new Response(b as ConstructorParameters[0]).text() + return (input instanceof Request) ? await input.clone().text() : "" +} +const rpcOf = async (input: unknown, init?: RequestInit) => { const b = await bodyOf(input, init); return b.includes('"Complete"') ? "Complete" : b.includes('"Lease"') ? "Lease" : b.includes('"Heartbeat"') ? "Heartbeat" : "?" } + +describe("KEEP-4 (r3-3): a floor-wide 500 is not charged to each verdict", () => { + let dir: string + const worlds: Array = [] + beforeEach(() => { dir = mkdtempSync(join(tmpdir(), "rl-link-")) }) + afterEach(() => { rmSync(dir, { recursive: true, force: true }) }) + const scenario = async (over: Partial, bursts: number) => { + const w = world({ cap: 4 }); worlds.push(w); w.floor.enqueue(job("1")) + let outage = false, served500 = 0 + const fetch: typeof globalThis.fetch = async (input, init) => { + if (outage) { await bodyOf(input, init); served500++; return new Response(JSON.stringify({ error: "LinkTokensInvalid" }) + "\n", { status: 500 }) } + return w.floor.fetch(input, init) + } + const l = start(w, dir, over, fetch) + await until("a1 running", () => w.ax.tasks.get("wf-test-1-a1")?.phase === "Running") + outage = true // every /rpc answers 500 before any RPC runs, as index.ts did for LinkTokensInvalid + w.ax.finish("wf-test-1-a1", 0, { answer: 42 }) + await until("verdict written", () => l.has("verdict", "wf-test-1-a1")) + await until("the outage served many 500s", () => served500 >= bursts, 8000) + outage = false + await until("job 1 closed", () => J(w).state === "done", 8000) + const out = { result: J(w).result, reason: (J(w).output as any)?.reason, served500, replaced: l.has("verdict-replaced", "wf-test-1-a1") } + await l.drain() + return out + } + it("A: a short outage (verdictAttempts 4, backoff [20,60] ms)", async () => { + expect(await scenario({ outboxBackoffMs: [20, 60], verdictAttempts: 4 }, 16)).toMatchObject({ result: "success", replaced: false }) + }, 20_000) + it("B: default verdictAttempts (8), an outage longer than grace", async () => { + expect(await scenario({ outboxBackoffMs: [20, 60] }, 32)).toMatchObject({ result: "success", replaced: false }) + }, 20_000) +}) diff --git a/pkgs/substrate-apps/src/apps/link/test/review-r1.test.ts b/pkgs/substrate-apps/src/apps/link/test/review-r1.test.ts new file mode 100644 index 00000000..5e49eeb7 --- /dev/null +++ b/pkgs/substrate-apps/src/apps/link/test/review-r1.test.ts @@ -0,0 +1,391 @@ +// Review round 1 (2026-09-23): the ten findings, each as a regression test that failed before its fix. The scratch +// repros are in /home/tom/today/evals-2026-09-23/link/scratch-r1-{0,1,2}; the log is link/REVIEW-LOG.md. +import { mkdtempSync, rmSync } from "node:fs" +import { tmpdir } from "node:os" +import { join } from "node:path" +import { Deferred, Effect, Fiber } from "effect" +import { afterEach, beforeEach, describe, expect, it } from "vitest" +import { AxError } from "../src/ax.ts" +import type { AgentJob, AxTask } from "../src/contract.ts" +import { rpcFloor } from "../src/floor.ts" +import type { FloorOptions } from "../src/floor.ts" +import { Journal } from "../src/journal.ts" +import { runLink } from "../src/link.ts" +import type { LinkConfig } from "../src/link.ts" +import { FakeAx } from "./fake-ax.ts" +import { FakeFloor } from "./fake-floor.ts" +import type { FloorConfig } from "./fake-floor.ts" +import { ConfigInvalid, readLinkEnv } from "../src/config.ts" +import type { FloorApi } from "../src/floor.ts" +import { axTaskFromGrant } from "../src/jobs.ts" + +const SEC = 20 +const UNAVAILABLE = 14 +const A = "ultracode.mecattaf.dev/" +const JK = `${"cd".repeat(32)}:1` // round 4: FIELD-MAP 5a journal key (jobs.ts refuses a grant without it) +const job = (n: string, spec: Partial = {}): AgentJob => ({ + apiVersion: "ultracode.mecattaf.dev/v1alpha1", kind: "AgentJob", + metadata: { + name: `wf-test-${n}`, + labels: { [A + "run-id"]: "wf-test", [A + "workflow"]: "link-test", [A + "phase-index"]: "1" }, + annotations: { [A + "run-id-raw"]: "wf_test", [A + "label"]: `probe:${n}`, [A + "item-key"]: `wf_test#${n}`, [A + "journal-key"]: JK, [A + "phase-title"]: "Probe" } + }, + spec: { + "runs-on": ["seat:halogen", "runtime:gvisor"], + with: { prompt: `say ${n}`, prompt_ref: { sha256: "ab".repeat(32), bytes: 5, uri: `journal://wf_test/${n}/prompt.md` }, model: "halogen-qwen3.8-flash-next" }, + ...spec + } +}) +const world = (o: Partial = {}) => ({ + floor: new FakeFloor({ cap: 2, leaseSeconds: 6, graceSeconds: 15, pollSeconds: 1, heartbeatSeconds: 2, maxAttempts: 3, secondMs: SEC, tokens: { "tok-nas": "nas-link-1" }, ...o }), + ax: new FakeAx() +}) +type World = ReturnType +const foreign = (name: string): AxTask => ({ apiVersion: "ax.io/v1alpha1", kind: "Task", metadata: { name, atespace: "fleet" }, spec: { image: "someone-else", command: ["true"], env: [] } }) + +interface Extra { fetch?: typeof globalThis.fetch; timeoutsMs?: FloorOptions["timeoutsMs"]; onEndpoint?: (u: string) => void } +function start(w: World, dir: string, over: Partial = {}, x: Extra = {}) { + const logs: Array> = [] + const stop = Effect.runSync(Deferred.make()) + const cfg: LinkConfig = { + holder: "nas-link-1", maxInFlight: 2, servedLabels: ["seat:halogen", "runtime:gvisor"], + shape: { atespace: "fleet", image: "ax-agent", gateway: "halogen", command: () => ["ax-agent", "pi"] }, + completion: "auto", secondMs: SEC, resyncMs: 30, pendingTimeoutMs: 1500, deleteAfterMs: 0, deadlineBackstopMs: 300, + createAttempts: 3, outboxBackoffMs: [20, 100], initialPollSeconds: 1, initialHeartbeatSeconds: 2, fenceTimeoutMs: 300, resultReadTries: 3, ...over + } + const fiber = Effect.runFork(Effect.scoped(Effect.gen(function*() { + const floor = yield* rpcFloor({ url: "http://floor.test", token: "tok-nas", sessionId: `s:${dir}`, fetch: x.fetch ?? w.floor.fetch, ...(x.timeoutsMs ? { timeoutsMs: x.timeoutsMs } : {}) }) + return yield* runLink(cfg, { ax: w.ax, floor, journal: Journal.open(dir), log: (ev, f) => logs.push({ ev, ...f }), stop, floorUrl: "http://floor.test", ...(x.onEndpoint ? { onEndpoint: x.onEndpoint } : {}) }) + }))) + return { + logs, + has: (ev: string, leaseId?: string) => logs.some((l) => l.ev === ev && (leaseId === undefined || l.leaseId === leaseId)), + verdict: (reason: string) => logs.some((l) => l.ev === "verdict" && l.reason === reason), + crash: () => Effect.runPromise(Fiber.interrupt(fiber)), + drain: () => { Effect.runSync(Deferred.succeed(stop, undefined)); return Promise.race([Effect.runPromise(Fiber.await(fiber)), sleep(1500)]) } + } +} +const until = async (what: string, pred: () => boolean, ms = 5000) => { + const t0 = Date.now() + while (!pred()) { if (Date.now() - t0 > ms) throw new Error(`timeout waiting for: ${what}`); await new Promise((r) => setTimeout(r, 5)) } +} +const sleep = (ms: number) => new Promise((r) => setTimeout(r, ms)) +const J = (w: World, n = "1") => w.floor.jobs.get(`wf-test-${n}`)! +const liveOwn = (w: World) => [...w.ax.tasks.keys()].filter((k) => k.startsWith("wf-test-") && w.ax.live(k) && !["Completed", "Failed"].includes(w.ax.tasks.get(k)!.phase)).length + + +const bodyOf = async (input: unknown, init?: RequestInit) => { + const b = init?.body + if (typeof b === "string") return b + if (b instanceof Uint8Array) return new TextDecoder().decode(b) + if (b) return await new Response(b as ConstructorParameters[0]).text() + return (input instanceof Request) ? await input.clone().text() : "" +} +const isComplete = async (input: unknown, init?: RequestInit) => (await bodyOf(input, init)).includes('"Complete"') +const stats = { seq: 1, cap: 2, wip: 0, queued: 0, done: 0 } +const grantOf = (n: string, extra: Record = {}) => ({ + leaseId: `wf-test-${n}-a1`, attempt: 1, job: job(n), + lease: { holderIdentity: "nas-link-1", leaseDurationSeconds: 30, acquireTime: Date.now(), renewTime: Date.now(), leaseTransitions: 0 }, ...extra +}) as any +/** A floor that hands out `grants` once and renews everything: for link-side properties the double cannot stage. */ +const stubFloor = (grants: Array) => { + let sent = false + const completes: Array = [] + const api: FloorApi = { + lease: (p) => Effect.sync(() => { const g = !sent && p.capacity > 0 ? grants : []; if (g.length) sent = true; return { grants: g, stats, nextPollSeconds: 1, heartbeatSeconds: 2 } }), + heartbeat: (p) => Effect.succeed({ renewed: [...p.leaseIds], lost: [], cancelRequested: [], stats }), + complete: (p) => Effect.sync(() => { completes.push(p); return { duplicate: false, stats } }) + } + return { api, completes } +} +const cfgOf = (over: Partial = {}): LinkConfig => ({ + holder: "nas-link-1", maxInFlight: 2, servedLabels: ["seat:halogen", "runtime:gvisor"], + shape: { atespace: "fleet", image: "ax-agent", gateway: "halogen", command: () => ["ax-agent", "pi"] }, + completion: "auto", secondMs: SEC, resyncMs: 30, pendingTimeoutMs: 60_000, deleteAfterMs: 60_000, deadlineBackstopMs: 300, + createAttempts: 3, outboxBackoffMs: [20, 100], initialPollSeconds: 1, initialHeartbeatSeconds: 2, fenceTimeoutMs: 300, resultReadTries: 3, ...over +}) +const runStub = async (cfg: LinkConfig, ax: FakeAx, floor: FloorApi, ms: number) => { + const d = mkdtempSync(join(tmpdir(), "conwip-link-r1s-")) + const logs: Array> = [] + const stop = Effect.runSync(Deferred.make()) + const f = Effect.runFork(runLink(cfg, { ax, floor, journal: Journal.open(d), log: (ev, x) => logs.push({ ev, ...x }), stop })) + await sleep(ms) + Effect.runSync(Deferred.succeed(stop, undefined)); await Effect.runPromise(Fiber.await(f)); rmSync(d, { recursive: true, force: true }) + return logs +} + +describe("review round 1: fixed", () => { + let dir: string + let worlds: Array = [] + const W = (o: Partial = {}) => { const w = world(o); worlds.push(w); return w } + beforeEach(() => { dir = mkdtempSync(join(tmpdir(), "conwip-link-r1-")) }) + afterEach(() => { for (const w of worlds) w.floor.close(); worlds = []; rmSync(dir, { recursive: true, force: true }) }) + + // ------------------------------------------------------------------ findings 1 and 2: one verdict, one application + it("R1-1 no fault: every verdict costs exactly one Complete (single-flight outbox)", async () => { + const w = W({ cap: 8 }) + for (let i = 1; i <= 8; i++) w.floor.enqueue(job(String(i))) + const l = start(w, dir, { maxInFlight: 8, resyncMs: 5 }) + await until("all running", () => [1, 2, 3, 4, 5, 6, 7, 8].every((i) => w.ax.tasks.get(`wf-test-${i}-a1`)?.phase === "Running")) + for (let i = 1; i <= 8; i++) w.ax.finish(`wf-test-${i}-a1`, 0, { i }) + await until("all done", () => [1, 2, 3, 4, 5, 6, 7, 8].every((i) => J(w, String(i)).state === "done")) + await sleep(100) + const per = new Map() + for (const c of w.floor.calls.filter((c) => c.rpc === "Complete")) per.set(c.leaseIds![0]!, (per.get(c.leaseIds![0]!) ?? 0) + 1) + expect([...per.values()]).toEqual([1, 1, 1, 1, 1, 1, 1, 1]) + expect(l.logs.filter((x) => x.ev === "complete").length).toBe(8) + await l.drain() + }) + + it("R1-1 (D5) one retryable failure with a budget of two releases leases attempt 2", async () => { + const w = W({ infraAttempts: 2 }); w.floor.enqueue(job("1")); w.ax.holdPending.add("wf-test-1-a1") + const l = start(w, dir, { pendingTimeoutMs: 150 }) + await until("attempt 2 leased", () => J(w).state === "leased" && J(w).attempt === 2, 4000) + expect(J(w).infraSpent).toBe(1) + expect(J(w).history).not.toContain("withdrawn:a2") + await l.drain() + }) + + it("R1-2 (D2) a Complete whose reply is lost after the floor committed is resent and answered duplicate, not applied twice", async () => { + const w = W({ infraAttempts: 2 }); w.floor.enqueue(job("1")); w.ax.holdPending.add("wf-test-1-a1") + let dropped = 0 + const fetch: typeof globalThis.fetch = async (input, init) => { + const c = await isComplete(input, init) + const res = await w.floor.fetch(input, init) + if (c && dropped === 0) { dropped++; throw new TypeError("fetch failed: reply lost after the floor committed") } + return res + } + const l = start(w, dir, { pendingTimeoutMs: 150 }, { fetch }) + await until("attempt 2 leased", () => J(w).state === "leased" && J(w).attempt === 2, 4000) + await until("the resend is acknowledged", () => l.logs.some((x) => x.ev === "complete" && x.leaseId === "wf-test-1-a1")) + expect(dropped).toBe(1) + expect(J(w).infraSpent).toBe(1) + expect(l.logs.find((x) => x.ev === "complete" && x.leaseId === "wf-test-1-a1")!.duplicate).toBe(true) + await l.drain() + }) + + // ------------------------------------------------------------------ finding 3: the create window is write-ahead + it("R1-3 (D1) crash after UpdateTask landed, before `created`, then a cancel: the Task is deleted and NotFound precedes `cancelled`", async () => { + const w = W(); w.floor.enqueue(job("1")); w.ax.hangAfterCreate = true + const l1 = start(w, dir) + await until("Task written to ax", () => w.ax.tasks.has("wf-test-1-a1")) + await l1.crash() + w.ax.hangAfterCreate = false + w.floor.cancel("wf-test-1") + const l2 = start(w, dir) + await until("floor done", () => J(w).state === "done") + expect(J(w).result).toBe("cancelled") + expect(w.ax.deletes).toContain("wf-test-1-a1") + expect(w.ax.tasks.has("wf-test-1-a1")).toBe(false) // NotFound before the verdict (reconcile reports on gone) + const evs = l2.logs.filter((x) => x.leaseId === "wf-test-1-a1" || x.task === "wf-test-1-a1").map((x) => x.ev) + expect(evs.indexOf("delete")).toBeLessThan(evs.indexOf("verdict")) + await l2.drain() + expect(Journal.open(dir).recs.has("wf-test-1-a1")).toBe(false) // compacted only once ax is clean + }) + + it("R1-3 (D1 lost variant) crash in the create window, the floor gives up on the job: `lost` deletes the maybe-created Task", async () => { + const w = W({ infraAttempts: 1 }); w.floor.enqueue(job("1")); w.ax.hangAfterCreate = true + const l1 = start(w, dir) + await until("Task written to ax", () => w.ax.tasks.has("wf-test-1-a1")) + await l1.crash() + w.ax.hangAfterCreate = false + await sleep(7 * SEC); w.floor.sweep(); await sleep(16 * SEC); w.floor.sweep() + expect(J(w).state).toBe("done") // budget spent at the first release: no re-grant + const l2 = start(w, dir) + await until("lost", () => l2.has("lost", "wf-test-1-a1")) + await until("Task gone", () => !w.ax.tasks.has("wf-test-1-a1")) + expect(w.ax.deletes).toContain("wf-test-1-a1") + await l2.drain() + }) + + // ------------------------------------------------------------------ findings 4 and 8: B15 + it("R1-8 (B15 deterministic) the first Complete after the floor returns is lost in transit: no Lease runs before it lands", async () => { + const w = W(); w.floor.enqueue(job("1")) + let failCompletes = 0 + const fetch: typeof globalThis.fetch = async (input, init) => { + if (failCompletes > 0 && await isComplete(input, init)) { failCompletes--; throw new TypeError("fetch failed") } + return w.floor.fetch(input, init) + } + const l = start(w, dir, {}, { fetch }) + await until("running", () => w.ax.tasks.get("wf-test-1-a1")?.phase === "Running") + w.floor.down = true + w.ax.finish("wf-test-1-a1", 0, { v: 1 }) + await until("outbox keeps", () => l.has("outbox-keep", "wf-test-1-a1")) + await until("requeued", () => J(w).history.includes("requeue:grace"), 3000) + failCompletes = 1 + const mark = w.floor.calls.length + w.floor.down = false + await until("floor done", () => J(w).state === "done", 4000) + expect([J(w).result, J(w).output, J(w).attempt]).toEqual(["success", { v: 1 }, 1]) + // round 3 (finding 12): the gate is asserted on the floor's side, not on the log line + const after = w.floor.calls.slice(mark), landed = after.findIndex((c) => c.rpc === "Complete") + expect(after.slice(0, landed).filter((c) => c.rpc === "Lease").map((c) => c.capacity).filter((c) => c !== 0)).toEqual([]) + expect(J(w).history).not.toContain("leased:a2") + expect(w.ax.updates.has("wf-test-1-a2")).toBe(false) + expect(l.has("complete-dropped", "wf-test-1-a1")).toBe(false) + await l.drain() + }) + + it("R1-4 (D3) Complete answers 503 while Lease answers: the gate asks for nothing until the success lands", async () => { + const w = W(); w.floor.enqueue(job("1")) + let failComplete = false + const fetch: typeof globalThis.fetch = async (input, init) => { + if (failComplete && await isComplete(input, init)) return new Response("upstream timeout", { status: 503 }) + return w.floor.fetch(input, init) + } + const l = start(w, dir, {}, { fetch }) + await until("running", () => w.ax.tasks.get("wf-test-1-a1")?.phase === "Running") + w.floor.down = true + w.ax.finish("wf-test-1-a1", 0, { answer: 42 }) + await until("verdict written", () => l.has("verdict", "wf-test-1-a1")) + await until("floor requeued attempt 2", () => J(w).state === "queued" && J(w).attempt === 2, 4000) + failComplete = true + const mark = w.floor.calls.length + w.floor.down = false + await sleep(400) // several polls while only Complete fails + expect(l.has("lease-gated-by-outbox")).toBe(true) + // round 3 (finding 12): every Lease while the success is undelivered asks for nothing, and nothing is leased + const leases = w.floor.calls.slice(mark).filter((c) => c.rpc === "Lease") + expect([leases.length > 0, leases.filter((c) => c.capacity !== 0).length, J(w).history.includes("leased:a2")]).toEqual([true, 0, false]) + expect(w.ax.tasks.has("wf-test-1-a2")).toBe(false) + // a1 may be deleted as `lost` (its verdict is already journaled); it is never fenced as `superseded` + expect(l.logs.some((x) => x.ev === "delete" && x.task === "wf-test-1-a1" && x.why === "superseded")).toBe(false) + failComplete = false + await until("floor done", () => J(w).state === "done", 4000) + expect([J(w).result, J(w).output, J(w).attempt]).toEqual(["success", { answer: 42 }, 1]) + await l.drain() + }) + + it("R1-4 (D3 at dispatch) a grant that supersedes an undelivered success is withdrawn, never fenced or created", async () => { + const w = W(); w.floor.enqueue(job("1")) + let failComplete = false + const fetch: typeof globalThis.fetch = async (input, init) => { + if (failComplete && await isComplete(input, init)) return new Response("upstream timeout", { status: 503 }) + return w.floor.fetch(input, init) + } + const l = start(w, dir, {}, { fetch, }) + await until("running", () => w.ax.tasks.get("wf-test-1-a1")?.phase === "Running") + w.floor.down = true + w.ax.finish("wf-test-1-a1", 0, { answer: 42 }) + await until("verdict written", () => l.has("verdict", "wf-test-1-a1")) + await until("floor requeued attempt 2", () => J(w).state === "queued" && J(w).attempt === 2, 4000) + failComplete = true + w.floor.o.overGrant = 1 // a floor that grants on a capacity-0 Lease: the link's own defence must hold + w.floor.down = false + await until("a2 granted", () => l.has("leased", "wf-test-1-a2"), 4000) + w.floor.o.overGrant = 0 + await until("dispatch waits for the verdict", () => l.has("supersede-waits-for-verdict", "wf-test-1-a2")) + failComplete = false + await until("floor done", () => J(w).state === "done", 4000) + expect([J(w).result, J(w).output, J(w).attempt]).toEqual(["success", { answer: 42 }, 1]) + expect(J(w).history).toContain("withdrawn:a2") + await until("a2 withdrawn locally", () => l.has("withdrawn", "wf-test-1-a2")) + expect(w.ax.updates.has("wf-test-1-a2")).toBe(false) + // the finished Task was not fenced before its verdict (a `lost` delete is fine: the verdict is already journaled) + expect(l.logs.some((x) => x.ev === "delete" && x.task === "wf-test-1-a1" && x.why === "superseded")).toBe(false) + await l.drain() + }) + + // ------------------------------------------------------------------ finding 5: supersedes is scoped to own Tasks + it("R1-5 supersedes naming a Task this link never created: no DeleteTask, the grant is refused", async () => { + const ax = new FakeAx() + ax.put(foreign("someone-elses-task"), "Running") + const f = stubFloor([grantOf("1", { attempt: 2, leaseId: "wf-test-1-a2", supersedes: ["someone-elses-task"] })]) + const logs = await runStub(cfgOf(), ax, f.api, 600) + expect(ax.deletes).toEqual([]) + expect(ax.live("someone-elses-task")).toBe(true) + expect(ax.tasks.has("wf-test-1-a2")).toBe(false) + expect(f.completes.map((c) => [c.leaseId, c.output?.reason])).toEqual([["wf-test-1-a2", "pre-start/supersedes-foreign"]]) + expect(logs.some((x) => x.ev === "supersedes-foreign")).toBe(true) + }) + + it("R1-5 an earlier attempt of the same job that carries another holder's mark is not deleted either", async () => { + const ax = new FakeAx() + const built = axTaskFromGrant(grantOf("1"), { atespace: "fleet", image: "ax-agent", gateway: "halogen", command: () => ["ax-agent", "pi"], holder: "other-link" }) + if (built._tag !== "ok") throw new Error("build") + ax.put(built.task, "Running") + const f = stubFloor([grantOf("1", { attempt: 2, leaseId: "wf-test-1-a2", supersedes: ["wf-test-1-a1"] })]) + await runStub(cfgOf(), ax, f.api, 600) + expect(ax.deletes).toEqual([]) + expect(f.completes.map((c) => c.output?.reason)).toEqual(["pre-start/supersedes-foreign"]) + }) + + // ------------------------------------------------------------------ finding 6: the gate is checked per UpdateTask + it("R1-6 the Gateway disappears during the UpdateTask retry window: nothing is created while it is missing", async () => { + const ax = new FakeAx() + let failed = false + const orig = ax.createTask + ax.createTask = (task) => { if (!failed) { failed = true; ax.gateways.delete("halogen"); return Effect.fail(new AxError(UNAVAILABLE, "transient")) } return orig(task) } + const f = stubFloor([grantOf("3")]) + const logs = await runStub(cfgOf(), ax, f.api, 1000) + expect(logs.some((x) => x.ev === "gateway-missing")).toBe(true) + expect(ax.tasks.has("wf-test-3-a1")).toBe(false) + expect(logs.some((x) => x.ev === "gate-closed-before-create")).toBe(true) + }) + + it("R1-6 ax returns without P1 during the retry window: nothing is created in p1 mode", async () => { + const ax = new FakeAx() + let failed = false + const orig = ax.createTask + ax.createTask = (task) => { + if (!failed) { failed = true; ax.up = false; setTimeout(() => { ax.p1 = false; ax.up = true }, 60); return Effect.fail(new AxError(UNAVAILABLE, "transient")) } + return orig(task) + } + const f = stubFloor([grantOf("4")]) + const logs = await runStub(cfgOf(), ax, f.api, 1200) + expect(logs.some((x) => x.ev === "p1-missing")).toBe(true) + expect(ax.tasks.has("wf-test-4-a1")).toBe(false) + }) + + // ------------------------------------------------------------------ finding 7: guest mode fails closed + it("R1-7 guest mode with no leaseToken in the grant: refused before UpdateTask, logged once", async () => { + const ax = new FakeAx(); ax.p1 = false + const f = stubFloor([grantOf("2")]) + const logs = await runStub(cfgOf({ completion: "guest", shape: { atespace: "fleet", image: "ax-agent", gateway: "halogen", command: () => ["ax-agent", "pi"], completeUrl: "http://link.fleet.internal/guest" } }), ax, f.api, 600) + expect(ax.tasks.has("wf-test-2-a1")).toBe(false) + expect(f.completes.map((c) => c.output?.reason)).toEqual(["pre-start/lease-token-missing"]) + expect(logs.filter((x) => x.ev === "guest-token-missing").length).toBe(1) + }) + + // ------------------------------------------------------------------ finding 9: one bad grant is refused alone + it("R1-9 one undecodable grant next to a good one: the good one runs, the bad one is refused, no crash loop", async () => { + const w = W({ skewedEncode: true }) // the floor sends what the link's contract calls malformed + w.floor.enqueue(job("good")) + w.floor.enqueue(job("bad", { "timeout-minutes": 1.5 } as any)) // a float where the contract says Schema.Int + const l = start(w, dir) + await until("good created", () => w.ax.tasks.has("wf-test-good-a1")) + await until("bad refused", () => w.floor.jobs.get("wf-test-bad")!.state === "done") + expect((w.floor.jobs.get("wf-test-bad")!.output as any).reason).toBe("pre-start/invalid-spec") + expect(l.has("grant-invalid", "wf-test-bad-a1")).toBe(true) + await l.drain() + const j = Journal.open(dir) + expect(j.pendingLeaseKey).toBeUndefined() + expect(j.invalid.size).toBe(0) // told and compacted + }) + + it("R1-9 a Lease reply the floor cannot even encode is a typed transient error: the link stays up and heartbeats", async () => { + const w = W() // strict floor: its own encode of the float fails + w.floor.enqueue(job("bad", { "timeout-minutes": 1.5 } as any)) + const l = start(w, dir) + await until("lease errors", () => l.logs.filter((x) => x.ev === "lease-error").length >= 2) + await until("still heartbeating", () => w.floor.calls.filter((c) => c.rpc === "Heartbeat").length >= 2) + expect(l.logs.some((x) => String(x.ev).endsWith("-died"))).toBe(false) + expect(await l.drain()).toBeDefined() // exits on the stop signal, not on a defect + }) + + // ------------------------------------------------------------------ finding 10: configuration fails closed + it("R1-10 numeric and enum configuration is validated: a typo is ConfigInvalid naming the key, never a NaN interval", () => { + const bad = (env: Record) => { try { readLinkEnv(env); return undefined } catch (e) { return e instanceof ConfigInvalid ? e.key : "other" } } + expect(bad({ LINK_RESYNC_SECONDS: "15s" })).toBe("LINK_RESYNC_SECONDS") + expect(bad({ LINK_MAX_IN_FLIGHT: "0" })).toBe("LINK_MAX_IN_FLIGHT") + expect(bad({ LINK_CREATE_ATTEMPTS: "NaN" })).toBe("LINK_CREATE_ATTEMPTS") + expect(bad({ LINK_PENDING_TIMEOUT_SECONDS: "1.5" })).toBe("LINK_PENDING_TIMEOUT_SECONDS") + expect(bad({ LINK_COMPLETION: "gues" })).toBe("LINK_COMPLETION") + expect(bad({ LINK_SEAT_COMMANDS: "{halogen:[ax-agent]}" })).toBe("LINK_SEAT_COMMANDS") + expect(bad({ LINK_SEAT_COMMANDS: '{"halogen":[]}' })).toBe("LINK_SEAT_COMMANDS") + expect(bad({ LINK_OUTBOX_BACKOFF_MIN_SECONDS: "900", LINK_OUTBOX_BACKOFF_MAX_SECONDS: "60" })).toBe("LINK_OUTBOX_BACKOFF_MAX_SECONDS") + const ok = readLinkEnv({}) + expect([ok.resyncSeconds, ok.maxInFlight, ok.createAttempts, ok.completion, ok.seatCommands]).toEqual([15, 2, 7, "auto", { halogen: ["ax-agent", "pi"] }]) + try { readLinkEnv({ LINK_RESYNC_SECONDS: "secret-looking-value" }) } catch (e) { expect(String((e as Error).message)).not.toContain("secret-looking-value") } + }) +}) diff --git a/pkgs/substrate-apps/src/apps/link/test/review-r2.test.ts b/pkgs/substrate-apps/src/apps/link/test/review-r2.test.ts new file mode 100644 index 00000000..1bf2d149 --- /dev/null +++ b/pkgs/substrate-apps/src/apps/link/test/review-r2.test.ts @@ -0,0 +1,423 @@ +// Review round 2 (2026-09-23): the twelve findings, each as a regression test. The scratch repros they restate are in +// /home/tom/today/evals-2026-09-23/link/scratch-r2-{0,1,2}; they failed against c5ed453. The log is link/REVIEW-LOG.md. +import { mkdtempSync, rmSync } from "node:fs" +import { tmpdir } from "node:os" +import { join } from "node:path" +import { Deferred, Effect, Fiber } from "effect" +import { afterEach, beforeEach, describe, expect, it } from "vitest" +import { cidrIsOpen, gatewayAllowsAll } from "../src/ax.ts" +import type { AgentJob } from "../src/contract.ts" +import { rpcFloor } from "../src/floor.ts" +import { Journal } from "../src/journal.ts" +import { isFleetInternalUrl, runLink } from "../src/link.ts" +import type { LinkConfig } from "../src/link.ts" +import { FakeAx } from "./fake-ax.ts" +import { FakeFloor } from "./fake-floor.ts" +import type { FloorConfig } from "./fake-floor.ts" + +const SEC = 20 +const A = "ultracode.mecattaf.dev/" +const JK = `${"cd".repeat(32)}:1` // round 4: FIELD-MAP 5a journal key (jobs.ts refuses a grant without it) +const job = (n: string): AgentJob => ({ + apiVersion: "ultracode.mecattaf.dev/v1alpha1", kind: "AgentJob", + metadata: { name: `wf-test-${n}`, labels: { [A + "run-id"]: "wf-test", [A + "workflow"]: "link-test", [A + "phase-index"]: "1" }, + annotations: { [A + "run-id-raw"]: "wf_test", [A + "label"]: `probe:${n}`, [A + "item-key"]: `wf_test#${n}`, [A + "journal-key"]: JK, [A + "phase-title"]: "Probe" } }, + spec: { "runs-on": ["seat:halogen", "runtime:gvisor"], + with: { prompt: `say ${n}`, prompt_ref: { sha256: "ab".repeat(32), bytes: 5, uri: `journal://wf_test/${n}/prompt.md` }, model: "halogen-qwen3.8-flash-next" } } +}) +const world = (o: Partial = {}) => ({ + floor: new FakeFloor({ cap: 2, leaseSeconds: 6, graceSeconds: 15, pollSeconds: 1, heartbeatSeconds: 2, maxAttempts: 3, secondMs: SEC, tokens: { "tok-nas": "nas-link-1" }, ...o }), + ax: new FakeAx() +}) +type World = ReturnType +function start(w: World, dir: string, over: Partial = {}, fetch?: typeof globalThis.fetch) { + const logs: Array> = [] + const stop = Effect.runSync(Deferred.make()) + const cfg: LinkConfig = { + holder: "nas-link-1", maxInFlight: 2, servedLabels: ["seat:halogen", "runtime:gvisor"], + shape: { atespace: "fleet", image: "ax-agent", gateway: "halogen", command: () => ["ax-agent", "pi"] }, + completion: "auto", secondMs: SEC, resyncMs: 30, pendingTimeoutMs: 1500, deleteAfterMs: 0, deadlineBackstopMs: 300, + createAttempts: 3, outboxBackoffMs: [20, 100], initialPollSeconds: 1, initialHeartbeatSeconds: 2, fenceTimeoutMs: 300, resultReadTries: 3, ...over + } + const fiber = Effect.runFork(Effect.scoped(Effect.gen(function*() { + const floor = yield* rpcFloor({ url: "http://floor.test", token: "tok-nas", sessionId: `s:${dir}`, fetch: fetch ?? w.floor.fetch }) + return yield* runLink(cfg, { ax: w.ax, floor, journal: Journal.open(dir), log: (ev, f) => logs.push({ t: Date.now(), ev, ...f }), stop, floorUrl: "http://floor.test" }) + }))) + return { + logs, fiber, + has: (ev: string, leaseId?: string) => logs.some((l) => l.ev === ev && (leaseId === undefined || l.leaseId === leaseId)), + trace: () => logs.filter((x) => typeof x.leaseId === "string" || typeof x.task === "string").map((x) => `${x.ev}:${x.leaseId ?? x.task}${x.why ? ":" + x.why : ""}`), + crash: () => Effect.runPromise(Fiber.interrupt(fiber)), + drain: () => { Effect.runSync(Deferred.succeed(stop, undefined)); return Promise.race([Effect.runPromise(Fiber.await(fiber)), sleep(1500)]) } + } +} +const until = async (what: string, pred: () => boolean, ms = 5000) => { + const t0 = Date.now() + while (!pred()) { if (Date.now() - t0 > ms) throw new Error(`timeout waiting for: ${what}`); await new Promise((r) => setTimeout(r, 5)) } +} +const sleep = (ms: number) => new Promise((r) => setTimeout(r, ms)) +const J = (w: World, n = "1") => w.floor.jobs.get(`wf-test-${n}`)! +const bodyOf = async (input: unknown, init?: RequestInit) => { + const b = init?.body + if (typeof b === "string") return b + if (b instanceof Uint8Array) return new TextDecoder().decode(b) + if (b) return await new Response(b as ConstructorParameters[0]).text() + return (input instanceof Request) ? await input.clone().text() : "" +} +const rpcOf = async (input: unknown, init?: RequestInit) => { const b = await bodyOf(input, init); return b.includes('"Complete"') ? "Complete" : b.includes('"Lease"') ? "Lease" : b.includes('"Heartbeat"') ? "Heartbeat" : "?" } +/** A capacity race: Failed with ResourceExhausted, which the link maps to retryable pre-start/resource-exhausted. */ +const failRetryable = (w: World, name: string) => { + const t = w.ax.tasks.get(name)! + t.exited = true; t.phase = "Failed" + t.conditions = [{ type: "Ready", status: "False", reason: "ResourceExhausted", message: "ResourceExhausted: no free worker" }] +} +const why = (w: World, l: ReturnType) => JSON.stringify({ floor: J(w).history, link: l.trace() }) + +// A floor stub for the fail-closed tests (scratch-r2-1/repro.mts): hands out its grants once, renews everything. +const stats = { seq: 1, cap: 2, wip: 0, queued: 0, done: 0 } +const grantOf = (n: string, extra: Record = {}) => ({ leaseId: `wf-test-${n}-a1`, attempt: 1, job: job(n), + lease: { holderIdentity: "nas-link-1", leaseDurationSeconds: 30, acquireTime: Date.now(), renewTime: Date.now(), leaseTransitions: 0 }, ...extra }) +const stubFloor = (grants: Array, gate: () => boolean = () => true) => { + let sent = false + return { + lease: (p: any) => Effect.sync(() => { const g = !sent && p.capacity > 0 && gate() ? grants : []; if (g.length) sent = true; return { grants: g, invalid: [], stats, nextPollSeconds: 1, heartbeatSeconds: 2 } }), + heartbeat: (p: any) => Effect.succeed({ renewed: [...p.leaseIds], lost: [], cancelRequested: [], stats }), + complete: (_p: any) => Effect.succeed({ duplicate: false, stats }) + } +} +const runStub = async (over: Partial, ax: FakeAx, floor: any, ms: number) => { + const dir = mkdtempSync(join(tmpdir(), "conwip-link-r2s-")) + const logs: Array> = [] + const stop = Effect.runSync(Deferred.make()) + const cfg: LinkConfig = { holder: "nas-link-1", maxInFlight: 2, servedLabels: ["seat:halogen", "runtime:gvisor"], + shape: { atespace: "fleet", image: "ax-agent", gateway: "halogen", command: () => ["ax-agent", "pi"] }, + completion: "auto", secondMs: SEC, resyncMs: 30, pendingTimeoutMs: 60_000, deleteAfterMs: 60_000, deadlineBackstopMs: 300, + createAttempts: 3, outboxBackoffMs: [20, 100], initialPollSeconds: 1, initialHeartbeatSeconds: 2, ...over } + const f = Effect.runFork(runLink(cfg, { ax, floor, journal: Journal.open(dir), log: (ev, x) => logs.push({ t: Date.now(), ev, ...x }), stop })) + await sleep(ms) + Effect.runSync(Deferred.succeed(stop, undefined)); await Effect.runPromise(Fiber.await(f)); rmSync(dir, { recursive: true, force: true }) + return logs +} + +describe("review round 2: fixed", () => { + let dir: string + let worlds: Array = [] + const W = (o: Partial = {}) => { const w = world(o); worlds.push(w); return w } + beforeEach(() => { dir = mkdtempSync(join(tmpdir(), "conwip-link-r2-")) }) + afterEach(() => { for (const w of worlds) w.floor.close(); worlds = []; rmSync(dir, { recursive: true, force: true }) }) + + // ---------------------------------------------------------------- R2-1: heartbeat `lost` and the janitor read first + it("R2-1a kill -9, the agent finishes, the floor requeues past grace, restart while ax is down: the success is delivered, a2 never runs", async () => { + const w = W(); w.floor.enqueue(job("1")) + const l1 = start(w, dir) + await until("running", () => w.ax.tasks.get("wf-test-1-a1")?.phase === "Running") + await l1.crash() + w.ax.finish("wf-test-1-a1", 0, { answer: 42 }) + await until("requeued by the alarm", () => J(w).history.includes("requeue:grace"), 3000) + w.ax.up = false + const l2 = start(w, dir) + await until("heartbeat says lost", () => l2.has("lost", "wf-test-1-a1")) + w.ax.up = true + await until("floor settles", () => J(w).state === "done" || w.ax.updates.has("wf-test-1-a2"), 4000) + await sleep(200) + expect(w.ax.updates.has("wf-test-1-a2"), why(w, l2)).toBe(false) + expect([J(w).result, J(w).output, J(w).attempt]).toEqual(["success", { answer: 42 }, 1]) + expect(l2.logs.findIndex((x) => x.ev === "verdict")).toBeLessThan(l2.logs.findIndex((x) => x.ev === "delete" && x.task === "wf-test-1-a1")) + await l2.drain() + }) + + it("R2-1b no restart: the agent finishes as the partition heals and the heartbeat's lost beats the resync: the success is delivered", async () => { + const w = W(); w.floor.enqueue(job("1")) + const l = start(w, dir, { resyncMs: 300 }) + await until("running", () => w.ax.tasks.get("wf-test-1-a1")?.phase === "Running") + w.floor.down = true + await until("requeued by the alarm", () => J(w).history.includes("requeue:grace"), 3000) + w.ax.finish("wf-test-1-a1", 0, { answer: 42 }) + w.floor.down = false + await until("floor settles", () => J(w).state === "done" || w.ax.updates.has("wf-test-1-a2"), 4000) + await sleep(200) + expect(w.ax.updates.has("wf-test-1-a2"), why(w, l)).toBe(false) + expect([J(w).result, J(w).output, J(w).attempt]).toEqual(["success", { answer: 42 }, 1]) + await l.drain() + }) + + // ---------------------------------------------------------------- R2-2: the supersede fence reads first + it("R2-2 the Lease wins the race (heartbeats failing): a2's fence reads a1's Completed result, the floor withdraws a2", async () => { + const w = W(); w.floor.enqueue(job("1")) + let failHeartbeat = false + const fetch: typeof globalThis.fetch = async (input, init) => { + if (failHeartbeat && await rpcOf(input, init) === "Heartbeat") throw new TypeError("fetch failed") + return w.floor.fetch(input, init) + } + const l = start(w, dir, { resyncMs: 300 }, fetch) + await until("running", () => w.ax.tasks.get("wf-test-1-a1")?.phase === "Running") + w.floor.down = true + await until("requeued by the alarm", () => J(w).history.includes("requeue:grace"), 3000) + failHeartbeat = true + w.ax.finish("wf-test-1-a1", 0, { answer: 42 }) + w.floor.down = false + await until("floor settles", () => J(w).state === "done" || w.ax.updates.has("wf-test-1-a2"), 4000) + failHeartbeat = false + await sleep(200) + expect(w.ax.updates.has("wf-test-1-a2"), why(w, l)).toBe(false) + expect([J(w).result, J(w).output, J(w).attempt]).toEqual(["success", { answer: 42 }, 1]) + await l.drain() + }) + + // ---------------------------------------------------------------- R2-3: one verdict never wedges the link + it("R2-3a a result over the link's output cap is refused agent/output-too-large; later jobs are leased", async () => { + const w = W({ cap: 4 }); w.floor.enqueue(job("1")) + const l = start(w, dir, { maxInFlight: 2, outboxBackoffMs: [20, 60] }) + await until("a1 running", () => w.ax.tasks.get("wf-test-1-a1")?.phase === "Running") + w.ax.finish("wf-test-1-a1", 0, { report: "x".repeat(2 * 1024 * 1024 + 1000) }) + await until("verdict written", () => l.has("verdict", "wf-test-1-a1")) + w.floor.enqueue(job("2")); w.floor.enqueue(job("3")) + await until("job 2 leased", () => J(w, "2").state !== "queued", 3000) + expect([J(w).result, (J(w).output as any)?.reason]).toEqual(["failure", "agent/output-too-large"]) + await l.drain() + }) + + it("R2-3b a verdict the floor answers 500 every time is replaced by infra/verdict-undeliverable, and never gates Lease", async () => { + const w = W({ cap: 4 }); w.floor.enqueue(job("1")) + let refused = 0 + const fetch: typeof globalThis.fetch = async (input, init) => { + const b = await bodyOf(input, init) + if (b.includes('"Complete"') && b.includes('"success"')) { refused++; return new Response("internal error", { status: 500 }) } + return w.floor.fetch(input, init) + } + const l = start(w, dir, { outboxBackoffMs: [20, 60], verdictAttempts: 4 }, fetch) + await until("a1 running", () => w.ax.tasks.get("wf-test-1-a1")?.phase === "Running") + w.ax.finish("wf-test-1-a1", 0, { answer: 42 }) + await until("verdict written", () => l.has("verdict", "wf-test-1-a1")) + w.floor.enqueue(job("2")) + await until("job 2 leased", () => J(w, "2").state !== "queued", 3000) + await until("job 1 closed", () => J(w).state === "done", 3000) + expect([J(w).result, (J(w).output as any)?.reason]).toEqual(["failure", "infra/verdict-undeliverable"]) + // critique pass 2026-09-24 (durability-r2-8): a try is charged only when a Lease or Heartbeat succeeded since the + // previous one, so the ceiling of 4 charges takes at least 4 refusals (the first try is never charged) + expect(refused).toBeGreaterThanOrEqual(4) + expect(l.has("verdict-replaced", "wf-test-1-a1")).toBe(true) + await l.drain() + }) + + it("R2-3c an RPC Defect reply (SQLITE_TOOBIG) is a server-error, not an encode defect", async () => { + const fetch: typeof globalThis.fetch = async (input, init) => { + const req = JSON.parse(await bodyOf(input, init)), id = (Array.isArray(req) ? req[0] : req).id + return new Response(JSON.stringify([{ _tag: "Exit", requestId: id, exit: { _tag: "Failure", cause: [{ _tag: "Die", defect: { message: "string or blob too big: SQLITE_TOOBIG" } }] } }]), { status: 200, headers: { "content-type": "application/json" } }) + } + const e = await Effect.runPromise(Effect.scoped(Effect.gen(function*() { + const floor = yield* rpcFloor({ url: "http://floor.test", token: "t", sessionId: "s", fetch }) + return yield* floor.complete({ leaseId: "x-a1", attempt: 1, result: "success", output: {} }).pipe(Effect.flip) + }))) + expect([e._tag, (e as any).kind, /SQLITE_TOOBIG/.test((e as any).message ?? "")]).toEqual(["FloorError", "server-error", true]) + }) + + it("R2-3d Complete classification: 413 and 422 drop the verdict, 404 is fatal misrouted (verdict kept), 500 is server-error, 503 transient", async () => { + const out: Record = {} + for (const status of [413, 422, 404, 500, 503, 429]) { + const e = await Effect.runPromise(Effect.scoped(Effect.gen(function*() { + const floor = yield* rpcFloor({ url: "http://floor.test", token: "t", sessionId: "s", fetch: async () => new Response("x", { status, headers: { "retry-after": "7" } }) }) + return yield* floor.complete({ leaseId: "x-a1", attempt: 1, result: "success" }).pipe(Effect.flip) + }))) + out[status] = e._tag === "CompleteRefused" ? `refused:${e.code}` : `${e.kind}:${e.fatal}${e.kind === "rate-limited" ? `:${e.retryAfterMs}` : ""}` + } + // round 3 (finding 13): 429 is rate-limited with its Retry-After, the verdict kept (not `rejected`, which drops it) + expect(out).toEqual({ 413: "refused:http-413", 422: "refused:http-422", 404: "misrouted:true", 500: "server-error:false", 503: "transient:false", 429: "rate-limited:false:7000" }) + }) + + // ---------------------------------------------------------------- R2-4: withdrawal is explicit on the wire + it("R2-4a a Lease crosses a1's retryable verdict, whose reply is lost; the resend answers duplicate: a2 is created", async () => { + const w = W(); w.floor.enqueue(job("1")) + let holdLease = false, loseCompleteReply = false, failComplete = false + const fetch: typeof globalThis.fetch = async (input, init) => { + const rpc = await rpcOf(input, init) + if (rpc === "Lease") while (holdLease) await sleep(2) + if (rpc === "Complete" && failComplete) return new Response("upstream timeout", { status: 503 }) + if (rpc === "Complete" && loseCompleteReply) { await w.floor.fetch(input, init); loseCompleteReply = false; failComplete = true; throw new TypeError("fetch failed") } + return w.floor.fetch(input, init) + } + const l = start(w, dir, {}, fetch) + await until("running", () => w.ax.tasks.get("wf-test-1-a1")?.phase === "Running") + holdLease = true + await sleep(60) + loseCompleteReply = true + failRetryable(w, "wf-test-1-a1") + await until("floor applied a1 and requeued a2", () => J(w).state === "queued" && J(w).attempt === 2, 4000) + holdLease = false + await until("a2 granted", () => l.has("leased", "wf-test-1-a2"), 4000) + await until("dispatch waits for the verdict", () => l.has("supersede-waits-for-verdict", "wf-test-1-a2")) + failComplete = false + await until("a2 created or withdrawn", () => w.ax.updates.has("wf-test-1-a2") || l.has("withdrawn", "wf-test-1-a2"), 4000) + await sleep(300) + expect(l.has("withdrawn", "wf-test-1-a2"), why(w, l)).toBe(false) + expect(w.ax.updates.get("wf-test-1-a2")).toBe(1) + expect(J(w).history).not.toContain("requeue:omitted") + await l.drain() + }) + + it("R2-4b the same race with an infrastructure budget of 2: the retry runs once, the job is not ended retry-budget-spent", async () => { + const w = W({ infraAttempts: 2 }); w.floor.enqueue(job("1")) + let armed = false, dropped = 0, committed!: () => void + const done = new Promise((r) => { committed = r }) + const fetch: typeof globalThis.fetch = async (input, init) => { + const body = await bodyOf(input, init) + if (armed && body.includes('"Lease"') && /"capacity":[1-9]/.test(body)) { armed = false; await done } + if (body.includes('"Complete"') && body.includes("wf-test-1-a1") && dropped > 0) await sleep(200) + const res = await w.floor.fetch(input, init) + if (body.includes('"Complete"') && body.includes("wf-test-1-a1") && dropped === 0) { dropped++; committed(); throw new TypeError("fetch failed: reply lost") } + return res + } + const l = start(w, dir, { outboxBackoffMs: [150, 150], fenceTimeoutMs: 3000 }, fetch) + await until("a1 running", () => w.ax.tasks.get("wf-test-1-a1")?.phase === "Running") + armed = true + await until("a Lease with capacity is held", () => !armed, 3000) + failRetryable(w, "wf-test-1-a1") + await until("a2 leased", () => l.has("leased", "wf-test-1-a2"), 4000) + await until("a2 created", () => w.ax.updates.has("wf-test-1-a2"), 4000).catch(() => {}) + expect(w.ax.updates.get("wf-test-1-a2") ?? 0, why(w, l)).toBe(1) + expect(l.has("withdrawn", "wf-test-1-a2")).toBe(false) + expect((J(w).output as any)?.reason).not.toBe("infra/retry-budget-spent") + await l.drain() + }) + + // ---------------------------------------------------------------- R2-5: a regrant of a withdrawn leaseId is new work + it("R2-5a rule 4b withdraws a2 for a1's retryable verdict and the floor grants a2 again: the new generation is created", async () => { + const w = W(); w.floor.enqueue(job("1")) + let holdLease = false, failHeartbeat = false, failComplete = false + const fetch: typeof globalThis.fetch = async (input, init) => { + const rpc = await rpcOf(input, init) + if (rpc === "Heartbeat" && failHeartbeat) throw new TypeError("fetch failed") + if (rpc === "Complete" && failComplete) return new Response("upstream timeout", { status: 503 }) + if (rpc === "Lease") while (holdLease) await sleep(2) + return w.floor.fetch(input, init) + } + const l = start(w, dir, {}, fetch) + await until("running", () => w.ax.tasks.get("wf-test-1-a1")?.phase === "Running") + w.floor.down = true + await until("floor requeued attempt 2", () => J(w).state === "queued" && J(w).attempt === 2, 4000) + failHeartbeat = true; holdLease = true; failComplete = true + w.floor.down = false + await sleep(80) + failRetryable(w, "wf-test-1-a1") + await until("verdict written", () => l.has("verdict", "wf-test-1-a1")) + holdLease = false + await until("a2 received", () => l.has("leased", "wf-test-1-a2"), 4000) + await until("dispatch waits for the verdict", () => l.has("supersede-waits-for-verdict", "wf-test-1-a2")) + failComplete = false; failHeartbeat = false + await until("a2 created, or the floor gives up on a2", () => w.ax.updates.has("wf-test-1-a2") || J(w).attempt >= 3 || J(w).state === "done", 4000).catch(() => {}) + expect(w.ax.updates.get("wf-test-1-a2") ?? 0, why(w, l)).toBe(1) + expect(l.has("duplicate-grant", "wf-test-1-a2")).toBe(false) + await l.drain() + }) + + it("R2-5b the Lease reply carrying a2 is slow; a1's retryable verdict lands first and withdraws it; the regrant is created", async () => { + const w = W(); w.floor.enqueue(job("1")) + let holdLeaseReply = false, failHeartbeat = false + const fetch: typeof globalThis.fetch = async (input, init) => { + const rpc = await rpcOf(input, init) + if (rpc === "Heartbeat" && failHeartbeat) throw new TypeError("fetch failed") + if (rpc === "Lease" && holdLeaseReply) { const r = await w.floor.fetch(input, init); while (holdLeaseReply) await sleep(2); return r } + return w.floor.fetch(input, init) + } + const l = start(w, dir, {}, fetch) + await until("running", () => w.ax.tasks.get("wf-test-1-a1")?.phase === "Running") + w.floor.down = true + await until("floor requeued attempt 2", () => J(w).state === "queued" && J(w).attempt === 2, 4000) + failHeartbeat = true; holdLeaseReply = true + w.floor.down = false + await until("floor leased a2 to this link", () => J(w).state === "leased" && J(w).attempt === 2, 4000) + failRetryable(w, "wf-test-1-a1") + await until("verdict written", () => l.has("verdict", "wf-test-1-a1")) + holdLeaseReply = false + await until("a2 received", () => l.has("leased", "wf-test-1-a2"), 4000) + failHeartbeat = false + await until("a2 created, or the floor gives up", () => w.ax.updates.has("wf-test-1-a2") || J(w).attempt >= 3 || J(w).state === "done", 4000).catch(() => {}) + expect(w.ax.updates.get("wf-test-1-a2") ?? 0, why(w, l)).toBe(1) + await l.drain() + }) + + // ---------------------------------------------------------------- R2-6: redirects are never followed + it("R2-6 the floor answers 307 to another origin: nothing is leased or created and the link stops fatal (redirect)", async () => { + const w = W(); w.floor.enqueue(job("1")) + let rogue = 0 + const fetch: typeof globalThis.fetch = async (input, init) => { + if ((init as RequestInit | undefined)?.redirect !== "manual") { rogue++; return w.floor.fetch(input, init) } // a follower would land on the rogue origin + return new Response(null, { status: 307, headers: { location: "http://127.0.0.1:9/rpc" } }) + } + const l = start(w, dir, {}, fetch) + const exit = await Promise.race([Effect.runPromise(Fiber.await(l.fiber)), sleep(1500).then(() => undefined)]) + expect([rogue, w.ax.tasks.size, l.has("leased")]).toEqual([0, 0, false]) + expect(exit?._tag).toBe("Failure") + expect(JSON.stringify(exit)).toContain("redirect") + }) + + // ---------------------------------------------------------------- R2-7, R2-8: fail-closed checks + it("R2-7 isFleetInternalUrl matches private ranges on IP literals only", () => { + for (const u of ["https://10.evil.example/guest", "https://127.0.0.1.nip.io/guest", "https://192.168.0.1.attacker.com/c", "https://link/guest", "https://x.workers.dev/", "https://10.internal.example.com/"]) + expect([u, isFleetInternalUrl(u)]).toEqual([u, false]) + for (const u of ["http://10.201.0.80:8080/c", "http://127.0.0.1/c", "http://100.64.0.7/c", "http://[fd7a:115c:a1e0::1]/c", "http://[::1]/c", "http://link.fleet.internal/guest", "http://nas.lan/c", "http://localhost:1/c", "http://0x7f.1/c"]) + expect([u, isFleetInternalUrl(u)]).toEqual([u, true]) + expect(isFleetInternalUrl("http://link/guest", ["link"])).toBe(true) + }) + + it("R2-7b guest mode with a public name that starts with a private prefix: no Task, no token handed out", async () => { + const ax = new FakeAx(); ax.p1 = false + const logs = await runStub({ completion: "guest", shape: { atespace: "fleet", image: "ax-agent", gateway: "halogen", command: () => ["ax-agent", "pi"], completeUrl: "https://10.evil.example/guest" } }, + ax, stubFloor([grantOf("g", { leaseToken: "tok" })]), 400) + expect([logs.some((l) => l.ev === "guest-url-invalid"), ax.tasks.has("wf-test-g-a1")]).toEqual([true, false]) + }) + + it("R2-8 a Gateway open by CIDR arithmetic is refused: ::/0, 0.0.0.0/1 + 128.0.0.0/1, and unparseable CIDRs", async () => { + const gw = (hosts: Array) => ({ hasAllowlist: true, hosts: hosts.map((host) => ({ host, port: 443 })) }) + expect(gatewayAllowsAll(gw(["worker", "::/0"]))).toBe(true) + expect(gatewayAllowsAll(gw(["0.0.0.0/1", "128.0.0.0/1"]))).toBe(true) + expect(gatewayAllowsAll(gw([" * "]))).toBe(true) + expect(gatewayAllowsAll(gw(["10.0.0.0/x"]))).toBe(true) + expect(gatewayAllowsAll(gw(["worker", "10.201.0.0/24", "fd7a:115c:a1e0::/48"]))).toBe(false) + expect([cidrIsOpen("8.0.0.0/8"), cidrIsOpen("10.1.0.0/16"), cidrIsOpen("2000::/3"), cidrIsOpen("fd00::/64")]).toEqual([true, false, true, false]) + const ax = new FakeAx() + ax.gateways.set("halogen", gw(["worker", "::/0"])) + const logs = await runStub({}, ax, stubFloor([grantOf("w")]), 400) + // critique D.2 item 5: an existing Gateway that allows everything is logged gateway-open, not gateway-missing + expect([logs.some((l) => l.ev === "gateway-open"), ax.tasks.has("wf-test-w-a1")]).toEqual([true, false]) + }) + + // ---------------------------------------------------------------- R2-9: nothing from before an ax outage opens the gate + it("R2-9 guest mode: ax returns without the Gateway and GetGateway is slow; nothing is created on pre-outage state", async () => { + const ax = new FakeAx(); ax.p1 = false + const origGw = ax.getGateway + let outageOver = false, release = false + ax.getGateway = (name: string) => outageOver ? Effect.sleep(300).pipe(Effect.andThen(origGw(name))) : origGw(name) + setTimeout(() => { ax.up = false }, 150) + setTimeout(() => { release = true }, 170) + setTimeout(() => { ax.gateways.delete("halogen"); outageOver = true; ax.up = true }, 400) + const logs = await runStub({ completion: "guest", shape: { atespace: "fleet", image: "ax-agent", gateway: "halogen", command: () => ["ax-agent", "pi"], completeUrl: "http://link.fleet.internal/guest" } }, + ax, stubFloor([grantOf("u", { leaseToken: "tok" })], () => release), 1200) + expect(ax.tasks.has("wf-test-u-a1"), JSON.stringify(logs.map((l) => l.ev))).toBe(false) + expect(logs.some((l) => l.ev === "gateway-missing")).toBe(true) + }) + + // ---------------------------------------------------------------- R2-11: a permanent Complete refusal is dropped + for (const variant of ["http-413", "unknown-error-code"] as const) + it(`R2-11 one Complete refused for good (${variant}) is dropped and job 2 is leased`, async () => { + const w = W(); w.floor.enqueue(job("1")) + const fetch: typeof globalThis.fetch = async (input, init) => { + const body = await bodyOf(input, init) + if (body.includes('"Complete"') && body.includes("wf-test-1-a1")) { + if (variant === "http-413") return new Response("payload too large", { status: 413 }) + const req = JSON.parse(body), id = (Array.isArray(req) ? req[0] : req).id + return new Response(JSON.stringify([{ _tag: "Exit", requestId: id, exit: { _tag: "Failure", cause: [{ _tag: "Fail", error: { code: "output-too-large" } }] } }]), { status: 200, headers: { "content-type": "application/json" } }) + } + return w.floor.fetch(input, init) + } + const l = start(w, dir, { outboxBackoffMs: [20, 40] }, fetch) + await until("a1 running", () => w.ax.tasks.get("wf-test-1-a1")?.phase === "Running") + w.ax.finish("wf-test-1-a1", 0, { big: "x" }) + await until("verdict", () => l.has("verdict", "wf-test-1-a1")) + w.floor.enqueue(job("2")) + await until("job 2 leased", () => J(w, "2").state !== "queued", 3000) + const d = l.logs.find((x) => x.ev === "complete-dropped" && x.leaseId === "wf-test-1-a1") + expect(d?.code).toBe(variant === "http-413" ? "http-413" : "output-too-large") + await l.drain() + }) +}) diff --git a/pkgs/substrate-apps/src/apps/link/test/review-r3.test.ts b/pkgs/substrate-apps/src/apps/link/test/review-r3.test.ts new file mode 100644 index 00000000..057b6faa --- /dev/null +++ b/pkgs/substrate-apps/src/apps/link/test/review-r3.test.ts @@ -0,0 +1,484 @@ +// Review round 3 (2026-09-23): the thirteen findings, each as a regression test. The scratch repros they restate are +// in /home/tom/today/evals-2026-09-23/link/scratch-r3-{0,1,2}; they failed against d582ead. The log is +// link/REVIEW-LOG.md. Test adequacy findings 11 to 13 also strengthen R1-4, R1-8 and R2-3d in place. +import { appendFileSync, mkdtempSync, rmSync } from "node:fs" +import { tmpdir } from "node:os" +import { join } from "node:path" +import { Deferred, Effect, Fiber } from "effect" +import { afterEach, beforeEach, describe, expect, it } from "vitest" +import { AxError } from "../src/ax.ts" +import type { AgentJob } from "../src/contract.ts" +import { rpcFloor } from "../src/floor.ts" +import { Journal } from "../src/journal.ts" +import { boundedSeconds, isFleetInternalUrl, runLink } from "../src/link.ts" +import type { LinkConfig } from "../src/link.ts" +import { FakeAx } from "./fake-ax.ts" +import { FakeFloor } from "./fake-floor.ts" +import type { FloorConfig } from "./fake-floor.ts" + +const SEC = 20 +const A = "ultracode.mecattaf.dev/" +const JK = `${"cd".repeat(32)}:1` // round 4: FIELD-MAP 5a journal key (jobs.ts refuses a grant without it) +const job = (n: string, spec: Record = {}): AgentJob => ({ + apiVersion: "ultracode.mecattaf.dev/v1alpha1", kind: "AgentJob", + metadata: { name: `wf-test-${n}`, labels: { [A + "run-id"]: "wf-test", [A + "workflow"]: "link-test", [A + "phase-index"]: "1" }, + annotations: { [A + "run-id-raw"]: "wf_test", [A + "label"]: `probe:${n}`, [A + "item-key"]: `wf_test#${n}`, [A + "journal-key"]: JK, [A + "phase-title"]: "Probe" } }, + spec: { "runs-on": ["seat:halogen", "runtime:gvisor"], + with: { prompt: `say ${n}`, prompt_ref: { sha256: "ab".repeat(32), bytes: 5, uri: `journal://wf_test/${n}/prompt.md` }, model: "halogen-qwen3.8-flash-next" }, ...spec } as AgentJob["spec"] +}) +const world = (o: Partial = {}) => ({ + floor: new FakeFloor({ cap: 2, leaseSeconds: 6, graceSeconds: 15, pollSeconds: 1, heartbeatSeconds: 2, maxAttempts: 3, secondMs: SEC, tokens: { "tok-nas": "nas-link-1" }, ...o }), + ax: new FakeAx() +}) +type World = ReturnType +function start(w: World, dir: string, over: Partial = {}, fetch?: typeof globalThis.fetch) { + const logs: Array> = [] + const stop = Effect.runSync(Deferred.make()) + const cfg: LinkConfig = { + holder: "nas-link-1", maxInFlight: 2, servedLabels: ["seat:halogen", "runtime:gvisor"], + shape: { atespace: "fleet", image: "ax-agent", gateway: "halogen", command: () => ["ax-agent", "pi"] }, + completion: "auto", secondMs: SEC, resyncMs: 30, pendingTimeoutMs: 1500, deleteAfterMs: 0, deadlineBackstopMs: 300, + createAttempts: 3, outboxBackoffMs: [20, 100], initialPollSeconds: 1, initialHeartbeatSeconds: 2, fenceTimeoutMs: 300, resultReadTries: 3, ...over + } + const fiber = Effect.runFork(Effect.scoped(Effect.gen(function*() { + const floor = yield* rpcFloor({ url: "http://floor.test", token: "tok-nas", sessionId: `s:${dir}`, fetch: fetch ?? w.floor.fetch }) + return yield* runLink(cfg, { ax: w.ax, floor, journal: Journal.open(dir), log: (ev, f) => logs.push({ t: Date.now(), ev, ...f }), stop, floorUrl: "http://floor.test" }) + }))) + return { + logs, fiber, + has: (ev: string, leaseId?: string) => logs.some((l) => l.ev === ev && (leaseId === undefined || l.leaseId === leaseId)), + trace: () => logs.filter((x) => typeof x.leaseId === "string" || typeof x.task === "string").map((x) => `${x.ev}:${x.leaseId ?? x.task}${x.why ? ":" + x.why : ""}`), + crash: () => Effect.runPromise(Fiber.interrupt(fiber)), + drain: () => { Effect.runSync(Deferred.succeed(stop, undefined)); return Promise.race([Effect.runPromise(Fiber.await(fiber)), sleep(1500)]) } + } +} +const until = async (what: string, pred: () => boolean, ms = 5000) => { + const t0 = Date.now() + while (!pred()) { if (Date.now() - t0 > ms) throw new Error(`timeout waiting for: ${what}`); await new Promise((r) => setTimeout(r, 5)) } +} +const sleep = (ms: number) => new Promise((r) => setTimeout(r, ms)) +const J = (w: World, n = "1") => w.floor.jobs.get(`wf-test-${n}`)! +const bodyOf = async (input: unknown, init?: RequestInit) => { + const b = init?.body + if (typeof b === "string") return b + if (b instanceof Uint8Array) return new TextDecoder().decode(b) + if (b) return await new Response(b as ConstructorParameters[0]).text() + return (input instanceof Request) ? await input.clone().text() : "" +} +const rpcOf = async (input: unknown, init?: RequestInit) => { const b = await bodyOf(input, init); return b.includes('"Complete"') ? "Complete" : b.includes('"Lease"') ? "Lease" : b.includes('"Heartbeat"') ? "Heartbeat" : "?" } +const failRetryable = (w: World, name: string) => { + const t = w.ax.tasks.get(name)! + t.exited = true; t.phase = "Failed" + t.conditions = [{ type: "Ready", status: "False", reason: "ResourceExhausted", message: "ResourceExhausted: no free worker" }] +} +const why = (w: World, l: ReturnType) => JSON.stringify({ floor: J(w).history, link: l.trace() }) +/** Rewrite every JSON reply of one RPC with `patch` (a floor whose wire drifted from the link's). */ +const rewriting = (w: World, rpc: string, patch: (x: any) => any): typeof globalThis.fetch => async (input, init) => { + const b = await bodyOf(input, init) + const res = await w.floor.fetch(input, init) + if (!b.includes(`"${rpc}"`)) return res + const txt = await res.text() + let j: unknown; try { j = JSON.parse(txt) } catch { return new Response(txt, { status: res.status, headers: res.headers }) } + return new Response(JSON.stringify(patch(j)), { status: res.status, headers: { "content-type": "application/json" } }) +} +const deep = (f: (o: Record) => void) => { const walk = (x: any): any => { + if (Array.isArray(x)) return x.map(walk) + if (x && typeof x === "object") { const o: Record = {}; for (const [k, v] of Object.entries(x)) o[k] = walk(v); f(o); return o } + return x +}; return walk } + +// A floor stub that grants one lease named after a Task someone else owns (scratch-r3-1). +const stats = { seq: 1, cap: 2, wip: 0, queued: 0, done: 0 } +const foreignTask = (name: string) => ({ apiVersion: "ax.io/v1alpha1" as const, kind: "Task" as const, metadata: { name, atespace: "fleet" }, + spec: { image: "someone-else", command: ["long-job"], env: [{ name: "AX_CONWIP_HOLDER", value: "nas-link-2" }], gateway: { name: "halogen" } } }) +const foreignGrant = (name: string) => ({ leaseId: name, attempt: 1, job: job("x"), + lease: { holderIdentity: "nas-link-1", leaseDurationSeconds: 30, acquireTime: Date.now(), renewTime: Date.now(), leaseTransitions: 0 } }) +const runStub = async (dir: string, ax: FakeAx, floor: any, ms: number, over: Partial = {}) => { + const logs: Array> = [] + const stop = Effect.runSync(Deferred.make()) + const cfg: LinkConfig = { holder: "nas-link-1", maxInFlight: 2, servedLabels: ["seat:halogen", "runtime:gvisor"], + shape: { atespace: "fleet", image: "ax-agent", gateway: "halogen", command: () => ["ax-agent", "pi"] }, + completion: "auto", secondMs: SEC, resyncMs: 30, pendingTimeoutMs: 60_000, deleteAfterMs: 600_000, deadlineBackstopMs: 300, + createAttempts: 3, outboxBackoffMs: [20, 100], initialPollSeconds: 1, initialHeartbeatSeconds: 2, ...over } + const f = Effect.runFork(runLink(cfg, { ax, floor, journal: Journal.open(dir), log: (ev, x) => logs.push({ t: Date.now(), ev, ...x }), stop })) + await sleep(ms) + Effect.runSync(Deferred.succeed(stop, undefined)); await Effect.runPromise(Fiber.await(f)) + return logs +} + +describe("review round 3", () => { + let dir: string + let worlds: Array = [] + const W = (o: Partial = {}) => { const w = world(o); worlds.push(w); return w } + beforeEach(() => { dir = mkdtempSync(join(tmpdir(), "conwip-link-r3-")) }) + afterEach(() => { for (const w of worlds) w.floor.close(); worlds = []; rmSync(dir, { recursive: true, force: true }) }) + + // ---------------------------------------------------------------- R3-1: a withdrawal names a generation + for (const sendWithdrewGen of [true, false]) { + it(`R3-1a (E) rule 4b withdraws a QUEUED a2 for a1's retryable verdict; the requeued a2 is created (floor names the generation: ${sendWithdrewGen})`, async () => { + const w = W({ sendWithdrewGen }); w.floor.enqueue(job("1")) + // Deterministic E: no Lease reaches the floor between its return and a1's verdict landing, so a2 gen 1 is withdrawn + // while still QUEUED and this link never receives it (a Lease retried across the return could carry a capacity + // computed before the verdict and receive gen 1 first: that is R2-5a's interleaving, not this one). + let gate = false, landed = false + const fetch: typeof globalThis.fetch = async (input, init) => { + const rpc = await rpcOf(input, init) + if (rpc === "Lease" && gate && !landed) throw new TypeError("fetch failed") + const r = await w.floor.fetch(input, init) + if (rpc === "Complete" && gate) landed = true + return r + } + const l = start(w, dir, {}, fetch) + await until("running", () => w.ax.tasks.get("wf-test-1-a1")?.phase === "Running") + gate = true + w.floor.down = true + await until("floor requeued attempt 2", () => J(w).state === "queued" && J(w).attempt === 2, 4000) + failRetryable(w, "wf-test-1-a1") + await until("verdict written", () => l.has("verdict", "wf-test-1-a1")) + w.floor.down = false + await until("a2 created", () => w.ax.updates.has("wf-test-1-a2"), 4000).catch(() => { throw new Error(why(w, l)) }) + expect(J(w).history).toContain("withdrawn:a2") + expect(l.logs.some((x) => x.ev === "leased" && x.leaseId === "wf-test-1-a2" && l.logs.indexOf(x) < l.logs.findIndex((y) => y.ev === "complete"))).toBe(false) + w.ax.finish("wf-test-1-a2", 0, { ok: 2 }) + await until("floor done", () => J(w).state === "done", 4000) + expect([J(w).result, J(w).attempt, w.ax.updates.get("wf-test-1-a2")]).toEqual(["success", 2, 1]) + await l.drain() + }) + + it(`R3-1b (C) kill -9 between the withdrawal and the regrant: the regrant of a2 is created after restart (floor names the generation: ${sendWithdrewGen})`, async () => { + const w = W({ sendWithdrewGen }); w.floor.enqueue(job("1")) + let holdLease = false, failHeartbeat = false, failComplete = false, holdAfterComplete = false + const fetch: typeof globalThis.fetch = async (input, init) => { + const rpc = await rpcOf(input, init) + if (rpc === "Heartbeat" && failHeartbeat) throw new TypeError("fetch failed") + if (rpc === "Complete" && failComplete) return new Response("upstream timeout", { status: 503 }) + if (rpc === "Lease") while (holdLease) await sleep(2) + const r = await w.floor.fetch(input, init) + if (rpc === "Complete" && holdAfterComplete) holdLease = true // the regrant never reaches this process + return r + } + const over = { deleteAfterMs: 60_000 } // a1's record survives the restart's compaction; a2 gen 1 does not + const l1 = start(w, dir, over, fetch) + await until("running", () => w.ax.tasks.get("wf-test-1-a1")?.phase === "Running") + w.floor.down = true + await until("floor requeued attempt 2", () => J(w).state === "queued" && J(w).attempt === 2, 4000) + failHeartbeat = true; holdLease = true; failComplete = true + w.floor.down = false + await sleep(80) + failRetryable(w, "wf-test-1-a1") + await until("verdict written", () => l1.has("verdict", "wf-test-1-a1")) + holdLease = false + await until("a2 received", () => l1.has("leased", "wf-test-1-a2"), 4000) + await until("dispatch waits for the verdict", () => l1.has("supersede-waits-for-verdict", "wf-test-1-a2")) + holdAfterComplete = true; failComplete = false; failHeartbeat = false + await until("floor withdrew a2 and the link journaled it", () => l1.has("withdrawn", "wf-test-1-a2"), 4000) + await l1.crash() + const l2 = start(w, dir, over) + await until("a2 created", () => w.ax.updates.has("wf-test-1-a2"), 4000).catch(() => { throw new Error(why(w, l2)) }) + expect(w.ax.updates.get("wf-test-1-a2")).toBe(1) + expect(J(w).history.filter((h) => h === "leased:a2").length).toBe(2) // gen 1 withdrawn, gen 2 run + await l2.drain() + }) + } + + // ---------------------------------------------------------------- R3-2: the cancel intent is journaled first + it("R3-2 (A) a cancel whose DeleteTask lands but whose reply is lost is reported cancelled; no attempt 2 is started", async () => { + const w = W({ leaseSeconds: 20, heartbeatSeconds: 6, graceSeconds: 40 }); w.floor.enqueue(job("1")) + const l = start(w, dir, { initialHeartbeatSeconds: 6 }) + await until("running", () => w.ax.tasks.get("wf-test-1-a1")?.phase === "Running") + const real = w.ax.deleteTask + let lostReplies = 0 + w.ax.deleteTask = (name: string) => real(name).pipe(Effect.andThen(() => lostReplies++ === 0 ? Effect.fail(new AxError(4, "deadline exceeded")) : Effect.void)) + w.floor.cancel("wf-test-1") + await until("floor done", () => J(w).state === "done", 4000).catch(() => { throw new Error(why(w, l)) }) + await sleep(200) + const v = l.logs.find((x) => x.ev === "verdict" && x.leaseId === "wf-test-1-a1")! + expect([v.result, J(w).result, J(w).attempt, J(w).infraSpent]).toEqual(["cancelled", "cancelled", 1, 0]) + expect(w.ax.updates.has("wf-test-1-a2")).toBe(false) + expect(l.logs.some((x) => x.ev === "delete-deferred" && x.task === "wf-test-1-a1")).toBe(true) + await l.drain() + }) + + it("R3-2b a journaled DeleteTask that did not land is sent again by the resync", async () => { + const w = W({ leaseSeconds: 20, heartbeatSeconds: 6, graceSeconds: 40 }); w.floor.enqueue(job("1")) + const l = start(w, dir, { initialHeartbeatSeconds: 6 }) + await until("running", () => w.ax.tasks.get("wf-test-1-a1")?.phase === "Running") + const real = w.ax.deleteTask + let refused = 0 + w.ax.deleteTask = (name: string) => refused++ === 0 ? Effect.fail(new AxError(14, "unavailable")) : real(name) + w.floor.cancel("wf-test-1") + await until("floor done", () => J(w).state === "done", 4000).catch(() => { throw new Error(why(w, l)) }) + expect([J(w).result, w.ax.deletes.filter((d) => d === "wf-test-1-a1").length]).toEqual(["cancelled", 1]) + await l.drain() + }) + + // ---------------------------------------------------------------- R3-3: finished leases are tombstoned + for (const restart of [false, true]) it(`R3-3 (D) a redelivered grant of a finished lease is never run again (restart=${restart})`, async () => { + const w = W(); w.floor.enqueue(job("1")) + const l1 = start(w, dir) + await until("running", () => w.ax.tasks.get("wf-test-1-a1")?.phase === "Running") + w.ax.finish("wf-test-1-a1", 0, { answer: 42 }) + await until("done", () => J(w).state === "done") + await until("janitor deleted a1", () => !w.ax.tasks.has("wf-test-1-a1"), 3000) + await sleep(100) + let l = l1 + if (restart) { await l1.crash(); l = start(w, dir); await sleep(100) } + w.floor.redeliver = ["wf-test-1-a1"] + await until("duplicate-grant", () => l.has("duplicate-grant", "wf-test-1-a1"), 2000) + await sleep(200) + expect(w.ax.updates.get("wf-test-1-a1")).toBe(1) + await l.drain() + }) + + it("R3-3b the tombstone survives compaction and expires after TOMB_MS; a later generation of the same leaseId is not refused", () => { + const j = Journal.open(dir) + const g = { leaseId: "x-a1", attempt: 1, job: job("x"), lease: { holderIdentity: "h", leaseDurationSeconds: 30, acquireTime: 1000, renewTime: 1000, leaseTransitions: 0 } } + j.append({ ev: "grant", leaseId: "x-a1", grant: g }) + j.append({ ev: "report", leaseId: "x-a1", attempt: 1, result: "success" }) + j.append({ ev: "reported", leaseId: "x-a1", duplicate: false }) + j.compact() + expect(j.recs.has("x-a1")).toBe(false) + expect(Journal.open(dir).tombs.get("x-a1")?.gen).toEqual({ leaseTransitions: 0, acquireTime: 1000 }) + const keep = Journal.TOMB_MS + try { Journal.TOMB_MS = -1; Journal.open(dir).compact(); expect(Journal.open(dir).tombs.has("x-a1")).toBe(false) } finally { Journal.TOMB_MS = keep } + }) + + // ---------------------------------------------------------------- R3-4: renewal does not wait for the first Lease + it("R3-4 (B) a restart whose first Lease is rate-limited keeps heartbeating; the running a1 is not requeued", async () => { + const w = W(); w.floor.enqueue(job("1")) + const l1 = start(w, dir) + await until("running", () => w.ax.tasks.get("wf-test-1-a1")?.phase === "Running") + await l1.crash() + let leases = 0 + const hbAt: Array = [] + const f: typeof globalThis.fetch = async (input, init) => { + const rpc = await rpcOf(input, init) + if (rpc === "Heartbeat") hbAt.push(Date.now()) + if (rpc === "Lease" && leases++ === 0) return new Response("slow down", { status: 429, headers: { "retry-after": "2" } }) + return w.floor.fetch(input, init) + } + const t0 = Date.now() + const l2 = start(w, dir, {}, f) + await sleep(2300) + expect(hbAt.filter((t) => t - t0 > 200 && t - t0 < 2000).length).toBeGreaterThan(5) + expect(J(w).history).not.toContain("requeue:grace") + expect([J(w).attempt, w.ax.tasks.get("wf-test-1-a1")?.phase]).toEqual([1, "Running"]) + await l2.drain() + }) + + // ---------------------------------------------------------------- R3-5: a Task the link did not write is never touched + for (const variant of ["janitor", "lost", "cancel"] as const) for (const phase of ["Running", "Completed"] as const) + it(`R3-5a a grant named after a foreign ${phase} Task: refused name-conflict, never deleted or read (${variant})`, async () => { + const ax = new FakeAx() + ax.put(foreignTask("other-team-build"), "Running") + if (phase === "Completed") ax.finish("other-team-build", 0, { secret: "other tenant's output" }) + let sent = false + const completes: Array = [] + const floor = { + lease: (p: any) => Effect.sync(() => { const g = !sent && p.capacity > 0 ? [foreignGrant("other-team-build")] : []; if (g.length) sent = true; return { grants: g, invalid: [], stats, nextPollSeconds: 1, heartbeatSeconds: 2 } }), + heartbeat: (p: any) => Effect.sync(() => { + const lost = variant === "lost" ? p.leaseIds.filter((x: string) => x === "other-team-build") : [] + const cancel = variant === "cancel" ? p.leaseIds.filter((x: string) => x === "other-team-build") : [] + return { renewed: p.leaseIds.filter((x: string) => !lost.includes(x)), lost, cancelRequested: cancel, stats } + }), + complete: (p: any) => Effect.sync(() => { completes.push(p); return { duplicate: false, stats } }) + } + const logs = await runStub(dir, ax, floor, 800) + expect(ax.deletes).toEqual([]) + expect(ax.tasks.get("other-team-build")?.phase).toBe(phase) + expect(completes.every((c) => c.result !== "success" && JSON.stringify(c).includes("secret") === false)).toBe(true) + expect(logs.some((x) => x.ev === "not-mine" && x.leaseId === "other-team-build")).toBe(true) + }) + + it("R3-5b (salvage) a crash in the create window, then `lost`: a foreign finished Task's result is never reported, the Task never deleted", async () => { + const g = foreignGrant("other-team-build") + const t = Date.now() + appendFileSync(join(dir, "journal.jsonl"), JSON.stringify({ t, ev: "grant", leaseId: g.leaseId, grant: g }) + "\n" + JSON.stringify({ t, ev: "creating", leaseId: g.leaseId }) + "\n") + const ax = new FakeAx() + ax.put(foreignTask("other-team-build"), "Running") + ax.finish("other-team-build", 0, { secret: "other tenant's output" }) + const completes: Array = [] + const floor = { + lease: (_p: any) => Effect.succeed({ grants: [], invalid: [], stats, nextPollSeconds: 1, heartbeatSeconds: 2 }), + heartbeat: (p: any) => Effect.succeed({ renewed: [], lost: p.leaseIds.filter((x: string) => x === "other-team-build"), cancelRequested: [], stats }), + complete: (p: any) => Effect.sync(() => { completes.push(p); return { duplicate: false, stats } }) + } + const logs = await runStub(dir, ax, floor, 600) + expect([completes, ax.deletes]).toEqual([[], []]) + expect(logs.some((x) => x.ev === "not-mine")).toBe(true) + }) + + it("R3-5c the link's own Task under a `creating`-only record (crash before `created`) is still salvaged and deleted", async () => { + const w = W(); w.floor.enqueue(job("1")) + w.ax.hangAfterCreate = true + const l1 = start(w, dir) + await until("created in ax", () => w.ax.tasks.has("wf-test-1-a1")) + await l1.crash() + w.ax.hangAfterCreate = false + w.ax.tick() + w.ax.finish("wf-test-1-a1", 0, { mine: true }) + const l2 = start(w, dir) + await until("floor done", () => J(w).state === "done", 4000).catch(() => { throw new Error(why(w, l2)) }) + expect([J(w).result, J(w).output]).toEqual(["success", { mine: true }]) + await until("deleted", () => !w.ax.tasks.has("wf-test-1-a1"), 3000) + expect(l2.has("not-mine")).toBe(false) + await l2.drain() + }) + + // ---------------------------------------------------------------- R3-6 and R3-7: server-driven intervals are bounded + for (const v of [0, -5]) it(`R3-6 a floor answering nextPollSeconds=${v} and heartbeatSeconds=${v} gets a bounded call rate (production secondMs)`, async () => { + const w = W({ pollSeconds: v, heartbeatSeconds: v, secondMs: 1000, leaseSeconds: 60, graceSeconds: 60 }) + w.floor.enqueue(job("1")) + const l = start(w, dir, { secondMs: 1000, resyncMs: 1000, initialPollSeconds: 15, initialHeartbeatSeconds: 30 }) + await sleep(1500) + const n = (rpc: string) => w.floor.calls.filter((c) => c.rpc === rpc).length + expect(n("Lease")).toBeLessThanOrEqual(4) + expect(n("Heartbeat")).toBeLessThanOrEqual(4) + expect(l.has("interval-clamped")).toBe(true) + await l.drain() + }) + + it("R3-7 boundedSeconds: default for missing, non-finite and not positive; clamped to [1, hi]", () => { + expect([boundedSeconds(undefined, 15, 300), boundedSeconds(0, 15, 300), boundedSeconds(-5, 15, 300), boundedSeconds(Number.NaN, 15, 300), + boundedSeconds(0.2, 15, 300), boundedSeconds(10 / 3, 15, 300), boundedSeconds(1e9, 15, 300), boundedSeconds(60, 30, 10)]).toEqual([15, 15, 15, 15, 1, 10 / 3, 300, 10]) + }) + + // ---------------------------------------------------------------- R3-8: the Lease envelope decodes leniently + it("R3-8 a Lease reply with heartbeatSeconds 3.333, endpoint null and no stats is used; the job runs", async () => { + const w = W(); w.floor.enqueue(job("1")) + const fetch = rewriting(w, "Lease", deep((o) => { if ("heartbeatSeconds" in o) { o.heartbeatSeconds = 10 / 3; o.endpoint = null; delete o.stats } })) + const l = start(w, dir, {}, fetch) + await until("created", () => w.ax.updates.has("wf-test-1-a1"), 4000).catch(() => { throw new Error(why(w, l)) }) + w.ax.finish("wf-test-1-a1", 0, { ok: 1 }) + await until("floor done", () => J(w).state === "done", 4000) + expect([J(w).result, J(w).attempt, l.has("lease-error")]).toEqual(["success", 1, false]) + await l.drain() + }) + + it("R3-8b a Lease key whose replies never decode is abandoned after leaseKeyAttempts, not replayed for ever; the job then runs", async () => { + const w = W(); w.floor.enqueue(job("1")) + let broken = true + const keys: Array = [] + const inner = rewriting(w, "Lease", deep((o) => { if (broken && "grants" in o && "nextPollSeconds" in o) o.grants = "not-an-array" })) + const fetch: typeof globalThis.fetch = async (input, init) => { + const b = await bodyOf(input, init) + const k = /"requestKey":"([^"]+)"/.exec(b)?.[1] + if (k && b.includes('"Lease"') && broken) keys.push(k) + return inner(input, init) + } + const l = start(w, dir, { leaseKeyAttempts: 3 }, fetch) + await until("key abandoned twice", () => l.logs.filter((x) => x.ev === "lease-key-abandoned").length >= 2, 4000).catch(() => { throw new Error(why(w, l)) }) + broken = false + expect(new Set(keys).size).toBeGreaterThan(1) // before round 3: one key, `transient`, for as long as the floor answers so + expect(l.logs.filter((x) => x.ev === "lease-error").every((x) => x.kind === "server-error")).toBe(true) + await until("created", () => w.ax.updates.size > 0, 4000).catch(() => { throw new Error(why(w, l)) }) + await l.drain() + }) + + it("R3-8c (R1-9b scenario) a floor that cannot encode one job: the link stays up and a job enqueued later runs", async () => { + // The good job co-granted with the bad one is only ever sent inside replies the strict floor cannot encode, so no + // link-side change can deliver it (REFUTED for the link; the floor port must encode per grant, round 1 H). + const w = W() + w.floor.enqueue(job("bad", { "timeout-minutes": 1.5 })) + w.floor.enqueue(job("good")) + const l = start(w, dir) + await sleep(500) + w.floor.enqueue(job("later")) + await until("later created", () => w.ax.updates.has("wf-test-later-a1"), 8000).catch(() => { throw new Error(why(w, l)) }) + expect(l.logs.filter((x) => x.ev === "lease-error").every((x) => x.kind === "server-error")).toBe(true) + await l.drain() + }) + + // ---------------------------------------------------------------- R3-9: Complete's success decodes leniently + it("R3-9 a Complete success reply with `withdrew: null` is reported once; the next jobs run", async () => { + const w = W(); w.floor.enqueue(job("1")); w.floor.enqueue(job("2")) + let completes = 0 + const patch = deep((o) => { if ("duplicate" in o && "stats" in o && !("withdrew" in o)) o.withdrew = null }) + const inner = rewriting(w, "Complete", patch) + const fetch: typeof globalThis.fetch = async (input, init) => { if (await rpcOf(input, init) === "Complete") completes++; return inner(input, init) } + const l = start(w, dir, { maxInFlight: 1 }, fetch) + await until("a1", () => w.ax.tasks.has("wf-test-1-a1")) + w.ax.finish("wf-test-1-a1", 0, { ok: true }) + await until("job 2 created", () => w.ax.updates.has("wf-test-2-a1"), 4000).catch(() => { throw new Error(why(w, l)) }) + expect([J(w).state, J(w).result, completes, l.has("outbox-keep")]).toEqual(["done", "success", 1, false]) + await l.drain() + }) + + it("R3-9b a Complete success reply that is not the protocol at all is a server-error: bounded by verdictAttempts, never gates Lease", async () => { + const w = W(); w.floor.enqueue(job("1")); w.floor.enqueue(job("2")) + // the floor applies the verdict; its reply's success value is replaced by a number + const fetch = rewriting(w, "Complete", deep((o) => { if (o.value && typeof o.value === "object" && "duplicate" in (o.value as object)) o.value = 7 })) + const l = start(w, dir, { maxInFlight: 1, verdictAttempts: 2 }, fetch) + await until("a1", () => w.ax.tasks.has("wf-test-1-a1")) + w.ax.finish("wf-test-1-a1", 0, { ok: true }) + await until("job 2 created", () => w.ax.updates.has("wf-test-2-a1"), 4000).catch(() => { throw new Error(why(w, l)) }) + expect(l.logs.filter((x) => x.ev === "outbox-keep").every((x) => x.kind === "server-error")).toBe(true) + await l.drain() + }) + + // ---------------------------------------------------------------- R3-10: B2 (n+1 only after NotFound for n) + it("R3-10 (F8 shape) attempt n+1 is written only once attempt n is absent from ax; while n stays Terminating it is never written", async () => { + const w = W() + const create = w.ax.createTask + const seenAtCreate: Array = [] + w.ax.createTask = (t) => { if (t.metadata.name === "wf-test-1-a2") seenAtCreate.push(...[...w.ax.tasks].map(([n, x]) => `${n}:${x.phase}`)); return create(t) } + w.floor.enqueue(job("1")) + const l1 = start(w, dir) + await until("running", () => w.ax.tasks.get("wf-test-1-a1")?.phase === "Running") + await l1.crash() + rmSync(dir, { recursive: true, force: true }) // the journal is lost: a2 is fenced on the Task's own marks + await sleep(7 * SEC); w.floor.sweep() + const tick = w.ax.tick.bind(w.ax) + let hold = true + w.ax.tick = () => { if (hold) { for (const t of w.ax.tasks.values()) if (t.phase === "Pending") t.phase = "Running" } else tick() } + const l2 = start(w, mkdtempSync(join(tmpdir(), "conwip-link-r3-")), { fenceTimeoutMs: 4000 }) + await until("a1 Terminating", () => w.ax.tasks.get("wf-test-1-a1")?.phase === "Terminating", 4000).catch(() => { throw new Error(why(w, l2)) }) + await sleep(250) + expect(w.ax.updates.has("wf-test-1-a2")).toBe(false) // the delete was asked, NotFound not yet seen + hold = false + await until("a2 written", () => w.ax.updates.has("wf-test-1-a2"), 6000).catch(() => { throw new Error(why(w, l2)) }) + expect(seenAtCreate.filter((x) => x.startsWith("wf-test-1-a1"))).toEqual([]) + await l2.drain() + }) + + // ---------------------------------------------------------------- R3-11: 429, Retry-After, endpoint and address rows + it("R3-11 a Complete answered 429 Retry-After: 1 keeps the verdict and waits at least Retry-After before the next try", async () => { + const w = W(); w.floor.enqueue(job("1")) + const at: Array = [] + const fetch: typeof globalThis.fetch = async (input, init) => { + if (await rpcOf(input, init) !== "Complete") return w.floor.fetch(input, init) + at.push(Date.now()) + if (at.length === 1) return new Response("slow down", { status: 429, headers: { "retry-after": "1" } }) + return w.floor.fetch(input, init) + } + const l = start(w, dir, {}, fetch) + await until("a1", () => w.ax.tasks.get("wf-test-1-a1")?.phase === "Running") + w.ax.finish("wf-test-1-a1", 0, { v: 1 }) + await until("floor done", () => J(w).state === "done", 5000).catch(() => { throw new Error(why(w, l)) }) + expect([J(w).result, J(w).attempt, l.has("complete-dropped")]).toEqual(["success", 1, false]) + expect(l.logs.some((x) => x.ev === "outbox-keep" && x.kind === "rate-limited")).toBe(true) + expect(at[1]! - at[0]!).toBeGreaterThanOrEqual(950) + await l.drain() + }) + + it("R3-11b an http:// endpoint is ignored even when it is in the declared list; 100.x outside 100.64/10 is not fleet-internal", async () => { + const w = W() + const fetch = rewriting(w, "Lease", deep((o) => { if ("nextPollSeconds" in o) o.endpoint = "http://floor2.test" })) + const switched: Array = [] + const logs: Array> = [] + const stop = Effect.runSync(Deferred.make()) + const cfg: LinkConfig = { holder: "nas-link-1", maxInFlight: 2, servedLabels: ["seat:halogen", "runtime:gvisor"], + shape: { atespace: "fleet", image: "ax-agent", gateway: "halogen", command: () => ["ax-agent", "pi"] }, + completion: "auto", secondMs: SEC, resyncMs: 30, pendingTimeoutMs: 1500, deleteAfterMs: 0, deadlineBackstopMs: 300, + createAttempts: 3, outboxBackoffMs: [20, 100], initialPollSeconds: 1, initialHeartbeatSeconds: 2, floorUrls: ["http://floor.test", "http://floor2.test"] } + const fiber = Effect.runFork(Effect.scoped(Effect.gen(function*() { + const floor = yield* rpcFloor({ url: "http://floor.test", token: "tok-nas", sessionId: "s", fetch }) + return yield* runLink(cfg, { ax: w.ax, floor, journal: Journal.open(dir), log: (ev, f) => logs.push({ ev, ...f }), stop, floorUrl: "http://floor.test", onEndpoint: (u) => switched.push(u) }) + }))) + await until("endpoint seen", () => logs.some((x) => x.ev === "endpoint-ignored" || x.ev === "endpoint-accepted"), 3000) + Effect.runSync(Deferred.succeed(stop, undefined)); await Promise.race([Effect.runPromise(Fiber.await(fiber)), sleep(1500)]) + expect([switched, logs.some((x) => x.ev === "endpoint-ignored")]).toEqual([[], true]) + expect(["http://100.1.2.3/c", "http://100.128.0.1/c", "http://100.63.255.255/c", "http://100.64.0.1/c", "http://100.127.255.254/c"].map((u) => isFleetInternalUrl(u))) + .toEqual([false, false, false, true, true]) + }) +}) diff --git a/pkgs/substrate-apps/src/apps/link/test/review-r4-expired.test.ts b/pkgs/substrate-apps/src/apps/link/test/review-r4-expired.test.ts new file mode 100644 index 00000000..c2b499f5 --- /dev/null +++ b/pkgs/substrate-apps/src/apps/link/test/review-r4-expired.test.ts @@ -0,0 +1,86 @@ +// Round 4 regression test, ported from /home/tom/today/evals-2026-09-23/link/scratch-r4-1/expired.repro.ts (assertions kept; additions marked). +// Round 4, lens fail-closed and capacity gate: a grant that waits at the slot gate (B11: gate closed, e.g. the Gateway +// check failing) is created whenever the gate reopens, with no check that its lease is still live. The floor, which +// the link could not reach (heartbeats failing), has already expired it, requeued it, and may grant attempt 2. +// Buildkite's pattern the design adopts (row 8, "reserve with expiry, finish -1 before start") would refuse it. +import { expect, it } from "vitest" +import { bodyOf, job, sleep, start, until, world } from "./review-r4-harness.ts" + +it("a grant whose lease expired while it waited at the gate is still created", async () => { + const w = world({ leaseSeconds: 6, graceSeconds: 15 }) // at secondMs 20: lease 120 ms, grace 300 ms + w.floor.enqueue(job("1")) + const gw = w.ax.gateways.get("halogen")! + let holdOnce = true + let l: ReturnType + const fetch: typeof globalThis.fetch = async (input, init) => { + const b = await bodyOf(input, init) + const res = await w.floor.fetch(input, init) + if (holdOnce && b.includes('"Lease"')) { + const txt = await res.clone().text() + if (txt.includes("wf-test-1-a1")) { + holdOnce = false + // the Gateway is edited away while the reply is in flight; the next resync closes the gate (B10) + w.ax.gateways.delete("halogen") + await until("gateway-missing", () => l.has("gateway-missing"), 3000) + } + } + return res + } + l = start(w, undefined, { maxInFlight: 1 }, fetch) + await until("waiting-for-slot", () => l.has("waiting-for-slot", "wf-test-1-a1"), 3000) + // the NAS loses its route to Cloudflare; ax stays reachable + w.floor.down = true + const J = w.floor.jobs.get("wf-test-1")! + await until("floor requeued a1 after lease + grace", () => J.state === "queued" && J.attempt === 2, 3000) + const floorStateAtReopen = { state: J.state, attempt: J.attempt, history: [...J.history] } + w.ax.gateways.set("halogen", gw) // the Gateway is back; the floor is still unreachable + await until("a1 created", () => (w.ax.updates.get("wf-test-1-a1") ?? 0) > 0, 3000).catch(() => {}) + const createdA1WhileExpired = (w.ax.updates.get("wf-test-1-a1") ?? 0) > 0 + await sleep(200) + const a1BeforeFloorReturned = w.ax.updates.get("wf-test-1-a1") ?? 0 // round 4 addition + w.floor.down = false + await until("a2 created", () => w.ax.tasks.has("wf-test-1-a2"), 3000).catch(() => {}) + await sleep(300) + const out = { + floorStateAtReopen, createdA1WhileExpired, axEvents: w.ax.events, + linkTrace: l!.logs.filter((x) => typeof x.leaseId === "string" || typeof x.task === "string").map((x) => `${x.ev}:${x.leaseId ?? x.task}${x.why ? ":" + x.why : ""}`), + heartbeatErrors: l!.logs.filter((x) => x.ev === "heartbeat-error").length, floorHistory: J.history + } + console.log(JSON.stringify(out)) + await l!.drain(); w.floor.close() + expect(createdA1WhileExpired).toBe(false) // fail closed: a lease known to be past renewTime + duration is not started + expect(a1BeforeFloorReturned).toBe(0) // round 4 addition: nothing of a1 before the floor returns + expect([...w.ax.updates.keys()]).toEqual(["wf-test-1-a2"]) // exactly one created attempt, a2, afterwards + expect(l!.has("lease-unconfirmed", "wf-test-1-a1")).toBe(true) +}) + +// Round 4 addition (finding 5, deadline half): a grant whose deadline has passed is refused with no create. The gate +// closes while the grant is in flight and the Heartbeats fail, so the floor cannot cancel at its deadline (the lease is +// orphaned, grace is long); only the link's own check can end it. +it("R4-5b: a grant past its deadline is failed deadline-exceeded before any UpdateTask", async () => { + const w = world({ graceSeconds: 400 }) + w.floor.enqueue(job("d", { "timeout-minutes": 1 })) // 60 s x 20 ms = 1.2 s deadline + const gw = w.ax.gateways.get("halogen")! + let hbDown = false + let l: ReturnType + const fetch: typeof globalThis.fetch = async (input, init) => { + const b = await bodyOf(input, init) + if (hbDown && b.includes('"Heartbeat"')) throw new TypeError("fetch failed") + const res = await w.floor.fetch(input, init) + if (!hbDown && b.includes('"Lease"') && (await res.clone().text()).includes("wf-test-d-a1")) { + hbDown = true + w.ax.gateways.delete("halogen") + await until("gateway-missing", () => l.has("gateway-missing"), 3000) + } + return res + } + l = start(w, undefined, { maxInFlight: 1 }, fetch) + await until("waiting", () => l.has("waiting-for-slot", "wf-test-d-a1") || l.has("lease-unconfirmed", "wf-test-d-a1"), 3000) + await sleep(1400) + w.ax.gateways.set("halogen", gw) + await until("verdict", () => l.has("verdict", "wf-test-d-a1"), 3000) + const v = l.logs.find((x) => x.ev === "verdict" && x.leaseId === "wf-test-d-a1")! + await l.drain(); w.floor.close() + expect(v.reason).toBe("deadline-exceeded") + expect(w.ax.updates.get("wf-test-d-a1") ?? 0).toBe(0) +}) diff --git a/pkgs/substrate-apps/src/apps/link/test/review-r4-fiber-defect.test.ts b/pkgs/substrate-apps/src/apps/link/test/review-r4-fiber-defect.test.ts new file mode 100644 index 00000000..f68fcaa3 --- /dev/null +++ b/pkgs/substrate-apps/src/apps/link/test/review-r4-fiber-defect.test.ts @@ -0,0 +1,96 @@ +// Round 4 regression test, ported from /home/tom/today/evals-2026-09-23/link/scratch-r4-2/fiber-defect.repro.ts (assertions kept; additions marked). +// Effect idiom: dispatch ends in `Effect.catchCause(... log("job-fiber-died"))`, so a defect in one job fiber is logged +// and swallowed while the process lives on. Here one journal write fails once (ENOSPC, then the disk recovers). The +// grant stays in `held()`, every Heartbeat renews it on the floor, nothing re-dispatches it, and nothing reports it: +// the job never runs and never ends, and it holds a floor WIP slot and a local slot until the process restarts. +import { mkdtempSync } from "node:fs" +import { tmpdir } from "node:os" +import { join } from "node:path" +import { Deferred, Effect, Fiber } from "effect" +import { expect, it } from "vitest" +import { rpcFloor } from "../src/floor.ts" +import { Journal } from "../src/journal.ts" +import { runLink } from "../src/link.ts" +import type { LinkConfig } from "../src/link.ts" +import { job, sleep, until, world, SEC } from "./review-r4-harness.ts" + +it("one failed journal write in a dispatch fiber strands the lease for the life of the process", async () => { + const w = world({ cap: 1 }) + w.floor.enqueue(job("1")) + w.floor.enqueue(job("2")) + const dir = mkdtempSync(join(tmpdir(), "r4-2-defect-")) + const j = Journal.open(dir) + const orig = j.append.bind(j) + let failed = 0 + ;(j as any).append = (e: any, t?: number) => { + if (e.ev === "creating" && failed === 0) { failed++; throw Object.assign(new Error("ENOSPC: no space left on device, write"), { code: "ENOSPC" }) } + return orig(e, t) + } + const logs: Array = [] + const stop = Effect.runSync(Deferred.make()) + const cfg: LinkConfig = { + holder: "nas-link-1", maxInFlight: 1, servedLabels: ["seat:halogen", "runtime:gvisor"], + shape: { atespace: "fleet", image: "ax-agent", gateway: "halogen", command: () => ["ax-agent", "pi"] }, + completion: "auto", secondMs: SEC, resyncMs: 30, pendingTimeoutMs: 1500, deleteAfterMs: 0, deadlineBackstopMs: 300, + createAttempts: 3, outboxBackoffMs: [20, 100], initialPollSeconds: 1, initialHeartbeatSeconds: 2, fenceTimeoutMs: 300, resultReadTries: 3 + } + const fiber = Effect.runFork(Effect.scoped(Effect.gen(function*() { + const floor = yield* rpcFloor({ url: "http://floor.test", token: "tok-nas", sessionId: "s:defect", fetch: w.floor.fetch }) + return yield* runLink(cfg, { ax: w.ax, floor, journal: j, log: (ev, f) => logs.push({ ev, ...f }), stop, floorUrl: "http://floor.test" }) + }))) + try { + await until("fiber died", () => logs.some((x) => x.ev === "job-fiber-died")) + await sleep(3000) // 25 lease durations (6 s x 20 ms); the disk is fine again + const J1 = w.floor.jobs.get("wf-test-1")!, J2 = w.floor.jobs.get("wf-test-2")! + const out = { + linkAlive: fiber.pollUnsafe() === undefined, failedWrites: failed, + job1: { state: J1.state, attempt: J1.attempt, history: J1.history, task: w.ax.tasks.get("wf-test-1-a1")?.phase ?? "absent" }, + job2: { state: J2.state, history: J2.history }, + held: j.held(), heartbeats: w.floor.calls.filter((c) => c.rpc === "Heartbeat").length, + completes: w.floor.calls.filter((c) => c.rpc === "Complete").length, + events: logs.map((x) => x.ev).filter((e, i, a) => a.indexOf(e) === i) + } + console.log(JSON.stringify(out)) + // Expected: the lease is failed back (retryable) or the process crashes and systemd restarts it; either frees job 1. + expect(J1.state === "done" || J1.attempt > 1 || w.ax.tasks.has("wf-test-1-a1") || !out.linkAlive).toBe(true) + } finally { + Effect.runSync(Deferred.succeed(stop, undefined)) + await Promise.race([Effect.runPromise(Fiber.await(fiber)), sleep(1500)]) + w.floor.close() + } +}) + +// Round 4 addition (finding 7): when even the fail-back cannot be journaled, the process dies (exit 1 under systemd) +// instead of renewing a lease nothing will ever run. +it("R4-7b: a dispatch defect whose fail-back also fails ends the link", async () => { + const w = world({ cap: 1 }) + w.floor.enqueue(job("1")) + const dir = mkdtempSync(join(tmpdir(), "r4-fix-defect-")) + const j = Journal.open(dir) + const orig = j.append.bind(j) + ;(j as any).append = (e: any, t?: number) => { + if (e.ev === "creating" || e.ev === "report") throw Object.assign(new Error("ENOSPC: no space left on device, write"), { code: "ENOSPC" }) + return orig(e, t) + } + const logs: Array = [] + const stop = Effect.runSync(Deferred.make()) + const cfg: LinkConfig = { + holder: "nas-link-1", maxInFlight: 1, servedLabels: ["seat:halogen", "runtime:gvisor"], + shape: { atespace: "fleet", image: "ax-agent", gateway: "halogen", command: () => ["ax-agent", "pi"] }, + completion: "auto", secondMs: SEC, resyncMs: 30, pendingTimeoutMs: 1500, deleteAfterMs: 0, deadlineBackstopMs: 300, + createAttempts: 3, outboxBackoffMs: [20, 100], initialPollSeconds: 1, initialHeartbeatSeconds: 2, fenceTimeoutMs: 300, resultReadTries: 3 + } + const fiber = Effect.runFork(Effect.scoped(Effect.gen(function*() { + const floor = yield* rpcFloor({ url: "http://floor.test", token: "tok-nas", sessionId: "s:defect2", fetch: w.floor.fetch }) + return yield* runLink(cfg, { ax: w.ax, floor, journal: j, log: (ev, f) => logs.push({ ev, ...f }), stop, floorUrl: "http://floor.test" }) + }))) + try { + const exit = await Promise.race([Effect.runPromise(Fiber.await(fiber)), sleep(3000).then(() => undefined)]) + expect(exit).toBeDefined() + expect(exit!._tag).toBe("Failure") + expect(logs.some((x) => x.ev === "link-defect-fatal")).toBe(true) + } finally { + Effect.runSync(Deferred.succeed(stop, undefined)) + w.floor.close() + } +}) diff --git a/pkgs/substrate-apps/src/apps/link/test/review-r4-harness.ts b/pkgs/substrate-apps/src/apps/link/test/review-r4-harness.ts new file mode 100644 index 00000000..387a1612 --- /dev/null +++ b/pkgs/substrate-apps/src/apps/link/test/review-r4-harness.ts @@ -0,0 +1,59 @@ +// Round 4 fix pass: the reviewers' scratch harness (scratch-r4-0/1/2, identical but for the temp prefix), moved into +// the worktree. The only change: the job carries the FIELD-MAP 5a journal-key annotation jobs.ts now requires. +import { mkdtempSync } from "node:fs" +import { tmpdir } from "node:os" +import { join } from "node:path" +import { Deferred, Effect, Fiber } from "effect" +import type { AgentJob } from "../src/contract.ts" +import { rpcFloor } from "../src/floor.ts" +import { Journal } from "../src/journal.ts" +import { runLink } from "../src/link.ts" +import type { LinkConfig, LinkDeps } from "../src/link.ts" +import { FakeAx } from "./fake-ax.ts" +import { FakeFloor } from "./fake-floor.ts" +import type { FloorConfig } from "./fake-floor.ts" +export const SEC = 20 +const A = "ultracode.mecattaf.dev/" +export const job = (n: string, spec: Record = {}): AgentJob => ({ + apiVersion: "ultracode.mecattaf.dev/v1alpha1", kind: "AgentJob", + metadata: { name: `wf-test-${n}`, labels: { [A + "run-id"]: "wf-test", [A + "workflow"]: "link-test", [A + "phase-index"]: "1" }, + annotations: { [A + "run-id-raw"]: "wf_test", [A + "label"]: `probe:${n}`, [A + "item-key"]: `wf_test#${n}`, [A + "journal-key"]: `${"cd".repeat(32)}:1`, [A + "phase-title"]: "Probe" } }, + spec: { "runs-on": ["seat:halogen", "runtime:gvisor"], + with: { prompt: `say ${n}`, prompt_ref: { sha256: "ab".repeat(32), bytes: 5, uri: `journal://wf_test/${n}/prompt.md` }, model: "halogen-qwen3.8-flash-next" }, ...spec } as any +}) +export const world = (o: Partial = {}) => ({ + floor: new FakeFloor({ cap: 2, leaseSeconds: 6, graceSeconds: 15, pollSeconds: 1, heartbeatSeconds: 2, maxAttempts: 3, secondMs: SEC, tokens: { "tok-nas": "nas-link-1" }, ...o }), + ax: new FakeAx() +}) +export type World = ReturnType +export function start(w: World, dir = mkdtempSync(join(tmpdir(), "r4-fix-")), over: Partial = {}, fetch?: typeof globalThis.fetch, extra: Pick = {}) { + const logs: Array> = [] + const stop = Effect.runSync(Deferred.make()) + const cfg: LinkConfig = { + holder: "nas-link-1", maxInFlight: 2, servedLabels: ["seat:halogen", "runtime:gvisor"], + shape: { atespace: "fleet", image: "ax-agent", gateway: "halogen", command: () => ["ax-agent", "pi"] }, + completion: "auto", secondMs: SEC, resyncMs: 30, pendingTimeoutMs: 1500, deleteAfterMs: 0, deadlineBackstopMs: 300, + createAttempts: 3, outboxBackoffMs: [20, 100], initialPollSeconds: 1, initialHeartbeatSeconds: 2, fenceTimeoutMs: 300, resultReadTries: 3, ...over + } + const fiber = Effect.runFork(Effect.scoped(Effect.gen(function*() { + const floor = yield* rpcFloor({ url: "http://floor.test", token: "tok-nas", sessionId: `s:${dir}`, fetch: fetch ?? w.floor.fetch }) + return yield* runLink(cfg, { ax: w.ax, floor, journal: Journal.open(dir), log: (ev, f) => logs.push({ t: Date.now(), ev, ...f }), stop, floorUrl: "http://floor.test", ...extra }) + }))) + return { + dir, logs, fiber, + has: (ev: string, leaseId?: string) => logs.some((l) => l.ev === ev && (leaseId === undefined || l.leaseId === leaseId)), + drain: () => { Effect.runSync(Deferred.succeed(stop, undefined)); return Promise.race([Effect.runPromise(Fiber.await(fiber)), sleep(1500)]) } + } +} +export const sleep = (ms: number) => new Promise((r) => setTimeout(r, ms)) +export const until = async (what: string, pred: () => boolean, ms = 5000) => { + const t0 = Date.now() + while (!pred()) { if (Date.now() - t0 > ms) throw new Error(`timeout waiting for: ${what}`); await new Promise((r) => setTimeout(r, 5)) } +} +export const bodyOf = async (input: unknown, init?: RequestInit) => { + const b = init?.body + if (typeof b === "string") return b + if (b instanceof Uint8Array) return new TextDecoder().decode(b) + if (b) return await new Response(b as any).text() + return (input instanceof Request) ? await input.clone().text() : "" +} diff --git a/pkgs/substrate-apps/src/apps/link/test/review-r4-heartbeat-envelope.test.ts b/pkgs/substrate-apps/src/apps/link/test/review-r4-heartbeat-envelope.test.ts new file mode 100644 index 00000000..81b5de39 --- /dev/null +++ b/pkgs/substrate-apps/src/apps/link/test/review-r4-heartbeat-envelope.test.ts @@ -0,0 +1,134 @@ +// Round 4 regression test, ported from /home/tom/today/evals-2026-09-23/link/scratch-r4-2/heartbeat-envelope.repro.ts (assertions kept; additions marked). +// Schema compatibility: the round-3 fix made the Lease and Complete success envelopes lenient ("a changed stats shape +// would otherwise fail the decode"), but Heartbeat still decodes with the strict L1 schema. The same drift on the +// Heartbeat reply (here `stats: null`, which Lease tolerates) makes every heartbeat a transient error: held leases are +// never renewed, the floor requeues a healthy running attempt as a2, and the link fences a1 and runs the job again. +import { expect, it } from "vitest" +import { job, start, until, world, bodyOf, sleep } from "./review-r4-harness.ts" + +const drift = (rpc: string, floorFetch: typeof globalThis.fetch): typeof globalThis.fetch => async (input, init) => { + const body = await bodyOf(input, init) + const res = await floorFetch(input, init) + if (!body.includes(`"${rpc}"`)) return res + const text = await res.text() + let m: any + try { m = JSON.parse(text) } catch { return new Response(text, { status: res.status, headers: res.headers }) } + const walk = (x: any) => { if (x && typeof x === "object") { if ("stats" in x && "seq" in (x.stats ?? {})) x.stats = null; for (const v of Object.values(x)) walk(v) } } + walk(m) + return new Response(JSON.stringify(m), { status: res.status, headers: res.headers }) +} + +for (const rpc of ["Lease", "Heartbeat"]) { + it(`stats: null on every ${rpc} reply`, async () => { + const w = world() + w.floor.enqueue(job("1")) + const l = start(w, undefined, {}, drift(rpc, w.floor.fetch)) + try { + await until("a1 running", () => w.ax.tasks.get("wf-test-1-a1")?.phase === "Running") + await sleep(1500) // about 12 lease durations (6 s x 20 ms) with a1 healthy and running + const J = w.floor.jobs.get("wf-test-1")! + const hbErr = l.logs.filter((x) => x.ev === "heartbeat-error").length + const out = { rpc, floorState: J.state, attempt: J.attempt, history: J.history, heartbeatErrors: hbErr, + a1: w.ax.tasks.get("wf-test-1-a1")?.phase ?? "deleted", a2: w.ax.tasks.get("wf-test-1-a2")?.phase ?? "absent", + events: l.logs.map((x) => x.ev).filter((e, i, a) => a.indexOf(e) === i) } + console.log(JSON.stringify(out)) + // The expected behaviour, as for the Lease drift: a healthy a1 keeps its lease and nothing reruns. + expect(J.attempt).toBe(1) + expect(w.ax.tasks.get("wf-test-1-a2")).toBeUndefined() + } finally { await l.drain(); w.floor.close() } + }) +} + +// The consequence: the floor renews on its side, but the link never reads `cancelRequested`, `lost` or `renewed`. +for (const rpc of ["Lease", "Heartbeat"]) { + it(`cancel with stats: null on every ${rpc} reply`, async () => { + const w = world() + w.floor.enqueue(job("1")) + const l = start(w, undefined, {}, drift(rpc, w.floor.fetch)) + try { + await until("a1 running", () => w.ax.tasks.get("wf-test-1-a1")?.phase === "Running") + w.floor.cancel("wf-test-1") + await sleep(1500) + const J = w.floor.jobs.get("wf-test-1")! + const out = { rpc, floorState: J.state, result: J.result ?? null, cancel: J.cancel ?? null, history: J.history, + a1: w.ax.tasks.get("wf-test-1-a1")?.phase ?? "deleted", cancelLogged: l.has("cancel") } + console.log(JSON.stringify(out)) + expect(J.state).toBe("done") + expect(J.result).toBe("cancelled") + } finally { await l.drain(); w.floor.close() } + }) +} + +// B13: a grant journaled by a previous process and never created is resumed only once a Heartbeat reply renews it. +for (const rpc of ["Lease", "Heartbeat"]) { + it(`restart resume with stats: null on every ${rpc} reply`, async () => { + const { Effect } = await import("effect") + const w = world() + w.floor.enqueue(job("1")) + const orig = w.ax.createTask + ;(w.ax as any).createTask = () => Effect.never // first process: UpdateTask never answers, then the process stops + const l1 = start(w) + const dir = l1.dir + await until("a1 creating", () => l1.has("leased")) + await sleep(100) + await l1.drain() + ;(w.ax as any).createTask = orig + const J = w.floor.jobs.get("wf-test-1")! + const l2 = start(w, dir, {}, drift(rpc, w.floor.fetch)) + try { + await sleep(2000) + const out = { rpc, floorState: J.state, attempt: J.attempt, history: J.history, result: J.result ?? null, + a1: w.ax.tasks.get("wf-test-1-a1")?.phase ?? "absent", resumed: l2.has("resume"), + heartbeatErrors: l2.logs.filter((x) => x.ev === "heartbeat-error").length, + events: l2.logs.map((x) => x.ev).filter((e, i, a) => a.indexOf(e) === i) } + console.log(JSON.stringify(out)) + expect(J.state === "done" || w.ax.tasks.get("wf-test-1-a1") !== undefined).toBe(true) + } finally { await l2.drain(); w.floor.close() } + }) +} + +// Round 4 addition (finding 6): a Heartbeat reply that omits `cancelRequested` (or has it null) is read for the rest. +const omit = (key: string, value: "omit" | null, floorFetch: typeof globalThis.fetch): typeof globalThis.fetch => async (input, init) => { + const body = await bodyOf(input, init) + const res = await floorFetch(input, init) + if (!body.includes('"Heartbeat"')) return res + const text = await res.text() + let m: any + try { m = JSON.parse(text) } catch { return new Response(text, { status: res.status, headers: res.headers }) } + const walk = (x: any) => { if (x && typeof x === "object") { if (key in x && "renewed" in x) { if (value === "omit") delete x[key]; else x[key] = value } for (const v of Object.values(x)) walk(v) } } + walk(m) + return new Response(JSON.stringify(m), { status: res.status, headers: res.headers }) +} +for (const value of ["omit", null] as const) { + it(`R4-6b: restart resume with cancelRequested ${value === "omit" ? "omitted" : "null"} on every Heartbeat reply`, async () => { + const { Effect } = await import("effect") + const w = world() + w.floor.enqueue(job("1")) + const orig = w.ax.createTask + ;(w.ax as any).createTask = () => Effect.never + const l1 = start(w) + await until("a1 creating", () => l1.has("leased")) + await sleep(100) + await l1.drain() + ;(w.ax as any).createTask = orig + const l2 = start(w, l1.dir, {}, omit("cancelRequested", value, w.floor.fetch)) + try { + await until("a1 resumed and created", () => w.ax.tasks.get("wf-test-1-a1") !== undefined, 3000) + expect(l2.has("resume", "wf-test-1-a1")).toBe(true) + expect(l2.logs.filter((x) => x.ev === "heartbeat-error").length).toBe(0) + } finally { await l2.drain(); w.floor.close() } + }) +} +it("R4-6c: a lost lease is still acted on when the Heartbeat reply omits stats and cancelRequested", async () => { + const w = world() + w.floor.enqueue(job("1")) + const both: typeof globalThis.fetch = omit("cancelRequested", "omit", drift("Heartbeat", w.floor.fetch)) + const l = start(w, undefined, {}, both) + try { + await until("a1 running", () => w.ax.tasks.get("wf-test-1-a1")?.phase === "Running") + const J = w.floor.jobs.get("wf-test-1")! + J.holder = "someone-else" // the floor answers lost:[a1] from now on + await until("lost", () => l.has("lost", "wf-test-1-a1"), 2000) + await until("a1 deleted", () => w.ax.deletes.includes("wf-test-1-a1"), 2000) + } finally { await l.drain(); w.floor.close() } +}) diff --git a/pkgs/substrate-apps/src/apps/link/test/review-r4-journal-key.test.ts b/pkgs/substrate-apps/src/apps/link/test/review-r4-journal-key.test.ts new file mode 100644 index 00000000..0b101904 --- /dev/null +++ b/pkgs/substrate-apps/src/apps/link/test/review-r4-journal-key.test.ts @@ -0,0 +1,48 @@ +// Round 4 regression test for finding 8 (repro: /home/tom/today/evals-2026-09-23/link/scratch-r4-2/journal-key.repro.ts). +// AX_CONWIP_JOURNAL_KEY must be the FIELD-MAP 5a journal key ("sha256 of canonical [prompt, opts minus label] plus the +// occurrence"), carried on the AgentJob as the `ultracode.mecattaf.dev/journal-key` annotation, never the prompt +// digest. The expected values are pinned from the substrate taskspec (journalKey in +// /home/tom/mecattaf/ax-conwip-wt-integration/src/taskspec.ts at b42b310, run 2026-09-23), so this test does not +// import another worktree. +import { createHash } from "node:crypto" +import { expect, it } from "vitest" +import { axTaskFromGrant } from "../src/jobs.ts" +import { job } from "./review-r4-harness.ts" + +const TASKSPEC = { + halogen1: "29a9c6cb8956c07073345109161d0e7eaa08708f4c2039f4e4af7a09d212ee07:1", // journalKey("same prompt", {model: halogen}, 1) + opus1: "bd381a2d66f7a1d93c7df296954218de4c9749a0feb9d94719b9c484f8c06718:1", // journalKey("same prompt", {model: opus}, 1) + halogen2: "29a9c6cb8956c07073345109161d0e7eaa08708f4c2039f4e4af7a09d212ee07:2" // the same call, second occurrence +} +const sha = (s: string) => createHash("sha256").update(s).digest("hex") +const A = "ultracode.mecattaf.dev/" +const grant = (n: string, model: string, key: string | undefined) => { + const j: any = job(n) + j.spec.with.prompt = "same prompt" + j.spec.with.prompt_ref = { sha256: sha("same prompt"), bytes: 11, uri: `journal://wf_test/${n}/prompt.md` } + j.spec.with.model = model + if (key === undefined) delete j.metadata.annotations[A + "journal-key"] + else j.metadata.annotations[A + "journal-key"] = key + return { leaseId: `wf-test-${n}-a1`, attempt: 1, job: j, lease: { holderIdentity: "h", leaseDurationSeconds: 90, acquireTime: 1, renewTime: 1, leaseTransitions: 0 } } +} +const shape = { atespace: "fleet", image: "ax-agent", gateway: "halogen", command: () => ["ax-agent", "pi"] } +const env = (g: any) => { const b: any = axTaskFromGrant(g, shape); expect(b._tag).toBe("ok"); return Object.fromEntries(b.task.spec.env.map((e: any) => [e.name, e.value])) } + +it("R4-8a: AX_CONWIP_JOURNAL_KEY is the taskspec journal key for two calls that share a prompt", () => { + const a = env(grant("1", "halogen-qwen3.8-flash-next", TASKSPEC.halogen1)) + const b = env(grant("2", "claude-opus-5-5", TASKSPEC.opus1)) + const c = env(grant("3", "halogen-qwen3.8-flash-next", TASKSPEC.halogen2)) + expect(a.AX_CONWIP_JOURNAL_KEY).toBe(TASKSPEC.halogen1) + expect(b.AX_CONWIP_JOURNAL_KEY).toBe(TASKSPEC.opus1) + expect(c.AX_CONWIP_JOURNAL_KEY).toBe(TASKSPEC.halogen2) + expect(a.AX_CONWIP_JOURNAL_KEY).not.toBe(a.AX_CONWIP_PROMPT_SHA256) + expect(a.AX_CONWIP_PROMPT_SHA256).toBe(b.AX_CONWIP_PROMPT_SHA256) // the prompt digest is shared; the journal key is not +}) + +it("R4-8b: a grant without the annotation, or with a malformed one, is refused pre-start/invalid-spec", () => { + for (const key of [undefined, "", sha("same prompt"), `${sha("x")}:`, `${sha("x").toUpperCase()}:1`]) { + const b = axTaskFromGrant(grant("4", "halogen-qwen3.8-flash-next", key) as any, shape) + expect(b._tag).toBe("refused") + if (b._tag === "refused") expect(b.reason).toBe("pre-start/invalid-spec") + } +}) diff --git a/pkgs/substrate-apps/src/apps/link/test/review-r4-overtake.test.ts b/pkgs/substrate-apps/src/apps/link/test/review-r4-overtake.test.ts new file mode 100644 index 00000000..e1274ca3 --- /dev/null +++ b/pkgs/substrate-apps/src/apps/link/test/review-r4-overtake.test.ts @@ -0,0 +1,78 @@ +// Round 4 regression test, ported from /home/tom/today/evals-2026-09-23/link/scratch-r4-0/overtake.repro.ts (assertions kept; additions marked). +// Reply ordering: the regrant of a2 (gen 2) overtakes the Complete reply that withdrew a2 (gen 1). Set-up is R2-5a +// (review-r2.test.ts): a1 fails retryable while a2 gen 1 is leased to this link and waits for a1's verdict. a1's first +// Complete answers 500 once (server-error), which by design stops gating Lease (round 2), so a Lease with capacity 1 is +// in flight while the retried Complete is applied. The floor withdraws gen 1, requeues gen 2 and grants it on that +// Lease; the Lease reply lands before the Complete reply. The link still holds gen 1 unreleased, so gen 2 is dropped as +// `duplicate-grant`; the Complete reply then releases gen 1. Nothing of a2 is ever created, although the floor has +// gen 2 leased to this link: the job waits for lease expiry and is released `omitted` (an infra release). +import { expect, it } from "vitest" +import { bodyOf, job, sleep, start, until, world } from "./review-r4-harness.ts" +import type { World } from "./review-r4-harness.ts" + +const rpcOf = async (input: unknown, init?: RequestInit) => { const b = await bodyOf(input, init); return b.includes('"Complete"') ? "Complete" : b.includes('"Lease"') ? "Lease" : b.includes('"Heartbeat"') ? "Heartbeat" : "?" } +const failRetryable = (w: World, name: string) => { + const t = w.ax.tasks.get(name)! + t.exited = true; t.phase = "Failed" + t.conditions = [{ type: "Ready", status: "False", reason: "ResourceExhausted", message: "ResourceExhausted: no free worker" }] +} + +// Round 4 addition: also against a floor that omits withdrewTransitions (the link then uses the generation of a2 it +// held when it SENT a1's verdict, not the regrant it holds when the reply lands). +for (const sendWithdrewGen of [true, false]) +it(`the regrant of a2 is created even when its Lease reply lands before the Complete reply that withdrew gen 1 (withdrewTransitions ${sendWithdrewGen ? "sent" : "omitted"})`, async () => { + const w = world({ sendWithdrewGen }); w.floor.enqueue(job("1")) + const J = () => w.floor.jobs.get("wf-test-1")! + let holdLease = false, failHeartbeat = false, failComplete = false, fiveHundredOnce = false + let l: ReturnType + let armed = false, leaseHeld = false + const fetch: typeof globalThis.fetch = async (input, init) => { + const body = await bodyOf(input, init) + const rpc = await rpcOf(input, init) + if (rpc === "Heartbeat" && (failHeartbeat || armed)) throw new TypeError("fetch failed") // no Heartbeat in the window + if (rpc === "Complete" && failComplete) return new Response("upstream timeout", { status: 503 }) + if (rpc === "Complete" && fiveHundredOnce) { fiveHundredOnce = false; armed = true; return new Response("internal error", { status: 500 }) } + if (rpc === "Lease") while (holdLease) await sleep(2) + if (rpc === "Lease" && armed && /"capacity":[1-9]/.test(body) && !leaseHeld) { + leaseHeld = true // a Lease sent before the retried Complete, delivered after it (network reordering) + await until("the retried Complete was applied", () => J().history.includes("withdrawn:a2"), 3000).catch(() => {}) + // Round 4 addition: longer than the dispatch wait period (at most 2 x SEC), so the gen-1 dispatch is inside, or + // waiting on, the outbox lock when the Complete reply lands, and reaches its withdrawn branch deterministically + await sleep(80) + } + const res = await w.floor.fetch(input as any, { ...init, body }) + if (rpc === "Complete" && armed) { + await until("the Lease reply reached the link", () => l.has("duplicate-grant", "wf-test-1-a2") || l.has("regrant", "wf-test-1-a2"), 1500).catch(() => {}) + armed = false + } + return res + } + l = start(w, undefined, {}, fetch) + await until("running", () => w.ax.tasks.get("wf-test-1-a1")?.phase === "Running") + w.floor.down = true + await until("floor requeued attempt 2", () => J().state === "queued" && J().attempt === 2, 4000) + failHeartbeat = true; holdLease = true; failComplete = true + w.floor.down = false + await sleep(80) + failRetryable(w, "wf-test-1-a1") + await until("verdict written", () => l.has("verdict", "wf-test-1-a1")) + holdLease = false + await until("a2 received", () => l.has("leased", "wf-test-1-a2"), 4000) + await until("dispatch waits for the verdict", () => l.has("supersede-waits-for-verdict", "wf-test-1-a2")) + fiveHundredOnce = true + failComplete = false; failHeartbeat = false + await until("a2 created, or 1.5 s", () => w.ax.updates.has("wf-test-1-a2"), 1500).catch(() => {}) + await sleep(300) + const trace = l.logs.filter((x) => x.leaseId === "wf-test-1-a2" || x.task === "wf-test-1-a2" || x.ev === "complete" || x.ev === "outbox-keep" || x.ev === "lease-gated-by-outbox").map((x) => `${x.ev}:${x.leaseId ?? x.task}${x.why ? ":" + x.why : ""}${x.withdrew ? ":withdrew=" + x.withdrew : ""}`) + const out = { leaseHeld, floor: J().history, state: J().state, attempt: J().attempt, infraSpent: J().infraSpent, a2Updates: w.ax.updates.get("wf-test-1-a2") ?? 0, trace } + console.log(JSON.stringify(out)) + if (out.a2Updates !== 1 || out.floor.includes("requeue:omitted")) console.log("FULL", JSON.stringify(l.logs.map((x) => ({ ...x, cause: undefined }))), JSON.stringify(w.floor.calls.slice(-40))) + await l.drain(); w.floor.close() + expect(out.a2Updates).toBe(1) + expect(out.floor).not.toContain("requeue:omitted") // round 4 addition: no infra release spent on the stall + expect(out.leaseHeld).toBe(true) + // Round 4 addition: the gen-1 dispatch never releases the regrant (the withdrawn branch is bound to its own grant) + const a2 = l.logs.filter((x) => x.leaseId === "wf-test-1-a2").map((x) => String(x.ev)) + expect(a2.slice(a2.indexOf("regrant"))).not.toContain("withdrawn") + expect(w.ax.deletes).not.toContain("wf-test-1-a2") +}) diff --git a/pkgs/substrate-apps/src/apps/link/test/review-r4-p1stale.test.ts b/pkgs/substrate-apps/src/apps/link/test/review-r4-p1stale.test.ts new file mode 100644 index 00000000..17e77600 --- /dev/null +++ b/pkgs/substrate-apps/src/apps/link/test/review-r4-p1stale.test.ts @@ -0,0 +1,74 @@ +// Round 4 regression test, ported from /home/tom/today/evals-2026-09-23/link/scratch-r4-1/p1stale.repro.ts (assertions kept; additions marked). +// Round 4, lens capacity gate: canCreate() needs "P1 proven", but the probe runs only on an ax-up transition +// (link.ts resync: `if (wasDown) serverP1 = undefined`). ax-server is a replicas:1 Deployment with the default +// RollingUpdate strategy (upstream deploy/ax-server.yaml), so a rollout to a server without P1 happens with no failed +// ListTasks, and the link keeps creating Tasks it can never read a verdict from. +import { Effect } from "effect" +import { expect, it } from "vitest" +import { job, sleep, start, until, world } from "./review-r4-harness.ts" + +it("A: ax rolled to stock v0.3.0 (server and controller) with no ListTasks failure", async () => { + const w = world() + w.floor.enqueue(job("1")) + const l = start(w) + await until("a1 running", () => w.ax.tasks.get("wf-test-1-a1")?.phase === "Running") + const probesBefore = l.logs.filter((x) => x.ev === "completion-probe").length + w.ax.p1 = false // the rollout: GetTaskResult is now UNIMPLEMENTED and no terminal phase is ever written + await until("p1-missing logged", () => l.has("p1-missing"), 1000) // round 4 fix: the probe runs on every resync + const leasesAtFlip = w.floor.calls.filter((c) => c.rpc === "Lease").length + await sleep(100) // several resyncs (30 ms) and Lease polls pass; none fails + w.floor.enqueue(job("2")) + await sleep(1500) // round 4: job 2 is never created on the stock server (the repro waited for its create here) + const J2 = w.floor.jobs.get("wf-test-2")! + const out = { + probesBefore, probesAfter: l.logs.filter((x) => x.ev === "completion-probe").length, + p1MissingLogged: l.has("p1-missing"), + job2: { state: J2.state, attempt: J2.attempt, history: J2.history }, + task2Phase: w.ax.tasks.get("wf-test-2-a1")?.phase, task2Exited: (w.ax.tasks.get("wf-test-2-a1") as any)?.exited, + verdictJob2: l.has("verdict", "wf-test-2-a1"), + leaseCapacitiesAfterRollout: [...new Set(w.floor.calls.filter((c) => c.rpc === "Lease").slice(-10).map((c) => c.capacity))] + } + console.log("A", JSON.stringify(out)) + const leasesAfter = w.floor.calls.filter((c) => c.rpc === "Lease").slice(leasesAtFlip + 1) // round 4 addition + await l.drain(); w.floor.close() + // Fail-closed expectation: after the P1 route disappears the link leases and creates nothing new. + expect(w.ax.updates.get("wf-test-2-a1") ?? 0).toBe(0) + expect(out.p1MissingLogged).toBe(true) + expect(leasesAfter.length).toBeGreaterThan(0) + expect(leasesAfter.every((c) => c.capacity === 0)).toBe(true) + expect(J2.state).toBe("queued") +}) + +it("B: ax-server rolled to stock, the P1 controller still writes Completed", async () => { + const w = world() + w.floor.enqueue(job("1")) + const l = start(w) + await until("a1 running", () => w.ax.tasks.get("wf-test-1-a1")?.phase === "Running") + let stockServer = false + const orig = w.ax.getTaskResult + ;(w.ax as any).getTaskResult = (n: string) => stockServer ? Effect.succeed("unimplemented" as const) : orig(n) + stockServer = true + w.ax.finish("wf-test-1-a1", 0, { answer: 1 }) // Completed, result stored by the controller, unreadable via ax-server + for (let i = 0; i < 40; i++) { // every attempt the floor grants finishes with a success the link cannot read + for (const [name, t] of w.ax.tasks) if (t.phase === "Running" && !t.exited) w.ax.finish(name, 0, { answer: name }) + await sleep(50) + } + const J = w.floor.jobs.get("wf-test-1")! + const out = { + probes: l.logs.filter((x) => x.ev === "completion-probe").length, p1MissingLogged: l.has("p1-missing"), + creates: [...w.ax.updates.keys()], job1: { state: J.state, result: (J as any).result, output: (J as any).output, history: J.history }, + verdicts: l.logs.filter((x) => x.ev === "verdict").map((x) => `${x.leaseId}:${x.result}:${x.reason ?? ""}`) + } + console.log("B", JSON.stringify(out)) + // Round 4 addition: the Task is kept, uncharged, and read once P1 is back: the success is reported, not thrown away + stockServer = false + await until("job 1 done", () => J.state === "done", 3000) + const leasesUnimpl = w.floor.calls.filter((c) => c.rpc === "Lease") + await l.drain(); w.floor.close() + expect(out.creates.length).toBe(1) // fail closed: nothing created once the server answered UNIMPLEMENTED + expect(out.p1MissingLogged).toBe(true) + expect(out.verdicts.some((v) => v.includes("result-unreadable"))).toBe(false) + expect((J as any).result).toBe("success") + expect((J as any).output).toEqual({ answer: 1 }) + expect(leasesUnimpl.length).toBeGreaterThan(0) +}) diff --git a/pkgs/substrate-apps/src/apps/link/test/review-r4-restart-replay.test.ts b/pkgs/substrate-apps/src/apps/link/test/review-r4-restart-replay.test.ts new file mode 100644 index 00000000..9307a75a --- /dev/null +++ b/pkgs/substrate-apps/src/apps/link/test/review-r4-restart-replay.test.ts @@ -0,0 +1,143 @@ +// Round 4 regression test, ported from /home/tom/today/evals-2026-09-23/link/scratch-r4-0/restart-replay.repro.ts (assertions kept; additions marked). +// Kill -9 mid-Lease (B8): the floor applied the Lease and granted a1, the link journaled the requestKey but died +// before the reply was journaled. The restart takes longer than the lease (orphaned) and less than lease + grace. +// B8 says the journaled key is replayed so the grant is recovered. At startup the link sends its first Heartbeat +// (held set without a1) BEFORE it replays the key, so the floor releases the omitted orphan (rule 7 / L3 a), and the +// replay then returns nothing: one infra release is spent and the job reruns as a2. +import { readFileSync } from "node:fs" +import { Effect, Fiber } from "effect" +import { expect, it } from "vitest" +import { job, SEC, sleep, start, until, world, bodyOf } from "./review-r4-harness.ts" + +it("a journaled requestKey whose grant outlived the lease is recovered on restart", async () => { + const w = world() + w.floor.enqueue(job("r1")) + // link #1: the first capacity>0 Lease reaches the floor, then the process "dies" before the reply lands + let leased = false + const dying: typeof globalThis.fetch = async (input, init) => { + const body = await bodyOf(input, init) + if (/"Lease"/.test(body) && /"capacity":[1-9]/.test(body)) { + await w.floor.fetch(input as any, { ...init, body }) + leased = true + return new Promise(() => {}) // the reply never reaches the journal + } + return w.floor.fetch(input as any, { ...init, body }) + } + const l1 = start(w, undefined, {}, dying) + await until("floor granted a1", () => leased) + Effect.runFork(Fiber.interrupt(l1.fiber)) // kill -9: nothing after this point runs in link #1 + await sleep(50) + const journal1 = readFileSync(l1.dir + "/journal.jsonl", "utf8") + const keyJournaled = /"ev":"lease-key"/.test(journal1) && !/"ev":"lease-replied"/.test(journal1) + // downtime: past the lease (6 s * SEC = 120 ms) and well inside lease + grace (21 s * SEC = 420 ms) + await sleep(8 * SEC) + const stateAtRestart = w.floor.jobs.get("wf-test-r1")!.state + const l2 = start(w, l1.dir) + await until("a Task exists", () => w.ax.tasks.size > 0, 3000).catch(() => {}) + await sleep(200) + const j = w.floor.jobs.get("wf-test-r1")! + const out = { keyJournaled, stateAtRestart, history: j.history, infraSpent: j.infraSpent, tasks: [...w.ax.tasks.keys()], + l2: l2.logs.filter((l) => ["leased", "duplicate-grant", "created", "resume"].includes(String(l.ev))).map((l) => `${l.ev}:${l.leaseId}`) } + console.log(JSON.stringify(out)) + await l2.drain() + w.floor.close() + expect(keyJournaled).toBe(true) + expect(stateAtRestart).toBe("orphaned") + expect(j.history).not.toContain("requeue:omitted") // the grant under the journaled key was released by the first heartbeat + expect(out.tasks).toContain("wf-test-r1-a1") + expect(j.infraSpent).toBe(0) // round 4 addition + expect(out.tasks).not.toContain("wf-test-r1-a2") +}) + +it("control: the same kill with a restart shorter than the lease recovers a1", async () => { + const w = world() + w.floor.enqueue(job("r2")) + let leased = false + const dying: typeof globalThis.fetch = async (input, init) => { + const body = await bodyOf(input, init) + if (/"Lease"/.test(body) && /"capacity":[1-9]/.test(body)) { await w.floor.fetch(input as any, { ...init, body }); leased = true; return new Promise(() => {}) } + return w.floor.fetch(input as any, { ...init, body }) + } + const l1 = start(w, undefined, {}, dying) + await until("floor granted a1", () => leased) + Effect.runFork(Fiber.interrupt(l1.fiber)) + await sleep(2 * SEC) + const l2 = start(w, l1.dir) + await until("a Task exists", () => w.ax.tasks.size > 0, 3000).catch(() => {}) + const j = w.floor.jobs.get("wf-test-r2")! + console.log(JSON.stringify({ control: true, history: j.history, tasks: [...w.ax.tasks.keys()] })) + await l2.drain(); w.floor.close() + expect([...w.ax.tasks.keys()]).toContain("wf-test-r2-a1") +}) + +it("running process: the reply is lost and Lease stays unanswered past the lease while Heartbeat works", async () => { + const w = world() + w.floor.enqueue(job("r3")) + let first = true, blockUntil = 0 + const f: typeof globalThis.fetch = async (input, init) => { + const body = await bodyOf(input, init) + if (/"Lease"/.test(body)) { + if (first && /"capacity":[1-9]/.test(body)) { first = false; await w.floor.fetch(input as any, { ...init, body }); blockUntil = Date.now() + 10 * SEC; throw new TypeError("fetch failed") } + if (Date.now() < blockUntil) throw new TypeError("fetch failed") + } + return w.floor.fetch(input as any, { ...init, body }) + } + const l = start(w, undefined, {}, f) + await until("a Task exists", () => w.ax.tasks.size > 0, 4000).catch(() => {}) + await sleep(100) + const j = w.floor.jobs.get("wf-test-r3")! + console.log(JSON.stringify({ running: true, history: j.history, infraSpent: j.infraSpent, tasks: [...w.ax.tasks.keys()] })) + await l.drain(); w.floor.close() + expect(j.history).not.toContain("requeue:omitted") + expect([...w.ax.tasks.keys()]).toEqual(["wf-test-r3-a1"]) // round 4 addition +}) + +// Round 4 addition: the running-process half needs the floor's rule 6b. Against an L1-only floor (vouchByKey false) +// the residual is still there; this pins that the link-side fix alone does not claim it. +it("control: an L1-only floor that ignores pendingRequestKey still releases the omitted orphan", async () => { + const w = world() + w.floor.vouchByKey = false + w.floor.enqueue(job("r4")) + let first = true, blockUntil = 0 + const f: typeof globalThis.fetch = async (input, init) => { + const body = await bodyOf(input, init) + if (/"Lease"/.test(body)) { + if (first && /"capacity":[1-9]/.test(body)) { first = false; await w.floor.fetch(input as any, { ...init, body }); blockUntil = Date.now() + 10 * SEC; throw new TypeError("fetch failed") } + if (Date.now() < blockUntil) throw new TypeError("fetch failed") + } + return w.floor.fetch(input as any, { ...init, body }) + } + const l = start(w, undefined, {}, f) + await until("a Task exists", () => w.ax.tasks.size > 0, 4000).catch(() => {}) + const j = w.floor.jobs.get("wf-test-r4")! + await l.drain(); w.floor.close() + expect(j.history).toContain("requeue:omitted") +}) + +// Round 4 addition: the startup half does not depend on the floor. Against an L1-only floor (no rule 6b) the journaled +// key is still answered before the first Heartbeat, so the orphan granted under it is listed and re-adopted. +it("restart past the lease against an L1-only floor: the startup replay recovers a1", async () => { + const w = world() + w.floor.vouchByKey = false + w.floor.enqueue(job("r5")) + let leased = false + const dying: typeof globalThis.fetch = async (input, init) => { + const body = await bodyOf(input, init) + if (/"Lease"/.test(body) && /"capacity":[1-9]/.test(body)) { await w.floor.fetch(input as any, { ...init, body }); leased = true; return new Promise(() => {}) } + return w.floor.fetch(input as any, { ...init, body }) + } + const l1 = start(w, undefined, {}, dying) + await until("floor granted a1", () => leased) + Effect.runFork(Fiber.interrupt(l1.fiber)) + await sleep(8 * SEC) + const stateAtRestart = w.floor.jobs.get("wf-test-r5")!.state + const l2 = start(w, l1.dir) + await until("a Task exists", () => w.ax.tasks.size > 0, 3000).catch(() => {}) + await sleep(100) + const j = w.floor.jobs.get("wf-test-r5")! + await l2.drain(); w.floor.close() + expect(stateAtRestart).toBe("orphaned") + expect(l2.has("lease-key-replayed")).toBe(true) + expect(j.history).not.toContain("requeue:omitted") + expect([...w.ax.tasks.keys()]).toEqual(["wf-test-r5-a1"]) +}) diff --git a/pkgs/substrate-apps/src/apps/link/test/review-r4-stale-lost.test.ts b/pkgs/substrate-apps/src/apps/link/test/review-r4-stale-lost.test.ts new file mode 100644 index 00000000..a999b9fd --- /dev/null +++ b/pkgs/substrate-apps/src/apps/link/test/review-r4-stale-lost.test.ts @@ -0,0 +1,67 @@ +// Round 4 regression test, ported from /home/tom/today/evals-2026-09-23/link/scratch-r4-0/stale-lost.repro.ts (assertions kept; additions marked). +// Duplicate/stale delivery: a Heartbeat reply is applied to whatever record holds that leaseId when the reply lands, +// not to the generation the Heartbeat vouched for. Set-up is R2-5a (review-r2.test.ts): a1 fails retryable while a2 +// gen 1 is leased to this link and waits for a1's verdict. One Heartbeat carrying [a2] (gen 1) reaches the floor +// after rule 4b withdrew a2 gen 1 and requeued a2 gen 2, so the floor answers lost:[a2]. Its reply lands after the +// link took the regrant of a2 (gen 2). The link then releases gen 2 as `lost` and never creates it, although the floor +// has gen 2 leased to it: the job is stranded until the lease expires, then released `omitted` (an infra release). +import { Effect } from "effect" +import { expect, it } from "vitest" +import { bodyOf, job, sleep, start, until, world } from "./review-r4-harness.ts" +import type { World } from "./review-r4-harness.ts" + +const rpcOf = async (input: unknown, init?: RequestInit) => { const b = await bodyOf(input, init); return b.includes('"Complete"') ? "Complete" : b.includes('"Lease"') ? "Lease" : b.includes('"Heartbeat"') ? "Heartbeat" : "?" } +const failRetryable = (w: World, name: string) => { + const t = w.ax.tasks.get(name)! + t.exited = true; t.phase = "Failed" + t.conditions = [{ type: "Ready", status: "False", reason: "ResourceExhausted", message: "ResourceExhausted: no free worker" }] +} + +it("a Heartbeat reply computed for a2 gen 1 does not release a2 gen 2", async () => { + const w = world(); w.floor.enqueue(job("1")) + const J = () => w.floor.jobs.get("wf-test-1")! + let holdLease = false, failHeartbeat = false, failComplete = false + let armed = false, hbForwarded = false, hbIntercepted = false + let l: ReturnType + const fetch: typeof globalThis.fetch = async (input, init) => { + const body = await bodyOf(input, init) + const rpc = await rpcOf(input, init) + if (rpc === "Heartbeat" && failHeartbeat) throw new TypeError("fetch failed") + if (rpc === "Complete" && failComplete) return new Response("upstream timeout", { status: 503 }) + if (rpc === "Complete" && armed) await until("a Heartbeat vouching for a2 gen 1 is in flight", () => hbIntercepted, 4000) + if (rpc === "Lease") { while (holdLease) await sleep(2); while (armed && !hbForwarded) await sleep(2) } + if (rpc === "Heartbeat" && armed && !hbIntercepted && body.includes("wf-test-1-a2")) { + hbIntercepted = true + await until("floor withdrew a2 gen 1 and requeued a2 gen 2", () => J().history.includes("withdrawn:a2") && J().state === "queued", 4000) + const res = await w.floor.fetch(input as any, { ...init, body }) + hbForwarded = true // Lease may go now: the floor regrants a2 (gen 2) + await until("link took the regrant", () => l.has("regrant", "wf-test-1-a2"), 4000).catch(() => {}) + return res // the reply (lost:[a2]) lands after the regrant + } + return w.floor.fetch(input as any, { ...init, body }) + } + l = start(w, undefined, {}, fetch) + await until("running", () => w.ax.tasks.get("wf-test-1-a1")?.phase === "Running") + w.floor.down = true + await until("floor requeued attempt 2", () => J().state === "queued" && J().attempt === 2, 4000) + failHeartbeat = true; holdLease = true; failComplete = true + w.floor.down = false + await sleep(80) + failRetryable(w, "wf-test-1-a1") + await until("verdict written", () => l.has("verdict", "wf-test-1-a1")) + holdLease = false + await until("a2 received", () => l.has("leased", "wf-test-1-a2"), 4000) + await until("dispatch waits for the verdict", () => l.has("supersede-waits-for-verdict", "wf-test-1-a2")) + armed = true + failComplete = false; failHeartbeat = false + await until("a2 created, or 1 s", () => w.ax.updates.has("wf-test-1-a2"), 1000).catch(() => {}) + await sleep(300) + const trace = l.logs.filter((x) => x.leaseId === "wf-test-1-a2" || x.task === "wf-test-1-a2").map((x) => `${x.ev}${x.why ? ":" + x.why : ""}`) + const out = { hbIntercepted, floor: J().history, state: J().state, attempt: J().attempt, infraSpent: J().infraSpent, a2Updates: w.ax.updates.get("wf-test-1-a2") ?? 0, a2trace: trace } + console.log(JSON.stringify(out)) + await l.drain(); w.floor.close() + expect(hbIntercepted).toBe(true) + expect(out.a2Updates).toBe(1) + expect(out.floor).not.toContain("requeue:omitted") // round 4 addition + expect(out.a2trace).toContain("stale-heartbeat-reply") +}) diff --git a/pkgs/substrate-apps/src/apps/link/test/workerd.integration.test.ts b/pkgs/substrate-apps/src/apps/link/test/workerd.integration.test.ts new file mode 100644 index 00000000..05142b89 --- /dev/null +++ b/pkgs/substrate-apps/src/apps/link/test/workerd.integration.test.ts @@ -0,0 +1,191 @@ +// Integration: the link against a REAL floor process and a REAL gRPC ax wire. +// floor the option-A prototype Durable Object (../arc/proto/floor, Effect's RpcServer.layerHttp inside a DO, DO +// SQLite, a storage alarm) served by `wrangler dev --local` on loopback (workerd). Copied to a temp dir first, +// so nothing is written into the prototype's own tree. It is NOT the floor port (FL1 to FL8 are not in it). +// ax a gRPC server on loopback built from the vendored ax.proto plus P1's GetTaskResult (fixtures/ax-p1.proto), +// with ax's own ListTasks paging (50 rows, newest first) and a Gateway. The link uses its production grpcAx. +// Opt-in: LINK_WORKERD=1, run inside ~/.local/bin/runtime-test (it starts workerd). Skipped otherwise. +import { spawn } from "node:child_process" +import type { ChildProcess } from "node:child_process" +import { createHash } from "node:crypto" +import { cpSync, mkdtempSync, rmSync, symlinkSync } from "node:fs" +import { tmpdir } from "node:os" +import { join } from "node:path" +import * as grpc from "@grpc/grpc-js" +import * as protoLoader from "@grpc/proto-loader" +import { Deferred, Effect, Exit, Fiber } from "effect" +import { afterAll, beforeAll, describe, expect, it } from "vitest" +import { grpcAx } from "../src/ax.ts" +import type { AgentJob } from "../src/contract.ts" +import { rpcFloor } from "../src/floor.ts" +import { Journal } from "../src/journal.ts" +import { runLink } from "../src/link.ts" +import type { LinkConfig } from "../src/link.ts" + +const RUN = process.env.LINK_WORKERD === "1" && (process.env.LINK_PROTO_DIR ?? "") !== "" +// The ARC prototype floor directory (LINK_PROTO_DIR); the test is skipped without it. +const PROTO_DIR = process.env.LINK_PROTO_DIR ?? "" +const P1_PROTO = join(import.meta.dirname, "fixtures", "ax-p1.proto") +const TOKEN = "local-dev-only-not-a-secret" // the prototype's wrangler.jsonc var, a public test value +const PORT = 18000 + Math.floor(Math.random() * 900) +const F = `http://127.0.0.1:${PORT}` +const A = "ultracode.mecattaf.dev/" + +const job = (n: string): AgentJob => ({ + apiVersion: "ultracode.mecattaf.dev/v1alpha1", kind: "AgentJob", + metadata: { name: `wf-int-${n}`, labels: { [A + "workflow"]: "link-int", [A + "phase-index"]: "1" }, annotations: { [A + "run-id-raw"]: "wf_int", [A + "label"]: `int:${n}`, [A + "item-key"]: `wf_int#${n}`, [A + "journal-key"]: `${"cd".repeat(32)}:1`, [A + "phase-title"]: "Integration" } }, + spec: { "runs-on": ["seat:halogen", "runtime:gvisor"], with: { prompt: `say ${n}`, prompt_ref: { sha256: "cd".repeat(32), bytes: 5, uri: `journal://wf_int/${n}/prompt.md` }, model: "halogen-qwen3.8-flash-next" } } +}) + +// ---------------------------------------------------------------- the ax gRPC server (P1 wire) +const tasks = new Map() +const results = new Map() +let axUnavailable = false +let axServer: grpc.Server +let axPort = 0 +const unavailable = { code: grpc.status.UNAVAILABLE, details: "ax-server unavailable" } +const nf = (what: string) => ({ code: grpc.status.NOT_FOUND, details: `${what} not found` }) +const guard = (cb: grpc.sendUnaryData, f: () => void) => axUnavailable ? cb(unavailable) : f() + +async function startAx() { + const def = protoLoader.loadSync(P1_PROTO, { keepCase: false, longs: String, enums: String, defaults: true, oneofs: true }) + const svc = (grpc.loadPackageDefinition(def) as any).ax.v1alpha1.AX.service + axServer = new grpc.Server() + axServer.addService(svc, { + GetTask: (c: any, cb: any) => guard(cb, () => { const t = tasks.get(c.request.name); t ? cb(null, t) : cb(nf(c.request.name)) }), + UpdateTask: (c: any, cb: any) => guard(cb, () => { tasks.set(c.request.task.metadata.name, { ...c.request.task, status: { phase: "Pending" } }); cb(null, c.request.task) }), + ListTasks: (c: any, cb: any) => guard(cb, () => { const n = Number(c.request.limit) > 0 ? Number(c.request.limit) : 50, o = Number(c.request.offset) || 0; cb(null, { tasks: [...tasks.values()].reverse().slice(o, o + n) }) }), + DeleteTask: (c: any, cb: any) => guard(cb, () => { tasks.delete(c.request.name); results.delete(c.request.name); cb(null, {}) }), + GetGateway: (c: any, cb: any) => guard(cb, () => c.request.name === "halogen" + ? cb(null, { apiVersion: "ax.io/v1alpha1", kind: "Gateway", metadata: { name: "halogen", atespace: "fleet" }, spec: { egress: { allowlist: { hosts: [{ host: "worker", port: 8731 }] } } } }) + : cb(nf("gateway"))), + GetTaskResult: (c: any, cb: any) => guard(cb, () => { const b = results.get(c.request.name); b ? cb(null, { content: b, sha256: createHash("sha256").update(b).digest("hex") }) : cb(nf("result")) }) + }) + axPort = await new Promise((res, rej) => axServer.bindAsync("127.0.0.1:0", grpc.ServerCredentials.createInsecure(), (e, p) => e ? rej(e) : res(p))) +} +const run = (name: string) => { tasks.get(name).status = { phase: "Running" } } +const finish = (name: string, out: unknown) => { + results.set(name, Buffer.from(JSON.stringify(out))) + tasks.get(name).status = { phase: "Completed", usage: { promptTokens: 5, completionTokens: 3, toolCalls: 1 }, conditions: [{ type: "Ready", status: "False", reason: "CommandExited", message: "ExitCode=0" }] } +} + +// ---------------------------------------------------------------- the floor under wrangler dev --local +let work = "", wrangler: ChildProcess | undefined +const floorLog: Array = [] +async function floorUp() { + wrangler = spawn("wrangler", ["dev", "--local", "--ip", "127.0.0.1", "--port", String(PORT), "--persist-to", join(work, "state"), + "--var", "LEASE_SECONDS:30", "--show-interactive-dev-session=false"], + { cwd: join(work, "floor"), env: { ...process.env, WRANGLER_SEND_METRICS: "false", CI: "1" }, detached: true, stdio: ["ignore", "pipe", "pipe"] }) + wrangler.stdout!.on("data", (d) => floorLog.push(String(d))) + wrangler.stderr!.on("data", (d) => floorLog.push(String(d))) + await until("floor up", async () => (await admin("GET", "/admin/state").catch(() => undefined)) !== undefined, 60_000) +} +async function floorDown() { + if (wrangler?.pid) { try { process.kill(-wrangler.pid, "SIGTERM") } catch { /* gone */ } } + await until("floor down", async () => (await admin("GET", "/admin/state").catch(() => undefined)) === undefined, 20_000) +} +async function admin(method: string, path: string, body?: unknown): Promise { + const r = await fetch(F + path, { method, headers: { authorization: `Bearer ${TOKEN}`, "content-type": "application/json" }, ...(body ? { body: JSON.stringify(body) } : {}), signal: AbortSignal.timeout(3000) }) + if (!r.ok) throw new Error(`admin ${path}: ${r.status}`) + return r.json() +} +const jobRow = async (n: string) => ((await admin("GET", "/admin/state")).jobs as Array).find((j) => j.name === `wf-int-${n}`) + +async function until(what: string, pred: () => boolean | Promise, ms = 30_000) { + const t0 = Date.now() + while (!(await pred())) { if (Date.now() - t0 > ms) throw new Error(`timeout waiting for: ${what}`); await new Promise((r) => setTimeout(r, 100)) } +} +const sleep = (ms: number) => new Promise((r) => setTimeout(r, ms)) + +function startLink(token = TOKEN) { + const logs: Array> = [] + const dir = mkdtempSync(join(tmpdir(), "conwip-link-int-")) + const stop = Effect.runSync(Deferred.make()) + const ax = grpcAx(`127.0.0.1:${axPort}`, "fleet", 5_000, P1_PROTO) + const cfg: LinkConfig = { + holder: "nas-link-1", maxInFlight: 2, servedLabels: ["seat:halogen", "runtime:gvisor"], + shape: { atespace: "fleet", image: "ax-agent", gateway: "halogen", command: () => ["ax-agent", "pi"] }, + completion: "p1", secondMs: 1000, resyncMs: 500, pendingTimeoutMs: 60_000, deleteAfterMs: 0, deadlineBackstopMs: 60_000, + createAttempts: 3, outboxBackoffMs: [200, 1000], fenceTimeoutMs: 5_000, initialPollSeconds: 1, initialHeartbeatSeconds: 2 + } + const fiber = Effect.runFork(Effect.scoped(Effect.gen(function*() { + const floor = yield* rpcFloor({ url: F, token, sessionId: `s:${dir}`, timeoutsMs: { lease: 5_000, heartbeat: 5_000, complete: 10_000 } }) + return yield* runLink(cfg, { ax, floor, journal: Journal.open(dir), log: (ev, f) => logs.push({ ev, ...f }), stop, floorUrl: F }) + }))) + return { + logs, + has: (ev: string) => logs.some((l) => l.ev === ev), + exit: () => Effect.runPromise(Fiber.await(fiber)), + drain: async () => { Effect.runSync(Deferred.succeed(stop, undefined)); const e = await Effect.runPromise(Fiber.await(fiber)); ax.close(); rmSync(dir, { recursive: true, force: true }); return e } + } +} + +describe.skipIf(!RUN)("link against the prototype floor under wrangler dev --local and a P1 gRPC ax", () => { + beforeAll(async () => { + work = mkdtempSync(join(tmpdir(), "conwip-floor-int-")) + cpSync(join(PROTO_DIR, "floor"), join(work, "floor"), { recursive: true, filter: (s) => !s.includes(".wrangler") }) + cpSync(join(PROTO_DIR, "contract.ts"), join(work, "contract.ts")) + symlinkSync(join(PROTO_DIR, "node_modules"), join(work, "node_modules")) + await startAx() + await floorUp() + }, 90_000) + afterAll(async () => { + await floorDown().catch(() => undefined) + axServer?.forceShutdown() + if (process.env.LINK_DEBUG) console.log(floorLog.join("").slice(-4000)) + if (work) rmSync(work, { recursive: true, force: true }) + }, 30_000) + + it("I1 happy path over real HTTP RPC and real gRPC: lease, create-only, P1 result with sha256, Complete, janitor delete", async () => { + await admin("POST", "/admin/enqueue", job("1")) + const l = startLink() + await until("created", () => tasks.has("wf-int-1-a1")) + expect(tasks.get("wf-int-1-a1").spec.env.find((e: any) => e.name === "AX_CONWIP_PROMPT").value).toBe("say 1") + run("wf-int-1-a1"); finish("wf-int-1-a1", { answer: 42 }) + await until("floor done", async () => (await jobRow("1"))?.state === "done") + const r = await jobRow("1") + expect([r.result, JSON.parse(r.output), JSON.parse(r.usage), r.attempt]).toEqual(["success", { answer: 42 }, { prompt_tokens: 5, completion_tokens: 3, tool_calls: 1 }, 1]) + await until("janitor deleted the Task", () => !tasks.has("wf-int-1-a1")) + expect(l.has("completion-probe") && l.has("gateway-ok")).toBe(true) + expect(Exit.isSuccess(await l.drain())).toBe(true) + }, 60_000) + + it("I2 floor process restarted while a verdict is pending: the outbox keeps it and it lands once, attempt 1", async () => { + await admin("POST", "/admin/enqueue", job("2")) + const l = startLink() + await until("created", () => tasks.has("wf-int-2-a1")) + run("wf-int-2-a1") + await floorDown() + finish("wf-int-2-a1", { restarted: true }) + await until("outbox keeps", () => l.has("outbox-keep")) + await floorUp() // same --persist-to: the DO's SQLite and alarm survive + await until("floor done", async () => (await jobRow("2"))?.state === "done", 45_000) + const r = await jobRow("2") + expect([r.result, JSON.parse(r.output), r.attempt, r.transitions]).toEqual(["success", { restarted: true }, 1, 0]) + expect(Exit.isSuccess(await l.drain())).toBe(true) + }, 120_000) + + it("I3 ax answers UNAVAILABLE: every Lease asks for 0, the job stays queued; ax returns and the job runs", async () => { + axUnavailable = true + const l = startLink() + await sleep(1_500) // the start-up resync fails; nothing is probed, capacity stays 0 + expect(l.has("completion-probe")).toBe(false) + await admin("POST", "/admin/enqueue", job("3")) + await sleep(6_000) // three polls at the floor's 2 s + expect((await jobRow("3")).state).toBe("queued") + axUnavailable = false + await until("created", () => tasks.has("wf-int-3-a1")) + run("wf-int-3-a1"); finish("wf-int-3-a1", { back: true }) + await until("floor done", async () => (await jobRow("3"))?.state === "done") + expect((await jobRow("3")).result).toBe("success") + expect(Exit.isSuccess(await l.drain())).toBe(true) + }, 60_000) + + it("I4 the real Worker entry answers 401 to a wrong bearer: the link stops with kind auth (exit 78 in main)", async () => { + const l = startLink("not-the-token") + const exit = await l.exit() + expect(Exit.isFailure(exit)).toBe(true) + expect(JSON.stringify(exit)).toContain("\"kind\":\"auth\"") + await l.drain() + }, 30_000) +}) diff --git a/pkgs/substrate-apps/src/apps/link/tsconfig.json b/pkgs/substrate-apps/src/apps/link/tsconfig.json new file mode 100644 index 00000000..eb9a0298 --- /dev/null +++ b/pkgs/substrate-apps/src/apps/link/tsconfig.json @@ -0,0 +1,16 @@ +{ + "compilerOptions": { + "target": "ES2022", + "module": "NodeNext", + "moduleResolution": "NodeNext", + "lib": ["ES2022"], + "strict": true, + "exactOptionalPropertyTypes": false, + "noUncheckedIndexedAccess": true, + "allowImportingTsExtensions": true, + "noEmit": true, + "skipLibCheck": true, + "types": ["node"] + }, + "include": ["src", "test"] +} diff --git a/pkgs/substrate-apps/src/apps/link/vitest.config.ts b/pkgs/substrate-apps/src/apps/link/vitest.config.ts new file mode 100644 index 00000000..7ef18d4f --- /dev/null +++ b/pkgs/substrate-apps/src/apps/link/vitest.config.ts @@ -0,0 +1,8 @@ +import { defineConfig } from "vitest/config" + +export default defineConfig({ + test: { + include: ["test/**/*.test.ts"], + environment: "node" + } +}) diff --git a/pkgs/substrate-apps/src/apps/puller/bin/substrate-puller.mjs b/pkgs/substrate-apps/src/apps/puller/bin/substrate-puller.mjs new file mode 100755 index 00000000..6160ba08 --- /dev/null +++ b/pkgs/substrate-apps/src/apps/puller/bin/substrate-puller.mjs @@ -0,0 +1,10 @@ +#!/usr/bin/env node +// Entry point: registers tsx's ESM loader, then runs src/main.ts. +import { register } from "tsx/esm/api"; +import { fileURLToPath } from "node:url"; +import { dirname, join } from "node:path"; + +register(); +const here = dirname(fileURLToPath(import.meta.url)); +const { main } = await import(join(here, "..", "src", "main.ts")); +process.exitCode = await main(process.env); diff --git a/pkgs/substrate-apps/src/apps/puller/package.json b/pkgs/substrate-apps/src/apps/puller/package.json new file mode 100644 index 00000000..f75a8c72 --- /dev/null +++ b/pkgs/substrate-apps/src/apps/puller/package.json @@ -0,0 +1,14 @@ +{ + "name": "@substrate/puller", + "version": "0.1.0", + "private": true, + "type": "module", + "description": "The coordinator-side interpreter host: leases runs (runtime:interpreter) from the floor over /rpc, runs each with the interpreter and the repo's runners (or dispatches its nodes back to the floor), heartbeats, completes, and resumes from the run's journal after a kill. A plain long-lived process with a pidfile.", + "license": "UNLICENSED", + "bin": { "substrate-puller": "./bin/substrate-puller.mjs" }, + "exports": { ".": "./src/puller.ts", "./*": "./src/*" }, + "scripts": { "check": "tsc --noEmit -p .", "test": "vitest run" }, + "dependencies": { "substrate": "workspace:*", "@substrate/api": "workspace:*", "@substrate/link": "workspace:*", "@substrate/interpreter": "workspace:*", "@substrate/runners": "workspace:*", "tsx": "^4.19.2" }, + "peerDependencies": { "effect": "4.0.0-rc.112" }, + "devDependencies": { "@substrate/floor": "workspace:*", "@types/node": "^22.10.2", "effect": "4.0.0-rc.112", "typescript": "^5.9.3", "vitest": "^3.2.4" } +} diff --git a/pkgs/substrate-apps/src/apps/puller/src/connect.ts b/pkgs/substrate-apps/src/apps/puller/src/connect.ts new file mode 100644 index 00000000..393d4d16 --- /dev/null +++ b/pkgs/substrate-apps/src/apps/puller/src/connect.ts @@ -0,0 +1,13 @@ +// The puller's floor connection: the link's own rpcFloor client (Effect RPC over HTTP, its retry classifier and +// per-call ceilings), held open in a scope for the life of the process. +import { Effect, Exit, Scope } from "effect" +import { rpcFloor } from "@substrate/link/floor.ts" +import type { FloorOptions } from "@substrate/link/floor.ts" +import { floorPort } from "./puller.ts" +import type { FloorPort } from "./puller.ts" + +export const connectFloor = async (o: FloorOptions): Promise<{ port: FloorPort; close: () => Promise }> => { + const scope = await Effect.runPromise(Scope.make()) + const api = await Effect.runPromise(Scope.provide(scope)(rpcFloor(o))) + return { port: floorPort(api), close: () => Effect.runPromise(Scope.close(scope, Exit.void)) } +} diff --git a/pkgs/substrate-apps/src/apps/puller/src/execute.ts b/pkgs/substrate-apps/src/apps/puller/src/execute.ts new file mode 100644 index 00000000..c7ec9641 --- /dev/null +++ b/pkgs/substrate-apps/src/apps/puller/src/execute.ts @@ -0,0 +1,327 @@ +// What a leased run becomes. Two executors, one contract (puller.ts Executor): +// +// local the interpreter with the repo's runners on this host (src/integrated.ts runIntegrated): host, +// runtime-test, herdr, gvisor, ssh:worker for pi on Halogen, microvm, workerd, as runtimes.toml names them. +// The run directory is the puller's runs//, so a second attempt resumes from the first one's journal. +// An ax runtime is refused with the reason: ax Tasks are rendered and dispatched by the link on the NAS, +// which this host is not; use node_dispatch = "floor" to send nodes there. +// floor the interpreter here, every agent() node back to the floor as an AgentJob (POST /runs/:id/jobs), leased +// by whichever link serves its runs-on labels (the NAS link renders and dispatches it to ax) and read back +// from GET /jobs/:name/output. The journal stays here, so resume works the same way. +import { createHash } from "node:crypto" +import { connect } from "node:net" +import { existsSync, lstatSync, mkdirSync, readdirSync, readFileSync, utimesSync, writeFileSync } from "node:fs" +import { join } from "node:path" +import type { SubstrateClient, AgentJob } from "@substrate/api" +import { FileJournal, parseJournal, runWorkflow } from "@substrate/interpreter" +import type { AgentCall, AgentOutcome, Backend } from "@substrate/interpreter" +import { lookupRuntime, refusal, seatOf, seatShadowRoot, transcriptRootFor } from "@substrate/runners" +import type { Runner, RuntimesConfig } from "@substrate/runners" +import type { CapacityGate } from "substrate/src/capacity/gate.ts" +import { runIntegrated } from "substrate/src/integrated.ts" +import { runOutcome } from "substrate/src/run-outcome.ts" +import { Aborted, raceAbort, transient } from "./puller.ts" +import type { Executor, RunVerdict } from "./puller.ts" + +const A = "ultracode.mecattaf.dev/" + +/** Can a TCP connection to host:port open within the timeout? */ +export const reachable = (hostPort: string, timeoutMs = 1500): Promise => new Promise((ok) => { + const i = hostPort.lastIndexOf(":") + const s = connect({ host: hostPort.slice(0, i), port: Number(hostPort.slice(i + 1)) }) + const t = setTimeout(() => { s.destroy(); ok(false) }, timeoutMs) + s.once("connect", () => { clearTimeout(t); s.destroy(); ok(true) }) + s.once("error", () => { clearTimeout(t); ok(false) }) +}) + +export const axRefusal = (axServer: string | undefined, isReachable: boolean | undefined): string => + `ax runtime: this puller does not dispatch ax Tasks; the NAS link renders and dispatches them (create-only, durable verdict outbox). ` + + (axServer === undefined ? "No ax_server is configured here. " : isReachable ? `ax-server ${axServer} is reachable, but dispatch from the coordinator is not its path. ` : `ax-server ${axServer} is not reachable. `) + + `Run with node_dispatch = "floor" so the node is enqueued for the link that serves its runs-on labels.` + +export interface LocalOptions { + readonly runtimes: RuntimesConfig + readonly seat: string + readonly defaultModel: string + readonly cap: number + readonly capacity?: CapacityGate + readonly concurrency?: number + readonly maxAttempts?: number + readonly axServer?: string + /** + * The capacity pusher's demand dir (apps/pusher --demand-dir). While a run executes, a marker there is touched + * every minute, so the pusher reads at its active cadence (300 s), inside the gate's dispatch bound (360 s). + * Without it an idle pusher reads every 900 s and the first node after a quiet spell is refused as stale. + */ + readonly demandDir?: string + /** How long a node waits for capacity before it fails (the CONWIP's capacityWait). Default: no wait. */ + readonly capacityWait?: { readonly delayMs: number; readonly maxWaitMs: number } + /** Runner overrides by runtime name (tests). */ + readonly runners?: Readonly> + /** + * AUDIT-transcripts TX2 for local runs: after the run and before its verdict, every node's archived transcript + * (runners' transcript root, //) is uploaded to the floor under the run's own job, one part + * per file named `__`. Best effort: a failure is named in the verdict, never fails the run; the local + * archive stays the long-term copy. + */ + readonly transcripts?: { readonly client: Pick; readonly root?: string; readonly maxBytes?: number } +} + +export interface TranscriptUpload { readonly uploaded: number; readonly bytes: number; readonly skipped: ReadonlyArray; readonly failed: ReadonlyArray<{ part: string; error: string }> } + +/** A floor part name for one archived file: one segment of [A-Za-z0-9_.-], at most 200 characters. */ +export const transcriptPartName = (jobId: string, file: string): string => { + const p = `${jobId}__${file}`.replace(/[^A-Za-z0-9_.-]/g, "_").replace(/^[^A-Za-z0-9_]/, "_") + return p.length <= 200 ? p : `${p.slice(0, 120)}~${p.slice(p.length - 79)}` +} + +/** Upload `//` (regular files only, never through a link) under job `name`, up to `maxBytes`. */ +export const uploadRunTranscripts = async (client: Pick, name: string, dir: string, maxBytes = 30_000_000): Promise => { + const files: Array<{ part: string; path: string; size: number }> = [] + const ls = (d: string) => { try { return readdirSync(d).sort() } catch { return [] } } + for (const job of ls(dir)) { + const jd = join(dir, job) + if (!lstatSync(jd).isDirectory()) continue + for (const f of ls(jd)) { + const st = lstatSync(join(jd, f)) + if (st.isFile()) files.push({ part: transcriptPartName(job, f), path: join(jd, f), size: st.size }) + } + } + let bytes = 0, uploaded = 0 + const skipped: Array = [], failed: Array<{ part: string; error: string }> = [] + for (const f of files) { + if (bytes + f.size > maxBytes) { skipped.push(f.part); continue } + try { + const r = await client.uploadTranscript(name, f.part, readFileSync(f.path, "utf8")) + bytes += r.bytes; uploaded++ + } catch (e) { + failed.push({ part: f.part, error: String((e as Error).message ?? e).slice(0, 200) }) + } + } + return { uploaded, bytes, skipped, failed } +} + +/** Touch `/` now; never throws (demand is a hint, never a reason to fail a run). */ +export const touchDemand = (dir: string, name: string): void => { + try { + mkdirSync(dir, { recursive: true }) + const f = join(dir, name) + if (existsSync(f)) { const now = new Date(); utimesSync(f, now, now) } else writeFileSync(f, "") + } catch { /* a hint only */ } +} + +export const localExecutor = (o: LocalOptions): Executor => async (t) => { + let demand: ReturnType | undefined + if (o.demandDir !== undefined) { + const dir = o.demandDir + touchDemand(dir, "substrate-puller") + demand = setInterval(() => touchDemand(dir, "substrate-puller"), 60_000) + } + try { return await runLocal(o, t) } finally { if (demand !== undefined) clearInterval(demand) } +} + +const runLocal = async (o: LocalOptions, t: Parameters[0]): ReturnType => { + const axNames = Object.entries(o.runtimes.runtimes).filter(([, r]) => r.type === "ax").map(([n]) => n) + const up = axNames.length > 0 && o.axServer !== undefined ? await reachable(o.axServer) : undefined + const why = axRefusal(o.axServer, up) + const axRunners = Object.fromEntries(axNames.map((n): [string, Runner] => [n, { name: n, type: "ax", refuses: () => why, run: async (job) => refusal(n, job, why) }])) + const run = await runIntegrated({ + scriptPath: t.scriptPath, dir: t.dir, runtimes: o.runtimes, seat: o.seat, defaultModel: o.defaultModel, cap: o.cap, + ...(t.args !== undefined ? { args: t.args } : {}), + ...(o.capacity ? { capacity: o.capacity } : {}), + ...(o.concurrency !== undefined ? { concurrency: o.concurrency } : {}), + ...(o.maxAttempts !== undefined ? { maxAttempts: o.maxAttempts } : {}), + ...(o.capacityWait !== undefined ? { capacityWait: o.capacityWait } : {}), + runners: { ...axRunners, ...(o.runners ?? {}) }, + runIdPrefix: `${t.runId}-`, + signal: t.signal + }) + let transcripts: TranscriptUpload | undefined + if (o.transcripts !== undefined) { + const root = transcriptRootFor(seatShadowRoot(o.runtimes), o.transcripts.root) + const dir = join(root, run.runId.replace(/[^A-Za-z0-9_.-]/g, "_").slice(0, 128)) + try { transcripts = await uploadRunTranscripts(o.transcripts.client, t.job.metadata.name, dir, o.transcripts.maxBytes) } catch (e) { + transcripts = { uploaded: 0, bytes: 0, skipped: [], failed: [{ part: "*", error: String((e as Error).message ?? e).slice(0, 200) }] } + } + } + // codex review 3, C3-9: success is conwip-run's exit 0 (every call done or cached), not "the script returned": + // a refused, failed or nulled call makes the run partial, all-failed or refused, and the verdict says failure + const ok = run.exitCode === 0 + return { + result: ok ? "success" : "failure", + output: { + runId: t.runId, interpreterRunId: run.runId, resumed: run.resumed, outcome: run.outcome, callCounts: run.callCounts, dispatched: run.dispatched, + status: run.result.status, result: run.result.result ?? null, error: run.result.error ?? null, + ...(transcripts !== undefined ? { transcripts } : {}), + ...(ok ? {} : { reason: "agent/script-failed", message: run.result.error ?? `the run ended ${run.outcome}` }) + } + } +} + +// ---------------------------------------------------------------- floor dispatch + +export interface FloorBackendOptions { + readonly client: Pick + readonly runId: string + readonly workflow: string + /** runs-on for a node that names none (opts.runsOn, or opts.seat and opts.runtime). */ + readonly defaultRunsOn: ReadonlyArray + readonly defaultModel: string + /** + * The puller's runtimes file, when it has one: agent({runtime: X}) is labelled with the seat X's table spends + * (critique pass 2026-09-24, RG-3: it borrowed node_runs_on's seat, so {runtime:'codex'} became seat:halogen). + */ + readonly runtimes?: RuntimesConfig + readonly pollMs?: number + readonly signal: AbortSignal +} + +const sha = (s: string) => createHash("sha256").update(s).digest("hex") +const str = (v: unknown) => (typeof v === "string" && v !== "" ? v : undefined) + +/** + * The call's content identity: `:`, from the interpreter's cid (prompt, keyed opts and route, + * plus the occurrence among identical calls). It does not depend on the order earlier calls were invoked or finished, + * so a resume that releases a pipeline in another order names the same node (codex review 3, C3-7). It is also the + * form the NAS link requires of the journal-key annotation (apps/link jobs.ts JOURNAL_KEY). + */ +export const contentKey = (call: AgentCall): string => { + const m = /^c1:([0-9a-f]{64})#([1-9][0-9]*)$/.exec(call.cid ?? "") + return m ? `${m[1]}:${m[2]}` : `${sha(`${call.key}\0${call.index}`)}:1` +} + +/** One agent() call as an AgentJob. The name is fixed by (content identity, attempt, run), so a resumed call + * re-enqueues the same job (idempotent by name) and reads the verdict the floor may already hold. */ +export class RouteRefused extends Error {} + +/** + * The seat label a node spends. A named seat is taken as named. A named runtime takes the seat its table spends in the + * puller's runtimes file; with no file, or a runtime the file cannot bind to a seat, it is refused rather than given + * node_runs_on's seat, so local and floor dispatch never disagree on what a node spends (RG-3). + */ +const seatLabelFor = (opts: Record, o: Pick): string => { + const seat = str(opts.seat), runtime = str(opts.runtime) + if (runtime !== undefined) { + const r = o.runtimes !== undefined ? lookupRuntime(o.runtimes, runtime) : undefined + const spends = r !== undefined && o.runtimes !== undefined ? seatOf(o.runtimes, r) : undefined + if (seat !== undefined) { + if (spends !== undefined && spends !== seat) throw new RouteRefused(`agent({runtime: ${runtime}, seat: ${seat}}) refused: runtime ${runtime} spends seat ${spends}`) + return seat + } + if (spends === undefined) throw new RouteRefused(`agent({runtime: ${runtime}}) refused: no seat is resolvable for runtime ${runtime} (${o.runtimes === undefined ? "the puller has no runtimes file" : "its table binds no seat"}); name agent({seat}) or bind one`) + return spends + } + return seat ?? o.defaultRunsOn.find((l) => l.startsWith("seat:"))?.slice(5) ?? "unknown" +} + +// Critique pass 2026-09-24 (red team durability-r1-1): nothing in a node's body depends on its invocation index, so a +// resumed run whose calls land in another order re-enqueues the same body under the same name. +export const nodeJob = (call: AgentCall, o: Pick): AgentJob => { + const opts = call.opts as Record + const runsOn = Array.isArray(opts.runsOn) ? opts.runsOn.map(String) + : str(opts.seat) || str(opts.runtime) ? [`seat:${seatLabelFor(opts, o)}`, ...(str(opts.runtime) ? [`runtime:${str(opts.runtime)}`] : o.defaultRunsOn.filter((l) => !l.startsWith("seat:")))] + : [...o.defaultRunsOn] + const digest = sha(call.prompt) + const w: Record = { prompt: call.prompt, prompt_ref: { sha256: digest, bytes: Buffer.byteLength(call.prompt), uri: `journal://${o.runId}/${contentKey(call)}/prompt.md` }, model: str(opts.model) ?? o.defaultModel } + if (str(opts.effort)) w.effort = opts.effort + if (opts.schema !== undefined) w.schema = opts.schema + if (str(opts.isolation)) w.isolation = opts.isolation + if (str(opts.agentType)) w["agent-type"] = opts.agentType + return { + apiVersion: "ultracode.mecattaf.dev/v1alpha1", kind: "AgentJob", + metadata: { + name: `n${contentKey(call).slice(0, 16)}-o${contentKey(call).split(":")[1]}-a${call.attempt}-${o.runId}`.toLowerCase(), + labels: { [A + "run-id"]: o.runId, [A + "workflow"]: o.workflow.toLowerCase().replace(/[^a-z0-9.-]+/g, "-").slice(0, 63), [A + "kind"]: "agent" }, + annotations: { [A + "run-id-raw"]: o.runId, [A + "journal-key"]: contentKey(call), ...(call.opts.label ? { [A + "label"]: String(call.opts.label) } : {}), ...(call.phase ? { [A + "phase"]: call.phase } : {}) } + }, + spec: { "runs-on": runsOn, with: w as AgentJob["spec"]["with"] } + } +} + +/** The verdict as the interpreter's AgentOutcome. A failure is the call's error; the interpreter owns retries. */ +export const outcomeOf = (o: { result: string; output: unknown; usage: unknown }, wantObject: boolean): AgentOutcome => { + const u = o.usage as { prompt_tokens?: number; completion_tokens?: number } | null + // TX3: the harness session id, when the executor reported one, is journaled as the call's agentId + const sid = (o.output as { sessionId?: unknown } | null)?.sessionId + const usage = { ...(u && typeof u.prompt_tokens === "number" ? { usage: { inputTokens: u.prompt_tokens, outputTokens: u.completion_tokens ?? 0 } } : {}), + ...(typeof sid === "string" && sid !== "" ? { agentId: sid } : {}) } + if (o.result !== "success") { + const f = (o.output ?? {}) as { reason?: string; message?: string } + return { error: `${o.result}: ${f.reason ?? "no reason"}${f.message ? `: ${f.message}` : ""}`, ...usage } + } + const v = o.output as { object?: unknown; text?: unknown } | string | null + if (wantObject) return { object: v !== null && typeof v === "object" && "object" in v ? v.object : v, ...usage } + return { text: typeof v === "string" ? v : v !== null && typeof v === "object" && typeof v.text === "string" ? v.text : JSON.stringify(v), ...usage } +} + +export class FloorBackend implements Backend { + readonly name = "floor" + readonly enqueued: Array = [] + constructor(readonly o: FloorBackendOptions) {} + /** + * After an abort nothing more happens: no enqueue, no read, no answer, so the interpreter journals nothing more and + * dispatches no continuation (codex review 3, C3-11). The call stays started-only, as a real kill leaves it. + */ + private parked = (): Promise => new Promise(() => undefined) + /** One floor request, retried while the run lives when it fails transiently (codex review 3, C3-10). */ + private async req(f: () => Promise): Promise { + for (let wait = this.o.pollMs ?? 2000; ; wait = Math.min(wait * 2, 30_000)) { + if (this.o.signal.aborted) return this.parked() + try { return await raceAbort(f(), this.o.signal) } catch (e) { + if (e instanceof Aborted || this.o.signal.aborted) return this.parked() + // the floor refuses nodes for a run that ended or is being cancelled (C2-13): the heartbeat brings the cancel + // or the loss that aborts this run, so the call waits for it instead of failing the script first + const code = (e as { code?: unknown })?.code + if (code === "run-cancelling" || code === "run-done") return this.parked() + if (!transient(e)) throw e + await new Promise((ok) => { const t = setTimeout(ok, wait); this.o.signal.addEventListener("abort", () => { clearTimeout(t); ok(undefined) }, { once: true }) }) + } + } + } + async run(call: AgentCall): Promise { + let job: AgentJob + try { job = nodeJob(call, this.o) } catch (e) { if (e instanceof RouteRefused) return { error: e.message }; throw e } + const name = job.metadata.name + await this.req(() => this.o.client.enqueue(this.o.runId, [job])) // idempotent by name: a lost answer is resent safely + this.enqueued.push(name) + for (;;) { + const v = await this.req(() => this.o.client.job(name)) + if (v.state === "done") { const out = await this.req(() => this.o.client.output(name)); return outcomeOf(out, call.opts.schema !== undefined) } + await new Promise((ok) => { const t = setTimeout(ok, this.o.pollMs ?? 2000); this.o.signal.addEventListener("abort", () => { clearTimeout(t); ok(undefined) }, { once: true }) }) + if (this.o.signal.aborted) return this.parked() + } + } +} + +export interface FloorExecOptions extends Omit { + readonly concurrency?: number + readonly maxAttempts?: number + /** Observe each run's backend (tests). */ + readonly onBackend?: (b: FloorBackend) => void +} + +export const floorExecutor = (o: FloorExecOptions): Executor => async (t) => { + // G-BK5: every node is enqueued under this run's live lease, so a fenced or superseded attempt cannot add nodes + const client = typeof (o.client as { underLease?: unknown }).underLease === "function" + ? (o.client as unknown as { underLease: (l: string, a: number) => FloorBackendOptions["client"] }).underLease(t.leaseId, t.attempt) : o.client + const backend = new FloorBackend({ ...o, client, runId: t.runId, workflow: t.workflowName, signal: t.signal }) + o.onBackend?.(backend) + const journalPath = join(t.dir, "journal.jsonl") + const resumed = existsSync(journalPath) && readFileSync(journalPath, "utf8").trim() !== "" + const aborted = new Promise<"aborted">((ok) => { if (t.signal.aborted) ok("aborted"); else t.signal.addEventListener("abort", () => ok("aborted"), { once: true }) }) + const r = await Promise.race([runWorkflow(t.script, { + backend, runId: `wf_${t.runId}`, scriptPath: t.scriptPath, journal: new FileJournal(journalPath), cacheIdentity: "content", defaultModel: o.defaultModel, + ...(resumed ? { resumeFrom: parseJournal(readFileSync(journalPath, "utf8")) } : {}), + ...(t.args !== undefined ? { args: t.args } : {}), + ...(o.concurrency !== undefined ? { concurrency: o.concurrency } : {}), + ...(o.maxAttempts !== undefined ? { maxAttempts: o.maxAttempts } : {}) + }), aborted]) + if (r === "aborted") return { result: "cancelled", output: { runId: t.runId, reason: "aborted" } } satisfies RunVerdict + const calls = r.calls.map((c) => c.state) + const oc = runOutcome(r.status, r.calls, r.events, r.result) // codex review 3, C3-9: success only when every call is done or cached + return { + result: oc.code === 0 ? "success" : "failure", + output: { runId: t.runId, resumed, status: r.status, result: r.result ?? null, error: r.error ?? null, calls: { total: calls.length, done: calls.filter((s) => s === "done").length, cached: calls.filter((s) => s === "cached").length, null: calls.filter((s) => s === "null").length }, enqueued: backend.enqueued.length, + outcome: oc.outcome, ...(oc.code === 0 ? {} : { reason: "agent/script-failed", message: r.error ?? `the run ended ${oc.outcome}` }) } + } +} diff --git a/pkgs/substrate-apps/src/apps/puller/src/health.ts b/pkgs/substrate-apps/src/apps/puller/src/health.ts new file mode 100644 index 00000000..3fcc704c --- /dev/null +++ b/pkgs/substrate-apps/src/apps/puller/src/health.ts @@ -0,0 +1,37 @@ +// G-BK7: the puller's loopback health endpoint. GET /status.json is Puller.status(); GET /metrics is the same in +// Prometheus text. Loopback only: a non-loopback address is refused, and nothing here carries a secret. +import { createServer } from "node:http" +import type { Puller, PullerStatus } from "./puller.ts" + +export const prometheusOf = (s: PullerStatus): string => { + const h = JSON.stringify(s.holder) + const lines = [ + "# TYPE substrate_puller_held gauge", `substrate_puller_held{holder=${h}} ${s.held}`, + "# TYPE substrate_puller_active gauge", `substrate_puller_active{holder=${h}} ${s.active.length}`, + "# TYPE substrate_puller_outbox gauge", `substrate_puller_outbox{holder=${h}} ${s.outbox}`, + "# TYPE substrate_puller_draining gauge", `substrate_puller_draining{holder=${h}} ${s.state === "draining" ? 1 : 0}`, + "# TYPE substrate_puller_heartbeat_failures_consecutive gauge", `substrate_puller_heartbeat_failures_consecutive{holder=${h}} ${s.heartbeatFailures}`, + "# TYPE substrate_puller_events_total counter", + ...Object.entries(s.counters).sort().map(([k, v]) => `substrate_puller_events_total{holder=${h},event=${JSON.stringify(k)}} ${v}`) + ] + return lines.join("\n") + "\n" +} + +export const serveHealth = (addr: string, puller: Pick): Promise<{ addr: string; close: () => void }> => { + const m = /^(127\.0\.0\.1|localhost|\[::1\]):(\d+)$/.exec(addr) + if (!m) return Promise.reject(new Error(`health_addr ${addr} must be a loopback host:port`)) + const host = m[1] === "[::1]" ? "::1" : m[1]! + const srv = createServer((req, res) => { + const path = (req.url ?? "/").split("?")[0] + if (req.method === "GET" && path === "/status.json") { res.writeHead(200, { "content-type": "application/json" }); res.end(JSON.stringify(puller.status())); return } + if (req.method === "GET" && path === "/metrics") { res.writeHead(200, { "content-type": "text/plain; version=0.0.4" }); res.end(prometheusOf(puller.status())); return } + res.writeHead(404).end() + }) + return new Promise((ok, no) => { + srv.once("error", no) + srv.listen(Number(m[2]), host, () => { + const a = srv.address() + ok({ addr: typeof a === "object" && a ? `${host}:${a.port}` : addr, close: () => srv.close() }) + }) + }) +} diff --git a/pkgs/substrate-apps/src/apps/puller/src/logship.ts b/pkgs/substrate-apps/src/apps/puller/src/logship.ts new file mode 100644 index 00000000..3b7fbe67 --- /dev/null +++ b/pkgs/substrate-apps/src/apps/puller/src/logship.ts @@ -0,0 +1,93 @@ +// G-BK3, Buildkite's log chunk upload: while a run holds its lease, the puller tails a file of the run's directory +// (the interpreter's events.jsonl by default) and appends it to the floor in chunks, so the CLI, MCP and the operator +// see progress before the verdict. Each chunk names its lease and attempt; the floor takes it only while that attempt +// is live (a fenced puller cannot write), and (leaseId, seq) makes a resend a duplicate, never a second copy. +// Progress is persisted per lease, so a restart of the same attempt continues where it stopped. +import { closeSync, existsSync, openSync, readFileSync, readSync, statSync, writeFileSync } from "node:fs" +import { join } from "node:path" + +export type AppendLog = (name: string, c: { leaseId: string; attempt: number; seq: number; data: string }) => Promise + +export interface LogShipperOptions { + readonly append: AppendLog + readonly name: string + readonly leaseId: string + readonly attempt: number + readonly dir: string + readonly file?: string + readonly intervalMs?: number + readonly maxChunkBytes?: number + readonly log?: (ev: string, f?: Record) => void +} + +export class LogShipper { + #seq = 0 + #offset = 0 + #timer: ReturnType | undefined + #busy: Promise | undefined + #stopped: string | undefined + failures = 0 + readonly #state: string + constructor(readonly o: LogShipperOptions) { + this.#state = join(o.dir, `log-ship.${o.leaseId}.json`) + try { const s = JSON.parse(readFileSync(this.#state, "utf8")) as { seq: number; offset: number }; this.#seq = s.seq; this.#offset = s.offset } catch { /* fresh */ } + } + get stoppedWhy() { return this.#stopped } + get shipped() { return { seq: this.#seq, offset: this.#offset } } + start() { this.#timer = setInterval(() => void this.tick(), this.o.intervalMs ?? 2_000); this.#timer.unref?.() } + + /** Ship what is new, one chunk at a time, cut at the last newline (a line is never split across chunks). */ + tick(final = false): Promise { + if (this.#busy) return this.#busy + const run = async () => { + await Promise.resolve() // #busy is set before the body runs, so its finally clears this pass and no other + try { + for (;;) { + if (this.#stopped !== undefined) return + const data = this.read(final) + if (data === undefined) return + try { + await this.o.append(this.o.name, { leaseId: this.o.leaseId, attempt: this.o.attempt, seq: this.#seq, data: data.toString("utf8") }) + } catch (e) { + const status = (e as { status?: number })?.status, code = (e as { code?: string })?.code + if (status === 409 || status === 413) { this.#stopped = code ?? `http-${status}`; this.o.log?.("log-ship-stopped", { leaseId: this.o.leaseId, code: this.#stopped }); return } + this.failures++ + return // transient: the next tick sends the same seq again + } + this.#seq++; this.#offset += data.length + writeFileSync(this.#state, JSON.stringify({ seq: this.#seq, offset: this.#offset })) + } + } finally { this.#busy = undefined } + } + this.#busy = run() + return this.#busy + } + + private read(final: boolean): Buffer | undefined { + const path = join(this.o.dir, this.o.file ?? "events.jsonl") + if (!existsSync(path)) return undefined + const size = statSync(path).size + if (size <= this.#offset) return undefined + const max = this.o.maxChunkBytes ?? 64_000 + const n = Math.min(max, size - this.#offset) + const buf = Buffer.alloc(n) + const fd = openSync(path, "r") + try { readSync(fd, buf, 0, n, this.#offset) } finally { closeSync(fd) } + const nl = buf.lastIndexOf(0x0a) + if (nl >= 0) return buf.subarray(0, nl + 1) + // no newline: a line longer than a chunk goes whole-chunk; a short partial line waits unless this is the last pass + return n === max || final ? buf : undefined + } + + /** The run ended: ship the rest (bounded), then stop. Returns how many sends failed transiently. */ + async stop(): Promise { + if (this.#timer) clearInterval(this.#timer) + for (let i = 0; i < 3 && this.#stopped === undefined; i++) { + const before = this.#seq + await this.tick(true) + if (this.#seq === before) break + } + this.#stopped ??= "stopped" + return this.failures + } +} diff --git a/pkgs/substrate-apps/src/apps/puller/src/main.ts b/pkgs/substrate-apps/src/apps/puller/src/main.ts new file mode 100644 index 00000000..ef4ccea3 --- /dev/null +++ b/pkgs/substrate-apps/src/apps/puller/src/main.ts @@ -0,0 +1,81 @@ +// substrate-puller: a plain long-lived process. Configuration is the [puller] table of +// ~/.config/substrate/config.toml (see deploy/client.config.example.toml); the operator bearer and the per-link +// token are files (mode 600), read once and never logged. Logs are JSON lines on stdout. +// +// Exit codes: 0 stopped on SIGTERM or SIGINT; 3 another puller holds the pidfile; 75 another session holds this +// holder identity (L5); 78 configuration invalid, or the floor refused the token or answered a redirect. +import { homedir } from "node:os" +import { join } from "node:path" +import { ConfigError, clientFromConfig, expandHome, loadClientConfig, readSecretFile } from "@substrate/api" +import { loadRuntimes } from "@substrate/runners" +import { CapacityGate } from "substrate/src/capacity/gate.ts" +import { defaultCapacitySource } from "substrate/src/capacity/config.ts" +import { killLiveGroups } from "@substrate/runners" +import { connectFloor } from "./connect.ts" +import { floorExecutor, localExecutor } from "./execute.ts" +import { acquirePidfile, PidfileHeld } from "./pidfile.ts" +import { Puller } from "./puller.ts" +import type { Executor } from "./puller.ts" +import { PullerState } from "./state.ts" +import { serveHealth } from "./health.ts" + +const log = (ev: string, f: Record = {}) => process.stdout.write(JSON.stringify({ t: new Date().toISOString(), ev, ...f }) + "\n") + +export const main = async (env: NodeJS.ProcessEnv = process.env): Promise => { + let release: (() => void) | undefined + try { + const cfg = loadClientConfig(env) + const p = cfg.puller ?? {} + if (!p.holder) throw new ConfigError("[puller] holder is required (the holder name its link token is bound to)") + if (!p.link_token_file) throw new ConfigError("[puller] link_token_file is required") + if (!cfg.floor_url) throw new ConfigError("floor_url is required") + const stateDir = expandHome(p.state_dir ?? "~/.local/state/substrate/puller") + const client = clientFromConfig(cfg) + const linkToken = readSecretFile(expandHome(p.link_token_file), "[puller] link_token_file") + release = acquirePidfile(expandHome(p.pidfile ?? join(homedir(), ".local/state/substrate/puller.pid"))) + const state = new PullerState(stateDir) + let execute: Executor + if ((p.node_dispatch ?? "local") === "floor") { + execute = floorExecutor({ client, defaultRunsOn: p.node_runs_on ?? ["seat:halogen", "runtime:gvisor"], defaultModel: p.default_model ?? "claude-opus-5-5", pollMs: 5000, + ...(p.runtimes ? { runtimes: loadRuntimes(expandHome(p.runtimes)) } : {}) }) + } else { + if (!p.seat) throw new ConfigError("[puller] seat is required for node_dispatch = \"local\" (the seat the capacity gate reads)") + const runtimes = loadRuntimes(p.runtimes ? expandHome(p.runtimes) : undefined) + let capacity: CapacityGate + try { capacity = new CapacityGate(defaultCapacitySource(), p.seat) } catch (e) { throw new ConfigError(String((e as Error).message)) } + const waitS = p.capacity_wait_s ?? 600 + execute = localExecutor({ + runtimes, seat: p.seat, defaultModel: p.default_model ?? "claude-opus-5-5", cap: p.cap ?? 2, capacity, + demandDir: expandHome(p.demand_dir ?? join(homedir(), ".local/state/substrate/demand")), + ...(waitS > 0 ? { capacityWait: { delayMs: 30_000, maxWaitMs: waitS * 1000 } } : {}), + ...(p.ax_server ? { axServer: p.ax_server } : {}), + transcripts: { client } // TX2: node transcripts reach the floor before the run's verdict + }) + } + const floor = await connectFloor({ url: cfg.floor_url, token: linkToken, sessionId: state.sessionId() }) + const puller = new Puller({ holder: p.holder, maxRuns: p.max_runs ?? 1, stateDir, pollMs: 5000, heartbeatMs: 10_000, serverPoll: true }, + { floor: floor.port, client, execute, log, appendLog: (name, c) => client.appendLog(name, c) }) + // G-BK4, Buildkite's stop ladder: the first signal drains (no new leases; runs finish up to drain_timeout_s), the + // second aborts every run (each harness gets its cancel grace, G-BK1), the third SIGKILLs every group and exits. + let signals = 0 + const drainS = p.drain_timeout_s ?? 60 + for (const sig of ["SIGTERM", "SIGINT", "SIGHUP"] as const) process.on(sig, () => { + signals++ + if (signals >= 3) { killLiveGroups("SIGKILL"); process.exit(130) } + if (signals === 2) { log("stop-requested", { signal: sig }); puller.stop(); return } + log("drain-requested", { signal: sig, drainTimeoutS: drainS }); puller.drain(drainS * 1000) + }) + const health = p.health_addr ? await serveHealth(p.health_addr, puller) : undefined + if (health) log("health", { addr: health.addr }) + log("start", { holder: p.holder, stateDir, nodeDispatch: p.node_dispatch ?? "local", maxRuns: p.max_runs ?? 1, held: Object.keys(puller.held).length }) + try { await puller.run() } finally { await floor.close(); health?.close() } + log("stopped") + return 0 + } catch (e) { + if (e instanceof PidfileHeld) { log("pidfile-held", { pid: e.pid }); return 3 } + if (e instanceof ConfigError) { log("config-invalid", { why: e.message }); return 78 } + const kind = (e as { kind?: string })?.kind + log("fatal", { kind: kind ?? "defect", message: String((e as Error)?.message ?? e) }) + return kind === "session-conflict" ? 75 : kind === "auth" || kind === "redirect" || kind === "misrouted" ? 78 : 1 + } finally { release?.() } +} diff --git a/pkgs/substrate-apps/src/apps/puller/src/pidfile.ts b/pkgs/substrate-apps/src/apps/puller/src/pidfile.ts new file mode 100644 index 00000000..7b4d8246 --- /dev/null +++ b/pkgs/substrate-apps/src/apps/puller/src/pidfile.ts @@ -0,0 +1,34 @@ +// One puller per state directory and holder: a pidfile, taken at start and removed at exit. A pidfile whose pid is +// gone, or is no longer a substrate-puller, is stale and replaced. +import { linkSync, readFileSync, rmSync, writeFileSync, mkdirSync } from "node:fs" +import { dirname } from "node:path" + +export class PidfileHeld extends Error { constructor(readonly pid: number, path: string) { super(`another puller (pid ${pid}) holds ${path}`) } } + +const alive = (pid: number, marker: string): boolean => { + try { process.kill(pid, 0) } catch (e) { return (e as NodeJS.ErrnoException).code === "EPERM" } + try { return readFileSync(`/proc/${pid}/cmdline`, "utf8").includes(marker) } catch { return true } +} + +export const acquirePidfile = (path: string, marker = "substrate-puller", self = process.pid): (() => void) => { + mkdirSync(dirname(path), { recursive: true }) + // codex review 3, C3-3: the pid is written to a private file and hard-linked into place. link(2) fails with EEXIST + // when the pidfile exists, so two starts racing past a missing or stale pidfile cannot both win, and a reader + // never sees a pidfile without its pid. A stale pidfile is removed and the link is tried again. + const tmp = `${path}.${self}.tmp` + writeFileSync(tmp, `${self}\n`, { mode: 0o644 }) + try { + for (let tries = 0; ; tries++) { + try { linkSync(tmp, path); break } catch (e) { + if ((e as NodeJS.ErrnoException).code !== "EEXIST" || tries >= 5) throw e + let text: string + try { text = readFileSync(path, "utf8").trim() } catch { continue } // removed meanwhile: race again + const pid = Number(text) + if (pid === self) break + if (Number.isInteger(pid) && pid > 0 && alive(pid, marker)) throw new PidfileHeld(pid, path) + try { if (readFileSync(path, "utf8").trim() === text) rmSync(path) } catch { /* gone */ } + } + } + } finally { rmSync(tmp, { force: true }) } + return () => { try { if (readFileSync(path, "utf8").trim() === String(self)) rmSync(path) } catch { /* gone */ } } +} diff --git a/pkgs/substrate-apps/src/apps/puller/src/puller.ts b/pkgs/substrate-apps/src/apps/puller/src/puller.ts new file mode 100644 index 00000000..65089cb9 --- /dev/null +++ b/pkgs/substrate-apps/src/apps/puller/src/puller.ts @@ -0,0 +1,493 @@ +// The puller: the coordinator-side interpreter host (LINK-DESIGN: a holder advertising runtime:interpreter). +// +// It speaks the link's wire, unchanged: Lease with a journaled requestKey (Forgejo request key; a restart re-sends +// it and the floor answers the same grants, rule 2b), Heartbeat with the complete set it holds (level-triggered, +// ARC; an omitted lease is released, L3), Complete from a durable verdict outbox (Temporal RespondActivityTask*; a +// verdict survives a kill and is resent until the floor accepts or refuses it for good). +// +// A grant is one run. The puller fetches the run's script from the floor, checks it against the grant's content +// address, and hands it to an Executor with the run's own directory. The directory is keyed by run id, not by +// lease, so attempt n+1 of a run (after a kill, a lost lease or a restart) finds attempt n's journal and resumes +// from it: finished agent() calls are cache hits and are never dispatched again. +import { createHash, randomUUID } from "node:crypto" +import { writeFileSync } from "node:fs" +import { join } from "node:path" +import { Effect } from "effect" +import type { SubstrateClient } from "@substrate/api" +import type { AgentJob, Grant } from "@substrate/link/contract.ts" +import type { CompletePayload, FloorApi, LeaseReply } from "@substrate/link/floor.ts" +import { CompleteRefused, FloorError } from "@substrate/link/floor.ts" +import { PullerState } from "./state.ts" +import { LogShipper } from "./logship.ts" +import type { AppendLog } from "./logship.ts" + +export const A = "ultracode.mecattaf.dev/" +export const INTERPRETER_LABEL = "runtime:interpreter" + +/** The floor's three calls as promises; a failure rejects with the link's own FloorError or CompleteRefused. */ +export interface FloorPort { + lease(p: { holderIdentity: string; capacity: number; requestKey: string }): Promise + heartbeat(p: { holderIdentity: string; leaseIds: ReadonlyArray; pendingRequestKey?: string }): Promise<{ renewed: ReadonlyArray; lost: ReadonlyArray; cancelRequested: ReadonlyArray }> + complete(p: CompletePayload): Promise<{ duplicate: boolean }> +} +const settle = (eff: Effect.Effect): Promise => + Effect.runPromise(Effect.match(eff, { onFailure: (e) => ({ e }), onSuccess: (a) => ({ a }) })).then((r) => { if ("e" in r) throw r.e; return r.a }) +export const floorPort = (api: FloorApi): FloorPort => ({ + lease: (p) => settle(api.lease(p)), + heartbeat: (p) => settle(api.heartbeat(p)), + complete: (p) => settle(api.complete(p)) +}) + +export interface RunTask { + readonly runId: string + readonly leaseId: string + readonly attempt: number + readonly job: AgentJob + readonly workflowName: string + readonly args: unknown + /** The run's directory; `script.js` is written there before the executor starts. */ + readonly dir: string + readonly scriptPath: string + readonly script: string + readonly signal: AbortSignal +} +export interface RunVerdict { readonly result: "success" | "failure" | "cancelled"; readonly output: unknown } +export type Executor = (t: RunTask) => Promise + +export interface PullerConfig { + readonly holder: string + readonly maxRuns: number + readonly stateDir: string + readonly pollMs?: number + readonly heartbeatMs?: number + /** Verdict outputs above this are cut to a preview (the floor's row limit is 2 MB). */ + readonly maxOutputBytes?: number + /** + * G-BK4: honour the floor's nextPollSeconds (a paused holder is told 300 s) with +-10 % jitter, bounded below by + * pollMs and above by maxPollMs (default 300000). Off: poll every pollMs, as before. A freed slot always wakes the loop. + */ + readonly serverPoll?: boolean + readonly maxPollMs?: number + /** G-BK3: tail this file of the run's directory to the floor's live log while the run holds its lease (default + * events.jsonl). Needs deps.appendLog; false turns it off. */ + readonly logFile?: string | false + readonly logIntervalMs?: number + /** Red team loss-and-wip r2-2 (KEEP-3): server errors one verdict may collect before it is replaced by a small + * `infra/verdict-undeliverable` failure naming its output's sha256, and that replacement before it is dropped. Default 8. */ + readonly verdictAttempts?: number +} +export interface PullerDeps { + readonly floor: FloorPort + readonly client: Pick + readonly execute: Executor + readonly log?: (ev: string, f?: Record) => void + /** G-BK3: the floor's live-log append (SubstrateClient.appendLog). Absent: no live log. */ + readonly appendLog?: AppendLog +} + +type Why = "lost" | "cancel" | "stop" | "superseded" + +/** The run was aborted (lost, cancel, stop, superseded) while waiting on the floor. */ +export class Aborted extends Error { constructor() { super("aborted") } } +/** Resolve with `p`, or reject with Aborted as soon as `signal` aborts (the request itself is left to settle). */ +export const raceAbort = (p: Promise, signal: AbortSignal): Promise => signal.aborted ? Promise.reject(new Aborted()) : new Promise((ok, no) => { + const on = () => no(new Aborted()) + signal.addEventListener("abort", on, { once: true }) + p.then((v) => { signal.removeEventListener("abort", on); ok(v) }, (e: unknown) => { signal.removeEventListener("abort", on); no(e) }) +}) +/** A failure worth retrying: the network, a 5xx, a 408 or 429; never a refusal (4xx) of the request itself. */ +export const transient = (e: unknown): boolean => { + const status = (e as { status?: unknown })?.status + if (typeof status === "number") return status >= 500 || status === 408 || status === 429 + return !(e instanceof CompleteRefused) +} +/** Red team loss-and-wip r2-1 (KEEP-7): errnos of the host, not of the script or the puller's code. A run that dies of + * one is retried by the floor (`infra/puller-io` is an infra reason); any other executor exception stays final. */ +export const HOST_ERRNOS: ReadonlySet = new Set(["ENOSPC", "EDQUOT", "EIO", "EMFILE", "ENFILE", "ENOMEM", "EAGAIN", "EBUSY", "EROFS", "ETXTBSY"]) +export const hostErrno = (e: unknown): string | undefined => { + const code = (e as { code?: unknown })?.code + if (typeof code === "string" && HOST_ERRNOS.has(code)) return code + const m = /^(E[A-Z]+)\b/.exec(String((e as Error)?.message ?? "")) + return m && HOST_ERRNOS.has(m[1]!) ? m[1] : undefined +} +const UNDELIVERABLE = "infra/verdict-undeliverable" +const isReplacement = (v: CompletePayload) => (v.output as { reason?: unknown; replaces?: unknown } | undefined)?.reason === UNDELIVERABLE + && (v.output as { replaces?: unknown }).replaces !== undefined +interface Active { readonly grant: Grant; readonly ac: AbortController; why?: Why; readonly done: Promise; readonly startedAt: number } + +/** The puller's own view for a health endpoint (G-BK7): what it holds, runs, owes and when it last renewed. */ +export interface PullerStatus { + readonly holder: string; readonly state: "running" | "draining" | "stopping"; readonly held: number + readonly active: ReadonlyArray<{ leaseId: string; attempt: number; runningMs: number; fenceInMs: number }> + readonly outbox: number; readonly lastHeartbeatOkMs: number | null; readonly heartbeatFailures: number; readonly pollMs: number + readonly counters: Readonly> +} + +const sha256 = async (s: string) => [...new Uint8Array(await crypto.subtle.digest("SHA-256", new TextEncoder().encode(s)))].map((b) => b.toString(16).padStart(2, "0")).join("") +const sleep = (ms: number, signal?: AbortSignal) => new Promise((ok) => { + const t = setTimeout(ok, ms) + signal?.addEventListener("abort", () => { clearTimeout(t); ok() }, { once: true }) +}) + +export class Puller { + readonly state: PullerState + readonly active = new Map() + #held: Record + #stop = new AbortController() + #hbMs: number + #pollMs: number + #draining = false + #wake: (() => void) | undefined + /** G-BK2: per lease, the send time of the last call the floor answered with it renewed (or its grant arrived). */ + readonly #renewedAt = new Map() + #lastHbOk: number | null = null + #hbFailures = 0 + readonly counters: Record = {} + #fatal: unknown + readonly #log: NonNullable + readonly #floor: FloorPort + /** KEEP-3: floor replies that were not errors; an HTTP 5xx after one of them is about that call's bytes. */ + #answered = 0 + readonly #verdictFailures = new Map() + readonly #failMark = new Map() + + constructor(readonly cfg: PullerConfig, readonly deps: PullerDeps) { + this.state = new PullerState(cfg.stateDir) + this.#held = this.state.held() + this.#hbMs = cfg.heartbeatMs ?? 10_000 + this.#pollMs = cfg.pollMs ?? 5_000 + this.#log = deps.log ?? (() => undefined) + const seen = (p: Promise) => p.then((v) => { this.#answered++; return v }) + this.#floor = { lease: (p) => seen(deps.floor.lease(p)), heartbeat: (p) => seen(deps.floor.heartbeat(p)), complete: (p) => seen(deps.floor.complete(p)) } + } + + get held(): Readonly> { return this.#held } + + /** Runs until stop() or a fatal floor answer (bad token, second session, redirect, misroute), which it rethrows. */ + async run(): Promise { + const stop = this.#stop.signal + await this.flushOutbox() + await this.readopt() + const hb = this.heartbeatLoop() + const fence = this.fenceLoop() + while (!stop.aborted && this.#fatal === undefined) { + await this.flushOutbox() + // G-BK4: a draining puller leases nothing and stops once its runs have ended and their verdicts are sent + if (this.#draining && this.active.size === 0 && this.state.outbox().length === 0) break + // Red team loss-and-wip r2-2 (KEEP-3): no new run while a verdict is undelivered. Red team r3-1 (KEEP-12): a + // held grant with no verdict waiting is a run of this puller whether or not it is running here yet. + const waiting = new Set(this.state.outbox().map((v) => v.leaseId)) + const mine = new Set([...this.active.keys(), ...Object.keys(this.#held).filter((id) => !waiting.has(id))]) + const free = this.cfg.maxRuns - mine.size + if (waiting.size === 0 && free > 0 && !stop.aborted && !this.#draining) await this.lease(free) + await new Promise((ok) => { this.#wake = ok; void sleep(this.pollDelay(), stop).then(ok) }) + this.#wake = undefined + } + this.#stop.abort() + await hb + await fence + for (const a of this.active.values()) { a.why ??= "stop"; a.ac.abort(a.why) } + await Promise.all([...this.active.values()].map((a) => a.done)) + if (this.#fatal !== undefined) throw this.#fatal + } + + /** Stop leasing and abort every run; nothing is completed, so each lease is re-adopted by the next start. */ + stop() { this.#stop.abort() } + + /** + * G-BK4, Buildkite's graceful stop: lease nothing more, keep heartbeating, let every run finish and its verdict go + * out, and stop then. Past `timeoutMs` the remaining runs are aborted as by stop() (each harness gets its cancel + * grace, G-BK1). Resolves when the stop has been requested (run() resolves when it is done). + */ + drain(timeoutMs: number): void { + if (this.#draining) return + this.#draining = true + this.#log("drain", { active: this.active.size, timeoutMs }) + this.#wake?.() + if (timeoutMs <= 0) { this.stop(); return } + const t = setTimeout(() => { this.#log("drain-timeout", { active: this.active.size }); this.stop() }, timeoutMs) + t.unref?.() + this.#stop.signal.addEventListener("abort", () => clearTimeout(t), { once: true }) + } + get draining() { return this.#draining } + + status(now = Date.now()): PullerStatus { + return { + holder: this.cfg.holder, state: this.#stop.signal.aborted ? "stopping" : this.#draining ? "draining" : "running", + held: Object.keys(this.#held).length, + active: [...this.active.values()].map((a) => ({ leaseId: a.grant.leaseId, attempt: a.grant.attempt, runningMs: now - a.startedAt, fenceInMs: this.fenceAt(a.grant) - now })), + outbox: this.state.outbox().length, lastHeartbeatOkMs: this.#lastHbOk, heartbeatFailures: this.#hbFailures, pollMs: this.#pollMs, + counters: { ...this.counters } + } + } + private count(k: string) { this.counters[k] = (this.counters[k] ?? 0) + 1 } + + private pollDelay(): number { + if (!this.cfg.serverPoll) return this.#pollMs + return Math.round(this.#pollMs * (0.9 + Math.random() * 0.2)) + } + + /** + * G-BK2 (red team double-run-r1-1, ported from link.ts selfFenceAt): past reassignSeconds the floor may grant + * attempt n+1 to another holder, so a run this puller has not seen renewed for reassignSeconds less half a lease + * (one lease when the grant carries no bound) is aborted as lost: no verdict, its journal kept for the next attempt. + */ + fenceAt(g: Grant): number { + const lease = g.lease.leaseDurationSeconds * 1000 + const bound = g.reassignSeconds !== undefined && Number.isFinite(g.reassignSeconds) ? g.reassignSeconds * 1000 : lease + const base = this.#renewedAt.get(g.leaseId) ?? Date.now() + return base + Math.max(lease / 2, bound - lease / 2) + } + selfFence(now = Date.now()) { + for (const [id, a] of this.active) { + if (a.why !== undefined) continue + if (now >= this.fenceAt(a.grant)) { + this.#log("lease-expired", { leaseId: id, lastRenewal: this.#renewedAt.get(id) ?? null }) + this.count("self_fences") + this.abort(id, "lost") + } + } + } + private async fenceLoop() { + const stop = this.#stop.signal + while (!stop.aborted) { + await sleep(Math.min(1_000, this.#hbMs), stop) + if (!stop.aborted) this.selfFence() + } + } + + private fatal(e: unknown) { + if (e instanceof FloorError && e.fatal) { + // Red team double-run-r1-2, as in the link: a 409 means another session holds this identity and may run + // attempt n+1 of every lease held here; each run is fenced (lost, no verdict) before the exit. + if (e.kind === "session-conflict") for (const id of [...this.active.keys()]) this.abort(id, "lost") + this.#fatal = e; this.#stop.abort(); this.#wake?.() + } + } + private persistHeld() { this.state.putHeld(this.#held) } + + /** + * May this held grant start (or restart) here? Not while its own verdict waits in the outbox (codex review 3, C3-2: + * a restart would run it again and overwrite that verdict), and not while a verdict of an attempt it supersedes + * waits there (codex review 2, C2-2: the floor may still accept that verdict and withdraw this attempt, rule 4b). + */ + private launchable(g: Grant): boolean { + if (this.active.has(g.leaseId)) return false + const pending = new Set(this.state.outbox().map((v) => v.leaseId)) + return !pending.has(g.leaseId) && !(g.supersedes ?? []).some((old) => pending.has(old)) + } + + /** After a restart: vouch for every held lease; relaunch the renewed ones (they resume from their journal). */ + private async readopt() { + const ids = Object.keys(this.#held) + const pendingRequestKey = this.state.pendingKey() + if (ids.length === 0 && pendingRequestKey === undefined) return + try { + const sentAt = Date.now() + const r = await this.#floor.heartbeat({ holderIdentity: this.cfg.holder, leaseIds: ids, ...(pendingRequestKey ? { pendingRequestKey } : {}) }) + for (const id of r.renewed) this.#renewedAt.set(id, sentAt) + for (const id of r.lost) delete this.#held[id] + this.persistHeld() + for (const id of r.renewed) { const g = this.#held[id]; if (g && this.launchable(g)) { this.#log("readopt", { leaseId: id }); this.launch(g) } } + for (const id of r.cancelRequested) this.abort(id, "cancel") + } catch (e) { this.#log("readopt-error", { message: String((e as Error)?.message ?? e) }); this.fatal(e) } + } + + private async lease(capacity: number) { + let key = this.state.pendingKey() + if (key === undefined) { key = randomUUID(); this.state.setPendingKey(key) } + let r: LeaseReply + const sentAt = Date.now() + try { r = await this.#floor.lease({ holderIdentity: this.cfg.holder, capacity, requestKey: key }) } catch (e) { + this.#log("lease-error", { kind: (e as FloorError)?.kind, message: String((e as Error)?.message ?? e) }); this.fatal(e); return + } + if (typeof r.heartbeatSeconds === "number" && r.heartbeatSeconds > 0) this.#hbMs = Math.min(this.cfg.heartbeatMs ?? Infinity, r.heartbeatSeconds * 1000) + if (typeof r.nextPollSeconds === "number" && r.nextPollSeconds > 0 && this.cfg.serverPoll) + this.#pollMs = Math.min(this.cfg.maxPollMs ?? 300_000, Math.max(this.cfg.pollMs ?? 5_000, r.nextPollSeconds * 1000)) + for (const g of r.grants) { this.#held[g.leaseId] = g; this.#renewedAt.set(g.leaseId, sentAt); this.count("grants") } + this.persistHeld() + this.state.setPendingKey(undefined) + for (const bad of r.invalid ?? []) { + if (bad.leaseId !== undefined && bad.attempt !== undefined) this.queueVerdict({ leaseId: bad.leaseId, attempt: bad.attempt, result: "failure", output: { reason: "pre-start/invalid-spec", message: bad.message } }) + } + for (const g of r.grants) { + for (const old of g.supersedes ?? []) { // L3: the earlier attempt stops before this one starts + const a = this.active.get(old) + if (a) { this.abort(old, "superseded"); await a.done } + delete this.#held[old] + } + this.persistHeld() + if (this.launchable(g)) this.launch(g) + else this.#log("launch-deferred", { leaseId: g.leaseId }) + } + } + + private abort(leaseId: string, why: Why) { + const a = this.active.get(leaseId) + if (a) { a.why ??= why; a.ac.abort(why) } + else if (why === "cancel" && this.#held[leaseId]) { + const g = this.#held[leaseId]! + this.queueVerdict({ leaseId, attempt: g.attempt, result: "cancelled", output: { reason: "cancelled" } }) + } + } + + private launch(g: Grant) { + const ac = new AbortController() + const entry: Active = { grant: g, ac, done: undefined as never, startedAt: Date.now() } + const work = async (): Promise => { + const ann = g.job.metadata.annotations ?? {} + const runId = ann[A + "run-id-raw"] + if (!g.job.spec["runs-on"].includes(INTERPRETER_LABEL) || runId === undefined) + return { result: "failure", output: { reason: "pre-start/invalid-spec", message: "not a run: the puller takes runtime:interpreter jobs with a run-id-raw annotation" } } + const script = await this.fetchScript(runId, ac.signal) + const want = g.job.spec.with.prompt_ref.sha256 + if (await sha256(script) !== want) return { result: "failure", output: { reason: "pre-start/script-mismatch", message: `the floor's script does not hash to ${want}` } } + const dir = this.state.runDir(runId) + const scriptPath = join(dir, "script.js") + writeFileSync(scriptPath, script) + let args: unknown + try { args = ann[A + "args"] === undefined ? undefined : JSON.parse(ann[A + "args"]!) } catch { return { result: "failure", output: { reason: "pre-start/invalid-spec", message: "the args annotation is not JSON" } } } + this.#log("run-start", { runId, leaseId: g.leaseId, attempt: g.attempt }) + const ship = this.deps.appendLog && this.cfg.logFile !== false + ? new LogShipper({ append: this.deps.appendLog, name: g.job.metadata.name, leaseId: g.leaseId, attempt: g.attempt, dir, log: this.#log, + ...(typeof this.cfg.logFile === "string" ? { file: this.cfg.logFile } : {}), ...(this.cfg.logIntervalMs !== undefined ? { intervalMs: this.cfg.logIntervalMs } : {}) }) + : undefined + ship?.start() + let v: RunVerdict + try { + v = await this.deps.execute({ runId, leaseId: g.leaseId, attempt: g.attempt, job: g.job, workflowName: ann[A + "workflow-name"] ?? runId, args, dir, scriptPath, script, signal: ac.signal }) + } finally { if (ship) { const failed = await ship.stop(); if (failed > 0) this.count("log_chunks_failed") } } + return v + } + const done = work().then((v) => v, (e: unknown): RunVerdict => { + if (e instanceof Aborted) return { result: "cancelled", output: { reason: "aborted" } } + if (e instanceof FloorError) return { result: "failure", output: { reason: "infra/script-unreadable", message: e.message } } + const errno = hostErrno(e) + if (errno !== undefined) return { result: "failure", output: { reason: "infra/puller-io", errno, message: String((e as Error)?.message ?? e).slice(0, 2000) } } + return { result: "failure", output: { reason: "infra/puller-defect", message: String((e as Error)?.message ?? e).slice(0, 2000) } } + }).then((v) => { + const why = entry.why + this.#log("run-end", { leaseId: g.leaseId, result: v.result, ...(why ? { aborted: why } : {}) }) + this.count(`runs_ended_${why ?? v.result}`) + if (why === "lost" || why === "stop" || why === "superseded") return // no verdict: the lease is the floor's again, or re-adopted on restart + // G-BK1: a cancelled verdict names why the run was signalled (Buildkite's signal_reason) + const output = why === "cancel" && typeof v.output === "object" && v.output !== null && !Array.isArray(v.output) ? { ...v.output, signal_reason: "cancel" } : v.output + this.queueVerdict({ leaseId: g.leaseId, attempt: g.attempt, result: why === "cancel" ? "cancelled" : v.result, output: this.cap(output) }) + }).finally(() => { this.active.delete(g.leaseId); this.#wake?.() }) + Object.assign(entry, { done }) + this.active.set(g.leaseId, entry) + } + + /** + * The run's script, retried while the run is live (codex review 3, C3-10 and C3-13): a floor restart or a partition + * is not a verdict, and an abort (lost, cancel, stop) ends the wait at once instead of pinning the active slot. + */ + private async fetchScript(runId: string, signal: AbortSignal): Promise { + for (let wait = 250; ; wait = Math.min(wait * 2, 15_000)) { + if (signal.aborted) throw new Aborted() + try { + return await raceAbort(this.deps.client.runScript(runId), signal) + } catch (e) { + if (e instanceof Aborted || signal.aborted) throw new Aborted() + if (!transient(e)) throw e + this.#log("script-retry", { runId, message: String((e as Error)?.message ?? e) }) + await sleep(wait, signal) + } + } + } + + private cap(output: unknown): unknown { + const text = JSON.stringify(output ?? null) + const max = this.cfg.maxOutputBytes ?? 1_500_000 + return Buffer.byteLength(text) <= max ? output : { truncated: true, bytes: Buffer.byteLength(text), preview: text.slice(0, 4000) } + } + + /** Red team loss-and-wip r2-2 (KEEP-3): the first verdict for a lease is the one delivered; a later one (a cancel + * that arrives while a success waits) never overwrites it. */ + private queueVerdict(v: CompletePayload) { + if (this.state.hasVerdict(v.leaseId)) this.#log("verdict-kept", { leaseId: v.leaseId, ignored: v.result }) + else this.state.putVerdict(v) + void this.flushOutbox() + } + + /** KEEP-3: a server error is charged to the verdict when it is an RPC Defect reply, or an HTTP 5xx after the floor + * answered some other call since this verdict last failed; at the ceiling the verdict is replaced, then dropped. */ + private serverError(v: CompletePayload, e: FloorError) { + const charged = e.status === undefined || (this.#failMark.get(v.leaseId) ?? -1) < this.#answered + this.#failMark.set(v.leaseId, this.#answered) + const n = (this.#verdictFailures.get(v.leaseId) ?? 0) + (charged ? 1 : 0) + this.#verdictFailures.set(v.leaseId, n) + if (n < (this.cfg.verdictAttempts ?? 8)) { this.#log("verdict-retry", { leaseId: v.leaseId, charged, try: n, message: e.message.slice(0, 200) }); return } + this.#verdictFailures.delete(v.leaseId); this.#failMark.delete(v.leaseId) + if (!isReplacement(v)) { + const outputSha256 = createHash("sha256").update(JSON.stringify(v.output ?? null)).digest("hex") + this.state.putVerdict({ leaseId: v.leaseId, attempt: v.attempt, result: "failure", + output: { reason: UNDELIVERABLE, replaces: v.result, outputSha256, message: `${v.result} verdict refused ${n} times: ${e.message.slice(0, 200)}` } }) + this.#log("verdict-replaced", { leaseId: v.leaseId, result: v.result, tries: n, outputSha256 }) + return + } + this.state.dropVerdict(v.leaseId) + delete this.#held[v.leaseId] + this.persistHeld() + this.#log("verdict-dead-letter", { leaseId: v.leaseId, tries: n }) + } + + #flushing: Promise | undefined + #flushGen = 0 + /** Sends every verdict in the outbox; a transient failure leaves it for the next pass. */ + flushOutbox(): Promise { + if (this.#flushing) return this.#flushing + const gen = ++this.#flushGen + const p = (async () => { + await Promise.resolve() // the pass starts after #flushing is set, so its finally clears this pass and no other + try { + for (const v of this.state.outbox()) { + try { await this.#floor.complete(v) } catch (e) { + if (e instanceof CompleteRefused) this.#log("verdict-refused", { leaseId: v.leaseId, code: e.code }) + else if (e instanceof FloorError && e.kind === "rejected") this.#log("verdict-rejected", { leaseId: v.leaseId, message: e.message }) + else if (e instanceof FloorError && e.kind === "server-error") { this.serverError(v, e); continue } + else { this.#log("verdict-retry", { leaseId: v.leaseId, message: String((e as Error)?.message ?? e) }); this.fatal(e); continue } + } + this.state.dropVerdict(v.leaseId) + this.#verdictFailures.delete(v.leaseId); this.#failMark.delete(v.leaseId) + delete this.#held[v.leaseId] + this.persistHeld() + } + } finally { if (this.#flushGen === gen) this.#flushing = undefined } + })() + this.#flushing = p + return p + } + + private async heartbeatLoop() { + const stop = this.#stop.signal + while (!stop.aborted) { + await sleep(this.#hbMs, stop) + if (stop.aborted) break + const ids = Object.keys(this.#held) + const pendingRequestKey = this.state.pendingKey() + if (ids.length === 0 && pendingRequestKey === undefined) continue + try { + const sentAt = Date.now() + const r = await this.#floor.heartbeat({ holderIdentity: this.cfg.holder, leaseIds: ids, ...(pendingRequestKey ? { pendingRequestKey } : {}) }) + this.#lastHbOk = Date.now(); this.#hbFailures = 0 + for (const id of r.renewed) this.#renewedAt.set(id, sentAt) + for (const id of r.lost) { + this.#log("lease-lost", { leaseId: id }) + this.abort(id, "lost") + if (!this.state.outbox().some((v) => v.leaseId === id)) delete this.#held[id] + } + if (r.lost.length > 0) this.persistHeld() + for (const id of r.cancelRequested) { this.#log("cancel-requested", { leaseId: id }); this.abort(id, "cancel") } + // codex review 3, C3-1: a renewed grant that runs nowhere here (the start-up re-adoption failed, or its launch + // waited on an earlier attempt's verdict) starts now; level-triggered, so one failed call never strands it + const cancelled = new Set(r.cancelRequested) + for (const id of r.renewed) { + const g = this.#held[id] + if (g && !cancelled.has(id) && !stop.aborted && this.launchable(g) && this.active.size < this.cfg.maxRuns) { this.#log("readopt", { leaseId: id }); this.launch(g) } + } + } catch (e) { this.#hbFailures++; this.count("heartbeat_failures"); this.#log("heartbeat-error", { message: String((e as Error)?.message ?? e) }); this.fatal(e) } + this.selfFence() + } + } +} diff --git a/pkgs/substrate-apps/src/apps/puller/src/state.ts b/pkgs/substrate-apps/src/apps/puller/src/state.ts new file mode 100644 index 00000000..693215a1 --- /dev/null +++ b/pkgs/substrate-apps/src/apps/puller/src/state.ts @@ -0,0 +1,36 @@ +// The puller's durable state, all under one directory, written before the act it records: +// session-id stable per directory: a restart is the same session (L5), a second copy is not +// held.json leaseId -> the grant, written when a Lease answer arrives, removed when its verdict lands +// pending-request-key the Lease requestKey in flight; a restart re-sends it and gets the same grants (rule 2b) +// outbox/.json a verdict not yet accepted by the floor; resent until it is +// runs// the run's own directory: script.js, journal.jsonl, events, jobs, result.json +import { randomUUID } from "node:crypto" +import { existsSync, mkdirSync, readdirSync, readFileSync, renameSync, rmSync, writeFileSync } from "node:fs" +import { join } from "node:path" +import type { Grant } from "@substrate/link/contract.ts" +import type { CompletePayload } from "@substrate/link/floor.ts" + +const atomic = (path: string, text: string) => { const tmp = `${path}.tmp-${process.pid}`; writeFileSync(tmp, text, { mode: 0o600 }); renameSync(tmp, path) } + +export class PullerState { + constructor(readonly dir: string) { mkdirSync(join(dir, "outbox"), { recursive: true }); mkdirSync(join(dir, "runs"), { recursive: true }) } + sessionId(): string { + const p = join(this.dir, "session-id") + if (!existsSync(p)) writeFileSync(p, randomUUID() + "\n", { mode: 0o600 }) + return readFileSync(p, "utf8").trim() + } + held(): Record { + const p = join(this.dir, "held.json") + return existsSync(p) ? JSON.parse(readFileSync(p, "utf8")) as Record : {} + } + putHeld(h: Record) { atomic(join(this.dir, "held.json"), JSON.stringify(h)) } + pendingKey(): string | undefined { const p = join(this.dir, "pending-request-key"); return existsSync(p) ? readFileSync(p, "utf8").trim() || undefined : undefined } + setPendingKey(k: string | undefined) { const p = join(this.dir, "pending-request-key"); if (k === undefined) rmSync(p, { force: true }); else atomic(p, k) } + outbox(): Array { + return readdirSync(join(this.dir, "outbox")).filter((f) => f.endsWith(".json")).sort().map((f) => JSON.parse(readFileSync(join(this.dir, "outbox", f), "utf8")) as CompletePayload) + } + putVerdict(v: CompletePayload) { atomic(join(this.dir, "outbox", `${v.leaseId}.json`), JSON.stringify(v)) } + hasVerdict(leaseId: string) { return existsSync(join(this.dir, "outbox", `${leaseId}.json`)) } + dropVerdict(leaseId: string) { rmSync(join(this.dir, "outbox", `${leaseId}.json`), { force: true }) } + runDir(runId: string) { const d = join(this.dir, "runs", runId); mkdirSync(d, { recursive: true }); return d } +} diff --git a/pkgs/substrate-apps/src/apps/puller/test/buildkite-family.test.ts b/pkgs/substrate-apps/src/apps/puller/test/buildkite-family.test.ts new file mode 100644 index 00000000..12bbbe7b --- /dev/null +++ b/pkgs/substrate-apps/src/apps/puller/test/buildkite-family.test.ts @@ -0,0 +1,214 @@ +// Buildkite-family critique pass (2026-09-24), puller side: self-fence (G-BK2, audit probe P2 inverted), graceful +// drain (G-BK4), nextPollSeconds with jitter (G-BK4), the loopback health endpoint (G-BK7) and signal_reason (G-BK1). +import { createHash } from "node:crypto" +import { mkdtempSync } from "node:fs" +import { tmpdir } from "node:os" +import { join } from "node:path" +import { describe, expect, it } from "vitest" +import { FloorError } from "@substrate/link/floor.ts" +import { Puller } from "../src/puller.ts" +import type { FloorPort, RunTask } from "../src/puller.ts" +import { prometheusOf, serveHealth } from "../src/health.ts" + +const script = "export const meta = {}" +const sha = createHash("sha256").update(script).digest("hex") +const grantOf = (id: string, o: { lease?: number; reassign?: number } = {}) => ({ + leaseId: `run-${id}-a1`, attempt: 1, + lease: { holderIdentity: "coord", leaseDurationSeconds: o.lease ?? 1, acquireTime: Date.now(), renewTime: Date.now(), leaseTransitions: 0 }, + ...(o.reassign !== undefined ? { reassignSeconds: o.reassign } : {}), + job: { apiVersion: "ultracode.mecattaf.dev/v1alpha1", kind: "AgentJob", + metadata: { name: `run-${id}`, labels: {}, annotations: { "ultracode.mecattaf.dev/run-id-raw": id, "ultracode.mecattaf.dev/journal-key": "k" } }, + spec: { "runs-on": ["seat:interp", "runtime:interpreter"], with: { prompt_ref: { sha256: sha }, model: "m" } } } +}) +const stats = { seq: 1, cap: 1, wip: 1, queued: 0, done: 0 } +const wait = (ms: number) => new Promise((r) => setTimeout(r, ms)) +const stateDir = () => mkdtempSync(join(tmpdir(), "bk-pull-")) + +type Fake = { floor: FloorPort; beats: number; leases: number; completes: Array<{ leaseId: string; result: string; output: unknown }> } +const fakeFloor = (grants: Array>, o: { beat?: (ids: ReadonlyArray) => Promise<{ renewed: string[]; lost: string[]; cancelRequested: string[] }>; nextPollSeconds?: number } = {}): Fake => { + const f: Fake = { beats: 0, leases: 0, completes: [], floor: undefined as never } + let given = false + f.floor = { + lease: async () => { f.leases++; const g = given ? [] : grants; given = true; return { grants: g, stats, ...(o.nextPollSeconds !== undefined ? { nextPollSeconds: o.nextPollSeconds } : {}), heartbeatSeconds: 1 } as never }, + heartbeat: async (p) => { f.beats++; return o.beat ? o.beat(p.leaseIds) : { renewed: [...p.leaseIds], lost: [], cancelRequested: [] } }, + complete: async (p) => { f.completes.push({ leaseId: p.leaseId, result: p.result, output: p.output }); return { duplicate: false } } + } + return f +} +const hanging = (seen: { aborted?: unknown }) => (t: RunTask) => new Promise((_, no) => t.signal.addEventListener("abort", () => { seen.aborted = t.signal.reason; no(new Error("aborted")) })) + +describe("G-BK2 puller self-fence (audit probe P2 inverted)", () => { + it("a run whose heartbeats all fail is aborted as lost before reassignSeconds, with no verdict", async () => { + const f = fakeFloor([grantOf("fence1", { lease: 1, reassign: 2 })], { beat: async () => { throw new Error("network down (partition)") } }) + const seen: { aborted?: unknown } = {} + const events: string[] = [] + const p = new Puller({ holder: "coord", maxRuns: 1, stateDir: stateDir(), pollMs: 50, heartbeatMs: 200 }, + { floor: f.floor, client: { runScript: async () => script } as never, execute: hanging(seen) as never, log: (ev) => events.push(ev) }) + const done = p.run().catch(() => undefined) + const t0 = Date.now() + while (!events.includes("lease-expired") && Date.now() - t0 < 5_000) await wait(50) + const fencedAfter = Date.now() - t0 + await wait(100) + expect(events).toContain("lease-expired") + expect(fencedAfter).toBeLessThan(2_000) // reassignSeconds 2, lease 1: fence at 1.5 s after the grant + expect(seen.aborted).toBe("lost") + expect(p.active.size).toBe(0) + expect(f.completes).toEqual([]) // lost: no verdict, the journal is kept for attempt n+1 + expect(p.status().counters["self_fences"]).toBe(1) + p.stop(); await done + }, 15_000) + + it("renewing heartbeats keep the run alive past the bound", async () => { + const f = fakeFloor([grantOf("fence2", { lease: 1, reassign: 1 })]) + const seen: { aborted?: unknown } = {} + const p = new Puller({ holder: "coord", maxRuns: 1, stateDir: stateDir(), pollMs: 50, heartbeatMs: 100 }, + { floor: f.floor, client: { runScript: async () => script } as never, execute: hanging(seen) as never }) + const done = p.run().catch(() => undefined) + await wait(1_500) + expect(p.active.size).toBe(1) + expect(seen.aborted).toBeUndefined() + p.stop(); await done + expect(seen.aborted).toBe("stop") + }, 10_000) + + it("a 409 session conflict fences every run as lost before the puller exits", async () => { + const f = fakeFloor([grantOf("fence3", { lease: 30, reassign: 300 })], { beat: async () => { throw new FloorError("session-conflict", "another session") } }) + const seen: { aborted?: unknown } = {} + const p = new Puller({ holder: "coord", maxRuns: 1, stateDir: stateDir(), pollMs: 50, heartbeatMs: 150 }, + { floor: f.floor, client: { runScript: async () => script } as never, execute: hanging(seen) as never }) + await expect(p.run()).rejects.toMatchObject({ kind: "session-conflict" }) + expect(seen.aborted).toBe("lost") + expect(f.completes).toEqual([]) + }, 10_000) +}) + +describe("G-BK4 graceful drain and server poll", () => { + it("drain: no new leases, the in-flight run finishes and its verdict is sent, then run() resolves", async () => { + const f = fakeFloor([grantOf("drain1", { lease: 30, reassign: 300 })]) + let finish!: () => void + const exec = (t: RunTask) => new Promise<{ result: "success"; output: unknown }>((ok, no) => { finish = () => ok({ result: "success", output: { v: 1 } }); t.signal.addEventListener("abort", () => no(new Error("aborted"))) }) + const p = new Puller({ holder: "coord", maxRuns: 2, stateDir: stateDir(), pollMs: 30, heartbeatMs: 100 }, + { floor: f.floor, client: { runScript: async () => script } as never, execute: exec as never }) + const done = p.run() + while (p.active.size === 0) await wait(20) + p.drain(10_000) + const leasesAtDrain = f.leases + await wait(200) + expect(f.leases).toBe(leasesAtDrain) // a draining puller leases nothing + expect(p.status().state).toBe("draining") + finish() + await done + expect(f.completes).toEqual([{ leaseId: "run-drain1-a1", result: "success", output: { v: 1 } }]) + }, 10_000) + + it("drain past its timeout aborts the run as stop (no verdict), then run() resolves", async () => { + const f = fakeFloor([grantOf("drain2", { lease: 30, reassign: 300 })]) + const seen: { aborted?: unknown } = {} + const p = new Puller({ holder: "coord", maxRuns: 1, stateDir: stateDir(), pollMs: 30, heartbeatMs: 100 }, + { floor: f.floor, client: { runScript: async () => script } as never, execute: hanging(seen) as never }) + const done = p.run() + while (p.active.size === 0) await wait(20) + const t0 = Date.now() + p.drain(300) + await done + expect(Date.now() - t0).toBeGreaterThanOrEqual(250) + expect(seen.aborted).toBe("stop") + expect(f.completes).toEqual([]) + }, 10_000) + + it("serverPoll honours nextPollSeconds (a paused holder polls slowly) within the bounds", async () => { + const f = fakeFloor([], { nextPollSeconds: 2 }) + const p = new Puller({ holder: "coord", maxRuns: 1, stateDir: stateDir(), pollMs: 20, heartbeatMs: 100, serverPoll: true, maxPollMs: 400 }, + { floor: f.floor, client: { runScript: async () => script } as never, execute: (async () => ({ result: "success", output: null })) as never }) + const done = p.run() + await wait(1_000) + p.stop(); await done + expect(p.status().pollMs).toBe(400) // 2 s asked, capped at maxPollMs + expect(f.leases).toBeLessThanOrEqual(4) // not the 50 a 20 ms poll would make + const g = new Puller({ holder: "coord", maxRuns: 1, stateDir: stateDir(), pollMs: 20, heartbeatMs: 100 }, + { floor: fakeFloor([], { nextPollSeconds: 2 }).floor, client: {} as never, execute: (async () => ({ result: "success", output: null })) as never }) + const gd = g.run(); await wait(200); g.stop(); await gd + expect(g.status().pollMs).toBe(20) // off by default: the old fixed poll + }, 10_000) +}) + +describe("G-BK1 signal_reason and G-BK7 health", () => { + it("a cancelled run's verdict names signal_reason cancel", async () => { + let n = 0 + const f = fakeFloor([grantOf("cancel1", { lease: 30, reassign: 300 })], { beat: async (ids) => ({ renewed: [...ids], lost: [], cancelRequested: n++ > 0 ? [...ids] : [] }) }) + const exec = (t: RunTask) => new Promise((ok) => t.signal.addEventListener("abort", () => ok({ result: "cancelled", output: { reason: "aborted" } }))) + const p = new Puller({ holder: "coord", maxRuns: 1, stateDir: stateDir(), pollMs: 30, heartbeatMs: 80 }, + { floor: f.floor, client: { runScript: async () => script } as never, execute: exec as never }) + const done = p.run() + while (f.completes.length === 0) await wait(20) + p.stop(); await done + expect(f.completes[0]).toMatchObject({ result: "cancelled", output: { reason: "aborted", signal_reason: "cancel" } }) + }, 10_000) + + it("the loopback health endpoint serves status.json and Prometheus text; a public address is refused", async () => { + const status = { holder: "coord", state: "running" as const, held: 1, active: [{ leaseId: "run-x-a1", attempt: 1, runningMs: 5, fenceInMs: 100 }], outbox: 0, + lastHeartbeatOkMs: 1, heartbeatFailures: 0, pollMs: 5000, counters: { grants: 3, self_fences: 1 } } + const h = await serveHealth("127.0.0.1:0", { status: () => status }) + try { + const js = await (await fetch(`http://${h.addr}/status.json`)).json() + expect(js).toMatchObject({ holder: "coord", held: 1, counters: { grants: 3 } }) + const text = await (await fetch(`http://${h.addr}/metrics`)).text() + expect(text).toContain('substrate_puller_events_total{holder="coord",event="self_fences"} 1') + expect(text).toBe(prometheusOf(status)) + } finally { h.close() } + await expect(serveHealth("0.0.0.0:9", { status: () => status })).rejects.toThrow(/loopback/) + }) +}) + +describe("G-BK3 live log shipping", () => { + const { appendFileSync, writeFileSync: wf } = require("node:fs") as typeof import("node:fs") + it("ships whole lines in order, resumes after a transient failure without a gap or a copy, and stops on 409", async () => { + const { LogShipper } = await import("../src/logship.ts") + const dir = stateDir() + const got: Array<{ seq: number; data: string }> = [] + let fail = 0, gone = false + const append = async (_n: string, c: { seq: number; data: string }) => { + if (gone) throw Object.assign(new Error("lease-not-live"), { status: 409, code: "lease-not-live" }) + if (fail > 0) { fail--; throw Object.assign(new Error("502"), { status: 502 }) } + got.push({ seq: c.seq, data: c.data }) + } + const s = new LogShipper({ append, name: "run-x", leaseId: "run-x-a1", attempt: 1, dir, maxChunkBytes: 16 }) + wf(join(dir, "events.jsonl"), "one\ntwo\npart") + await s.tick() + expect(got.map((g) => g.data).join("")).toBe("one\ntwo\n") // the partial line waits + fail = 1 + appendFileSync(join(dir, "events.jsonl"), "ial\nthree\n") + await s.tick() // fails transiently + await s.tick() + expect(got.map((g) => g.data).join("")).toBe("one\ntwo\npartial\nthree\n") + expect(got.map((g) => g.seq)).toEqual(got.map((_, i) => i)) + // a restart of the same attempt continues from the persisted cursor + const again = new LogShipper({ append, name: "run-x", leaseId: "run-x-a1", attempt: 1, dir, maxChunkBytes: 16 }) + expect(again.shipped.offset).toBe(Buffer.byteLength("one\ntwo\npartial\nthree\n")) + gone = true + appendFileSync(join(dir, "events.jsonl"), "four\n") + await again.tick() + expect(again.stoppedWhy).toBe("lease-not-live") + }) + + it("the puller streams the run's events.jsonl while it runs and flushes the tail before the verdict", async () => { + const f = fakeFloor([grantOf("log1", { lease: 30, reassign: 300 })]) + const chunks: string[] = [] + let verdictAt = -1 + const floor = { ...f.floor, complete: async (p: any) => { verdictAt = chunks.length; return f.floor.complete(p) } } + const exec = async (t: RunTask) => { + wf(join(t.dir, "events.jsonl"), `{"ev":"call-start","i":1}\n`) + await wait(250) + appendFileSync(join(t.dir, "events.jsonl"), `{"ev":"call-end","i":1}\n`) + return { result: "success" as const, output: null } + } + const p = new Puller({ holder: "coord", maxRuns: 1, stateDir: stateDir(), pollMs: 30, heartbeatMs: 100, logIntervalMs: 50 }, + { floor, client: { runScript: async () => script } as never, execute: exec as never, appendLog: async (name, c) => { expect([name, c.leaseId, c.attempt]).toEqual(["run-log1", "run-log1-a1", 1]); chunks.push(c.data) } }) + const done = p.run() + while (f.completes.length === 0) await wait(20) + p.stop(); await done + expect(chunks.length).toBeGreaterThanOrEqual(2) // the first line went out while the run was still running + expect(chunks.join("")).toBe(`{"ev":"call-start","i":1}\n{"ev":"call-end","i":1}\n`) + expect(verdictAt).toBe(chunks.length) // every chunk before the verdict + }, 10_000) +}) diff --git a/pkgs/substrate-apps/src/apps/puller/test/critique-durability-pipeline.test.ts b/pkgs/substrate-apps/src/apps/puller/test/critique-durability-pipeline.test.ts new file mode 100644 index 00000000..be6291f5 --- /dev/null +++ b/pkgs/substrate-apps/src/apps/puller/test/critique-durability-pipeline.test.ts @@ -0,0 +1,103 @@ +// Critique pass 2026-09-24, red team durability-r1-1 (and double-run-r2-4), promoted from the audit probe and inverted. A puller killed while a no-barrier pipeline's second stage is +// in flight: does attempt 2 of the run resume, as puller.ts promises ("finished agent() calls are cache hits and are +// never dispatched again")? The real floor (Worker entry and Floor object over node:sqlite), the real FloorBackend. +import { mkdtempSync, readFileSync, writeFileSync } from "node:fs" +import { tmpdir } from "node:os" +import { join } from "node:path" +import { afterEach, describe, expect, it } from "vitest" +import { SubstrateClient } from "@substrate/api" +import { localFloor } from "@substrate/floor/local.ts" +import { connectFloor } from "../src/connect.ts" +import { FloorBackend, floorExecutor } from "../src/execute.ts" + +const T = { floor: "rt-floor", nas: "rt-nas", interp: "rt-interp" } +const URL_ = "http://floor.test" +const cleanups: Array<() => unknown> = [] +afterEach(async () => { for (const c of cleanups.splice(0).reverse()) await c() }) +const until = async (what: string, f: () => Promise | boolean, ms = 15_000) => { + const end = Date.now() + ms + while (!(await f())) { if (Date.now() > end) throw new Error(`timed out waiting for ${what}`); await new Promise((r) => setTimeout(r, 15)) } +} + +const PIPELINE = `export const meta = { name: "pipe", description: "two independent two-stage chains, no barrier" } +const second = (tag) => (x) => agent("second " + tag + " " + x, { label: "s2-" + tag }) +const pa = agent("first A", { label: "a" }).then(second("A")) +const pb = agent("first B", { label: "b" }).then(second("B")) +return await Promise.all([pa, pb]) +` + +describe("durability-r1: kill during a pipeline, then resume (puller attempt 2)", () => { + it("durability-r1-1: stage-2 nodes resume under the same name and body, and the run ends success", async () => { + const f = localFloor({ floorToken: T.floor, links: [{ token: T.nas, holder: "nas", labels: ["seat:t", "runtime:*"] }, { token: T.interp, holder: "coordinator", labels: ["runtime:interpreter"] }], + vars: { FLOOR_UNGATED_SEATS: "*", FLOOR_INTERPRETER_SEAT: "interp", FLOOR_HEARTBEAT_SECONDS: "1", FLOOR_LEASE_SECONDS: "30", FLOOR_POLL_SECONDS: "1" } }) + cleanups.push(() => f.close()) + const client = new SubstrateClient({ url: URL_, token: T.floor, fetch: f.fetch }) + const nas = await connectFloor({ url: URL_, token: T.nas, sessionId: "nas-s", fetch: f.fetch }) + cleanups.push(() => nas.close()) + const id = "rtpipe0001" + await client.submit({ id, script: PIPELINE }) + // G-BK5 (integrate 2026-09-24): node enqueues name the run's live interpreter lease, so the run is really leased + const interp = await connectFloor({ url: URL_, token: T.interp, sessionId: "interp-s", fetch: f.fetch }) + cleanups.push(() => interp.close()) + const runGrant = (await interp.port.lease({ holderIdentity: "coordinator", capacity: 1, requestKey: crypto.randomUUID() })).grants.find((g) => g.leaseId === `run-${id}-a1`) + expect(runGrant).toBeDefined() + const dir = mkdtempSync(join(tmpdir(), "rt-pipe-")) + const scriptPath = join(dir, "script.js") + writeFileSync(scriptPath, PIPELINE) + const backends: Array = [] + const exec = floorExecutor({ client, defaultRunsOn: ["seat:t"], defaultModel: "test-model", pollMs: 10, onBackend: (b) => backends.push(b) }) + // Both attempts run under the one live interpreter lease: attempt 2 is the puller re-adopting it after the kill. + const task = (_attempt: number, signal: AbortSignal) => ({ runId: id, leaseId: runGrant!.leaseId, attempt: runGrant!.attempt, job: {} as never, workflowName: "pipe", args: undefined, dir, scriptPath, script: PIPELINE, signal }) + const promptOf = async (name: string) => ((await client.job(name)).job as { spec: { with: { prompt?: string } } }).spec.with.prompt ?? "" + const names = async () => (await client.runJobs(id)).map((j) => j.name).filter((n) => n.startsWith("n")) + /** The NAS link's part by hand: lease what is queued (the grants are kept), answer a prompt when told. */ + const granted = new Map() + const serve = async () => { + const r = await nas.port.lease({ holderIdentity: "nas", capacity: 10, requestKey: crypto.randomUUID() }) + for (const g of r.grants) granted.set(g.job.spec.with.prompt ?? "", { leaseId: g.leaseId, attempt: g.attempt }) + } + const answer = async (prompt: string, out: string) => { + const g = granted.get(prompt) + if (!g) throw new Error(`no grant for ${prompt}`) + await nas.port.complete({ leaseId: g.leaseId, attempt: g.attempt, result: "success", output: out, usage: { prompt_tokens: 1, completion_tokens: 1, tool_calls: 0 } }) + } + + // Attempt 1. Stage 1 is enqueued as n1 (first A) and n2 (first B); B finishes first, so "second B" is invoked + // first and becomes n3; A finishes, "second A" becomes n4. + const ac1 = new AbortController() + const run1 = exec(task(1, ac1.signal)) + await until("stage 1 enqueued", async () => (await names()).length === 2) + await serve() + await answer("first B", "B") + await until("second B enqueued", async () => (await names()).length === 3) + await answer("first A", "A") + await until("second A enqueued", async () => (await names()).length === 4) + await serve() // the NAS link takes both stage-2 nodes: they are running when the puller dies + // The kill, with both stage-2 nodes in flight (queued on the floor for the NAS link, which may already run them). + ac1.abort() + expect((await run1).result).toBe("cancelled") + const journal1 = readFileSync(join(dir, "journal.jsonl"), "utf8") + expect(journal1.split("\n").filter((l) => l.includes('"result"')).length).toBe(2) // stage 1 is finished work + + // Attempt 2 (the puller re-adopts the lease, or the floor requeues the run): the same directory, the journal + // found, cacheIdentity content. Stage 1 is a cache hit, resolved in invocation order, so "second A" is now + // invoked first and takes index 3: the name n3 carries a different prompt than the job the floor holds. + const ac2 = new AbortController() + const p2 = exec(task(2, ac2.signal)) + // audit: if the stage-2 names are stable, attempt 2 re-enqueues them as enqueued:false and waits on the NAS grants + await new Promise((r) => setTimeout(r, 300)) + await answer("second A A", "AA").catch(() => undefined) + await answer("second B B", "BB").catch(() => undefined) + const run2 = await Promise.race([p2, new Promise((_, no) => setTimeout(() => no(new Error("attempt 2 hung")), 8000))]) + // Fixed: the name and the body of every node are independent of invocation order, so attempt 2's re-enqueue is + // enqueued:false and it waits on the NAS link's answers. + expect((run2.output as { error: string | null }).error ?? null).toBeNull() + expect(run2.result).toBe("success") + expect(JSON.stringify(run2.output)).toContain("AA") + expect(JSON.stringify(run2.output)).toContain("BB") + for (const n of await names()) { + const j = (await client.job(n)).job as { metadata: { annotations: Record } } + expect(j.metadata.annotations["ultracode.mecattaf.dev/call-index"]).toBeUndefined() + } + }) +}) diff --git a/pkgs/substrate-apps/src/apps/puller/test/demand.test.ts b/pkgs/substrate-apps/src/apps/puller/test/demand.test.ts new file mode 100644 index 00000000..d28d44a8 --- /dev/null +++ b/pkgs/substrate-apps/src/apps/puller/test/demand.test.ts @@ -0,0 +1,24 @@ +// Prove lane 2026-09-23: an idle pusher reads every 900 s, past the gate's 360 s dispatch bound, so the first node +// after a quiet spell was refused stale-for-dispatch. The puller now marks demand while it holds a run. +import { mkdtempSync, statSync, utimesSync, writeFileSync } from "node:fs" +import { tmpdir } from "node:os" +import { join } from "node:path" +import { describe, expect, it } from "vitest" +import { touchDemand } from "../src/execute.ts" + +describe("demand marker", () => { + it("creates the marker, then refreshes its mtime", () => { + const dir = join(mkdtempSync(join(tmpdir(), "demand-")), "nested") + touchDemand(dir, "substrate-puller") + const f = join(dir, "substrate-puller") + const old = new Date(Date.now() - 3_600_000) + utimesSync(f, old, old) + touchDemand(dir, "substrate-puller") + expect(Date.now() - statSync(f).mtimeMs).toBeLessThan(60_000) + }) + it("never throws when the dir cannot be made (its parent is a file)", () => { + const file = join(mkdtempSync(join(tmpdir(), "demand-")), "plain") + writeFileSync(file, "") + expect(() => touchDemand(join(file, "sub"), "x")).not.toThrow() + }) +}) diff --git a/pkgs/substrate-apps/src/apps/puller/test/puller.test.ts b/pkgs/substrate-apps/src/apps/puller/test/puller.test.ts new file mode 100644 index 00000000..2a0d4512 --- /dev/null +++ b/pkgs/substrate-apps/src/apps/puller/test/puller.test.ts @@ -0,0 +1,206 @@ +// The puller's lease loop against the real floor (the Worker entry and Floor object over node:sqlite, +// apps/floor/src/local.ts), through the link's own rpcFloor client: lease, run, heartbeat, complete; a kill and a +// restart that re-adopts the lease and resumes from the run's journal; a cancel; the local executor with the repo's +// runners (a fake claude binary) and the ax refusal; the pidfile. +import { chmodSync, existsSync, mkdtempSync, readFileSync, writeFileSync } from "node:fs" +import { tmpdir } from "node:os" +import { join } from "node:path" +import { afterAll, afterEach, beforeAll, describe, expect, it } from "vitest" +import { SubstrateClient } from "@substrate/api" +import { localFloor } from "@substrate/floor/local.ts" +import { parseRuntimesToml } from "@substrate/runners" +import { connectFloor } from "../src/connect.ts" +import { FloorBackend, floorExecutor, localExecutor, outcomeOf, transcriptPartName } from "../src/execute.ts" +import { acquirePidfile, PidfileHeld } from "../src/pidfile.ts" +import { Puller } from "../src/puller.ts" +import type { Executor } from "../src/puller.ts" +import { main } from "../src/main.ts" + +const T = { floor: "puller-test-floor", coord: "puller-test-coord", nas: "puller-test-nas" } +const URL_ = "http://floor.test" +const cleanups: Array<() => unknown> = [] +afterEach(async () => { for (const c of cleanups.splice(0).reverse()) await c() }) + +const until = async (what: string, f: () => Promise | boolean, ms = 15_000) => { + const end = Date.now() + ms + while (!(await f())) { if (Date.now() > end) throw new Error(`timed out waiting for ${what}`); await new Promise((r) => setTimeout(r, 20)) } +} + +const world = async () => { + const f = localFloor({ floorToken: T.floor, links: [ + { token: T.coord, holder: "coord", labels: ["runtime:interpreter", "seat:interp"] }, + { token: T.nas, holder: "nas", labels: ["seat:t", "runtime:*"] } + ], vars: { FLOOR_UNGATED_SEATS: "*", FLOOR_INTERPRETER_SEAT: "interp", FLOOR_HEARTBEAT_SECONDS: "1", FLOOR_LEASE_SECONDS: "30", FLOOR_POLL_SECONDS: "1" } }) + cleanups.push(() => f.close()) + const client = new SubstrateClient({ url: URL_, token: T.floor, fetch: f.fetch }) + const stateDir = mkdtempSync(join(tmpdir(), "substrate-puller-")) + const nas = await connectFloor({ url: URL_, token: T.nas, sessionId: "nas-session", fetch: f.fetch }) + cleanups.push(() => nas.close()) + /** The NAS link's part, by hand: lease what is queued for it and answer each node by its prompt. */ + const serveNodes = async (answer: (prompt: string) => unknown | undefined) => { + const r = await nas.port.lease({ holderIdentity: "nas", capacity: 10, requestKey: crypto.randomUUID() }) + for (const g of r.grants) { + const out = answer(g.job.spec.with.prompt ?? "") + if (out !== undefined) await nas.port.complete({ leaseId: g.leaseId, attempt: g.attempt, result: "success", output: out, usage: { prompt_tokens: 3, completion_tokens: 1, tool_calls: 0 } }) + } + return r.grants.length + } + const start = async (execute: Executor, o: { hbMs?: number } = {}) => { + const conn = await connectFloor({ url: URL_, token: T.coord, sessionId: readFileSync(join(stateDir, "session-id"), "utf8").trim(), fetch: f.fetch }) + const events: Array<{ ev: string; f?: Record }> = [] + const p = new Puller({ holder: "coord", maxRuns: 1, stateDir, pollMs: 25, heartbeatMs: o.hbMs ?? 150 }, { floor: conn.port, client, execute, log: (ev, fields) => { if (process.env.PULLER_DEBUG) console.error(ev, JSON.stringify(fields)); events.push({ ev, ...(fields ? { f: fields } : {}) }) } }) + const done = p.run().finally(() => conn.close()) + cleanups.push(async () => { p.stop(); await done.catch(() => undefined) }) + return { p, done, events } + } + // The session id file is created by the first PullerState; make it exist before start() reads it. + const { PullerState } = await import("../src/state.ts") + new PullerState(stateDir).sessionId() + return { f, client, stateDir, serveNodes, start } +} + +const TWO_STEP = `export const meta = { name: "two-step", description: "d" } +const a = await agent("first", { label: "a" }) +const b = await agent("second after " + a, { label: "b" }) +return { a, b } +` + +describe("lease loop, floor dispatch", () => { + it("leases the run, dispatches its nodes to the floor, completes; after a kill it re-adopts the lease and resumes from the journal", async () => { + const w = await world() + await w.client.submitScript(TWO_STEP, { id: "pullrun0001" }) + const backends: Array = [] + const exec = floorExecutor({ client: w.client, defaultRunsOn: ["seat:t", "runtime:gvisor"], defaultModel: "test-model", pollMs: 25, onBackend: (b) => backends.push(b) }) + + const first = await w.start(exec) + await until("node 1 enqueued", async () => (await w.client.runJobs("pullrun0001")).filter((j) => j.kind === "agent").length >= 1) + expect((await w.client.job("run-pullrun0001")).holder).toBe("coord") + expect(await w.serveNodes((p) => (p === "first" ? "A" : undefined))).toBe(1) + await until("node 2 enqueued", async () => (await w.client.runJobs("pullrun0001")).filter((j) => j.kind === "agent").length >= 2) + // The kill: the process stops with node 2 in flight. No verdict is sent; the lease is kept for re-adoption. + first.p.stop(); await first.done + expect(Object.keys(JSON.parse(readFileSync(join(w.stateDir, "held.json"), "utf8")))).toEqual(["run-pullrun0001-a1"]) + const journal = readFileSync(join(w.stateDir, "runs", "pullrun0001", "journal.jsonl"), "utf8") + expect(journal.split("\n").filter((l) => l.includes("\"result\"")).length).toBe(1) + expect((await w.client.run("pullrun0001")).state).toBe("running") + + const second = await w.start(exec) + await until("re-adopted", () => second.events.some((e) => e.ev === "readopt")) + await until("node 2 served", async () => (await w.serveNodes((p) => (p.startsWith("second") ? "B" : undefined))) > 0) + const run = await w.client.waitRun("pullrun0001", { timeoutMs: 15_000, pollMs: 25 }) + expect([run.state, run.result]).toEqual(["done", "success"]) + const out = await w.client.output("run-pullrun0001") + expect(out).toMatchObject({ result: "success", attempt: 1, output: { resumed: true, status: "completed", result: { a: "A", b: "B" } } }) + // Node 1 was a journal hit on the resume: never enqueued again. Node 2 was re-enqueued under the same name. + expect(backends[1]!.enqueued).toEqual([backends[0]!.enqueued[1]]) + expect((await w.client.runJobs("pullrun0001")).filter((j) => j.kind === "agent").map((j) => j.name)).toHaveLength(2) + await until("verdict flushed", () => !existsSync(join(w.stateDir, "outbox", "run-pullrun0001-a1.json")) && Object.keys(second.p.held).length === 0) + const pullers = await w.client.pullers() + expect(pullers.map((p) => p.holder).sort()).toEqual(["coord", "nas"]) + }) + + it("a cancel reaches the running run through the heartbeat and is completed as cancelled", async () => { + const w = await world() + await w.client.submitScript(TWO_STEP, { id: "pullrun0002" }) + const exec = floorExecutor({ client: w.client, defaultRunsOn: ["seat:t"], defaultModel: "test-model", pollMs: 25 }) + const s = await w.start(exec) + await until("node 1 enqueued", async () => (await w.client.runJobs("pullrun0002")).filter((j) => j.kind === "agent").length >= 1) + await w.client.cancelRun("pullrun0002") + await until("cancel requested", () => s.events.some((e) => e.ev === "cancel-requested")) + await until("interpreter job done", async () => (await w.client.job("run-pullrun0002")).state === "done") + expect((await w.client.output("run-pullrun0002")).result).toBe("cancelled") + expect(s.events.find((e) => e.ev === "run-end")?.f).toMatchObject({ aborted: "cancel" }) + }) + + it("refuses a script that does not hash to the grant's content address", async () => { + const w = await world() + await w.client.submitScript(TWO_STEP, { id: "pullrun0003" }) + const s = await w.start(async () => { throw new Error("must not run") }) + // Tamper with what the floor serves: the grant still carries the original sha256. + const orig = w.client.runScript.bind(w.client) + ;(s.p.deps.client as { runScript: (id: string) => Promise }).runScript = async (id) => (await orig(id)) + "\n// tampered" + await until("done", async () => (await w.client.job("run-pullrun0003")).state === "done") + expect((await w.client.output("run-pullrun0003")).output).toMatchObject({ reason: "pre-start/script-mismatch" }) + }) +}) + +describe("local executor (the repo's runners)", () => { + const bin = mkdtempSync(join(tmpdir(), "substrate-puller-bin-")) + const saved = process.env.PATH + beforeAll(() => { + writeFileSync(join(bin, "claude"), `#!/bin/sh\np=$(cat)\ncase "$p" in\n *gold*) echo '{"type":"result","subtype":"success","is_error":false,"session_id":"s-gold","result":"Au","usage":{"input_tokens":3,"output_tokens":1}}' ;;\nesac\n`) + chmodSync(join(bin, "claude"), 0o755) + process.env.PATH = `${bin}:${saved}` + }) + afterAll(() => { process.env.PATH = saved }) + + it("runs a leased run with runIntegrated on a host runtime and completes it with the return value", async () => { + const w = await world() + await w.client.submitScript(`export const meta = { name: "gold", description: "d" }\nreturn await agent("say gold")\n`, { id: "pullrun0004" }) + const runtimes = parseRuntimesToml(`default = "h"\n[runtime.h]\ntype = "host"\nharness = "claude"\n`, "test") + await w.start(localExecutor({ runtimes, seat: "fake", defaultModel: "fake-model", cap: 1, maxAttempts: 1 })) + const run = await w.client.waitRun("pullrun0004", { timeoutMs: 30_000, pollMs: 25 }) + expect([run.state, run.result]).toEqual(["done", "success"]) + expect((await w.client.output("run-pullrun0004")).output).toMatchObject({ status: "completed", result: "Au", outcome: "all-done" }) + expect(existsSync(join(w.stateDir, "runs", "pullrun0004", "journal.jsonl"))).toBe(true) + }) + + it("AUDIT-transcripts TX2: every node's archived transcript reaches the floor under the run's job before the verdict", async () => { + const w = await world() + await w.client.submitScript(`export const meta = { name: "txs", description: "d" }\nreturn await agent("say gold")\n`, { id: "pullrun0006" }) + const runtimes = parseRuntimesToml(`default = "h"\n[runtime.h]\ntype = "host"\nharness = "claude"\n`, "test") + await w.start(localExecutor({ runtimes, seat: "fake", defaultModel: "fake-model", cap: 1, maxAttempts: 1, transcripts: { client: w.client } })) + const run = await w.client.waitRun("pullrun0006", { timeoutMs: 30_000, pollMs: 25 }) + expect([run.state, run.result]).toEqual(["done", "success"]) + const out = (await w.client.output("run-pullrun0006")).output as { transcripts: { uploaded: number; failed: Array } } + expect(out.transcripts.uploaded).toBeGreaterThan(0) + expect(out.transcripts.failed).toEqual([]) + const m = await w.client.transcripts("run-pullrun0006") + const stdout = m.parts.find((p) => p.part.endsWith("__harness.stdout"))! + expect(stdout.committed).toBe(true) + expect(await w.client.transcript("run-pullrun0006", stdout.part)).toContain("Au") + }) + + it("the floor-dispatch outcome carries a reported session id as agentId (TX3)", () => { + expect(outcomeOf({ result: "success", output: { text: "x", sessionId: "sess-1" }, usage: null }, false)).toEqual({ text: "x", agentId: "sess-1" }) + expect(outcomeOf({ result: "success", output: { text: "x" }, usage: null }, false)).toEqual({ text: "x" }) + expect(transcriptPartName("wf_a-1-a1", "claude-x/y.jsonl")).toBe("wf_a-1-a1__claude-x_y.jsonl") + expect(transcriptPartName("j", "f".repeat(300)).length).toBe(200) + }) + + it("an ax runtime is refused with the reason, and the run fails with it", async () => { + const w = await world() + await w.client.submitScript(`export const meta = { name: "axed", description: "d" }\nconst r = await agent("say gold")\nif (r === null) throw new Error("the ax node was refused")\nreturn r\n`, { id: "pullrun0005" }) + const runtimes = parseRuntimesToml(`default = "axr"\n[runtime.axr]\ntype = "ax"\n`, "test") + await w.start(localExecutor({ runtimes, seat: "fake", defaultModel: "fake-model", cap: 1, maxAttempts: 1, axServer: "127.0.0.1:1" })) + const run = await w.client.waitRun("pullrun0005", { timeoutMs: 30_000, pollMs: 25 }) + expect([run.state, run.result]).toEqual(["done", "failure"]) + expect((await w.client.output("run-pullrun0005")).output).toMatchObject({ reason: "agent/script-failed" }) + const ledger = readFileSync(join(w.stateDir, "runs", "pullrun0005", "ledger.jsonl"), "utf8") + expect(ledger).toMatch(/ax-server 127\.0\.0\.1:1 is not reachable/) + }) +}) + +describe("process", () => { + it("holds a pidfile: a live holder refuses a second copy, a dead one is replaced", () => { + const dir = mkdtempSync(join(tmpdir(), "substrate-pid-")) + const path = join(dir, "puller.pid") + writeFileSync(path, `${process.pid}\n`) + expect(() => acquirePidfile(path, "node", 2_000_000_000)).toThrow(PidfileHeld) + writeFileSync(path, "2000000001\n") + const release = acquirePidfile(path, "node", 2_000_000_000) + expect(readFileSync(path, "utf8").trim()).toBe("2000000000") + release() + expect(existsSync(path)).toBe(false) + }) + + it("main refuses to start without a [puller] config (exit 78)", async () => { + const dir = mkdtempSync(join(tmpdir(), "substrate-pcfg-")) + writeFileSync(join(dir, "config.toml"), `floor_url = "http://floor.test"\n`) + const writes: Array = [] + const orig = process.stdout.write.bind(process.stdout) + process.stdout.write = ((s: string) => { writes.push(s); return true }) as typeof process.stdout.write + try { expect(await main({ SUBSTRATE_CLIENT_CONFIG: join(dir, "config.toml") })).toBe(78) } finally { process.stdout.write = orig } + expect(writes.join("")).toMatch(/config-invalid.*holder is required/) + }) +}) diff --git a/pkgs/substrate-apps/src/apps/puller/test/redteam-loss-r3-1.test.ts b/pkgs/substrate-apps/src/apps/puller/test/redteam-loss-r3-1.test.ts new file mode 100644 index 00000000..ecab0007 --- /dev/null +++ b/pkgs/substrate-apps/src/apps/puller/test/redteam-loss-r3-1.test.ts @@ -0,0 +1,126 @@ +// Red team loss-and-wip r3-1 (critique pass KEEP-12): a failed start-up re-adoption never lets the puller hold more runs than maxRuns. +// CONTROL: an identical kill and restart with no Heartbeat failure must finish the run (so any failure below is the +// transient readopt error, not the harness). CASE A: one transient failure on the restart's first Heartbeat. +// CASE B: the same, then a second run is submitted; maxRuns is 1. +import { mkdtempSync, readFileSync } from "node:fs" +import { tmpdir } from "node:os" +import { join } from "node:path" +import { afterEach, describe, expect, it } from "vitest" +import { SubstrateClient } from "@substrate/api" +import { localFloor } from "@substrate/floor/local.ts" +import { FloorError } from "@substrate/link/floor.ts" +import { connectFloor } from "../src/connect.ts" +import { floorExecutor } from "../src/execute.ts" +import { Puller } from "../src/puller.ts" +import type { Executor, FloorPort } from "../src/puller.ts" +import { PullerState } from "../src/state.ts" + +const T = { floor: "rf-floor", coord: "rf-coord", nas: "rf-nas" } +const URL_ = "http://floor.test" +const cleanups: Array<() => unknown> = [] +afterEach(async () => { for (const c of cleanups.splice(0).reverse()) await c() }) +const until = async (what: string, f: () => Promise | boolean, ms = 15_000) => { + const end = Date.now() + ms + while (!(await f())) { if (Date.now() > end) throw new Error(`timed out waiting for ${what}`); await new Promise((r) => setTimeout(r, 20)) } +} +const sleep = (ms: number) => new Promise((r) => setTimeout(r, ms)) +const TWO_STEP = `export const meta = { name: "two-step", description: "d" } +const a = await agent("first", { label: "a" }) +const b = await agent("second after " + a, { label: "b" }) +return { a, b } +` +const world = async () => { + const f = localFloor({ floorToken: T.floor, links: [ + { token: T.coord, holder: "coord", labels: ["runtime:interpreter", "seat:interp"] }, + { token: T.nas, holder: "nas", labels: ["seat:t", "runtime:*"] } + ], vars: { FLOOR_UNGATED_SEATS: "*", FLOOR_INTERPRETER_SEAT: "interp", FLOOR_HEARTBEAT_SECONDS: "1", FLOOR_LEASE_SECONDS: "2", FLOOR_GRACE_SECONDS: "2", FLOOR_POLL_SECONDS: "1" } }) + cleanups.push(() => f.close()) + const client = new SubstrateClient({ url: URL_, token: T.floor, fetch: f.fetch }) + const stateDir = mkdtempSync(join(tmpdir(), "rf-puller-")) + new PullerState(stateDir).sessionId() + const nas = await connectFloor({ url: URL_, token: T.nas, sessionId: "nas-session", fetch: f.fetch }) + cleanups.push(() => nas.close()) + const serveNodes = async () => { + const r = await nas.port.lease({ holderIdentity: "nas", capacity: 10, requestKey: crypto.randomUUID() }) + for (const g of r.grants) { + const p = g.job.spec.with.prompt ?? "" + const out = p === "first" ? "A" : p.startsWith("second") ? "B" : undefined + if (out !== undefined) await nas.port.complete({ leaseId: g.leaseId, attempt: g.attempt, result: "success", output: out }) + } + } + const start = async (execute: Executor, wrap: (p: FloorPort) => FloorPort = (p) => p) => { + const conn = await connectFloor({ url: URL_, token: T.coord, sessionId: readFileSync(join(stateDir, "session-id"), "utf8").trim(), fetch: f.fetch }) + const events: Array<{ ev: string; f?: Record }> = [] + const p = new Puller({ holder: "coord", maxRuns: 1, stateDir, pollMs: 25, heartbeatMs: 150 }, { floor: wrap(conn.port), client, execute, log: (ev, fields) => { events.push({ ev, ...(fields ? { f: fields } : {}) }) } }) + const done = p.run().finally(() => conn.close()) + cleanups.push(async () => { p.stop(); await done.catch(() => undefined) }) + return { p, done, events } + } + return { client, serveNodes, start } +} +const failFirstHeartbeat = (port: FloorPort): FloorPort => { + let failed = 0 + return { ...port, heartbeat: async (p) => { if (failed++ === 0) throw new FloorError("transient", "connection reset at boot"); return port.heartbeat(p) } } +} +const killAfterNode1 = async (w: Awaited>, id: string, exec: Executor) => { + await w.client.submitScript(TWO_STEP, { id }) + const first = await w.start(exec) + await until("node 1 enqueued", async () => (await w.client.runJobs(id)).filter((j: any) => j.kind === "agent").length >= 1) + first.p.stop(); await first.done +} +const mkExec = (w: Awaited>) => floorExecutor({ client: w.client, defaultRunsOn: ["seat:t", "runtime:gvisor"], defaultModel: "test-model", pollMs: 25 }) + +describe("loss-and-wip-r3-1: re-adoption and maxRuns", () => { + it("CONTROL: a clean restart re-adopts and finishes the run", async () => { + const w = await world(); const exec = mkExec(w) + await killAfterNode1(w, "rfrun00000", exec) + const second = await w.start(exec) + const end = Date.now() + 8_000 + while (Date.now() < end && (await w.client.run("rfrun00000")).state !== "done") { await w.serveNodes(); await sleep(100) } + const run = await w.client.run("rfrun00000") + console.log("CONTROL", JSON.stringify({ runState: run.state, readopt: second.events.some((e) => e.ev === "readopt") })) + expect(run.state).toBe("done") + }, 30_000) + + it("CASE A: one transient first Heartbeat after restart; the run must still finish", async () => { + const w = await world(); const exec = mkExec(w) + await killAfterNode1(w, "rfrun00001", exec) + const second = await w.start(exec, failFirstHeartbeat) + const end = Date.now() + 10_000 + while (Date.now() < end && (await w.client.run("rfrun00001")).state !== "done") { await w.serveNodes(); await sleep(100) } + const job = await w.client.job("run-rfrun00001"), run = await w.client.run("rfrun00001") + console.log("CASE A", JSON.stringify({ runState: run.state, jobState: job.state, holder: job.holder, attempt: job.attempt, + readopt: second.events.some((e) => e.ev === "readopt"), runStart: second.events.filter((e) => e.ev === "run-start").length, + readoptErrors: second.events.filter((e) => e.ev === "readopt-error").length, heartbeatErrors: second.events.filter((e) => e.ev === "heartbeat-error").length, + held: Object.keys(second.p.held), active: [...second.p.active.keys()] })) + expect(run.state).toBe("done") + }, 30_000) + + it("CASE B: with the un-readopted run held, maxRuns 1 must not hold a second run", async () => { + const w = await world(); const exec = mkExec(w) + await killAfterNode1(w, "rfrun00002", exec) + const second = await w.start(exec, failFirstHeartbeat) + await w.client.submitScript(TWO_STEP.replace("two-step", "other"), { id: "rfrun00003" }) + let leased = true + try { await until("second run leased by coord", async () => (await w.client.job("run-rfrun00003")).holder === "coord", 2_000) } catch { leased = false } + await sleep(1_000) + const a = await w.client.job("run-rfrun00002"), b = await w.client.job("run-rfrun00003") + console.log("CASE B", JSON.stringify({ leased, zombie: [a.state, a.holder, a.attempt], other: [b.state, b.holder], held: Object.keys(second.p.held), active: [...second.p.active.keys()] })) + expect(Object.keys(second.p.held).length).toBeLessThanOrEqual(1) + }, 30_000) + it("CASE C: after CASE B, both runs finish one after the other, never both held", async () => { + const w = await world(); const exec = mkExec(w) + await killAfterNode1(w, "rfrun00012", exec) + const second = await w.start(exec, failFirstHeartbeat) + await w.client.submitScript(TWO_STEP.replace("two-step", "other"), { id: "rfrun00013" }) + let maxHeld = 0 + const end = Date.now() + 20_000 + while (Date.now() < end && ((await w.client.run("rfrun00012")).state !== "done" || (await w.client.run("rfrun00013")).state !== "done")) { + maxHeld = Math.max(maxHeld, Object.keys(second.p.held).length); await w.serveNodes(); await sleep(100) + } + const a = await w.client.run("rfrun00012"), b = await w.client.run("rfrun00013") + console.log("CASE C", JSON.stringify({ maxHeld, zombie: [a.state, a.result], other: [b.state, b.result] })) + expect([a.state, b.state]).toEqual(["done", "done"]) + expect(maxHeld).toBeLessThanOrEqual(1) + }, 40_000) +}) diff --git a/pkgs/substrate-apps/src/apps/puller/test/redteam-loss.test.ts b/pkgs/substrate-apps/src/apps/puller/test/redteam-loss.test.ts new file mode 100644 index 00000000..d6eaa6b7 --- /dev/null +++ b/pkgs/substrate-apps/src/apps/puller/test/redteam-loss.test.ts @@ -0,0 +1,155 @@ +// Red team loss-and-wip, the puller's leftovers (critique pass 2026-09-24, KEEP-3 and KEEP-7): the coordinator puller +// against the real floor (localFloor over node:sqlite). Each case failed on 2882545. +import { existsSync, mkdtempSync, readFileSync } from "node:fs" +import { tmpdir } from "node:os" +import { join } from "node:path" +import { afterEach, describe, expect, it } from "vitest" +import { SubstrateClient, SubstrateError } from "@substrate/api" +import { localFloor } from "@substrate/floor/local.ts" + +import { connectFloor } from "../src/connect.ts" +import { floorExecutor } from "../src/execute.ts" +import { Puller } from "../src/puller.ts" +import { FloorError } from "@substrate/link/floor.ts" +import type { Executor } from "../src/puller.ts" +import type { FloorPort } from "../src/puller.ts" +import { PullerState } from "../src/state.ts" + +const T = { floor: "rt-floor", coord: "rt-coord", nas: "rt-nas" } +const URL_ = "http://floor.test" +const cleanups: Array<() => unknown> = [] +afterEach(async () => { for (const c of cleanups.splice(0).reverse()) await c() }) + +const until = async (what: string, f: () => Promise | boolean, ms = 15_000) => { + const end = Date.now() + ms + while (!(await f())) { if (Date.now() > end) throw new Error(`timed out waiting for ${what}`); await new Promise((r) => setTimeout(r, 20)) } +} + +const world = async (vars: Record = {}) => { + const f = localFloor({ floorToken: T.floor, links: [ + { token: T.coord, holder: "coord", labels: ["runtime:interpreter", "seat:interp"] }, + { token: T.nas, holder: "nas", labels: ["seat:t", "runtime:*"] } + ], vars: { FLOOR_UNGATED_SEATS: "*", FLOOR_INTERPRETER_SEAT: "interp", FLOOR_HEARTBEAT_SECONDS: "1", FLOOR_LEASE_SECONDS: "30", FLOOR_POLL_SECONDS: "1", ...vars } as never }) + cleanups.push(() => f.close()) + const client = new SubstrateClient({ url: URL_, token: T.floor, fetch: f.fetch }) + const stateDir = mkdtempSync(join(tmpdir(), "substrate-rt-puller-")) + new PullerState(stateDir).sessionId() + const nas = await connectFloor({ url: URL_, token: T.nas, sessionId: "nas-session", fetch: f.fetch }) + cleanups.push(() => nas.close()) + const serveNodes = async (answer: (prompt: string) => unknown | undefined) => { + const r = await nas.port.lease({ holderIdentity: "nas", capacity: 10, requestKey: crypto.randomUUID() }) + for (const g of r.grants) { + const out = answer(g.job.spec.with.prompt ?? "") + if (out !== undefined) await nas.port.complete({ leaseId: g.leaseId, attempt: g.attempt, result: "success", output: out }) + } + return r.grants.length + } + const start = async (execute: Executor, o: { maxRuns?: number; verdictAttempts?: number; port?: (real: FloorPort) => FloorPort; client?: SubstrateClient } = {}) => { + const conn = await connectFloor({ url: URL_, token: T.coord, sessionId: readFileSync(join(stateDir, "session-id"), "utf8").trim(), fetch: f.fetch }) + const events: Array<{ ev: string; f?: Record }> = [] + const port = o.port ? o.port(conn.port) : conn.port + const p = new Puller({ holder: "coord", maxRuns: o.maxRuns ?? 1, stateDir, pollMs: 25, heartbeatMs: 150, ...(o.verdictAttempts !== undefined ? { verdictAttempts: o.verdictAttempts } : {}) }, + { floor: port, client: o.client ?? client, execute, log: (ev, fields) => { if (process.env.PULLER_DEBUG) console.error(ev, JSON.stringify(fields)); events.push({ ev, ...(fields ? { f: fields } : {}) }) } }) + const done = p.run().finally(() => conn.close()) + cleanups.push(async () => { p.stop(); await done.catch(() => undefined) }) + return { p, done, events } + } + return { f, client, stateDir, serveNodes, start } +} + +const ONE_STEP = `export const meta = { name: "one-step", description: "d" } +const a = await agent("first", { label: "a" }) +return { a } +` + + +/** A server error for Complete calls carrying this lease's verdict of this result; everything else is the real floor. */ +const refuse = (leaseId: string, result: string, status: number | undefined, counter: { n: number }) => (port: FloorPort): FloorPort => ({ + ...port, complete: async (p) => { if (p.leaseId === leaseId && p.result === result) { counter.n++; throw new FloorError("server-error", `floor answered ${status ?? "a Defect"}`, undefined, status) } return port.complete(p) } +}) +const serveUntilDone = async (w: Awaited>, id: string, ms = 15_000) => { + const end = Date.now() + ms + let run: any + while (Date.now() < end) { await w.serveNodes(() => "ok"); run = await w.client.run(id); if (run.state === "done") break; await new Promise((r) => setTimeout(r, 25)) } + return run +} + +describe("KEEP-3 (r2-2): a verdict the floor keeps refusing with a server error", () => { + it("is replaced by a typed failure naming its output after verdictAttempts, and no new run is leased while it waits", async () => { + const w = await world() + const exec = floorExecutor({ client: w.client, defaultRunsOn: ["seat:t"], defaultModel: "test-model", pollMs: 25 }) + const hits = { n: 0 } + const s = await w.start(exec, { verdictAttempts: 3, port: refuse("run-rlk3run001-a1", "success", 500, hits) }) + await w.client.submitScript(ONE_STEP, { id: "rlk3run001" }) + await until("run 1 node", async () => (await w.client.runJobs("rlk3run001")).filter((j: any) => j.kind === "agent").length >= 1) + await w.client.submitScript(ONE_STEP.replace("one-step", "second"), { id: "rlk3run002" }) + await until("run 1 finished here", async () => { await w.serveNodes(() => "ok"); return s.events.some((e) => e.ev === "run-end" && e.f?.leaseId === "run-rlk3run001-a1") }) + const run1 = await serveUntilDone(w, "rlk3run001") + const out = await w.client.output("run-rlk3run001") as { result: string; output: { reason?: string; replaces?: string; outputSha256?: string } } + expect([run1.state, out.result, out.output.reason, out.output.replaces]).toEqual(["done", "failure", "infra/verdict-undeliverable", "success"]) + expect(out.output.outputSha256).toMatch(/^[0-9a-f]{64}$/) + expect(hits.n).toBeGreaterThanOrEqual(3) // every send is refused; only those after a floor answer are charged + // maxRuns 1: run 2 starts only after run 1's verdict left the outbox + const idx = (ev: string, lease: string) => s.events.findIndex((e) => e.ev === ev && e.f?.leaseId === lease) + expect(idx("verdict-replaced", "run-rlk3run001-a1")).toBeGreaterThanOrEqual(0) + const run2 = await serveUntilDone(w, "rlk3run002") + expect(run2.result).toBe("success") + expect(idx("run-start", "run-rlk3run002-a1")).toBeGreaterThan(idx("verdict-replaced", "run-rlk3run001-a1")) + }, 40_000) + + it("a later cancel never overwrites the success waiting in the outbox", async () => { + const w = await world() + const exec = floorExecutor({ client: w.client, defaultRunsOn: ["seat:t"], defaultModel: "test-model", pollMs: 25 }) + const hits = { n: 0 } + const s = await w.start(exec, { verdictAttempts: 1_000_000, port: refuse("run-rlk3run003-a1", "success", 500, hits) }) + await w.client.submitScript(ONE_STEP, { id: "rlk3run003" }) + await until("run finished here", async () => { await w.serveNodes(() => "ok"); return s.events.some((e) => e.ev === "run-end" && e.f?.leaseId === "run-rlk3run003-a1") }) + await until("a few refusals", () => hits.n >= 3) + await w.client.cancelJob("run-rlk3run003") + await until("the cancel was heard", () => s.events.some((e) => e.ev === "cancel-requested" && e.f?.leaseId === "run-rlk3run003-a1")) + await until("one more refusal", () => { const n = hits.n; return new Promise((r) => setTimeout(() => r(hits.n > n), 200)) }) + const file = JSON.parse(readFileSync(join(w.stateDir, "outbox", "run-rlk3run003-a1.json"), "utf8")) as { result: string } + expect(file.result).toBe("success") + expect((await w.client.job("run-rlk3run003")).state).not.toBe("done") + }, 40_000) + + it("control: an HTTP 5xx with no floor answer since the last one is not charged (a floor-wide outage)", async () => { + const w = await world() + const exec = floorExecutor({ client: w.client, defaultRunsOn: ["seat:t"], defaultModel: "test-model", pollMs: 25 }) + let down = false, tripped = false, n = 0 + const outage = (port: FloorPort): FloorPort => { + const gate = async (f: () => Promise) => { if (down) { n++; throw new FloorError("server-error", "floor answered 500", undefined, 500) } return f() } + return { lease: (p) => gate(() => port.lease(p)), heartbeat: (p) => gate(() => port.heartbeat(p)), complete: (p) => { if (!tripped && p.result === "success" && p.leaseId === "run-rlk3run004-a1") { tripped = true; down = true } return gate(() => port.complete(p)) } } + } + const s = await w.start(exec, { verdictAttempts: 2, port: outage }) + await w.client.submitScript(ONE_STEP, { id: "rlk3run004" }) + await until("run finished here", async () => { await w.serveNodes(() => "ok"); return s.events.some((e) => e.ev === "run-end" && e.f?.leaseId === "run-rlk3run004-a1") }) + await until("many 500s", () => n >= 10) + down = false + const run = await serveUntilDone(w, "rlk3run004") + expect([run.state, run.result]).toEqual(["done", "success"]) + expect(s.events.some((e) => e.ev === "verdict-replaced")).toBe(false) + }, 40_000) +}) + +describe("KEEP-7 (r2-1 residual): a host I/O error in the executor is retried, not final", () => { + it("the executor throws ENOSPC once; the floor requeues and the run finishes success", async () => { + const w = await world() + await w.client.submitScript(ONE_STEP, { id: "rlk7run001" }) + const real = floorExecutor({ client: w.client, defaultRunsOn: ["seat:t"], defaultModel: "test-model", pollMs: 25 }) + let n = 0 + const exec: Executor = async (x) => { if (n++ === 0) throw Object.assign(new Error("ENOSPC: no space left on device, write"), { code: "ENOSPC" }); return real(x) } + await w.start(exec) + const run = await serveUntilDone(w, "rlk7run001") + const job = await w.client.job("run-rlk7run001") + expect([run.state, run.result, job.attempt]).toEqual(["done", "success", 2]) + }, 30_000) + it("control: any other executor exception stays final (infra/puller-defect)", async () => { + const w = await world() + await w.client.submitScript(ONE_STEP, { id: "rlk7run002" }) + await w.start(async () => { throw new TypeError("x is not a function") }) + await until("run done", async () => (await w.client.run("rlk7run002")).state === "done") + const out = await w.client.output("run-rlk7run002") as { result: string; output: { reason: string } } + expect([out.result, out.output.reason]).toEqual(["failure", "infra/puller-defect"]) + }, 30_000) +}) diff --git a/pkgs/substrate-apps/src/apps/puller/test/reverify-2026-09-24.test.ts b/pkgs/substrate-apps/src/apps/puller/test/reverify-2026-09-24.test.ts new file mode 100644 index 00000000..b2f03699 --- /dev/null +++ b/pkgs/substrate-apps/src/apps/puller/test/reverify-2026-09-24.test.ts @@ -0,0 +1,95 @@ +// Critique pass 2026-09-24 (AUDIT-reverify-interp): C3-4 and C3-5 on the floor-dispatch path against the REAL floor +// (the Worker entry and Floor object over node:sqlite, apps/floor/src/local.ts), not a double. The floor must keep a +// finished node's name and verdict for as long as a run can resume, so a resumed executor adopts it by name. +import { mkdtempSync, writeFileSync } from "node:fs" +import { tmpdir } from "node:os" +import { join } from "node:path" +import { afterEach, describe, expect, it } from "vitest" +import { SubstrateClient } from "@substrate/api" +import { localFloor } from "@substrate/floor/local.ts" +import { connectFloor } from "../src/connect.ts" +import { floorExecutor } from "../src/execute.ts" +import type { RunTask } from "../src/puller.ts" + +const T = { floor: "rv-floor", nas: "rv-nas", coord: "rv-coord" } +const cleanups: Array<() => unknown> = [] +afterEach(async () => { for (const c of cleanups.splice(0).reverse()) await c() }) +const until = async (what: string, f: () => Promise | boolean, ms = 10_000) => { + const end = Date.now() + ms + while (!(await f())) { if (Date.now() > end) throw new Error(`timed out waiting for ${what}`); await new Promise((r) => setTimeout(r, 20)) } +} +const S = `export const meta = { name: "rv", description: "d" }\nconst a = await agent("prep")\nconst b = await agent("act " + a)\nreturn b\n` + +const world = async (id: string) => { + const f = localFloor({ floorToken: T.floor, links: [ + { token: T.coord, holder: "coord", labels: ["runtime:interpreter", "seat:interp"] }, + { token: T.nas, holder: "nas", labels: ["seat:t", "runtime:*"] } + ], vars: { FLOOR_UNGATED_SEATS: "*", FLOOR_INTERPRETER_SEAT: "interp", FLOOR_HEARTBEAT_SECONDS: "1", FLOOR_LEASE_SECONDS: "30", FLOOR_POLL_SECONDS: "1" } }) + cleanups.push(() => f.close()) + const client = new SubstrateClient({ url: "http://floor.test", token: T.floor, fetch: f.fetch }) + const nas = await connectFloor({ url: "http://floor.test", token: T.nas, sessionId: "rv-nas-session", fetch: f.fetch }) + cleanups.push(() => nas.close()) + await client.submitScript(S, { id }) + // G-BK5 (merged after this test was written): node enqueues name the run's live interpreter lease, so the run is + // really leased. Both executors run under that one lease: the second is the puller re-adopting it after the kill + // (the node name carries the CALL's attempt, not the run's, so adoption by name is what is tested either way). + const interp = await connectFloor({ url: "http://floor.test", token: T.coord, sessionId: "rv-coord-session", fetch: f.fetch }) + cleanups.push(() => interp.close()) + const runGrant = (await interp.port.lease({ holderIdentity: "coord", capacity: 1, requestKey: crypto.randomUUID() })).grants.find((g) => g.leaseId === `run-${id}-a1`) + if (!runGrant) throw new Error("the run was not leased to the interpreter link") + const dir = mkdtempSync(join(tmpdir(), "substrate-rv-run-")) + const scriptPath = join(dir, "script.js"); writeFileSync(scriptPath, S) + const task = (_attempt: number, signal: AbortSignal): RunTask => + ({ runId: id, leaseId: runGrant.leaseId, attempt: runGrant.attempt, job: {} as never, workflowName: "rv", args: undefined, dir, scriptPath, script: S, signal }) + /** The NAS link by hand: lease what is queued and answer each node by its prompt; counts grants. */ + let grants = 0 + const serve = async (answer: (prompt: string) => { result: "success" | "failure"; output: unknown } | undefined) => { + const r = await nas.port.lease({ holderIdentity: "nas", capacity: 10, requestKey: crypto.randomUUID() }) + for (const g of r.grants) { + grants++ + const a = answer(g.job.spec.with.prompt ?? "") + if (a) await nas.port.complete({ leaseId: g.leaseId, attempt: g.attempt, result: a.result, output: a.output, usage: { prompt_tokens: 3, completion_tokens: 1, tool_calls: 0 } }) + } + return r.grants.length + } + const agentJobs = async () => (await client.runJobs(id)).filter((j) => j.kind === "agent") + const exec = floorExecutor({ client, defaultRunsOn: ["seat:t"], defaultModel: "m", pollMs: 20, maxAttempts: 2 }) + return { client, task, serve, agentJobs, exec, grants: () => grants } +} + +describe("floor retention: a resumed executor adopts finished nodes by name", () => { + it("C3-4: a node that finished while no executor watched is adopted, with no new node and no new lease", async () => { + const w = await world("rvrun0034") + const ac = new AbortController() + const p1 = w.exec(w.task(1, ac.signal)) + await until("prep enqueued", async () => (await w.agentJobs()).length >= 1) + await until("prep served", async () => (await w.serve((p) => (p === "prep" ? { result: "success", output: "A" } : undefined))) === 1) + await until("act enqueued", async () => (await w.agentJobs()).length >= 2) + ac.abort() // the executor is killed with act in flight; act keeps its lease on the NAS link + await p1.catch(() => undefined) + await until("act served", async () => (await w.serve((p) => (p.startsWith("act") ? { result: "success", output: "B" } : undefined))) === 1) + const grantsBefore = w.grants() + const v2 = await w.exec(w.task(2, new AbortController().signal)) + expect(v2.result).toBe("success") + expect((v2.output as { result?: unknown }).result).toBe("B") + expect(await w.agentJobs()).toHaveLength(2) + expect(await w.serve(() => undefined)).toBe(0) + expect(w.grants()).toBe(grantsBefore) + }) + + it("C3-5: a node that failed is the same failure on the next run attempt, with no new node", async () => { + const w = await world("rvrun0035") + const p1 = w.exec(w.task(1, new AbortController().signal)) + await until("prep enqueued", async () => (await w.agentJobs()).length >= 1) + await until("prep served", async () => (await w.serve((p) => (p === "prep" ? { result: "success", output: "A" } : undefined))) === 1) + await until("act enqueued", async () => (await w.agentJobs()).length >= 2) + await until("act failed", async () => (await w.serve((p) => (p.startsWith("act") ? { result: "failure", output: { reason: "agent/x", message: "boom" } } : undefined))) >= 1) + const v1 = await p1 + expect(v1.result).toBe("failure") + const nodes = (await w.agentJobs()).length + const v2 = await w.exec(w.task(2, new AbortController().signal)) + expect(v2.result).toBe("failure") + expect(await w.agentJobs()).toHaveLength(nodes) + expect(await w.serve(() => undefined)).toBe(0) + }) +}) diff --git a/pkgs/substrate-apps/src/apps/puller/test/review3.test.ts b/pkgs/substrate-apps/src/apps/puller/test/review3.test.ts new file mode 100644 index 00000000..320b29cd --- /dev/null +++ b/pkgs/substrate-apps/src/apps/puller/test/review3.test.ts @@ -0,0 +1,156 @@ +// Codex review 3 (2026-09-24): each confirmed finding as the test that failed before its fix. +import { existsSync, mkdtempSync, readFileSync, writeFileSync } from "node:fs" +import { tmpdir } from "node:os" +import { join } from "node:path" +import { describe, expect, it } from "vitest" +import { SubstrateClient } from "@substrate/api" +import { localFloor } from "@substrate/floor/local.ts" +import { contentHash, FileJournal, parseJournal } from "@substrate/interpreter" +import { connectFloor } from "../src/connect.ts" +import { floorExecutor, nodeJob } from "../src/execute.ts" +import { Puller } from "../src/puller.ts" +import type { Executor, FloorPort, RunTask } from "../src/puller.ts" +import { PullerState } from "../src/state.ts" + +const until = async (what: string, f: () => Promise | boolean, ms = 10_000) => { + const end = Date.now() + ms + while (!(await f())) { if (Date.now() > end) throw new Error(`timed out waiting for ${what}`); await new Promise((r) => setTimeout(r, 20)) } +} +const T = { floor: "r3-floor", coord: "r3-coord" } +const world = async () => { + const f = localFloor({ floorToken: T.floor, links: [{ token: T.coord, holder: "coord", labels: ["runtime:interpreter", "seat:interp"] }], + vars: { FLOOR_UNGATED_SEATS: "*", FLOOR_INTERPRETER_SEAT: "interp", FLOOR_HEARTBEAT_SECONDS: "1", FLOOR_LEASE_SECONDS: "30", FLOOR_POLL_SECONDS: "1" } }) + const client = new SubstrateClient({ url: "http://floor.test", token: T.floor, fetch: f.fetch }) + const stateDir = mkdtempSync(join(tmpdir(), "substrate-r3-")) + const sid = new PullerState(stateDir).sessionId() + const conns: Array<{ close: () => unknown }> = [] + const start = async (execute: Executor, wrap: (p: FloorPort) => FloorPort = (p) => p) => { + const conn = await connectFloor({ url: "http://floor.test", token: T.coord, sessionId: sid, fetch: f.fetch }); conns.push(conn) + const events: Array = [] + const p = new Puller({ holder: "coord", maxRuns: 1, stateDir, pollMs: 25, heartbeatMs: 100 }, { floor: wrap(conn.port), client, execute, log: (ev) => events.push(ev) }) + const done = p.run().catch(() => undefined) + return { p, done, events } + } + const close = async () => { for (const c of conns) await c.close(); f.close() } + return { f, client, stateDir, start, close } +} +const SCRIPT = `export const meta = { name: "r3", description: "d" }\nreturn "ok"\n` +/** A run leased by a first puller whose executor never ends, then that puller stopped: the grant stays in held.json. */ +const heldRun = async (w: Awaited>, id: string) => { + await w.client.submitScript(SCRIPT, { id }) + const first = await w.start((t) => new Promise((ok) => t.signal.addEventListener("abort", () => ok({ result: "failure", output: null }), { once: true }))) + await until("leased", () => Object.keys(first.p.held).length === 1) + first.p.stop(); await first.done + return Object.keys(JSON.parse(readFileSync(join(w.stateDir, "held.json"), "utf8")))[0]! +} + +describe("codex review 3: the puller", () => { + it("C3-1: a failed start-up heartbeat does not strand a held run; the next renewal launches it", async () => { + const w = await world() + try { + await heldRun(w, "r3run0001") + let runs = 0, first = true + const s = await w.start(async () => { runs++; return { result: "success", output: "ok" } }, + (p) => ({ ...p, heartbeat: (q) => { if (first) { first = false; return Promise.reject(new Error("socket hang up")) } return p.heartbeat(q) } })) + await until("the run executed", () => runs === 1) + await until("done", async () => (await w.client.job("run-r3run0001")).state === "done") + expect((await w.client.output("run-r3run0001")).result).toBe("success") + s.p.stop(); await s.done + } finally { await w.close() } + }) + it("C3-2: a held run whose verdict waits in the outbox is never executed again, and that verdict is the one delivered", async () => { + const w = await world() + try { + const leaseId = await heldRun(w, "r3run0002") + new PullerState(w.stateDir).putVerdict({ leaseId, attempt: 1, result: "failure", output: { reason: "agent/script-failed", message: "the first execution" } }) + let runs = 0, fails = 2 + const s = await w.start(async () => { runs++; return { result: "success", output: "second execution" } }, + (p) => ({ ...p, complete: (q) => fails-- > 0 ? Promise.reject(new Error("socket hang up")) : p.complete(q) })) + await until("done", async () => (await w.client.job("run-r3run0002")).state === "done") + expect(runs).toBe(0) + expect((await w.client.output("run-r3run0002")).result).toBe("failure") + s.p.stop(); await s.done + } finally { await w.close() } + }) +}) + +// ---------------------------------------------------------------- the floor executor with a scripted client +type J = { name: string; state: string; result?: { result: string; output: unknown } } +const fakeClient = (answer: (prompt: string) => { result: string; output: unknown } | "hold", hooks: { enqueueFailures?: number; jobGate?: Promise } = {}) => { + const jobs = new Map() + let enqueueFailures = hooks.enqueueFailures ?? 0 + const client = { + enqueued: 0, + async enqueue(_run: string, list: Array) { + for (const j of list) if (!jobs.has(j.metadata.name)) { client.enqueued++; jobs.set(j.metadata.name, { name: j.metadata.name, state: "queued", prompt: j.spec.with.prompt }) } + if (enqueueFailures-- > 0) throw new TypeError("fetch failed") // committed, then the answer was lost + return { enqueued: [] } + }, + async job(name: string) { + if (hooks.jobGate) await hooks.jobGate + const j = jobs.get(name)! + const a = answer(j.prompt) + if (a !== "hold") { j.state = "done"; j.result = a } + return { name, state: j.state } as never + }, + async output(name: string) { const j = jobs.get(name)!; return { ...j.result!, usage: null, attempt: 1, leaseId: "", bytes: 0 } as never } + } + return client +} +const task = (script: string, signal: AbortSignal): RunTask => { + const dir = mkdtempSync(join(tmpdir(), "substrate-r3-run-")) + const scriptPath = join(dir, "script.js"); writeFileSync(scriptPath, script) + return { runId: "r3x", leaseId: "run-r3x-a1", attempt: 1, job: {} as never, workflowName: "r3", args: undefined, dir, scriptPath, script, signal } +} + +describe("codex review 3: the floor executor", () => { + it("C3-9: a node that failed and a script that returned anyway is a failed run, not a success", async () => { + const c = fakeClient(() => ({ result: "failure", output: { reason: "agent/failed" } })) + const v = await floorExecutor({ client: c as never, defaultRunsOn: ["seat:t"], defaultModel: "m", pollMs: 5 })( + task(`export const meta = { name: "r3", description: "d" }\nconst a = await agent("x")\nreturn a\n`, new AbortController().signal)) + expect(v.result).toBe("failure") + }) + it("C3-10: a lost enqueue answer is retried, not a failed run", async () => { + const c = fakeClient(() => ({ result: "success", output: "A" }), { enqueueFailures: 1 }) + const v = await floorExecutor({ client: c as never, defaultRunsOn: ["seat:t"], defaultModel: "m", pollMs: 5 })( + task(`export const meta = { name: "r3", description: "d" }\nreturn await agent("x")\n`, new AbortController().signal)) + expect([v.result, (v.output as any).result, c.enqueued]).toEqual(["success", "A", 1]) + }) + it("C3-11: after an abort the interpreter dispatches no continuation", async () => { + let open!: () => void + const gate = new Promise((ok) => { open = ok }) + const c = fakeClient(() => ({ result: "success", output: "A" }), { jobGate: gate }) + const ac = new AbortController() + const p = floorExecutor({ client: c as never, defaultRunsOn: ["seat:t"], defaultModel: "m", pollMs: 5 })( + task(`export const meta = { name: "r3", description: "d" }\nconst a = await agent("x")\nconst b = await agent("y " + a)\nreturn b\n`, ac.signal)) + await until("node 1 enqueued", () => c.enqueued === 1) + ac.abort(); await p + open(); await new Promise((r) => setTimeout(r, 100)) + expect(c.enqueued).toBe(1) + }) + it("C3-7: a node's floor name and body do not depend on its invocation index", () => { + const call = (index: number, key: string) => ({ index, key, prompt: "act A", opts: {}, phase: undefined, attempt: 1, occurrence: 1, cid: `c1:${contentHash("act A", {})}#1` }) + const o = { runId: "r3x", workflow: "w", defaultRunsOn: ["seat:t"], defaultModel: "m" } + const a = nodeJob(call(4, "v2:" + "a".repeat(64)) as never, o), b = nodeJob(call(3, "v2:" + "b".repeat(64)) as never, o) + expect(a.metadata.name).toBe(b.metadata.name) + // critique pass 2026-09-24 (durability-r1-1): the whole body, not only the name, since the floor compares bodies + expect(JSON.stringify(a)).toBe(JSON.stringify(b)) + expect(a.metadata.annotations!["ultracode.mecattaf.dev/journal-key"]).toMatch(/^[0-9a-f]{64}:1$/) // the NAS link's JOURNAL_KEY form + }) + it("C3-8: one prompt on two runtimes is two contents; without a route the content hash is unchanged", () => { + expect(contentHash("act", { runtime: "a" })).not.toBe(contentHash("act", { runtime: "b" })) + expect(contentHash("act", { label: "x" })).toBe(contentHash("act", {})) + }) + it("C3-6: a journal whose last line lost its newline keeps that event after the next append", () => { + const dir = mkdtempSync(join(tmpdir(), "substrate-r3-j-")) + const path = join(dir, "journal.jsonl") + const k = (c: string) => "v2:" + c.repeat(64) + writeFileSync(path, JSON.stringify({ type: "launched" }) + "\n" + JSON.stringify({ type: "started", key: k("a"), agentId: "x" }) + "\n" + JSON.stringify({ type: "result", key: k("a"), agentId: "x", result: "A" })) + expect(parseJournal(readFileSync(path, "utf8")).results.size).toBe(1) + new FileJournal(path).append({ type: "started", key: k("b"), agentId: "y" } as never) + const after = parseJournal(readFileSync(path, "utf8")) + expect(after.results.size).toBe(1) + expect(after.skippedLines ?? 0).toBe(0) + expect(existsSync(path)).toBe(true) + }) +}) diff --git a/pkgs/substrate-apps/src/apps/puller/test/seat-routing.test.ts b/pkgs/substrate-apps/src/apps/puller/test/seat-routing.test.ts new file mode 100644 index 00000000..178fdf5b --- /dev/null +++ b/pkgs/substrate-apps/src/apps/puller/test/seat-routing.test.ts @@ -0,0 +1,44 @@ +// Critique pass 2026-09-24, RG-3: floor dispatch labels agent({runtime: X}) with the seat X's table spends, never +// node_runs_on's seat, and refuses a runtime with no resolvable seat; the refusal is the call's error, nothing enqueued. +import { describe, expect, it } from "vitest" +import { parseRuntimesToml } from "@substrate/runners" +import { FloorBackend, nodeJob } from "../src/execute.ts" + +const runtimes = parseRuntimesToml(`default = "halogen" +[seats] +pi = "halogen" +codex = "codex" +[runtime.halogen] +type = "host" +harness = "pi" +[runtime.codex] +type = "host" +harness = "codex" +[runtime.bare] +type = "host" +harness = "pi" +seat = "worker-pi" +`, "t", "/home/u") +const call = (opts: Record) => ({ index: 1, key: "k", prompt: "p", opts, phase: undefined, attempt: 1, occurrence: 1, cid: undefined }) as never +const o = { runId: "r", workflow: "w", defaultRunsOn: ["seat:halogen", "runtime:gvisor"], defaultModel: "m" } + +describe("RG-3: floor runs-on carries the seat the runtime spends", () => { + it("runtime codex is labelled seat:codex (it was seat:halogen), runtime halogen seat:halogen, a seat alone keeps the default runtime labels", () => { + expect(nodeJob(call({ runtime: "codex" }), { ...o, runtimes }).spec["runs-on"]).toEqual(["seat:codex", "runtime:codex"]) + expect(nodeJob(call({ runtime: "halogen" }), { ...o, runtimes }).spec["runs-on"]).toEqual(["seat:halogen", "runtime:halogen"]) + expect(nodeJob(call({ runtime: "bare" }), { ...o, runtimes }).spec["runs-on"]).toEqual(["seat:worker-pi", "runtime:bare"]) + expect(nodeJob(call({ seat: "codex" }), { ...o, runtimes }).spec["runs-on"]).toEqual(["seat:codex", "runtime:gvisor"]) + expect(nodeJob(call({ runsOn: ["seat:x", "runtime:y"] }), o).spec["runs-on"]).toEqual(["seat:x", "runtime:y"]) + expect(nodeJob(call({}), o).spec["runs-on"]).toEqual(["seat:halogen", "runtime:gvisor"]) + }) + + it("a runtime with no resolvable seat, or a seat its table contradicts, is refused as the call's error and nothing is enqueued", async () => { + expect(() => nodeJob(call({ runtime: "codex" }), o)).toThrow(/the puller has no runtimes file/) + expect(() => nodeJob(call({ runtime: "nope" }), { ...o, runtimes })).toThrow(/no seat is resolvable for runtime nope/) + expect(() => nodeJob(call({ runtime: "codex", seat: "halogen" }), { ...o, runtimes })).toThrow(/runtime codex spends seat codex/) + const enq: unknown[] = [] + const b = new FloorBackend({ ...o, runtimes, signal: new AbortController().signal, client: { enqueue: async (...a: unknown[]) => { enq.push(a); return {} as never }, job: async () => ({}) as never, output: async () => ({}) as never } }) + expect(await b.run(call({ runtime: "nope" }))).toMatchObject({ error: expect.stringMatching(/no seat is resolvable/) }) + expect(enq).toHaveLength(0) + }) +}) diff --git a/pkgs/substrate-apps/src/apps/puller/tsconfig.json b/pkgs/substrate-apps/src/apps/puller/tsconfig.json new file mode 100644 index 00000000..4edbf3bf --- /dev/null +++ b/pkgs/substrate-apps/src/apps/puller/tsconfig.json @@ -0,0 +1,5 @@ +{ + "extends": "../../tsconfig.base.json", + "compilerOptions": { "exactOptionalPropertyTypes": false, "types": ["node"], "allowImportingTsExtensions": true, "lib": ["ES2022", "DOM"] }, + "include": ["src/**/*.ts", "test/**/*.ts", "../floor/src/cf-types.d.ts"] +} diff --git a/pkgs/substrate-apps/src/apps/puller/vitest.config.ts b/pkgs/substrate-apps/src/apps/puller/vitest.config.ts new file mode 100644 index 00000000..49b4806f --- /dev/null +++ b/pkgs/substrate-apps/src/apps/puller/vitest.config.ts @@ -0,0 +1,2 @@ +import { defineConfig } from "vitest/config"; +export default defineConfig({ test: { name: "puller", include: ["test/**/*.test.ts"], environment: "node", testTimeout: 60000 } }); diff --git a/pkgs/substrate-apps/src/apps/pusher/bin/seat-pusher.mjs b/pkgs/substrate-apps/src/apps/pusher/bin/seat-pusher.mjs new file mode 100644 index 00000000..afdf7a62 --- /dev/null +++ b/pkgs/substrate-apps/src/apps/pusher/bin/seat-pusher.mjs @@ -0,0 +1,182 @@ +#!/usr/bin/env node +// apps/pusher/bin/seat-pusher.mjs: LEGACY. Kept for its tests; not the package bin. The pusher to run is +// bin/substrate-pusher.mjs (docs/pusher.md), which reads `seats --json` and never calls a usage endpoint itself. +// +// pushes seat capacity to the CONWIP floor. +// +// node apps/pusher/bin/seat-pusher.mjs --url URL --token-file PATH [options] +// node apps/pusher/bin/seat-pusher.mjs --dry-run [options] +// +// Each seat named under `usage` in the config is read through its own source +// (src/usageSource.mjs): "oauth" reads the usage endpoint directly with the +// seat's credentials file, under the gentle refresh policy +// (src/refreshPolicy.mjs: 120 s hard minimum, 900 s idle, 300 s active, 180 s +// near cap, a read at each reset + 90 s, 429 backoff, no call after an auth +// failure until the credentials change); "cache-file" reads a file another +// reader wrote. A legacy meters directory (--meters) is read only when named. +// The snapshot (`seat-capacity/2`) goes OUTBOUND to `/capacity/seats`. +// +// Push policy: push when any seat's reading changed (a Claude seat's provider +// observed_at, grade, windows, dispatchability, slots, plan) and at least every +// --heartbeat seconds (default 1800), or every --slot-heartbeat seconds +// (default 600) while a dispatchable Halogen slot row is published, so the +// floor never grades a live slot row STALE; never twice within --min-interval +// seconds (default 60). A restamp with the same reading is not a change. +// +// Options: +// --url URL the floor's base URL (required unless --dry-run) +// --token-file PATH the floor's bearer, read from a file and never printed +// --config PATH default ~/.config/substrate/seat-capacity.json (optional; the +// dry run reads the same path, and the ruled seats cc3 +// and gpu-coordinator are never dispatchable either way) +// --state PATH default ~/.local/state/substrate/pusher/state.json +// --usage-dir DIR the direct reader's cache and refresh state +// (default ~/.local/state/substrate/usage) +// --demand-dir DIR demand markers (default ~/.local/state/substrate/demand) +// --meters DIR a legacy meters directory; none by default +// --capacity-dir DIR a directory holding a feeder-written seats.json; none by default +// --host NAME overrides the config's host and the hostname +// --watch SECONDS loop, one pass every SECONDS (minimum 30); default: one pass +// --heartbeat SECONDS --slot-heartbeat SECONDS --min-interval SECONDS +// --force --instance NAME +// --pre-dispatch treat this pass as a pre-dispatch read (reading older than 300 s) +// --no-network never call a usage endpoint (cached readings only) +// --dry-run print the snapshot and the decision; no network, no state +// +// Exit: 0 on a pass that pushed or had nothing due; 1 on a failed push; 2 usage. + +import { readdirSync } from "node:fs" +import { join } from "node:path" +import { parseArgs } from "node:util" + +import { tokenFromFile } from "../src/token.mjs" +import { + DEFAULT_POLICY, + defaultConfigPath, + defaultPusherDir, + readConfig, + runOnce +} from "../src/seatCapacity.mjs" + +const usage = (message) => { + process.stderr.write(`seat-pusher: ${message}\n`) + process.exit(2) +} + +let args +try { + args = parseArgs({ + options: { + url: { type: "string" }, + "token-file": { type: "string" }, + meters: { type: "string" }, + "capacity-dir": { type: "string" }, + state: { type: "string" }, + config: { type: "string" }, + host: { type: "string" }, + watch: { type: "string" }, + heartbeat: { type: "string" }, + "slot-heartbeat": { type: "string" }, + "min-interval": { type: "string" }, + instance: { type: "string" }, + "usage-dir": { type: "string" }, + "demand-dir": { type: "string" }, + "pre-dispatch": { type: "boolean", default: false }, + "no-network": { type: "boolean", default: false }, + force: { type: "boolean", default: false }, + "dry-run": { type: "boolean", default: false } + }, + strict: true + }).values +} catch (error) { + usage(error.message) +} + +const seconds = (name, fallback, minimum = 0) => { + const raw = args[name] + if (raw === undefined) return fallback + const value = Number(raw) + if (!Number.isFinite(value) || value < minimum) usage(`--${name} must be a number of seconds >= ${minimum}`) + return value +} + +const dryRun = args["dry-run"] +if (!dryRun && (args.url === undefined || args["token-file"] === undefined)) { + usage("--url and --token-file are required (or --dry-run)") +} +const metersDir = args.meters === undefined || args.meters === "" ? null : args.meters +const capacityDir = args["capacity-dir"] === undefined || args["capacity-dir"] === "" ? null : args["capacity-dir"] +const statePath = args.state ?? join(defaultPusherDir(), "state.json") +// The dry run reads the same config as the real run, so it previews what a +// real run would push. +const config = readConfig(args.config ?? defaultConfigPath()) +if (!config.present) { + process.stderr.write( + "seat-pusher: no seat-capacity config; using built-in rulings only (cc3, gpu-coordinator not dispatchable)\n" + ) +} +if (args.host !== undefined) config.host = args.host +const policy = { + heartbeatSeconds: seconds("heartbeat", DEFAULT_POLICY.heartbeatSeconds, 60), + slotHeartbeatSeconds: seconds("slot-heartbeat", DEFAULT_POLICY.slotHeartbeatSeconds, 60), + minIntervalSeconds: seconds("min-interval", DEFAULT_POLICY.minIntervalSeconds, 0) +} +const watch = args.watch === undefined ? null : seconds("watch", 300, 30) +let token = "" +if (!dryRun) { + try { + token = tokenFromFile(args["token-file"]) + } catch (error) { + process.stderr.write(`seat-pusher: ${error.message}\n`) + process.exit(1) + } +} + +const listMeters = () => { + if (metersDir === null) return [] + try { + return readdirSync(metersDir) + } catch (error) { + process.stderr.write(`seat-pusher: cannot list ${metersDir}: ${error.code ?? error.name}\n`) + return [] + } +} + +const pass = async () => { + const result = await runOnce({ + metersDir, + capacityDir, + statePath, + config, + base: args.url ?? "", + token, + list: listMeters(), + policy, + force: args.force, + dryRun, + instance: args.instance ?? "", + allowNetwork: !args["no-network"], + preDispatch: args["pre-dispatch"], + ...(args["usage-dir"] === undefined ? {} : { usageDir: args["usage-dir"] }), + ...(args["demand-dir"] === undefined ? {} : { demandDir: args["demand-dir"] }) + }) + process.stdout.write(`${JSON.stringify(result)}\n`) +} + +if (watch === null) { + try { + await pass() + } catch (error) { + process.stderr.write(`seat-pusher: ${error.message}\n`) + process.exit(1) + } +} else { + for (;;) { + try { + await pass() + } catch (error) { + process.stderr.write(`seat-pusher: ${error.message}\n`) + } + await new Promise((resolve) => setTimeout(resolve, watch * 1000)) + } +} diff --git a/pkgs/substrate-apps/src/apps/pusher/bin/substrate-pusher.mjs b/pkgs/substrate-apps/src/apps/pusher/bin/substrate-pusher.mjs new file mode 100755 index 00000000..340379f7 --- /dev/null +++ b/pkgs/substrate-apps/src/apps/pusher/bin/substrate-pusher.mjs @@ -0,0 +1,213 @@ +#!/usr/bin/env node +// apps/pusher/bin/substrate-pusher.mjs: the gentle capacity pusher, a plain +// long-lived process. It asks the box's capacity oracle (`seats --json`, +// seat-capacity/1) under the repo's refresh policy, converts the answer to +// seat-capacity/2 and posts it to the floor's POST /capacity/snapshots. It +// never calls a provider's usage endpoint itself (src/seatsOracle.mjs, +// src/gentle.mjs). +// +// substrate-pusher [--config PATH] [--url URL] [--token-file PATH] [options] +// substrate-pusher --dry-run --once [--seats-json FILE] +// +// Options (each overrides the config file; see apps/pusher/pusher.example.json): +// --config PATH default ~/.config/substrate/pusher.json (optional) +// --url URL the floor's base URL (config floorUrl) +// --token-file PATH the floor's bearer, read from a file, never printed +// (config tokenFile; default ~/.local/state/substrate/pusher-token) +// --seats-bin PATH the oracle (config seatsBin; default ~/.local/bin/seats) +// --seats-json FILE read the oracle's answer from a file instead of running it +// --peer-cache-dir D the oracle's peer cache (config peerCacheDir; default +// ~/.local/state/substrate/seats-peer-cache; "inherit" keeps +// the oracle's own default) +// --state PATH default ~/.local/state/substrate/pusher/gentle-state.json +// --pidfile PATH default ~/.local/state/substrate/pusher.pid +// --demand-dir DIR default ~/.local/state/substrate/demand; a file touched here marks the box active +// --once one tick, then exit (the pidfile is still taken and released) +// --dry-run read and convert, print the snapshot, post nothing, keep no state +// +// Signals: SIGUSR1 asks for a pre-dispatch read (still never within 120 s of the +// last one); SIGTERM and SIGINT stop the loop and remove the pidfile. +// Exit: 0 stopped cleanly or --once done; 1 a failed --once tick; 2 usage; +// 3 another pusher holds the pidfile. + +import { execFile } from "node:child_process" +import { closeSync, existsSync, mkdirSync, openSync, readFileSync, readdirSync, renameSync, statSync, unlinkSync, writeFileSync, writeSync } from "node:fs" +import { homedir } from "node:os" +import { dirname, join } from "node:path" +import { parseArgs } from "node:util" + +import { initialGentleState, postSnapshot, tick } from "../src/gentle.mjs" +import { oracleEnv } from "../src/seatsOracle.mjs" +import { tokenFromFile } from "../src/token.mjs" + +const home = homedir() +const expand = (p) => (typeof p !== "string" ? p : p === "~" ? home : p.startsWith("~/") ? join(home, p.slice(2)) : p) +const stateDir = join(home, ".local", "state", "substrate") +const fail = (code, message) => { + process.stderr.write(`substrate-pusher: ${message}\n`) + process.exit(code) +} + +let args +try { + args = parseArgs({ + options: { + config: { type: "string" }, url: { type: "string" }, "token-file": { type: "string" }, + "seats-bin": { type: "string" }, "seats-json": { type: "string" }, state: { type: "string" }, + pidfile: { type: "string" }, "demand-dir": { type: "string" }, "peer-cache-dir": { type: "string" }, + once: { type: "boolean", default: false }, "dry-run": { type: "boolean", default: false } + }, + strict: true + }).values +} catch (error) { + fail(2, error.message) +} + +const configPath = expand(args.config ?? join(home, ".config", "substrate", "pusher.json")) +let config = {} +if (existsSync(configPath)) { + try { + config = JSON.parse(readFileSync(configPath, "utf8")) + } catch (error) { + fail(2, `cannot parse the pusher config ${configPath}: ${error.name}`) + } +} else if (args.config !== undefined) { + fail(2, `no pusher config at ${configPath}`) +} + +const dryRun = args["dry-run"] +const url = args.url ?? config.floorUrl +const tokenFile = expand(args["token-file"] ?? config.tokenFile ?? join(stateDir, "pusher-token")) +const seatsBin = expand(args["seats-bin"] ?? config.seatsBin ?? join(home, ".local", "bin", "seats")) +let seatsEnv +try { + const peer = args["peer-cache-dir"] ?? config.peerCacheDir + seatsEnv = oracleEnv({ ...config, peerCacheDir: peer === undefined || peer === "inherit" ? peer : expand(peer) }, stateDir, process.env) +} catch (error) { + fail(2, error.message) +} +const seatsJson = args["seats-json"] === undefined ? null : expand(args["seats-json"]) +const statePath = expand(args.state ?? join(stateDir, "pusher", "gentle-state.json")) +const pidfile = expand(args.pidfile ?? join(stateDir, "pusher.pid")) +const demandDir = expand(args["demand-dir"] ?? join(stateDir, "demand")) +if (!dryRun && (typeof url !== "string" || !/^https?:\/\/[^/]/.test(url))) fail(2, "--url (or floorUrl in the config) must be http(s)://host") + +let token = "" +if (!dryRun) { + try { + token = tokenFromFile(tokenFile) + } catch (error) { + fail(1, error.message) + } +} + +// ---- the pidfile: one pusher per box. A pidfile whose process is gone, or is not a pusher, is stale and replaced. +const alivePusher = (pid) => { + try { + process.kill(pid, 0) + } catch { + return false + } + try { + return readFileSync(`/proc/${pid}/cmdline`, "utf8").includes("substrate-pusher") + } catch { + return true + } +} +const takePidfile = () => { + mkdirSync(dirname(pidfile), { recursive: true, mode: 0o700 }) + for (let tries = 0; tries < 2; tries++) { + try { + const fd = openSync(pidfile, "wx", 0o600) + writeSync(fd, `${process.pid}\n`) + closeSync(fd) + return + } catch (error) { + if (error.code !== "EEXIST") fail(1, `cannot write the pidfile: ${error.code}`) + const held = Number.parseInt(readFileSync(pidfile, "utf8").trim(), 10) + if (Number.isInteger(held) && held !== process.pid && alivePusher(held)) fail(3, `another pusher (pid ${held}) holds ${pidfile}`) + try { unlinkSync(pidfile) } catch { /* raced with its owner's exit */ } + } + } + fail(3, `cannot take ${pidfile}`) +} +const releasePidfile = () => { + try { + if (readFileSync(pidfile, "utf8").trim() === String(process.pid)) unlinkSync(pidfile) + } catch { /* already gone */ } +} + +// ---- state, 0600 by atomic rename +const readState = () => { + try { + return { ...initialGentleState(), ...JSON.parse(readFileSync(statePath, "utf8")) } + } catch { + return initialGentleState() + } +} +const writeState = (state) => { + mkdirSync(dirname(statePath), { recursive: true, mode: 0o700 }) + const temporary = `${statePath}.tmp-${process.pid}` + writeFileSync(temporary, `${JSON.stringify(state)}\n`, { mode: 0o600 }) + renameSync(temporary, statePath) +} + +const newestDemandMs = () => { + try { + const times = readdirSync(demandDir).map((name) => statSync(join(demandDir, name)).mtimeMs) + return times.length === 0 ? null : Math.max(...times) + } catch { + return null + } +} + +// ---- the oracle: `seats --json --no-spend` (no transcript scan); its own cache and 429 handling stand. +const runOracle = () => + seatsJson !== null + ? Promise.resolve(JSON.parse(readFileSync(seatsJson, "utf8"))) + : new Promise((resolve, reject) => { + execFile(seatsBin, ["--json", "--no-spend"], { timeout: 90_000, maxBuffer: 16 * 1024 * 1024, env: seatsEnv }, (error, stdout) => { + if (error) return reject(new Error(`the oracle failed: ${error.code ?? error.signal ?? error.name}`)) + try { + resolve(JSON.parse(stdout)) + } catch { + reject(new Error("the oracle answered no JSON")) + } + }) + }) + +const post = (snapshot) => postSnapshot({ base: url, token, snapshot }) + +let stopping = false +let preDispatch = false +let wake = null +const sleep = (ms) => new Promise((resolve) => { const t = setTimeout(resolve, ms); wake = () => { clearTimeout(t); resolve() } }) +const stop = () => { stopping = true; wake?.() } +process.on("SIGTERM", stop) +process.on("SIGINT", stop) +process.on("SIGUSR1", () => { preDispatch = true; wake?.() }) +process.on("exit", releasePidfile) + +takePidfile() +let state = dryRun ? initialGentleState() : readState() +let failedOnce = false +for (;;) { + const nowMs = Date.now() + const asked = preDispatch + preDispatch = false + const result = await tick(state, { nowMs, demandAtMs: newestDemandMs(), preDispatch: asked, config, runOracle, post, dryRun }) + state = result.state + for (const event of result.events) { + if (event.event === "wait" && !args.once) continue + process.stdout.write(`${JSON.stringify(event)}\n`) + if (event.event === "read-failed" || event.event === "push-failed") failedOnce = true + } + if (!dryRun) writeState(state) + if (args.once || stopping) break + // Wake at the next due instant, at most every 30 s (a demand marker or a signal can make a read due sooner). + const next = result.events.map((event) => (event.event === "wait" && event.next_at ? Date.parse(event.next_at) : Number.NaN)).find(Number.isFinite) + await sleep(Math.max(1_000, Math.min(30_000, Number.isFinite(next) ? next - Date.now() : 30_000))) + if (stopping) break +} +releasePidfile() +process.exit(args.once && failedOnce ? 1 : 0) diff --git a/pkgs/substrate-apps/src/apps/pusher/package.json b/pkgs/substrate-apps/src/apps/pusher/package.json new file mode 100644 index 00000000..863db745 --- /dev/null +++ b/pkgs/substrate-apps/src/apps/pusher/package.json @@ -0,0 +1,28 @@ +{ + "name": "@substrate/pusher", + "private": true, + "type": "module", + "version": "0.1.0", + "description": "The box-side gentle capacity pusher: reads the box's capacity oracle (seats --json, seat-capacity/1) under the refresh policy, converts to seat-capacity/2 and posts to the floor's POST /capacity/snapshots. The legacy seat-pusher (meters and direct usage reads) is kept for its tests only.", + "license": "UNLICENSED", + "bin": { + "substrate-pusher": "./bin/substrate-pusher.mjs" + }, + "exports": { + ".": "./src/seatCapacity.mjs", + "./*": "./src/*" + }, + "scripts": { + "test": "vitest run" + }, + "peerDependencies": { + "effect": "4.0.0-rc.112" + }, + "devDependencies": { + "fast-check": "^4.9.0", + "@substrate/factory": "workspace:*", + "@substrate/planning": "workspace:*", + "effect": "4.0.0-rc.112", + "vitest": "^3.2.4" + } +} diff --git a/pkgs/substrate-apps/src/apps/pusher/pusher.example.json b/pkgs/substrate-apps/src/apps/pusher/pusher.example.json new file mode 100644 index 00000000..a8429c12 --- /dev/null +++ b/pkgs/substrate-apps/src/apps/pusher/pusher.example.json @@ -0,0 +1,29 @@ +{ + "comment": "Example gentle-pusher config. Install as ~/.config/substrate/pusher.json (a dotfiles module will own it). Values here are placeholders, not a deployment.", + "floorUrl": "https://substrate.example.dev", + "tokenFile": "~/.local/state/substrate/pusher-token", + "seatsBin": "~/.local/bin/seats", + "peerCacheDir": "~/.local/state/substrate/seats-peer-cache", + "host": "coordinator", + "seats": [ + "cc", + "cc2", + "codex", + "pi-qwencloud", + "halogen" + ], + "seatIds": { + "gpu-worker": "halogen" + }, + "owners": {}, + "not_dispatchable": { + "cc3": "evicted" + }, + "plans": {}, + "slots": { + "halogen": 1 + }, + "estimatedProviders": [ + "qwen" + ] +} diff --git a/pkgs/substrate-apps/src/apps/pusher/seat-capacity.example.json b/pkgs/substrate-apps/src/apps/pusher/seat-capacity.example.json new file mode 100644 index 00000000..7b536559 --- /dev/null +++ b/pkgs/substrate-apps/src/apps/pusher/seat-capacity.example.json @@ -0,0 +1,10 @@ +{ + "host": "coordinator", + "not_dispatchable": { + "cc3": "evicted", + "gpu-coordinator": "halogen is declared but not resident on the coordinator" + }, + "plans": { + "pi-qwencloud": "2026-11-07T00:00:00Z" + } +} diff --git a/pkgs/substrate-apps/src/apps/pusher/seat-capacity.usage.example.json b/pkgs/substrate-apps/src/apps/pusher/seat-capacity.usage.example.json new file mode 100644 index 00000000..a45f37d1 --- /dev/null +++ b/pkgs/substrate-apps/src/apps/pusher/seat-capacity.usage.example.json @@ -0,0 +1,9 @@ +{ + "host": "coordinator", + "usage": { + "cc": { "kind": "oauth", "credentials": "/.credentials.json", "config_dir": "" }, + "cc2": { "kind": "cache-file", "path": "" } + }, + "not_dispatchable": { "cc3": "evicted" }, + "plans": { "pi-qwencloud": "2026-11-07T00:00:00Z" } +} diff --git a/pkgs/substrate-apps/src/apps/pusher/src/gentle.mjs b/pkgs/substrate-apps/src/apps/pusher/src/gentle.mjs new file mode 100644 index 00000000..dfd3e0c5 --- /dev/null +++ b/pkgs/substrate-apps/src/apps/pusher/src/gentle.mjs @@ -0,0 +1,145 @@ +// apps/pusher/src/gentle.mjs: the gentle pusher's loop, one tick at a time. +// Pure decisions plus one `tick` that takes every effect as a parameter +// (run the oracle, post, read the clock), so the tests drive it on a fake clock. +// +// WHEN THE ORACLE IS ASKED (refreshPolicy.mjs, the policy already in the repo): +// never twice within 120 s; every 900 s when idle; every 300 s when active; +// every 180 s when active and a binding window is at or above 85 percent; +// once at each reset + 90 s; a pre-dispatch ask (SIGUSR1) when the reading is +// older than 300 s. "Active" means a demand marker younger than 300 s, or the +// last two readings differed in any utilization (someone is spending). +// WHEN THE FLOOR IS TOLD (seatCapacity.mjs decidePush): after a read, when a +// seat's reading changed, or as a heartbeat (1800 s; 600 s while a Halogen +// slot row is published); never twice within 60 s. A failed post is retried +// on the next tick after 60 s with the same snapshot, without a new read. +// A failed oracle run (non-zero exit, timeout, not seat-capacity/1) counts as +// an attempt: the 120 s spacing holds, nothing is posted, and the floor's copy +// ages to STALE on its own, which refuses admission (fail closed). + +import { REFRESH_POLICY, afterAttempt, initialRefreshState, refreshDue } from "./refreshPolicy.mjs" +import { DEFAULT_POLICY, decidePush, stateAfterPush } from "./seatCapacity.mjs" +import { snapshotFromSeatsV1 } from "./seatsOracle.mjs" + +export const PUSH_RETRY_SECONDS = 60 + +export const initialGentleState = () => ({ refresh: initialRefreshState(), push: {}, last: null, previous_utilization: null, pending: null }) + +/** Utilization per (seat, window) of a snapshot, for "is anyone spending". */ +export const utilizationKey = (snapshot) => + snapshot === null + ? null + : JSON.stringify( + snapshot.seats.map((seat) => [seat.seat, seat.windows.map((w) => [w.kind, w.model, w.utilization_pct])]) + ) + +/** The highest utilization over binding seat-wide windows of dispatchable seats; null when none is known. */ +export const bindingMaxPct = (snapshot) => { + if (snapshot === null) return null + const known = snapshot.seats + .filter((seat) => seat.dispatchable) + .flatMap((seat) => seat.windows) + .filter((w) => w.binding && w.kind !== "model_scoped" && typeof w.utilization_pct === "number") + .map((w) => w.utilization_pct) + return known.length === 0 ? null : Math.max(...known) +} + +/** Every reset instant the last snapshot states. */ +export const resetsOf = (snapshot) => + snapshot === null ? [] : snapshot.seats.flatMap((seat) => seat.windows.map((w) => w.resets_at)).filter((r) => r !== null) + +/** + * Whether the oracle is due now. + * @param input `{ nowMs, demandAtMs, preDispatch }`; demandAtMs is the newest demand marker's mtime or null. + */ +export const planRead = (state, input, policy = REFRESH_POLICY) => { + const s = state ?? initialGentleState() + const demand = typeof input.demandAtMs === "number" && input.nowMs - input.demandAtMs < policy.activeSeconds * 1000 + const moving = s.previous_utilization !== null && s.last !== null && utilizationKey(s.last) !== s.previous_utilization + return refreshDue(s.refresh, { + nowMs: input.nowMs, + active: demand || moving, + preDispatch: input.preDispatch === true, + bindingMaxPct: bindingMaxPct(s.last), + resets: resetsOf(s.last) + }, policy) +} + +/** + * One tick. Effects: + * runOracle() -> Promise the parsed `seats --json` answer (throws on failure) + * post(snapshot) -> Promise the floor's answer (throws on failure) + * @returns `{ state, events }`; events are log lines with no secret in them. + */ +export const tick = async (state, { nowMs, demandAtMs = null, preDispatch = false, config, runOracle, post, dryRun = false, pushPolicy = DEFAULT_POLICY, refreshPolicy = REFRESH_POLICY }) => { + let s = { ...initialGentleState(), ...(state ?? {}) } + const now = new Date(nowMs).toISOString() + const events = [] + // A post that failed is retried with the same snapshot before any new read. + if (s.pending !== null && !dryRun) { + if (nowMs < Date.parse(s.pending.retry_at)) return { state: s, events: [{ at: now, event: "wait", reason: "push-retry", next_at: s.pending.retry_at }] } + try { + await post(s.pending.snapshot) + s = { ...s, push: stateAfterPush(s.pending.snapshot, now), pending: null } + events.push({ at: now, event: "pushed", reason: "retry", seats: s.last?.seats.map((seat) => seat.seat) ?? [] }) + } catch (error) { + s = { ...s, pending: { ...s.pending, retry_at: new Date(nowMs + PUSH_RETRY_SECONDS * 1000).toISOString() } } + return { state: s, events: [{ at: now, event: "push-failed", error: String(error?.message ?? error).slice(0, 200) }] } + } + } + const due = planRead(s, { nowMs, demandAtMs, preDispatch }, refreshPolicy) + if (!due.due) return { state: s, events: [...events, { at: now, event: "wait", reason: due.reason, next_at: due.next_at }] } + let built + try { + const doc = await runOracle() + built = snapshotFromSeatsV1(doc, config, now) + } catch (error) { + s = { ...s, refresh: afterAttempt(s.refresh, { kind: "error" }, nowMs, null, refreshPolicy) } + return { state: s, events: [...events, { at: now, event: "read-failed", reason: due.reason, error: String(error?.message ?? error).slice(0, 200) }] } + } + s = { + ...s, + refresh: afterAttempt(s.refresh, { kind: "success" }, nowMs, null, refreshPolicy), + previous_utilization: utilizationKey(s.last), + last: built.snapshot + } + events.push({ at: now, event: "read", reason: due.reason, seats: built.snapshot.seats.map((seat) => seat.seat), skipped: built.skipped }) + const decision = decidePush(s.push, built.snapshot, nowMs, pushPolicy) + if (dryRun) return { state: s, events: [...events, { at: now, event: "dry-run", decision, snapshot: built.snapshot }] } + if (!decision.push) return { state: s, events: [...events, { at: now, event: "not-pushed", reason: decision.reason }] } + try { + const answer = await post(built.snapshot) + s = { ...s, push: stateAfterPush(built.snapshot, now) } + const results = answer?.report?.results ?? answer?.results ?? null + events.push({ at: now, event: "pushed", reason: decision.reason, changed: decision.changed, results }) + } catch (error) { + s = { ...s, pending: { snapshot: built.snapshot, retry_at: new Date(nowMs + PUSH_RETRY_SECONDS * 1000).toISOString() } } + events.push({ at: now, event: "push-failed", error: String(error?.message ?? error).slice(0, 200) }) + } + return { state: s, events } +} + +/** + * Posts a snapshot to `/capacity/snapshots` with the bearer. The token is + * sent and never echoed: an error names the status, not the header. + */ +export const postSnapshot = async ({ base, token, snapshot, fetchImpl = fetch }) => { + const url = `${base.replace(/\/+$/, "")}/capacity/snapshots` + let response + try { + response = await fetchImpl(url, { + method: "POST", + headers: { authorization: `Bearer ${token}`, "content-type": "application/json" }, + body: JSON.stringify(snapshot), + signal: AbortSignal.timeout(20_000) + }) + } catch (error) { + throw new Error(`POST /capacity/snapshots did not complete: ${error?.name ?? "Error"}`) + } + const body = await response.text() + if (response.status !== 200) throw new Error(`POST /capacity/snapshots answered ${response.status}: ${body.slice(0, 200)}`) + try { + return JSON.parse(body) + } catch { + throw new Error("POST /capacity/snapshots answered 200 with a body that is not JSON") + } +} diff --git a/pkgs/substrate-apps/src/apps/pusher/src/refreshPolicy.mjs b/pkgs/substrate-apps/src/apps/pusher/src/refreshPolicy.mjs new file mode 100644 index 00000000..f5f2fab6 --- /dev/null +++ b/pkgs/substrate-apps/src/apps/pusher/src/refreshPolicy.mjs @@ -0,0 +1,113 @@ +// apps/pusher/src/refreshPolicy.mjs: when a direct usage read is worth a +// network call. Pure: every instant is a parameter, nothing reads a clock. +// +// The values are the capacity scout's refresh policy (evals-2026-09-23 +// capacity/SCOUT.md section 5.5), one enforcer per credential: +// +// hard minimum between network calls 120 s (sustained success spacing was +// MEASURED at 123 s; faster is 429s) +// idle floor 900 s +// active cadence 300 s +// active and a binding window >= 85 % 180 s +// pre-dispatch read if the reading is older than 300 s +// reset boundary one read at resets_at + 90 s +// 429 one attempt, no in-tick retry; next +// allowed = now + max(Retry-After, +// min(120 s x 2^k, 960 s)); k resets on success +// auth failure no call until the credentials file's +// mtime changes + +export const REFRESH_POLICY = Object.freeze({ + minSpacingSeconds: 120, + idleSeconds: 900, + activeSeconds: 300, + nearCapSeconds: 180, + nearCapPct: 85, + preDispatchAgeSeconds: 300, + resetGraceSeconds: 90, + backoffBaseSeconds: 120, + backoffMaxSeconds: 960 +}) + +/** The state a fresh credential starts from. */ +export const initialRefreshState = () => ({ + last_attempt_at: null, + last_success_at: null, + backoff_k: 0, + next_allowed_at: null, + auth_failed_mtime_ms: null +}) + +const ms = (iso) => { + if (typeof iso !== "string") return null + const value = Date.parse(iso) + return Number.isFinite(value) ? value : null +} + +const iso = (value) => new Date(value).toISOString() + +/** + * Decides whether a network read is due. + * + * @param state the credential's refresh state (initialRefreshState() shape). + * @param input `{ nowMs, active, preDispatch, bindingMaxPct, resets, credentialsMtimeMs }`: + * `resets` are the binding windows' `resets_at` instants from the last reading; + * `bindingMaxPct` the highest binding utilization, or null when unknown. + * @returns `{ due, reason, next_at }`, `next_at` an ISO instant or null. + */ +export const refreshDue = (state, input, policy = REFRESH_POLICY) => { + const s = { ...initialRefreshState(), ...(state ?? {}) } + const now = input.nowMs + if (s.auth_failed_mtime_ms !== null && input.credentialsMtimeMs === s.auth_failed_mtime_ms) { + return { due: false, reason: "auth-failed", next_at: null } + } + const allowed = ms(s.next_allowed_at) + if (allowed !== null && now < allowed) return { due: false, reason: "backoff", next_at: iso(allowed) } + const attempted = ms(s.last_attempt_at) + const earliest = attempted === null ? null : attempted + policy.minSpacingSeconds * 1000 + if (earliest !== null && now < earliest) return { due: false, reason: "min-spacing", next_at: iso(earliest) } + const succeeded = ms(s.last_success_at) + if (succeeded === null) return { due: true, reason: "first-read", next_at: null } + // A reset boundary read turns a PROJECTED window into a MEASURED one. + const boundaries = (input.resets ?? []) + .map(ms) + .filter((value) => value !== null) + .map((value) => value + policy.resetGraceSeconds * 1000) + if (boundaries.some((at) => at > succeeded && at <= now)) { + return { due: true, reason: "reset-boundary", next_at: null } + } + const ageSeconds = (now - succeeded) / 1000 + if (input.preDispatch === true && ageSeconds > policy.preDispatchAgeSeconds) { + return { due: true, reason: "pre-dispatch", next_at: null } + } + const nearCap = typeof input.bindingMaxPct === "number" && input.bindingMaxPct >= policy.nearCapPct + const cadence = input.active === true ? (nearCap ? policy.nearCapSeconds : policy.activeSeconds) : policy.idleSeconds + if (ageSeconds >= cadence) return { due: true, reason: input.active === true ? "active" : "idle", next_at: null } + const candidates = [succeeded + cadence * 1000, ...boundaries.filter((at) => at > now)] + const next = Math.max(Math.min(...candidates), earliest ?? 0) + return { due: false, reason: "fresh", next_at: iso(next) } +} + +/** + * The state after one attempt. + * + * @param outcome `{ kind: "success" }`, `{ kind: "rate-limited", retryAfterSeconds }`, + * `{ kind: "auth-failed" }` or `{ kind: "error" }`. + */ +export const afterAttempt = (state, outcome, nowMs, credentialsMtimeMs, policy = REFRESH_POLICY) => { + const s = { ...initialRefreshState(), ...(state ?? {}), last_attempt_at: iso(nowMs) } + if (outcome.kind === "success") { + return { ...s, last_success_at: iso(nowMs), backoff_k: 0, next_allowed_at: null, auth_failed_mtime_ms: null } + } + if (outcome.kind === "rate-limited") { + const k = Number.isInteger(s.backoff_k) && s.backoff_k >= 0 ? s.backoff_k : 0 + const exponential = Math.min(policy.backoffBaseSeconds * 2 ** k, policy.backoffMaxSeconds) + const retryAfter = + typeof outcome.retryAfterSeconds === "number" && Number.isFinite(outcome.retryAfterSeconds) + ? outcome.retryAfterSeconds + : 0 + return { ...s, backoff_k: k + 1, next_allowed_at: iso(nowMs + Math.max(retryAfter, exponential) * 1000) } + } + if (outcome.kind === "auth-failed") return { ...s, auth_failed_mtime_ms: credentialsMtimeMs ?? -1 } + return s +} diff --git a/pkgs/substrate-apps/src/apps/pusher/src/seatCapacity.mjs b/pkgs/substrate-apps/src/apps/pusher/src/seatCapacity.mjs new file mode 100644 index 00000000..2d7862a8 --- /dev/null +++ b/pkgs/substrate-apps/src/apps/pusher/src/seatCapacity.mjs @@ -0,0 +1,593 @@ +// apps/uplink/src/seatCapacity.mjs: EVAL-CAPACITY. The seat-capacity pusher's +// library: build a `seat-capacity/2` snapshot from the feeders' files, decide +// whether it is worth sending, and send it outbound. +// +// THREE CLOCKS, KEPT APART (SCOUT.md, evals-2026-09-23/capacity). The feeder +// restamps meter rows every 30 s for the kernel; the provider is read far less +// often; the lake projects from the last reading. This module sits between the +// first two and the third: it reads LOCAL FILES ONLY and never calls a +// provider, so running it often costs nothing upstream. What it carries is the +// instant the PROVIDER answered (`observed_at`), never the restamp. +// +// SOURCES, IN ORDER OF AUTHORITY: +// 1. `/seats.json`, when the feeder writes the snapshot itself +// (the proposed feeder change, not yet applied): carried as it is. +// 2. The meters directory: `.json` rows, and for a Claude seat the raw +// usage cache `.window-cache-.json` beside it, which is the only +// place the model-scoped rows (`weekly_scoped`, e.g. Fable) appear. Only +// the `limits[]` rows and `observed_at` are read from the cache; nothing +// else in it is copied anywhere. +// +// WHAT IT REFUSES TO INVENT. A cell the file does not state stays null +// (UNKNOWN, never headroom); a meter this module cannot classify is skipped +// with a named reason rather than guessed into a provider. + +import { existsSync, mkdirSync, readFileSync, renameSync, writeFileSync } from "node:fs" +import { homedir, hostname } from "node:os" +import { dirname, join } from "node:path" + +import { limitsComplete, readSeatUsage, windowsFromLimits, windowsFromUsage } from "./usageSource.mjs" + +const SCHEMA_VERSION = "seat-capacity/2" + +const MINUTES = { five_hour: 300, seven_day: 10080, model_scoped: 10080 } + +/** The pusher's own state; nothing under a tally path is a default any more. */ +export const defaultPusherDir = () => join(homedir(), ".local", "state", "substrate", "pusher") +export const defaultConfigPath = () => join(homedir(), ".config", "substrate", "seat-capacity.json") + +/** + * Refresh policy defaults, in seconds. + * + * `slotHeartbeatSeconds` is the beat for a dispatchable Halogen slot row. Its + * `observed_at` is the kernel's own observation, and the lake grades a reading + * STALE after 1200 s (MEASURED_MAX_AGE_S in packages/planning), so a slot row + * re-sent only every 1800 s would be refused for half of every beat. + */ +export const DEFAULT_POLICY = { heartbeatSeconds: 1800, slotHeartbeatSeconds: 600, minIntervalSeconds: 60 } + +/** + * Seats Tom has ruled out of dispatch. Built in, so a missing or partial config + * can never publish them as dispatchable; a config may add rulings and reword + * these, never lift them. cc3: evicted. gpu-coordinator: Halogen is declared on + * the coordinator but never resident there (ruling in ~/today/CLAUDE.md). + */ +const RULED_NOT_DISPATCHABLE = Object.freeze({ + cc3: "evicted", + "gpu-coordinator": "halogen is declared but not resident on the coordinator" +}) + +const ZONED = /^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}(:\d{2}(\.\d+)?)?(Z|[+-]\d{2}:\d{2})$/ + +/** An instant the lake's schema will accept, or null. */ +const instant = (value) => + typeof value === "string" && ZONED.test(value) && Number.isFinite(Date.parse(value)) ? value : null + +const percent = (value) => (typeof value === "number" && Number.isFinite(value) && value >= 0 ? value : null) + +const readJson = (path) => { + if (!existsSync(path)) return { missing: true } + try { + return { value: JSON.parse(readFileSync(path, "utf8")) } + } catch (error) { + return { error: `${error.name}` } + } +} + +/** + * Reads the pusher's config; a missing file is the defaults, a broken one is an + * error. The built-in rulings always apply (`RULED_NOT_DISPATCHABLE`): a config + * entry that is not a reason string is ignored, so it cannot lift one. + */ +export const readConfig = (path) => { + const read = path === null ? { missing: true } : readJson(path) + if (read.error) throw new Error(`cannot parse the seat-capacity config ${path}: ${read.error}`) + const stated = read.value ?? {} + const statedRulings = + stated.not_dispatchable !== null && typeof stated.not_dispatchable === "object" + ? Object.fromEntries( + Object.entries(stated.not_dispatchable).filter(([, reason]) => typeof reason === "string" && reason !== "") + ) + : {} + return { + present: read.value !== undefined, + host: typeof stated.host === "string" ? stated.host : null, + seats: Array.isArray(stated.seats) ? stated.seats.filter((seat) => typeof seat === "string") : null, + notDispatchable: { ...RULED_NOT_DISPATCHABLE, ...statedRulings }, + plans: stated.plans !== null && typeof stated.plans === "object" ? stated.plans : {}, + // seat -> { kind: "oauth", credentials, token_pointer?, config_dir? } | { kind: "cache-file", path } + usage: stated.usage !== null && typeof stated.usage === "object" ? stated.usage : {}, + owners: stated.owners !== null && typeof stated.owners === "object" ? stated.owners : {}, + providers: stated.providers !== null && typeof stated.providers === "object" ? stated.providers : {} + } +} + +/** + * A reason as it leaves the box: local absolute paths are replaced by ``. + * The lake is on the public internet (behind a bearer); where a file lives on + * the coordinator is none of its business. + */ +const scrubReason = (value) => + typeof value === "string" ? value.replace(/(?:~|\/(?:home|root|Users|nix|run|tmp|var|etc))\/[^\s,;:'"`)]+/g, "") : null + +const AUTH_FAILURE = /token expired|unauthori[sz]ed|\b401\b|invalid[_ ]grant|credentials/i + +const dispatchOf = (seat, owner, staleReason, config) => { + const ruled = config.notDispatchable[seat] + if (typeof ruled === "string") return { dispatchable: false, dispatchable_reason: ruled } + if (owner !== "tom") return { dispatchable: false, dispatchable_reason: "third-party" } + if (typeof staleReason === "string" && AUTH_FAILURE.test(staleReason)) { + return { dispatchable: false, dispatchable_reason: "auth-failed" } + } + return { dispatchable: true, dispatchable_reason: null } +} + +const planOf = (seat, config) => { + const expires = instant(config.plans[seat]) + return expires === null ? null : { expires_at: expires } +} + +const later = (left, right) => { + if (left === null) return right + if (right === null) return left + return Date.parse(right) > Date.parse(left) ? right : left +} + +/** The pusher's one limits[] mapping (usageSource.mjs windowsFromLimits): an unplaceable row is a binding UNKNOWN seven_day window. */ +const claudeWindowsFromLimits = (limits) => windowsFromLimits(limits).windows + +const claudeWindowsFromMeter = (window) => { + if (window === null || typeof window !== "object") return [] + return [ + ["five_hour", window.primary], + ["seven_day", window.secondary] + ].flatMap(([kind, entry]) => + entry === null || typeof entry !== "object" + ? [] + : [ + { + kind, + model: null, + binding: true, + minutes: MINUTES[kind], + utilization_pct: percent(entry.utilization_pct), + resets_at: instant(entry.resets_at), + severity: null, + grade: "MEASURED" + } + ] + ) +} + +/** + * The model-scoped rows a cache last showed, carried forward as UNKNOWN when the + * reading itself comes from the meter row: the model limit exists and was not + * read with this reading, so its utilization and severity are not known. The + * reset instant is kept (a passed one projects forward at the lake). The lake + * refuses a job for that model `window-unknown`, never admits it as headroom. + */ +const carriedModelWindows = (limits) => + Array.isArray(limits) + ? claudeWindowsFromLimits(limits) + // An unplaceable row (UNKNOWN seven_day) is carried too: dropping it read as headroom. + .filter((window) => window.kind === "model_scoped" || window.grade === "UNKNOWN") + .map((window) => ({ ...window, utilization_pct: null, severity: null, grade: "UNKNOWN" })) + : [] + +const claudeSeat = (seat, meter, cache, config) => { + const meterObserved = later(instant(meter.reading_observed_at), instant(meter.source?.source_observed_at)) + const cacheObserved = instant(cache?.observed_at) + const observed = later(meterObserved, cacheObserved) + if (observed === null) return { skip: "the meter states no reading instant" } + const limits = cache?.usage?.limits + // The cache's limits[] are the reading only when the cache IS the latest read. + // Otherwise (cache missing, torn mid-write, or behind the meter row) the + // meter row gives the seat-wide windows and the model limits stay unread. + const fromCache = + Array.isArray(limits) && cacheObserved !== null && Date.parse(cacheObserved) === Date.parse(observed) + const graded = meter.grade === "MEASURED" || meter.grade === "STALE-MEASURED" + const staleReason = scrubReason(meter.stale_reason) + return { + seat: { + seat, + provider: "claude", + owner: meter.owner === "tom" ? "tom" : "third-party", + ...dispatchOf(seat, meter.owner, staleReason, config), + plan: planOf(seat, config), + slots: null, + windows: fromCache + ? claudeWindowsFromLimits(limits) + : [...claudeWindowsFromMeter(meter.window), ...carriedModelWindows(limits)], + observed_at: observed, + source: { + kind: "oauth-usage-endpoint", + detail: fromCache ? "usage cache limits[]" : "meter row window" + }, + // The reading's age is the lake's to judge from observed_at: a row the + // feeder marked STALE-MEASURED is a MEASURED reading that has aged. + grade: graded ? "MEASURED" : "UNKNOWN", + stale_reason: staleReason, + // Only limits[] read with this very reading, every row placed, lists every model limit. + model_windows_complete: fromCache && windowsFromLimits(limits).unmapped === 0 + } + } +} + +const codexSeat = (seat, meter, config) => { + const observed = instant(meter.observed_at) + if (observed === null) return { skip: "the meter states no observed_at" } + const window = meter.window + const windows = + window !== null && typeof window === "object" && window.minutes === 10080 + ? [ + { + kind: "seven_day", + model: null, + binding: true, + minutes: 10080, + utilization_pct: percent(window.utilization_pct), + resets_at: instant(window.resets_at), + severity: null, + grade: "ESTIMATED" + } + ] + : [] + return { + seat: { + seat, + provider: "codex", + owner: meter.owner === "tom" ? "tom" : "third-party", + ...dispatchOf(seat, meter.owner, null, config), + plan: planOf(seat, config), + slots: null, + windows, + observed_at: observed, + // The meter row states no provider or rollout-event instant; observed_at + // is the feeder's read of a local rollout. Said so, rather than implied. + source: { kind: "codex-rollout-rate-limits", detail: "observed_at is the feeder's read instant" }, + // Read from a local rollout, not from the provider at a known instant. + grade: "ESTIMATED", + stale_reason: null + } + } +} + +const qwenSeat = (seat, meter, config) => { + const observed = instant(meter.observed_at) + if (observed === null) return { skip: "the meter states no observed_at" } + const window = meter.window + const windows = + window !== null && typeof window === "object" && instant(window.resets_at) !== null + ? [ + { + kind: "seven_day", + model: null, + binding: true, + minutes: 10080, + utilization_pct: null, + resets_at: instant(window.resets_at), + severity: null, + grade: "UNKNOWN" + } + ] + : [] + return { + seat: { + seat, + provider: "qwen", + owner: meter.owner === "tom" ? "tom" : "third-party", + ...dispatchOf(seat, meter.owner, null, config), + plan: planOf(seat, config), + slots: null, + windows, + observed_at: observed, + source: { kind: "hold-record", detail: "observed_at is the feeder's read instant" }, + grade: "UNKNOWN", + stale_reason: scrubReason(meter.window_reason) + } + } +} + +const halogenSeat = (seat, meter, config) => { + const observed = instant(meter.observed_at) + if (observed === null) return { skip: "the meter states no observed_at" } + const capacity = Number.isInteger(meter.capacity) && meter.capacity >= 0 ? meter.capacity : null + const holders = Number.isInteger(meter.holders) && meter.holders >= 0 ? meter.holders : null + return { + seat: { + seat, + provider: "halogen", + owner: "tom", + ...dispatchOf(seat, "tom", null, config), + plan: planOf(seat, config), + slots: capacity === null || holders === null ? null : { capacity, holders }, + windows: [], + observed_at: observed, + source: { kind: "kernel-row", detail: null }, + grade: meter.running_grade === "MEASURED" ? "MEASURED" : "UNKNOWN", + stale_reason: null + } + } +} + +/** + * A seat read through its own usage source (usageSource.mjs), no meter row + * needed. `read` is `{ reading, error }` from readSeatUsage. + */ +const seatFromUsage = (seat, read, sourceKind, config) => { + const reading = read.reading + if (reading === null) return { skip: read.error ?? "no usage reading yet" } + const owner = config.owners[seat] === "third-party" ? "third-party" : "tom" + const staleReason = scrubReason(read.error) + return { + seat: { + seat, + provider: typeof config.providers[seat] === "string" ? config.providers[seat] : "claude", + owner, + ...dispatchOf(seat, owner, staleReason, config), + plan: planOf(seat, config), + slots: null, + windows: windowsFromUsage(reading.usage), + // The reading lists every model limit only when limits[] is the reading; + // without the claim, admission refuses a model job with no window. + // An unplaceable row withdraws the claim (successor review r4). + model_windows_complete: limitsComplete(reading.usage), + observed_at: reading.observed_at, + source: { + kind: "oauth-usage-endpoint", + detail: sourceKind === "oauth" ? "direct read" : "usage cache file" + }, + grade: "MEASURED", + stale_reason: staleReason + } + } +} + +/** Classifies one meter file into a seat, or a named skip. */ +const seatFromMeter = (seat, meter, cache, config) => { + if (meter === null || typeof meter !== "object") return { skip: "the meter is not an object" } + const kind = meter.source?.kind + if (kind === "oauth-usage-endpoint") return claudeSeat(seat, meter, cache, config) + if (kind === "codex-rollout-rate-limits") return codexSeat(seat, meter, config) + if (typeof meter.source?.hold_record === "string" || seat.startsWith("pi-qwen")) { + return qwenSeat(seat, meter, config) + } + if (seat.startsWith("gpu-") && meter.window?.kind === "none") return halogenSeat(seat, meter, config) + return { skip: "not a compute seat this pusher knows" } +} + +const METER_FILE = /^([a-z0-9][a-z0-9-]{0,62})\.json$/ + +/** + * The feeder's own snapshot, held to the same local rules as one built here: + * the config's seat filter, the rulings (which only ever take dispatch away), + * and the path scrub on every reason that leaves the box. + */ +const governWritten = (snapshot, config) => ({ + ...snapshot, + seats: snapshot.seats + .filter((seat) => config.seats === null || config.seats.includes(seat?.seat)) + .map((seat) => { + if (seat === null || typeof seat !== "object") return seat + const staleReason = scrubReason(seat.stale_reason) + const ruled = dispatchOf(seat.seat, seat.owner, staleReason, config) + const dispatch = + seat.dispatchable === true && ruled.dispatchable + ? { dispatchable: true, dispatchable_reason: null } + : { + dispatchable: false, + dispatchable_reason: ruled.dispatchable + ? scrubReason(seat.dispatchable_reason) + : ruled.dispatchable_reason + } + return { ...seat, ...dispatch, stale_reason: staleReason } + }) +}) + +/** + * Builds the snapshot the lake ingests. + * + * @returns `{ snapshot, skipped }`: skipped names every meter file left out and why. + */ +export const buildSnapshot = ({ metersDir = null, capacityDir = null, config, now, list, usage = {} }) => { + const host = config.host ?? hostname().split(".")[0] + if (capacityDir !== null) { + const written = readJson(join(capacityDir, "seats.json")) + if (written.value?.schema_version === SCHEMA_VERSION && Array.isArray(written.value.seats)) { + return { snapshot: governWritten(written.value, config), skipped: [], source: "seats.json" } + } + } + const wanted = (name) => config.seats === null || config.seats.includes(name) + const seats = [] + const skipped = [] + // Seats with their own usage source come first and win over a meter row. + const direct = Object.keys(usage).filter(wanted).sort() + for (const name of direct) { + const built = seatFromUsage(name, usage[name], config.usage[name]?.kind ?? null, config) + if (built.skip !== undefined) skipped.push({ seat: name, reason: scrubReason(built.skip) }) + else seats.push(built.seat) + } + const names = (metersDir === null ? [] : (list ?? [])) + .map((name) => METER_FILE.exec(name)?.[1]) + .filter((name) => name !== undefined) + .filter((name) => wanted(name) && !direct.includes(name)) + .sort() + for (const name of names) { + const meter = readJson(join(metersDir, `${name}.json`)) + if (meter.value === undefined) { + skipped.push({ seat: name, reason: meter.error ? `unreadable: ${meter.error}` : "missing" }) + continue + } + const cache = readJson(join(metersDir, `.window-cache-${name}.json`)).value ?? null + const built = seatFromMeter(name, meter.value, cache, config) + if (built.skip !== undefined) skipped.push({ seat: name, reason: built.skip }) + else seats.push(built.seat) + } + return { + snapshot: { schema_version: SCHEMA_VERSION, host, published_at: now, seats }, + skipped, + source: direct.length > 0 ? (metersDir === null ? "usage" : "usage+meters") : "meters" + } +} + +/** Key-sorted JSON. */ +const canonical = (value) => + JSON.stringify(value, (_key, entry) => + entry !== null && typeof entry === "object" && !Array.isArray(entry) + ? Object.fromEntries(Object.entries(entry).sort(([left], [right]) => (left < right ? -1 : left > right ? 1 : 0))) + : entry + ) + +/** + * Providers whose `observed_at` is a provider's answer. For the others it is a + * feeder or kernel restamp (codex: the feeder's re-read of a local rollout; + * qwen: a hold record; halogen: the kernel's slot row), which moves every tick + * without the reading changing, so it is left out of the fingerprint and the + * heartbeats carry it instead. + */ +const PROVIDER_INSTANT = new Set(["claude"]) + +/** What decides a push: the reading's identity, not the instant it was restamped. */ +export const fingerprint = (seat) => + canonical({ + observed_at: PROVIDER_INSTANT.has(seat.provider) ? Date.parse(seat.observed_at) : null, + grade: seat.grade, + windows: seat.windows, + dispatchable: seat.dispatchable, + dispatchable_reason: seat.dispatchable_reason, + slots: seat.slots, + plan: seat.plan + }) + +/** + * The refresh policy: push when any seat's reading changed, and at least every + * `heartbeatSeconds` as a liveness beat (every `slotHeartbeatSeconds` while a + * dispatchable Halogen slot row is published); never twice within + * `minIntervalSeconds`. + */ +export const decidePush = (state, snapshot, nowMs, policy = DEFAULT_POLICY) => { + const last = state?.last_push_at === undefined ? null : Date.parse(state.last_push_at) + const since = last === null || !Number.isFinite(last) ? null : (nowMs - last) / 1000 + if (since !== null && since >= 0 && since < policy.minIntervalSeconds) { + return { push: false, reason: "min-interval", changed: [] } + } + const held = state?.fingerprints ?? {} + const current = Object.fromEntries(snapshot.seats.map((seat) => [seat.seat, fingerprint(seat)])) + const changed = Object.keys(current).filter((seat) => held[seat] !== current[seat]) + const removed = Object.keys(held).filter((seat) => !(seat in current)) + if (changed.length > 0 || removed.length > 0) { + return { push: true, reason: "changed", changed: [...changed, ...removed].sort() } + } + if (since === null || since < 0 || since >= policy.heartbeatSeconds) { + return { push: true, reason: "heartbeat", changed: [] } + } + const slotBeat = policy.slotHeartbeatSeconds ?? DEFAULT_POLICY.slotHeartbeatSeconds + const liveSlots = snapshot.seats.some((seat) => seat.provider === "halogen" && seat.dispatchable) + if (liveSlots && since >= slotBeat) { + return { push: true, reason: "heartbeat", changed: [] } + } + return { push: false, reason: "unchanged", changed: [] } +} + +/** The state to keep after a successful push. */ +export const stateAfterPush = (snapshot, now) => ({ + last_push_at: now, + fingerprints: Object.fromEntries(snapshot.seats.map((seat) => [seat.seat, fingerprint(seat)])) +}) + +export const readState = (path) => { + const read = readJson(path) + return read.value !== null && typeof read.value === "object" ? read.value : {} +} + +/** Atomic, 0600: a partial state file must never make the next run skip a push. */ +const writeState = (path, state) => { + mkdirSync(dirname(path), { recursive: true, mode: 0o700 }) + const temporary = `${path}.tmp-${process.pid}` + writeFileSync(temporary, `${JSON.stringify(state, null, 2)}\n`, { mode: 0o600 }) + renameSync(temporary, path) +} + +class PushError extends Error {} + +/** + * Sends the snapshot to `POST /capacity/seats`. The token is sent and + * never echoed: an error names the status and the route, not the header. + */ +const pushSnapshot = async ({ base, token, snapshot, fetchImpl = fetch, instance = "" }) => { + const url = `${base.replace(/\/+$/, "")}/capacity/seats` + let response + try { + response = await fetchImpl(url, { + method: "POST", + headers: { + authorization: `Bearer ${token}`, + "content-type": "application/json", + ...(instance === "" ? {} : { "x-conwip-factory-instance": instance }) + }, + body: JSON.stringify(snapshot) + }) + } catch (error) { + throw new PushError(`POST /capacity/seats did not complete: ${error?.name ?? "Error"}`) + } + const text = await response.text() + if (response.status !== 200) { + throw new PushError(`POST /capacity/seats answered ${response.status}: ${text.slice(0, 200)}`) + } + try { + return JSON.parse(text) + } catch { + throw new PushError("POST /capacity/seats answered 200 with a body that is not JSON") + } +} + +/** + * One pass: build, decide, push when due, and keep the state only after the + * lake has answered 200. + */ +export const runOnce = async ({ + metersDir, + capacityDir, + statePath, + config, + base, + token, + now = () => new Date().toISOString(), + list, + fetchImpl = fetch, + policy = DEFAULT_POLICY, + force = false, + dryRun = false, + instance = "", + allowNetwork = true, + preDispatch = false, + usageDir, + demandDir +}) => { + const at = now() + // Each seat's usage source first. A dry run never calls the network. + const usage = {} + for (const seat of Object.keys(config.usage ?? {}).sort()) { + usage[seat] = await readSeatUsage({ + seat, + source: config.usage[seat], + nowMs: Date.parse(at), + allowNetwork: allowNetwork && !dryRun, + preDispatch, + fetchImpl, + ...(usageDir === undefined ? {} : { usageDir }), + ...(demandDir === undefined ? {} : { demandDir }) + }) + } + const { snapshot, skipped, source } = buildSnapshot({ metersDir, capacityDir, config, now: at, list, usage }) + const state = readState(statePath) + const decision = force + ? { push: true, reason: "forced", changed: [] } + : decidePush(state, snapshot, Date.parse(at), policy) + const summary = { at, source, seats: snapshot.seats.map((seat) => seat.seat), skipped, decision } + if (dryRun) return { ...summary, pushed: false, snapshot } + if (!decision.push) return { ...summary, pushed: false } + const answer = await pushSnapshot({ base, token, snapshot, fetchImpl, instance }) + writeState(statePath, stateAfterPush(snapshot, at)) + return { ...summary, pushed: true, results: answer.results ?? null } +} diff --git a/pkgs/substrate-apps/src/apps/pusher/src/seatsOracle.mjs b/pkgs/substrate-apps/src/apps/pusher/src/seatsOracle.mjs new file mode 100644 index 00000000..3d3bb66f --- /dev/null +++ b/pkgs/substrate-apps/src/apps/pusher/src/seatsOracle.mjs @@ -0,0 +1,220 @@ +// apps/pusher/src/seatsOracle.mjs: the gentle pusher's one source. It reads the +// box's capacity oracle (`seats --json`, schema seat-capacity/1) and converts +// the answer to seat-capacity/2 for the floor. Pure: no clock, no file, no +// network; the process (bin/substrate-pusher.mjs) runs the oracle and posts. +// +// The pusher never calls a provider's usage endpoint itself. The oracle owns +// its own cache and its own 429 handling; the pusher only decides how often it +// asks the oracle (gentle.mjs, the repo's refresh policy). +// +// WHAT THE CONVERSION REFUSES TO INVENT (fail closed, as the /2 schema says): +// - a window the /2 kinds cannot place (a length other than 300 or 10080 +// minutes, a scope naming no model) becomes a binding seven_day window +// graded UNKNOWN with its reset kept, and the seat withdraws +// model_windows_complete; +// - a null or missing percentage stays null (UNKNOWN, never headroom); +// - a seat the oracle did not report is named in `skipped`, never carried +// forward; the floor's copy then ages to STALE and refuses on its own; +// - local paths in any reason are replaced by `` before they leave the +// box, and no oracle field other than the few named here is copied. + +import { hostname } from "node:os" + +export const SOURCE_SCHEMA = "seat-capacity/1" +export const TARGET_SCHEMA = "seat-capacity/2" + +/** The seats the pusher publishes unless its config says otherwise (the runs-on seat names). */ +export const DEFAULT_SEATS = Object.freeze(["cc", "cc2", "codex", "pi-qwencloud", "halogen"]) +/** Oracle seat ids onto runs-on seat names: the oracle calls the worker's Halogen server `gpu-worker`. */ +export const DEFAULT_SEAT_IDS = Object.freeze({ "gpu-worker": "halogen" }) +/** Providers whose percentages are the oracle's estimate, not a provider answer (Qwen Cloud: a credit count from local logs). */ +export const DEFAULT_ESTIMATED_PROVIDERS = Object.freeze(["qwen"]) +/** Seats ruled out of dispatch whatever a config says (a config may add rulings, never lift these). */ +export const RULED_NOT_DISPATCHABLE = Object.freeze({ + cc3: "evicted", + "gpu-coordinator": "halogen is declared but not resident on the coordinator" +}) + +export class SeatsFormatError extends Error {} + +const ZONED = /^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}(:\d{2}(\.\d+)?)?(Z|[+-]\d{2}:\d{2})$/ +const SEAT_ID = /^[a-z0-9][a-z0-9-]{0,62}$/ +const PROVIDER = /^[a-z][a-z0-9-]{0,31}$/ + +export const instant = (value) => + typeof value === "string" && ZONED.test(value) && Number.isFinite(Date.parse(value)) ? value : null +const percent = (value) => (typeof value === "number" && Number.isFinite(value) && value >= 0 ? value : null) +const text = (value) => (typeof value === "string" && value.trim() !== "" ? value.trim() : null) + +/** A reason as it leaves the box: local absolute paths become ``. */ +export const scrub = (value) => + typeof value === "string" + ? value.replace(/(?:~|\/(?:home|root|Users|nix|run|tmp|var|etc|mnt|srv))\/[^\s,;:'"`)]+/g, "").slice(0, 300) + : null + +/** + * The conversion's settings, from the pusher config (defaults filled in). + * `owners` overrides the oracle's owner field per seat (the oracle's own seat + * table can lag a ruling); `slots` is the slot count of a slot-counted seat. + */ +export const conversionSettings = (config = {}) => { + const object = (value) => (value !== null && typeof value === "object" && !Array.isArray(value) ? value : {}) + const stated = Object.fromEntries( + Object.entries(object(config.not_dispatchable)).filter(([, reason]) => typeof reason === "string" && reason !== "") + ) + return { + host: typeof config.host === "string" && config.host !== "" ? config.host : null, + seats: Array.isArray(config.seats) ? config.seats.filter((seat) => typeof seat === "string") : [...DEFAULT_SEATS], + seatIds: { ...DEFAULT_SEAT_IDS, ...object(config.seatIds) }, + owners: object(config.owners), + plans: object(config.plans), + slots: object(config.slots), + estimatedProviders: Array.isArray(config.estimatedProviders) ? config.estimatedProviders : [...DEFAULT_ESTIMATED_PROVIDERS], + notDispatchable: { ...RULED_NOT_DISPATCHABLE, ...stated } + } +} + +/** When the provider (or the local artefact) answered, as best the oracle states it. */ +const observedOf = (seat, generatedMs) => { + const stated = instant(seat?.source?.observed_at) + if (stated !== null) return new Date(Date.parse(stated)).toISOString() + const age = seat?.source?.reading_age_seconds + const ageS = typeof age === "number" && Number.isFinite(age) && age >= 0 ? age : 0 + return new Date(generatedMs - ageS * 1000).toISOString() +} + +/** One oracle window as a /2 window, or an UNKNOWN binding weekly window when it cannot be placed. */ +const windowOf = (entry, grade) => { + const minutes = entry?.minutes + const scope = text(entry?.scope) + const base = { + utilization_pct: percent(entry?.used_pct), + resets_at: instant(entry?.resets_at), + severity: text(entry?.severity), + grade + } + if (scope !== null && minutes === 10080) return { placed: true, window: { kind: "model_scoped", model: scope, binding: true, minutes: 10080, ...base } } + if (scope === null && minutes === 300) return { placed: true, window: { kind: "five_hour", model: null, binding: true, minutes: 300, ...base } } + if (scope === null && minutes === 10080) return { placed: true, window: { kind: "seven_day", model: null, binding: true, minutes: 10080, ...base } } + return { + placed: false, + window: { kind: "seven_day", model: null, binding: true, minutes: 10080, utilization_pct: null, resets_at: instant(entry?.resets_at), severity: null, grade: "UNKNOWN" } + } +} + +/** The /2 grade of an oracle seat. CACHED is a provider answer that has aged: MEASURED, dated by its age. */ +const gradeOf = (seat, provider, settings) => { + if (seat.grade === "UNKNOWN" || seat.grade === undefined) return "UNKNOWN" + if (settings.estimatedProviders.includes(provider)) return "ESTIMATED" + if (seat.grade === "MEASURED" || seat.grade === "CACHED") return "MEASURED" + return "UNKNOWN" +} + +const dispatchOf = (id, owner, seat, provider, settings) => { + const ruled = settings.notDispatchable[id] + if (typeof ruled === "string") return { dispatchable: false, dispatchable_reason: ruled } + if (owner !== "tom") return { dispatchable: false, dispatchable_reason: "third-party" } + if (seat.state === "unauth") return { dispatchable: false, dispatchable_reason: "auth-failed" } + if (seat.state === "n/a") return { dispatchable: false, dispatchable_reason: scrub(text(seat.detail) ?? "not applicable") } + if (provider === "halogen" && seat.state !== "open") return { dispatchable: false, dispatchable_reason: `down: ${scrub(text(seat.detail) ?? seat.state ?? "unknown")}` } + return { dispatchable: true, dispatchable_reason: null } +} + +/** + * Converts one `seats --json` document to a seat-capacity/2 snapshot. + * + * @param doc the parsed oracle answer (schema seat-capacity/1). + * @param config the pusher config (see conversionSettings). + * @param now the publishing instant (ISO), the pusher's clock. + * @returns `{ snapshot, skipped }`; skipped names every configured seat left out and why. + * @throws SeatsFormatError when the document is not seat-capacity/1. + */ +export const snapshotFromSeatsV1 = (doc, config, now) => { + if (doc === null || typeof doc !== "object" || doc.schema_version !== SOURCE_SCHEMA || !Array.isArray(doc.seats)) { + throw new SeatsFormatError(`the oracle's answer is not ${SOURCE_SCHEMA}`) + } + if (instant(now) === null) throw new RangeError(`now is not an instant: ${now}`) + const settings = conversionSettings(config) + const generated = instant(doc.generated_at) + const generatedMs = generated === null ? Date.parse(now) : Math.min(Date.parse(generated), Date.parse(now)) + const seats = [] + const skipped = [] + const seen = new Set() + for (const raw of doc.seats) { + if (raw === null || typeof raw !== "object" || typeof raw.id !== "string") continue + const id = settings.seatIds[raw.id] ?? raw.id + if (!settings.seats.includes(id) || seen.has(id)) continue + seen.add(id) + const provider = typeof raw.provider === "string" ? raw.provider : "" + if (!SEAT_ID.test(id)) { skipped.push({ seat: id, reason: "not a seat id the floor accepts" }); continue } + if (!PROVIDER.test(provider) || provider === "none") { skipped.push({ seat: id, reason: "the oracle names no provider" }); continue } + const configured = settings.owners[id] + const owner = configured === "tom" || configured === "third-party" ? configured : raw.owner === "tom" || raw.owner === "kernel" ? "tom" : "third-party" + const grade = gradeOf(raw, provider, settings) + const observed = observedOf(raw, generatedMs) + let unmapped = 0 + const windows = (Array.isArray(raw.windows) ? raw.windows : []).map((entry) => { + const placed = windowOf(entry, grade) + if (!placed.placed) unmapped++ + return placed.window + }) + const ageS = Math.round((generatedMs - Date.parse(observed)) / 1000) + const staleReason = + grade === "UNKNOWN" + ? scrub(text(raw.detail) ?? "the oracle could not read this seat") + : raw.grade === "CACHED" + ? `the oracle served a cached reading ${ageS} s old` + : null + const slotCount = settings.slots[id] + const slots = + provider === "halogen" && raw.state === "open" + ? { capacity: Number.isInteger(slotCount) && slotCount >= 0 ? slotCount : 1, holders: 0 } + : null + const plan = instant(settings.plans[id]) + seats.push({ + seat: id, + provider, + owner, + ...dispatchOf(id, owner, raw, provider, settings), + plan: plan === null ? null : { expires_at: plan }, + slots, + windows, + observed_at: observed, + source: { kind: "seats-oracle", detail: scrub(`${SOURCE_SCHEMA} ${text(raw.source?.kind) ?? "no-source"}`) }, + grade, + stale_reason: staleReason, + model_windows_complete: grade !== "UNKNOWN" && unmapped === 0 + }) + } + for (const id of settings.seats) { + if (!seen.has(id)) skipped.push({ seat: id, reason: "the oracle did not report this seat" }) + } + return { + snapshot: { + schema_version: TARGET_SCHEMA, + host: settings.host ?? (typeof doc.host === "string" && doc.host !== "" ? doc.host : hostname().split(".")[0]), + published_at: now, + seats: seats.sort((a, b) => (a.seat < b.seat ? -1 : a.seat > b.seat ? 1 : 0)) + }, + skipped + } +} + +// ---- the oracle's environment (parity gap PT-01, 2026-09-24). +// The dotfiles `seats` oracle consults a peer cache before the network; its default is the tally-rewrite feeder's +// meters directory. The pusher points it at a substrate-owned directory instead, so no substrate process reads a +// tally-era path. `peerCacheDir: "inherit"` keeps the oracle's own default (the old behaviour), for a box where the +// tally feeder still runs and its readings are wanted to spare the rate-limited usage endpoint. +export const DEFAULT_PEER_CACHE_SUBDIR = "seats-peer-cache" + +/** The environment the pusher runs `seats` with. Pure: `stateDir` is the substrate state directory. */ +export const oracleEnv = (config = {}, stateDir, env = {}) => { + const configured = config.peerCacheDir ?? undefined + if (configured === "inherit") return { ...env } + if (configured !== undefined && (typeof configured !== "string" || configured === "")) { + throw new SeatsFormatError("peerCacheDir must be a non-empty path or \"inherit\"") + } + const dir = configured ?? `${stateDir.replace(/\/+$/, "")}/${DEFAULT_PEER_CACHE_SUBDIR}` + if (/tally-rewrite/.test(dir)) throw new SeatsFormatError("peerCacheDir must not be a tally-rewrite path; use \"inherit\" to keep the oracle's own default") + return { ...env, SEATS_PEER_CACHE_DIR: dir } +} diff --git a/pkgs/substrate-apps/src/apps/pusher/src/token.mjs b/pkgs/substrate-apps/src/apps/pusher/src/token.mjs new file mode 100644 index 00000000..2a61bd70 --- /dev/null +++ b/pkgs/substrate-apps/src/apps/pusher/src/token.mjs @@ -0,0 +1,13 @@ +// apps/pusher/src/token.mjs: reads the floor's bearer from a file. The value is +// returned to the caller and never printed; an error names the path only. +import { readFileSync } from "node:fs" + +export class TokenError extends Error {} + +export const tokenFromFile = (path) => { + try { + return readFileSync(path, "utf8").trim() + } catch (error) { + throw new TokenError(`cannot read the floor token file ${path}: ${error.code ?? error.message}`) + } +} diff --git a/pkgs/substrate-apps/src/apps/pusher/src/usageSource.mjs b/pkgs/substrate-apps/src/apps/pusher/src/usageSource.mjs new file mode 100644 index 00000000..1c13cb3f --- /dev/null +++ b/pkgs/substrate-apps/src/apps/pusher/src/usageSource.mjs @@ -0,0 +1,352 @@ +// apps/pusher/src/usageSource.mjs: the Claude usage source, read directly or +// through an adapter, so the pusher needs no tally-rewrite feeder. +// +// TWO ADAPTERS, ONE READING SHAPE `{ observed_at, usage }`: +// +// "oauth" the direct reader. It calls the usage endpoint Claude Code's +// own /usage view calls, with the seat's OAuth bearer read from +// the seat's credentials file at call time. The bearer is held +// in memory for the one request and never written, logged or +// put in an error. Calls obey refreshPolicy.mjs. +// (Endpoint, headers and the token's place in the file are +// REPORTED from research-methods bin/stamp-receipt.py:103-126.) +// "cache-file" a file some other reader already wrote in the endpoint's +// answer shape: the tally feeder's `.window-cache-.json` +// while it still exists, or this module's own cache. +// +// THE CACHE KEEPS ONLY WHAT IS READ. The direct reader writes `observed_at` +// and `usage.{five_hour, seven_day, limits}`; spend and every other field of +// the answer are dropped before anything touches the disk. +import { existsSync, mkdirSync, readdirSync, readFileSync, renameSync, statSync, writeFileSync } from "node:fs" +import { homedir } from "node:os" +import { dirname, join } from "node:path" + +import { afterAttempt, initialRefreshState, REFRESH_POLICY, refreshDue } from "./refreshPolicy.mjs" + +export const USAGE_ENDPOINT = "https://api.anthropic.com/api/oauth/usage" +const USAGE_HEADERS = { "content-type": "application/json", "anthropic-beta": "oauth-2025-04-20" } +/** Where the bearer sits in a Claude credentials file. */ +export const DEFAULT_TOKEN_POINTER = ["claudeAiOauth", "accessToken"] + +export const defaultStateDir = () => join(homedir(), ".local", "state", "substrate") +export const defaultUsageDir = () => join(defaultStateDir(), "usage") +export const defaultDemandDir = () => join(defaultStateDir(), "demand") + +/** The usage endpoint's `limits[].kind`, mapped to the floor's window kinds. */ +const LIMIT_KINDS = { session: "five_hour", weekly_all: "seven_day", weekly_scoped: "model_scoped" } +const MINUTES = { five_hour: 300, seven_day: 10080, model_scoped: 10080 } +const ZONED = /^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}(:\d{2}(\.\d+)?)?(Z|[+-]\d{2}:\d{2})$/ + +export const instant = (value) => + typeof value === "string" && ZONED.test(value) && Number.isFinite(Date.parse(value)) ? value : null +const percent = (value) => (typeof value === "number" && Number.isFinite(value) && value >= 0 ? value : null) + +export class CredentialError extends Error {} + +/** + * The one mapping from the endpoint's `limits[]` to windows, for every pusher + * path (capacity review round 3, ported here by successor review r4: the + * pusher dropped rows it could not place and still claimed + * model_windows_complete, so the gate admitted past a 100 percent limit the + * tally adapter refused). A row it cannot place (a kind it does not know, or + * a scoped row naming no model, such as one scoped by surface only) is never + * dropped: it becomes a binding seven_day window graded UNKNOWN with its reset + * kept, and `unmapped` counts it, so the reading withdraws + * model_windows_complete. src/capacity/tally-meters.ts maps the same way; a + * root test pins the two together. + */ +export const windowsFromLimits = (limits) => { + let unmapped = 0 + const windows = limits.flatMap((limit) => { + const kind = LIMIT_KINDS[limit?.kind] + const model = + kind === "model_scoped" ? (limit.scope?.model?.display_name ?? limit.scope?.model?.id ?? null) : null + if (kind === undefined || (kind === "model_scoped" && typeof model !== "string")) { + unmapped++ + return [ + { + kind: "seven_day", + model: null, + binding: true, + minutes: MINUTES.seven_day, + utilization_pct: null, + resets_at: instant(limit?.resets_at), + severity: null, + grade: "UNKNOWN" + } + ] + } + return [ + { + kind, + model, + binding: true, + minutes: MINUTES[kind], + utilization_pct: percent(limit.percent), + resets_at: instant(limit.resets_at), + severity: typeof limit.severity === "string" ? limit.severity : null, + grade: "MEASURED" + } + ] + }) + return { windows, unmapped } +} + +/** A usage answer lists every model limit: limits[] is present and every row was placed. */ +export const limitsComplete = (usage) => + usage !== null && typeof usage === "object" && Array.isArray(usage.limits) && windowsFromLimits(usage.limits).unmapped === 0 + +/** + * The windows a usage answer states: `limits[]` when present (the only place + * model-scoped rows appear), else the flat `five_hour` / `seven_day` pair. + */ +export const windowsFromUsage = (usage) => { + if (usage === null || typeof usage !== "object") return [] + if (Array.isArray(usage.limits)) return windowsFromLimits(usage.limits).windows + return [ + ["five_hour", usage.five_hour], + ["seven_day", usage.seven_day] + ].flatMap(([kind, entry]) => + entry === null || typeof entry !== "object" + ? [] + : [ + { + kind, + model: null, + binding: true, + minutes: MINUTES[kind], + utilization_pct: percent(entry.utilization), + resets_at: instant(entry.resets_at), + severity: null, + grade: "MEASURED" + } + ] + ) +} + +/** Only the fields the pusher reads; spend and the rest never reach the disk. */ +export const minimalUsage = (usage) => { + const pick = (entry) => + entry !== null && typeof entry === "object" + ? { utilization: percent(entry.utilization), resets_at: instant(entry.resets_at) } + : null + const limits = Array.isArray(usage?.limits) + ? usage.limits.map((limit) => ({ + kind: typeof limit?.kind === "string" ? limit.kind : null, + percent: percent(limit?.percent), + severity: typeof limit?.severity === "string" ? limit.severity : null, + resets_at: instant(limit?.resets_at), + scope: + typeof limit?.scope?.model?.display_name === "string" || typeof limit?.scope?.model?.id === "string" + ? { + model: { + display_name: limit.scope.model.display_name ?? null, + id: limit.scope.model.id ?? null + } + } + : null + })) + : undefined + return { + five_hour: pick(usage?.five_hour), + seven_day: pick(usage?.seven_day), + ...(limits === undefined ? {} : { limits }) + } +} + +const readJson = (path) => { + if (!existsSync(path)) return { missing: true } + try { + return { value: JSON.parse(readFileSync(path, "utf8")) } + } catch (error) { + return { error: `${error.name}` } + } +} + +/** Atomic, 0600, directory 0700. */ +export const writePrivateJson = (path, value) => { + mkdirSync(dirname(path), { recursive: true, mode: 0o700 }) + const temporary = `${path}.tmp-${process.pid}` + writeFileSync(temporary, `${JSON.stringify(value, null, 2)}\n`, { mode: 0o600 }) + renameSync(temporary, path) +} + +/** A reading `{ observed_at, usage }` from a file in the endpoint's answer shape, or null. */ +export const readUsageFile = (path) => { + const read = readJson(path) + const value = read.value + if (value === null || typeof value !== "object") return null + const observed = instant(value.observed_at) + if (observed === null || value.usage === null || typeof value.usage !== "object") return null + return { observed_at: observed, usage: value.usage } +} + +/** + * The bearer from a credentials file. The value is returned and never shown: + * every error names the path and the pointer only. + */ +export const bearerFromCredentials = (path, pointer = DEFAULT_TOKEN_POINTER) => { + let parsed + try { + parsed = JSON.parse(readFileSync(path, "utf8")) + } catch (error) { + throw new CredentialError(`cannot read the credentials file ${path}: ${error.code ?? error.name}`) + } + let node = parsed + for (const key of pointer) node = node !== null && typeof node === "object" ? node[key] : undefined + if (typeof node !== "string" || node.length === 0) { + throw new CredentialError(`the credentials file ${path} has no string at ${pointer.join(".")}`) + } + return node +} + +const mtimeMs = (path) => { + try { + return statSync(path).mtimeMs + } catch { + return null + } +} + +/** + * Whether a seat is in active use: a demand marker touched within the window + * (dispatch touches it), or a transcript under the seat's config directory + * written within the window (interactive use). Looks at the 5 most recently + * touched project directories only, to keep a pass cheap. + */ +export const seatActive = ({ seat, demandDir, configDir, nowMs, windowSeconds = 600 }) => { + const recent = (value) => value !== null && nowMs - value <= windowSeconds * 1000 + if (demandDir !== null && demandDir !== undefined && recent(mtimeMs(join(demandDir, seat)))) return true + if (configDir === null || configDir === undefined) return false + const projects = join(configDir, "projects") + let dirs + try { + dirs = readdirSync(projects, { withFileTypes: true }).filter((entry) => entry.isDirectory()) + } catch { + return false + } + const newest = dirs + .map((entry) => ({ path: join(projects, entry.name), at: mtimeMs(join(projects, entry.name)) ?? 0 })) + .sort((left, right) => right.at - left.at) + .slice(0, 5) + for (const dir of newest) { + if (recent(dir.at)) return true + try { + for (const name of readdirSync(dir.path)) { + if (name.endsWith(".jsonl") && recent(mtimeMs(join(dir.path, name)))) return true + } + } catch { + // unreadable directory: not evidence of activity + } + } + return false +} + +const retryAfterSeconds = (response) => { + const raw = response.headers?.get?.("retry-after") + if (raw === null || raw === undefined) return null + const seconds = Number(raw) + if (Number.isFinite(seconds)) return seconds + const at = Date.parse(raw) + return Number.isFinite(at) ? Math.max(0, (at - Date.now()) / 1000) : null +} + +/** One GET of the usage endpoint. The bearer is only in the request header. */ +export const fetchUsage = async ({ token, fetchImpl = fetch, timeoutMs = 12000 }) => { + let response + try { + response = await fetchImpl(USAGE_ENDPOINT, { + method: "GET", + headers: { ...USAGE_HEADERS, authorization: `Bearer ${token}` }, + signal: AbortSignal.timeout(timeoutMs) + }) + } catch (error) { + return { kind: "error", detail: `the usage read did not complete: ${error?.name ?? "Error"}` } + } + if (response.status === 429) { + return { kind: "rate-limited", retryAfterSeconds: retryAfterSeconds(response), detail: "usage endpoint 429" } + } + if (response.status === 401 || response.status === 403) { + return { kind: "auth-failed", detail: `usage endpoint ${response.status} unauthorized` } + } + if (response.status !== 200) return { kind: "error", detail: `usage endpoint answered ${response.status}` } + try { + return { kind: "success", usage: JSON.parse(await response.text()) } + } catch { + return { kind: "error", detail: "usage endpoint answered 200 with a body that is not JSON" } + } +} + +const bindingFacts = (reading) => { + const windows = reading === null ? [] : windowsFromUsage(reading.usage).filter((w) => w.kind !== "model_scoped") + const known = windows.map((w) => w.utilization_pct).filter((value) => value !== null) + return { + bindingMaxPct: known.length === 0 ? null : Math.max(...known), + resets: windows.map((w) => w.resets_at).filter((value) => value !== null) + } +} + +/** + * Reads one seat's usage through its configured source. + * + * @param source `{ kind: "oauth", credentials, token_pointer?, config_dir? }` or + * `{ kind: "cache-file", path }`. + * @returns `{ reading, attempted, decision, error }`; `reading` is + * `{ observed_at, usage }` or null, `error` a line safe to publish or null. + */ +export const readSeatUsage = async ({ + seat, + source, + usageDir = defaultUsageDir(), + demandDir = defaultDemandDir(), + nowMs = Date.now(), + allowNetwork = true, + preDispatch = false, + fetchImpl = fetch, + policy = REFRESH_POLICY +}) => { + if (source?.kind === "cache-file") { + const reading = typeof source.path === "string" ? readUsageFile(source.path) : null + return { reading, attempted: false, decision: null, error: reading === null ? "no usage cache file reading" : null } + } + if (source?.kind !== "oauth" || typeof source.credentials !== "string") { + return { reading: null, attempted: false, decision: null, error: "no usage source configured" } + } + const cachePath = join(usageDir, `${seat}.json`) + const statePath = join(usageDir, `${seat}.state.json`) + const cached = readUsageFile(cachePath) + const stored = readJson(statePath).value + const state = stored !== null && typeof stored === "object" ? stored : initialRefreshState() + const credentialsMtimeMs = mtimeMs(source.credentials) + const decision = refreshDue( + state, + { + nowMs, + active: seatActive({ seat, demandDir, configDir: source.config_dir ?? null, nowMs }), + preDispatch, + credentialsMtimeMs, + ...bindingFacts(cached) + }, + policy + ) + const lastError = typeof state.last_error === "string" ? state.last_error : null + if (!decision.due || !allowNetwork) { + return { reading: cached, attempted: false, decision, error: lastError } + } + let outcome + try { + const token = bearerFromCredentials(source.credentials, source.token_pointer ?? DEFAULT_TOKEN_POINTER) + outcome = await fetchUsage({ token, fetchImpl }) + } catch (error) { + outcome = { kind: "auth-failed", detail: error instanceof CredentialError ? "credentials unreadable" : "credentials error" } + } + const next = { ...afterAttempt(state, outcome, nowMs, credentialsMtimeMs, policy), last_error: outcome.kind === "success" ? null : outcome.detail } + if (outcome.kind === "success") { + const reading = { observed_at: new Date(nowMs).toISOString(), usage: minimalUsage(outcome.usage) } + writePrivateJson(cachePath, reading) + writePrivateJson(statePath, next) + return { reading, attempted: true, decision, error: null } + } + writePrivateJson(statePath, next) + return { reading: cached, attempted: true, decision, error: outcome.detail } +} diff --git a/pkgs/substrate-apps/src/apps/pusher/test/fixtures/seats-v1.json b/pkgs/substrate-apps/src/apps/pusher/test/fixtures/seats-v1.json new file mode 100644 index 00000000..672b511a --- /dev/null +++ b/pkgs/substrate-apps/src/apps/pusher/test/fixtures/seats-v1.json @@ -0,0 +1,269 @@ +{ + "schema_version": "seat-capacity/1", + "generated_at": "2026-09-23T20:00:00Z", + "host": "example-box", + "recommend": "cc", + "thresholds": { + "wall_pct": 95.0, + "warn_pct": 80.0 + }, + "allowance_changes": [], + "seats": [ + { + "id": "cc", + "provider": "claude", + "label": "Claude (personal)", + "owner": "tom", + "state": "open", + "usable": true, + "grade": "MEASURED", + "detail": "", + "note": "", + "source": { + "kind": "seat-feeder-cache", + "config_dir": "/home/example/.claude", + "reading_age_seconds": 40 + }, + "windows": [ + { + "binding": true, + "id": "five_hour", + "is_active": false, + "label": "session (5h)", + "minutes": 300, + "pace_pct_per_hour": null, + "projected_pct_at_reset": null, + "remaining_pct": 100.0, + "resets_at": "", + "resets_in_seconds": null, + "scope": null, + "severity": "normal", + "used_pct": 0.0, + "wall_eta": "" + }, + { + "binding": true, + "id": "seven_day", + "is_active": false, + "label": "weekly (7d)", + "minutes": 10080, + "pace_pct_per_hour": null, + "projected_pct_at_reset": null, + "remaining_pct": 94.0, + "resets_at": "2026-09-30T09:59:59Z", + "resets_in_seconds": null, + "scope": null, + "severity": "normal", + "used_pct": 6.0, + "wall_eta": "" + }, + { + "binding": false, + "id": "weekly:fable", + "is_active": false, + "label": "weekly \u00b7 Fable", + "minutes": 10080, + "pace_pct_per_hour": null, + "projected_pct_at_reset": null, + "remaining_pct": 98.0, + "resets_at": "2026-09-30T09:59:59Z", + "resets_in_seconds": null, + "scope": "Fable", + "severity": "normal", + "used_pct": 2.0, + "wall_eta": "" + } + ] + }, + { + "id": "cc2", + "provider": "claude", + "label": "Claude (work)", + "owner": "tom", + "state": "tight", + "usable": true, + "grade": "CACHED", + "detail": "", + "note": "", + "source": { + "kind": "own-cache", + "reading_age_seconds": 1500 + }, + "windows": [ + { + "binding": true, + "id": "five_hour", + "is_active": false, + "label": "session (5h)", + "minutes": 300, + "pace_pct_per_hour": null, + "projected_pct_at_reset": null, + "remaining_pct": 86.0, + "resets_at": "2026-09-23T23:40:00Z", + "resets_in_seconds": null, + "scope": null, + "severity": "normal", + "used_pct": 14.0, + "wall_eta": "" + }, + { + "binding": true, + "id": "seven_day", + "is_active": false, + "label": "weekly (7d)", + "minutes": 10080, + "pace_pct_per_hour": null, + "projected_pct_at_reset": null, + "remaining_pct": 24.0, + "resets_at": "2026-09-24T06:00:00Z", + "resets_in_seconds": null, + "scope": null, + "severity": "warning", + "used_pct": 76.0, + "wall_eta": "" + }, + { + "binding": false, + "id": "weekly:fable", + "is_active": false, + "label": "weekly \u00b7 Fable", + "minutes": 10080, + "pace_pct_per_hour": null, + "projected_pct_at_reset": null, + "remaining_pct": 0.0, + "resets_at": "2026-09-24T06:00:00Z", + "resets_in_seconds": null, + "scope": "Fable", + "severity": "critical", + "used_pct": 100.0, + "wall_eta": "" + } + ] + }, + { + "id": "cc3", + "provider": "claude", + "label": "Claude (third)", + "owner": "tom", + "state": "unauth", + "usable": false, + "grade": "UNKNOWN", + "detail": "sign-in expired, see /home/example/.claude-3/.credentials.json", + "note": "", + "source": {}, + "windows": [] + }, + { + "id": "codex", + "provider": "codex", + "label": "Codex CLI", + "owner": "third-party", + "state": "open", + "usable": true, + "grade": "MEASURED", + "detail": "", + "note": "", + "source": { + "kind": "codex-rollout-rate-limits", + "path": "/home/example/.codex/sessions/x.jsonl", + "observed_at": "2026-09-23T19:50:00Z" + }, + "windows": [ + { + "binding": true, + "id": "codex:primary", + "is_active": false, + "label": "weekly (7d)", + "minutes": 10080, + "pace_pct_per_hour": null, + "projected_pct_at_reset": null, + "remaining_pct": 96.0, + "resets_at": "2026-09-28T19:07:31Z", + "resets_in_seconds": null, + "scope": null, + "severity": "", + "used_pct": 4.0, + "wall_eta": "" + }, + { + "binding": true, + "id": "codex:odd", + "is_active": false, + "label": "odd (1d)", + "minutes": 1440, + "pace_pct_per_hour": null, + "projected_pct_at_reset": null, + "remaining_pct": 90.0, + "resets_at": "2026-09-24T19:00:00Z", + "resets_in_seconds": null, + "scope": null, + "severity": "", + "used_pct": 10.0, + "wall_eta": "" + } + ] + }, + { + "id": "pi-qwencloud", + "provider": "qwen", + "label": "Qwen Cloud", + "owner": "tom", + "state": "open", + "usable": true, + "grade": "MEASURED-FROM-REFUSAL", + "detail": "", + "note": "", + "source": { + "kind": "429-refusal-text", + "key_source": "/run/agenix/example-token" + }, + "windows": [ + { + "binding": true, + "id": "qwen:weekly", + "is_active": false, + "label": "weekly (7d)", + "minutes": 10080, + "pace_pct_per_hour": null, + "projected_pct_at_reset": null, + "remaining_pct": null, + "resets_at": "", + "resets_in_seconds": null, + "scope": null, + "severity": "", + "used_pct": null, + "wall_eta": "" + } + ] + }, + { + "id": "gpu-coordinator", + "provider": "none", + "label": "Local GPU", + "owner": "kernel", + "state": "n/a", + "usable": false, + "grade": "UNKNOWN", + "detail": "serves nothing", + "note": "", + "source": {}, + "windows": [] + }, + { + "id": "gpu-worker", + "provider": "halogen", + "label": "Halogen", + "owner": "kernel", + "state": "open", + "usable": true, + "grade": "MEASURED", + "detail": "up", + "note": "", + "source": { + "kind": "halogen-health", + "endpoint": "http://worker.example:8731/health" + }, + "windows": [] + } + ] +} \ No newline at end of file diff --git a/pkgs/substrate-apps/src/apps/pusher/test/gentle.test.mjs b/pkgs/substrate-apps/src/apps/pusher/test/gentle.test.mjs new file mode 100644 index 00000000..27c8d8fa --- /dev/null +++ b/pkgs/substrate-apps/src/apps/pusher/test/gentle.test.mjs @@ -0,0 +1,257 @@ +// The gentle pusher: seat-capacity/1 (the box's oracle, `seats --json`) to +// seat-capacity/2, the refresh policy it reads under, and the stale handling. +import { spawn } from "node:child_process" +import { existsSync, mkdtempSync, readFileSync, readdirSync, rmSync, statSync, writeFileSync } from "node:fs" +import { createServer } from "node:http" +import { tmpdir } from "node:os" +import { join } from "node:path" +import { fileURLToPath } from "node:url" +import { Schema } from "effect" +import { afterEach, describe, expect, it } from "vitest" + +import { SeatCapacitySnapshot } from "@substrate/planning/schema/seatCapacity.ts" +import { admitSeat } from "@substrate/planning/capacity/project.ts" +import { SeatsFormatError, snapshotFromSeatsV1 } from "../src/seatsOracle.mjs" +import { initialGentleState, planRead, tick } from "../src/gentle.mjs" + +const FIXTURE = fileURLToPath(new URL("./fixtures/seats-v1.json", import.meta.url)) +const BIN = fileURLToPath(new URL("../bin/substrate-pusher.mjs", import.meta.url)) +const SRC = fileURLToPath(new URL("../src/", import.meta.url)) +const v1 = () => JSON.parse(readFileSync(FIXTURE, "utf8")) +const NOW = "2026-09-23T20:00:05Z" +const decode = Schema.decodeUnknownSync(SeatCapacitySnapshot) +const convert = (config = {}, doc = v1(), now = NOW) => snapshotFromSeatsV1(doc, config, now) +const bySeat = (snapshot) => Object.fromEntries(snapshot.seats.map((seat) => [seat.seat, seat])) + +describe("seat-capacity/1 to seat-capacity/2", () => { + it("covers cc, cc2, codex, pi-qwencloud and halogen, and decodes as seat-capacity/2", () => { + const { snapshot, skipped } = convert() + expect(decode(snapshot)).toBeTruthy() + expect(snapshot.seats.map((seat) => seat.seat)).toEqual(["cc", "cc2", "codex", "halogen", "pi-qwencloud"]) + expect(skipped).toEqual([]) + expect(snapshot.published_at).toBe(NOW) + expect(snapshot.host).toBe("example-box") + }) + + it("places the 5h, 7d and model-scoped windows, with the reset instants and severities", () => { + const cc2 = bySeat(convert().snapshot).cc2 + expect(cc2.windows.map((w) => [w.kind, w.model, w.minutes, w.utilization_pct, w.resets_at, w.severity])).toEqual([ + ["five_hour", null, 300, 14, "2026-09-23T23:40:00Z", "normal"], + ["seven_day", null, 10080, 76, "2026-09-24T06:00:00Z", "warning"], + ["model_scoped", "Fable", 10080, 100, "2026-09-24T06:00:00Z", "critical"] + ]) + expect(cc2.model_windows_complete).toBe(true) + // An empty reset is "opens on first use"; an empty severity is none. + expect(bySeat(convert().snapshot).cc.windows[0]).toMatchObject({ kind: "five_hour", resets_at: null, utilization_pct: 0 }) + }) + + it("dates each reading by the provider's answer, not by the oracle's run", () => { + const seats = bySeat(convert().snapshot) + expect(seats.cc.observed_at).toBe("2026-09-23T19:59:20.000Z") // generated_at - 40 s + expect(seats.cc2.observed_at).toBe("2026-09-23T19:35:00.000Z") // a CACHED reading 1500 s old + expect(seats.cc2.stale_reason).toMatch(/cached reading 1500 s old/) + expect(seats.codex.observed_at).toBe("2026-09-23T19:50:00.000Z") // the rollout's own instant + }) + + it("a window it cannot place becomes a binding UNKNOWN weekly window and withdraws model_windows_complete", () => { + const codex = bySeat(convert().snapshot).codex + expect(codex.windows[1]).toMatchObject({ kind: "seven_day", binding: true, utilization_pct: null, grade: "UNKNOWN", resets_at: "2026-09-24T19:00:00Z" }) + expect(codex.model_windows_complete).toBe(false) + }) + + it("owners, rulings, slots and estimated providers follow the config; the oracle's owner field can be overridden", () => { + const plain = bySeat(convert().snapshot) + expect([plain.codex.owner, plain.codex.dispatchable, plain.codex.dispatchable_reason]).toEqual(["third-party", false, "third-party"]) + const ruled = bySeat(convert({ owners: { codex: "tom" }, slots: { halogen: 2 }, plans: { "pi-qwencloud": "2026-11-07T00:00:00Z" } }).snapshot) + expect([ruled.codex.owner, ruled.codex.dispatchable]).toEqual(["tom", true]) + expect(ruled.halogen).toMatchObject({ provider: "halogen", owner: "tom", slots: { capacity: 2, holders: 0 }, grade: "MEASURED", dispatchable: true }) + expect(ruled["pi-qwencloud"]).toMatchObject({ grade: "ESTIMATED", plan: { expires_at: "2026-11-07T00:00:00Z" } }) + const cc3 = bySeat(convert({ seats: ["cc3"] }).snapshot).cc3 + expect([cc3.dispatchable, cc3.dispatchable_reason, cc3.grade]).toEqual([false, "evicted", "UNKNOWN"]) + const unauth = bySeat(convert({ seats: ["cc3"], not_dispatchable: {} }).snapshot).cc3 + expect(unauth.dispatchable_reason).toBe("evicted") // a config cannot lift a built-in ruling + const skip = convert({ seats: ["gpu-coordinator", "cc"] }) + expect(skip.skipped).toEqual([{ seat: "gpu-coordinator", reason: "the oracle names no provider" }]) + }) + + it("no local path, endpoint or credential location leaves the box", () => { + const out = JSON.stringify(convert({ seats: ["cc", "cc2", "cc3", "codex", "pi-qwencloud", "halogen"] }).snapshot) + expect(out).not.toMatch(/\/home\/|\/run\/|agenix|credentials|worker\.example|config_dir|key_source/) + expect(out).toContain("") + }) + + it("refuses a document that is not seat-capacity/1", () => { + expect(() => convert({}, { schema_version: "seat-capacity/2", seats: [] })).toThrow(SeatsFormatError) + expect(() => convert({}, null)).toThrow(SeatsFormatError) + }) + + it("the admit rule on the converted seats: model-scoped refusal, stale refusal, estimated refusal, slot admission", () => { + const seats = bySeat(convert({ owners: { codex: "tom" }, slots: { halogen: 1 } }).snapshot) + const at = NOW + expect(admitSeat("cc", seats.cc, { model: "claude-fable-5-1", min_headroom_pct: 0 }, at)).toMatchObject({ admit: true }) + // cc2's reading is 1505 s old at NOW: STALE, which refuses and asks for a fresh read. + expect(admitSeat("cc2", seats.cc2, { model: "claude-opus-5", min_headroom_pct: 0 }, at)).toMatchObject({ admit: false, reason: "stale", raise_demand: true }) + // The same reading when fresh: Fable refused by its own window, Opus admitted. + const fresh = { ...seats.cc2, observed_at: "2026-09-23T20:00:00Z" } + expect(admitSeat("cc2", fresh, { model: "claude-fable-5-1", min_headroom_pct: 0 }, at).admit).toBe(false) + expect(admitSeat("cc2", fresh, { model: "claude-opus-5", min_headroom_pct: 0 }, at)).toMatchObject({ admit: true }) + expect(admitSeat("pi-qwencloud", seats["pi-qwencloud"], { model: null, min_headroom_pct: 0 }, at)).toMatchObject({ admit: false, reason: "estimated" }) + expect(admitSeat("halogen", seats.halogen, { model: null, min_headroom_pct: 0 }, at)).toMatchObject({ admit: true }) + }) +}) + +describe("the refresh policy the pusher reads under", () => { + const T0 = Date.parse("2026-09-23T20:00:00Z") + const S = 1000 + const read = (state, nowMs, extra = {}) => tick(state, { nowMs, config: {}, runOracle: async () => v1(), post: async () => ({ report: { results: [] } }), ...extra }) + + it("first read at once; never again within 120 s; idle every 900 s", async () => { + expect(planRead(initialGentleState(), { nowMs: T0 })).toMatchObject({ due: true, reason: "first-read" }) + const first = await read(initialGentleState(), T0) + expect(first.events.map((e) => e.event)).toEqual(["read", "pushed"]) + expect(planRead(first.state, { nowMs: T0 + 60 * S })).toMatchObject({ due: false, reason: "min-spacing" }) + expect(planRead(first.state, { nowMs: T0 + 60 * S, preDispatch: true })).toMatchObject({ due: false, reason: "min-spacing" }) + expect(planRead(first.state, { nowMs: T0 + 899 * S })).toMatchObject({ due: false, reason: "fresh" }) + expect(planRead(first.state, { nowMs: T0 + 900 * S })).toMatchObject({ due: true, reason: "idle" }) + }) + + it("active every 300 s (a demand marker, or readings that move); 180 s near a cap; pre-dispatch after 300 s", async () => { + const first = await read(initialGentleState(), T0) + expect(planRead(first.state, { nowMs: T0 + 300 * S, demandAtMs: T0 + 250 * S })).toMatchObject({ due: true, reason: "active" }) + expect(planRead(first.state, { nowMs: T0 + 299 * S, demandAtMs: T0 + 250 * S }).due).toBe(false) + expect(planRead(first.state, { nowMs: T0 + 301 * S, preDispatch: true })).toMatchObject({ due: true, reason: "pre-dispatch" }) + // Two reads whose utilization differs: the box is active without a marker. cc2's 7d is 76 %, below 85. + const moved = v1() + moved.seats[0].windows[1].used_pct = 7.0 + const second = await read(first.state, T0 + 900 * S, { runOracle: async () => moved }) + expect(planRead(second.state, { nowMs: T0 + 1200 * S })).toMatchObject({ due: true, reason: "active" }) + const hot = v1() + hot.seats[1].windows[1].used_pct = 90.0 + const third = await read(second.state, T0 + 1200 * S, { runOracle: async () => hot }) + expect(planRead(third.state, { nowMs: T0 + 1380 * S })).toMatchObject({ due: true, reason: "active" }) + expect(planRead(third.state, { nowMs: T0 + 1379 * S }).due).toBe(false) + }) + + it("an unchanged reading is not re-posted until the heartbeat", async () => { + const claudeOnly = { config: { seats: ["cc", "cc2"] } } + const first = await read(initialGentleState(), T0, claudeOnly) + const again = await read(first.state, T0 + 900 * S, claudeOnly) + expect(again.events.map((e) => e.event)).toEqual(["read", "not-pushed"]) + const beat = await read(again.state, T0 + 1800 * S, claudeOnly) + expect(beat.events.at(-1)).toMatchObject({ event: "pushed", reason: "heartbeat" }) + // A published Halogen slot row beats every 600 s, so the floor never grades it STALE (1200 s). + const slots = await read((await read(initialGentleState(), T0)).state, T0 + 900 * S) + expect(slots.events.at(-1)).toMatchObject({ event: "pushed", reason: "heartbeat" }) + }) +}) + +describe("stale and failed reads", () => { + const T0 = Date.parse("2026-09-23T20:00:00Z") + const S = 1000 + + it("a failed oracle run posts nothing, keeps the 120 s spacing, and the next due read recovers", async () => { + const posts = [] + const failed = await tick(initialGentleState(), { nowMs: T0, config: {}, runOracle: async () => { throw new Error("the oracle failed: 1") }, post: async (s) => posts.push(s) }) + expect(failed.events.at(-1)).toMatchObject({ event: "read-failed" }) + expect(posts).toEqual([]) + expect(planRead(failed.state, { nowMs: T0 + 119 * S })).toMatchObject({ due: false, reason: "min-spacing" }) + const wrong = await tick(failed.state, { nowMs: T0 + 120 * S, config: {}, runOracle: async () => ({ schema_version: "nope" }), post: async (s) => posts.push(s) }) + expect(wrong.events.at(-1)).toMatchObject({ event: "read-failed" }) + const ok = await tick(wrong.state, { nowMs: T0 + 240 * S, config: {}, runOracle: async () => v1(), post: async (s) => { posts.push(s); return {} } }) + expect(ok.events.map((e) => e.event)).toEqual(["read", "pushed"]) + expect(posts).toHaveLength(1) + }) + + it("a seat the oracle stops reporting is named, not carried forward", async () => { + const doc = v1() + doc.seats = doc.seats.filter((seat) => seat.id !== "cc") + const r = await tick(initialGentleState(), { nowMs: T0, config: {}, runOracle: async () => doc, post: async () => ({}) }) + expect(r.events[0].skipped).toEqual([{ seat: "cc", reason: "the oracle did not report this seat" }]) + expect(r.state.last.seats.map((seat) => seat.seat)).not.toContain("cc") + }) + + it("a failed post is retried with the same snapshot after 60 s, without a new read", async () => { + let reads = 0 + const runOracle = async () => { reads++; return v1() } + const down = await tick(initialGentleState(), { nowMs: T0, config: {}, runOracle, post: async () => { throw new Error("answered 503") } }) + expect(down.events.at(-1)).toMatchObject({ event: "push-failed" }) + const early = await tick(down.state, { nowMs: T0 + 30 * S, config: {}, runOracle, post: async () => ({}) }) + expect(early.events).toEqual([expect.objectContaining({ event: "wait", reason: "push-retry" })]) + const sent = [] + const retry = await tick(down.state, { nowMs: T0 + 60 * S, config: {}, runOracle, post: async (s) => { sent.push(s); return {} } }) + expect(retry.events[0]).toMatchObject({ event: "pushed", reason: "retry" }) + expect(sent[0]).toEqual(down.state.pending.snapshot) + expect(reads).toBe(1) + }) +}) + +describe("the pusher never calls a usage endpoint itself", () => { + it("its modules name no provider endpoint and do not import the legacy direct reader", () => { + for (const file of ["seatsOracle.mjs", "gentle.mjs"]) { + const text = readFileSync(join(SRC, file), "utf8") + expect(text, file).not.toMatch(/api\.anthropic\.com|oauth\/usage|usageSource/) + } + const bin = readFileSync(BIN, "utf8") + expect(bin).not.toMatch(/api\.anthropic\.com|oauth\/usage|usageSource|seatCapacity\.mjs"/) + }) +}) + +describe("the process", () => { + const dirs = [] + afterEach(() => { for (const d of dirs.splice(0)) rmSync(d, { recursive: true, force: true }) }) + const run = (args, env = {}) => new Promise((resolve) => { + const child = spawn(process.execPath, [BIN, ...args], { env: { ...process.env, ...env }, stdio: ["ignore", "pipe", "pipe"] }) + let out = "", err = "" + child.stdout.on("data", (b) => { out += b }) + child.stderr.on("data", (b) => { err += b }) + child.on("exit", (code) => resolve({ code, out, err, child })) + }) + + it("--once posts to /capacity/snapshots with the bearer, keeps 0600 state, and releases its pidfile", async () => { + const dir = mkdtempSync(join(tmpdir(), "gentle-")); dirs.push(dir) + writeFileSync(join(dir, "token"), "test-token-value\n", { mode: 0o600 }) + const seen = [] + const server = createServer((req, res) => { + let body = "" + req.on("data", (b) => { body += b }) + req.on("end", () => { + seen.push({ method: req.method, url: req.url, auth: req.headers.authorization, body: JSON.parse(body) }) + res.writeHead(200, { "content-type": "application/json" }).end(JSON.stringify({ report: { results: [], changed: true } })) + }) + }) + await new Promise((r) => server.listen(0, "127.0.0.1", r)) + const port = server.address().port + const pid = join(dir, "pusher.pid") + const r = await run(["--once", "--url", `http://127.0.0.1:${port}`, "--token-file", join(dir, "token"), "--seats-json", FIXTURE, + "--state", join(dir, "state.json"), "--pidfile", pid, "--demand-dir", join(dir, "demand"), "--config", join(dir, "absent.json")].slice(0, -2)) + server.close() + expect(r.code, r.err).toBe(0) + expect(seen).toHaveLength(1) + expect([seen[0].method, seen[0].url, seen[0].auth]).toEqual(["POST", "/capacity/snapshots", "Bearer test-token-value"]) + expect(decode(seen[0].body).seats.length).toBe(5) + expect(r.out).not.toContain("test-token-value") + expect(existsSync(pid)).toBe(false) + expect(statSync(join(dir, "state.json")).mode & 0o777).toBe(0o600) + }) + + it("a second pusher refuses while the first holds the pidfile; a stale pidfile is replaced", async () => { + const dir = mkdtempSync(join(tmpdir(), "gentle-")); dirs.push(dir) + const pid = join(dir, "pusher.pid") + const common = ["--dry-run", "--seats-json", FIXTURE, "--pidfile", pid, "--demand-dir", join(dir, "demand")] + const first = spawn(process.execPath, [BIN, ...common], { stdio: "ignore" }) + for (let i = 0; i < 100 && !existsSync(pid); i++) await new Promise((r) => setTimeout(r, 50)) + expect(readFileSync(pid, "utf8").trim()).toBe(String(first.pid)) + const second = await run([...common, "--once"]) + expect(second.code).toBe(3) + expect(second.err).toMatch(/another pusher/) + const exited = new Promise((r) => first.on("exit", r)) + first.kill("SIGTERM") + await exited + expect(existsSync(pid)).toBe(false) + writeFileSync(pid, "999999999\n") + const third = await run([...common, "--once"]) + expect(third.code, third.err).toBe(0) + expect(JSON.parse(third.out.trim().split("\n").at(-1))).toMatchObject({ event: "dry-run" }) + expect(readdirSync(dir)).not.toContain("pusher.pid") + }) +}) diff --git a/pkgs/substrate-apps/src/apps/pusher/test/oracleEnv.test.mjs b/pkgs/substrate-apps/src/apps/pusher/test/oracleEnv.test.mjs new file mode 100644 index 00000000..109aaa09 --- /dev/null +++ b/pkgs/substrate-apps/src/apps/pusher/test/oracleEnv.test.mjs @@ -0,0 +1,50 @@ +// PT-01 (2026-09-24): the pusher runs the `seats` oracle with a substrate-owned peer cache, never a tally-rewrite path. +import { execFileSync } from "node:child_process" +import { chmodSync, mkdtempSync, writeFileSync } from "node:fs" +import { tmpdir } from "node:os" +import { join, resolve } from "node:path" +import { describe, expect, it } from "vitest" +import { oracleEnv, SeatsFormatError } from "../src/seatsOracle.mjs" + +describe("oracleEnv", () => { + it("defaults the peer cache under the substrate state directory", () => { + expect(oracleEnv({}, "/s/state/substrate/", { PATH: "/bin" })).toEqual({ PATH: "/bin", SEATS_PEER_CACHE_DIR: "/s/state/substrate/seats-peer-cache" }) + expect(oracleEnv({ peerCacheDir: undefined }, "/s").SEATS_PEER_CACHE_DIR).toBe("/s/seats-peer-cache") + }) + it("takes a configured directory, and \"inherit\" leaves the oracle's own default alone", () => { + expect(oracleEnv({ peerCacheDir: "/x/cache" }, "/s").SEATS_PEER_CACHE_DIR).toBe("/x/cache") + expect(oracleEnv({ peerCacheDir: "inherit" }, "/s", { SEATS_PEER_CACHE_DIR: "/pre" })).toEqual({ SEATS_PEER_CACHE_DIR: "/pre" }) + expect("SEATS_PEER_CACHE_DIR" in oracleEnv({ peerCacheDir: "inherit" }, "/s", {})).toBe(false) + }) + it("refuses a tally-rewrite path and an empty value", () => { + expect(() => oracleEnv({ peerCacheDir: "/home/u/.local/state/tally-rewrite/meters" }, "/s")).toThrow(SeatsFormatError) + expect(() => oracleEnv({ peerCacheDir: "" }, "/s")).toThrow(/non-empty/) + }) +}) + +describe("substrate-pusher passes the peer cache to the oracle", () => { + const bin = resolve(import.meta.dirname, "../bin/substrate-pusher.mjs") + // A stand-in oracle that reports the directory it was given, as a seat-capacity/1 document is not needed: + // --dry-run --once prints the converted snapshot or fails, and the stand-in writes its env to a file first. + const fake = (dir) => { + const seen = join(dir, "seen") + const path = join(dir, "seats") + writeFileSync(path, `#!/bin/sh\nprintf '%s' "\${SEATS_PEER_CACHE_DIR-UNSET}" > "${seen}"\necho '{}'\n`) + chmodSync(path, 0o755) + return { path, seen } + } + const runOnce = (dir, extra) => { + const f = fake(dir) + try { + execFileSync(process.execPath, [bin, "--dry-run", "--once", "--config", join(dir, "none.json"), "--seats-bin", f.path, "--pidfile", join(dir, "pid"), "--state", join(dir, "st.json"), "--demand-dir", join(dir, "d"), ...extra], { stdio: "pipe", env: { PATH: process.env.PATH, HOME: dir } }) + } catch { /* '{}' is not a seat-capacity/1 document; only the env the oracle saw matters here */ } + return execFileSync("cat", [f.seen], { encoding: "utf8" }) + } + it("defaults to $HOME/.local/state/substrate/seats-peer-cache, and --peer-cache-dir inherit leaves it unset", () => { + const dir = mkdtempSync(join(tmpdir(), "pusher-env-")) + writeFileSync(join(dir, "none.json"), "{}") + expect(runOnce(dir, [])).toBe(join(dir, ".local/state/substrate/seats-peer-cache")) + expect(runOnce(dir, ["--peer-cache-dir", "inherit"])).toBe("UNSET") + expect(runOnce(dir, ["--peer-cache-dir", "~/pc"])).toBe(join(dir, "pc")) + }) +}) diff --git a/pkgs/substrate-apps/src/apps/pusher/test/refreshPolicy.test.mjs b/pkgs/substrate-apps/src/apps/pusher/test/refreshPolicy.test.mjs new file mode 100644 index 00000000..be3385cb --- /dev/null +++ b/pkgs/substrate-apps/src/apps/pusher/test/refreshPolicy.test.mjs @@ -0,0 +1,117 @@ +// The gentle refresh policy (SCOUT.md 5.5), pinned case by case and by property. +import fc from "fast-check" +import { describe, expect, it } from "vitest" + +import { afterAttempt, initialRefreshState, REFRESH_POLICY, refreshDue } from "../src/refreshPolicy.mjs" + +const T0 = Date.parse("2026-09-23T06:00:00Z") +const s = (seconds) => T0 + seconds * 1000 +const ok = (at) => afterAttempt(initialRefreshState(), { kind: "success" }, at, 1) +const due = (state, input) => refreshDue(state, { active: false, preDispatch: false, bindingMaxPct: null, resets: [], credentialsMtimeMs: 1, ...input }) + +describe("refreshDue", () => { + it("P1 the first read is due", () => { + expect(due(initialRefreshState(), { nowMs: T0 })).toMatchObject({ due: true, reason: "first-read" }) + }) + + it("P2 idle: 900 s between reads", () => { + const state = ok(T0) + expect(due(state, { nowMs: s(899) }).due).toBe(false) + expect(due(state, { nowMs: s(900) })).toMatchObject({ due: true, reason: "idle" }) + expect(due(state, { nowMs: s(600) }).next_at).toBe(new Date(s(900)).toISOString()) + }) + + it("P3 active: 300 s, and 180 s once a binding window is at 85 percent", () => { + const state = ok(T0) + expect(due(state, { nowMs: s(299), active: true }).due).toBe(false) + expect(due(state, { nowMs: s(300), active: true }).due).toBe(true) + expect(due(state, { nowMs: s(179), active: true, bindingMaxPct: 85 }).due).toBe(false) + expect(due(state, { nowMs: s(180), active: true, bindingMaxPct: 85 }).due).toBe(true) + expect(due(state, { nowMs: s(180), active: false, bindingMaxPct: 99 }).due).toBe(false) + }) + + it("P4 the hard 120 s minimum holds over every other reason", () => { + const failed = afterAttempt(ok(T0 - 3600_000), { kind: "error" }, T0, 1) + expect(due(failed, { nowMs: s(119), active: true, preDispatch: true }).reason).toBe("min-spacing") + expect(due(failed, { nowMs: s(120), active: true }).due).toBe(true) + }) + + it("P5 a reset boundary is read at resets_at + 90 s", () => { + const state = ok(T0) + const resets = [new Date(s(200)).toISOString()] + expect(due(state, { nowMs: s(289), resets }).due).toBe(false) + expect(due(state, { nowMs: s(290), resets })).toMatchObject({ due: true, reason: "reset-boundary" }) + expect(due(state, { nowMs: s(150), resets }).next_at).toBe(new Date(s(290)).toISOString()) + // a boundary already covered by a later read is not read again + expect(due(ok(s(300)), { nowMs: s(500), resets }).due).toBe(false) + }) + + it("P6 pre-dispatch reads a reading older than 300 s", () => { + const state = ok(T0) + expect(due(state, { nowMs: s(300), preDispatch: true }).due).toBe(false) + expect(due(state, { nowMs: s(301), preDispatch: true })).toMatchObject({ due: true, reason: "pre-dispatch" }) + }) + + it("P7 429: Retry-After or 120 s x 2^k capped at 960 s, reset on success", () => { + let state = initialRefreshState() + const waits = [] + let at = T0 + for (let k = 0; k < 5; k += 1) { + state = afterAttempt(state, { kind: "rate-limited", retryAfterSeconds: null }, at, 1) + const next = Date.parse(state.next_allowed_at) + waits.push((next - at) / 1000) + expect(due(state, { nowMs: next - 1 }).reason).toBe("backoff") + at = next + } + expect(waits).toEqual([120, 240, 480, 960, 960]) + const honoured = afterAttempt(initialRefreshState(), { kind: "rate-limited", retryAfterSeconds: 700 }, T0, 1) + expect(Date.parse(honoured.next_allowed_at) - T0).toBe(700_000) + expect(afterAttempt(state, { kind: "success" }, at, 1).backoff_k).toBe(0) + }) + + it("P8 an auth failure makes no call until the credentials file changes", () => { + const state = afterAttempt(ok(T0 - 3600_000), { kind: "auth-failed" }, T0, 42) + expect(due(state, { nowMs: s(100000), credentialsMtimeMs: 42 }).reason).toBe("auth-failed") + expect(due(state, { nowMs: s(100000), credentialsMtimeMs: 43 }).due).toBe(true) + }) + + it("property: two network attempts are never closer than the hard minimum", () => { + fc.assert( + fc.property( + fc.array( + fc.record({ + step: fc.integer({ min: 1, max: 400 }), + active: fc.boolean(), + pre: fc.boolean(), + pct: fc.option(fc.integer({ min: 0, max: 100 })), + outcome: fc.constantFrom("success", "rate-limited", "error") + }), + { maxLength: 60 } + ), + (steps) => { + let state = initialRefreshState() + let now = T0 + let last = null + for (const step of steps) { + now += step.step * 1000 + const d = due(state, { nowMs: now, active: step.active, preDispatch: step.pre, bindingMaxPct: step.pct }) + if (!d.due) continue + if (last !== null && now - last < REFRESH_POLICY.minSpacingSeconds * 1000) return false + last = now + state = afterAttempt(state, { kind: step.outcome, retryAfterSeconds: null }, now, 1) + } + return true + } + ) + ) + }) + + it("property: next_at is never at or before now when a read is not due", () => { + fc.assert( + fc.property(fc.integer({ min: 0, max: 5000 }), fc.boolean(), fc.integer({ min: -2000, max: 5000 }), (after, active, reset) => { + const d = due(ok(T0), { nowMs: s(after), active, resets: [new Date(s(reset)).toISOString()] }) + return d.due || d.next_at === null || Date.parse(d.next_at) > s(after) + }) + ) + }) +}) diff --git a/pkgs/substrate-apps/src/apps/pusher/test/seatCapacity.test.mjs b/pkgs/substrate-apps/src/apps/pusher/test/seatCapacity.test.mjs new file mode 100644 index 00000000..b53cf5ee --- /dev/null +++ b/pkgs/substrate-apps/src/apps/pusher/test/seatCapacity.test.mjs @@ -0,0 +1,494 @@ +// The seat-capacity pusher: snapshot building from today's meter shapes, the +// refresh policy, the push, and a round trip into the real Factory handler. +import { spawnSync } from "node:child_process" +import { mkdtempSync, readdirSync, readFileSync, rmSync, statSync, writeFileSync, mkdirSync } from "node:fs" +import { tmpdir } from "node:os" +import { join } from "node:path" +import { fileURLToPath } from "node:url" +import { Schema } from "effect" +import { describe, expect, it } from "vitest" + +import { + buildSnapshot, + decidePush, + fingerprint, + readConfig, + readState, + runOnce, + stateAfterPush +} from "../src/seatCapacity.mjs" +import { SeatCapacitySnapshot } from "@substrate/planning/schema/seatCapacity.ts" +import { admitSeat } from "@substrate/planning/capacity/project.ts" +import { + emptyFactory, + artifactBytes, + artifactDigest +} from "../../../packages/factory/test/fixtures.ts" +import { makeFactoryHttpHandler, makeTableArtifactHasher } from "@substrate/factory" + +const decodeSnapshot = Schema.decodeUnknownSync(SeatCapacitySnapshot) +const PUSHER = fileURLToPath(new URL("../bin/seat-pusher.mjs", import.meta.url)) +const EXAMPLE_CONFIG = fileURLToPath(new URL("../seat-capacity.example.json", import.meta.url)) + +const OBSERVED = "2026-09-23T04:52:18.363166+00:00" +const NOW = "2026-09-23T04:53:18.000Z" + +// Shapes as MEASURED on 2026-09-23 (SCOUT.md section 2); values synthetic. +const ccCache = (observed = OBSERVED, fable = 100) => ({ + observed_at: observed, + usage: { + five_hour: { utilization: 8, resets_at: "2026-09-23T08:49:59.526442+00:00", used_dollars: 1.5 }, + seven_day: { utilization: 72, resets_at: "2026-09-23T09:59:59.526461+00:00" }, + limits: [ + { kind: "session", group: "session", percent: 8, severity: "normal", resets_at: "2026-09-23T08:49:59.526442+00:00", scope: null, is_active: false }, + { kind: "weekly_all", group: "weekly", percent: 72, severity: "normal", resets_at: "2026-09-23T09:59:59.526461+00:00", scope: null, is_active: false }, + { kind: "weekly_scoped", group: "weekly", percent: fable, severity: fable >= 100 ? "critical" : "warning", resets_at: "2026-09-23T09:59:59.526615+00:00", scope: { model: { id: null, display_name: "Fable" }, surface: null }, is_active: true } + // An unknown kind ("something_new") is no longer dropped: it withdraws + // model_windows_complete and refuses (successor review r4); that case + // lives in unmappedLimits.test.mjs. + ], + spend: { limit_dollars: 999, used_dollars: 123 } + } +}) + +const claudeMeter = (seat, overrides = {}) => ({ + schema_version: "seat-meter/1", + seat, + owner: "tom", + grade: "MEASURED", + observed_at: "2026-09-23T04:53:00Z", + reading_observed_at: "2026-09-23T04:52:18Z", + reading_age_seconds: 42, + source: { kind: "oauth-usage-endpoint", reader: "stamp-receipt.py window", seat, source_observed_at: "2026-09-23T04:52:18Z" }, + utilization_pct: 8, + weekly_utilization_pct: 72, + window: { + kind: "nested", + primary: { minutes: 300, resets_at: "2026-09-23T08:49:59.526442+00:00", utilization_pct: 8 }, + secondary: { minutes: 10080, resets_at: "2026-09-23T09:59:59.526461+00:00", utilization_pct: 72 } + }, + ...overrides +}) + +const fleet = () => { + const dir = mkdtempSync(join(tmpdir(), "seat-pusher-")) + const meters = join(dir, "meters") + mkdirSync(meters) + const put = (name, value) => writeFileSync(join(meters, name), `${JSON.stringify(value)}\n`) + put("cc.json", claudeMeter("cc")) + put(".window-cache-cc.json", ccCache()) + put("cc2.json", claudeMeter("cc2", { grade: "STALE-MEASURED", stale_reason: "the reader could not be run: TimeoutExpired" })) + put(".window-cache-cc2.json", ccCache(OBSERVED, 81)) + put("cc3.json", claudeMeter("cc3", { + stale_reason: "token expired", + reading_observed_at: "2026-09-11T15:15:00Z", + source: { kind: "oauth-usage-endpoint", seat: "cc3", source_observed_at: "2026-09-11T15:15:00Z" }, + window: { kind: "nested", primary: { minutes: 300, resets_at: "2026-09-11T19:00:00Z", utilization_pct: 12 }, secondary: { minutes: 10080, resets_at: "2026-09-12T11:00:00.199042+00:00", utilization_pct: 80 } } + })) + put("codex.json", { + schema_version: "seat-meter/1", seat: "codex", owner: "third-party", grade: "MEASURED", observed_at: "2026-09-23T04:52:18Z", + source: { kind: "codex-rollout-rate-limits", path: "/elsewhere/rollout.jsonl" }, + window: { kind: "rolling", minutes: 10080, resets_at: "2026-09-28T19:07:31Z", utilization_pct: 3 } + }) + put("pi-qwencloud.json", { + schema_version: "exp002-meter/1", seat: "pi-qwencloud", owner: "tom", observed_at: "2026-09-23T04:52:18Z", + capacity: { grade: "UNKNOWN", reason: "TL-17 is unset" }, + source: { hold_record: "/elsewhere/pi-hold.json", kind: "unavailable" }, + window: "UNKNOWN", window_reason: "the hold record at /home/someone/plan/pi-hold.json is not held" + }) + put("gpu-worker.json", { + schema_version: "seat-meter/1", seat: "gpu-worker", owner: "kernel", observed_at: "2026-09-23T04:47:32.509Z", + holders: 0, capacity: 1, running_grade: "MEASURED", window: { kind: "none" } + }) + put("mechanical.json", { schema_version: "seat-meter/1", seat: "mechanical", owner: "kernel", observed_at: "2026-09-23T04:47:32.509Z", window: { kind: "none" } }) + writeFileSync(join(meters, "broken.json"), "{") + return { dir, meters, capacity: join(dir, "capacity"), state: join(dir, "capacity", "pusher-state.json") } +} + +const config = () => readConfig(EXAMPLE_CONFIG) +const build = (paths, now = NOW) => + buildSnapshot({ metersDir: paths.meters, capacityDir: paths.capacity, config: config(), now, list: readdirSync(paths.meters) }) + +describe("building the snapshot from today's meter shapes", () => { + it("decodes against the lake's schema and classifies every seat", () => { + const paths = fleet() + const { snapshot, skipped } = build(paths) + const decoded = decodeSnapshot(snapshot) + expect(decoded.seats.map((seat) => [seat.seat, seat.provider, seat.dispatchable, seat.dispatchable_reason])).toEqual([ + ["cc", "claude", true, null], + ["cc2", "claude", true, null], + ["cc3", "claude", false, "evicted"], + ["codex", "codex", false, "third-party"], + ["gpu-worker", "halogen", true, null], + ["pi-qwencloud", "qwen", true, null] + ]) + expect(skipped).toEqual([ + { seat: "broken", reason: "unreadable: SyntaxError" }, + { seat: "mechanical", reason: "not a compute seat this pusher knows" } + ]) + }) + + it("carries the model-scoped Fable row from the usage cache, and the provider's instant", () => { + const cc = build(fleet()).snapshot.seats.find((seat) => seat.seat === "cc") + expect(cc.observed_at).toBe(OBSERVED) + expect(cc.windows.map((window) => [window.kind, window.model, window.utilization_pct, window.severity])).toEqual([ + ["five_hour", null, 8, "normal"], + ["seven_day", null, 72, "normal"], + ["model_scoped", "Fable", 100, "critical"] + ]) + }) + + it("never copies spend, or anything else outside limits[], from the cache", () => { + const text = JSON.stringify(build(fleet()).snapshot) + expect(text).not.toMatch(/spend|dollars|123|999/) + expect(text).not.toContain("/elsewhere/") + }) + + it("a STALE-MEASURED row is a MEASURED reading that has aged; the lake judges its age", () => { + const cc2 = build(fleet()).snapshot.seats.find((seat) => seat.seat === "cc2") + expect(cc2.grade).toBe("MEASURED") + expect(cc2.stale_reason).toBe("the reader could not be run: TimeoutExpired") + }) + + it("falls back to the meter row when the cache is older than the reading", () => { + const paths = fleet() + writeFileSync(join(paths.meters, ".window-cache-cc.json"), JSON.stringify(ccCache("2026-09-23T03:00:00Z"))) + const cc = build(paths).snapshot.seats.find((seat) => seat.seat === "cc") + expect(cc.source.detail).toBe("meter row window") + expect(cc.observed_at).toBe("2026-09-23T04:52:18Z") + // The cache's model rows are carried forward as UNKNOWN (review round 1). + expect(cc.windows.map((window) => window.kind)).toEqual(["five_hour", "seven_day", "model_scoped"]) + expect(cc.model_windows_complete).toBe(false) + }) + + it("an auth failure makes a seat not dispatchable even without a config ruling", () => { + const paths = fleet() + // cc3 is ruled out by the built-in rulings; cc2 carries the auth failure here. + writeFileSync(join(paths.meters, "cc2.json"), JSON.stringify(claudeMeter("cc2", { stale_reason: "token expired" }))) + const { snapshot } = buildSnapshot({ + metersDir: paths.meters, capacityDir: null, config: readConfig(null), now: NOW, list: readdirSync(paths.meters) + }) + const cc2 = snapshot.seats.find((seat) => seat.seat === "cc2") + expect([cc2.dispatchable, cc2.dispatchable_reason]).toEqual([false, "auth-failed"]) + expect(snapshot.seats.find((seat) => seat.seat === "pi-qwencloud").plan).toBeNull() + }) + + it("prefers the feeder's own seats.json when it writes one", () => { + const paths = fleet() + mkdirSync(paths.capacity, { recursive: true }) + const written = { ...build(paths).snapshot, host: "from-feeder", seats: [] } + writeFileSync(join(paths.capacity, "seats.json"), JSON.stringify(written)) + expect(build(paths)).toMatchObject({ source: "seats.json", snapshot: { host: "from-feeder" } }) + }) + + it("carries the qwen plan end from the config file", () => { + const qwen = build(fleet()).snapshot.seats.find((seat) => seat.seat === "pi-qwencloud") + expect(qwen.plan).toEqual({ expires_at: "2026-11-07T00:00:00Z" }) + expect(qwen.windows).toEqual([]) + expect(qwen.stale_reason).toBe("the hold record at is not held") + }) +}) + +describe("the refresh policy", () => { + const minute = 60_000 + const at = Date.parse(NOW) + + it("pushes the first time, then only on a changed reading or the 30 minute heartbeat", () => { + const paths = fleet() + const first = build(paths).snapshot + expect(decidePush({}, first, at)).toMatchObject({ push: true, reason: "changed" }) + const state = stateAfterPush(first, NOW) + // A restamp: the meter row's observed_at moves, the reading does not. + writeFileSync(join(paths.meters, "cc.json"), JSON.stringify(claudeMeter("cc", { observed_at: "2026-09-23T04:55:00Z" }))) + const restamped = build(paths, new Date(at + 5 * minute).toISOString()).snapshot + expect(decidePush(state, restamped, at + 5 * minute)).toMatchObject({ push: false, reason: "unchanged" }) + expect(decidePush(state, restamped, at + 30 * minute)).toMatchObject({ push: true, reason: "heartbeat" }) + // A new reading on one seat. + writeFileSync(join(paths.meters, ".window-cache-cc.json"), JSON.stringify(ccCache("2026-09-23T04:57:18Z"))) + const read = build(paths, new Date(at + 5 * minute).toISOString()).snapshot + expect(decidePush(state, read, at + 5 * minute)).toMatchObject({ push: true, reason: "changed", changed: ["cc"] }) + }) + + it("never pushes twice within the minimum interval, and notices a removed seat", () => { + const snapshot = build(fleet()).snapshot + const state = stateAfterPush(snapshot, NOW) + const fewer = { ...snapshot, seats: snapshot.seats.slice(1) } + expect(decidePush(state, fewer, at + 30_000)).toMatchObject({ push: false, reason: "min-interval" }) + expect(decidePush(state, fewer, at + 2 * minute)).toMatchObject({ push: true, changed: ["cc"] }) + }) + + it("the fingerprint ignores how an instant is spelled", () => { + const cc = build(fleet()).snapshot.seats[0] + expect(fingerprint({ ...cc, observed_at: "2026-09-23T06:52:18.363+02:00" })).toBe( + fingerprint({ ...cc, observed_at: "2026-09-23T04:52:18.363Z" }) + ) + }) +}) + +describe("pushing", () => { + const recorder = (status = 200, body = { results: [], changed: true }) => { + const calls = [] + const fetchImpl = async (url, init) => { + calls.push({ url, init }) + return new Response(JSON.stringify(body), { status }) + } + return { calls, fetchImpl } + } + const pass = (paths, fetchImpl, now = NOW) => + runOnce({ + metersDir: paths.meters, capacityDir: paths.capacity, statePath: paths.state, config: config(), + base: "https://lake.invalid/", token: "secret-token-value", now: () => now, + list: readdirSync(paths.meters), fetchImpl + }) + + it("posts once, keeps a 0600 state, and posts nothing on the next pass", async () => { + const paths = fleet() + const { calls, fetchImpl } = recorder() + expect((await pass(paths, fetchImpl)).pushed).toBe(true) + expect(calls).toHaveLength(1) + expect(calls[0].url).toBe("https://lake.invalid/capacity/seats") + expect(calls[0].init.headers.authorization).toBe("Bearer secret-token-value") + expect(statSync(paths.state).mode & 0o777).toBe(0o600) + const second = await pass(paths, fetchImpl, "2026-09-23T04:58:18.000Z") + expect(second).toMatchObject({ pushed: false, decision: { reason: "unchanged" } }) + expect(calls).toHaveLength(1) + }) + + it("a refused push keeps no state and never prints the token", async () => { + const paths = fleet() + const { fetchImpl } = recorder(401, { error: "Unauthorized" }) + const failure = await pass(paths, fetchImpl).then(() => null, (error) => error) + expect(failure.message).toContain("401") + expect(failure.message).not.toContain("secret-token-value") + expect(readState(paths.state)).toEqual({}) + }) + + it("round trip: the pusher's snapshot lands in the real Factory handler and answers admission", async () => { + const paths = fleet() + const factory = await emptyFactory() + const handler = makeFactoryHttpHandler({ + factory, + token: "secret-token-value", + artifactHasher: makeTableArtifactHasher([[artifactBytes, artifactDigest]]), + now: () => NOW + }) + const fetchImpl = (url, init) => handler(new Request(url, init)) + const result = await pass(paths, fetchImpl) + expect(result.results.every((entry) => entry.outcome === "accepted")).toBe(true) + const ask = async (query) => (await handler(new Request(`https://lake.invalid/capacity/admit?${query}`))).json() + expect(await ask("seat=cc&model=Opus")).toMatchObject({ admit: true, headroom_pct: 28 }) + expect(await ask("seat=cc&model=Fable")).toMatchObject({ admit: false, reason: "severity-critical" }) + expect(await ask("seat=cc2&model=Fable")).toMatchObject({ admit: true, signal: "SLOW" }) + expect(await ask("seat=cc3&model=Opus")).toMatchObject({ admit: false, reason: "not-dispatchable" }) + expect(await ask("seat=codex")).toMatchObject({ admit: false, reason: "not-dispatchable" }) + expect(await ask("seat=pi-qwencloud")).toMatchObject({ admit: false, reason: "unknown" }) + expect(await ask("seat=gpu-worker")).toMatchObject({ admit: true }) + // A repeat pass with --force is idempotent at the lake. + const again = await runOnce({ + metersDir: paths.meters, capacityDir: paths.capacity, statePath: paths.state, config: config(), + base: "https://lake.invalid", token: "secret-token-value", now: () => NOW, + list: readdirSync(paths.meters), fetchImpl, force: true + }) + expect(again.results.every((entry) => entry.outcome === "duplicate")).toBe(true) + }) +}) + +describe("the CLI", () => { + it("--dry-run prints the snapshot and touches no network and no state", () => { + const paths = fleet() + const run = spawnSync(process.execPath, [PUSHER, "--dry-run", "--meters", paths.meters, "--capacity-dir", "", "--config", EXAMPLE_CONFIG], { encoding: "utf8" }) + expect(run.status).toBe(0) + const out = JSON.parse(run.stdout) + expect(out.pushed).toBe(false) + expect(out.snapshot.schema_version).toBe("seat-capacity/2") + expect(out.snapshot.host).toBe("coordinator") + expect(() => statSync(paths.state)).toThrow() + }) + + it("refuses to run without a url and a token file", () => { + const run = spawnSync(process.execPath, [PUSHER], { encoding: "utf8" }) + expect(run.status).toBe(2) + expect(run.stderr).toContain("--url and --token-file are required") + }) +}) + +describe("review round 1: the pusher never drops a model limit silently", () => { + const FABLE = { model: "Fable", min_headroom_pct: 0 } + const ANY = { model: null, min_headroom_pct: 0 } + const AT = "2026-09-23T04:53:30Z" + const ccOf = (paths) => decodeSnapshot(build(paths).snapshot).seats.find((seat) => seat.seat === "cc") + + it("the usage cache read from the same provider call states every model limit", () => { + const cc = ccOf(fleet()) + expect(cc.model_windows_complete).toBe(true) + expect(admitSeat("cc", cc, { model: "Opus", min_headroom_pct: 0 }, AT).admit).toBe(true) + }) + + it("a meter row newer than the cache carries the Fable row forward as UNKNOWN and refuses Fable", () => { + const paths = fleet() + // The cache (and its Fable row at 100 percent critical) is from 04:40; the + // meter row's provider read is 04:52:18 (the feeder tests' shape). + writeFileSync(join(paths.meters, ".window-cache-cc.json"), JSON.stringify(ccCache("2026-09-23T04:40:00Z"))) + const cc = ccOf(paths) + expect(cc.model_windows_complete).toBe(false) + expect(cc.windows.find((window) => window.kind === "model_scoped")).toMatchObject({ + model: "Fable", + utilization_pct: null, + severity: null, + grade: "UNKNOWN" + }) + expect(admitSeat("cc", cc, FABLE, AT)).toMatchObject({ admit: false, reason: "window-unknown", raise_demand: true }) + expect(admitSeat("cc", cc, ANY, AT).admit).toBe(true) + }) + + it("a cache 1 s behind the meter row refuses Fable", () => { + const paths = fleet() + writeFileSync(join(paths.meters, ".window-cache-cc.json"), JSON.stringify(ccCache("2026-09-23T04:52:17Z"))) + writeFileSync(join(paths.meters, "cc.json"), JSON.stringify(claudeMeter("cc", { + reading_observed_at: "2026-09-23T04:52:18Z", + source: { kind: "oauth-usage-endpoint", source_observed_at: "2026-09-23T04:52:18Z" } + }))) + expect(admitSeat("cc", ccOf(paths), FABLE, AT)).toMatchObject({ admit: false, reason: "window-unknown" }) + }) + + it("a torn (unparseable) or missing cache refuses every model job and still admits a job for no model", () => { + for (const cache of ['{"observed_at": "2026-09-23T04:52:18.365', null]) { + const paths = fleet() + const cachePath = join(paths.meters, ".window-cache-cc.json") + if (cache === null) rmSync(cachePath) + else writeFileSync(cachePath, cache) + const cc = ccOf(paths) + expect(cc.model_windows_complete).toBe(false) + expect(cc.windows.map((window) => window.kind)).toEqual(["five_hour", "seven_day"]) + expect(admitSeat("cc", cc, FABLE, AT)).toMatchObject({ admit: false, reason: "window-unknown" }) + expect(admitSeat("cc", cc, { model: "Opus", min_headroom_pct: 0 }, AT)).toMatchObject({ admit: false, reason: "window-unknown" }) + expect(admitSeat("cc", cc, ANY, AT).admit).toBe(true) + } + }) +}) + +describe("review round 1: the ruled seats are never dispatchable, config or not", () => { + const withCoordinatorGpu = () => { + const paths = fleet() + writeFileSync(join(paths.meters, "gpu-coordinator.json"), JSON.stringify({ + schema_version: "seat-meter/1", seat: "gpu-coordinator", owner: "kernel", observed_at: "2026-09-23T04:52:18Z", + holders: 0, capacity: 1, running_grade: "MEASURED", window: { kind: "none" } + })) + // cc3 after a re-login: no auth failure left to catch it. + writeFileSync(join(paths.meters, "cc3.json"), JSON.stringify(claudeMeter("cc3"))) + writeFileSync(join(paths.meters, ".window-cache-cc3.json"), JSON.stringify(ccCache())) + return paths + } + const dispatch = (snapshot) => + Object.fromEntries(snapshot.seats.map((seat) => [seat.seat, [seat.dispatchable, seat.dispatchable_reason]])) + + it("with no config file, cc3 and gpu-coordinator are published not dispatchable and refused", () => { + const paths = withCoordinatorGpu() + const { snapshot } = buildSnapshot({ + metersDir: paths.meters, capacityDir: null, config: readConfig(null), now: NOW, list: readdirSync(paths.meters) + }) + const seats = decodeSnapshot(snapshot).seats + expect(dispatch(snapshot)["cc3"]).toEqual([false, "evicted"]) + expect(dispatch(snapshot)["gpu-coordinator"][0]).toBe(false) + for (const id of ["cc3", "gpu-coordinator"]) { + expect(admitSeat(id, seats.find((seat) => seat.seat === id), { model: null, min_headroom_pct: 0 }, NOW)).toMatchObject({ + admit: false, + reason: "not-dispatchable" + }) + } + }) + + it("a config can extend the rulings but never lift one", () => { + const dir = mkdtempSync(join(tmpdir(), "seat-config-")) + const path = join(dir, "seat-capacity.json") + writeFileSync(path, JSON.stringify({ not_dispatchable: { cc3: null, "gpu-coordinator": false, cc2: "paused" } })) + const ruled = readConfig(path).notDispatchable + expect(typeof ruled.cc3).toBe("string") + expect(typeof ruled["gpu-coordinator"]).toBe("string") + expect(ruled.cc2).toBe("paused") + }) + + it("the feeder's seats.json is held to the same rulings, seat filter and path scrub", () => { + const paths = fleet() + mkdirSync(paths.capacity, { recursive: true }) + const built = build(paths).snapshot + const written = { + ...built, + seats: built.seats.map((seat) => + seat.seat === "cc3" + ? { ...seat, dispatchable: true, dispatchable_reason: null, stale_reason: "read /home/tom/.claude/x failed" } + : seat + ) + } + writeFileSync(join(paths.capacity, "seats.json"), JSON.stringify(written)) + const cfg = { ...readConfig(EXAMPLE_CONFIG), seats: ["cc", "cc3"] } + const { snapshot, source } = buildSnapshot({ + metersDir: paths.meters, capacityDir: paths.capacity, config: cfg, now: NOW, list: readdirSync(paths.meters) + }) + expect(source).toBe("seats.json") + expect(snapshot.seats.map((seat) => seat.seat)).toEqual(["cc", "cc3"]) + const cc3 = snapshot.seats.find((seat) => seat.seat === "cc3") + expect([cc3.dispatchable, cc3.dispatchable_reason]).toEqual([false, "evicted"]) + expect(cc3.stale_reason).toBe("read failed") + expect(() => decodeSnapshot(snapshot)).not.toThrow() + }) + + it("--dry-run reads the same default config path as a real run", () => { + const paths = fleet() + const home = mkdtempSync(join(tmpdir(), "seat-home-")) + mkdirSync(join(home, ".config", "substrate"), { recursive: true }) + writeFileSync(join(home, ".config", "substrate", "seat-capacity.json"), JSON.stringify({ host: "from-default-config" })) + const run = spawnSync(process.execPath, [PUSHER, "--dry-run", "--meters", paths.meters, "--capacity-dir", ""], { + encoding: "utf8", + env: { ...process.env, HOME: home } + }) + expect(run.status).toBe(0) + expect(JSON.parse(run.stdout).snapshot.host).toBe("from-default-config") + }) +}) + +describe("review round 1: a restamp is not a change for any provider", () => { + const minute = 60_000 + const at = Date.parse(NOW) + const restampAll = (paths, instant) => { + for (const name of ["codex.json", "pi-qwencloud.json", "gpu-worker.json"]) { + const path = join(paths.meters, name) + const row = JSON.parse(readFileSync(path, "utf8")) + writeFileSync(path, JSON.stringify({ ...row, observed_at: instant })) + } + } + + it("moving only the codex, qwen and halogen rows' observed_at does not push", () => { + const paths = fleet() + const first = build(paths).snapshot + const state = stateAfterPush(first, NOW) + restampAll(paths, "2026-09-23T04:57:48Z") + const later = build(paths, new Date(at + 5 * minute).toISOString()).snapshot + expect(decidePush(state, later, at + 5 * minute)).toMatchObject({ push: false, reason: "unchanged" }) + // No provider instant exists in those rows; the snapshot says so. + for (const id of ["codex", "pi-qwencloud"]) { + expect(later.seats.find((seat) => seat.seat === id).source.detail).toBe("observed_at is the feeder's read instant") + } + }) + + it("a changed codex utilization or halogen holder count still pushes", () => { + const paths = fleet() + const state = stateAfterPush(build(paths).snapshot, NOW) + const path = join(paths.meters, "gpu-worker.json") + writeFileSync(path, JSON.stringify({ ...JSON.parse(readFileSync(path, "utf8")), holders: 1 })) + expect(decidePush(state, build(paths).snapshot, at + 2 * minute)).toMatchObject({ push: true, changed: ["gpu-worker"] }) + }) + + it("a dispatchable slot row is re-sent before the lake would grade it STALE (1200 s)", () => { + const paths = fleet() + const state = stateAfterPush(build(paths).snapshot, NOW) + restampAll(paths, "2026-09-23T05:03:18Z") + const snapshot = build(paths).snapshot + expect(decidePush(state, snapshot, at + 9 * minute)).toMatchObject({ push: false }) + expect(decidePush(state, snapshot, at + 10 * minute)).toMatchObject({ push: true, reason: "heartbeat" }) + // Without a dispatchable slot row, the 1800 s heartbeat stands. + const noSlots = { ...snapshot, seats: snapshot.seats.filter((seat) => seat.provider !== "halogen") } + const stateNoSlots = stateAfterPush({ ...build(paths).snapshot, seats: noSlots.seats }, NOW) + expect(decidePush(stateNoSlots, noSlots, at + 10 * minute)).toMatchObject({ push: false, reason: "unchanged" }) + }) +}) diff --git a/pkgs/substrate-apps/src/apps/pusher/test/unmappedLimits.test.mjs b/pkgs/substrate-apps/src/apps/pusher/test/unmappedLimits.test.mjs new file mode 100644 index 00000000..b7bd1007 --- /dev/null +++ b/pkgs/substrate-apps/src/apps/pusher/test/unmappedLimits.test.mjs @@ -0,0 +1,32 @@ +// Successor review r4: the pusher placed limit rows the way capacity round 3 +// placed them in src/capacity/tally-meters.ts. A row it cannot place (an +// unknown kind, a scoped row naming no model) is a binding seven_day window +// graded UNKNOWN with its reset kept, and model_windows_complete is withdrawn. +import { describe, expect, it } from "vitest" +import { windowsFromLimits, windowsFromUsage, limitsComplete } from "../src/usageSource.mjs" + +const reset = "2026-09-23T09:59:59Z" +const limits = [ + { kind: "session", percent: 10, resets_at: reset, severity: "normal", scope: null }, + { kind: "weekly_all", percent: 20, resets_at: reset, severity: "normal", scope: null }, + { kind: "weekly_scoped", percent: 100, resets_at: reset, severity: "critical", scope: { model: null, surface: "claude_code" } }, + { kind: "something_new", percent: 100, resets_at: reset, severity: "critical", scope: null } +] + +describe("r4: the pusher never drops a limit row it cannot place", () => { + it("unknown kind and a model-less scoped row become UNKNOWN seven_day windows with their resets", () => { + const { windows, unmapped } = windowsFromLimits(limits) + expect(unmapped).toBe(2) + const unknown = windows.filter((w) => w.grade === "UNKNOWN") + expect(unknown).toHaveLength(2) + for (const w of unknown) { + expect(w).toMatchObject({ kind: "seven_day", model: null, binding: true, utilization_pct: null, resets_at: reset }) + } + expect(windowsFromUsage({ limits })).toEqual(windows) + }) + it("model_windows_complete is claimed only when every row was placed", () => { + expect(limitsComplete({ limits })).toBe(false) + expect(limitsComplete({ limits: limits.slice(0, 2) })).toBe(true) + expect(limitsComplete({ five_hour: {}, seven_day: {} })).toBe(false) + }) +}) diff --git a/pkgs/substrate-apps/src/apps/pusher/test/usageSource.test.mjs b/pkgs/substrate-apps/src/apps/pusher/test/usageSource.test.mjs new file mode 100644 index 00000000..e9fad6b9 --- /dev/null +++ b/pkgs/substrate-apps/src/apps/pusher/test/usageSource.test.mjs @@ -0,0 +1,298 @@ +// The Claude usage source: the direct reader under the refresh policy, the +// cache-file adapter, and a snapshot built with no tally path at all. +import { spawnSync } from "node:child_process" +import { mkdirSync, mkdtempSync, readFileSync, statSync, utimesSync, writeFileSync } from "node:fs" +import { tmpdir } from "node:os" +import { join } from "node:path" +import { fileURLToPath } from "node:url" +import { Schema } from "effect" +import { describe, expect, it } from "vitest" + +import { SeatCapacitySnapshot } from "@substrate/planning/schema/seatCapacity.ts" +import { admitSeat } from "@substrate/planning/capacity/project.ts" +import { buildSnapshot, defaultConfigPath, defaultPusherDir, readConfig, runOnce } from "../src/seatCapacity.mjs" +import { + bearerFromCredentials, + defaultUsageDir, + readSeatUsage, + USAGE_ENDPOINT, + windowsFromUsage +} from "../src/usageSource.mjs" + +const decode = Schema.decodeUnknownSync(SeatCapacitySnapshot) +const PUSHER = fileURLToPath(new URL("../bin/seat-pusher.mjs", import.meta.url)) +const FAKE = "fake-bearer-for-tests-only" +const T0 = Date.parse("2026-09-23T06:00:00Z") + +// The endpoint's answer shape as MEASURED in the feeder cache (SCOUT.md section 2); values synthetic. +const answer = (fable = 100) => ({ + five_hour: { utilization: 8, resets_at: "2026-09-23T08:49:59.526442+00:00" }, + seven_day: { utilization: 72, resets_at: "2026-09-23T09:59:59.526461+00:00" }, + limits: [ + { kind: "session", percent: 8, severity: "normal", resets_at: "2026-09-23T08:49:59.526442+00:00", scope: null }, + { kind: "weekly_all", percent: 72, severity: "normal", resets_at: "2026-09-23T09:59:59.526461+00:00", scope: null }, + { kind: "weekly_scoped", percent: fable, severity: "critical", resets_at: "2026-09-23T09:59:59.526615+00:00", scope: { model: { id: null, display_name: "Fable" } } } + ], + spend: { limit_dollars: 999, used_dollars: 123 } +}) + +const box = () => { + const dir = mkdtempSync(join(tmpdir(), "usage-source-")) + const credentials = join(dir, "credentials.json") + writeFileSync(credentials, JSON.stringify({ claudeAiOauth: { accessToken: FAKE } })) + return { dir, credentials, usageDir: join(dir, "usage"), demandDir: join(dir, "demand") } +} + +const fakeFetch = (responses) => { + const calls = [] + const impl = async (url, init) => { + calls.push({ url, init }) + const next = responses.shift() ?? { status: 200, body: answer() } + return { + status: next.status, + headers: { get: (name) => (name === "retry-after" ? (next.retryAfter ?? null) : null) }, + text: async () => JSON.stringify(next.body ?? {}) + } + } + return { impl, calls } +} + +const read = (b, fetchImpl, nowMs, extra = {}) => + readSeatUsage({ + seat: "cc", + source: { kind: "oauth", credentials: b.credentials }, + usageDir: b.usageDir, + demandDir: b.demandDir, + nowMs, + fetchImpl, + ...extra + }) + +describe("windowsFromUsage", () => { + it("maps limits[] to five_hour, seven_day and a model-scoped row", () => { + const windows = windowsFromUsage(answer()) + expect(windows.map((w) => [w.kind, w.model, w.utilization_pct])).toEqual([ + ["five_hour", null, 8], + ["seven_day", null, 72], + ["model_scoped", "Fable", 100] + ]) + }) + it("falls back to the flat pair when limits[] is absent", () => { + const { limits, ...flat } = answer() + expect(windowsFromUsage(flat).map((w) => [w.kind, w.utilization_pct])).toEqual([ + ["five_hour", 8], + ["seven_day", 72] + ]) + }) +}) + +describe("bearerFromCredentials", () => { + it("reads the pointer and never puts the value in an error", () => { + const b = box() + expect(bearerFromCredentials(b.credentials)).toBe(FAKE) + let message = "" + try { + bearerFromCredentials(b.credentials, ["claudeAiOauth", "refreshToken"]) + } catch (error) { + message = error.message + } + expect(message).toMatch(/refreshToken/) + expect(message).not.toContain(FAKE) + }) +}) + +describe("the direct reader", () => { + it("reads once, sends the bearer only in the header, caches 0600 without spend", async () => { + const b = box() + const f = fakeFetch([{ status: 200, body: answer() }]) + const r = await read(b, f.impl, T0) + expect(r.attempted).toBe(true) + expect(f.calls).toHaveLength(1) + expect(f.calls[0].url).toBe(USAGE_ENDPOINT) + expect(f.calls[0].init.headers.authorization).toBe(`Bearer ${FAKE}`) + expect(f.calls[0].init.headers["anthropic-beta"]).toBe("oauth-2025-04-20") + const cache = readFileSync(join(b.usageDir, "cc.json"), "utf8") + expect(cache).not.toContain("spend") + expect(cache).not.toContain(FAKE) + expect(readFileSync(join(b.usageDir, "cc.state.json"), "utf8")).not.toContain(FAKE) + expect(statSync(join(b.usageDir, "cc.json")).mode & 0o777).toBe(0o600) + expect(r.reading.observed_at).toBe(new Date(T0).toISOString()) + }) + + it("is gentle: no second call inside the idle floor, one at 900 s, 300 s when active", async () => { + const b = box() + const f = fakeFetch([]) + await read(b, f.impl, T0) + for (const after of [30, 119, 120, 600, 899]) await read(b, f.impl, T0 + after * 1000) + expect(f.calls).toHaveLength(1) + const idle = await read(b, f.impl, T0 + 900_000) + expect(idle.decision.reason).toBe("idle") + expect(f.calls).toHaveLength(2) + mkdirSync(b.demandDir, { recursive: true }) + writeFileSync(join(b.demandDir, "cc"), "") + const markerAt = (T0 + 1_200_000) / 1000 + utimesSync(join(b.demandDir, "cc"), markerAt, markerAt) + const active = await read(b, f.impl, T0 + 1_200_000) + expect(active.decision.reason).toBe("active") + expect(f.calls).toHaveLength(3) + }) + + it("backs off after a 429, honouring Retry-After, and keeps serving the cached reading", async () => { + const b = box() + const f = fakeFetch([{ status: 200, body: answer() }, { status: 429, retryAfter: "1500" }]) + await read(b, f.impl, T0) + const limited = await read(b, f.impl, T0 + 900_000) + expect(limited.attempted).toBe(true) + expect(limited.error).toMatch(/429/) + expect(limited.reading.observed_at).toBe(new Date(T0).toISOString()) + await read(b, f.impl, T0 + (900 + 1499) * 1000) + expect(f.calls).toHaveLength(2) + await read(b, f.impl, T0 + (900 + 1500) * 1000) + expect(f.calls).toHaveLength(3) + }) + + it("after an auth failure, calls again only once the credentials file changes", async () => { + const b = box() + const f = fakeFetch([{ status: 401 }]) + const failed = await read(b, f.impl, T0) + expect(failed.error).toMatch(/401/) + await read(b, f.impl, T0 + 86_400_000) + expect(f.calls).toHaveLength(1) + const later = (T0 + 90_000_000) / 1000 + utimesSync(b.credentials, later, later) + await read(b, f.impl, T0 + 90_000_000) + expect(f.calls).toHaveLength(2) + }) + + it("a dry run never calls the network, even with a first read due", async () => { + const b = box() + const f = fakeFetch([]) + const config = readConfig(null) + config.usage = { cc: { kind: "oauth", credentials: b.credentials } } + const dry = await runOnce({ + metersDir: null, + capacityDir: null, + statePath: join(b.dir, "state.json"), + config, + base: "", + token: "", + now: () => new Date(T0).toISOString(), + fetchImpl: f.impl, + dryRun: true, + usageDir: b.usageDir, + demandDir: b.demandDir + }) + expect(f.calls).toHaveLength(0) + expect(dry.skipped.map((entry) => entry.seat)).toEqual(["cc"]) + }) + + it("never calls the network when told not to", async () => { + const b = box() + const f = fakeFetch([]) + const r = await read(b, f.impl, T0, { allowNetwork: false }) + expect(f.calls).toHaveLength(0) + expect(r.reading).toBeNull() + }) +}) + +describe("the snapshot without tally-rewrite", () => { + it("builds from usage sources alone, decodes, and admission honours the model-scoped row", async () => { + const b = box() + const cacheFile = join(b.dir, "feeder-cache.json") + writeFileSync(cacheFile, JSON.stringify({ observed_at: "2026-09-23T05:59:00Z", usage: answer(81) })) + const config = readConfig(null) + config.host = "coordinator" + config.usage = { cc: { kind: "oauth", credentials: b.credentials }, cc2: { kind: "cache-file", path: cacheFile } } + const f = fakeFetch([{ status: 200, body: answer(100) }]) + const result = await runOnce({ + metersDir: null, + capacityDir: null, + statePath: join(b.dir, "state.json"), + config, + base: "http://floor.invalid", + token: "floor-token", + now: () => new Date(T0).toISOString(), + fetchImpl: f.impl, + dryRun: false, + usageDir: b.usageDir, + demandDir: b.demandDir + }).catch((error) => ({ error: error.message })) + // The push itself fails against the fake (it answers usage JSON, not the floor's); + // what matters here is the usage read, below, and the dry-run snapshot. + expect(f.calls[0].url).toBe(USAGE_ENDPOINT) + const dry = await runOnce({ + metersDir: null, + capacityDir: null, + statePath: join(b.dir, "state.json"), + config, + base: "", + token: "", + now: () => new Date(T0 + 1000).toISOString(), + fetchImpl: f.impl, + dryRun: true, + usageDir: b.usageDir, + demandDir: b.demandDir + }) + expect(result).toBeDefined() + expect(f.calls).toHaveLength(2) // the failed push, never a second usage read + expect(dry.source).toBe("usage") + const snapshot = decode(dry.snapshot) + expect(snapshot.seats.map((seat) => [seat.seat, seat.source.detail])).toEqual([ + ["cc", "direct read"], + ["cc2", "usage cache file"] + ]) + expect(JSON.stringify(snapshot)).not.toContain("spend") + const asOf = new Date(T0 + 2000).toISOString() + const cc = snapshot.seats[0] + expect(admitSeat("cc", cc, { model: "opus-class", min_headroom_pct: 0 }, asOf).admit).toBe(true) + expect(admitSeat("cc", cc, { model: "Fable", min_headroom_pct: 0 }, asOf).admit).toBe(false) + expect(cc.model_windows_complete).toBe(true) + }) + + it("a reading without limits[] does not claim every model limit, so a model job is refused", () => { + const b = box() + const { limits, ...flat } = answer() + const cacheFile = join(b.dir, "flat.json") + writeFileSync(cacheFile, JSON.stringify({ observed_at: new Date(T0).toISOString(), usage: flat })) + const config = readConfig(null) + config.usage = { cc: { kind: "cache-file", path: cacheFile } } + const usage = { cc: { reading: { observed_at: new Date(T0).toISOString(), usage: flat }, error: null } } + const seat = decode(buildSnapshot({ config, now: new Date(T0).toISOString(), usage }).snapshot).seats[0] + expect(seat.model_windows_complete).toBe(false) + const asOf = new Date(T0 + 1000).toISOString() + const refused = admitSeat("cc", seat, { model: "Fable", min_headroom_pct: 0 }, asOf) + expect(refused.admit).toBe(false) + expect(refused.reason).toBe("window-unknown") + expect(admitSeat("cc", seat, { model: null, min_headroom_pct: 0 }, asOf).admit).toBe(true) + }) + + it("the example configs name no real credentials path (tests load the first one)", () => { + const example = fileURLToPath(new URL("../seat-capacity.example.json", import.meta.url)) + expect(readConfig(example).usage).toEqual({}) + const usageExample = readConfig(fileURLToPath(new URL("../seat-capacity.usage.example.json", import.meta.url))) + for (const source of Object.values(usageExample.usage)) { + for (const value of Object.values(source)) expect(String(value).startsWith("/")).toBe(false) + } + }) + + it("names no tally path in any default", () => { + for (const path of [defaultConfigPath(), defaultPusherDir(), defaultUsageDir()]) { + expect(path).not.toMatch(/tally/) + } + const built = buildSnapshot({ config: readConfig(null), now: new Date(T0).toISOString(), list: ["cc.json"] }) + expect(built.snapshot.seats).toEqual([]) + }) + + it("the CLI dry run reads a cache-file source and calls no network", () => { + const b = box() + const cacheFile = join(b.dir, "cache.json") + writeFileSync(cacheFile, JSON.stringify({ observed_at: "2026-09-23T05:59:00Z", usage: answer() })) + const configPath = join(b.dir, "config.json") + writeFileSync(configPath, JSON.stringify({ host: "coordinator", usage: { cc: { kind: "cache-file", path: cacheFile } } })) + const run = spawnSync(process.execPath, [PUSHER, "--dry-run", "--config", configPath], { encoding: "utf8" }) + expect(run.status).toBe(0) + const out = JSON.parse(run.stdout) + expect(out.pushed).toBe(false) + expect(decode(out.snapshot).seats.map((seat) => seat.seat)).toEqual(["cc"]) + }) +}) diff --git a/pkgs/substrate-apps/src/apps/pusher/vitest.config.ts b/pkgs/substrate-apps/src/apps/pusher/vitest.config.ts new file mode 100644 index 00000000..1d909a98 --- /dev/null +++ b/pkgs/substrate-apps/src/apps/pusher/vitest.config.ts @@ -0,0 +1,8 @@ +import { defineConfig } from "vitest/config" + +export default defineConfig({ + test: { + include: ["test/**/*.test.mjs", "test/**/*.test.ts"], + environment: "node" + } +}) diff --git a/pkgs/substrate-apps/src/deploy/client.config.example.toml b/pkgs/substrate-apps/src/deploy/client.config.example.toml new file mode 100644 index 00000000..edfb7be0 --- /dev/null +++ b/pkgs/substrate-apps/src/deploy/client.config.example.toml @@ -0,0 +1,28 @@ +# ~/.config/substrate/config.toml: what the substrate CLI, the MCP server and the puller read. +# It names the floor and WHERE each credential is, never a credential. Every file named here is mode 600. +# SUBSTRATE_CLIENT_CONFIG moves this file; SUBSTRATE_URL and SUBSTRATE_TOKEN_FILE override the first two keys. + +floor_url = "https://substrate.example.dev" +token_file = "~/.local/state/substrate/floor-token" # the operator bearer (FLOOR_TOKEN) +# access_client_id_file = "~/.local/state/substrate/access-client-id" # a Cloudflare Access service +# access_client_secret_file = "~/.local/state/substrate/access-client-secret" # token pair, instead of or beside it + +[puller] +holder = "coordinator" # the holder its link token is bound to (LINK_TOKENS) +link_token_file = "~/.local/state/substrate/link-token-coordinator" +state_dir = "~/.local/state/substrate/puller" # held leases, verdict outbox, runs// journals +pidfile = "~/.local/state/substrate/puller.pid" +max_runs = 1 # runs held at once (the Lease capacity) +node_dispatch = "local" # "local": the runners here; "floor": nodes go back to the floor +# local +runtimes = "~/.config/substrate/runtimes.toml" +seat = "cc" # the seat the capacity gate reads for this host's calls +cap = 2 # the WIP cap across this puller's calls +default_model = "claude-opus-5-5" +demand_dir = "~/.local/state/substrate/demand" # marked while a run is held: the pusher reads at its active cadence +drain_timeout_s = 60 # first SIGTERM: runs may finish this long; 2nd signal aborts (cancel grace), 3rd kills +# health_addr = "127.0.0.1:9311" # loopback GET /status.json and /metrics +capacity_wait_s = 600 # a node waits this long for capacity (a stale reading) before it fails +# ax_server = "nas:7070" # probed only to say why an ax node is refused here +# floor +node_runs_on = ["seat:halogen", "runtime:gvisor"] # runs-on for a node that names none diff --git a/pkgs/substrate-apps/src/deploy/substrate.config.example.json b/pkgs/substrate-apps/src/deploy/substrate.config.example.json new file mode 100644 index 00000000..d06a235d --- /dev/null +++ b/pkgs/substrate-apps/src/deploy/substrate.config.example.json @@ -0,0 +1,21 @@ +{ + "comment": "Copy to ~/.config/substrate/substrate.json or point SUBSTRATE_CONFIG at it. Every key is optional; see src/deploy-config.ts.", + "forbiddenReadPrefixes": [ + "~/private", + "~/.local/state/secrets" + ], + "forbiddenLedgerPrefixes": [ + "/run/user", + "~/.local/state/substrate/meters" + ], + "metersDir": "~/.local/state/substrate/meters", + "axProtoFallbackPath": null, + "realSeats": [ + "claude-main", + "codex", + "gpu-worker" + ], + "codexWorkdir": "~/src/my-project", + "capacityFloorUrl": "https://substrate.example.dev", + "capacityFloorTokenFile": "~/.local/state/substrate/floor-token" +} diff --git a/pkgs/substrate-apps/src/deploy/wrangler.local.example.jsonc b/pkgs/substrate-apps/src/deploy/wrangler.local.example.jsonc new file mode 100644 index 00000000..dccbaab9 --- /dev/null +++ b/pkgs/substrate-apps/src/deploy/wrangler.local.example.jsonc @@ -0,0 +1,43 @@ +{ + // deploy/wrangler.local.example.jsonc: the deploy-local Worker config. + // + // Copy to apps/floor/wrangler.local.jsonc (gitignored by *.local.jsonc) and + // fill in YOUR account, route and seat names. Deploy with: + // cd apps/floor && wrangler deploy --config wrangler.local.jsonc + // The committed apps/floor/wrangler.jsonc names no account and no route, so + // no command run from a fresh clone can publish anything by accident. + "name": "substrate-floor", + "main": "src/index.ts", + "compatibility_date": "2026-05-18", + "account_id": "", + "routes": [{ "pattern": "substrate.example.com", "custom_domain": true }], + "durable_objects": { + "bindings": [ + { "name": "FACTORY", "class_name": "Factory" }, + { "name": "FLOOR", "class_name": "Floor" } + ] + }, + "migrations": [ + { "tag": "v1", "new_sqlite_classes": ["Factory"] }, + { "tag": "v2", "new_sqlite_classes": ["Floor"] } + ], + // Secrets are never vars. Set each with `wrangler secret put --config wrangler.local.jsonc`: + // FLOOR_TOKEN operator bearer (intake API, CLI, capacity pusher) + // LINK_TOKENS JSON {"": {"holder": "...", "labels": ["seat:...", "runtime:..."], "guest": false}} + // LEASE_TOKEN_KEY HMAC key for guest lease tokens (only needed when a holder has "guest": true) + // ACCESS_CLIENT_ID optional: a Cloudflare Access service token accepted in place of FLOOR_TOKEN + // ACCESS_CLIENT_SECRET + "vars": { + // The CONWIP cap over agent jobs, and the runs held by interpreter hosts at once. + "FLOOR_CAP": "2", + "FLOOR_RUN_CAP": "4", + // The seat an interpreter host (runtime:interpreter) serves; runs are handed out on it. + "FLOOR_INTERPRETER_SEAT": "coordinator", + // Seats admitted without a capacity reading (comma list). Every other seat needs a fresh seat-capacity/2 row. + "FLOOR_UNGATED_SEATS": "coordinator", + // runs-on seat name -> capacity seat id, when they differ (e.g. a GPU slot row). + "FLOOR_SEAT_CAPACITY_MAP": "{\"gpu\": \"gpu-worker\"}", + // Per-label caps, JSON. + "FLOOR_LABEL_CAPS": "{}" + } +} diff --git a/pkgs/substrate-apps/src/package.json b/pkgs/substrate-apps/src/package.json new file mode 100644 index 00000000..5541c17d --- /dev/null +++ b/pkgs/substrate-apps/src/package.json @@ -0,0 +1,31 @@ +{ + "name": "substrate", + "version": "0.1.0", + "private": true, + "type": "module", + "description": "CONWIP factory: turns Claude ultracode workflow run records into ax Tasks under a WIP cap, plus the Effect workspace (schema, serializer, planning, factory) and the Cloudflare floor.", + "scripts": { + "test": "vitest run", + "typecheck": "tsc --noEmit -p . && tsc --noEmit -p packages/schema && tsc --noEmit -p packages/serializer && tsc --noEmit -p packages/planning && tsc --noEmit -p packages/factory && tsc --noEmit -p apps/floor && tsc --noEmit -p packages/interpreter && tsc --noEmit -p packages/runners && tsc --noEmit -p apps/link && tsc --noEmit -p packages/api && tsc --noEmit -p apps/floor/tsconfig.node.json && tsc --noEmit -p apps/cli && tsc --noEmit -p apps/mcp && tsc --noEmit -p apps/puller", + "check:fences": "node tools/check-fences.mjs", + "check": "pnpm typecheck && pnpm check:fences && pnpm test", + "conwip": "tsx src/cli.ts", + "conwip-run": "tsx src/run-cli.ts" + }, + "dependencies": { + "@grpc/grpc-js": "^1.12.6", + "@grpc/proto-loader": "^0.7.13", + "effect": "4.0.0-rc.112", + "@substrate/planning": "workspace:*", + "@substrate/schema": "workspace:*", + "@substrate/interpreter": "workspace:*", + "@substrate/runners": "workspace:*" + }, + "devDependencies": { + "@types/node": "^22.10.2", + "tsx": "^4.19.2", + "typescript": "^5.9.3", + "vitest": "^3.2.4", + "fast-check": "^4.9.0" + } +} diff --git a/pkgs/substrate-apps/src/packages/api/package.json b/pkgs/substrate-apps/src/packages/api/package.json new file mode 100644 index 00000000..69673b8b --- /dev/null +++ b/pkgs/substrate-apps/src/packages/api/package.json @@ -0,0 +1,18 @@ +{ + "name": "@substrate/api", + "version": "0.1.0", + "private": true, + "type": "module", + "description": "The floor's HTTP API described once as Effect Schema (HttpApi): the generated OpenAPI 3.1 document, a typed Promise client, and the operator config shared by the CLI, the MCP server and the puller.", + "license": "UNLICENSED", + "exports": { ".": "./src/index.ts", "./*": "./src/*" }, + "scripts": { "check": "tsc --noEmit -p .", "test": "vitest run" }, + "dependencies": { "smol-toml": "^1.4.2", "@substrate/link": "workspace:*" }, + "peerDependencies": { "effect": "4.0.0-rc.112" }, + "devDependencies": { + "@types/node": "^22.10.2", + "effect": "4.0.0-rc.112", + "typescript": "^5.9.3", + "vitest": "^3.2.4" + } +} diff --git a/pkgs/substrate-apps/src/packages/api/src/api.ts b/pkgs/substrate-apps/src/packages/api/src/api.ts new file mode 100644 index 00000000..315b5524 --- /dev/null +++ b/pkgs/substrate-apps/src/packages/api/src/api.ts @@ -0,0 +1,86 @@ +// The floor's routes as one Effect HttpApi. Nothing serves this definition directly: the floor's own router +// (apps/floor/src/link/floor-object.ts) is the implementation and apps/floor/test/openapi.test.ts holds the two to +// the same route table. What this buys is one description with two outputs: GET /openapi.json (OpenApi.fromApi) +// and the client's decoders (client.ts). /rpc is Effect RPC (Lease, Heartbeat, Complete) and is described in the +// document's info, not as REST paths. +import { Schema } from "effect" +import { HttpApi, HttpApiEndpoint, HttpApiGroup, HttpApiSchema, OpenApi } from "effect/unstable/httpapi" +import * as S from "./schema.ts" + +const Id = { id: Schema.String } +const Name = { name: Schema.String } +const errors = [S.ErrorBody] +const doc = unknown }>(e: E, summary: string, description?: string): E => + // eslint-disable-next-line @typescript-eslint/no-explicit-any + { let r: any = (e as any).annotate(OpenApi.Summary, summary); if (description) r = r.annotate(OpenApi.Description, description); return r } + +export const RunsGroup = HttpApiGroup.make("runs") + .add(doc(HttpApiEndpoint.post("submitRun", "/runs", { payload: S.SubmitRequest, success: S.SubmitReply.pipe(HttpApiSchema.status(201)), error: errors }), + "Submit a run", "A workflow script (JSON { script } or a text/javascript body) or a list of AgentJob. Answers 201 when created, 200 when the id already exists.")) + .add(doc(HttpApiEndpoint.get("listRuns", "/runs", { query: { limit: Schema.optionalKey(Schema.String) }, success: Schema.Struct({ runs: Schema.Array(S.RunView) }), error: errors }), "List runs, newest first")) + .add(doc(HttpApiEndpoint.get("getRun", "/runs/:id", { params: Id, success: Schema.Struct({ run: S.RunView }), error: errors }), "One run")) + .add(doc(HttpApiEndpoint.get("runJobs", "/runs/:id/jobs", { params: Id, success: Schema.Struct({ jobs: Schema.Array(S.JobSummary) }), error: errors }), "The jobs of a run")) + .add(doc(HttpApiEndpoint.post("enqueueRunJobs", "/runs/:id/jobs", { params: Id, payload: Schema.Union([S.AgentJob, Schema.Array(S.AgentJob)]), success: S.Enqueued.pipe(HttpApiSchema.status(201)), error: errors }), + "Enqueue nodes of a run", "The interpreter host's call: each agent() node as an AgentJob, idempotent by metadata.name.")) + .add(doc(HttpApiEndpoint.get("runScript", "/runs/:id/script", { params: Id, success: Schema.String.pipe(HttpApiSchema.asText({ contentType: "text/javascript" })), error: errors }), "The script of a script run")) + .add(doc(HttpApiEndpoint.get("runEvents", "/runs/:id/events", { params: Id, query: { after: Schema.optionalKey(Schema.String) }, success: Schema.Struct({ events: Schema.Array(S.FloorEvent) }), error: errors }), "Verdict events of a run")) + .add(doc(HttpApiEndpoint.post("cancelRun", "/runs/:id/cancel", { params: Id, success: Schema.Struct({ run: S.RunView }), error: errors }), "Cancel a run and every unfinished job in it")) + +export const JobsGroup = HttpApiGroup.make("jobs") + .add(doc(HttpApiEndpoint.get("getJob", "/jobs/:name", { params: Name, success: Schema.Struct({ job: S.JobView }), error: errors }), "One job")) + .add(doc(HttpApiEndpoint.get("jobOutput", "/jobs/:name/output", { params: Name, success: S.JobOutput, error: errors }), "A job's verdict and output", "404 no-output while the job has no verdict.")) + .add(doc(HttpApiEndpoint.get("jobTranscripts", "/jobs/:name/transcript", { params: Name, success: S.TranscriptManifest, error: errors }), "A job's transcript parts", "The harness transcripts (session file, rollout, stdout and stderr) stored for the job.")) + .add(doc(HttpApiEndpoint.get("jobTranscriptPart", "/jobs/:name/transcript/:part", { params: { name: Schema.String, part: Schema.String }, query: { idx: Schema.optionalKey(Schema.String), partial: Schema.optionalKey(Schema.String) }, success: Schema.String.pipe(HttpApiSchema.asText({ contentType: "text/plain" })), error: errors }), "One transcript part as text", "409 transcript-uncommitted until its commit; ?idx=N reads one chunk, ?partial=1 reads an uncommitted part.")) + .add(doc(HttpApiEndpoint.put("putTranscriptChunk", "/jobs/:name/transcript/:part/:idx", { params: { name: Schema.String, part: Schema.String, idx: Schema.String }, payload: Schema.String.pipe(HttpApiSchema.asText({ contentType: "text/plain" })), success: Schema.Unknown, error: errors }), "Store one transcript chunk", "At most 1 MB per chunk and 32 MB per job (413); 409 once the part is committed.")) + .add(doc(HttpApiEndpoint.post("commitTranscript", "/jobs/:name/transcript/:part/commit", { params: { name: Schema.String, part: Schema.String }, payload: S.TranscriptCommit, success: Schema.Unknown, error: errors }), "Seal a transcript part", "422 sha-mismatch or transcript-incomplete unless bytes, chunk count and sha256 match what is stored.")) + .add(doc(HttpApiEndpoint.post("cancelJob", "/jobs/:name/cancel", { params: Name, success: Schema.Struct({ job: S.JobView }), error: errors }), "Cancel one job")) + .add(doc(HttpApiEndpoint.get("events", "/events", { query: { after: Schema.optionalKey(Schema.String) }, success: Schema.Struct({ events: Schema.Array(S.FloorEvent) }), error: errors }), "Every verdict event after a sequence number")) + +export const HoldersGroup = HttpApiGroup.make("holders") + .add(doc(HttpApiEndpoint.get("listHolders", "/holders", { success: Schema.Struct({ holders: Schema.Array(S.Holder) }), error: errors }), "Pullers and links known to the floor")) + .add(doc(HttpApiEndpoint.post("pauseHolder", "/holders/:holder/pause", { params: { holder: Schema.String }, success: Schema.Struct({ holder: Schema.String, paused: Schema.Boolean }), error: errors }), "Stop granting to a holder")) + .add(doc(HttpApiEndpoint.post("resumeHolder", "/holders/:holder/resume", { params: { holder: Schema.String }, success: Schema.Struct({ holder: Schema.String, paused: Schema.Boolean }), error: errors }), "Grant to a holder again")) + +export const CapacityGroup = HttpApiGroup.make("capacity") + .add(doc(HttpApiEndpoint.get("capacity", "/capacity", { query: { model: Schema.optionalKey(Schema.String), min_headroom_pct: Schema.optionalKey(Schema.String), asOf: Schema.optionalKey(Schema.String) }, success: S.CapacityView, error: errors }), + "Every seat's projected capacity", "Grade, staleness, headroom, resets, the floor's WIP per seat and admit for ?model.")) + .add(doc(HttpApiEndpoint.get("capacityAdmit", "/capacity/admit", { query: { seat: Schema.String, model: Schema.optionalKey(Schema.String), min_headroom_pct: Schema.optionalKey(Schema.String) }, success: S.AdmitAnswer, error: errors }), "Would one seat admit a job now")) + .add(doc(HttpApiEndpoint.post("capacitySnapshot", "/capacity/snapshots", { payload: Schema.Unknown.annotate({ description: "A seat-capacity/2 snapshot." }), success: Schema.Record(Schema.String, Schema.Unknown), error: errors }), "Push a seat-capacity/2 snapshot (the pusher)")) + +export const FloorGroup = HttpApiGroup.make("floor") + .add(doc(HttpApiEndpoint.get("floorState", "/floor/state", { success: S.FloorState, error: errors }), "Floor statistics, run WIP, alarm and config")) + .add(doc(HttpApiEndpoint.get("openapi", "/openapi.json", { success: Schema.Record(Schema.String, Schema.Unknown) }), "This document", "Served without a credential: it carries no data.")) + +export const SubstrateApi = HttpApi.make("substrate") + .add(RunsGroup).add(JobsGroup).add(HoldersGroup).add(CapacityGroup).add(FloorGroup) + .annotate(OpenApi.Title, "substrate floor") + .annotate(OpenApi.Description, "The job-of-record floor. Operator routes take `Authorization: Bearer ` or a Cloudflare Access service token pair. " + + "Pullers and links use POST /rpc (Effect RPC over HTTP, JSON serialization) with a per-link token: Lease, Heartbeat, Complete (see apps/link/src/contract.ts).") + +/** Operator routes as (method, path template) pairs; the floor test holds the router to this list. */ +export const ROUTES: ReadonlyArray<{ readonly method: string; readonly path: string; readonly id: string; readonly public: boolean }> = + Object.values(SubstrateApi.groups).flatMap((g) => Object.values(g.endpoints).map((e) => ({ method: e.method, path: e.path, id: e.name, public: e.path === "/openapi.json" }))) + +let cached: Record | undefined +/** The OpenAPI 3.1 document, with the two credential schemes the door accepts. */ +export const openApiDocument = (): Record => { + if (cached) return cached + const spec = OpenApi.fromApi(SubstrateApi) as unknown as Record & { components?: Record; paths: Record>> } + const components = (spec.components ?? {}) as Record + components.securitySchemes = { + bearer: { type: "http", scheme: "bearer", description: "FLOOR_TOKEN (operator)." }, + accessClientId: { type: "apiKey", in: "header", name: "CF-Access-Client-Id" }, + accessClientSecret: { type: "apiKey", in: "header", name: "CF-Access-Client-Secret" } + } + spec.components = components + for (const [path, ops] of Object.entries(spec.paths)) for (const op of Object.values(ops)) + op.security = path === "/openapi.json" ? [] : [{ bearer: [] }, { accessClientId: [], accessClientSecret: [] }] + spec.servers = [{ url: "/" }] + // Refusals are the floor's 4xx (400, 401, 403, 404, 409, 413, 422), not the generator's default 500. + for (const ops of Object.values(spec.paths)) for (const op of Object.values(ops)) { + const r = op.responses as Record | undefined + if (r && r["500"] !== undefined) { r["4XX"] = r["500"]; delete r["500"] } + } + cached = spec + return spec +} diff --git a/pkgs/substrate-apps/src/packages/api/src/client.ts b/pkgs/substrate-apps/src/packages/api/src/client.ts new file mode 100644 index 00000000..3451f1e2 --- /dev/null +++ b/pkgs/substrate-apps/src/packages/api/src/client.ts @@ -0,0 +1,174 @@ +// The typed client over the floor's operator routes. Promise-based so a CLI verb, an MCP tool and a code-mode +// snippet all call it the same way; every answer is decoded with the schema api.ts publishes, so a drifted floor +// fails here, loudly, rather than in a table three layers up. +import { Schema } from "effect" +import * as S from "./schema.ts" + +export class SubstrateError extends Error { + override readonly name = "SubstrateError" + constructor(readonly status: number, readonly code: string, message: string) { super(`${status} ${code}: ${message}`) } +} + +export interface ClientOptions { + /** The floor's base URL, e.g. https://substrate.example.dev (no trailing path). */ + readonly url: string + /** The operator bearer (FLOOR_TOKEN). Never logged. */ + readonly token?: string + /** A Cloudflare Access service token pair, sent instead of or beside the bearer. */ + readonly access?: { readonly clientId: string; readonly clientSecret: string } + readonly fetch?: typeof globalThis.fetch + readonly timeoutMs?: number + /** G-BK5: the live lease a run-scoped write is made under, sent as x-substrate-lease: :. */ + readonly lease?: { readonly leaseId: string; readonly attempt: number } +} + +export interface SeatRow { + readonly seat: string; readonly grade: string; readonly staleness: string; readonly headroom_pct: number | null + readonly next_reset_at: string | null; readonly wip: number; readonly admit: boolean | null; readonly reason: string | null +} + +const decode = (schema: Schema.Codec, v: unknown, what: string): A => { + try { return Schema.decodeUnknownSync(schema as never)(v) as A } catch (e) { + throw new SubstrateError(0, "decode", `${what}: the floor's answer does not match the published schema: ${String((e as Error).message).slice(0, 600)}`) + } +} + +const RunsReply = Schema.Struct({ runs: Schema.Array(S.RunView) }) +const RunReply = Schema.Struct({ run: S.RunView }) +const JobsReply = Schema.Struct({ jobs: Schema.Array(S.JobSummary) }) +const JobReply = Schema.Struct({ job: S.JobView }) +const EventsReply = Schema.Struct({ events: Schema.Array(S.FloorEvent) }) +const HoldersReply = Schema.Struct({ holders: Schema.Array(S.Holder) }) +const PauseReply = Schema.Struct({ holder: Schema.String, paused: Schema.Boolean }) + +export class SubstrateClient { + readonly url: string + readonly #o: ClientOptions + constructor(o: ClientOptions) { this.url = o.url.replace(/\/+$/, ""); this.#o = o } + + private headers(extra: Record = {}): Headers { + const h = new Headers(extra) + if (this.#o.token) h.set("authorization", `Bearer ${this.#o.token}`) + if (this.#o.access) { h.set("cf-access-client-id", this.#o.access.clientId); h.set("cf-access-client-secret", this.#o.access.clientSecret) } + if (this.#o.lease) h.set("x-substrate-lease", `${this.#o.lease.leaseId}:${this.#o.lease.attempt}`) + return h + } + + /** One request; a non-2xx answer becomes a SubstrateError carrying the floor's own code. */ + async request(method: string, path: string, body?: { json?: unknown; text?: string; contentType?: string }): Promise<{ status: number; text: string }> { + const f = this.#o.fetch ?? globalThis.fetch + const init: RequestInit = { method, headers: this.headers(body?.json !== undefined ? { "content-type": "application/json" } : body?.text !== undefined ? { "content-type": body.contentType ?? "text/plain" } : {}) } + if (body?.json !== undefined) init.body = JSON.stringify(body.json) + else if (body?.text !== undefined) init.body = body.text + if (this.#o.timeoutMs) init.signal = AbortSignal.timeout(this.#o.timeoutMs) + const res = await f(`${this.url}${path}`, init) + const text = await res.text() + if (!res.ok) { + let code = `http-${res.status}`, message = text.slice(0, 500) + try { + const e = (JSON.parse(text) as { error?: unknown }).error + if (typeof e === "string") { code = e; message = e } else if (e && typeof e === "object") { code = String((e as { code?: unknown }).code ?? code); message = String((e as { message?: unknown }).message ?? message) } + } catch { /* not JSON */ } + throw new SubstrateError(res.status, code, message) + } + return { status: res.status, text } + } + private async json(method: string, path: string, json?: unknown): Promise { + const r = await this.request(method, path, json === undefined ? undefined : { json }) + try { return JSON.parse(r.text) } catch { throw new SubstrateError(r.status, "not-json", `${method} ${path} answered a non-JSON body`) } + } + private p = (s: string) => encodeURIComponent(s) + /** G-BK5: the same client, its run-scoped writes made under a live lease (Buildkite's job token). */ + underLease(leaseId: string, attempt: number): SubstrateClient { return new SubstrateClient({ ...this.#o, lease: { leaseId, attempt } }) } + + // ---- runs + /** Submit a workflow script (source text) or AgentJobs. Idempotent when `id` is given. */ + async submit(req: S.SubmitRequest): Promise { return decode(S.SubmitReply, await this.json("POST", "/runs", req), "submit") } + submitScript(script: string, o: { args?: unknown; id?: string } = {}): Promise { return this.submit({ script, ...o }) } + submitJobs(jobs: ReadonlyArray, o: { name?: string; id?: string } = {}): Promise { return this.submit({ jobs, ...o }) } + async runs(o: { limit?: number } = {}): Promise> { return decode(RunsReply, await this.json("GET", `/runs?limit=${o.limit ?? 50}`), "runs").runs } + async run(id: string): Promise { return decode(RunReply, await this.json("GET", `/runs/${this.p(id)}`), "run").run } + async runJobs(id: string): Promise> { return decode(JobsReply, await this.json("GET", `/runs/${this.p(id)}/jobs`), "runJobs").jobs } + async runScript(id: string): Promise { return (await this.request("GET", `/runs/${this.p(id)}/script`)).text } + async runEvents(id: string, after = 0): Promise> { return decode(EventsReply, await this.json("GET", `/runs/${this.p(id)}/events?after=${after}`), "runEvents").events } + async cancelRun(id: string): Promise { return decode(RunReply, await this.json("POST", `/runs/${this.p(id)}/cancel`), "cancelRun").run } + /** The interpreter host's call: enqueue agent() nodes of a script run as AgentJobs (idempotent by name). */ + async enqueue(id: string, jobs: ReadonlyArray): Promise { return decode(S.Enqueued, await this.json("POST", `/runs/${this.p(id)}/jobs`, jobs), "enqueue") } + /** Poll a run until it is done or the timeout passes; answers the last view either way. */ + async waitRun(id: string, o: { timeoutMs?: number; pollMs?: number } = {}): Promise { + const until = Date.now() + (o.timeoutMs ?? 60_000) + for (;;) { + const r = await this.run(id) + if (r.state === "done" || Date.now() >= until) return r + await new Promise((ok) => setTimeout(ok, o.pollMs ?? 2000)) + } + } + + // ---- jobs + async job(name: string): Promise { return decode(JobReply, await this.json("GET", `/jobs/${this.p(name)}`), "job").job } + async output(name: string): Promise { return decode(S.JobOutput, await this.json("GET", `/jobs/${this.p(name)}/output`), "output") } + /** G-BK3: append one live log chunk for the live attempt `leaseId`/`attempt` of job `name`. Idempotent by (leaseId, seq). */ + async appendLog(name: string, c: { leaseId: string; attempt: number; seq: number; data: string }): Promise<{ duplicate: boolean; id: number; offset: number }> { + return await this.json("POST", `/jobs/${this.p(name)}/log`, c) as { duplicate: boolean; id: number; offset: number } + } + /** G-BK3: a job's live log chunks after cursor `after`. */ + async jobLog(name: string, after = 0): Promise<{ chunks: ReadonlyArray<{ id: number; leaseId: string; attempt: number; seq: number; offset: number; bytes: number; data: string; at: number }>; next: number }> { + return await this.json("GET", `/jobs/${this.p(name)}/log?after=${after}`) as never + } + async cancelJob(name: string): Promise { return decode(JobReply, await this.json("POST", `/jobs/${this.p(name)}/cancel`), "cancelJob").job } + // ---- transcripts (AUDIT-transcripts TX2) + async transcripts(name: string): Promise { return decode(S.TranscriptManifest, await this.json("GET", `/jobs/${this.p(name)}/transcript`), "transcripts") } + async transcript(name: string, part: string, o: { partial?: boolean; idx?: number } = {}): Promise { + const q = new URLSearchParams({ ...(o.partial ? { partial: "1" } : {}), ...(o.idx !== undefined ? { idx: String(o.idx) } : {}) }).toString() + return (await this.request("GET", `/jobs/${this.p(name)}/transcript/${this.p(part)}${q ? `?${q}` : ""}`)).text + } + /** Upload one part in chunks (each under the floor's 1 MB) and seal it with its sha256. Idempotent: a replay of the + * same text re-stores the same chunks and the commit answers duplicate. */ + async uploadTranscript(name: string, part: string, text: string, o: { chunkChars?: number } = {}): Promise<{ sha256: string; bytes: number; chunks: number }> { + const step = o.chunkChars ?? 300_000 // UTF-16 units; at most 3 bytes each, so a chunk stays under 1 MB + const chunks: Array = [] + for (let i = 0; i < text.length;) { + let end = Math.min(text.length, i + step) + if (end < text.length && end - i > 1) { const c = text.charCodeAt(end - 1); if (c >= 0xd800 && c <= 0xdbff) end-- } // never split a pair + chunks.push(text.slice(i, end)); i = end + } + if (chunks.length === 0) chunks.push("") + for (const [i, c] of chunks.entries()) await this.request("PUT", `/jobs/${this.p(name)}/transcript/${this.p(part)}/${i}`, { text: c, contentType: "text/plain; charset=utf-8" }) + const bytes = new TextEncoder().encode(text).length + const sha256 = [...new Uint8Array(await crypto.subtle.digest("SHA-256", new TextEncoder().encode(text)))].map((x) => x.toString(16).padStart(2, "0")).join("") + await this.json("POST", `/jobs/${this.p(name)}/transcript/${this.p(part)}/commit`, { sha256, bytes, chunks: chunks.length }) + return { sha256, bytes, chunks: chunks.length } + } + async events(after = 0): Promise> { return decode(EventsReply, await this.json("GET", `/events?after=${after}`), "events").events } + + // ---- capacity and seats + async capacity(o: { model?: string; minHeadroomPct?: number } = {}): Promise { + const q = new URLSearchParams() + if (o.model) q.set("model", o.model) + if (o.minHeadroomPct !== undefined) q.set("min_headroom_pct", String(o.minHeadroomPct)) + const qs = q.toString() + return decode(S.CapacityView, await this.json("GET", `/capacity${qs ? `?${qs}` : ""}`), "capacity") + } + async admit(seat: string, o: { model?: string } = {}): Promise> { + const q = new URLSearchParams({ seat, ...(o.model ? { model: o.model } : {}) }) + return decode(S.AdmitAnswer, await this.json("GET", `/capacity/admit?${q}`), "admit") as Record + } + /** One row per seat: the `seats` view of GET /capacity. */ + async seats(o: { model?: string } = {}): Promise> { + const v = await this.capacity(o) + return v.seats.map((s) => { + const a = (s.admit ?? {}) as { admit?: unknown; reason?: unknown } + return { seat: s.seat, grade: s.grade, staleness: s.staleness, headroom_pct: s.headroom_pct, next_reset_at: s.next_reset_at, wip: s.wip, + admit: typeof a.admit === "boolean" ? a.admit : null, reason: typeof a.reason === "string" ? a.reason : null } + }) + } + + // ---- pullers (holders) + async pullers(): Promise> { return decode(HoldersReply, await this.json("GET", "/holders"), "pullers").holders } + async pausePuller(holder: string): Promise<{ holder: string; paused: boolean }> { return decode(PauseReply, await this.json("POST", `/holders/${this.p(holder)}/pause`), "pause") } + async resumePuller(holder: string): Promise<{ holder: string; paused: boolean }> { return decode(PauseReply, await this.json("POST", `/holders/${this.p(holder)}/resume`), "resume") } + + // ---- floor + async floorState(): Promise { return decode(S.FloorState, await this.json("GET", "/floor/state"), "floorState") } + async openapi(): Promise> { return await this.json("GET", "/openapi.json") as Record } +} diff --git a/pkgs/substrate-apps/src/packages/api/src/config.ts b/pkgs/substrate-apps/src/packages/api/src/config.ts new file mode 100644 index 00000000..7c354874 --- /dev/null +++ b/pkgs/substrate-apps/src/packages/api/src/config.ts @@ -0,0 +1,94 @@ +// The operator config every interface reads: ~/.config/substrate/config.toml (SUBSTRATE_CLIENT_CONFIG overrides the +// path). It names the floor and WHERE the credential is, never the credential itself. Environment overrides: +// SUBSTRATE_URL, SUBSTRATE_TOKEN_FILE. A deployment's own values (domain, paths) live only in that file; the repo +// carries deploy/client.config.example.toml. +import { existsSync, readFileSync, statSync } from "node:fs" +import { homedir } from "node:os" +import { parse } from "smol-toml" +import { SubstrateClient } from "./client.ts" + +export interface PullerSection { + readonly holder?: string + readonly link_token_file?: string + readonly state_dir?: string + readonly runtimes?: string + readonly max_runs?: number + readonly seat?: string + readonly default_model?: string + readonly cap?: number + readonly node_dispatch?: "local" | "floor" + readonly ax_server?: string + readonly pidfile?: string + readonly node_runs_on?: ReadonlyArray + /** The capacity pusher's demand dir: a marker here while a run is held makes the pusher read at its active cadence. */ + readonly demand_dir?: string + /** How long a node waits for capacity (a stale or refused reading) before it fails, in seconds. Default 600. */ + readonly capacity_wait_s?: number + /** G-BK4: on the first SIGTERM, how long in-flight runs may finish before they are aborted (with their cancel grace). Default 60; 0 aborts at once. */ + readonly drain_timeout_s?: number + /** G-BK7: loopback host:port serving GET /status.json and /metrics (Prometheus text). Absent: no endpoint. */ + readonly health_addr?: string +} +export interface ClientConfig { + readonly floor_url?: string + readonly token_file?: string + /** TX4: the read-only bearer (FLOOR_READ_TOKEN) file; `substrate watch` runs on it. */ + readonly read_token_file?: string + readonly access_client_id_file?: string + readonly access_client_secret_file?: string + readonly puller?: PullerSection + /** Where it was read from, or undefined when no file exists. */ + readonly source?: string +} + +export class ConfigError extends Error { override readonly name = "ConfigError" } + +export const expandHome = (p: string, home = homedir()) => (p === "~" ? home : p.startsWith("~/") ? home + p.slice(1) : p) +export const DEFAULT_CONFIG_PATH = "~/.config/substrate/config.toml" + +export const loadClientConfig = (env: NodeJS.ProcessEnv = process.env, home = homedir()): ClientConfig => { + const path = expandHome(env.SUBSTRATE_CLIENT_CONFIG ?? DEFAULT_CONFIG_PATH, home) + let file: Record = {} + if (existsSync(path)) { + try { file = parse(readFileSync(path, "utf8")) as Record } catch (e) { throw new ConfigError(`${path}: ${(e as Error).message}`) } + } + const str = (k: string) => (typeof file[k] === "string" ? (file[k] as string) : undefined) + const url = env.SUBSTRATE_URL ?? str("floor_url") + const tokenFile = env.SUBSTRATE_TOKEN_FILE ?? str("token_file") + const readTokenFile = env.SUBSTRATE_READ_TOKEN_FILE ?? str("read_token_file") + return { + ...(readTokenFile !== undefined ? { read_token_file: expandHome(readTokenFile, home) } : {}), + ...(url !== undefined ? { floor_url: url } : {}), + ...(tokenFile !== undefined ? { token_file: expandHome(tokenFile, home) } : {}), + ...(str("access_client_id_file") !== undefined ? { access_client_id_file: expandHome(str("access_client_id_file")!, home) } : {}), + ...(str("access_client_secret_file") !== undefined ? { access_client_secret_file: expandHome(str("access_client_secret_file")!, home) } : {}), + ...(typeof file.puller === "object" && file.puller !== null ? { puller: file.puller as PullerSection } : {}), + ...(existsSync(path) ? { source: path } : {}) + } +} + +/** Reads a secret file. Refuses a group- or world-readable one; the value never enters an error message. */ +export const readSecretFile = (path: string, what: string): string => { + if (!existsSync(path)) throw new ConfigError(`${what}: ${path} does not exist`) + const mode = statSync(path).mode & 0o077 + if (mode !== 0) throw new ConfigError(`${what}: ${path} is readable by group or others (chmod 600 it)`) + const v = readFileSync(path, "utf8").trim() + if (v === "") throw new ConfigError(`${what}: ${path} is empty`) + return v +} + +/** TX4: a client on the read-only credential only (never the operator bearer or the Access pair). */ +export const readClientFromConfig = (c: ClientConfig, o: { fetch?: typeof globalThis.fetch } = {}): SubstrateClient => { + if (!c.floor_url) throw new ConfigError("no floor_url") + if (!c.read_token_file) throw new ConfigError("no read-only credential: set read_token_file (or SUBSTRATE_READ_TOKEN_FILE) to a file holding FLOOR_READ_TOKEN") + return new SubstrateClient({ url: c.floor_url, token: readSecretFile(c.read_token_file, "read_token_file"), ...(o.fetch ? { fetch: o.fetch } : {}) }) +} +export const clientFromConfig = (c: ClientConfig, o: { fetch?: typeof globalThis.fetch; home?: string } = {}): SubstrateClient => { + if (!c.floor_url) throw new ConfigError(`no floor_url: set it in ${DEFAULT_CONFIG_PATH} or SUBSTRATE_URL`) + const token = c.token_file ? readSecretFile(c.token_file, "token_file") : undefined + const access = c.access_client_id_file && c.access_client_secret_file + ? { clientId: readSecretFile(c.access_client_id_file, "access_client_id_file"), clientSecret: readSecretFile(c.access_client_secret_file, "access_client_secret_file") } + : undefined + if (token === undefined && access === undefined) throw new ConfigError("no credential: set token_file (or the Access pair files)") + return new SubstrateClient({ url: c.floor_url, ...(token ? { token } : {}), ...(access ? { access } : {}), ...(o.fetch ? { fetch: o.fetch } : {}) }) +} diff --git a/pkgs/substrate-apps/src/packages/api/src/index.ts b/pkgs/substrate-apps/src/packages/api/src/index.ts new file mode 100644 index 00000000..2972561a --- /dev/null +++ b/pkgs/substrate-apps/src/packages/api/src/index.ts @@ -0,0 +1,5 @@ +export * from "./schema.ts" +export * from "./api.ts" +export * from "./client.ts" +export * from "./config.ts" +export * from "./seats.ts" diff --git a/pkgs/substrate-apps/src/packages/api/src/schema.ts b/pkgs/substrate-apps/src/packages/api/src/schema.ts new file mode 100644 index 00000000..34da6bfd --- /dev/null +++ b/pkgs/substrate-apps/src/packages/api/src/schema.ts @@ -0,0 +1,121 @@ +// The floor's HTTP bodies, described once. The floor (apps/floor) serves them, the OpenAPI document (api.ts) is +// generated from them, and the client (client.ts) decodes every answer with them. Field names are the floor's own +// (engine.ts run(), jobView(), output(), events(); capacity.ts capacityView()). +import { AgentJob } from "@substrate/link/contract.ts" +import { Schema } from "effect" + +export { AgentJob } + +export const ErrorBody = Schema.Struct({ + error: Schema.Union([Schema.String, Schema.Struct({ code: Schema.String, message: Schema.String })]) +}).annotate({ identifier: "ErrorBody", description: "Every refusal: { error: { code, message } }, or { error: \"Unauthorized\" } at the door." }) +export type ErrorBody = typeof ErrorBody.Type + +export const JobSummary = Schema.Struct({ + name: Schema.String, + runId: Schema.NullOr(Schema.String), + kind: Schema.String, // agent | run + state: Schema.String, // queued | leased | orphaned | done + attempt: Schema.Number, + leaseId: Schema.NullOr(Schema.String), + holder: Schema.NullOr(Schema.String), + runsOn: Schema.Array(Schema.String), + cancel: Schema.NullOr(Schema.String), + result: Schema.NullOr(Schema.String), + reason: Schema.NullOr(Schema.String), + supersedes: Schema.Array(Schema.String), + createdAt: Schema.Number, + doneAt: Schema.NullOr(Schema.Number) +}).annotate({ identifier: "JobSummary" }) +export type JobSummary = typeof JobSummary.Type + +export const JobView = Schema.Struct({ ...JobSummary.fields, job: Schema.Unknown }).annotate({ identifier: "JobView", description: "A job with its AgentJob body." }) +export type JobView = typeof JobView.Type + +export const RunView = Schema.Struct({ + id: Schema.String, + kind: Schema.String, // script | jobs + name: Schema.NullOr(Schema.String), + meta: Schema.Unknown, + args: Schema.Unknown, + scriptSha256: Schema.NullOr(Schema.String), + state: Schema.String, // running | done + result: Schema.NullOr(Schema.String), + createdAt: Schema.Number, + doneAt: Schema.NullOr(Schema.Number), + jobs: Schema.Record(Schema.String, Schema.Number), + interpreter: Schema.NullOr(JobView) +}).annotate({ identifier: "RunView" }) +export type RunView = typeof RunView.Type + +export const JobOutput = Schema.Struct({ + result: Schema.String, attempt: Schema.Number, leaseId: Schema.String, + output: Schema.Unknown, usage: Schema.Unknown, bytes: Schema.Number +}).annotate({ identifier: "JobOutput", description: "A job's recorded verdict and output." }) +export type JobOutput = typeof JobOutput.Type + +export const TranscriptPart = Schema.Struct({ + part: Schema.String, sha256: Schema.NullOr(Schema.String), bytes: Schema.Number, chunks: Schema.Number, + committed: Schema.Boolean, committedAt: Schema.NullOr(Schema.Number) +}).annotate({ identifier: "TranscriptPart", description: "One part of a job's harness transcript (AUDIT-transcripts TX2): sealed by a sha256 commit." }) +export type TranscriptPart = typeof TranscriptPart.Type +export const TranscriptManifest = Schema.Struct({ name: Schema.String, parts: Schema.Array(TranscriptPart) }) + .annotate({ identifier: "TranscriptManifest", description: "A job's stored transcript parts." }) +export type TranscriptManifest = typeof TranscriptManifest.Type +export const TranscriptCommit = Schema.Struct({ sha256: Schema.String, bytes: Schema.Number, chunks: Schema.Number }) + .annotate({ identifier: "TranscriptCommit", description: "What an uploader says the assembled part is; the floor checks it." }) + +export const FloorEvent = Schema.Struct({ + seq: Schema.Number, run_id: Schema.NullOr(Schema.String), name: Schema.String, lease_id: Schema.NullOr(Schema.String), + attempt: Schema.Number, result: Schema.String, reason: Schema.NullOr(Schema.String), at_ms: Schema.Number +}).annotate({ identifier: "FloorEvent", description: "One row of the verdict outbox." }) +export type FloorEvent = typeof FloorEvent.Type + +export const ScriptSubmit = Schema.Struct({ + script: Schema.String.annotate({ description: "An ultracode workflow script whose `export const meta` is a pure literal." }), + args: Schema.optionalKey(Schema.Unknown), + id: Schema.optionalKey(Schema.String.annotate({ description: "6 to 40 lowercase letters or digits; makes the submit idempotent." })) +}).annotate({ identifier: "ScriptSubmit" }) +export const JobsSubmit = Schema.Struct({ + jobs: Schema.Array(AgentJob), + name: Schema.optionalKey(Schema.String), + id: Schema.optionalKey(Schema.String) +}).annotate({ identifier: "JobsSubmit" }) +export const SubmitRequest = Schema.Union([ScriptSubmit, JobsSubmit]).annotate({ identifier: "SubmitRequest" }) +export type SubmitRequest = typeof SubmitRequest.Type +export const SubmitReply = Schema.Struct({ run: RunView, created: Schema.Boolean }).annotate({ identifier: "SubmitReply" }) +export type SubmitReply = typeof SubmitReply.Type + +export const Enqueued = Schema.Struct({ enqueued: Schema.Array(Schema.Struct({ enqueued: Schema.Boolean, name: Schema.String })) }) +export type Enqueued = typeof Enqueued.Type + +export const SeatView = Schema.Struct({ + seat: Schema.String, host: Schema.String, published_at: Schema.String, received_at: Schema.String, + provider: Schema.Unknown, dispatchable: Schema.Unknown, dispatchable_reason: Schema.Unknown, + grade: Schema.String, staleness: Schema.String, age_seconds: Schema.NullOr(Schema.Number), + headroom_pct: Schema.NullOr(Schema.Number), next_reset_at: Schema.NullOr(Schema.String), wip: Schema.Number, + reading: Schema.Unknown, lapses: Schema.Unknown, plan_lapses_at: Schema.Unknown, admit: Schema.Unknown +}).annotate({ identifier: "SeatView" }) +export type SeatView = typeof SeatView.Type +export const CapacityView = Schema.Struct({ + as_of: Schema.String, + job: Schema.Struct({ model: Schema.NullOr(Schema.String), min_headroom_pct: Schema.Number }), + seats: Schema.Array(SeatView), + next_transition_at: Schema.NullOr(Schema.String) +}).annotate({ identifier: "CapacityView" }) +export type CapacityView = typeof CapacityView.Type +export const AdmitAnswer = Schema.Record(Schema.String, Schema.Unknown).annotate({ identifier: "AdmitAnswer" }) + +export const Holder = Schema.Struct({ + holder: Schema.String, + paused: Schema.Boolean, + live: Schema.Boolean.annotate({ description: "A session renewed within three heartbeat intervals." }), + sessionUntil: Schema.NullOr(Schema.Number), + leased: Schema.Number, + labels: Schema.Array(Schema.String).annotate({ description: "The runs-on labels of the jobs it holds now." }), + lastLeaseAt: Schema.NullOr(Schema.Number) +}).annotate({ identifier: "Holder", description: "A puller or link: anything that leases over /rpc." }) +export type Holder = typeof Holder.Type + +export const FloorState = Schema.Struct({ stats: Schema.Unknown, runWip: Schema.Unknown, alarm: Schema.Unknown, config: Schema.Unknown }) +export type FloorState = typeof FloorState.Type diff --git a/pkgs/substrate-apps/src/packages/api/src/seats.ts b/pkgs/substrate-apps/src/packages/api/src/seats.ts new file mode 100644 index 00000000..1437405d --- /dev/null +++ b/pkgs/substrate-apps/src/packages/api/src/seats.ts @@ -0,0 +1,50 @@ +// The `seats` CLI's oracle verbs, over the floor's seat rows (parity with the dotfiles `seats` CLI, 2026-09-24). +// Pure: every function reads SeatRow values the client already decoded, so the CLI and the MCP server answer alike. +// +// pickSeat the seat with the most headroom that the floor would admit now, or null (`seats --pick`, exit 1) +// checkSeat exit-code oracle for one seat: 0 headroom, 1 spent, 2 unmeasurable (`seats --check SEAT`) +// bar the ten-cell headroom bar the table prints +import type { SeatRow } from "./client.ts" + +/** Grades that carry a number worth planning on. UNKNOWN never does. */ +const NUMBERED = new Set(["MEASURED", "STALE", "PROJECTED", "ESTIMATED"]) + +/** A row with a number: a grade other than UNKNOWN and a headroom the floor computed. */ +export const measurable = (r: SeatRow): boolean => NUMBERED.has(r.grade) && r.headroom_pct !== null + +/** + * Would this row take a job now. The floor's own admission decides when it answered (admit is boolean when the + * view was asked for a model); without an answer, a measurable row with headroom above zero does, except a + * STALE or UNKNOWN staleness, which fails closed as admitSeat does. + */ +export const admits = (r: SeatRow): boolean => { + if (r.admit !== null) return r.admit + return measurable(r) && (r.headroom_pct ?? 0) > 0 && r.staleness === "MEASURED" +} + +/** The admitting seat with the most headroom; ties go to the fewer jobs in flight, then the seat name. */ +export const pickSeat = (rows: ReadonlyArray): SeatRow | null => { + const ok = rows.filter(admits) + if (ok.length === 0) return null + return [...ok].sort((a, b) => (b.headroom_pct ?? -1) - (a.headroom_pct ?? -1) || a.wip - b.wip || a.seat.localeCompare(b.seat))[0]! +} + +export interface SeatCheck { readonly code: 0 | 1 | 2; readonly line: string } + +/** The `--check` oracle. 2 when the seat is absent or has no number; 1 when it would not admit; 0 otherwise. */ +export const checkSeat = (rows: ReadonlyArray, seat: string): SeatCheck => { + const r = rows.find((x) => x.seat === seat) + if (r === undefined) return { code: 2, line: `${seat}: no such seat` } + if (!measurable(r)) return { code: 2, line: `${seat}: unmeasurable (grade ${r.grade}, staleness ${r.staleness})` } + const resets = r.next_reset_at ?? "unknown" + const head = `${seat}: ${r.headroom_pct!.toFixed(1)}% headroom, grade ${r.grade}, resets ${resets}` + return admits(r) ? { code: 0, line: head } : { code: 1, line: `${head}, refused${r.reason ? `: ${r.reason}` : ""}` } +} + +/** Ten cells, filled in proportion to the part of the window already used; `?` cells for no number. */ +export const bar = (headroomPct: number | null, width = 10): string => { + if (headroomPct === null) return "?".repeat(width) + const used = Math.min(100, Math.max(0, 100 - headroomPct)) + const filled = Math.round((used / 100) * width) + return "#".repeat(filled) + ".".repeat(width - filled) +} diff --git a/pkgs/substrate-apps/src/packages/api/test/api.test.ts b/pkgs/substrate-apps/src/packages/api/test/api.test.ts new file mode 100644 index 00000000..35073b55 --- /dev/null +++ b/pkgs/substrate-apps/src/packages/api/test/api.test.ts @@ -0,0 +1,66 @@ +// The client's error mapping and decode guard, and the operator config loader. +import { chmodSync, mkdtempSync, writeFileSync } from "node:fs" +import { tmpdir } from "node:os" +import { join } from "node:path" +import { describe, expect, it } from "vitest" +import { clientFromConfig, ConfigError, loadClientConfig, openApiDocument, SubstrateClient, SubstrateError } from "../src/index.ts" + +const answer = (status: number, body: unknown): typeof fetch => (async () => new Response(typeof body === "string" ? body : JSON.stringify(body), { status })) as typeof fetch + +describe("SubstrateClient", () => { + it("carries the floor's own error code, in both refusal shapes", async () => { + await expect(new SubstrateClient({ url: "http://f", token: "t", fetch: answer(404, { error: { code: "not-found", message: "no run x" } }) }).run("x")) + .rejects.toMatchObject({ status: 404, code: "not-found" }) + await expect(new SubstrateClient({ url: "http://f", fetch: answer(401, { error: "Unauthorized" }) }).runs()).rejects.toMatchObject({ status: 401, code: "Unauthorized" }) + await expect(new SubstrateClient({ url: "http://f", fetch: answer(502, "bad gateway") }).runs()).rejects.toMatchObject({ status: 502, code: "http-502" }) + }) + it("refuses an answer that does not match the published schema", async () => { + const e = await new SubstrateClient({ url: "http://f", fetch: answer(200, { runs: [{ id: 1 }] }) }).runs().catch((x: unknown) => x) + expect(e).toBeInstanceOf(SubstrateError) + expect((e as SubstrateError).code).toBe("decode") + }) + it("sends the bearer and the Access pair", async () => { + let seen: Headers | undefined + const f = (async (_: unknown, init?: RequestInit) => { seen = new Headers(init?.headers); return new Response(JSON.stringify({ runs: [] })) }) as typeof fetch + await new SubstrateClient({ url: "http://f/", token: "tok", access: { clientId: "id", clientSecret: "sec" }, fetch: f }).runs() + expect([seen!.get("authorization"), seen!.get("cf-access-client-id"), seen!.get("cf-access-client-secret")]).toEqual(["Bearer tok", "id", "sec"]) + }) +}) + +describe("config", () => { + it("reads the file, expands ~, lets the environment override, and refuses a loose secret file", () => { + const home = mkdtempSync(join(tmpdir(), "substrate-home-")) + const cfg = join(home, "c.toml") + writeFileSync(cfg, `floor_url = "https://floor.example"\ntoken_file = "~/tok"\n[puller]\nholder = "coord"\nmax_runs = 2\n`) + const c = loadClientConfig({ SUBSTRATE_CLIENT_CONFIG: cfg }, home) + expect(c).toMatchObject({ floor_url: "https://floor.example", token_file: join(home, "tok"), puller: { holder: "coord", max_runs: 2 }, source: cfg }) + expect(loadClientConfig({ SUBSTRATE_CLIENT_CONFIG: cfg, SUBSTRATE_URL: "http://other" }, home).floor_url).toBe("http://other") + expect(() => clientFromConfig(c)).toThrow(/does not exist/) + writeFileSync(join(home, "tok"), "secret-value\n") + chmodSync(join(home, "tok"), 0o644) + let msg = "" + try { clientFromConfig(c) } catch (e) { expect(e).toBeInstanceOf(ConfigError); msg = (e as Error).message } + expect(msg).toMatch(/chmod 600/) + expect(msg).not.toContain("secret-value") + chmodSync(join(home, "tok"), 0o600) + expect(clientFromConfig(c).url).toBe("https://floor.example") + expect(() => clientFromConfig(loadClientConfig({ SUBSTRATE_CLIENT_CONFIG: join(home, "none.toml") }, home))).toThrow(/no floor_url/) + }) + it("the OpenAPI document declares both credential schemes and no 500 responses", () => { + const d = openApiDocument() as { components: { securitySchemes: Record }; paths: Record } + expect(Object.keys(d.components.securitySchemes).sort()).toEqual(["accessClientId", "accessClientSecret", "bearer"]) + expect(JSON.stringify(d.paths)).not.toContain("\"500\"") + }) +}) + +describe("G-BK5 underLease", () => { + it("sends x-substrate-lease on every request of the derived client, and only there", async () => { + const seen: Array = [] + const fetch = (async (_u: unknown, init?: RequestInit) => { seen.push(new Headers(init?.headers).get("x-substrate-lease")); return new Response("{\"enqueued\":[]}", { status: 201 }) }) as typeof globalThis.fetch + const { SubstrateClient } = await import("../src/client.ts") + const c = new SubstrateClient({ url: "http://f.test", token: "t", fetch }) + await c.request("POST", "/runs/r1/jobs", { json: [] }) + await c.underLease("run-r1-a2", 2).request("POST", "/runs/r1/jobs", { json: [] }) + expect(seen).toEqual([null, "run-r1-a2:2"]) + }) +}) diff --git a/pkgs/substrate-apps/src/packages/api/test/seats.test.ts b/pkgs/substrate-apps/src/packages/api/test/seats.test.ts new file mode 100644 index 00000000..29bd73a7 --- /dev/null +++ b/pkgs/substrate-apps/src/packages/api/test/seats.test.ts @@ -0,0 +1,45 @@ +// The seats oracle verbs (parity with the dotfiles `seats --pick` and `--check`). +import { describe, expect, it } from "vitest" +import { admits, bar, checkSeat, pickSeat } from "../src/seats.ts" +import type { SeatRow } from "../src/client.ts" + +const row = (seat: string, o: Partial = {}): SeatRow => ({ + seat, grade: "MEASURED", staleness: "MEASURED", headroom_pct: 50, next_reset_at: "2026-09-24T12:00:00Z", wip: 0, admit: null, reason: null, ...o +}) + +describe("pickSeat", () => { + it("takes the most headroom among admitting seats", () => { + expect(pickSeat([row("a", { headroom_pct: 20 }), row("b", { headroom_pct: 70 }), row("c", { headroom_pct: 90, admit: false })])!.seat).toBe("b") + }) + it("breaks a headroom tie by fewer jobs in flight, then by name", () => { + expect(pickSeat([row("z", { wip: 2 }), row("y", { wip: 1 }), row("x", { wip: 1 })])!.seat).toBe("x") + }) + it("returns null when nothing admits: an empty floor, all refused, all unknown or stale", () => { + expect(pickSeat([])).toBeNull() + expect(pickSeat([row("a", { admit: false }), row("b", { grade: "UNKNOWN", headroom_pct: null }), row("c", { staleness: "STALE" }), row("d", { headroom_pct: 0 })])).toBeNull() + }) + it("defers to the floor's own admission when it answered", () => { + expect(admits(row("a", { staleness: "STALE", admit: true }))).toBe(true) + expect(admits(row("a", { headroom_pct: 99, admit: false }))).toBe(false) + }) +}) + +describe("checkSeat", () => { + const rows = [row("cc", { headroom_pct: 42.25 }), row("cc2", { admit: false, reason: "headroom 3% below 10%", headroom_pct: 3 }), row("qwen", { grade: "UNKNOWN", headroom_pct: null })] + it("0 with headroom, 1 when refused, 2 when unmeasurable or absent", () => { + expect(checkSeat(rows, "cc")).toEqual({ code: 0, line: "cc: 42.3% headroom, grade MEASURED, resets 2026-09-24T12:00:00Z" }) + expect(checkSeat(rows, "cc2")).toMatchObject({ code: 1, line: expect.stringContaining("refused: headroom 3% below 10%") }) + expect(checkSeat(rows, "qwen")).toMatchObject({ code: 2, line: expect.stringContaining("unmeasurable") }) + expect(checkSeat(rows, "nope")).toEqual({ code: 2, line: "nope: no such seat" }) + }) +}) + +describe("bar", () => { + it("fills in proportion to the used part of the window", () => { + expect(bar(100)).toBe("..........") + expect(bar(0)).toBe("##########") + expect(bar(35)).toBe("#######...") + expect(bar(null)).toBe("??????????") + expect(bar(-5)).toBe("##########") + }) +}) diff --git a/pkgs/substrate-apps/src/packages/api/tsconfig.json b/pkgs/substrate-apps/src/packages/api/tsconfig.json new file mode 100644 index 00000000..40fe4b28 --- /dev/null +++ b/pkgs/substrate-apps/src/packages/api/tsconfig.json @@ -0,0 +1,5 @@ +{ + "extends": "../../tsconfig.base.json", + "compilerOptions": { "types": ["node"], "allowImportingTsExtensions": true, "lib": ["ES2022", "DOM"] }, + "include": ["src/**/*.ts", "test/**/*.ts"] +} diff --git a/pkgs/substrate-apps/src/packages/api/vitest.config.ts b/pkgs/substrate-apps/src/packages/api/vitest.config.ts new file mode 100644 index 00000000..11387e4c --- /dev/null +++ b/pkgs/substrate-apps/src/packages/api/vitest.config.ts @@ -0,0 +1,2 @@ +import { defineConfig } from "vitest/config"; +export default defineConfig({ test: { name: "api", include: ["test/**/*.test.ts"], environment: "node" } }); diff --git a/pkgs/substrate-apps/src/packages/factory/README.md b/pkgs/substrate-apps/src/packages/factory/README.md new file mode 100644 index 00000000..d25d6cac --- /dev/null +++ b/pkgs/substrate-apps/src/packages/factory/README.md @@ -0,0 +1,42 @@ +# @substrate/factory + +U-A11 supplies the host-neutral Factory service over the `PlanningStore` +interface. Tests compose the in-memory store, one constant `PriceVector`, and an +authored fake capacity feed. The package has no Worker entry and no SQLite +binding; those are U-A14's composition-root work. + +The service owns the `unclaimed → released → inflight → closed` lifecycle. +`handOut` makes the first transition, `observeOutcome` applies `Accepted`, +`Rejected`, or `NotYet`, and `observeVerdict` mirrors a chained kernel record, +deduplicates `(executor, seq, hash)`, updates WIP once, and wakes the evaluator. +Only an `inflight` item may consume a verdict; a merely `released` item is +refused with its current state and keeps its WIP unchanged. A non-pass verdict +retries under the plan's `attemptCap` with a new armed-plan-and-attempt dedup +identity; a pass closes the item and can release an item whose `dependsOn` edge +has just become satisfied. + +The service schema-encodes its mutable state under `FACTORY_STATE_KEY` after +each accepted mutation and restores it when another Factory is constructed over +the same `PlanningStore`. Inflight work, WIP, readings, mirrored chain state, +alarms, outcome idempotence, deferrals, heartbeats, receipts, and facts therefore +survive an in-memory recreation today and the U-A14 SQLite Layer later. +The backlog schema includes optional `argv_ref` and `mutation_hint` cells, so +the command reference and evaluator control carried by an armed item survive +that same boundary and are unchanged when the item is proposed after restart. + +`makeFactoryHttpHandler` exposes `/plans`, `/capacity`, `/proposals`, `/outcomes`, +`/verdicts`, `/receipts`, `/heartbeats`, and `/state` without instantiating a +Worker. A proposal pull returns `{ proposals, next_wake_at }`, including the wake +instant when every row is stopped and the proposal list is empty. Every +state-changing request—including that GET proposal pull—requires the bearer +token supplied by the host. Receipt ingestion first uses the strict §2.3 decoder, +then requires `oracle_rc`, `mutation_rc`, and +`oracle_output_sha256` to match one mirrored verdict exactly. + +The `/plans` route accepts the acceptor's producer body +`{ planHash, script_bytes_base64, args_bytes_base64, plan }`, hashes the decoded +script-plus-args bytes, and expands the emitted items into the Factory backlog. +The same plan hash is also the stable internal plan id, so posting the identical +producer payload again is append-only and duplicate-free. + +The dominant checks are documented in [docs/factory.md](../../docs/factory.md). diff --git a/pkgs/substrate-apps/src/packages/factory/package.json b/pkgs/substrate-apps/src/packages/factory/package.json new file mode 100644 index 00000000..6c8c2a34 --- /dev/null +++ b/pkgs/substrate-apps/src/packages/factory/package.json @@ -0,0 +1,30 @@ +{ + "name": "@substrate/factory", + "private": true, + "type": "module", + "version": "0.1.0", + "description": "The Factory Durable Object as a service: backlog, WIP cap, release evaluator, capacity routes. Ported from tally-ts-sdk packages/factory.", + "license": "UNLICENSED", + "exports": { + ".": "./src/index.ts", + "./*": "./src/*" + }, + "scripts": { + "check": "tsc --noEmit -p .", + "test": "vitest run" + }, + "dependencies": { + "@substrate/planning": "workspace:*", + "@substrate/schema": "workspace:*", + "@substrate/serializer": "workspace:*" + }, + "peerDependencies": { + "effect": "4.0.0-rc.112" + }, + "devDependencies": { + "effect": "4.0.0-rc.112", + "fast-check": "^4.9.0", + "typescript": "^5.9.3", + "vitest": "^3.2.4" + } +} diff --git a/pkgs/substrate-apps/src/packages/factory/src/cardWriter.ts b/pkgs/substrate-apps/src/packages/factory/src/cardWriter.ts new file mode 100644 index 00000000..d4d03f42 --- /dev/null +++ b/pkgs/substrate-apps/src/packages/factory/src/cardWriter.ts @@ -0,0 +1,192 @@ +/** + * cardWriter.ts — U-A13 LAKE-PROJECTION, the writer. + * + * §2.2e: "`--write` fills only `actuals`, `result.grade`, `result.observed`, + * `result.receipt_sha256`, `prior_gap`, `outcome_for_calibration` on cards (the + * operator's fields); `status` and `outcome_ruled` never." + * + * LINE-ORIENTED, NOT A YAML EMITTER. A card is a file a human wrote: it carries + * trailing comments (`status: DRAFT # DRAFT -> ARMED -> …`), block + * scalars, and a field order someone chose. Re-emitting its front matter from a + * parsed value would rewrite every one of those and the diff would be a diff of + * this tool's formatting opinions. So the writer finds the ONE line a cell lives + * on and replaces the text after the colon, keeping the trailing comment byte + * for byte; a cell with no line yet is appended to its block. + * + * THE FENCE IS THE VOCABULARY. `applyCells` writes the keys it is handed and has + * no path that constructs a key of its own. `status` and `outcome_ruled` are + * never in `PROJECTION_CELLS`, so no value the tool computes can name them — + * and `fencedLines` lets the caller re-read the result and refuse anyway, + * because a fence that is only an argument about scope is not a fence. + * + * IDEMPOTENT BY CONSTRUCTION. Values render through one function, and a cell + * already carrying its rendered form is replaced by the same bytes. Applying the + * same card twice is a no-op, which is the issue's "Re-applying the same card + * reports zero changes" read at the level of the file. + */ +import { Frontmatter } from "@substrate/schema"; +import { NEVER_WRITTEN } from "./projection.ts"; + +/** One cell value, rendered as the front matter spells it. */ +const renderValue = (value: string | number | boolean | null): string => { + if (value === null) return "null"; + if (typeof value === "number") return String(value); + if (typeof value === "boolean") return value ? "true" : "false"; + return JSON.stringify(value); +}; + +/** + * The trailing `# …` comment of a front-matter line, with the whitespace that + * separated it from the value, or the empty string. + * + * The padding comes along because it is part of what the human wrote: a card + * that reads `status: DRAFT # DRAFT -> ARMED -> …` keeps its column + * when a value beside it changes, and a line with no comment gains no trailing + * space. + */ +const trailingComment = (line: string): string => { + const stripped = Frontmatter.stripComment(line); + const comment = line.slice(stripped.length); + if (comment === "") return ""; + return stripped.slice(stripped.trimEnd().length) + comment; +}; + +const indentOf = (line: string): number => line.length - line.replace(/^ +/, "").length; + +/** The half-open line range of the block opened at `head`, by indentation. */ +const blockRange = ( + lines: ReadonlyArray, + head: number +): { readonly start: number; readonly end: number; readonly indent: number } => { + const indent = indentOf(lines[head] ?? ""); + let end = head + 1; + while (end < lines.length) { + const line = lines[end] ?? ""; + if (line.trim() === "" || line.trimStart().startsWith("#")) { + end += 1; + continue; + } + if (indentOf(line) <= indent) break; + end += 1; + } + // A trailing run of blank or comment lines belongs to whatever comes next, + // not to this block: appending after them would move a comment's subject. + while (end > head + 1) { + const line = lines[end - 1] ?? ""; + if (line.trim() === "" || line.trimStart().startsWith("#")) end -= 1; + else break; + } + return { start: head + 1, end, indent: indent + 2 }; +}; + +/** The index of the line opening `key` at indent 0, or -1. */ +const headOf = (lines: ReadonlyArray, key: string): number => + lines.findIndex((line) => line === `${key}:` || line.startsWith(`${key}: `) || line.startsWith(`${key}:#`)); + +/** Set `key` inside `[start, end)`, or insert it at `end`. Returns the new end. */ +const setInBlock = ( + lines: Array, + start: number, + end: number, + indent: number, + key: string, + value: string | number | boolean | null +): number => { + const prefix = `${" ".repeat(indent)}${key}:`; + for (let i = start; i < end; i++) { + const line = lines[i] ?? ""; + if (!line.startsWith(prefix)) continue; + const rest = line.slice(prefix.length); + if (rest !== "" && !rest.startsWith(" ") && !rest.startsWith("\t") && !rest.startsWith("#")) continue; + lines[i] = `${prefix} ${renderValue(value)}${trailingComment(line)}`; + return end; + } + lines.splice(end, 0, `${" ".repeat(indent)}${key}: ${renderValue(value)}`); + return end + 1; +}; + +/** + * Put the operator's cells on one card's front matter. + * + * @param text - The card, whole. + * @param cells - The cells to set, keyed as `PROJECTION_CELLS` keys them + * (`actuals.tokens`, `result.grade`, `outcome_for_calibration`, …). + * @returns The card with those cells set and nothing else touched. + */ +export const applyCells = ( + text: string, + cells: ReadonlyMap +): string => { + const split = Frontmatter.split(text); + if (split === null) throw new Error("the card opens with no front-matter fence"); + const lines = split.frontmatter.split("\n"); + + const group = (name: string, after: number): number => { + let head = headOf(lines, name); + if (head === -1) { + lines.splice(after, 0, `${name}:`); + head = after; + } + let range = blockRange(lines, head); + for (const [cell, value] of cells) { + if (!cell.startsWith(`${name}.`)) continue; + range = { + ...range, + end: setInBlock(lines, range.start, range.end, range.indent, cell.slice(name.length + 1), value) + }; + } + return range.end; + }; + + const afterActuals = group("actuals", lines.length); + group("result", afterActuals); + + for (const [cell, value] of cells) { + if (cell.includes(".")) continue; + setInBlock(lines, 0, lines.length, 0, cell, value); + } + + return `---\n${lines.join("\n")}\n---\n${split.body}`; +}; + +/** The front-matter lines whose key is one this projection never writes. */ +export const fencedLines = (text: string): ReadonlyArray => { + const split = Frontmatter.split(text); + if (split === null) return []; + return split.frontmatter + .split("\n") + .filter((line) => NEVER_WRITTEN.some((key) => line.startsWith(`${key}:`))); +}; + +/** + * `git diff --stat`'s shape for one file. + * + * Computed here because git is not on the wrapped PATH: `scripts/node-env.sh` + * puts node on it and nothing else, and the evaluator runs the oracle on a bare + * PATH where the only executables are `/usr/bin/env` and `/bin/sh` (U-A2, + * MEASURED). The oracle's parenthetical asks for the reading, not for the + * program that usually prints it. + */ +export const statOf = ( + name: string, + before: string, + after: string +): { readonly name: string; readonly added: number; readonly removed: number } => { + const a = before.split("\n"); + const b = after.split("\n"); + // Longest common subsequence, so an inserted block counts as an insertion and + // not as a rewrite of every line below it. A card is ~120 lines; the quadratic + // table is the honest reading and costs nothing at that size. + const table: Array> = Array.from({ length: a.length + 1 }, () => + new Array(b.length + 1).fill(0) + ); + for (let i = a.length - 1; i >= 0; i--) { + for (let j = b.length - 1; j >= 0; j--) { + table[i]![j] = a[i] === b[j] + ? (table[i + 1]![j + 1] ?? 0) + 1 + : Math.max(table[i + 1]![j] ?? 0, table[i]![j + 1] ?? 0); + } + } + const common = table[0]![0] ?? 0; + return { name, added: b.length - common, removed: a.length - common }; +}; diff --git a/pkgs/substrate-apps/src/packages/factory/src/http.ts b/pkgs/substrate-apps/src/packages/factory/src/http.ts new file mode 100644 index 00000000..40df649d --- /dev/null +++ b/pkgs/substrate-apps/src/packages/factory/src/http.ts @@ -0,0 +1,454 @@ +/** + * The Factory HTTP surface as a library handler. + * + * U-A11 built every route but one and left the Worker entry to U-A14, which + * added `GET /projection` — §2.2b's last unbuilt line — and the store the route + * reads. The handler still instantiates no Worker and names no binding. + */ +import { Effect, Schema } from "effect"; +import { + ArtifactHasher, + type IArtifactHasher +} from "@substrate/planning/objects/hasher.ts"; +import type { IFactory } from "@substrate/planning/objects/factory.ts"; +import { parseAdmitOutcomeReport } from "@substrate/planning/schema/admit.ts"; +import { parseBacklog } from "@substrate/planning/schema/backlog.ts"; +import { parseCapacityReading } from "@substrate/planning/schema/capacity.ts"; +import { + Instant, + parseSeatCapacitySnapshot, + SeatJob +} from "@substrate/planning/schema/seatCapacity.ts"; +import { FactoryError } from "@substrate/planning/schema/errors.ts"; +import { ExecutorId, PlanId } from "@substrate/planning/schema/ids.ts"; +import { parsePlanArm } from "@substrate/planning/schema/plan.ts"; +import { parseVerdict } from "@substrate/planning/schema/records.ts"; +import { Heartbeat } from "@substrate/planning/schema/uplink.ts"; +import type { IPlanningStore } from "@substrate/planning/objects/storage.ts"; +import { serializeValue } from "@substrate/serializer"; +import { NONE } from "./projection.ts"; +import { + mirrorProjectionRows, + projectionBody, + retainedReceiptOf, + retainedReceipts, + retainReceipt +} from "./projectionMirror.ts"; +import { postReceipt } from "./receipt.ts"; + +const parseExecutor = Schema.decodeUnknownEffect(ExecutorId); +const parsePlanId = Schema.decodeUnknownEffect(PlanId); +const parseHeartbeat = Schema.decodeUnknownEffect(Heartbeat); +const parseInstant = Schema.decodeUnknownEffect(Instant); +const parseSeatJob = Schema.decodeUnknownEffect(SeatJob); + +interface FactoryHttpOptions { + readonly factory: IFactory; + readonly token: string; + readonly artifactHasher: IArtifactHasher; + /** + * The same store the object was built over, for `GET /projection` (U-A14). + * + * Optional because the route is the only thing that needs it and a host that + * does not serve the projection should not have to name a store. Without it, + * an accepted receipt's measured cells are not retained and the projection is + * empty — which is an honest answer and not a silent one: an empty projection + * says "this object mirrors no receipt", which is exactly true of a handler + * that was given nowhere to keep one. + */ + readonly store?: IPlanningStore; + /** + * The host's clock, as an RFC 3339 instant. + * + * The object never reads a clock; the host hands one in. It is the default + * `asOf` for `GET /capacity`, the receipt time for pushed seats (a reading + * dated beyond the skew tolerance ahead of it is refused) and the floor under + * every alarm a `POST /capacity` computes. Without it, `POST /capacity/seats` + * and a `POST /capacity` that carries `seats` are refused (501 NoClock) + * rather than accepted against no clock (the publisher's own `published_at` + * is never the receiving clock), and `GET /capacity` requires `asOf`. + */ + readonly now?: () => string; +} + +type FactoryHttpHandler = (request: Request) => Promise; + +const response = (status: number, body: unknown): Response => + new Response(`${JSON.stringify(body)}\n`, { + status, + headers: { "content-type": "application/json; charset=utf-8" } + }); + +/** + * The projection's own response. + * + * Its body is JSON Lines produced by U-A12's serializer and is served verbatim: + * re-encoding it through `response` above would put `JSON.stringify`'s + * insertion order back on the wire and undo the one property the route exists + * to have. + */ +const ndjson = (body: string): Response => + new Response(body, { + status: 200, + headers: { "content-type": "application/x-ndjson; charset=utf-8" } + }); + +const asRecord = (value: unknown): Record => { + if (value === null || typeof value !== "object" || Array.isArray(value)) { + throw new TypeError("request body must be a JSON object"); + } + return value as Record; +}; + +const decodeBase64 = (value: unknown, field: string): Uint8Array => { + if (typeof value !== "string") throw new TypeError(`${field} must be base64 text`); + try { + const binary = atob(value); + return Uint8Array.from(binary, (character) => character.charCodeAt(0)); + } catch { + throw new TypeError(`${field} must be valid base64`); + } +}; + +const concatenate = (...parts: ReadonlyArray): Uint8Array => { + const joined = new Uint8Array(parts.reduce((length, part) => length + part.length, 0)); + let offset = 0; + for (const part of parts) { + joined.set(part, offset); + offset += part.length; + } + return joined; +}; + +const statusFor = (error: unknown): number => { + if (error instanceof FactoryError) { + if (error.reason === "ReceiptMismatch" || error.reason === "ContinuityGap") return 409; + if (error.reason === "InvalidTransition" || error.reason === "DedupMismatch") return 409; + if (error.reason === "ReceiptInvalid") return 400; + if (error.reason === "UnknownTask" || error.reason === "PlanNotArmed") return 404; + } + return 400; +}; + +const errorBody = (error: unknown): Record => { + if (error instanceof FactoryError) { + return { error: error.reason, operation: error.operation, subject: error.subject }; + } + if ( + error !== null && + typeof error === "object" && + "_tag" in error && + typeof (error as { readonly _tag?: unknown })._tag === "string" + ) { + return { error: String((error as { readonly _tag: string })._tag) }; + } + return { error: error instanceof Error ? error.message : "invalid request" }; +}; + +/** + * Builds the route handler without creating a Worker entry or selecting a + * persistence binding. Every state-changing request checks the supplied bearer + * token, including the proposal pull that releases backlog items. + */ +export const makeFactoryHttpHandler = (options: FactoryHttpOptions): FactoryHttpHandler => { + const run = (effect: Effect.Effect): Promise => + Effect.runPromise( + effect.pipe( + Effect.provideService( + ArtifactHasher, + ArtifactHasher.of(options.artifactHasher) + ) + ) + ); + + /** `?asOf=` when given (validated), else the host's clock, else nothing. */ + const asOfFrom = async (url: URL): Promise => { + const raw = url.searchParams.get("asOf"); + if (raw !== null) return Effect.runPromise(parseInstant(raw)); + return options.now?.(); + }; + + return async (request: Request): Promise => { + const url = new URL(request.url); + const changesState = + (request.method !== "GET" && request.method !== "HEAD") || + (request.method === "GET" && url.pathname === "/proposals"); + if ( + changesState && + request.headers.get("authorization") !== `Bearer ${options.token}` + ) { + // Drain before refusing. The refusal itself needs nothing from the body, + // but a Worker that answers while the request stream is still open makes + // workerd log `Can't read from request stream after response has been + // sent` — an uncaught TypeError in the dev log of an otherwise green + // smoke, which is exactly the kind of noise a reader learns to ignore. + await request.arrayBuffer().catch(() => undefined); + return response(401, { error: "Unauthorized" }); + } + + try { + if (request.method === "GET" && url.pathname === "/projection") { + if (options.store === undefined) return ndjson(""); + const retained = await Effect.runPromise(retainedReceipts(options.store)); + const state = await run(options.factory.stateView); + return ndjson(projectionBody(mirrorProjectionRows(retained, state.receipts))); + } + + if (request.method === "GET" && url.pathname === "/state") { + return response(200, await run(options.factory.stateView)); + } + + if (request.method === "GET" && url.pathname === "/proposals") { + const raw = url.searchParams.get("executor"); + if (raw === null) return response(400, { error: "executor is required" }); + const executor = await Effect.runPromise(parseExecutor(raw)); + const proposals = await run(options.factory.handOut(executor)); + const state = await run(options.factory.stateView); + return response(200, { + proposals, + next_wake_at: state.alarm_at + }); + } + + if (request.method === "POST" && url.pathname === "/capacity") { + const body = await request.json(); + const record = asRecord(body); + const { seats, ...reading } = await Effect.runPromise( + parseCapacityReading(record["reading"] ?? body) + ); + const asOf = options.now?.(); + // Seats need a receiving clock, the same as on POST /capacity/seats. + // Refused whole, before anything is folded in, so the two doors agree. + if (seats !== undefined && asOf === undefined) { + return response(501, { error: "NoClock", subject: "/capacity" }); + } + // The seats are the ledger's: folded in on their own monotonic rule, + // whatever becomes of the reading's seq. + const seatReport = + seats === undefined || asOf === undefined + ? undefined + : await run(options.factory.observeSeats(seats, asOf)); + const accepted = await run(options.factory.observeCapacity(reading, asOf)); + return response(200, seatReport === undefined ? { accepted } : { accepted, seats: seatReport }); + } + + if (request.method === "POST" && url.pathname === "/capacity/seats") { + const receivedAt = options.now?.(); + if (receivedAt === undefined) { + await request.arrayBuffer().catch(() => undefined); + return response(501, { error: "NoClock", subject: "/capacity/seats" }); + } + const body = asRecord(await request.json()); + const snapshot = await Effect.runPromise( + parseSeatCapacitySnapshot(body["snapshot"] ?? body) + ); + return response(200, await run(options.factory.observeSeats(snapshot, receivedAt))); + } + + if (request.method === "GET" && url.pathname === "/capacity") { + const asOf = await asOfFrom(url); + if (asOf === undefined) return response(400, { error: "asOf is required" }); + return response(200, await run(options.factory.capacityAt(asOf))); + } + + if (request.method === "GET" && url.pathname === "/capacity/admit") { + const seat = url.searchParams.get("seat"); + if (seat === null) return response(400, { error: "seat is required" }); + const asOf = await asOfFrom(url); + if (asOf === undefined) return response(400, { error: "asOf is required" }); + const minimum = url.searchParams.get("min_headroom_pct"); + const job = await Effect.runPromise( + parseSeatJob({ + model: url.searchParams.get("model"), + min_headroom_pct: minimum === null ? 0 : Number(minimum) + }) + ); + return response(200, await run(options.factory.admitSeat(seat, job, asOf))); + } + + if (request.method === "POST" && url.pathname === "/outcomes") { + const report = await Effect.runPromise( + parseAdmitOutcomeReport(await request.json()) + ); + return response(200, await run(options.factory.observeOutcome(report))); + } + + if (request.method === "POST" && url.pathname === "/verdicts") { + const verdict = await Effect.runPromise(parseVerdict(await request.json())); + return response(200, await run(options.factory.observeVerdict(verdict))); + } + + if (request.method === "POST" && url.pathname === "/receipts") { + const body = asRecord(await request.json()); + const verdictHash = body["verdict_hash"]; + if (typeof verdictHash !== "string") { + return response(400, { error: "verdict_hash is required" }); + } + let receipt: unknown = body["receipt"]; + if (receipt === undefined) { + const inline = { ...body }; + delete inline["verdict_hash"]; + receipt = inline; + } + const observation = await run(postReceipt(options.factory, receipt, verdictHash)); + + // Accepted. The three evidence cells are now in the mirror; the + // measured cells (`seconds`, `tokens`, `disposition`) live only in + // these bytes, so they are retained beside the object's state for + // `GET /projection`. The digest is taken over the receipt's CANONICAL + // bytes — U-A12's serializer — so the same receipt yields the same + // `receipt_sha256` whichever order its keys arrived in. + if (options.store !== undefined) { + const path = body["receipt_path"]; + const canonical = serializeValue(receipt); + const digest = await Effect.runPromise( + options.artifactHasher.hash( + new TextEncoder().encode(canonical), + "receipt" + ) + ); + const retained = retainedReceiptOf( + receipt, + typeof path === "string" ? path : NONE, + digest + ); + if (retained.ok) { + await Effect.runPromise(retainReceipt(options.store, retained.retained)); + } + } + return response(200, observation); + } + + if (request.method === "POST" && url.pathname === "/heartbeats") { + const body = asRecord(await request.json()); + const heartbeat = await Effect.runPromise(parseHeartbeat(body["heartbeat"] ?? body)); + const lane = typeof body["lane"] === "string" ? body["lane"] : undefined; + const observedAt = + typeof body["observed_at"] === "string" ? body["observed_at"] : undefined; + await run(options.factory.observeHeartbeat(heartbeat, lane, observedAt)); + return response(200, { accepted: true }); + } + + if (request.method === "POST" && url.pathname === "/plans") { + const body = asRecord(await request.json()); + + // The acceptor owns this producer shape. Its planHash is over the + // concatenated script and args bytes, and its plan items are enriched + // here with the release-state fields that are Factory concerns. + if (body["plan"] !== undefined) { + const plan = asRecord(body["plan"]); + const meta = plan["meta"] === undefined ? {} : asRecord(plan["meta"]); + const arm = await Effect.runPromise( + parsePlanArm({ + label: + typeof meta["name"] === "string" && meta["name"] !== "none" + ? meta["name"] + : "conwip plan arm", + kind: "workflow", + level: plan["level"], + namespace: plan["namespace"], + planHash: body["planHash"], + author: + typeof plan["acceptor"] === "string" + ? plan["acceptor"] + : "conwip plan arm" + }) + ); + const planId = await Effect.runPromise(parsePlanId(arm.planHash)); + if (plan["planHash"] !== arm.planHash) { + throw new FactoryError({ + reason: "PlanHashMismatch", + operation: "FactoryHttp.POST /plans", + subject: planId + }); + } + + const rawItems = plan["items"]; + if (!Array.isArray(rawItems)) throw new TypeError("plan.items must be an array"); + const current = await run(options.factory.releaseState); + const itemRecords = rawItems.map((value) => asRecord(value)); + const decodedItems = await Effect.runPromise( + parseBacklog( + itemRecords.map((item) => { + if (item["planHash"] !== arm.planHash) { + throw new FactoryError({ + reason: "PlanHashMismatch", + operation: "FactoryHttp.POST /plans", + subject: planId + }); + } + const namespace = item["namespace"] ?? arm.namespace; + const namespaceValue = current.namespaces.namespaces.find( + (entry) => entry.name === namespace + )?.value; + const mutationHint = item["mutation_hint"]; + const argvRef = item["argv_ref"]; + return { + taskId: item["taskId"], + planId, + namespace, + family: item["family"] ?? "build", + level: item["level"] ?? arm.level, + rank: item["rank"], + needs: item["needs"], + dependsOn: item["dependsOn"] ?? [], + evidence: item["evidence"] ?? [], + briefHash: item["briefHash"], + planHash: item["planHash"], + dedupKey: item["dedupKey"], + state: item["state"], + deferrals: item["deferrals"] ?? 0, + attempt: item["attempt"] ?? 1, + outcome: item["outcome"] ?? null, + value: item["value"] ?? namespaceValue ?? 1, + estimate: item["estimate"] ?? current.estimates.mainEffect, + gate: item["gate"], + subassembly: item["subassembly"] ?? null, + dueBy: item["dueBy"] ?? null, + float: item["float"] ?? null, + envelopeParent: item["envelopeParent"] ?? null, + ...(typeof mutationHint === "string" && mutationHint !== "none" + ? { mutation_hint: mutationHint } + : {}), + ...(typeof argvRef === "string" ? { argv_ref: argvRef } : {}) + }; + }) + ) + ); + const bytes = concatenate( + decodeBase64(body["script_bytes_base64"], "script_bytes_base64"), + decodeBase64(body["args_bytes_base64"], "args_bytes_base64") + ); + const row = await run( + options.factory.armPlan(arm, { planId, bytes, items: decodedItems }) + ); + return response(200, row); + } + + const artifactBody = asRecord(body["artifact"]); + const arm = await Effect.runPromise(parsePlanArm(body["arm"])); + const planId = await Effect.runPromise(parsePlanId(artifactBody["planId"])); + const items = await Effect.runPromise(parseBacklog(artifactBody["items"])); + const bytes = artifactBody["bytes"]; + if ( + !Array.isArray(bytes) || + !bytes.every((entry) => Number.isInteger(entry) && entry >= 0 && entry <= 255) + ) { + return response(400, { error: "artifact.bytes must be byte values" }); + } + const row = await run( + options.factory.armPlan(arm, { + planId, + bytes: Uint8Array.from(bytes as ReadonlyArray), + items + }) + ); + return response(200, row); + } + + return response(404, { error: "NotFound" }); + } catch (error) { + return response(statusFor(error), errorBody(error)); + } + }; +}; diff --git a/pkgs/substrate-apps/src/packages/factory/src/index.ts b/pkgs/substrate-apps/src/packages/factory/src/index.ts new file mode 100644 index 00000000..2ae8d0a6 --- /dev/null +++ b/pkgs/substrate-apps/src/packages/factory/src/index.ts @@ -0,0 +1,11 @@ +/** Public surface of the host-neutral Factory package. */ +export * from "@substrate/planning/objects/factory.ts"; +export * from "@substrate/planning/objects/storage.ts"; +export * from "@substrate/planning/objects/hasher.ts"; +export * from "./inputs.ts"; +export * from "./receipt.ts"; +export * from "./http.ts"; +export * from "./projection.ts"; +export * from "./cardWriter.ts"; +export * from "./projectionLoad.ts"; +export * from "./projectionMirror.ts"; diff --git a/pkgs/substrate-apps/src/packages/factory/src/inputs.ts b/pkgs/substrate-apps/src/packages/factory/src/inputs.ts new file mode 100644 index 00000000..7015cdbd --- /dev/null +++ b/pkgs/substrate-apps/src/packages/factory/src/inputs.ts @@ -0,0 +1,130 @@ +/** + * Host-neutral input Layers used to compose the Factory. + * + * Capacity arrives two ways and both cross `FactoryCapacityFeed`: authored + * readings and seat snapshots (`fakeCapacityLayer`, the hermetic bench) and the + * seats the capacity ledger holds in the object's own store + * (`ledgerCapacityFeedLayer`, backed by Durable Object storage in the Worker and + * by the in-memory store in tests). + */ +import { Context, Effect, Layer } from "effect"; +import { + CapacityLedger, + capacityLedgerLayer +} from "@substrate/planning/capacity/ledger.ts"; +import { PlanningStore } from "@substrate/planning/objects/storage.ts"; +import type { CapacityReading } from "@substrate/planning/schema/capacity.ts"; +import { + SEAT_CAPACITY_SCHEMA_VERSION, + type SeatCapacitySnapshot +} from "@substrate/planning/schema/seatCapacity.ts"; +import type { PriceVector } from "@substrate/planning/schema/prices.ts"; +import { Factory } from "@substrate/planning/objects/factory.ts"; +import { FactoryError } from "@substrate/planning/schema/errors.ts"; + +interface IFactoryCapacityFeed { + readonly readings: Effect.Effect>; + /** Per-seat capacity snapshots (`seat-capacity/2`). */ + readonly seats: Effect.Effect>; +} + +/** A capacity source; production and test sources cross the same interface. */ +export class FactoryCapacityFeed extends Context.Service< + FactoryCapacityFeed, + IFactoryCapacityFeed +>()("@substrate/factory/FactoryCapacityFeed") {} + +interface IFactoryPriceSource { + readonly prices: PriceVector; +} + +/** The immutable price vector used for every evaluation in one composition. */ +export class FactoryPriceSource extends Context.Service< + FactoryPriceSource, + IFactoryPriceSource +>()("@substrate/factory/FactoryPriceSource") {} + +/** Provides authored fake readings and seat snapshots without a live capacity plane. */ +export const fakeCapacityLayer = ( + readings: ReadonlyArray, + seats: ReadonlyArray = [] +): Layer.Layer => + Layer.succeed(FactoryCapacityFeed)( + FactoryCapacityFeed.of({ readings: Effect.succeed(readings), seats: Effect.succeed(seats) }) + ); + +/** + * The real capacity feed: the seats the capacity ledger holds, one snapshot per + * publishing host, read from whichever `PlanningStore` is provided. + * + * Readings are not replayed from here: the kernel's readings are the Factory's + * own stored state and arrive by `POST /capacity`. A store that cannot be read + * yields no seats, which the admission predicate answers `unknown-seat`, so a + * storage failure can only ever refuse work, never admit it. + */ +const ledgerCapacityFeedLayer: Layer.Layer = + Layer.effect( + FactoryCapacityFeed, + Effect.gen(function* () { + const ledger = yield* CapacityLedger; + const seats = ledger.latest.pipe( + Effect.map((held) => { + const byHost = new Map>(); + for (const entry of held) byHost.set(entry.host, [...(byHost.get(entry.host) ?? []), entry]); + return [...byHost].map( + ([host, entries]): SeatCapacitySnapshot => ({ + schema_version: SEAT_CAPACITY_SCHEMA_VERSION, + host, + published_at: entries + .map((entry) => entry.published_at) + .reduce((latest, next) => (Date.parse(next) > Date.parse(latest) ? next : latest)), + seats: entries.map((entry) => entry.seat) + }) + ); + }), + Effect.orElseSucceed((): ReadonlyArray => []) + ); + return FactoryCapacityFeed.of({ readings: Effect.succeed([]), seats }); + }) + ); + +/** The real capacity feed over the provided store, ledger included. */ +export const storedCapacityLayer: Layer.Layer = + ledgerCapacityFeedLayer.pipe(Layer.provide(capacityLedgerLayer)); + +/** Provides one constant, authored price vector. */ +export const constantPriceVectorLayer = ( + prices: PriceVector +): Layer.Layer => + Layer.succeed(FactoryPriceSource)(FactoryPriceSource.of({ prices })); + +/** + * Wires the service to its constant price vector and fake/read-only capacity + * source. The production Worker supplies different Layers; the state machine is + * unchanged. + * + * @param receivedAt the receiving clock for seat snapshots. Required when the + * feed carries any: a snapshot's own `published_at` is never the receipt + * time, so without a clock seats are refused `NoClock`. Re-feeding what the + * ledger already holds is a no-op (idempotent). + */ +export const loadFactoryInputs = (receivedAt?: string) => + Effect.gen(function* () { + const factory = yield* Factory; + const capacity = yield* FactoryCapacityFeed; + const price = yield* FactoryPriceSource; + const current = yield* factory.releaseState; + yield* factory.restore({ ...current, prices: price.prices }); + const readings = yield* capacity.readings; + for (const reading of readings) yield* factory.observeCapacity(reading, receivedAt); + const seats = yield* capacity.seats; + if (seats.length > 0) { + if (receivedAt === undefined) { + return yield* Effect.fail( + new FactoryError({ reason: "NoClock", operation: "loadFactoryInputs", subject: "capacity/seats" }) + ); + } + for (const snapshot of seats) yield* factory.observeSeats(snapshot, receivedAt); + } + return yield* factory.stateView; + }); diff --git a/pkgs/substrate-apps/src/packages/factory/src/projection.ts b/pkgs/substrate-apps/src/packages/factory/src/projection.ts new file mode 100644 index 00000000..435b046e --- /dev/null +++ b/pkgs/substrate-apps/src/packages/factory/src/projection.ts @@ -0,0 +1,410 @@ +/** + * projection.ts — U-A13 LAKE-PROJECTION, the part that touches nothing. + * + * §2.2e names one tool, `tools/ladder-projection.mjs`, and one thing it fills: + * "`--write` fills only `actuals`, `result.grade`, `result.observed`, + * `result.receipt_sha256`, `prior_gap`, `outcome_for_calibration` on cards (the + * operator's fields); `status` and `outcome_ruled` never." + * + * This module is the operator's fields as a value. It reads no path, opens no + * socket and writes no byte: the tool reads the register and hands what it read + * in here as data, and the tool writes back what comes out. Everything that can + * be decided without the disk is decided here so the suite can decide it too. + * + * THE THREE THINGS IT KNOWS + * + * 1. `bankedRowOf` — what one banked receipt says, normalised. The banked + * corpus is heterogeneous: MEASURED 2026-09-06 over + * `receipts/FACTORY-2026-09-06/*​/receipt.json`, 52 files carry `unit`, + * `disposition`, `verdict`, `oracle_rc`, `tokens{6 cells}` and `seconds`; + * 27 also carry `outcome_for_calibration`, `prior_gap` and `mutation_rc`; + * 47 carry `oracle_output_sha256`; and 0 of 52 decode under the strict + * §2.3 `Receipt` (U-A16) — they are the factory run's own receipts, written + * before that type existed. A projection that refused them would project + * nothing, so this reader takes the cells that are there, names the ones + * that are not, and never invents one. + * + * 2. `projectionRowOf` — the operator's fields for one unit, built from the + * banked cells AND from what the Factory object handed back. The evidence + * cells (`result.observed`, `result.grade`) come from the mirror, not from + * the file: a row exists only for a unit the object accepted, which is what + * makes `--negative-control drop-one-receipt` visible at all. + * + * 3. `diffProjections` — one line per disagreement, in a fixed order. Empty + * output is the green; every line names the unit and the cell. + * + * WHAT IS NOT HERE. `status` and `outcome_ruled` are absent from + * `PROJECTION_CELLS`, from `cardCellsOf` and from every writer path — not + * guarded against, absent. A field the writer has no name for cannot be written + * by a mistake in the writer. + */ + +/** The one release cycle this projection covers. */ +export const RELEASE_CYCLE = "FACTORY-2026-09-06"; + +/** + * The card cells `--write` may fill, spelled as §2.2e spells them. + * + * `status` and `outcome_ruled` are not here and never are. The tool's writer + * takes its whole vocabulary from this list. + */ +export const PROJECTION_CELLS: ReadonlyArray = [ + "actuals.tokens", + "actuals.seconds", + "actuals.outcome", + "actuals.receipt_sha256", + "actuals.prior_gap", + "result.grade", + "result.observed", + "result.receipt_sha256", + "outcome_for_calibration" +]; + +/** The cells no projection may ever write, named so the suite can assert it. */ +export const NEVER_WRITTEN: ReadonlyArray = ["status", "outcome_ruled"]; + +/** The string D-B22 gives an inapplicable field: never a null, never a guess. */ +export const NONE = "none"; + +/** One banked receipt, normalised to the cells the projection uses. */ +export interface BankedRow { + /** The card id the receipt is banked under — the directory name. */ + readonly unit: string; + /** The receipt path, relative to the register root. */ + readonly receipt_path: string; + /** sha256 over the receipt's bytes, computed by the caller. */ + readonly receipt_sha256: string; + /** `oracle_rc`, the gate metric of every card in this cycle. */ + readonly oracle_rc: number; + /** `oracle_output_sha256`, or `"none"` where the receipt carries none. */ + readonly oracle_output_sha256: string; + /** `mutation_rc` where the receipt carries one. */ + readonly mutation_rc: number | null; + /** `verdict`, normalised to the mirror's vocabulary. */ + readonly outcome: "pass" | "fail"; + /** `disposition`, verbatim. */ + readonly disposition: string; + /** `seconds`, measured. */ + readonly seconds: number; + /** `tokens.total`, the cell the ladder's `actuals.tokens` carries. */ + readonly tokens: number; + /** `tokens.out`, the declared cell (D-B19). */ + readonly tokens_out: number; + /** `outcome_for_calibration` where the receipt carries one. */ + readonly outcome_for_calibration: string | null; + /** The declared-cell prior gap, where it can be read or derived. */ + readonly prior_gap: number | null; + /** Which cells the receipt did not carry. Reported, never filled. */ + readonly absent: ReadonlyArray; +} + +/** The operator's fields for one unit, as one canonical record. */ +export interface ProjectionRow { + readonly actuals: { + readonly outcome: string; + readonly prior_gap: number | null; + readonly receipt_sha256: string; + readonly seconds: number; + readonly tokens: number; + }; + readonly outcome_for_calibration: string | null; + readonly receipt_path: string; + readonly result: { + readonly grade: "MEASURED" | "CLAIMED"; + readonly observed: number; + readonly receipt_sha256: string; + }; + readonly unit: string; +} + +const isRecord = (value: unknown): value is Record => + value !== null && typeof value === "object" && !Array.isArray(value); + +const numberAt = (source: Record, key: string): number | null => { + const value = source[key]; + return typeof value === "number" && Number.isFinite(value) ? value : null; +}; + +const stringAt = (source: Record, key: string): string | null => { + const value = source[key]; + return typeof value === "string" && value !== "" ? value : null; +}; + +/** + * The declared-cell prior gap. + * + * The corpus MEASURED two spellings of `prior_gap`: the strict §2.3 block + * `{tokens_by_cell{out, in_uncached+out, total}, ...}` (1 receipt) and the older + * flat `{tokens_out, p_pass_vs_outcome}` (26). Both name the same cell, `out`, + * which is the cell every prior in this cycle declares (D-B19), so both read as + * one number. A receipt carrying neither yields `null` and is named in `absent` + * — the gap is not derived from a prior the receipt does not carry, because a + * number computed against an unknown prediction is not a measurement. + */ +export const priorGapOf = (value: unknown): number | null => { + if (!isRecord(value)) return null; + const flat = numberAt(value, "tokens_out"); + if (flat !== null) return flat; + const byCell = value["tokens_by_cell"]; + if (isRecord(byCell)) { + const out = numberAt(byCell, "out"); + if (out !== null) return out; + } + return null; +}; + +/** The verdict vocabulary of the banked corpus, mapped onto the mirror's. */ +const outcomeOf = (verdict: unknown): "pass" | "fail" => + typeof verdict === "string" && verdict.trim().toUpperCase() === "PASS" ? "pass" : "fail"; + +/** + * Normalise one banked receipt. + * + * @param unit - The card id the receipt is banked under. + * @param receiptPath - The path, relative to the register root. + * @param receiptSha256 - sha256 over the file's bytes. + * @param parsed - The receipt's JSON, already parsed. + * @returns The row, or the reason it is not one. A receipt missing a cell the + * projection cannot do without (`oracle_rc`, `seconds`, `tokens.total`, + * `disposition`, `verdict`) is refused by name rather than filled with a zero. + */ +export const bankedRowOf = ( + unit: string, + receiptPath: string, + receiptSha256: string, + parsed: unknown +): { readonly ok: true; readonly row: BankedRow } | { readonly ok: false; readonly why: string } => { + if (!isRecord(parsed)) return { ok: false, why: "the receipt is not a JSON object" }; + const missing: Array = []; + const absent: Array = []; + + const oracleRc = numberAt(parsed, "oracle_rc"); + if (oracleRc === null) missing.push("oracle_rc"); + const seconds = numberAt(parsed, "seconds"); + if (seconds === null) missing.push("seconds"); + const disposition = stringAt(parsed, "disposition"); + if (disposition === null) missing.push("disposition"); + const verdict = stringAt(parsed, "verdict"); + if (verdict === null) missing.push("verdict"); + + const tokensBlock = parsed["tokens"]; + const tokens = isRecord(tokensBlock) ? numberAt(tokensBlock, "total") : null; + const tokensOut = isRecord(tokensBlock) ? numberAt(tokensBlock, "out") : null; + if (tokens === null) missing.push("tokens.total"); + if (tokensOut === null) missing.push("tokens.out"); + + if (missing.length > 0) { + return { ok: false, why: `carries no ${missing.join(", ")}` }; + } + + const outputSha = stringAt(parsed, "oracle_output_sha256"); + if (outputSha === null) absent.push("oracle_output_sha256"); + const mutationRc = numberAt(parsed, "mutation_rc"); + if (mutationRc === null) absent.push("mutation_rc"); + const forCalibration = stringAt(parsed, "outcome_for_calibration"); + if (forCalibration === null) absent.push("outcome_for_calibration"); + const gap = priorGapOf(parsed["prior_gap"]); + if (gap === null) absent.push("prior_gap"); + + return { + ok: true, + row: { + unit, + receipt_path: receiptPath, + receipt_sha256: receiptSha256, + oracle_rc: oracleRc as number, + oracle_output_sha256: outputSha ?? NONE, + mutation_rc: mutationRc, + outcome: outcomeOf(verdict), + disposition: disposition as string, + seconds: seconds as number, + tokens: tokens as number, + tokens_out: tokensOut as number, + outcome_for_calibration: forCalibration, + prior_gap: gap, + absent + } + }; +}; + +/** + * The banked cells one projection row is built from. + * + * A `BankedRow` is one of these and more: the extra cells (`oracle_rc`, + * `oracle_output_sha256`, `mutation_rc`, `outcome`) are the evidence cells, and + * the row below never reads them from the file — it reads them from the mirror, + * which is the whole mechanism of `--negative-control drop-one-receipt`. Naming + * the smaller shape lets the Worker serve `GET /projection` from the receipts it + * has itself accepted (U-A14) through this one definition of the operator's + * fields, rather than a second copy of it. + */ +type ProjectionCells = Pick< + BankedRow, + | "unit" + | "receipt_path" + | "receipt_sha256" + | "disposition" + | "seconds" + | "tokens" + | "outcome_for_calibration" + | "prior_gap" +>; + +/** What the Factory object handed back for one unit. */ +export interface MirroredEvidence { + readonly id: string; + readonly oracle_rc: number; + readonly oracle_output_sha256: string; + readonly mutation_rc?: number; + readonly verdict_hash: string; +} + +/** + * The operator's fields for one unit. + * + * `result.observed` and `result.grade` are read off the MIRROR — the evidence + * the object accepted — and not off the file the tool read. That is the whole + * mechanism of the negative control: a receipt dropped from the load is a unit + * the object never accepted, so no row exists for it, so `--diff` names it. + * + * `result.grade` is `MEASURED` when the object holds the receipt's evidence and + * the receipt carries a measured token total; `CLAIMED` otherwise. Nothing here + * decides `KEEP` — that is Tom's, and `outcome_ruled` is never written. + */ +export const projectionRowOf = ( + banked: ProjectionCells, + mirrored: MirroredEvidence +): ProjectionRow => ({ + actuals: { + outcome: banked.disposition, + prior_gap: banked.prior_gap, + receipt_sha256: banked.receipt_sha256, + seconds: banked.seconds, + tokens: banked.tokens + }, + outcome_for_calibration: banked.outcome_for_calibration, + receipt_path: banked.receipt_path, + result: { + grade: banked.tokens > 0 ? "MEASURED" : "CLAIMED", + observed: mirrored.oracle_rc, + receipt_sha256: banked.receipt_sha256 + }, + unit: banked.unit +}); + +/** The cells `--write` sets on one card, keyed by their §2.2e spelling. */ +export const cardCellsOf = (row: ProjectionRow): ReadonlyMap => + new Map([ + ["actuals.tokens", row.actuals.tokens], + ["actuals.seconds", row.actuals.seconds], + ["actuals.outcome", row.actuals.outcome], + ["actuals.receipt_sha256", row.actuals.receipt_sha256], + ["actuals.prior_gap", row.actuals.prior_gap], + ["result.grade", row.result.grade], + ["result.observed", row.result.observed], + ["result.receipt_sha256", row.result.receipt_sha256], + ["outcome_for_calibration", row.outcome_for_calibration] + ]); + +/** One card as the diff needs to see it: its id, whether it maps, its cells. */ +export interface CardView { + readonly id: string; + /** `repo:` on the card, or null. §2.2e's `unmapped`. */ + readonly repo: string | null; + /** The card's current value for each cell of `PROJECTION_CELLS`. */ + readonly cells: ReadonlyMap; +} + +const cellText = (value: string | number | boolean | null): string => + value === null ? "null" : String(value); + +/** + * Whether a card cell is unwritten. + * + * A card minted by U-E8 carries `actuals: {tokens: null, ...}` and no `result:` + * block at all: the operator's fields exist and are empty, which is what "not + * yet run" looks like. An unwritten cell is not a disagreement — it is what + * `--write` is for — so the diff is silent about it and loud about a cell that + * carries a DIFFERENT value. + */ +const unwritten = (value: string | number | boolean | null | undefined): boolean => + value === undefined || value === null || value === "" || value === NONE; + +/** + * Every disagreement between the mirror and the register, one line each. + * + * The line vocabulary, in the order the lines are emitted: + * + * `missing ` the register banks a receipt the mirror has no + * row for — the object never accepted it. This is + * what `drop-one-receipt` produces. + * `extra ` the mirror holds a row the register does not + * bank. The lake is a mirror and never a ledger; + * a row without a banked receipt is the lake + * having authored something. + * `cell mirror= register=` + * two projections of the same receipt disagree. + * `contradiction card= projection=` + * the card already carries a DIFFERENT value in + * a cell `--write` would fill. `--write` refuses + * to overwrite it; the operator settles it. + * `unmapped ` a card carries no `repo:` (§2.2e: expected to + * name 89 against the main checkout until + * TL-12's merge; 0 against the named worktree). + * + * @param mirror - Rows read back out of the Factory object. + * @param register - Rows computed straight off the register's banked receipts. + * @param cards - The register's ladder cards. + * @returns The lines, sorted within each kind by unit id. Empty is the green. + */ +export const diffProjections = ( + mirror: ReadonlyArray, + register: ReadonlyArray, + cards: ReadonlyArray +): ReadonlyArray => { + const lines: Array = []; + const byUnit = new Map(mirror.map((row) => [row.unit, row])); + const registerByUnit = new Map(register.map((row) => [row.unit, row])); + + for (const row of [...register].sort((a, b) => a.unit.localeCompare(b.unit))) { + if (!byUnit.has(row.unit)) lines.push(`missing ${row.unit} ${row.receipt_path}`); + } + for (const row of [...mirror].sort((a, b) => a.unit.localeCompare(b.unit))) { + if (!registerByUnit.has(row.unit)) lines.push(`extra ${row.unit}`); + } + for (const row of [...mirror].sort((a, b) => a.unit.localeCompare(b.unit))) { + const other = registerByUnit.get(row.unit); + if (other === undefined) continue; + const here = cardCellsOf(row); + const there = cardCellsOf(other); + for (const cell of PROJECTION_CELLS) { + const a = here.get(cell) ?? null; + const b = there.get(cell) ?? null; + if (cellText(a) !== cellText(b)) { + lines.push(`cell ${row.unit} ${cell} mirror=${cellText(a)} register=${cellText(b)}`); + } + } + } + + const cardById = new Map(cards.map((card) => [card.id, card])); + for (const row of [...mirror].sort((a, b) => a.unit.localeCompare(b.unit))) { + const card = cardById.get(row.unit); + if (card === undefined) continue; + for (const [cell, projected] of cardCellsOf(row)) { + const current = card.cells.get(cell); + if (unwritten(current)) continue; + if (projected === null) continue; + if (cellText(current ?? null) !== cellText(projected)) { + lines.push( + `contradiction ${row.unit} ${cell} card=${cellText(current ?? null)} projection=${cellText(projected)}` + ); + } + } + } + + for (const card of [...cards].sort((a, b) => a.id.localeCompare(b.id))) { + if (card.repo === null || card.repo === "") lines.push(`unmapped ${card.id}`); + } + + return lines; +}; diff --git a/pkgs/substrate-apps/src/packages/factory/src/projectionLoad.ts b/pkgs/substrate-apps/src/packages/factory/src/projectionLoad.ts new file mode 100644 index 00000000..e5e75953 --- /dev/null +++ b/pkgs/substrate-apps/src/packages/factory/src/projectionLoad.ts @@ -0,0 +1,186 @@ +/** + * projectionLoad.ts — U-A13 LAKE-PROJECTION, the load. + * + * `ACTION-PLAN.md` U-A13, verbatim: "loads the banked corpus into the Factory + * object under the in-memory storage Layer and projects the mirror back". This + * is that sentence, and it is the reason the projection is not a `cat` of the + * register into a different file. + * + * THE CORPUS GOES THROUGH THE STATE MACHINE, NOT AROUND IT. Every banked + * receipt is walked through the same four states an item walks in production — + * `unclaimed → released → inflight → closed` — by the same calls the uplink + * makes: + * + * observeCapacity(reading) a reading is what makes anything admissible + * handOut(executor) the release evaluator picks; this module does not + * observeOutcome(Accepted) the kernel's answer, which is what makes it inflight + * observeVerdict(verdict) the mirrored witness record, chained + * observeReceipt(evidence) §2.2b's exact chain match + * + * The object refuses a receipt whose three evidence cells do not equal a + * mirrored verdict's (`Factory.observeReceipt`, reason `ReceiptMismatch`) and + * refuses a verdict that breaks its executor's chain (`ContinuityGap`). So a + * unit reaches the mirror only if its evidence survived both, and the mirror is + * the thing the projection reads. A receipt dropped from the load is a unit the + * object never accepted, no row is projected for it, and `--diff` names it: + * that is the whole of `mutation_hint`, "drop one banked receipt → --diff + * non-empty", implemented rather than asserted. + * + * THIS MODULE AUTHORS NOTHING. It constructs no verdict of its own: the outcome, + * the rc, the mutation rc and the output digest all come from the banked bytes, + * and the hash chain is supplied by the caller. The lake is a mirror and never a + * ledger; what it mirrors here is a corpus that was banked by someone else. + * + * IN-MEMORY ONLY. The caller provides the object; the tool provides it under + * `planningStoreInMemoryLayer`. Nothing is persisted anywhere. + */ +import { Effect, Schema } from "effect"; +import type { CapacityReading } from "@substrate/planning/schema/capacity.ts"; +import type { ExecutorId } from "@substrate/planning/schema/ids.ts"; +import { Verdict } from "@substrate/planning/schema/records.ts"; +import { AdmitOutcomeReport } from "@substrate/planning/schema/admit.ts"; +import type { FactoryError } from "@substrate/planning/schema/errors.ts"; +import type { IFactory } from "@substrate/planning/objects/factory.ts"; +import { + projectionRowOf, + type BankedRow, + type MirroredEvidence, + type ProjectionRow +} from "./projection.ts"; + +const decodeVerdict = Schema.decodeUnknownSync(Verdict); +const decodeOutcome = Schema.decodeUnknownSync(AdmitOutcomeReport); + +/** One link of the witness chain the caller supplies. */ +interface ChainLink { + readonly seq: number; + readonly hash: string; + readonly prevHash: string; +} + +/** What one load produced. */ +interface LoadResult { + /** One row per unit the object accepted, sorted by unit id. */ + readonly rows: ReadonlyArray; + /** Units armed but never proposed before the loop stopped making progress. */ + readonly unreleased: ReadonlyArray; + /** Units the object refused, with the reason it gave. */ + readonly refused: ReadonlyArray<{ readonly unit: string; readonly reason: string }>; + /** Passes of the release loop the corpus needed. */ + readonly passes: number; +} + +/** + * Walk the banked corpus through one Factory object and read the mirror back. + * + * @param factory - The object, already built over the in-memory store. + * @param rows - The banked receipts, normalised. Order is the arming order. + * @param link - The witness chain link for the nth verdict, supplied by the + * caller so this module computes no hash and therefore mints no proof. + * @param readingFor - A capacity reading for pass `n`. One per pass, because a + * reading is a complete fixture and re-observing the same `seq` is a no-op. + * @param maxPasses - A bound on the release loop, so a corpus the evaluator will + * not drain stops rather than spins. + * @returns The mirror's rows plus everything that did not reach it, named. + */ +export const loadBankedIntoFactory = ( + factory: IFactory, + rows: ReadonlyArray, + link: (index: number) => ChainLink, + readingFor: (pass: number) => CapacityReading, + maxPasses = 512 +): Effect.Effect => + Effect.gen(function* () { + const pending = new Map(rows.map((row) => [row.unit, row])); + const witnessed = new Set(); + const refused: Array<{ unit: string; reason: string }> = []; + let written = 0; + let pass = 0; + + while (witnessed.size + refused.length < rows.length && pass < maxPasses) { + const reading = readingFor(pass); + pass += 1; + yield* factory.observeCapacity(reading); + const proposals = yield* factory.handOut(reading.executor as ExecutorId); + const fresh = proposals.filter( + (proposal) => pending.has(proposal.taskId) && !witnessed.has(proposal.taskId) + ); + if (fresh.length === 0) break; + + for (const proposal of fresh) { + const banked = pending.get(proposal.taskId); + if (banked === undefined) continue; + yield* factory.observeOutcome( + decodeOutcome({ + taskId: proposal.taskId, + dedupKey: proposal.dedupKey, + outcome: "Accepted", + row: proposal.rows[0]?.row ?? "build" + }) + ); + const chain = link(written); + written += 1; + const record = decodeVerdict({ + _tag: "Verdict", + taskId: banked.unit, + executor: reading.executor, + seq: chain.seq, + hash: chain.hash, + prevHash: chain.prevHash, + outcome: banked.outcome, + serviceSeconds: banked.seconds, + unit_id: banked.unit, + oracle_rc: banked.oracle_rc, + oracle_output_sha256: banked.oracle_output_sha256, + ...(banked.mutation_rc === null ? {} : { mutation_rc: banked.mutation_rc }) + }); + const observed = yield* factory.observeVerdict(record).pipe( + Effect.map(() => null), + Effect.catch((error: FactoryError) => Effect.succeed(error.reason)) + ); + if (observed !== null) { + refused.push({ unit: banked.unit, reason: observed }); + witnessed.add(banked.unit); + continue; + } + const accepted = yield* factory + .observeReceipt({ + id: banked.unit, + oracle_rc: banked.oracle_rc, + oracle_output_sha256: banked.oracle_output_sha256, + verdict_hash: chain.hash, + ...(banked.mutation_rc === null ? {} : { mutation_rc: banked.mutation_rc }) + }) + .pipe( + Effect.map(() => null), + Effect.catch((error: FactoryError) => Effect.succeed(error.reason)) + ); + if (accepted !== null) refused.push({ unit: banked.unit, reason: accepted }); + witnessed.add(banked.unit); + } + } + + const view = yield* factory.stateView; + const evidence = new Map(); + for (const receipt of view.receipts) evidence.set(receipt.id, receipt); + + const projected: Array = []; + for (const row of rows) { + const mirrored = evidence.get(row.unit); + if (mirrored === undefined) continue; + projected.push(projectionRowOf(row, mirrored)); + } + projected.sort((a, b) => a.unit.localeCompare(b.unit)); + + const unreleased = rows + .filter((row) => !witnessed.has(row.unit)) + .map((row) => row.unit) + .sort((a, b) => a.localeCompare(b)); + + return { + rows: projected, + unreleased, + refused: refused.sort((a, b) => a.unit.localeCompare(b.unit)), + passes: pass + }; + }); diff --git a/pkgs/substrate-apps/src/packages/factory/src/projectionMirror.ts b/pkgs/substrate-apps/src/packages/factory/src/projectionMirror.ts new file mode 100644 index 00000000..5ac1d9dd --- /dev/null +++ b/pkgs/substrate-apps/src/packages/factory/src/projectionMirror.ts @@ -0,0 +1,228 @@ +/** + * projectionMirror.ts — U-A14. `GET /projection`, served from the object's own + * mirror. + * + * §2.2b's route table has one line for it — `GET /projection` → §2.2e — and + * §2.2e says what a projection is: the operator's fields over U-A12's + * deterministic serializer (stable key order, LF, trailing newline, ISO-8601 + * UTC). U-A13 built that projection for the TOOL, which reads a banked corpus + * off the disk. This module builds the same rows for the WORKER, which has no + * disk and no register: what it has is the receipts it accepted. + * + * THE ROW IS THE SAME ROW. `projectionRowOf` is U-A13's and is not copied here. + * The two halves it joins are the two halves it always joins: + * + * the banked cells `actuals.*`, `outcome_for_calibration`, `receipt_path` + * — retained below from the receipt's own bytes + * the evidence cells `result.observed`, `result.grade` + * — read off `stateView.receipts`, the mirror + * + * so a unit has a row only if the object accepted its evidence against a + * mirrored kernel verdict (§2.2b's exact three-cell match). A receipt the object + * refused is a unit with no row, on the Worker for the same reason as in the + * tool. + * + * WHY THE CELLS ARE RETAINED AT ALL. `Factory.observeReceipt` keeps + * `ReceiptEvidence` — five cells, the ones the chain match is about — and + * deliberately not the receipt. But `actuals.seconds`, `actuals.tokens` and + * `actuals.outcome` are measurements that live nowhere else, and a projection + * that filled them with zeros would be inventing them. So the HTTP layer writes + * the receipt's own measured cells into the `PlanningStore` beside the object's + * state, under this module's prefix, and reads them back here. Durable Object + * SQLite is what makes that survive an eviction; the in-memory Layer makes it + * survive a test. + * + * NOTHING IS AUTHORED. Every cell below comes from the receipt's bytes or from + * the mirror. `status` and `outcome_ruled` are not reachable from here — they + * are not in `PROJECTION_CELLS`, not in `ProjectionRow`, and not named in this + * file. + */ +import { Effect, Option, Schema } from "effect"; +import { serializeRecords } from "@substrate/serializer"; +import { + StorageKey, + type IPlanningStore +} from "@substrate/planning/objects/storage.ts"; +import type { ReceiptEvidence } from "@substrate/planning/objects/factory.ts"; +import { + NONE, + projectionRowOf, + type ProjectionRow +} from "./projection.ts"; + +/** Where a retained receipt's measured cells live in the store. */ +const MIRROR_RECEIPT_PREFIX = "factory/projection/"; + +/** The key one unit's retained cells are written under. */ +const mirrorReceiptKey = (unit: string): StorageKey => + StorageKey(`${MIRROR_RECEIPT_PREFIX}${unit}`); + +/** + * The measured cells kept from an accepted receipt. + * + * Exactly the banked half of a projection row and not one field more. It is a + * schema rather than an interface because storage is a boundary: what comes back + * out of SQLite is parsed before any row is built from it, by the same rule the + * `PlanningStore` header states. + */ +const RetainedReceipt = Schema.Struct({ + unit: Schema.String, + receipt_path: Schema.String, + receipt_sha256: Schema.String, + disposition: Schema.String, + seconds: Schema.Number, + tokens: Schema.Number, + outcome_for_calibration: Schema.NullOr(Schema.String), + prior_gap: Schema.NullOr(Schema.Number) +}); + +/** The measured cells kept from an accepted receipt. */ +type RetainedReceipt = typeof RetainedReceipt.Type; + +const parseRetained = Schema.decodeUnknownEffect(RetainedReceipt); +const encodeRetained = Schema.encodeUnknownSync(RetainedReceipt); + +const isRecord = (value: unknown): value is Record => + value !== null && typeof value === "object" && !Array.isArray(value); + +const numberAt = (source: Record, key: string): number | null => { + const value = source[key]; + return typeof value === "number" && Number.isFinite(value) ? value : null; +}; + +const stringAt = (source: Record, key: string): string | null => { + const value = source[key]; + return typeof value === "string" && value !== "" ? value : null; +}; + +/** + * The declared-cell prior gap of a §2.3 receipt. + * + * The strict type spells it `prior_gap{tokens_by_cell{out, …}}`; `out` is the + * cell every prior in this cycle declares (D-B19), which is the same reading + * `projection.ts` takes over the banked corpus. + */ +const priorGapOf = (value: unknown): number | null => { + if (!isRecord(value)) return null; + const byCell = value["tokens_by_cell"]; + if (isRecord(byCell)) return numberAt(byCell, "out"); + return null; +}; + +/** + * Reads the measured cells off one receipt. + * + * The receipt has already been decoded strictly by `postReceipt` when this is + * called, so the required cells are there by construction; the reader is written + * defensively anyway, because "already validated upstream" is a claim about a + * caller and not about a value. + * + * @param receipt - The receipt as it arrived, parsed. + * @param receiptPath - Where the poster banked it, or `"none"` (D-B22). + * @param receiptSha256 - sha256 over the receipt's canonical bytes. + * @returns The retained cells, or the reason they are not a row. + */ +export const retainedReceiptOf = ( + receipt: unknown, + receiptPath: string, + receiptSha256: string +): + | { readonly ok: true; readonly retained: RetainedReceipt } + | { readonly ok: false; readonly why: string } => { + if (!isRecord(receipt)) return { ok: false, why: "the receipt is not a JSON object" }; + const missing: Array = []; + const unit = stringAt(receipt, "id"); + if (unit === null) missing.push("id"); + const seconds = numberAt(receipt, "seconds"); + if (seconds === null) missing.push("seconds"); + const disposition = stringAt(receipt, "disposition"); + if (disposition === null) missing.push("disposition"); + const tokensBlock = receipt["tokens"]; + const tokens = isRecord(tokensBlock) ? numberAt(tokensBlock, "total") : null; + if (tokens === null) missing.push("tokens.total"); + if (missing.length > 0) return { ok: false, why: `carries no ${missing.join(", ")}` }; + + return { + ok: true, + retained: { + unit: unit as string, + receipt_path: receiptPath === "" ? NONE : receiptPath, + receipt_sha256: receiptSha256, + disposition: disposition as string, + seconds: seconds as number, + tokens: tokens as number, + outcome_for_calibration: stringAt(receipt, "outcome_for_calibration"), + prior_gap: priorGapOf(receipt["prior_gap"]) + } + }; +}; + +/** Writes one unit's retained cells into the store. */ +export const retainReceipt = ( + store: IPlanningStore, + retained: RetainedReceipt +): Effect.Effect => + store + .put(mirrorReceiptKey(retained.unit), encodeRetained(retained)) + .pipe(Effect.catch(() => Effect.void)); + +/** + * Joins the retained cells with the mirror, in unit order. + * + * A retained receipt with no matching evidence yields no row: the object holds + * the cells only because it accepted the evidence, so the two disagreeing means + * the store is ahead of the mirror and the row would be a claim the object + * cannot back. Evidence with no retained cells yields no row either, and for the + * mirror image of the same reason. + */ +export const mirrorProjectionRows = ( + retained: ReadonlyArray, + evidence: ReadonlyArray +): ReadonlyArray => { + const byUnit = new Map(evidence.map((entry) => [entry.id, entry] as const)); + const rows: Array = []; + for (const cells of retained) { + const mirrored = byUnit.get(cells.unit); + if (mirrored === undefined) continue; + rows.push(projectionRowOf(cells, mirrored)); + } + return rows.sort((left, right) => (left.unit < right.unit ? -1 : left.unit > right.unit ? 1 : 0)); +}; + +/** + * Reads every retained receipt back out of the store, in key order. + * + * A record that does not parse is skipped rather than thrown on: the projection + * is a mirror, and a mirror that refuses to show anything because one record is + * malformed shows less truth, not more. It cannot be written by this repository + * in the first place — the only writer is `retainReceipt` above. + */ +export const retainedReceipts = ( + store: IPlanningStore +): Effect.Effect, never> => + Effect.gen(function* () { + const keys = yield* store + .list(MIRROR_RECEIPT_PREFIX) + .pipe(Effect.catch(() => Effect.succeed>([]))); + const found: Array = []; + for (const key of keys) { + const stored = yield* store + .get(key) + .pipe(Effect.catch(() => Effect.succeed(Option.none()))); + if (Option.isNone(stored)) continue; + const parsed = yield* parseRetained(stored.value).pipe(Effect.option); + if (Option.isSome(parsed)) found.push(parsed.value); + } + return found; + }); + +/** + * The projection body, byte for byte. + * + * `serializeRecords` is U-A12's: sorted keys, LF, one record per line, a + * trailing newline. That is what makes `cmp` against a banked fixture a + * statement about the projection rather than about `JSON.stringify`'s insertion + * order, and it is why the smoke's `cmp` step is worth running at all. + */ +export const projectionBody = (rows: ReadonlyArray): string => + serializeRecords(rows as ReadonlyArray); diff --git a/pkgs/substrate-apps/src/packages/factory/src/receipt.ts b/pkgs/substrate-apps/src/packages/factory/src/receipt.ts new file mode 100644 index 00000000..ec4d7298 --- /dev/null +++ b/pkgs/substrate-apps/src/packages/factory/src/receipt.ts @@ -0,0 +1,48 @@ +/** Strict receipt decoding followed by the Factory's exact verdict match. */ +import { Effect } from "effect"; +import { decodeReceipt } from "@substrate/schema"; +import { + type IFactory, + type ReceiptEvidence, + type ReceiptObservation +} from "@substrate/planning/objects/factory.ts"; +import { FactoryError } from "@substrate/planning/schema/errors.ts"; + +const possibleId = (input: unknown): string => { + if (input === null || typeof input !== "object") return "receipt"; + const id = (input as Record)["id"]; + return typeof id === "string" ? id : "receipt"; +}; + +/** + * Decodes the complete §2.3 receipt. Only then are its three evidence cells + * compared with a mirrored kernel verdict carrying `verdictHash`. + */ +export const postReceipt = ( + factory: IFactory, + incoming: unknown, + verdictHash: string +): Effect.Effect => { + const decoded = decodeReceipt(incoming); + if (!decoded.ok) { + return Effect.fail( + new FactoryError({ + reason: "ReceiptInvalid", + operation: "Factory.observeReceipt", + subject: possibleId(incoming) + }) + ); + } + const receipt = decoded.value; + const base = { + id: receipt.id, + oracle_rc: receipt.oracle_rc, + oracle_output_sha256: receipt.oracle_output_sha256, + verdict_hash: verdictHash + }; + const evidence: ReceiptEvidence = + receipt.mutation.kind === "hint" + ? { ...base, mutation_rc: receipt.mutation.rc } + : base; + return factory.observeReceipt(evidence); +}; diff --git a/pkgs/substrate-apps/src/packages/factory/test/capacity.test.ts b/pkgs/substrate-apps/src/packages/factory/test/capacity.test.ts new file mode 100644 index 00000000..09a52e48 --- /dev/null +++ b/pkgs/substrate-apps/src/packages/factory/test/capacity.test.ts @@ -0,0 +1,245 @@ +/** + * Seat capacity through the host-neutral surface: the ingestion routes, the + * projection and admission reads, the capacity feed Layers, and the Worker's + * alarm clamp. + */ +import { Effect, Schema } from "effect"; +import { describe, expect, it } from "vitest"; +import { + Factory, + FactoryCapacityFeed, + fakeCapacityLayer, + loadFactoryInputs, + makeFactory, + makeFactoryHttpHandler, + makeInMemoryPlanningStore, + makeTableArtifactHasher, + PlanningStore, + storedCapacityLayer, + constantPriceVectorLayer +} from "../src/index.ts"; +import { alarmToSet, ALARM_FLOOR_MS } from "../../../apps/floor/src/alarm.ts"; +import { CapacityReading } from "@substrate/planning/schema/capacity.ts"; +import { PriceVector } from "@substrate/planning/schema/prices.ts"; +import { + CC_OBSERVED, + cc3Seat, + claudeSeat, + halogenSeat, + plus, + qwenSeat, + snapshot +} from "../../planning/test/seatCapacityFixtures.ts"; +import { artifactBytes, artifactDigest, emptyFactory, reading, releaseState, run } from "./fixtures.ts"; + +const encodeReading = Schema.encodeSync(CapacityReading); +const T0 = plus(CC_OBSERVED, 60); +/** Today's fleet, as the pusher would publish it. */ +const today = () => snapshot([claudeSeat(), cc3Seat(), qwenSeat(), halogenSeat(0)]); + +const request = (path: string, method: "GET" | "POST", body?: unknown, authenticated = true) => + new Request(`https://factory.invalid${path}`, { + method, + headers: { + ...(authenticated ? { authorization: "Bearer test-token" } : {}), + ...(body === undefined ? {} : { "content-type": "application/json" }) + }, + ...(body === undefined ? {} : { body: JSON.stringify(body) }) + }); + +const handlerAt = async (clock: { now: string } | undefined) => { + const factory = await emptyFactory(); + const handler = makeFactoryHttpHandler({ + factory, + token: "test-token", + artifactHasher: makeTableArtifactHasher([[artifactBytes, artifactDigest]]), + ...(clock === undefined ? {} : { now: () => clock.now }) + }); + return { factory, handler }; +}; + +const json = async (response: Response) => ({ status: response.status, body: await response.json() }); + +describe("POST /capacity/seats", () => { + it("requires the bearer", async () => { + const { handler } = await handlerAt({ now: T0 }); + expect((await handler(request("/capacity/seats", "POST", today(), false))).status).toBe(401); + }); + + it("refuses without a host clock rather than accepting against none", async () => { + const { handler } = await handlerAt(undefined); + expect(await json(await handler(request("/capacity/seats", "POST", today())))).toEqual({ + status: 501, + body: { error: "NoClock", subject: "/capacity/seats" } + }); + }); + + it("validates the snapshot against the schema", async () => { + const { handler } = await handlerAt({ now: T0 }); + const broken = { ...today(), seats: [{ ...claudeSeat(), observed_at: "2026-09-23 04:52" }] }; + expect((await handler(request("/capacity/seats", "POST", broken))).status).toBe(400); + const badVersion = { ...today(), schema_version: "seat-capacity/1" }; + expect((await handler(request("/capacity/seats", "POST", badVersion))).status).toBe(400); + }); + + it("is idempotent, monotonic, and refuses a reading dated in the future", async () => { + const clock = { now: T0 }; + const { handler } = await handlerAt(clock); + const first = await json(await handler(request("/capacity/seats", "POST", today()))); + expect(first.status).toBe(200); + expect(first.body.changed).toBe(true); + const again = await json(await handler(request("/capacity/seats", "POST", { snapshot: today() }))); + expect(again.body).toEqual({ + results: ["cc", "cc3", "pi-qwencloud", "gpu-worker"].map((seat) => ({ seat, outcome: "duplicate" })), + changed: false + }); + const older = snapshot([claudeSeat({ observed_at: plus(CC_OBSERVED, -300) })], plus(T0, 10)); + expect((await json(await handler(request("/capacity/seats", "POST", older)))).body.results).toEqual([ + { seat: "cc", outcome: "older" } + ]); + const future = snapshot([claudeSeat({ observed_at: plus(T0, 900) })], plus(T0, 900)); + expect((await json(await handler(request("/capacity/seats", "POST", future)))).body.results).toEqual([ + { seat: "cc", outcome: "future" } + ]); + }); +}); + +describe("POST /capacity with seats attached", () => { + it("keeps today's uplink body working and folds attached seats in", async () => { + const { handler, factory } = await handlerAt({ now: T0 }); + const plain = await json(await handler(request("/capacity", "POST", encodeReading(reading({}))))); + expect(plain.body).toEqual({ accepted: true }); + const withSeats = { ...encodeReading(reading({})), seats: today() }; + const attached = await json(await handler(request("/capacity", "POST", withSeats))); + expect(attached.body.accepted).toBe(false); + expect(attached.body.seats.changed).toBe(true); + const state = await run(factory.stateView); + expect(state.seat_readings).toHaveLength(4); + expect(state.latest_readings.coordinator).not.toHaveProperty("seats"); + }); + + it("never sets alarm_at at or before the host clock", async () => { + const { handler, factory } = await handlerAt({ now: T0 }); + const past = encodeReading(reading({ nextWindowAt: "2026-09-12T11:00:00Z" })); + await handler(request("/capacity", "POST", past)); + expect((await run(factory.stateView)).alarm_at).toBeNull(); + }); +}); + +describe("GET /capacity and GET /capacity/admit", () => { + const loaded = async () => { + const clock = { now: T0 }; + const context = await handlerAt(clock); + await context.handler(request("/capacity/seats", "POST", today())); + return { ...context, clock }; + }; + + it("projects every seat at the given asOf, defaulting to the host clock", async () => { + const { handler, clock } = await loaded(); + const now = await json(await handler(request("/capacity", "GET"))); + expect(now.body.as_of).toBe(T0); + expect(now.body.next_transition_at).toBe(plus(CC_OBSERVED, 1200)); + const headroom = Object.fromEntries( + now.body.seats.map((seat: { reading: { seat: string }; headroom_pct: number | null }) => [ + seat.reading.seat, + seat.headroom_pct + ]) + ); + expect(headroom).toEqual({ cc: 28, cc3: 100, "gpu-worker": null, "pi-qwencloud": null }); + clock.now = plus(T0, 3600); + const later = await json(await handler(request(`/capacity?asOf=${encodeURIComponent(T0)}`, "GET"))); + expect(later.body.as_of).toBe(T0); + }); + + it("answers the admission question per job class", async () => { + const { handler } = await loaded(); + const ask = async (query: string) => (await json(await handler(request(`/capacity/admit?${query}`, "GET")))).body; + expect(await ask("seat=cc&model=Opus")).toMatchObject({ admit: true, headroom_pct: 28 }); + expect(await ask("seat=cc&model=Fable")).toMatchObject({ admit: false, reason: "severity-critical" }); + expect(await ask("seat=cc&model=Opus&min_headroom_pct=30")).toMatchObject({ + admit: false, + reason: "window-exhausted" + }); + expect(await ask("seat=cc3&model=Opus")).toMatchObject({ admit: false, reason: "not-dispatchable" }); + expect(await ask("seat=nobody")).toMatchObject({ admit: false, reason: "unknown-seat" }); + expect(await ask(`seat=cc&model=Opus&asOf=${encodeURIComponent(plus(CC_OBSERVED, 2000))}`)).toMatchObject({ + admit: false, + reason: "stale" + }); + }); + + it("refuses a malformed asOf, a missing seat and a malformed headroom", async () => { + const { handler } = await loaded(); + expect((await handler(request("/capacity?asOf=tomorrow", "GET"))).status).toBe(400); + expect((await handler(request("/capacity/admit", "GET"))).status).toBe(400); + expect((await handler(request("/capacity/admit?seat=cc&min_headroom_pct=lots", "GET"))).status).toBe(400); + }); + + it("without a host clock, GET /capacity requires asOf", async () => { + const { handler } = await handlerAt(undefined); + expect((await handler(request("/capacity", "GET"))).status).toBe(400); + expect((await handler(request(`/capacity?asOf=${encodeURIComponent(T0)}`, "GET"))).status).toBe(200); + }); +}); + +describe("the capacity feed Layers", () => { + const prices = Schema.decodeUnknownSync(PriceVector)({ + hash: "7".repeat(64), + rows: [], + drum: 1, + stage: [] + }); + + it("the fake Layer carries seats built from today's shapes into the Factory", async () => { + const factory = await emptyFactory(); + const state = await Effect.runPromise( + loadFactoryInputs(T0).pipe( + Effect.provideService(Factory, Factory.of(factory)), + Effect.provide(fakeCapacityLayer([reading({})], [today()])), + Effect.provide(constantPriceVectorLayer(prices)) + ) + ); + expect(state.seat_readings.map((entry) => entry.seat.seat)).toEqual([ + "cc", + "cc3", + "gpu-worker", + "pi-qwencloud" + ]); + }); + + it("the stored Layer reads back what the object persisted, from the same store", async () => { + const store = await run(makeInMemoryPlanningStore); + const factory = await run( + makeFactory(releaseState([], { plans: [] })).pipe(Effect.provideService(PlanningStore, store)) + ); + await run(factory.observeSeats(today(), T0)); + await run(factory.observeSeats(snapshot([halogenSeat(1)], plus(T0, 1), "worker"), T0)); + const snapshots = await run( + Effect.gen(function* () { + const feed = yield* FactoryCapacityFeed; + return yield* feed.seats; + }).pipe(Effect.provide(storedCapacityLayer), Effect.provideService(PlanningStore, store)) + ); + expect(snapshots.map((entry) => [entry.host, entry.seats.map((seat) => seat.seat)])).toEqual([ + ["coordinator", ["cc", "cc3", "pi-qwencloud"]], + ["worker", ["gpu-worker"]] + ]); + // Feeding them back is a no-op: ingestion is idempotent. + const before = await run(factory.stateView); + for (const entry of snapshots) { + expect((await run(factory.observeSeats(entry, T0))).changed).toBe(false); + } + expect(await run(factory.stateView)).toEqual(before); + }); +}); + +describe("the Worker's alarm clamp", () => { + it("never hands setAlarm an instant at or before now", () => { + const now = Date.parse(T0); + expect(alarmToSet(null, now)).toBeNull(); + expect(alarmToSet(Number.NaN, now)).toBeNull(); + expect(alarmToSet(now + 60_000, now)).toBe(now + 60_000); + expect(alarmToSet(now, now)).toBe(now + ALARM_FLOOR_MS); + expect(alarmToSet(Date.parse("2026-09-12T11:00:00Z"), now)).toBe(now + ALARM_FLOOR_MS); + }); +}); diff --git a/pkgs/substrate-apps/src/packages/factory/test/capacityReview1.test.ts b/pkgs/substrate-apps/src/packages/factory/test/capacityReview1.test.ts new file mode 100644 index 00000000..9a48c637 --- /dev/null +++ b/pkgs/substrate-apps/src/packages/factory/test/capacityReview1.test.ts @@ -0,0 +1,283 @@ +/** + * Capacity review round 1 (evals-2026-09-23/capacity/IMPL.md, "Fix round 1"). + * + * Each case pins a defect the reviewers reproduced on the branch: + * + * - a request that lands after a due alarm and before its delivery must not + * cancel or replace that alarm (the Worker's post-request reconcile); + * - a clock-less host must not ingest seats against the publisher's own + * `published_at`, on any door; + * - an undecodable `capacity/seats` key must not take down routes that have + * nothing to do with seat capacity, nor the Worker's post-request step. + */ +import { Effect, Option, Schema } from "effect"; +import { describe, expect, it } from "vitest"; +import { + Factory, + fakeCapacityLayer, + loadFactoryInputs, + makeFactory, + makeFactoryHttpHandler, + makeInMemoryPlanningStore, + makeTableArtifactHasher, + PlanningStore, + StorageKey, + constantPriceVectorLayer, + type IFactory +} from "../src/index.ts"; +import { reconcileRuntimeAlarm, type AlarmStorage } from "../../../apps/floor/src/alarm.ts"; +import { CapacityReading } from "@substrate/planning/schema/capacity.ts"; +import { PriceVector } from "@substrate/planning/schema/prices.ts"; +import { FactoryError } from "@substrate/planning/schema/errors.ts"; +import { + CC_OBSERVED, + claudeSeat, + plus, + snapshot +} from "../../planning/test/seatCapacityFixtures.ts"; +import { artifactBytes, artifactDigest, emptyFactory, reading, releaseState, run } from "./fixtures.ts"; +import { cloudReading } from "../../planning/test/fixtures.ts"; + +const encodeReading = Schema.encodeSync(CapacityReading); +const T0 = plus(CC_OBSERVED, 60); + +const request = (path: string, method: "GET" | "POST", body?: unknown) => + new Request(`https://factory.invalid${path}`, { + method, + headers: { + authorization: "Bearer test-token", + ...(body === undefined ? {} : { "content-type": "application/json" }) + }, + ...(body === undefined ? {} : { body: JSON.stringify(body) }) + }); + +const handlerOver = (factory: IFactory, now?: () => string) => + makeFactoryHttpHandler({ + factory, + token: "test-token", + artifactHasher: makeTableArtifactHasher([[artifactBytes, artifactDigest]]), + ...(now === undefined ? {} : { now }) + }); + +const json = async (response: Response) => ({ status: response.status, body: await response.json() }); + +/** A Durable Object's alarm cell, as the Worker sees it through ctx.storage. */ +const fakeStorage = (alarm: number | null) => { + const log: Array = []; + const storage: AlarmStorage & { alarm: number | null; readonly log: Array } = { + alarm, + log, + getAlarm: async () => storage.alarm, + setAlarm: async (at: number) => { + storage.alarm = at; + log.push(`set ${new Date(at).toISOString()}`); + }, + deleteAlarm: async () => { + storage.alarm = null; + log.push("delete"); + } + }; + return storage; +}; + +describe("the Worker's reconcile never drops a due, undelivered alarm", () => { + const T = "2026-09-23T06:00:00.000Z"; + const armedAt = async (readings: ReadonlyArray>) => { + const factory = await emptyFactory(); + for (const entry of readings) { + await run(factory.observeCapacity(entry, "2026-09-23T05:00:00.000Z")); + } + expect((await run(factory.stateView)).alarm_at).toBe(T); + return factory; + }; + + it("a GET 20 ms after T, before delivery, leaves the alarm at T; its delivery wakes the object", async () => { + const factory = await armedAt([reading({ seq: 1, nextWindowAt: T })]); + const storage = fakeStorage(Date.parse(T)); + await reconcileRuntimeAlarm({ storage, factory, nowMs: Date.parse(T) + 20, path: "request" }); + expect(storage.alarm).toBe(Date.parse(T)); + expect(storage.log).toEqual([]); + // The runtime delivers the alarm (and clears it) 30 ms after T. + storage.alarm = null; + const firedAt = new Date(Date.parse(T) + 30).toISOString(); + await run(factory.alarm(firedAt)); + await reconcileRuntimeAlarm({ storage, factory, nowMs: Date.parse(firedAt), path: "alarm" }); + expect((await run(factory.stateView)).alarm_fired_at).toBe(firedAt); + expect(storage.alarm).toBeNull(); + }); + + it("with a later transition held, the due alarm is not replaced by the later one", async () => { + const later = "2026-09-23T07:00:00.000Z"; + const factory = await armedAt([reading({ seq: 1, nextWindowAt: T })]); + // A second executor's row carries the later transition. + await run(factory.observeCapacity(cloudReading({ seq: 1, nextWindowAt: later }), "2026-09-23T05:00:00.000Z")); + const storage = fakeStorage(Date.parse(T)); + await reconcileRuntimeAlarm({ storage, factory, nowMs: Date.parse(T) + 20, path: "request" }); + expect(storage.alarm).toBe(Date.parse(T)); + storage.alarm = null; + const firedAt = new Date(Date.parse(T) + 30).toISOString(); + await run(factory.alarm(firedAt)); + await reconcileRuntimeAlarm({ storage, factory, nowMs: Date.parse(firedAt), path: "alarm" }); + expect(storage.alarm).toBe(Date.parse(later)); + }); + + it("an alarm not yet due is still moved or cleared as before", async () => { + const factory = await armedAt([reading({ seq: 1, nextWindowAt: T })]); + const early = fakeStorage(Date.parse("2026-09-23T08:00:00.000Z")); + await reconcileRuntimeAlarm({ storage: early, factory, nowMs: Date.parse(T) - 60_000, path: "request" }); + expect(early.alarm).toBe(Date.parse(T)); + const empty = await emptyFactory(); + const stray = fakeStorage(Date.parse(T)); + await reconcileRuntimeAlarm({ storage: stray, factory: empty, nowMs: Date.parse(T) - 60_000, path: "request" }); + expect(stray.log).toEqual(["delete"]); + }); + + it("on the alarm path a stored instant at or before now is replaced, never kept", async () => { + const factory = await armedAt([reading({ seq: 1, nextWindowAt: T })]); + const firedAt = new Date(Date.parse(T) + 30).toISOString(); + await run(factory.alarm(firedAt)); + const storage = fakeStorage(Date.parse(T)); + await reconcileRuntimeAlarm({ storage, factory, nowMs: Date.parse(firedAt), path: "alarm" }); + expect(storage.log).toEqual(["delete"]); + }); + + it("a failing rearm on the request path is contained; on the alarm path it is raised", async () => { + const broken = { + rearm: () => + Effect.fail( + new FactoryError({ reason: "PersistenceFailed", operation: "Factory.rearm", subject: "capacity/seats" }) + ) + } as unknown as IFactory; + const storage = fakeStorage(null); + const outcome = await reconcileRuntimeAlarm({ storage, factory: broken, nowMs: Date.parse(T), path: "request" }); + expect(outcome).toMatchObject({ action: "failed" }); + await expect( + reconcileRuntimeAlarm({ storage, factory: broken, nowMs: Date.parse(T), path: "alarm" }) + ).rejects.toBeDefined(); + }); +}); + +describe("a clock-less host never ingests seats", () => { + const far = "2099-01-01T00:00:00Z"; + const poisoned = () => snapshot([claudeSeat({ observed_at: far })], far); + + it("POST /capacity with seats and no clock is refused 501 NoClock, and nothing is folded in", async () => { + const factory = await emptyFactory(); + const handler = handlerOver(factory); + const body = { ...encodeReading(reading({})), seats: poisoned() }; + expect(await json(await handler(request("/capacity", "POST", body)))).toEqual({ + status: 501, + body: { error: "NoClock", subject: "/capacity" } + }); + const state = await run(factory.stateView); + expect(state.seat_readings).toEqual([]); + expect(state.latest_readings).toEqual({}); + }); + + it("POST /capacity without seats still works without a clock", async () => { + const handler = handlerOver(await emptyFactory()); + expect(await json(await handler(request("/capacity", "POST", encodeReading(reading({})))))).toEqual({ + status: 200, + body: { accepted: true } + }); + }); + + it("with a clock, the future-dated seat is refused and an honest one is then accepted", async () => { + const handler = handlerOver(await emptyFactory(), () => T0); + const first = await json( + await handler(request("/capacity", "POST", { ...encodeReading(reading({ seq: 1 })), seats: poisoned() })) + ); + expect(first.body.seats.results).toEqual([{ seat: "cc", outcome: "future" }]); + const honest = await json( + await handler( + request("/capacity", "POST", { ...encodeReading(reading({ seq: 2 })), seats: snapshot([claudeSeat()], T0) }) + ) + ); + expect(honest.body.seats.results).toEqual([{ seat: "cc", outcome: "accepted" }]); + }); + + it("Factory.observeCapacity refuses seats without asOf, and takes them with one", async () => { + const factory = await emptyFactory(); + const withSeats = { ...reading({ seq: 1 }), seats: poisoned() }; + const failure = await Effect.runPromise(Effect.flip(factory.observeCapacity(withSeats))); + expect(failure).toMatchObject({ _tag: "FactoryError", reason: "NoClock", operation: "Factory.observeCapacity" }); + expect((await run(factory.stateView)).seat_readings).toEqual([]); + expect(await run(factory.observeCapacity({ ...reading({ seq: 1 }), seats: snapshot([claudeSeat()], T0) }, T0))).toBe( + true + ); + }); + + it("the bench loads authored seats only against an explicit clock", async () => { + const prices = Schema.decodeUnknownSync(PriceVector)({ hash: "7".repeat(64), rows: [], drum: 1, stage: [] }); + const load = async (clock: string | undefined) => { + const factory = await emptyFactory(); + return Effect.runPromise( + Effect.flip( + loadFactoryInputs(clock).pipe( + Effect.provideService(Factory, Factory.of(factory)), + Effect.provide(fakeCapacityLayer([reading({})], [poisoned()])), + Effect.provide(constantPriceVectorLayer(prices)) + ) + ) + ); + }; + expect(await load(undefined)).toMatchObject({ reason: "NoClock" }); + }); +}); + +describe("an undecodable capacity/seats key is contained to the capacity routes", () => { + const brokenFactory = async () => { + const store = await run(makeInMemoryPlanningStore); + // What a future seat-capacity/3 row, or any schema tightening, looks like here. + await run( + store.put(StorageKey("capacity/seats"), [ + { seat: { seat: "cc", provider: "gemini" }, host: "coordinator" } + ]) + ); + return run( + makeFactory(releaseState([], { plans: [] })).pipe(Effect.provideService(PlanningStore, store)) + ); + }; + + it("a plain kernel reading is accepted, GET /state answers with a marker, and rearm resolves", async () => { + const factory = await brokenFactory(); + const handler = handlerOver(factory, () => T0); + expect(await json(await handler(request("/capacity", "POST", encodeReading(reading({})))))).toEqual({ + status: 200, + body: { accepted: true } + }); + const state = await json(await handler(request("/state", "GET"))); + expect(state.status).toBe(200); + expect(state.body.seat_readings).toEqual([]); + expect(state.body.seat_readings_error).toMatch(/capacity\/seats/); + expect(Option.isOption(await run(factory.rearm(T0)))).toBe(true); + const storage = fakeStorage(null); + expect( + await reconcileRuntimeAlarm({ storage, factory, nowMs: Date.parse(T0), path: "request" }) + ).not.toMatchObject({ action: "failed" }); + }); + + it("the capacity reads say PersistenceFailed rather than answering from nothing", async () => { + const handler = handlerOver(await brokenFactory(), () => T0); + expect((await json(await handler(request("/capacity", "GET")))).body).toMatchObject({ + error: "PersistenceFailed" + }); + expect((await json(await handler(request("/capacity/admit?seat=cc", "GET")))).body).toMatchObject({ + error: "PersistenceFailed" + }); + }); + + it("the next accepted push replaces the unreadable value, and says so", async () => { + const factory = await brokenFactory(); + const handler = handlerOver(factory, () => T0); + const pushed = await json(await handler(request("/capacity/seats", "POST", snapshot([claudeSeat()], T0)))); + expect(pushed).toMatchObject({ + status: 200, + body: { results: [{ seat: "cc", outcome: "accepted" }], changed: true, discarded_unreadable: true } + }); + const state = await run(factory.stateView); + expect(state.seat_readings.map((entry) => entry.seat.seat)).toEqual(["cc"]); + expect(state.seat_readings_error).toBeNull(); + expect((await json(await handler(request("/capacity/admit?seat=cc", "GET")))).body).toMatchObject({ admit: true }); + }); +}); diff --git a/pkgs/substrate-apps/src/packages/factory/test/factory.test.ts b/pkgs/substrate-apps/src/packages/factory/test/factory.test.ts new file mode 100644 index 00000000..fcce2c7f --- /dev/null +++ b/pkgs/substrate-apps/src/packages/factory/test/factory.test.ts @@ -0,0 +1,384 @@ +import { Option } from "effect"; +import { describe, expect, it } from "vitest"; +import { cloudReading, armedPlan } from "../../planning/test/fixtures.ts"; +import { + armItems, + emptyFactory, + factoryFor, + heartbeat, + item, + reading, + report, + run, + verdict +} from "./fixtures.ts"; + +const capped = (cap = 1) => [ + { axis: "level" as const, subject: "approved", cap }, + { axis: "namespace" as const, subject: "mecattaf/conwip", cap }, + { axis: "family" as const, subject: "build", cap } +]; + +describe("the Factory state machine", () => { + it("an accepted, unclosed item is not proposed on the next evaluate", async () => { + const factory = await factoryFor( + [item({ taskId: "A", rank: 0 }), item({ taskId: "B", rank: 1 })], + { caps: capped(1) } + ); + const capacity = reading({}); + await run(factory.observeCapacity(capacity)); + expect((await run(factory.handOut(capacity.executor))).map((p) => p.taskId)).toEqual(["A"]); + await run(factory.observeOutcome(report("A", "Accepted"))); + + expect(await run(factory.handOut(capacity.executor))).toEqual([]); + expect((await run(factory.releaseState)).backlog[0]?.state).toBe("inflight"); + }); + + it("wip after verdict equals wip before admit", async () => { + const factory = await factoryFor([item({ taskId: "A" })], { caps: capped(1) }); + const before = (await run(factory.releaseState)).wip; + const capacity = reading({}); + await run(factory.observeCapacity(capacity)); + await run(factory.handOut(capacity.executor)); + await run(factory.observeOutcome(report("A", "Accepted"))); + const during = await run(factory.stateView); + expect(during.wip).toEqual({ + level: { approved: 1 }, + namespace: { "mecattaf/conwip": 1 }, + family: { build: 1 } + }); + + await run(factory.observeVerdict(verdict("A", "pass"))); + expect((await run(factory.releaseState)).wip).toEqual(before); + }); + + it("a NotYet returns the item with deferrals+1", async () => { + const factory = await factoryFor([item({ taskId: "A" })]); + const capacity = reading({}); + await run(factory.observeCapacity(capacity)); + await run(factory.handOut(capacity.executor)); + await run(factory.observeOutcome(report("A", "NotYet"))); + + const state = await run(factory.releaseState); + expect(state.backlog[0]?.state).toBe("unclaimed"); + expect(state.backlog[0]?.deferrals).toBe(1); + expect((await run(factory.stateView)).deferrals_by_row).toEqual({ + "gpu-coordinator": 1 + }); + }); + + it("applies the same NotYet report again after a new release cycle", async () => { + const factory = await factoryFor([item({ taskId: "A" })]); + const capacity = reading({}); + const notYet = report("A", "NotYet"); + await run(factory.observeCapacity(capacity)); + + await run(factory.handOut(capacity.executor)); + expect((await run(factory.observeOutcome(notYet))).idempotent).toBe(false); + expect((await run(factory.observeOutcome(notYet))).idempotent).toBe(true); + + await run(factory.handOut(capacity.executor)); + expect((await run(factory.observeOutcome(notYet))).idempotent).toBe(false); + const state = await run(factory.stateView); + expect(state.backlog[0]).toMatchObject({ state: "unclaimed", deferrals: 2 }); + expect(state.deferrals_by_row).toEqual({ "gpu-coordinator": 2 }); + }); + + it("the same verdict posted twice mirrors once and frees one slot", async () => { + const factory = await factoryFor([item({ taskId: "A" })], { caps: capped(1) }); + const capacity = reading({}); + await run(factory.observeCapacity(capacity)); + await run(factory.handOut(capacity.executor)); + await run(factory.observeOutcome(report("A", "Accepted"))); + const first = await run(factory.observeVerdict(verdict("A", "pass"))); + const second = await run(factory.observeVerdict(verdict("A", "pass"))); + + expect(first.idempotent).toBe(false); + expect(second.idempotent).toBe(true); + const state = await run(factory.stateView); + expect(state.mirrored_verdicts).toBe(1); + expect(state.wip.level.approved).toBe(0); + }); + + it("a re-arm with the same planHash adds only new ordinals", async () => { + const factory = await emptyFactory(); + const original = item({ taskId: "ordinal-3", rank: 3, value: 10 }); + await armItems(factory, [original]); + const capacity = reading({}); + await run(factory.observeCapacity(capacity)); + await run(factory.handOut(capacity.executor)); + await run(factory.observeOutcome(report("ordinal-3", "Accepted"))); + await run(factory.observeVerdict(verdict("ordinal-3", "pass"))); + + await armItems(factory, [ + item({ taskId: "ordinal-3-rewritten", rank: 3, value: 999 }), + item({ taskId: "ordinal-4", rank: 4 }) + ]); + const state = await run(factory.releaseState); + expect(state.backlog.map((entry) => entry.rank)).toEqual([3, 4]); + expect(state.backlog[0]).toMatchObject({ + taskId: "ordinal-3", + state: "closed", + value: 10 + }); + expect(state.backlog[1]).toMatchObject({ taskId: "ordinal-4", state: "unclaimed" }); + }); + + it("moves a handed-out item from unclaimed to released", async () => { + const factory = await factoryFor([item({ taskId: "A" })]); + const capacity = reading({}); + await run(factory.observeCapacity(capacity)); + const proposals = await run(factory.handOut(capacity.executor)); + expect(proposals[0]).toMatchObject({ + taskId: "A", + row: "gpu-coordinator", + argv_ref: "A" + }); + expect((await run(factory.releaseState)).backlog[0]?.state).toBe("released"); + }); + + it("reserves cap slack while a released proposal awaits its answer", async () => { + const factory = await factoryFor( + [item({ taskId: "A", rank: 0 }), item({ taskId: "B", rank: 1 })], + { caps: capped(1) } + ); + const capacity = reading({}); + await run(factory.observeCapacity(capacity)); + expect((await run(factory.handOut(capacity.executor))).map((p) => p.taskId)).toEqual(["A"]); + expect(await run(factory.handOut(capacity.executor))).toEqual([]); + expect((await run(factory.releaseState)).wip).toEqual([]); + }); + + it("refuses a verdict before Accepted and leaves released WIP unchanged", async () => { + const factory = await factoryFor([item({ taskId: "A" })]); + const capacity = reading({}); + await run(factory.observeCapacity(capacity)); + await run(factory.handOut(capacity.executor)); + const before = await run(factory.releaseState); + expect(before.backlog[0]).toMatchObject({ state: "released", outcome: Option.none() }); + + await expect(run(factory.observeVerdict(verdict("A", "pass")))).rejects.toMatchObject({ + reason: "InvalidTransition", + operation: "Factory.observeVerdict", + subject: "A (state released)" + }); + + const after = await run(factory.releaseState); + expect(after.backlog[0]).toMatchObject({ state: "released", outcome: Option.none() }); + expect(after.wip).toEqual(before.wip); + expect((await run(factory.stateView)).mirrored_verdicts).toBe(0); + }); + + it("closes a rejected release without minting a verdict outcome", async () => { + const factory = await factoryFor([item({ taskId: "A" })]); + const capacity = reading({}); + await run(factory.observeCapacity(capacity)); + await run(factory.handOut(capacity.executor)); + await run(factory.observeOutcome(report("A", "Rejected", { code: "not-armed" }))); + const state = await run(factory.stateView); + expect(state.backlog[0]).toMatchObject({ state: "closed", outcome: null }); + expect(state.facts.at(-1)).toMatchObject({ kind: "rejected", detail: "not-armed" }); + }); + + it("deduplicates an outcome report before changing WIP", async () => { + const factory = await factoryFor([item({ taskId: "A" })]); + const capacity = reading({}); + await run(factory.observeCapacity(capacity)); + await run(factory.handOut(capacity.executor)); + const first = await run(factory.observeOutcome(report("A", "Accepted", { lease_id: "L1" }))); + const second = await run(factory.observeOutcome(report("A", "Accepted", { lease_id: "L1" }))); + expect(first.idempotent).toBe(false); + expect(second.idempotent).toBe(true); + expect((await run(factory.releaseState)).wip[0]?.count).toBe(1); + }); + + it("does not confuse outcome identities containing embedded NULs", async () => { + const factory = await factoryFor([item({ taskId: "A" })]); + const capacity = reading({}); + const first = report("A", "NotYet", { + lease_id: "lease\u0000part", + row: "row" + }); + const boundaryShifted = report("A", "NotYet", { + lease_id: "lease", + row: "part\u0000row" + }); + await run(factory.observeCapacity(capacity)); + await run(factory.handOut(capacity.executor)); + await run(factory.observeOutcome(first)); + + await expect(run(factory.observeOutcome(boundaryShifted))).rejects.toMatchObject({ + reason: "InvalidTransition" + }); + await run(factory.handOut(capacity.executor)); + expect((await run(factory.observeOutcome(boundaryShifted))).idempotent).toBe(false); + expect((await run(factory.releaseState)).backlog[0]).toMatchObject({ + state: "unclaimed", + deferrals: 2 + }); + }); + + it("refuses an outcome carrying another proposal's dedup key", async () => { + const factory = await factoryFor([item({ taskId: "A" })]); + const capacity = reading({}); + await run(factory.observeCapacity(capacity)); + await run(factory.handOut(capacity.executor)); + await expect( + run(factory.observeOutcome(report("A", "Accepted", { dedupKey: "wrong" }))) + ).rejects.toMatchObject({ reason: "DedupMismatch" }); + }); +}); + +describe("dependency release and verdict replay", () => { + it("proposes B only after A closes pass", async () => { + const factory = await factoryFor( + [ + item({ taskId: "A", rank: 0 }), + item({ taskId: "B", rank: 1, dependsOn: ["A"] }) + ], + { caps: capped(2) } + ); + const capacity = reading({}); + await run(factory.observeCapacity(capacity)); + expect((await run(factory.handOut(capacity.executor))).map((p) => p.taskId)).toEqual(["A"]); + await run(factory.observeOutcome(report("A", "Accepted"))); + await run(factory.observeVerdict(verdict("A", "pass"))); + expect((await run(factory.handOut(capacity.executor))).map((p) => p.taskId)).toEqual(["B"]); + }); + + it("returns A to attempt 2 after fail and keeps B dependency-blocked", async () => { + const factory = await factoryFor( + [ + item({ taskId: "A", rank: 0 }), + item({ taskId: "B", rank: 1, dependsOn: ["A"] }) + ], + { caps: capped(2) } + ); + const capacity = reading({}); + await run(factory.observeCapacity(capacity)); + await run(factory.handOut(capacity.executor)); + await run(factory.observeOutcome(report("A", "Accepted"))); + await run(factory.observeVerdict(verdict("A", "fail"))); + const state = await run(factory.releaseState); + expect(state.backlog[0]).toMatchObject({ state: "unclaimed", attempt: 2 }); + expect((await run(factory.handOut(capacity.executor))).map((p) => p.taskId)).toEqual(["A"]); + }); + + it("gives a retried attempt a new admit identity", async () => { + const factory = await factoryFor([item({ taskId: "A" })]); + const capacity = reading({}); + await run(factory.observeCapacity(capacity)); + const first = await run(factory.handOut(capacity.executor)); + const firstKey = first[0]?.dedupKey; + await run( + factory.observeOutcome( + report("A", "Accepted", { dedupKey: firstKey, lease_id: "lease-1" }) + ) + ); + await run(factory.observeVerdict(verdict("A", "fail"))); + + const second = await run(factory.handOut(capacity.executor)); + const secondKey = second[0]?.dedupKey; + expect(secondKey).not.toBe(firstKey); + const accepted = await run( + factory.observeOutcome( + report("A", "Accepted", { dedupKey: secondKey, lease_id: "lease-2" }) + ) + ); + expect(accepted.idempotent).toBe(false); + expect((await run(factory.releaseState)).backlog[0]).toMatchObject({ + state: "inflight", + attempt: 2, + dedupKey: secondKey + }); + expect((await run(factory.stateView)).wip.level.approved).toBe(1); + }); + + it("closes a failed item at its authored attempt cap", async () => { + const factory = await factoryFor([item({ taskId: "A" })], { + plans: [{ ...armedPlan, attemptCap: Option.some(1) }] + }); + const capacity = reading({}); + await run(factory.observeCapacity(capacity)); + await run(factory.handOut(capacity.executor)); + await run(factory.observeOutcome(report("A", "Accepted"))); + await run(factory.observeVerdict(verdict("A", "fail"))); + expect((await run(factory.releaseState)).backlog[0]).toMatchObject({ + state: "closed", + attempt: 1 + }); + }); + + it("refuses a first mirrored verdict that skips the chain", async () => { + const factory = await factoryFor([item({ taskId: "A" })]); + const capacity = reading({}); + await run(factory.observeCapacity(capacity)); + await run(factory.handOut(capacity.executor)); + await run(factory.observeOutcome(report("A", "Accepted"))); + await expect(run(factory.observeVerdict(verdict("A", "pass", 2)))).rejects.toMatchObject({ + reason: "ContinuityGap" + }); + const state = await run(factory.stateView); + expect(state.continuity_gaps).toHaveLength(1); + expect(state.chains).toEqual({}); + }); +}); + +describe("capacity wake state", () => { + it("treats SLOW as STOP for proposals", async () => { + const factory = await factoryFor([item({ taskId: "A" })]); + const capacity = reading({ signal: "SLOW" }); + await run(factory.observeCapacity(capacity)); + expect(await run(factory.handOut(capacity.executor))).toEqual([]); + }); + + it("keeps only the newest reading from each executor", async () => { + const factory = await factoryFor([]); + expect(await run(factory.observeCapacity(reading({ seq: 2 })))).toBe(true); + expect(await run(factory.observeCapacity(reading({ seq: 1 })))).toBe(false); + expect((await run(factory.stateView)).latest_readings.coordinator?.seq).toBe(2); + }); + + it("sets alarm_at to the minimum next_window_at and records alarm firing", async () => { + const factory = await factoryFor([]); + await run( + factory.observeCapacity(reading({ nextWindowAt: "2026-09-06T18:00:00Z" })) + ); + await run( + factory.observeCapacity(cloudReading({ nextWindowAt: "2026-09-06T17:00:00Z" })) + ); + expect((await run(factory.stateView)).alarm_at).toBe("2026-09-06T17:00:00Z"); + await run(factory.alarm("2026-09-06T17:00:00Z")); + expect((await run(factory.stateView)).alarm_fired_at).toBe("2026-09-06T17:00:00Z"); + }); + + it("returns chain high-water marks and the constant price hash", async () => { + const factory = await factoryFor([]); + await run(factory.observeVerdict(verdict("external-task", "pass"))); + const state = await run(factory.stateView); + expect(state.chains.coordinator).toEqual({ + last_seq: 1, + last_hash: verdict("external-task", "pass").hash + }); + expect(state.price_hash).toBe("d".repeat(64)); + }); + + it("reports the latest heartbeat by lane", async () => { + const factory = await factoryFor([]); + await run( + factory.observeHeartbeat( + heartbeat("lane-A", 1), + "lane-A", + "2026-09-06T16:00:00Z" + ) + ); + await run(factory.observeHeartbeat(heartbeat("lane-A", 1), "lane-A")); + expect((await run(factory.stateView)).heartbeats_by_lane["lane-A"]).toEqual({ + executor: "coordinator", + task_id: "lane-A", + last_seq: 1, + status: "live", + observed_at: "2026-09-06T16:00:00Z" + }); + }); +}); diff --git a/pkgs/substrate-apps/src/packages/factory/test/fixtures.ts b/pkgs/substrate-apps/src/packages/factory/test/fixtures.ts new file mode 100644 index 00000000..e70e66ea --- /dev/null +++ b/pkgs/substrate-apps/src/packages/factory/test/fixtures.ts @@ -0,0 +1,109 @@ +import { Effect, Schema } from "effect"; +import { + artifactHasherTableLayer, + makeFactory, + planningStoreInMemoryLayer, + type ArmedArtifact, + type IFactory +} from "../src/index.ts"; +import { AdmitOutcomeReport } from "@substrate/planning/schema/admit.ts"; +import { Heartbeat } from "@substrate/planning/schema/uplink.ts"; +import { PlanArm } from "@substrate/planning/schema/plan.ts"; +import { Verdict } from "@substrate/planning/schema/records.ts"; +import { Sha256Hex } from "@substrate/planning/schema/ids.ts"; +import type { ReleaseState } from "@substrate/planning/release/evaluator.ts"; +import { + digest, + item, + reading, + releaseState +} from "../../planning/test/fixtures.ts"; + +const decodeOutcome = Schema.decodeUnknownSync(AdmitOutcomeReport); +const decodeHeartbeat = Schema.decodeUnknownSync(Heartbeat); +const decodePlanArm = Schema.decodeUnknownSync(PlanArm); +const decodeVerdict = Schema.decodeUnknownSync(Verdict); +const decodeSha = Schema.decodeUnknownSync(Sha256Hex); + +export { digest, item, reading, releaseState }; + +export const artifactBytes = Uint8Array.of(1, 2, 3, 4); +export const artifactDigest = decodeSha(digest("e")); + +export const armRequest = decodePlanArm({ + label: "night.js", + kind: "workflow", + level: "approved", + namespace: "mecattaf/conwip", + planHash: artifactDigest, + author: "tom" +}); + +export const factoryFor = ( + items: ReleaseState["backlog"], + overrides: Partial = {} +): Promise => + Effect.runPromise( + makeFactory(releaseState(items, overrides)).pipe( + Effect.provide(planningStoreInMemoryLayer) + ) + ); + +export const emptyFactory = (): Promise => + factoryFor([], { plans: [] }); + +export const armItems = ( + factory: IFactory, + items: ArmedArtifact["items"] +): Promise => + Effect.runPromise( + factory + .armPlan(armRequest, { + planId: items[0]?.planId ?? item({ taskId: "placeholder" }).planId, + bytes: artifactBytes, + items + }) + .pipe(Effect.provide(artifactHasherTableLayer([[artifactBytes, artifactDigest]]))) + ); + +export const report = ( + taskId: string, + outcome: "Accepted" | "Rejected" | "NotYet", + extra: Record = {} +) => + decodeOutcome({ + taskId, + dedupKey: `dedup-${taskId}`, + outcome, + row: "gpu-coordinator", + ...extra + }); + +export const verdict = ( + taskId: string, + outcome: "pass" | "fail" | "cancelled" | "preempted" | "expired", + sequence = 1, + extra: Record = {} +) => + decodeVerdict({ + _tag: "Verdict", + taskId, + executor: "coordinator", + seq: sequence, + hash: digest(String(sequence % 10)), + prevHash: digest(String(Math.max(0, sequence - 1) % 10)), + outcome, + serviceSeconds: 1, + ...extra + }); + +export const heartbeat = (taskId: string | null, sequence = 1) => + decodeHeartbeat({ + _tag: "Heartbeat", + executor: "coordinator", + taskId, + seq: sequence + }); + +export const run = (effect: Effect.Effect): Promise => + Effect.runPromise(effect); diff --git a/pkgs/substrate-apps/src/packages/factory/test/http.test.ts b/pkgs/substrate-apps/src/packages/factory/test/http.test.ts new file mode 100644 index 00000000..57bf5fa6 --- /dev/null +++ b/pkgs/substrate-apps/src/packages/factory/test/http.test.ts @@ -0,0 +1,241 @@ +import { readFileSync } from "node:fs"; +import { fileURLToPath } from "node:url"; +import { Schema } from "effect"; +import { describe, expect, it } from "vitest"; +import { + makeFactoryHttpHandler, + makeTableArtifactHasher +} from "../src/index.ts"; +import { BacklogItem } from "@substrate/planning/schema/backlog.ts"; +import { CapacityReading } from "@substrate/planning/schema/capacity.ts"; +import { + armRequest, + artifactBytes, + artifactDigest, + emptyFactory, + factoryFor, + item, + reading, + report, + run, + verdict +} from "./fixtures.ts"; + +const RECEIPT = fileURLToPath( + new URL("../../schema/test/fixtures/receipt-strict-full.json", import.meta.url) +); +const receipt = (): Record => JSON.parse(readFileSync(RECEIPT, "utf8")); +const encodeReading = Schema.encodeSync(CapacityReading); +const encodeItem = Schema.encodeSync(BacklogItem); + +const request = ( + path: string, + method: "GET" | "POST", + body?: unknown, + authenticated = true +) => + new Request(`https://factory.invalid${path}`, { + method, + headers: { + ...(authenticated ? { authorization: "Bearer test-token" } : {}), + ...(body === undefined ? {} : { "content-type": "application/json" }) + }, + ...(body === undefined ? {} : { body: JSON.stringify(body) }) + }); + +describe("the host-neutral HTTP surface", () => { + it("requires bearer authorization on every write", async () => { + const factory = await emptyFactory(); + const handler = makeFactoryHttpHandler({ + factory, + token: "test-token", + artifactHasher: makeTableArtifactHasher([[artifactBytes, artifactDigest]]) + }); + const result = await handler( + request("/capacity", "POST", encodeReading(reading({})), false) + ); + expect(result.status).toBe(401); + }); + + it("requires bearer authorization before a proposal pull can release work", async () => { + const factory = await factoryFor([item({ taskId: "A" })]); + await run(factory.observeCapacity(reading({}))); + const handler = makeFactoryHttpHandler({ + factory, + token: "test-token", + artifactHasher: makeTableArtifactHasher([]) + }); + + const refused = await handler( + request("/proposals?executor=coordinator", "GET", undefined, false) + ); + expect(refused.status).toBe(401); + expect((await run(factory.releaseState)).backlog[0]?.state).toBe("unclaimed"); + + const accepted = await handler( + request("/proposals?executor=coordinator", "GET") + ); + expect(accepted.status).toBe(200); + expect((await run(factory.releaseState)).backlog[0]?.state).toBe("released"); + }); + + it("serves plans, capacity, proposals, outcomes and state without a Worker entry", async () => { + const factory = await emptyFactory(); + const handler = makeFactoryHttpHandler({ + factory, + token: "test-token", + artifactHasher: makeTableArtifactHasher([[artifactBytes, artifactDigest]]) + }); + const a = item({ taskId: "A", rank: 0 }); + expect( + ( + await handler( + request("/plans", "POST", { + arm: armRequest, + artifact: { + planId: a.planId, + bytes: [...artifactBytes], + items: [encodeItem(a)] + } + }) + ) + ).status + ).toBe(200); + expect( + ( + await handler( + request("/capacity", "POST", encodeReading(reading({}))) + ) + ).status + ).toBe(200); + + const proposals = await handler(request("/proposals?executor=coordinator", "GET")); + expect(proposals.status).toBe(200); + expect( + (await proposals.json()) as { + proposals: Array<{ taskId: string }>; + next_wake_at: string | null; + } + ).toMatchObject({ proposals: [{ taskId: "A" }], next_wake_at: null }); + expect((await handler(request("/outcomes", "POST", report("A", "Accepted")))).status).toBe( + 200 + ); + const state = await handler(request("/state", "GET", undefined, false)); + expect(state.status).toBe(200); + expect((await state.json()) as { backlog: Array<{ state: string }> }).toMatchObject({ + backlog: [{ state: "inflight" }] + }); + }); + + it("returns next_wake_at when every row is STOP and no proposal is admissible", async () => { + const factory = await emptyFactory(); + const handler = makeFactoryHttpHandler({ + factory, + token: "test-token", + artifactHasher: makeTableArtifactHasher([[artifactBytes, artifactDigest]]) + }); + const a = item({ taskId: "A", rank: 0 }); + expect( + ( + await handler( + request("/plans", "POST", { + arm: armRequest, + artifact: { + planId: a.planId, + bytes: [...artifactBytes], + items: [encodeItem(a)] + } + }) + ) + ).status + ).toBe(200); + + const nextWakeAt = "2026-09-06T18:00:00Z"; + const encoded = encodeReading(reading({ signal: "STOP", nextWindowAt: nextWakeAt })); + const allStopped = { + ...encoded, + rows: encoded.rows.map((row) => ({ ...row, signal: "STOP" as const })) + }; + expect((await handler(request("/capacity", "POST", allStopped))).status).toBe(200); + + const response = await handler(request("/proposals?executor=coordinator", "GET")); + expect(response.status).toBe(200); + expect(await response.json()).toEqual({ proposals: [], next_wake_at: nextWakeAt }); + }); + + it("returns 409 when a strict receipt has no matching mirrored verdict", async () => { + const factory = await emptyFactory(); + const handler = makeFactoryHttpHandler({ + factory, + token: "test-token", + artifactHasher: makeTableArtifactHasher([]) + }); + const result = await handler( + request("/receipts", "POST", { + receipt: receipt(), + verdict_hash: "1".repeat(64) + }) + ); + expect(result.status).toBe(409); + }); + + it("accepts a receipt only when all three evidence cells match", async () => { + const factory = await emptyFactory(); + const handler = makeFactoryHttpHandler({ + factory, + token: "test-token", + artifactHasher: makeTableArtifactHasher([]) + }); + const line = receipt(); + const mirrored = verdict("receipt-evaluator", "pass", 1, { + unit_id: line["id"], + oracle_rc: line["oracle_rc"], + mutation_rc: (line["mutation"] as { rc: number }).rc, + oracle_output_sha256: line["oracle_output_sha256"] + }); + expect((await handler(request("/verdicts", "POST", mirrored))).status).toBe(200); + expect( + ( + await handler( + request("/receipts", "POST", { + receipt: line, + verdict_hash: mirrored.hash + }) + ) + ).status + ).toBe(200); + + const changed = { ...line, oracle_rc: 7 }; + expect( + ( + await handler( + request("/receipts", "POST", { + receipt: changed, + verdict_hash: mirrored.hash + }) + ) + ).status + ).toBe(409); + expect((await run(factory.stateView)).receipts).toHaveLength(1); + }); + + it("strictly rejects a receipt with a null measured cell", async () => { + const factory = await emptyFactory(); + const handler = makeFactoryHttpHandler({ + factory, + token: "test-token", + artifactHasher: makeTableArtifactHasher([]) + }); + const invalid = { ...receipt(), tokens: null }; + expect( + ( + await handler( + request("/receipts", "POST", { + receipt: invalid, + verdict_hash: "1".repeat(64) + }) + ) + ).status + ).toBe(400); + }); +}); diff --git a/pkgs/substrate-apps/src/packages/factory/test/inputs.test.ts b/pkgs/substrate-apps/src/packages/factory/test/inputs.test.ts new file mode 100644 index 00000000..5545f620 --- /dev/null +++ b/pkgs/substrate-apps/src/packages/factory/test/inputs.test.ts @@ -0,0 +1,58 @@ +import { Effect, Schema } from "effect"; +import { describe, expect, it } from "vitest"; +import { + Factory, + FactoryCapacityFeed, + FactoryPriceSource, + constantPriceVectorLayer, + fakeCapacityLayer, + loadFactoryInputs +} from "../src/index.ts"; +import { PriceVector } from "@substrate/planning/schema/prices.ts"; +import { factoryFor, item, reading } from "./fixtures.ts"; + +const decodePrices = Schema.decodeUnknownSync(PriceVector); + +describe("the in-memory composition Layers", () => { + it("wires a constant PriceVector and fake capacity readings into the service", async () => { + const factory = await factoryFor([item({ taskId: "A" })]); + const prices = decodePrices({ + hash: "9".repeat(64), + rows: [{ row: "gpu-coordinator", price: 0 }], + drum: 1, + stage: [{ family: "build", holdingCost: 0 }] + }); + const capacity = reading({ nextWindowAt: "2026-09-06T17:00:00Z" }); + const state = await Effect.runPromise( + loadFactoryInputs().pipe( + Effect.provideService(Factory, Factory.of(factory)), + Effect.provide(fakeCapacityLayer([capacity])), + Effect.provide(constantPriceVectorLayer(prices)) + ) + ); + expect(state.price_hash).toBe("9".repeat(64)); + expect(state.latest_readings.coordinator?.seq).toBe(1); + expect(state.alarm_at).toBe("2026-09-06T17:00:00Z"); + }); + + it("exposes the two test inputs as distinct Effect services", async () => { + const capacity = reading({}); + const prices = decodePrices({ + hash: "8".repeat(64), + rows: [], + drum: 1, + stage: [] + }); + const values = await Effect.runPromise( + Effect.gen(function* () { + const feed = yield* FactoryCapacityFeed; + const price = yield* FactoryPriceSource; + return [(yield* feed.readings).length, price.prices.hash] as const; + }).pipe( + Effect.provide(fakeCapacityLayer([capacity])), + Effect.provide(constantPriceVectorLayer(prices)) + ) + ); + expect(values).toEqual([1, "8".repeat(64)]); + }); +}); diff --git a/pkgs/substrate-apps/src/packages/factory/test/persistence.test.ts b/pkgs/substrate-apps/src/packages/factory/test/persistence.test.ts new file mode 100644 index 00000000..7456aee7 --- /dev/null +++ b/pkgs/substrate-apps/src/packages/factory/test/persistence.test.ts @@ -0,0 +1,70 @@ +import { Effect, Option } from "effect"; +import { describe, expect, it } from "vitest"; + +import { + FACTORY_STATE_KEY, + makeFactory, + makeInMemoryPlanningStore, + PlanningStore +} from "../src/index.ts"; +import { item, reading, releaseState, report, run } from "./fixtures.ts"; + +describe("Factory state in PlanningStore", () => { + it("restores inflight work, WIP, alarms, readings and outcome identity", async () => { + const store = await run(makeInMemoryPlanningStore); + const create = (initial: ReturnType) => + run(makeFactory(initial).pipe(Effect.provideService(PlanningStore, store))); + const alarmAt = "2026-09-07T03:04:05Z"; + + const first = await create(releaseState([item({ taskId: "A" })])); + const capacity = reading({ nextWindowAt: alarmAt }); + await run(first.observeCapacity(capacity)); + const proposals = await run(first.handOut(capacity.executor)); + const outcome = report("A", "Accepted", { + dedupKey: proposals[0]?.dedupKey, + lease_id: "lease-persisted" + }); + await run(first.observeOutcome(outcome)); + + expect(Option.isSome(await run(store.get(FACTORY_STATE_KEY)))).toBe(true); + const recreated = await create(releaseState([], { plans: [], wip: [] })); + const state = await run(recreated.stateView); + expect(state.backlog[0]?.state).toBe("inflight"); + expect(state.wip.level.approved).toBe(1); + expect(state.alarm_at).toBe(alarmAt); + expect(state.latest_readings.coordinator?.seq).toBe(1); + expect(await run(recreated.handOut(capacity.executor))).toEqual([]); + expect((await run(recreated.observeOutcome(outcome))).idempotent).toBe(true); + expect((await run(recreated.stateView)).wip.level.approved).toBe(1); + }); + + it("restores explicit proposal command and mutation references", async () => { + const store = await run(makeInMemoryPlanningStore); + const create = (initial: ReturnType) => + run(makeFactory(initial).pipe(Effect.provideService(PlanningStore, store))); + const enriched = item({ + taskId: "A", + argv_ref: "cards/U-A11.json#A", + mutation_hint: "delete dependencyUnmet" + }); + + const first = await create(releaseState([enriched])); + expect((await run(first.releaseState)).backlog[0]).toMatchObject({ + argv_ref: "cards/U-A11.json#A", + mutation_hint: "delete dependencyUnmet" + }); + + const recreated = await create(releaseState([], { plans: [], wip: [] })); + expect((await run(recreated.releaseState)).backlog[0]).toMatchObject({ + argv_ref: "cards/U-A11.json#A", + mutation_hint: "delete dependencyUnmet" + }); + const capacity = reading({}); + await run(recreated.observeCapacity(capacity)); + expect((await run(recreated.handOut(capacity.executor)))[0]).toMatchObject({ + taskId: "A", + argv_ref: "cards/U-A11.json#A", + mutation_hint: "delete dependencyUnmet" + }); + }); +}); diff --git a/pkgs/substrate-apps/src/packages/factory/test/projection.test.ts b/pkgs/substrate-apps/src/packages/factory/test/projection.test.ts new file mode 100644 index 00000000..553725fe --- /dev/null +++ b/pkgs/substrate-apps/src/packages/factory/test/projection.test.ts @@ -0,0 +1,317 @@ +/** + * U-A13 LAKE-PROJECTION — the suite. + * + * The unit's DOMINANT oracle runs against the register on this disk, which is a + * corpus that belongs to other lanes and moves while they bank into it. This + * suite is the part that does not: hermetic fixtures, no path opened, one + * assertion per sentence of §2.2e. + * + * What it asserts, in the order §2.2e says it: + * + * the banked corpus is read as it is, not as a schema wishes it were + * the mirror is what the Factory object accepted, and nothing else + * the diff sees a dropped receipt — the unit's own mutation_hint + * the diff sees a card that contradicts the projection, and is silent about + * one that is merely unwritten + * `--write` fills the nine cells and never `status` or `outcome_ruled` + * re-applying the same projection changes nothing + */ +import { Effect, Option } from "effect"; +import { describe, expect, it } from "vitest"; +import { + applyCells, + bankedRowOf, + cardCellsOf, + diffProjections, + fencedLines, + loadBankedIntoFactory, + makeFactory, + NEVER_WRITTEN, + planningStoreInMemoryLayer, + priorGapOf, + projectionRowOf, + PROJECTION_CELLS, + statOf, + type BankedRow, + type CardView, + type ProjectionRow +} from "../src/index.ts"; +import { armedPlan, item, reading, releaseState } from "../../planning/test/fixtures.ts"; + +/** + * One banked receipt in the shape the corpus actually has. + * + * MEASURED 2026-09-06 over `receipts/FACTORY-2026-09-06/*​/receipt.json`: every + * one of the 52 files carries `unit`, `disposition`, `verdict`, `oracle_rc`, + * `seconds` and a six-cell `tokens`; 27 also carry `outcome_for_calibration`, + * `prior_gap` and `mutation_rc`; 0 of 52 decode under the strict §2.3 `Receipt`. + */ +const receipt = (overrides: Record = {}) => ({ + id: "U-X1-2026-09-06-r0", + unit: "U-X1", + kind: "build", + disposition: "KEEP", + verdict: "PASS", + seconds: 1200, + oracle_rc: 0, + oracle_output_sha256: "a".repeat(64), + mutation_rc: 1, + outcome_for_calibration: "pass", + prior_gap: { tokens_out: 4200, p_pass_vs_outcome: "0.7 vs 1" }, + tokens: { + in_uncached: 1, + cache_read: 2, + cache_write: 0, + out: 3, + reasoning: 4, + total: 10 + }, + ...overrides +}); + +const rowOf = (unit: string, overrides: Record = {}): BankedRow => { + const read = bankedRowOf( + unit, + `receipts/FACTORY-2026-09-06/${unit}/receipt.json`, + unit.padEnd(64, "0"), + receipt({ unit, ...overrides }) + ); + if (!read.ok) throw new Error(`the fixture receipt did not read: ${read.why}`); + return read.row; +}; + +const chainLink = (index: number) => ({ + seq: index + 1, + hash: String(index + 1).padStart(64, "0"), + prevHash: String(index).padStart(64, "0") +}); + +const load = async (rows: ReadonlyArray) => { + const items = rows.map((row, index) => item({ taskId: row.unit, rank: index })); + const factory = await Effect.runPromise( + makeFactory( + releaseState(items, { plans: [{ ...armedPlan, attemptCap: Option.some(1) }] }) + ).pipe(Effect.provide(planningStoreInMemoryLayer)) + ); + return Effect.runPromise( + loadBankedIntoFactory(factory, rows, chainLink, (pass) => reading({ seq: pass + 1 })) + ); +}; + +const registerSide = (rows: ReadonlyArray): ReadonlyArray => + rows + .map((row) => + projectionRowOf(row, { + id: row.unit, + oracle_rc: row.oracle_rc, + oracle_output_sha256: row.oracle_output_sha256, + verdict_hash: "" + }) + ) + .sort((a, b) => a.unit.localeCompare(b.unit)); + +const card = (id: string, overrides: Partial = {}): CardView => ({ + id, + repo: "mecattaf/substrate", + cells: new Map(), + ...overrides +}); + +const CARD_TEXT = [ + "---", + 'id: "U-X1"', + "status: DRAFT # DRAFT -> ARMED -> RUNNING -> KEEP | DISCARD", + 'repo: "mecattaf/substrate"', + "prior:", + " p_pass: 0.55", + " predicted_tokens_cell: out # DECISIONS.md D-B19", + 'grade: "CLAIMED"', + "actuals: # written after the run", + " tokens: null", + " seconds: null", + " outcome: null", + " receipt_sha256: null", + " prior_gap: null", + "---", + "", + "# U-X1 — the body, which the writer never touches", + "" +].join("\n"); + +describe("reading the banked corpus", () => { + it("takes the cells the receipt carries and names the ones it does not", () => { + const read = bankedRowOf("U-X1", "p", "s", receipt({ mutation_rc: undefined })); + expect(read.ok).toBe(true); + if (!read.ok) return; + expect(read.row.absent).toContain("mutation_rc"); + expect(read.row.mutation_rc).toBeNull(); + expect(read.row.tokens).toBe(10); + expect(read.row.outcome).toBe("pass"); + }); + + it("refuses a receipt missing a cell it cannot do without, by name", () => { + const read = bankedRowOf("U-X1", "p", "s", receipt({ seconds: undefined, tokens: {} })); + expect(read.ok).toBe(false); + if (read.ok) return; + expect(read.why).toContain("seconds"); + expect(read.why).toContain("tokens.total"); + }); + + it("carries `none` rather than a null where §2.3 has no digest (D-B22)", () => { + const row = rowOf("U-X1", { oracle_output_sha256: undefined }); + expect(row.oracle_output_sha256).toBe("none"); + }); + + it("reads both spellings of prior_gap as the one declared cell", () => { + expect(priorGapOf({ tokens_out: 7, p_pass_vs_outcome: "0.5 vs 1" })).toBe(7); + expect(priorGapOf({ tokens_by_cell: { out: -9, "in_uncached+out": 1, total: 2 } })).toBe(-9); + expect(priorGapOf(undefined)).toBeNull(); + }); + + it("reads a FAIL verdict as the mirror's `fail`, not as an absence", () => { + expect(rowOf("U-X1", { verdict: "FAIL" }).outcome).toBe("fail"); + }); +}); + +describe("loading the corpus into the Factory object", () => { + it("walks every receipt through the state machine and mirrors it back", async () => { + const rows = [rowOf("U-X1"), rowOf("U-X2"), rowOf("U-X3")]; + const result = await load(rows); + expect(result.refused).toEqual([]); + expect(result.unreleased).toEqual([]); + expect(result.rows.map((row) => row.unit)).toEqual(["U-X1", "U-X2", "U-X3"]); + expect(result.rows[0]?.result.observed).toBe(0); + expect(result.rows[0]?.result.grade).toBe("MEASURED"); + }); + + it("agrees with the same rows read straight off the register", async () => { + const rows = [rowOf("U-X1"), rowOf("U-X2")]; + const result = await load(rows); + expect(diffProjections(result.rows, registerSide(rows), [])).toEqual([]); + }); + + it("names the unit when one banked receipt is dropped from the load", async () => { + // The unit's mutation_hint: "drop one banked receipt → --diff non-empty". + const rows = [rowOf("U-X1"), rowOf("U-X2"), rowOf("U-X3")]; + const result = await load(rows.filter((row) => row.unit !== "U-X2")); + const lines = diffProjections(result.rows, registerSide(rows), []); + expect(lines).toEqual(["missing U-X2 receipts/FACTORY-2026-09-06/U-X2/receipt.json"]); + }); + + it("names a mirrored row the register does not bank", async () => { + const rows = [rowOf("U-X1"), rowOf("U-X2")]; + const result = await load(rows); + const lines = diffProjections(result.rows, registerSide([rows[0]!]), []); + expect(lines).toEqual(["extra U-X2"]); + }); + + it("projects no row for a receipt whose evidence the object refuses", async () => { + // A verdict that skips the chain is a ContinuityGap; the object refuses it, + // the receipt then has no verdict to match, and the mirror holds neither. + const rows = [rowOf("U-X1"), rowOf("U-X2")]; + const items = rows.map((row, index) => item({ taskId: row.unit, rank: index })); + const factory = await Effect.runPromise( + makeFactory( + releaseState(items, { plans: [{ ...armedPlan, attemptCap: Option.some(1) }] }) + ).pipe(Effect.provide(planningStoreInMemoryLayer)) + ); + const skipping = (index: number) => ({ + seq: (index + 1) * 7, + hash: String(index + 1).padStart(64, "0"), + prevHash: String(index).padStart(64, "0") + }); + const result = await Effect.runPromise( + loadBankedIntoFactory(factory, rows, skipping, (pass) => reading({ seq: pass + 1 })) + ); + expect(result.rows).toEqual([]); + expect(result.refused.map((entry) => entry.reason)).toContain("ContinuityGap"); + }); +}); + +describe("the diff against the cards", () => { + it("is silent about a cell the card has not written yet", async () => { + const rows = [rowOf("U-X1")]; + const result = await load(rows); + const unwritten = card("U-X1", { + cells: new Map(PROJECTION_CELLS.map((cell) => [cell, null])) + }); + expect(diffProjections(result.rows, registerSide(rows), [unwritten])).toEqual([]); + }); + + it("names a card that carries a different value in a cell it would fill", async () => { + const rows = [rowOf("U-X1")]; + const result = await load(rows); + const stale = card("U-X1", { cells: new Map([["actuals.tokens", 999]]) }); + expect(diffProjections(result.rows, registerSide(rows), [stale])).toEqual([ + "contradiction U-X1 actuals.tokens card=999 projection=10" + ]); + }); + + it("names a card that carries no repo: (§2.2e's unmapped)", async () => { + const rows = [rowOf("U-X1")]; + const result = await load(rows); + const lines = diffProjections(result.rows, registerSide(rows), [ + card("U-X1"), + card("U-X9", { repo: null }) + ]); + expect(lines).toEqual(["unmapped U-X9"]); + }); +}); + +describe("writing the operator's fields", () => { + const cellsFor = (row: ProjectionRow) => { + const cells = new Map(); + for (const [cell, value] of cardCellsOf(row)) if (value !== null) cells.set(cell, value); + return cells; + }; + + it("fills exactly the cells §2.2e names and leaves the body alone", async () => { + const rows = [rowOf("U-X1")]; + const result = await load(rows); + const after = applyCells(CARD_TEXT, cellsFor(result.rows[0]!)); + expect(after).toContain(" tokens: 10"); + expect(after).toContain(" seconds: 1200"); + expect(after).toContain(' outcome: "KEEP"'); + expect(after).toContain(" prior_gap: 4200"); + expect(after).toContain(' grade: "MEASURED"'); + expect(after).toContain(" observed: 0"); + expect(after).toContain('outcome_for_calibration: "pass"'); + expect(after).toContain("# U-X1 — the body, which the writer never touches"); + }); + + it("never moves a status: or outcome_ruled: line", async () => { + const rows = [rowOf("U-X1")]; + const result = await load(rows); + const after = applyCells(CARD_TEXT, cellsFor(result.rows[0]!)); + expect(fencedLines(after)).toEqual(fencedLines(CARD_TEXT)); + expect(after).toContain("status: DRAFT # DRAFT -> ARMED -> RUNNING -> KEEP | DISCARD"); + for (const key of NEVER_WRITTEN) { + expect([...cardCellsOf(result.rows[0]!).keys()]).not.toContain(key); + expect(PROJECTION_CELLS).not.toContain(key); + } + }); + + it("keeps a trailing comment, and its column, when a value beside it changes", () => { + const after = applyCells(CARD_TEXT, new Map([["status", "KEEP"]])); + expect(after).toContain('status: "KEEP" # DRAFT -> ARMED -> RUNNING -> KEEP | DISCARD'); + }); + + it("is idempotent: re-applying the same cells changes no byte", async () => { + const rows = [rowOf("U-X1")]; + const result = await load(rows); + const once = applyCells(CARD_TEXT, cellsFor(result.rows[0]!)); + const twice = applyCells(once, cellsFor(result.rows[0]!)); + expect(twice).toBe(once); + }); + + it("reads the change as an insertion and not as a rewrite of the file", async () => { + const rows = [rowOf("U-X1")]; + const result = await load(rows); + const after = applyCells(CARD_TEXT, cellsFor(result.rows[0]!)); + const stat = statOf("U-X1.md", CARD_TEXT, after); + // five actuals lines replaced, plus `result:` with its three cells and + // `outcome_for_calibration:` inserted. + expect(stat.removed).toBe(5); + expect(stat.added).toBe(10); + }); +}); diff --git a/pkgs/substrate-apps/src/packages/factory/test/projectionMirror.test.ts b/pkgs/substrate-apps/src/packages/factory/test/projectionMirror.test.ts new file mode 100644 index 00000000..33ff3179 --- /dev/null +++ b/pkgs/substrate-apps/src/packages/factory/test/projectionMirror.test.ts @@ -0,0 +1,247 @@ +import { readFileSync } from "node:fs"; +import { fileURLToPath } from "node:url"; +import { Effect } from "effect"; +import { describe, expect, it } from "vitest"; +import { + makeFactoryHttpHandler, + makeInMemoryPlanningStore, + makeFactory, + mirrorProjectionRows, + NEVER_WRITTEN, + PlanningStore, + retainedReceiptOf, + type IFactory, + type IPlanningStore +} from "../src/index.ts"; +import { checkLineDiscipline } from "@substrate/serializer"; +import { releaseState, run, verdict } from "./fixtures.ts"; + +const RECEIPT = fileURLToPath( + new URL("../../schema/test/fixtures/receipt-strict-full.json", import.meta.url) +); +const receipt = (): Record => JSON.parse(readFileSync(RECEIPT, "utf8")); + +/** + * A hasher that answers for any bytes. + * + * The table hasher refuses bytes it has not been told about, which is right for + * an ARMING — an unknown artifact has no authority. A receipt digest is not an + * authority, it is a label on a row, so the fake here answers by length. It is + * still not sha-256 and still does not pretend to be: the digest's value is + * never asserted below, only its presence and its stability. + */ +const lengthHasher = { + hash: (bytes: Uint8Array) => + Effect.succeed(String(bytes.length).padStart(64, "0") as never) +}; + +const openHandler = async (): Promise<{ + readonly factory: IFactory; + readonly store: IPlanningStore; + readonly handler: (request: Request) => Promise; +}> => { + const store = await run(makeInMemoryPlanningStore); + const factory = await run( + makeFactory(releaseState([], { plans: [] })).pipe( + Effect.provideService(PlanningStore, store) + ) + ); + return { + factory, + store, + handler: makeFactoryHttpHandler({ + factory, + token: "test-token", + artifactHasher: lengthHasher, + store + }) + }; +}; + +const request = (path: string, method: "GET" | "POST", body?: unknown) => + new Request(`https://factory.invalid${path}`, { + method, + headers: { + authorization: "Bearer test-token", + ...(body === undefined ? {} : { "content-type": "application/json" }) + }, + ...(body === undefined ? {} : { body: JSON.stringify(body) }) + }); + +/** Posts one mirrored verdict and the receipt whose evidence matches it. */ +const ingest = async ( + handler: (request: Request) => Promise, + line: Record, + sequence = 1 +): Promise => { + const mirrored = verdict("receipt-evaluator", "pass", sequence, { + unit_id: line["id"], + oracle_rc: line["oracle_rc"], + mutation_rc: (line["mutation"] as { readonly rc: number }).rc, + oracle_output_sha256: line["oracle_output_sha256"] + }); + expect((await handler(request("/verdicts", "POST", mirrored))).status).toBe(200); + const posted = await handler( + request("/receipts", "POST", { + receipt: line, + verdict_hash: mirrored.hash, + receipt_path: `receipts/${String(line["id"])}/receipt.json` + }) + ); + expect(posted.status).toBe(200); + return mirrored.hash; +}; + +describe("the measured cells retained from an accepted receipt", () => { + it("reads exactly the banked half of a projection row", () => { + const read = retainedReceiptOf(receipt(), "receipts/U-A16/receipt.json", "a".repeat(64)); + expect(read.ok).toBe(true); + if (!read.ok) return; + expect(read.retained).toEqual({ + unit: "U-A16-FIXTURE-0", + receipt_path: "receipts/U-A16/receipt.json", + receipt_sha256: "a".repeat(64), + disposition: "CRASH", + seconds: 265, + tokens: 34206, + outcome_for_calibration: "fail", + prior_gap: -134694 + }); + }); + + it("names the cell it cannot do without rather than filling it with a zero", () => { + const withoutSeconds = { ...receipt() }; + delete withoutSeconds["seconds"]; + const read = retainedReceiptOf(withoutSeconds, "none", "a".repeat(64)); + expect(read.ok).toBe(false); + if (read.ok) return; + expect(read.why).toContain("seconds"); + }); + + it("carries an absent optional cell as null and never as a guess", () => { + const without = { ...receipt() }; + delete without["outcome_for_calibration"]; + delete without["prior_gap"]; + const read = retainedReceiptOf(without, "none", "a".repeat(64)); + expect(read.ok).toBe(true); + if (!read.ok) return; + expect(read.retained.outcome_for_calibration).toBeNull(); + expect(read.retained.prior_gap).toBeNull(); + }); +}); + +describe("the projection the object serves from its own mirror", () => { + it("yields a row only where the retained cells and the mirror agree on a unit", () => { + const cells = { + unit: "U-1", + receipt_path: "none", + receipt_sha256: "a".repeat(64), + disposition: "CRASH", + seconds: 1, + tokens: 2, + outcome_for_calibration: null, + prior_gap: null + }; + const evidence = { + id: "U-1", + oracle_rc: 0, + oracle_output_sha256: "sha256:x", + verdict_hash: "h" + }; + expect(mirrorProjectionRows([cells], [evidence])).toHaveLength(1); + expect(mirrorProjectionRows([cells], [])).toHaveLength(0); + expect(mirrorProjectionRows([], [evidence])).toHaveLength(0); + }); + + it("sorts by unit, so two objects holding the same receipts serve the same bytes", () => { + const cellsFor = (unit: string) => ({ + unit, + receipt_path: "none", + receipt_sha256: "a".repeat(64), + disposition: "CRASH", + seconds: 1, + tokens: 2, + outcome_for_calibration: null, + prior_gap: null + }); + const evidenceFor = (id: string) => ({ + id, + oracle_rc: 0, + oracle_output_sha256: "sha256:x", + verdict_hash: "h" + }); + const rows = mirrorProjectionRows( + [cellsFor("U-B"), cellsFor("U-A")], + [evidenceFor("U-A"), evidenceFor("U-B")] + ); + expect(rows.map((row) => row.unit)).toEqual(["U-A", "U-B"]); + }); + + it("serves an accepted receipt as one canonical line, twice identically", async () => { + const { handler } = await openHandler(); + await ingest(handler, receipt()); + + const first = await handler(request("/projection", "GET")); + expect(first.status).toBe(200); + const body = await first.text(); + expect(checkLineDiscipline(body)).toEqual([]); + expect(body.split("\n").filter((line) => line !== "")).toHaveLength(1); + expect(JSON.parse(body)).toMatchObject({ + unit: "U-A16-FIXTURE-0", + actuals: { seconds: 265, tokens: 34206, outcome: "CRASH" }, + result: { grade: "MEASURED", observed: 0 } + }); + + const second = await handler(request("/projection", "GET")); + expect(await second.text()).toBe(body); + }); + + it("never serves a cell the operator does not own", async () => { + const { handler } = await openHandler(); + // The fixture receipt carries `outcome_ruled: DISCARD`. It is Tom's field, + // and the projection has no name for it: §2.2e's `--write` fills only the + // operator's cells, and `status` and `outcome_ruled` never. + expect(receipt()["outcome_ruled"]).toBe("DISCARD"); + await ingest(handler, receipt()); + const body = await (await handler(request("/projection", "GET"))).text(); + for (const forbidden of NEVER_WRITTEN) { + expect(body).not.toContain(`"${forbidden}"`); + } + }); + + it("keeps the retained cells across a recreation over the same store", async () => { + const { handler, store } = await openHandler(); + await ingest(handler, receipt()); + const before = await (await handler(request("/projection", "GET"))).text(); + + const recreated = await run( + makeFactory(releaseState([], { plans: [], wip: [] })).pipe( + Effect.provideService(PlanningStore, store) + ) + ); + const second = makeFactoryHttpHandler({ + factory: recreated, + token: "test-token", + artifactHasher: lengthHasher, + store + }); + expect(await (await second(request("/projection", "GET"))).text()).toBe(before); + }); + + it("serves an empty body, not an error, when the host named no store", async () => { + const store = await run(makeInMemoryPlanningStore); + const factory = await run( + makeFactory(releaseState([], { plans: [] })).pipe( + Effect.provideService(PlanningStore, store) + ) + ); + const handler = makeFactoryHttpHandler({ + factory, + token: "test-token", + artifactHasher: lengthHasher + }); + const served = await handler(request("/projection", "GET")); + expect(served.status).toBe(200); + expect(await served.text()).toBe(""); + }); +}); diff --git a/pkgs/substrate-apps/src/packages/factory/tsconfig.json b/pkgs/substrate-apps/src/packages/factory/tsconfig.json new file mode 100644 index 00000000..553fbec5 --- /dev/null +++ b/pkgs/substrate-apps/src/packages/factory/tsconfig.json @@ -0,0 +1,9 @@ +{ + "extends": "../../tsconfig.base.json", + "compilerOptions": { + "lib": ["ES2022", "DOM"], + "types": [], + "allowImportingTsExtensions": true + }, + "include": ["src/**/*.ts"] +} diff --git a/pkgs/substrate-apps/src/packages/factory/vitest.config.ts b/pkgs/substrate-apps/src/packages/factory/vitest.config.ts new file mode 100644 index 00000000..21f4ddd0 --- /dev/null +++ b/pkgs/substrate-apps/src/packages/factory/vitest.config.ts @@ -0,0 +1,8 @@ +import { defineConfig } from "vitest/config" + +export default defineConfig({ + test: { + include: ["test/**/*.test.ts", "test-*/**/*.test.ts"], + environment: "node" + } +}) diff --git a/pkgs/substrate-apps/src/packages/interpreter/README.md b/pkgs/substrate-apps/src/packages/interpreter/README.md new file mode 100644 index 00000000..e12db485 --- /dev/null +++ b/pkgs/substrate-apps/src/packages/interpreter/README.md @@ -0,0 +1,66 @@ +# @substrate/interpreter + +Runs a Claude Code ultracode workflow script (`export const meta = ...` plus a body with top-level `await` and +`return`) against a backend, journals every call, and replays or resumes from that journal. It replaces tally-ts-sdk's +`packages/acceptor` and `tally-plan-arm.mjs` (ruling R44; see `DECISIONS.md` at the repo root). This page is the +written contract the acceptor kept in its README and `docs/conversion.md` (parity gap PT-15). + +## The realm contract + +A script runs in a fresh `node:vm` context created from `Object.create(null)` (`src/sandbox.ts`). It sees the +ECMAScript intrinsics of its own realm and the workflow builtins, nothing else. No `process`, `require`, `fetch`, +`setTimeout`, `performance`, `Buffer`, or any host object. Every builtin is defined inside the realm and talks to the +host through one captured bridge; values cross as JSON strings, and host errors are re-created as realm errors. + +Banned, because each would make a resumed run diverge from its journal (each throws +` is not available in a workflow script (it would break resume)`): + +| banned | since | +|---|---| +| `Date.now()` | D09 | +| `Date()` called as a function | D09 | +| argless `new Date()` | D09 | +| `Math.random()` | D09 | +| argless `Intl.DateTimeFormat().format()` and `.formatToParts()` | D17 (2026-09-24) | + +Pinned to UTC (D17): the default `timeZone` of `Intl.DateTimeFormat`, the `Date#toLocale*String` methods, the Date +local-time accessors (`getHours`, `setHours` and the rest, `getTimezoneOffset`, `toString`, `toDateString`, +`toTimeString`) and the multi-field `new Date(y, m, d, ...)` constructor. `new Date(value)`, `Date.UTC` and +`Date.parse` still work. The guards are non-writable and non-configurable; a script cannot undo them. + +Residue (known, not fixed): `Date.parse` of an ISO string without an offset (`"2026-09-23T00:00"`) is read in the host +zone, as ECMAScript specifies. Write the `Z`. + +This is isolation of globals and of the host object graph, not a security boundary: `vm` shares the process, the heap +and the event loop. Workflow scripts are the operator's own code. + +## Nothing is fabricated + +- A replay never reports `completed` after reaching a call its journal did not witness. It reports `diverged` + (`src/cli.ts`, exit 1). There is no suspension thrown through user code (D02). +- A failed call is journalled with its `error` and can be witnessed as failed (D08). +- A witnessed result is re-checked against the call's schema; a mismatch is a null, never a coerced value (D14). +- A throwing `parallel` thunk or `pipeline` stage becomes `null` for that item only, and is journalled as + `item_null` with its stage index (D12, D15). The run continues. + +## Caps + +| cap | value | where | +|---|---|---| +| nesting of `workflow()` | 1 level (depth 2 is refused) | `MAX_NESTING`, `src/interpreter.ts` | +| items per `parallel` / `pipeline` call | 4096 | `ITEMS_PER_CALL_CAP` | +| agent calls in one run's lifetime | 1000 | `LIFETIME_AGENT_CAP` | +| budget | a ceiling: a call that would pass it is refused at admission and journalled (D10) | `--budget` | + +## Resume identity + +A call's identity is either `chain` (the harness's chained key, the CLI default, for fidelity with the harness journal) +or `content` (`c1:#`, `src/key.ts`), selected with `--cache-identity`. The production paths (the +puller and the integrated gate) use `content`, so an early failure does not discard later cached work (D04, D07, D18). + +## CLI + +`bin/naive-run.mjs [--args ] [--backend mock|replay] [--journal ] +[--replay ] [--concurrency N] [--budget N] [--max-attempts N] [--cache-identity chain|content] +[--print-result]`. Exit codes: 0 completed; 1 failed or diverged; 2 usage; 3 crash; 4 the final recorded run was +killed. Run any test that spawns it under `~/.local/bin/runtime-test`. diff --git a/pkgs/substrate-apps/src/packages/interpreter/bin/naive-run.mjs b/pkgs/substrate-apps/src/packages/interpreter/bin/naive-run.mjs new file mode 100755 index 00000000..8666ab45 --- /dev/null +++ b/pkgs/substrate-apps/src/packages/interpreter/bin/naive-run.mjs @@ -0,0 +1,12 @@ +#!/usr/bin/env node +// Entry point: registers tsx's ESM loader, then runs src/cli.ts. +import { register } from "tsx/esm/api"; +import { fileURLToPath } from "node:url"; +import { dirname, join } from "node:path"; + +register(); +const here = dirname(fileURLToPath(import.meta.url)); +const { main } = await import(join(here, "..", "src", "cli.ts")); +const { code, out } = await main(process.argv.slice(2)); +(code === 2 ? process.stderr : process.stdout).write(out + "\n"); +process.exitCode = code; diff --git a/pkgs/substrate-apps/src/packages/interpreter/package.json b/pkgs/substrate-apps/src/packages/interpreter/package.json new file mode 100644 index 00000000..4b2d13be --- /dev/null +++ b/pkgs/substrate-apps/src/packages/interpreter/package.json @@ -0,0 +1,32 @@ +{ + "name": "@substrate/interpreter", + "version": "0.1.0", + "private": true, + "type": "module", + "exports": { + ".": "./src/index.ts", + "./meta": "./src/meta.ts" + }, + "description": "Naive clean-room interpreter for Claude Code workflow scripts: meta literal, agent/parallel/pipeline/phase/log/budget/workflow, chained resume journal, mock and replay backends. No live model calls.", + "bin": { + "naive-run": "./bin/naive-run.mjs" + }, + "scripts": { + "test": "vitest run", + "typecheck": "tsc -p tsconfig.json", + "naive-run": "node bin/naive-run.mjs" + }, + "dependencies": { + "ajv": "^8.17.1" + }, + "devDependencies": { + "@types/node": "^22.10.2", + "tsx": "^4.19.2", + "typescript": "^5.9.3", + "vitest": "^3.2.4", + "effect": "4.0.0-rc.112" + }, + "peerDependencies": { + "effect": "4.0.0-rc.112" + } +} diff --git a/pkgs/substrate-apps/src/packages/interpreter/src/backend.ts b/pkgs/substrate-apps/src/packages/interpreter/src/backend.ts new file mode 100644 index 00000000..1ceeb9a8 --- /dev/null +++ b/pkgs/substrate-apps/src/packages/interpreter/src/backend.ts @@ -0,0 +1,207 @@ +/** + * The Backend seam: what one agent() call becomes. + * + * The interpreter owns the script, the key chain, the journal, the caps and the + * retry loop. A backend owns exactly one attempt at one call and reports what + * happened. It never throws for an ordinary failure: it returns `error`. + */ +import { createHash } from "node:crypto"; + +export interface AgentOpts { + readonly label?: string; + readonly phase?: string; + readonly schema?: Record; + readonly model?: string; + readonly effort?: string; + readonly isolation?: string; + readonly agentType?: string; + readonly [k: string]: unknown; +} + +export interface AgentCall { + /** 1-based invocation index within the run (nested workflows share the counter). */ + readonly index: number; + /** The harness-compatible chained resume key. */ + readonly key: string; + readonly prompt: string; + readonly opts: AgentOpts; + /** The phase in force: opts.phase, else the last phase() title. */ + readonly phase: string | undefined; + /** 1-based attempt number; attempt > 1 is a retry after a schema mismatch or an error. */ + readonly attempt: number; + /** Validation errors from the previous attempt, for a backend that can feed them back. */ + readonly previousErrors?: readonly string[]; + readonly signal?: AbortSignal; + /** + * Which occurrence of the same content (prompt and keyed opts) this call is + * in the run, fixed at invocation: the same on every attempt and on every + * resume (successor review r3: a backend that counted its own occurrences + * gave a retry, or a resumed call, another call's identity). + */ + readonly occurrence?: number; + /** The call's content identity (key.ts contentId), fixed at invocation. */ + readonly cid?: string; + /** + * The run's budget ceiling at the moment a backend admits the call: a + * reason when the budget is spent, else undefined. A backend that queues + * calls (a WIP cap) asks it again when the call leaves the queue, and + * answers `budgetExhausted` instead of running (successor review r3: calls + * that passed the interpreter's check waited for a WIP slot and then ran + * after the budget was spent). + */ + readonly budgetExhausted?: () => string | undefined; +} + +export interface Usage { + readonly inputTokens: number; + readonly outputTokens: number; +} + +export interface AgentOutcome { + /** Unstructured result (no schema). */ + readonly text?: string; + /** Structured result (schema given); validated by the interpreter, not the backend. */ + readonly object?: unknown; + readonly usage?: Usage; + /** A terminal failure of this attempt. */ + readonly error?: string; + /** The backend's own id for the agent (session id, subagent id). */ + readonly agentId?: string; + /** A backend may declare the call skipped: the interpreter returns null without retrying. */ + readonly skipped?: boolean; + /** The backend refused to run the call because the run's budget was spent at admission. */ + readonly budgetExhausted?: boolean; +} + +export interface Backend { + readonly name: string; + run(call: AgentCall): Promise; +} + +// ---------------------------------------------------------------- MockBackend + +export interface MockOptions { + /** Length of every generated array (clamped to minItems/maxItems). Default 1. */ + readonly arrayLength?: number; + /** + * Simulated latency in wall-clock ms (setTimeout). When given, completion order + * follows the timers and is only as deterministic as they are. Default: none; + * see `turns`. + */ + readonly latency?: (call: AgentCall) => number; + /** + * Simulated latency in event-loop turns (setImmediate), used when `latency` is + * not given. Default: a function of the call ORDINAL, (index * 7) % 5, so calls + * finish out of invocation order (exercising pipelines) yet every run of the + * same script completes its calls in the same order: same keys, same results. + */ + readonly turns?: (call: AgentCall) => number; + /** Override the value for a call; return undefined to use the stub. */ + readonly respond?: (call: AgentCall) => AgentOutcome | undefined | Promise; + /** Boolean fields' stub value. Default true (so `filter(c => c.flag)` keeps items). */ + readonly booleanValue?: boolean; +} + +const digest = (s: string) => createHash("sha256").update(s).digest("hex"); + +/** + * A schema-satisfying stub. Deterministic: the same (schema, tag) gives the + * same value. Covers type (and type arrays), enum, const, properties, required, + * items/prefixItems, min/maxItems, anyOf/oneOf/allOf (first branch), minimum, + * minLength, and `$ref` into local `$defs`/`definitions`. + */ +export function stubFromSchema( + schema: unknown, + tag: string, + o: { arrayLength?: number; booleanValue?: boolean } = {}, + root: unknown = schema, + path = "$", + depth = 0, +): unknown { + if (depth > 32) return null; + if (schema === true || schema === undefined) return `${tag} ${path}`; + if (!schema || typeof schema !== "object") return null; + const s = schema as Record; + const rec = (sub: unknown, p: string) => stubFromSchema(sub, tag, o, root, p, depth + 1); + if (typeof s.$ref === "string") { + const m = /^#\/(\$defs|definitions)\/(.+)$/.exec(s.$ref); + const defs = m ? ((root as Record>)[m[1]!] ?? {}) : {}; + return rec(m ? defs[m[2]!] : undefined, path); + } + if ("const" in s) return s.const; + if (Array.isArray(s.enum) && s.enum.length) return s.enum[0]; + for (const k of ["anyOf", "oneOf"] as const) { + if (Array.isArray(s[k]) && (s[k] as unknown[]).length) { + const branches = s[k] as Record[]; + const nonNull = branches.find((b) => b && b.type !== "null") ?? branches[0]; + return rec(nonNull, path); + } + } + if (Array.isArray(s.allOf) && s.allOf.length) { + return rec(Object.assign({}, ...(s.allOf as object[])), path); + } + let t = s.type; + if (Array.isArray(t)) t = t.find((x) => x !== "null") ?? t[0]; + if (t === undefined && s.properties) t = "object"; + if (t === undefined && s.items) t = "array"; + switch (t) { + case "object": { + const out: Record = {}; + for (const [k, sub] of Object.entries((s.properties ?? {}) as Record)) out[k] = rec(sub, `${path}.${k}`); + return out; + } + case "array": { + const min = typeof s.minItems === "number" ? s.minItems : 0; + const max = typeof s.maxItems === "number" ? s.maxItems : Infinity; + const n = Math.min(max, Math.max(min, o.arrayLength ?? 1)); + const prefix = Array.isArray(s.prefixItems) ? (s.prefixItems as unknown[]) : []; + return Array.from({ length: n }, (_, i) => rec(prefix[i] ?? s.items, `${path}[${i}]`)); + } + case "boolean": + return o.booleanValue ?? true; + case "integer": + case "number": { + const min = typeof s.minimum === "number" ? s.minimum : typeof s.exclusiveMinimum === "number" ? s.exclusiveMinimum + 1 : 1; + const max = typeof s.maximum === "number" ? s.maximum : Infinity; + return Math.min(max, t === "integer" ? Math.ceil(min) : min); + } + case "null": + return null; + default: { + let v = `${tag} ${path}`; + if (typeof s.minLength === "number") while (v.length < s.minLength) v += "_"; + if (typeof s.maxLength === "number") v = v.slice(0, s.maxLength); + return v; + } + } +} + +/** + * Deterministic, offline. Every value carries a provenance tag `[[m: