From 8fa42a91099dd74afa90ee5a236b684fe39c8a93 Mon Sep 17 00:00:00 2001 From: mecattaf Date: Wed, 23 Sep 2026 23:23:28 +0200 Subject: [PATCH 1/9] substrate-apps: vendor apps/link and apps/pusher from agency-agency/substrate dc7cd1d, one pnpm package set pkgs/substrate-apps: the box-side programs of the Cloudflare Substrate as one package set (link, pusher; puller pending, absent upstream at this sha). Source taken with git archive dc7cd1d05b1e3938dace9d3d3cddc1a22d98d6cc (package.json, pnpm-lock.yaml, pnpm-workspace.yaml, tsconfig*.json, apps/link, apps/pusher), unchanged; SYNC.md records the sha and what is not vendored; sync.sh re-vendors from a sha and picks up apps/puller once it exists. link is bundled with esbuild exactly as pkgs/substrate-link (a021003) is; apps/link/src is identical between the two shas (MEASURED git diff: tests, tsconfig and vitest config only). pnpmDeps is fixed-output, fetcherVersion 3 (this nixpkgs pin removed 2), hash from a failing build (MEASURED). pusher needs no node_modules (its imports are node: builtins and its own src), so it is installed as files and its checkPhase runs one --dry-run --once tick against the vendored seat-capacity/1 fixture. Exposed as .#substrate-pusher and .#substrate-apps-link. pkgs/substrate (Agent Substrate, Go) is a different project and stays where it is, which is why this set is not at pkgs/substrate/. Co-Authored-By: Claude Fable 5.1 --- flake.nix | 9 + pkgs/substrate-apps/SYNC.md | 51 + pkgs/substrate-apps/default.nix | 122 + .../substrate-apps/src/apps/link/package.json | 21 + .../src/apps/link/proto/ax-p1.proto | 406 +++ pkgs/substrate-apps/src/apps/link/src/ax.ts | 121 + .../src/apps/link/src/config.ts | 103 + .../src/apps/link/src/contract.ts | 181 ++ .../substrate-apps/src/apps/link/src/floor.ts | 169 ++ pkgs/substrate-apps/src/apps/link/src/jobs.ts | 120 + .../src/apps/link/src/journal.ts | 187 ++ pkgs/substrate-apps/src/apps/link/src/link.ts | 921 ++++++ pkgs/substrate-apps/src/apps/link/src/main.ts | 105 + .../src/apps/link/test/ax-grpc.test.ts | 91 + .../src/apps/link/test/critique-fixed.test.ts | 406 +++ .../src/apps/link/test/fake-ax.ts | 84 + .../src/apps/link/test/fake-floor.ts | 304 ++ .../src/apps/link/test/final-pass.test.ts | 70 + .../src/apps/link/test/fixtures/ax-p1.proto | 405 +++ .../src/apps/link/test/link.test.ts | 357 +++ .../src/apps/link/test/review-r1.test.ts | 391 +++ .../src/apps/link/test/review-r2.test.ts | 419 +++ .../src/apps/link/test/review-r3.test.ts | 484 ++++ .../apps/link/test/review-r4-expired.test.ts | 86 + .../link/test/review-r4-fiber-defect.test.ts | 96 + .../src/apps/link/test/review-r4-harness.ts | 59 + .../test/review-r4-heartbeat-envelope.test.ts | 134 + .../link/test/review-r4-journal-key.test.ts | 48 + .../apps/link/test/review-r4-overtake.test.ts | 78 + .../apps/link/test/review-r4-p1stale.test.ts | 74 + .../test/review-r4-restart-replay.test.ts | 143 + .../link/test/review-r4-stale-lost.test.ts | 67 + .../link/test/workerd.integration.test.ts | 191 ++ .../src/apps/link/tsconfig.json | 16 + .../src/apps/link/vitest.config.ts | 8 + .../src/apps/pusher/bin/seat-pusher.mjs | 182 ++ .../src/apps/pusher/bin/substrate-pusher.mjs | 202 ++ .../src/apps/pusher/package.json | 28 + .../src/apps/pusher/pusher.example.json | 28 + .../apps/pusher/seat-capacity.example.json | 10 + .../pusher/seat-capacity.usage.example.json | 9 + .../src/apps/pusher/src/gentle.mjs | 145 + .../src/apps/pusher/src/refreshPolicy.mjs | 113 + .../src/apps/pusher/src/seatCapacity.mjs | 593 ++++ .../src/apps/pusher/src/seatsOracle.mjs | 201 ++ .../src/apps/pusher/src/token.mjs | 13 + .../src/apps/pusher/src/usageSource.mjs | 352 +++ .../apps/pusher/test/fixtures/seats-v1.json | 269 ++ .../src/apps/pusher/test/gentle.test.mjs | 257 ++ .../apps/pusher/test/refreshPolicy.test.mjs | 117 + .../apps/pusher/test/seatCapacity.test.mjs | 494 ++++ .../apps/pusher/test/unmappedLimits.test.mjs | 32 + .../src/apps/pusher/test/usageSource.test.mjs | 298 ++ .../src/apps/pusher/vitest.config.ts | 8 + pkgs/substrate-apps/src/package.json | 31 + pkgs/substrate-apps/src/pnpm-lock.yaml | 2551 +++++++++++++++++ pkgs/substrate-apps/src/pnpm-workspace.yaml | 12 + pkgs/substrate-apps/src/tsconfig.base.json | 20 + pkgs/substrate-apps/src/tsconfig.json | 16 + pkgs/substrate-apps/sync.sh | 23 + 60 files changed, 12531 insertions(+) create mode 100644 pkgs/substrate-apps/SYNC.md create mode 100644 pkgs/substrate-apps/default.nix create mode 100644 pkgs/substrate-apps/src/apps/link/package.json create mode 100644 pkgs/substrate-apps/src/apps/link/proto/ax-p1.proto create mode 100644 pkgs/substrate-apps/src/apps/link/src/ax.ts create mode 100644 pkgs/substrate-apps/src/apps/link/src/config.ts create mode 100644 pkgs/substrate-apps/src/apps/link/src/contract.ts create mode 100644 pkgs/substrate-apps/src/apps/link/src/floor.ts create mode 100644 pkgs/substrate-apps/src/apps/link/src/jobs.ts create mode 100644 pkgs/substrate-apps/src/apps/link/src/journal.ts create mode 100644 pkgs/substrate-apps/src/apps/link/src/link.ts create mode 100644 pkgs/substrate-apps/src/apps/link/src/main.ts create mode 100644 pkgs/substrate-apps/src/apps/link/test/ax-grpc.test.ts create mode 100644 pkgs/substrate-apps/src/apps/link/test/critique-fixed.test.ts create mode 100644 pkgs/substrate-apps/src/apps/link/test/fake-ax.ts create mode 100644 pkgs/substrate-apps/src/apps/link/test/fake-floor.ts create mode 100644 pkgs/substrate-apps/src/apps/link/test/final-pass.test.ts create mode 100644 pkgs/substrate-apps/src/apps/link/test/fixtures/ax-p1.proto create mode 100644 pkgs/substrate-apps/src/apps/link/test/link.test.ts create mode 100644 pkgs/substrate-apps/src/apps/link/test/review-r1.test.ts create mode 100644 pkgs/substrate-apps/src/apps/link/test/review-r2.test.ts create mode 100644 pkgs/substrate-apps/src/apps/link/test/review-r3.test.ts create mode 100644 pkgs/substrate-apps/src/apps/link/test/review-r4-expired.test.ts create mode 100644 pkgs/substrate-apps/src/apps/link/test/review-r4-fiber-defect.test.ts create mode 100644 pkgs/substrate-apps/src/apps/link/test/review-r4-harness.ts create mode 100644 pkgs/substrate-apps/src/apps/link/test/review-r4-heartbeat-envelope.test.ts create mode 100644 pkgs/substrate-apps/src/apps/link/test/review-r4-journal-key.test.ts create mode 100644 pkgs/substrate-apps/src/apps/link/test/review-r4-overtake.test.ts create mode 100644 pkgs/substrate-apps/src/apps/link/test/review-r4-p1stale.test.ts create mode 100644 pkgs/substrate-apps/src/apps/link/test/review-r4-restart-replay.test.ts create mode 100644 pkgs/substrate-apps/src/apps/link/test/review-r4-stale-lost.test.ts create mode 100644 pkgs/substrate-apps/src/apps/link/test/workerd.integration.test.ts create mode 100644 pkgs/substrate-apps/src/apps/link/tsconfig.json create mode 100644 pkgs/substrate-apps/src/apps/link/vitest.config.ts create mode 100644 pkgs/substrate-apps/src/apps/pusher/bin/seat-pusher.mjs create mode 100755 pkgs/substrate-apps/src/apps/pusher/bin/substrate-pusher.mjs create mode 100644 pkgs/substrate-apps/src/apps/pusher/package.json create mode 100644 pkgs/substrate-apps/src/apps/pusher/pusher.example.json create mode 100644 pkgs/substrate-apps/src/apps/pusher/seat-capacity.example.json create mode 100644 pkgs/substrate-apps/src/apps/pusher/seat-capacity.usage.example.json create mode 100644 pkgs/substrate-apps/src/apps/pusher/src/gentle.mjs create mode 100644 pkgs/substrate-apps/src/apps/pusher/src/refreshPolicy.mjs create mode 100644 pkgs/substrate-apps/src/apps/pusher/src/seatCapacity.mjs create mode 100644 pkgs/substrate-apps/src/apps/pusher/src/seatsOracle.mjs create mode 100644 pkgs/substrate-apps/src/apps/pusher/src/token.mjs create mode 100644 pkgs/substrate-apps/src/apps/pusher/src/usageSource.mjs create mode 100644 pkgs/substrate-apps/src/apps/pusher/test/fixtures/seats-v1.json create mode 100644 pkgs/substrate-apps/src/apps/pusher/test/gentle.test.mjs create mode 100644 pkgs/substrate-apps/src/apps/pusher/test/refreshPolicy.test.mjs create mode 100644 pkgs/substrate-apps/src/apps/pusher/test/seatCapacity.test.mjs create mode 100644 pkgs/substrate-apps/src/apps/pusher/test/unmappedLimits.test.mjs create mode 100644 pkgs/substrate-apps/src/apps/pusher/test/usageSource.test.mjs create mode 100644 pkgs/substrate-apps/src/apps/pusher/vitest.config.ts create mode 100644 pkgs/substrate-apps/src/package.json create mode 100644 pkgs/substrate-apps/src/pnpm-lock.yaml create mode 100644 pkgs/substrate-apps/src/pnpm-workspace.yaml create mode 100644 pkgs/substrate-apps/src/tsconfig.base.json create mode 100644 pkgs/substrate-apps/src/tsconfig.json create mode 100755 pkgs/substrate-apps/sync.sh diff --git a/flake.nix b/flake.nix index df96db44..ec00ec4c 100644 --- a/flake.nix +++ b/flake.nix @@ -659,6 +659,9 @@ let amdAi = inputs.nix-amd-ai.packages.${system}; strixAi = inputs.nix-strix-halo.packages.${system}; + # The Cloudflare Substrate's box-side programs, vendored from + # agency-agency/substrate at the sha in pkgs/substrate-apps/SYNC.md. + substrateApps = pkgs.callPackage ./pkgs/substrate-apps { }; in { inherit (pkgs) @@ -725,6 +728,12 @@ go_1_27 = inputs.nixpkgs-go.legacyPackages.${system}.go_1_27; }; }; + # `nix build .#substrate-pusher`, `.#substrate-apps-link`: the coordinator's + # gentle capacity pusher and the floor-to-ax link from the same pinned + # substrate sha (pkgs/substrate-apps). Installed by modules/substrate.nix + # (gates OFF) and hosts/nas/substrate-link.nix (still on pkgs/substrate-link). + substrate-pusher = substrateApps.pusher; + substrate-apps-link = substrateApps.link; }; # The same teardown the control and harness roles keep on their PATH diff --git a/pkgs/substrate-apps/SYNC.md b/pkgs/substrate-apps/SYNC.md new file mode 100644 index 00000000..158993e4 --- /dev/null +++ b/pkgs/substrate-apps/SYNC.md @@ -0,0 +1,51 @@ +# pkgs/substrate-apps/src: vendored source + +- Upstream: agency-agency/substrate (private; the repo formerly named ax-conwip, Tom 2026-09-23 E10), local + checkout `/home/tom/mecattaf/substrate`, branch `main`. +- Source sha: `dc7cd1d05b1e3938dace9d3d3cddc1a22d98d6cc` (synced 2026-09-23T21:40Z, apps/puller absent) + "capacity: floor capacity service, gentle seats pusher, floor as the gate's default (G4)". +- Taken with `git archive package.json pnpm-lock.yaml pnpm-workspace.yaml tsconfig.base.json tsconfig.json + apps/link apps/pusher`, unchanged (`./sync.sh /home/tom/mecattaf/substrate ` does exactly this). The root + `src/`, `test/`, `proto/`, `packages/*` and `apps/floor` are not vendored: the link bundle imports none of them, + the pusher's bin and src import only node: builtins and each other (MEASURED grep), and the floor is the + Cloudflare Worker, deployed from the substrate repo itself. +- Not vendored on purpose: the pusher's own vitest suite depends on `@substrate/planning` and `@substrate/factory` + (workspace packages); it runs upstream (1530 passed at this sha, REPORTED substrate/RECEIPT.md). The nix + checkPhase runs one dry tick against `apps/pusher/test/fixtures/seats-v1.json` instead. +- `apps/link` at this sha differs from the copy in `pkgs/substrate-link/src` (a021003) only in tests, + `tsconfig.json` and `vitest.config.ts` (MEASURED `git diff --stat a021003 dc7cd1d -- apps/link`: 7 files, no + `src/` change). The NAS module `hosts/nas/substrate-link.nix` keeps `pkgs/substrate-link` as its default until a + follow-up re-points it here; `.#substrate-apps-link` is the same program from the newer sha. + +## Programs and where they run + +| attr | upstream | host | dotfiles module | +|---|---|---|---| +| `link` | `apps/link` | NAS (system unit) | `hosts/nas/substrate-link.nix` (still on pkgs/substrate-link a021003) | +| `pusher` | `apps/pusher/bin/substrate-pusher.mjs` | coordinator (user unit) | `modules/substrate.nix` `services.substrate.pusher` | +| `puller` | `apps/puller` | coordinator (user unit) | `modules/substrate.nix` `services.substrate.puller` | + +## Pending: apps/puller + +At the pinned sha no branch of agency-agency/substrate has `apps/puller` (MEASURED `git ls-tree` over every ref, +2026-09-23 about 21:35Z; Lane B's Interfaces round was still open). The floor handoff names it: "the coordinator +puller (interpreter host). It leases runtime:interpreter, GETs /runs/:id/script, runs the interpreter with a floor +backend that POSTs /runs/:id/jobs and polls /runs/:id/events or /jobs/:id/output, then Completes the run." When it +lands: `./sync.sh ` picks it up, add a `puller` derivation here (bundle `apps/puller/src/main.ts` +the way `link` is bundled, or install its files the way `pusher` is), export it in the set, and drop the +`puller.package = null` default in `modules/substrate.nix`. + +## To resync + +1. `./sync.sh /home/tom/mecattaf/substrate ` (rewrites `src/`, the sha line above and `sourceSha`). +2. In `default.nix` set `pnpmDeps.hash = lib.fakeHash`, `nix build .#substrate-apps-link`, paste the `got:` value. +3. `nix build .#substrate-pusher .#substrate-apps-link --no-link` and `nix flake check --no-build`. +4. Commit with the sha in the message. + +## Unknowns and proposed defaults + +- Whether the vendored copy should become a flake input once the repo is public-shaped. Default: keep vendoring by + sha (E1: the code that wraps ax lives in dotfiles; a private input would also need a token on every build host). +- Whether `pkgs/substrate-link` (a021003) is retired in favour of this set's `link`. Default: yes, in the follow-up + that re-points `hosts/nas/substrate-link.nix`, after Lane A's 4-VM test is green on ax/fleet-zero; not in this lane + (hosts/ is Lane A's). diff --git a/pkgs/substrate-apps/default.nix b/pkgs/substrate-apps/default.nix new file mode 100644 index 00000000..fcd394bb --- /dev/null +++ b/pkgs/substrate-apps/default.nix @@ -0,0 +1,122 @@ +{ + lib, + stdenvNoCC, + nodejs_24, + pnpm_10, + esbuild, + makeWrapper, +}: +# substrate-apps: the box-side programs of the Cloudflare Substrate (agency-agency/substrate), +# vendored in ./src at the sha SYNC.md names and built here as one package set: +# +# link apps/link the NAS side: leases AgentJobs from the floor over /rpc and runs each as one +# ax Task. Bundled with esbuild exactly as pkgs/substrate-link does (that older +# copy, a021003, stays the NAS module's default until hosts/nas is re-pointed). +# pusher apps/pusher the coordinator's gentle capacity pusher: `seats --json --no-spend` under the +# refresh policy, converted to seat-capacity/2 and posted to the floor. +# puller apps/puller PENDING: not in the source tree at the pinned sha (SYNC.md). The module +# services.substrate.puller takes it as an option and refuses to arm without it. +# +# One pnpm workspace, one fixed-output dependency tree (pnpmDeps) shared by every program that needs +# node_modules. The pusher needs none: its bin and src import only node: builtins and each other +# (MEASURED grep, 2026-09-23), so it is installed as files and smoke-run in its checkPhase. +# +# Re-vendor: ./sync.sh , then refresh pnpmDeps.hash (lib.fakeHash, build, +# paste the got: value). E1: the code that wraps ax lives in dotfiles, pinned by sha, never a flake input. +let + version = "0.1.0-unstable-2026-09-23"; + src = ./src; + # The upstream commit ./src was taken from. Keep in step with SYNC.md. + sourceSha = "dc7cd1d05b1e3938dace9d3d3cddc1a22d98d6cc"; + + pnpmDeps = pnpm_10.fetchDeps { + pname = "substrate-apps"; + inherit version src; + # nix build .#substrate-apps-link with lib.fakeHash, then the "got:" value (MEASURED 2026-09-23). + hash = "sha256-/10ykNk5FGChNhjBxfP8lfrUJI4G5O4kE+phfewLODc="; + fetcherVersion = 3; + }; + + link = stdenvNoCC.mkDerivation { + pname = "substrate-link"; + inherit version src pnpmDeps; + nativeBuildInputs = [ + nodejs_24 + pnpm_10.configHook + esbuild + makeWrapper + ]; + buildPhase = '' + runHook preBuild + esbuild apps/link/src/main.ts --bundle --platform=node --format=esm --target=node22 \ + --outfile=dist/substrate-link.mjs \ + "--banner:js=import{createRequire as __cr}from'node:module';const require=__cr(import.meta.url);" + runHook postBuild + ''; + installPhase = '' + runHook preInstall + install -Dm444 dist/substrate-link.mjs $out/lib/substrate-link/substrate-link.mjs + install -Dm444 apps/link/proto/ax-p1.proto $out/share/substrate-link/ax-p1.proto + makeWrapper ${lib.getExe nodejs_24} $out/bin/substrate-link \ + --add-flags $out/lib/substrate-link/substrate-link.mjs \ + --set LINK_AX_PROTO_PATH $out/share/substrate-link/ax-p1.proto + runHook postInstall + ''; + passthru = { inherit sourceSha; }; + meta = { + description = "Outbound-only link from the Cloudflare Substrate floor to ax on the NAS"; + mainProgram = "substrate-link"; + platforms = lib.platforms.linux; + }; + }; + + pusher = stdenvNoCC.mkDerivation { + pname = "substrate-pusher"; + inherit version src; + nativeBuildInputs = [ + nodejs_24 + makeWrapper + ]; + dontBuild = true; + doCheck = true; + # One dry tick against the vendored seat-capacity/1 fixture: reads, converts, prints, posts nothing. + # HOME and every state path point into the build directory, so nothing outside the sandbox is touched. + checkPhase = '' + runHook preCheck + export HOME=$PWD/home + mkdir -p $HOME + node --check apps/pusher/bin/substrate-pusher.mjs + node apps/pusher/bin/substrate-pusher.mjs --dry-run --once \ + --seats-json apps/pusher/test/fixtures/seats-v1.json \ + --pidfile $PWD/pusher.pid --state $PWD/state.json --demand-dir $PWD/demand > tick.jsonl + grep -q '"event"' tick.jsonl + if grep -q '"read-failed"' tick.jsonl; then echo "the dry tick failed to read the fixture"; cat tick.jsonl; exit 1; fi + test ! -e $PWD/pusher.pid + runHook postCheck + ''; + installPhase = '' + runHook preInstall + mkdir -p $out/lib/substrate-pusher + cp -r apps/pusher/bin apps/pusher/src apps/pusher/package.json $out/lib/substrate-pusher/ + chmod -R a-w $out/lib/substrate-pusher + makeWrapper ${lib.getExe nodejs_24} $out/bin/substrate-pusher \ + --add-flags $out/lib/substrate-pusher/bin/substrate-pusher.mjs + runHook postInstall + ''; + passthru = { inherit sourceSha; }; + meta = { + description = "The gentle capacity pusher: seats --json to the Substrate floor's POST /capacity/snapshots"; + mainProgram = "substrate-pusher"; + platforms = lib.platforms.linux; + }; + }; +in +{ + inherit + link + pusher + pnpmDeps + sourceSha + ; + # puller: absent at ${sourceSha}; added by the next sync once apps/puller lands upstream. +} diff --git a/pkgs/substrate-apps/src/apps/link/package.json b/pkgs/substrate-apps/src/apps/link/package.json new file mode 100644 index 00000000..5e63fcc6 --- /dev/null +++ b/pkgs/substrate-apps/src/apps/link/package.json @@ -0,0 +1,21 @@ +{ + "name": "@substrate/link", + "private": true, + "type": "module", + "version": "0.1.0", + "description": "The outbound link on the hypervisor host: leases AgentJobs from the floor over /rpc (Lease, Heartbeat, Complete) and runs each as one ax Task (create-only, durable verdict outbox).", + "license": "UNLICENSED", + "exports": { ".": "./src/link.ts", "./*": "./src/*" }, + "scripts": { "check": "tsc --noEmit -p .", "test": "vitest run" }, + "dependencies": { + "@grpc/grpc-js": "^1.12.6", + "@grpc/proto-loader": "^0.7.13" + }, + "peerDependencies": { "effect": "4.0.0-rc.112" }, + "devDependencies": { + "@types/node": "^22.10.2", + "effect": "4.0.0-rc.112", + "typescript": "^5.9.3", + "vitest": "^3.2.4" + } +} diff --git a/pkgs/substrate-apps/src/apps/link/proto/ax-p1.proto b/pkgs/substrate-apps/src/apps/link/proto/ax-p1.proto new file mode 100644 index 00000000..134a0d24 --- /dev/null +++ b/pkgs/substrate-apps/src/apps/link/proto/ax-p1.proto @@ -0,0 +1,406 @@ +// RUNTIME PROTO of conwip-link (round 2): the vendored ax.proto (v0.3.0 d8ed0fe) plus the three hunks of carried patch +// P1 that the link reads: rpc GetTaskResult, messages GetTaskResultRequest and TaskResult, UsageStats.tool_calls. +// Copied from dotfiles branch ax/fleet-bringup pkgs/ax/patches/p1-completion.patch (lines 3240-3300) on 2026-09-23. +// A superset of the stock proto: against a stock server GetTaskResult reaches the server, which answers UNIMPLEMENTED, +// so the capability probe (B6) is decided by the server, never by the client's proto. Keep in sync with P1. +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +syntax = "proto3"; + +package ax.v1alpha1; + +import "google/protobuf/struct.proto"; +import "google/protobuf/timestamp.proto"; + +option go_package = "github.com/google/ax/pkg/apis/v1alpha1"; + +// AX defines the core control plane gRPC service for autonomous agent orchestration. +service AX { + // Manifests are parsed by the client (see `ax apply`) and submitted through the + // typed Update* RPCs below; the server never receives raw YAML. + + // Tasks + rpc GetTask(GetTaskRequest) returns (Task); + rpc ListTasks(ListTasksRequest) returns (ListTasksResponse); + rpc UpdateTask(UpdateTaskRequest) returns (Task); + rpc DeleteTask(DeleteTaskRequest) returns (DeleteTaskResponse); + rpc SuspendTask(SuspendTaskRequest) returns (Task); + rpc ResumeTask(ResumeTaskRequest) returns (Task); + rpc WatchTask(WatchTaskRequest) returns (stream WatchTaskResponse); + // GetTaskResult returns the result file the task's command wrote, as copied + // by the controller when the command exited. NotFound until then. + rpc GetTaskResult(GetTaskResultRequest) returns (TaskResult); + + // Gateways + rpc GetGateway(GetGatewayRequest) returns (Gateway); + rpc ListGateways(ListGatewaysRequest) returns (ListGatewaysResponse); + rpc UpdateGateway(UpdateGatewayRequest) returns (Gateway); + rpc DeleteGateway(DeleteGatewayRequest) returns (DeleteGatewayResponse); + + // Workspaces + rpc GetWorkspace(GetWorkspaceRequest) returns (Workspace); + rpc ListWorkspaces(ListWorkspacesRequest) returns (ListWorkspacesResponse); + rpc UpdateWorkspace(UpdateWorkspaceRequest) returns (Workspace); + rpc DeleteWorkspace(DeleteWorkspaceRequest) returns (DeleteWorkspaceResponse); + + // Models + rpc GetModel(GetModelRequest) returns (Model); + rpc ListModels(ListModelsRequest) returns (ListModelsResponse); + rpc UpdateModel(UpdateModelRequest) returns (Model); + rpc DeleteModel(DeleteModelRequest) returns (DeleteModelResponse); + +} + +// ObjectMeta is metadata for AX resources. +message ObjectMeta { + string name = 1; + string atespace = 2; + google.protobuf.Timestamp creation_timestamp = 3; +} + +// --- Task --- + +message Task { + string api_version = 1; + string kind = 2; + ObjectMeta metadata = 3; + TaskSpec spec = 4; + TaskStatus status = 5; +} + +message TaskSpec { + // Field 1 was `goal`, removed; the workspace goal lives on WorkspaceRef. + reserved 1; + reserved "goal"; + bool suspend = 2; + string image = 3; + repeated string command = 4; + repeated EnvVar env = 5; + ResourceReqs resources = 6; + // workspaces binds one or more Workspaces, each mounted at its own path + // under /workspace. The first entry is the task command's working directory. + repeated WorkspaceRef workspaces = 7; + GatewayRef gateway = 8; + // Field 9 was `policies` (budget and approval config), removed for now. + reserved 9; + reserved "policies"; + // debug enables the in-container guest services (process execution and file + // access) that back `ax ssh`. Off by default. + bool debug = 10; +} + +message EnvVar { + string name = 1; + string value = 2; +} + +message ResourceReqs { + ResourceList requests = 1; + ResourceList limits = 2; +} + +message ResourceList { + string cpu = 1; + string memory = 2; +} + +message WorkspaceRef { + string name = 1; + string path = 2; + string goal = 3; +} + +message GatewayRef { + string name = 1; +} + +message TaskStatus { + string phase = 1; + string id = 2; + string actor = 3; + // json_name keeps the established `workerIP` spelling in YAML and JSON. + string worker_ip = 4 [json_name = "workerIP"]; + PendingApproval pending_approval = 5; + UsageStats usage = 6; + repeated Condition conditions = 7; +} + +message PendingApproval { + string id = 1; + string action = 2; + google.protobuf.Timestamp requested_at = 3; +} + +message UsageStats { + int32 prompt_tokens = 1; + int32 completion_tokens = 2; + int32 tool_calls = 3; +} + +message Condition { + string type = 1; + string status = 2; + google.protobuf.Timestamp last_transition_time = 3; + string reason = 4; + string message = 5; +} + +// --- Gateway --- + +message Gateway { + string api_version = 1; + string kind = 2; + ObjectMeta metadata = 3; + GatewaySpec spec = 4; +} + +message GatewaySpec { + repeated Listener listeners = 1; + EgressConfig egress = 2; +} + +message Listener { + string name = 1; + int32 port = 2; + string protocol = 3; +} + +message EgressConfig { + EgressAllowlist allowlist = 1; +} + +message EgressAllowlist { + repeated HostRule hosts = 1; +} + +message HostRule { + string host = 1; + int32 port = 2; +} + +// --- Workspace --- + +message Workspace { + string api_version = 1; + string kind = 2; + ObjectMeta metadata = 3; + WorkspaceSpec spec = 4; +} + +message WorkspaceSpec { + repeated GitRepo git = 1; + MCPConfig mcp = 2; + SkillsConfig skills = 3; +} + +message GitRepo { + string name = 1; + string repo = 2; + string branch = 3; + string dir = 4; + int32 depth = 5; +} + +message MCPConfig { + repeated MCPRegistry registries = 1; + repeated MCPServer servers = 2; +} + +message MCPRegistry { + string provider = 1; + string project = 2; + string query = 3; + repeated MCPServer servers = 4; +} + +message MCPServer { + string name = 1; + string endpoint = 2; + string command = 3; + repeated string args = 4; +} + +message SkillsConfig { + repeated SkillRegistry registries = 1; + string path = 2; +} + +message SkillRegistry { + string provider = 1; + string project = 2; + string query = 3; +} + +// --- Model --- + +message Model { + string api_version = 1; + string kind = 2; + ObjectMeta metadata = 3; + ModelSpec spec = 4; +} + +message ModelSpec { + string provider = 1; + string model = 2; + // Fields 3-5 were typed temperature, max_tokens, and system_instruction; + // provider settings now live in the free-form `parameters` map. + reserved 3, 4, 5; + reserved "temperature", "max_tokens", "system_instruction"; + SecretKeyRef secret_key = 6; + // parameters are provider-specific generation settings passed through to the + // model API as-is, for example temperature or maxOutputTokens for Gemini. + google.protobuf.Struct parameters = 7; +} + +message SecretKeyRef { + string name = 1; + string key = 2; +} + +// --- RPC Request & Response Messages --- + +// Tasks +message GetTaskRequest { + string atespace = 1; + string name = 2; +} + +message GetTaskResultRequest { + string atespace = 1; + string name = 2; +} + +message TaskResult { + bytes content = 1; + string sha256 = 2; +} + +message ListTasksRequest { + string atespace = 1; + int64 limit = 2; + int64 offset = 3; +} + +message ListTasksResponse { + repeated Task tasks = 1; +} + +message UpdateTaskRequest { + Task task = 1; +} + +message DeleteTaskRequest { + string atespace = 1; + string name = 2; +} + +message DeleteTaskResponse {} + +message SuspendTaskRequest { + string atespace = 1; + string name = 2; +} + +message ResumeTaskRequest { + string atespace = 1; + string name = 2; +} + +message WatchTaskRequest { + string atespace = 1; + string name = 2; +} + +message WatchTaskResponse { + Task task = 1; + string action = 2; +} + +// Gateways +message GetGatewayRequest { + string atespace = 1; + string name = 2; +} + +message ListGatewaysRequest { + string atespace = 1; +} + +message ListGatewaysResponse { + repeated Gateway gateways = 1; +} + +message UpdateGatewayRequest { + Gateway gateway = 1; +} + +message DeleteGatewayRequest { + string atespace = 1; + string name = 2; +} + +message DeleteGatewayResponse {} + +// Workspaces +message GetWorkspaceRequest { + string atespace = 1; + string name = 2; +} + +message ListWorkspacesRequest { + string atespace = 1; +} + +message ListWorkspacesResponse { + repeated Workspace workspaces = 1; +} + +message UpdateWorkspaceRequest { + Workspace workspace = 1; +} + +message DeleteWorkspaceRequest { + string atespace = 1; + string name = 2; +} + +message DeleteWorkspaceResponse {} + +// Models +message GetModelRequest { + string atespace = 1; + string name = 2; +} + +message ListModelsRequest { + string atespace = 1; +} + +message ListModelsResponse { + repeated Model models = 1; +} + +message UpdateModelRequest { + Model model = 1; +} + +message DeleteModelRequest { + string atespace = 1; + string name = 2; +} + +message DeleteModelResponse {} + diff --git a/pkgs/substrate-apps/src/apps/link/src/ax.ts b/pkgs/substrate-apps/src/apps/link/src/ax.ts new file mode 100644 index 00000000..cdff0d5d --- /dev/null +++ b/pkgs/substrate-apps/src/apps/link/src/ax.ts @@ -0,0 +1,121 @@ +// The link's view of ax: five calls over plain gRPC (h2c), no CreateTask (ax has none; UpdateTask is an upsert, so +// the link only ever calls it after GetTask answered NotFound: decision L4, create-only). +// ListTasks is paged: ax-server answers limit 0 with 50 rows, newest first (MEASURED upstream +// internal/server/server.go:91-104), so `listAll` walks pages until a short one (B1, critique C1). +import { createHash } from "node:crypto" +import { isIPv4, isIPv6 } from "node:net" +import { dirname, resolve } from "node:path" +import { fileURLToPath } from "node:url" +import * as grpc from "@grpc/grpc-js" +import * as protoLoader from "@grpc/proto-loader" +import { Effect } from "effect" +import type { AxTask } from "./contract.ts" + +export class AxError { + readonly _tag = "AxError" + constructor(readonly code: number, readonly message: string) {} + get unavailable() { return this.code === grpc.status.UNAVAILABLE || this.code === grpc.status.DEADLINE_EXCEEDED || this.code === grpc.status.INTERNAL } + get resourceExhausted() { return this.code === grpc.status.RESOURCE_EXHAUSTED } +} +export const AX_PAGE = 50 // ax-server's own default page size + +export interface Condition { readonly type?: string; readonly status?: string; readonly reason?: string; readonly message?: string } +export interface AxObserved { + readonly name: string + readonly phase: string // "" and Pending before the controller acts; Running; Completed/Failed (P1); Terminating + readonly spec: { image?: string; command?: ReadonlyArray; env?: ReadonlyArray<{ name?: string; value?: string }>; gateway?: { name?: string } | null } + readonly conditions: ReadonlyArray + readonly usage?: { readonly promptTokens?: number; readonly completionTokens?: number; readonly toolCalls?: number } | null +} +/** `digestOk` is false when the server's sha256 does not match the bytes received, undefined when it sent none (B4). */ +export interface AxResult { readonly content: string; readonly sha256: string; readonly digestOk?: boolean } +/** A Gateway's egress, as ax applies it (MEASURED upstream internal/controller/reconciler.go:188-198 and + * internal/substrate/client.go:450-462): no egress or no allowlist means `*:443`, an empty host list applies no + * policy, and a `*` or `0.0.0.0/0` host allows everything. */ +export interface AxGateway { readonly hosts: ReadonlyArray<{ readonly host: string; readonly port: number }>; readonly hasAllowlist: boolean } +/** Round 2: ax sends every host containing "/" to Substrate as a CIDR rule (MEASURED upstream client.go:456-491), so an + * allowlist can be open by arithmetic (`::/0`, `0.0.0.0/1` + `128.0.0.0/1`). Fail closed: a CIDR that does not parse, + * or one at or wider than /8 (IPv4) or /16 (IPv6), counts as open. */ +export const cidrIsOpen = (host: string): boolean => { + const [addr = "", len, ...rest] = host.split("/") + if (rest.length > 0 || len === undefined || !/^\d{1,3}$/.test(len)) return true + const n = Number(len) + if (isIPv4(addr)) return n > 32 || n <= 8 + if (isIPv6(addr)) return n > 128 || n <= 16 + return true +} +export const gatewayAllowsAll = (g: AxGateway) => + !g.hasAllowlist || g.hosts.length === 0 || g.hosts.some((x) => { + const h = x.host.trim().toLowerCase() + return h === "*" || h === "" || (h.includes("/") && cidrIsOpen(h)) + }) + +/** Round 2: the link's own runtime proto, P1 included (see its header). `LINK_AX_PROTO_PATH` overrides it. */ +export const LINK_PROTO_PATH = resolve(dirname(fileURLToPath(import.meta.url)), "..", "proto", "ax-p1.proto") + +export interface AxApi { + readonly getTask: (name: string) => Effect.Effect // NotFound -> undefined + readonly createTask: (task: AxTask) => Effect.Effect // UpdateTask, called only after NotFound + /** One page of the executor's list (Buildkite informer); `limit` 0 means the server's 50. */ + readonly listTasks: (limit: number, offset: number) => Effect.Effect, AxError> + readonly getGateway: (name: string) => Effect.Effect // NotFound -> undefined + readonly deleteTask: (name: string) => Effect.Effect // NotFound counts as done + /** P1's GetTaskResult. `unimplemented` means the server predates P1 (stock v0.3.0): completion is the guest's (L7). */ + readonly getTaskResult: (name: string) => Effect.Effect +} + +/** Every page, until a short one. Rows can shift between pages while ax inserts or deletes; the link therefore + * confirms any "absent" Task with GetTask before acting on it (B1). */ +export const listAll = (ax: AxApi, page = AX_PAGE, maxPages = 400): Effect.Effect, AxError> => Effect.gen(function*() { + const out = new Map() + for (let i = 0; i < maxPages; i++) { + const rows = yield* ax.listTasks(page, i * page) + for (const t of rows) out.set(t.name, t) + if (rows.length < page) return [...out.values()] + } + return yield* Effect.fail(new AxError(grpc.status.OUT_OF_RANGE, `more than ${maxPages * page} Tasks in the atespace`)) +}) + +const observe = (t: any): AxObserved => ({ + name: t?.metadata?.name ?? "", + phase: t?.status?.phase ?? "", + spec: t?.spec ?? {}, + conditions: t?.status?.conditions ?? [], + usage: t?.status?.usage ?? null +}) + +/** The production AxApi: `address` is ax-server's `host:port` (the ClusterIP 10.201.0.80:8080 on the NAS). */ +export function grpcAx(address: string, atespace: string, deadlineMs = 10_000, protoPath: string = LINK_PROTO_PATH): AxApi & { close: () => void } { + const def = protoLoader.loadSync(protoPath, { keepCase: false, longs: String, enums: String, defaults: true, oneofs: true }) + const Ctor = (grpc.loadPackageDefinition(def) as any).ax?.v1alpha1?.AX + if (typeof Ctor !== "function") throw new Error(`ax.v1alpha1.AX not found in ${protoPath}`) + // Round 2: a proto without GetTaskResult would answer the P1 probe "unimplemented" locally, never asking the server + if (typeof Ctor.prototype?.GetTaskResult !== "function") throw new Error(`${protoPath} declares no GetTaskResult; the link needs its P1 proto`) + const client = new Ctor(address, grpc.credentials.createInsecure()) + const unary = (method: string, req: unknown) => Effect.callback((resume) => { + if (typeof client[method] !== "function") return resume(Effect.fail(new AxError(grpc.status.UNIMPLEMENTED, `${method} not in proto`))) + client[method](req, { deadline: new Date(Date.now() + deadlineMs) }, (err: grpc.ServiceError | null, res: T) => + resume(err ? Effect.fail(new AxError(err.code ?? grpc.status.UNKNOWN, err.details ?? err.message)) : Effect.succeed(res))) + }) + const notFound = (e: Effect.Effect, dflt: A) => + e.pipe(Effect.catchIf((x: AxError) => x.code === grpc.status.NOT_FOUND, () => Effect.succeed(dflt))) + return { + getTask: (name) => notFound(unary("GetTask", { atespace, name }).pipe(Effect.map(observe)), undefined), + createTask: (task) => unary("UpdateTask", { task }).pipe(Effect.asVoid), + listTasks: (limit, offset) => unary("ListTasks", { atespace, limit, offset }).pipe(Effect.map((r) => (r?.tasks ?? []).map(observe))), + getGateway: (name) => notFound(unary("GetGateway", { atespace, name }).pipe(Effect.map((g): AxGateway => { + const al = g?.spec?.egress?.allowlist + return { hasAllowlist: al != null, hosts: (al?.hosts ?? []).map((h: any) => ({ host: String(h?.host ?? ""), port: Number(h?.port ?? 0) })) } + })), undefined), + deleteTask: (name) => notFound(unary("DeleteTask", { atespace, name }).pipe(Effect.asVoid), undefined), + getTaskResult: (name) => notFound(unary("GetTaskResult", { atespace, name }).pipe( + Effect.map((r): AxResult => { + const bytes = Buffer.from(r?.content ?? "") + const sha256 = String(r?.sha256 ?? "") + return { content: bytes.toString("utf8"), sha256, ...(sha256 ? { digestOk: createHash("sha256").update(bytes).digest("hex") === sha256.toLowerCase() } : {}) } + }), + Effect.catchIf((x: AxError) => x.code === grpc.status.UNIMPLEMENTED, () => Effect.succeed("unimplemented" as const)) + ), undefined), + close: () => client.close() + } +} diff --git a/pkgs/substrate-apps/src/apps/link/src/config.ts b/pkgs/substrate-apps/src/apps/link/src/config.ts new file mode 100644 index 00000000..24198b65 --- /dev/null +++ b/pkgs/substrate-apps/src/apps/link/src/config.ts @@ -0,0 +1,103 @@ +// conwip-link configuration from the environment (round 1, finding "config"). Every numeric key must be a plain +// non-negative integer inside its range and every enum key one of its literals; anything else is ConfigInvalid, which +// main.ts turns into exit 78 so systemd's RestartPreventExitStatus holds the unit down instead of letting it spin +// (Effect.sleep(NaN) is a zero sleep) or crash-loop. Errors name the key and the rule, never the value. + +export class ConfigInvalid extends Error { + readonly _tag = "ConfigInvalid" + constructor(readonly key: string, readonly why: string) { super(`${key}: ${why}`) } +} + +type Env = Readonly> + +const str = (env: Env, key: string, d?: string): string => { + const v = env[key] + if (v !== undefined && v !== "") return v + if (d !== undefined) return d + throw new ConfigInvalid(key, "missing") +} + +export const int = (env: Env, key: string, d: number, min: number, max = 1_000_000): number => { + const v = str(env, key, String(d)).trim() + if (!/^\d+$/.test(v)) throw new ConfigInvalid(key, "not a non-negative integer") + const n = Number(v) + if (n < min || n > max) throw new ConfigInvalid(key, `outside ${min}..${max}`) + return n +} + +export const literal = (env: Env, key: string, d: L, allowed: ReadonlyArray): L => { + const v = str(env, key, d) + if (!(allowed as ReadonlyArray).includes(v)) throw new ConfigInvalid(key, `not one of ${allowed.join(", ")}`) + return v as L +} + +export const seatCommands = (env: Env, key: string, d: string): Record> => { + let parsed: unknown + try { parsed = JSON.parse(str(env, key, d)) } catch { throw new ConfigInvalid(key, "not JSON") } + if (parsed === null || typeof parsed !== "object" || Array.isArray(parsed)) throw new ConfigInvalid(key, "not a JSON object") + const out: Record> = {} + for (const [seat, cmd] of Object.entries(parsed)) { + if (!Array.isArray(cmd) || cmd.length === 0 || !cmd.every((c) => typeof c === "string" && c !== "")) + throw new ConfigInvalid(key, `seat ${JSON.stringify(seat)} is not a non-empty array of non-empty strings`) + out[seat] = cmd as Array + } + return out +} + +export interface LinkEnv { + readonly holder: string + readonly maxInFlight: number + readonly servedLabels: Array + readonly atespace: string + readonly image: string + readonly gateway: string + readonly axServer: string + readonly guestCompleteUrl: string | undefined + readonly seatCommands: Record> + readonly completion: "auto" | "p1" | "guest" + readonly resyncSeconds: number + readonly pendingTimeoutSeconds: number + readonly deleteAfterSeconds: number + readonly deadlineBackstopSeconds: number + readonly createAttempts: number + readonly outboxBackoffSeconds: readonly [number, number] + readonly fenceTimeoutSeconds: number + readonly resultReadTries: number + readonly verdictAttempts: number // round 2 + readonly maxOutputBytes: number // round 2: below the floor's 2 MB DO row limit + readonly internalHosts: Array // round 2: single-label names a guest Complete URL may use + readonly axProtoPath: string | undefined // round 2: the link's ax proto (must declare GetTaskResult) +} + +/** Throws ConfigInvalid on the first bad key. Defaults are the NixOS module's. */ +export const readLinkEnv = (env: Env): LinkEnv => { + const outMin = int(env, "LINK_OUTBOX_BACKOFF_MIN_SECONDS", 60, 1, 86_400) + const outMax = int(env, "LINK_OUTBOX_BACKOFF_MAX_SECONDS", 900, 1, 86_400) + if (outMax < outMin) throw new ConfigInvalid("LINK_OUTBOX_BACKOFF_MAX_SECONDS", "below LINK_OUTBOX_BACKOFF_MIN_SECONDS") + const servedLabels = str(env, "LINK_SERVED_LABELS", "seat:halogen,runtime:gvisor").split(",").map((l) => l.trim()).filter((l) => l !== "") + if (servedLabels.length === 0) throw new ConfigInvalid("LINK_SERVED_LABELS", "empty") + return { + holder: str(env, "LINK_HOLDER", "nas-link-1"), + maxInFlight: int(env, "LINK_MAX_IN_FLIGHT", 2, 1, 1000), + servedLabels, + atespace: str(env, "AX_ATESPACE", "fleet"), + image: str(env, "LINK_IMAGE", "ax-agent"), + gateway: str(env, "LINK_GATEWAY", "halogen"), + axServer: str(env, "AX_SERVER", "10.201.0.80:8080"), + guestCompleteUrl: env.LINK_GUEST_COMPLETE_URL ? env.LINK_GUEST_COMPLETE_URL : undefined, + seatCommands: seatCommands(env, "LINK_SEAT_COMMANDS", '{"halogen":["ax-agent","pi"]}'), + completion: literal(env, "LINK_COMPLETION", "auto", ["auto", "p1", "guest"]), + resyncSeconds: int(env, "LINK_RESYNC_SECONDS", 15, 1, 3600), + pendingTimeoutSeconds: int(env, "LINK_PENDING_TIMEOUT_SECONDS", 900, 1, 86_400), + deleteAfterSeconds: int(env, "LINK_DELETE_AFTER_SECONDS", 600, 0, 604_800), + deadlineBackstopSeconds: int(env, "LINK_DEADLINE_BACKSTOP_SECONDS", 300, 0, 86_400), + createAttempts: int(env, "LINK_CREATE_ATTEMPTS", 7, 1, 100), + outboxBackoffSeconds: [outMin, outMax], + fenceTimeoutSeconds: int(env, "LINK_FENCE_TIMEOUT_SECONDS", 120, 1, 86_400), + resultReadTries: int(env, "LINK_RESULT_READ_TRIES", 5, 1, 1000), + verdictAttempts: int(env, "LINK_VERDICT_ATTEMPTS", 8, 1, 1000), + maxOutputBytes: int(env, "LINK_MAX_OUTPUT_BYTES", 1_000_000, 1024, 2_000_000), + internalHosts: (env.LINK_INTERNAL_HOSTS ?? "").split(",").map((h) => h.trim()).filter((h) => h !== ""), + axProtoPath: env.LINK_AX_PROTO_PATH ? env.LINK_AX_PROTO_PATH : undefined + } +} diff --git a/pkgs/substrate-apps/src/apps/link/src/contract.ts b/pkgs/substrate-apps/src/apps/link/src/contract.ts new file mode 100644 index 00000000..40e93d8b --- /dev/null +++ b/pkgs/substrate-apps/src/apps/link/src/contract.ts @@ -0,0 +1,181 @@ +// The floor <-> link wire contract (LINK-DESIGN.md section 3). Starting point: the option-A prototype +// (/home/tom/today/evals-2026-09-23/arc/proto/contract.ts, decision L1). Every L1 name is kept as it was. +// Five keys are ADDED, each optional so an L1-only peer ignores them, each named after its source: +// Grant.supersedes L3 (two-stage expiry): the old attempt the link must DeleteTask first +// Grant.leaseToken L7 (completion before ax P1): a per-lease token for the guest (Buildkite job token) +// Grant.deadline Temporal start-to-close, as an absolute epoch ms the link can hold across a restart +// Lease.endpoint Buildkite ping `endpoint`: move the link to a new floor URL after a test call +// Complete.withdrew round 2: the leaseId of attempt n+1 that rule 4b withdrew when it accepted attempt n's verdict. +// The link releases n+1 only when it is named here; it never infers a withdrawal (a duplicate +// answer or a delivered retryable failure withdrew nothing) +// Complete.withdrewTransitions round 3: the leaseTransitions of the generation rule 4b withdrew. A floor that +// requeues the withdrawn attempt under the same leaseId grants it again with a higher value; the +// link treats a grant as withdrawn only when its generation matches +// Nothing here is GitHub-hosted at runtime. +import { Schema } from "effect" +import { Rpc, RpcGroup } from "effect/unstable/rpc" + +// Actions job result values (GitHub/Gitea job `result`). +export const Result = Schema.Literals(["success", "failure", "cancelled", "skipped"]) +export type Result = typeof Result.Type + +// ---- AgentJob: an ultracode agent() node as an Actions-style job in a Kubernetes object envelope (unchanged) ---- +const Labels = Schema.Record(Schema.String, Schema.String) +export const PromptRef = Schema.Struct({ + sha256: Schema.String, // content address; the guest refuses on mismatch (FIELD-MAP 5a AX_CONWIP_PROMPT_SHA256) + bytes: Schema.Int, + uri: Schema.String // journal:////prompt.md +}) +export const AgentJob = Schema.Struct({ + apiVersion: Schema.Literal("ultracode.mecattaf.dev/v1alpha1"), + kind: Schema.Literal("AgentJob"), + metadata: Schema.Struct({ name: Schema.String, labels: Schema.optionalKey(Labels), annotations: Schema.optionalKey(Labels) }), + spec: Schema.Struct({ + // CONWIP admission key, e.g. ["seat:halogen","runtime:gvisor"]. B7 asked for Schema.NonEmptyArray here; it stays + // Array on the wire because one malformed job would then fail the decode of a whole Lease reply and strand every + // grant in it (INFERRED). The link refuses it per job instead (`validRunsOn`, jobs.ts), and the floor at enqueue. + "runs-on": Schema.Array(Schema.String), + "timeout-minutes": Schema.optionalKey(Schema.Int), // Temporal start-to-close, enforced by the floor + "continue-on-error": Schema.optionalKey(Schema.Boolean), + environment: Schema.optionalKey(Schema.NullOr(Schema.String)), // Tom's ack gate + with: Schema.Struct({ + prompt: Schema.optionalKey(Schema.String), + prompt_ref: PromptRef, + model: Schema.String, + effort: Schema.optionalKey(Schema.String), + schema: Schema.optionalKey(Schema.Unknown), + isolation: Schema.optionalKey(Schema.String), + "agent-type": Schema.optionalKey(Schema.String) + }), + needs: Schema.optionalKey(Schema.Record(Schema.String, Schema.Struct({ result: Result, outputs_ref: Schema.String }))), + outputs: Schema.optionalKey(Schema.Array(Schema.String)) + }) +}) +export type AgentJob = typeof AgentJob.Type + +// ---- ax Task JSON (protojson camelCase of ax.proto v0.3.0 d8ed0fe). The link emits it; it never rides the wire ---- +// apiVersion is ax's own constant `ax.io/v1alpha1` (ax pkg/apis/v1alpha1/types.go:31); the prototype's +// `ax/v1alpha1` passed only because ax stores what it is given. +export const AxTask = Schema.Struct({ + apiVersion: Schema.Literal("ax.io/v1alpha1"), + kind: Schema.Literal("Task"), + metadata: Schema.Struct({ name: Schema.String, atespace: Schema.String }), + spec: Schema.Struct({ + image: Schema.String, + command: Schema.Array(Schema.String), + env: Schema.Array(Schema.Struct({ name: Schema.String, value: Schema.String })), + gateway: Schema.optionalKey(Schema.Struct({ name: Schema.String })) + }) +}) +export type AxTask = typeof AxTask.Type + +// ---- the link contract ---- +// Kubernetes coordination.k8s.io/v1 LeaseSpec field names. +export const LeaseSpec = Schema.Struct({ + holderIdentity: Schema.String, leaseDurationSeconds: Schema.Int, + acquireTime: Schema.Number, renewTime: Schema.Number, leaseTransitions: Schema.Int +}) +// Absolute counts on every response plus an increasing seq (ARC scaleset statistics). +export const Stats = Schema.Struct({ seq: Schema.Int, cap: Schema.Int, wip: Schema.Int, queued: Schema.Int, done: Schema.Int }) +export const Grant = Schema.Struct({ + leaseId: Schema.String, // `-a`; it is also the ax Task name (the idempotency key on ax) + attempt: Schema.Int, + job: AgentJob, + lease: LeaseSpec, + supersedes: Schema.optionalKey(Schema.Array(Schema.String)), // L3 + leaseToken: Schema.optionalKey(Schema.String), // L7, only while the guest completes (pre-P1) + deadline: Schema.optionalKey(Schema.Number) // epoch ms; acquireTime + timeout-minutes +}) +export type Grant = typeof Grant.Type +export const Usage = Schema.Struct({ prompt_tokens: Schema.Int, completion_tokens: Schema.Int, tool_calls: Schema.Int }) +export type Usage = typeof Usage.Type + +// Temporal PollActivityTaskQueue + ARC free capacity on every poll + Forgejo request key + Buildkite intervals. +export const Lease = Rpc.make("Lease", { + payload: { holderIdentity: Schema.String, capacity: Schema.Int, requestKey: Schema.String }, + success: Schema.Struct({ + grants: Schema.Array(Grant), stats: Stats, nextPollSeconds: Schema.Int, heartbeatSeconds: Schema.Int, + endpoint: Schema.optionalKey(Schema.String) // Buildkite ping endpoint switch + }) +}) +// Temporal RecordActivityTaskHeartbeat with cancel_requested, batched per session. `leaseIds` is the COMPLETE set +// the holder believes it holds (level-triggered, ARC): an orphaned lease it omits is requeued (L3 a). +// Round 4: `pendingRequestKey` (Forgejo request key, B8) is the Lease requestKey the holder journaled and has not +// seen answered. The holder cannot list leaseIds it never received, so it vouches for the key instead: the floor +// renews every lease granted under that key and never releases one as an omitted orphan (L3 a) while it is named. +export const Heartbeat = Rpc.make("Heartbeat", { + payload: { holderIdentity: Schema.String, leaseIds: Schema.Array(Schema.String), pendingRequestKey: Schema.optionalKey(Schema.String) }, + success: Schema.Struct({ + renewed: Schema.Array(Schema.String), lost: Schema.Array(Schema.String), + cancelRequested: Schema.Array(Schema.String), stats: Stats + }) +}) +// Temporal RespondActivityTask{Completed,Failed,Canceled}. The verdict travels here, never as an exit code (ARC). +// The two errors are terminal: stop retrying (Buildkite 422). +export const CompleteError = Schema.Struct({ code: Schema.Literals(["unknown-lease", "stale-attempt"]) }) +export const Complete = Rpc.make("Complete", { + payload: { + leaseId: Schema.String, attempt: Schema.Int, result: Result, + output: Schema.optionalKey(Schema.Unknown), usage: Schema.optionalKey(Usage) + }, + success: Schema.Struct({ duplicate: Schema.Boolean, stats: Stats, withdrew: Schema.optionalKey(Schema.String), + withdrewTransitions: Schema.optionalKey(Schema.Int) }), + error: CompleteError +}) +export class FloorLink extends RpcGroup.make(Lease, Heartbeat, Complete) {} + +// The link's own view of the same wire (round 1, finding "poison grant"): identical tags, payloads and JSON, but each +// grant is decoded by the link one at a time, so one malformed job fails only its own lease (pre-start/invalid-spec) +// instead of turning the whole Lease reply into a decode defect that the journaled requestKey would replay for ever. +// Round 3: the whole envelope is lenient too. A drifted interval (the prototype floor sends leaseMs / 3000, e.g. +// 3.333), a null endpoint or a changed stats shape would otherwise fail the decode of every replay of the journaled +// requestKey and strand the grants in it. The link bounds the intervals itself (link.ts `boundedSeconds`). +export const LeaseLenient = Rpc.make("Lease", { + payload: { holderIdentity: Schema.String, capacity: Schema.Int, requestKey: Schema.String }, + success: Schema.Struct({ + grants: Schema.Array(Schema.Unknown), stats: Schema.optionalKey(Schema.Unknown), + nextPollSeconds: Schema.optionalKey(Schema.Unknown), heartbeatSeconds: Schema.optionalKey(Schema.Unknown), + endpoint: Schema.optionalKey(Schema.Unknown) + }) +}) +// Round 2: the link decodes Complete's error code as any string, so a code the floor adds later is a permanent refusal +// (dropped, logged with the code), never a decode defect retried for ever at the head of the outbox. +// Round 3: its success is lenient as well. Any 2xx Exit Success means the floor applied the verdict; a drifted key +// (`withdrew: null`, a missing stats) must not keep an applied verdict at the head of the outbox. +export const CompleteLenient = Rpc.make("Complete", { + payload: Complete.payloadSchema.fields, + success: Schema.Struct({ duplicate: Schema.optionalKey(Schema.Unknown), stats: Schema.optionalKey(Schema.Unknown), + withdrew: Schema.optionalKey(Schema.Unknown), withdrewTransitions: Schema.optionalKey(Schema.Unknown) }), + error: Schema.Struct({ code: Schema.String }) +}) +// Round 4: Heartbeat is lenient as well. The strict reply schema turned a drifted `stats` into a heartbeat-error on +// every call, so `cancelRequested`, `lost` and `renewed` were never read while the floor kept renewing (a cancel was +// never carried out, a journaled grant never resumed). Every key is optional Unknown; floor.ts keeps the strings. +export const HeartbeatLenient = Rpc.make("Heartbeat", { + payload: Heartbeat.payloadSchema.fields, + success: Schema.Struct({ renewed: Schema.optionalKey(Schema.Unknown), lost: Schema.optionalKey(Schema.Unknown), + cancelRequested: Schema.optionalKey(Schema.Unknown), stats: Schema.optionalKey(Schema.Unknown) }) +}) +export class FloorLinkClient extends RpcGroup.make(LeaseLenient, HeartbeatLenient, CompleteLenient) {} + +// On result "failure" the link's `output` is this shape (Buildkite finish signal_reason, ARC failure reason). +// The reasons in RETRYABLE are the floor's to retry as attempt+1 (Temporal retry policy, Buildkite finish -1 on a +// pre-start failure); every other reason is final: a deterministic refusal, or the agent's own failure, which goes +// to the interpreter (null-on-failure). +export const FailureOutput = Schema.Struct({ + reason: Schema.String, // pre-start/env-over-budget, pre-start/name-conflict, infra/task-lost, deadline-exceeded, ... + message: Schema.optionalKey(Schema.String), + phase: Schema.optionalKey(Schema.String), + exitCode: Schema.optionalKey(Schema.Int) +}) +export type FailureOutput = typeof FailureOutput.Type +export const RETRYABLE = new Set([ + "pre-start/ax-unavailable", "pre-start/pending-timeout", "pre-start/resource-exhausted", + "infra/task-lost", "infra/actor-crashed", "infra/result-unreadable", + "pre-start/superseded-verdict-pending", // round 1: n+1 refused while attempt n's verdict could not be delivered + "infra/link-defect" // round 4: a dispatch fiber died on a defect before the Task was created; the floor requeues +]) +// Round 2, final (not in RETRYABLE): `infra/verdict-undeliverable` (the floor answered every delivery of the real +// verdict with a server error; a rerun would most likely produce the same bytes) and `agent/output-too-large` (the +// result is over the link's cap, below the floor's 2 MB row limit). +export const isRetryableReason = (reason: string) => RETRYABLE.has(reason) diff --git a/pkgs/substrate-apps/src/apps/link/src/floor.ts b/pkgs/substrate-apps/src/apps/link/src/floor.ts new file mode 100644 index 00000000..caef5f23 --- /dev/null +++ b/pkgs/substrate-apps/src/apps/link/src/floor.ts @@ -0,0 +1,169 @@ +// The link's view of the floor: the three L1 RPCs over Effect's stock HTTP RPC client (decision L2), outbound only. +// HTTP statuses the floor's Worker entry answers before any RPC runs are classified here (Buildkite's retry +// classifier, api/retryable.go): 401/403 fatal (token), 409 fatal (a second session for this identity, L5), +// 429 back off honouring Retry-After. Round 2: any 3xx is fatal (`redirect`: redirects are never followed, so the only +// way to another floor URL is the Lease `endpoint` checked against the Nix list, B18); 404 and 405 are fatal +// (`misrouted`: a wrong route answers every call so, and dropping verdicts for it would lose them); any other 4xx except 408 is +// `rejected` (permanent for these bytes: Complete drops the verdict); a 500, 501 or 505+ and an RPC Defect in the reply +// are `server-error` (the floor answered and refused these bytes; the outbox counts them per verdict); 502, 503, 504, +// 408, network errors and timeouts stay `transient`. Every call has a ceiling (B9). +import { Effect, Layer, Schema, Scope } from "effect" +import { FetchHttpClient, HttpClient, HttpClientRequest } from "effect/unstable/http" +import { RpcClient, RpcSerialization } from "effect/unstable/rpc" +import { Complete, FloorLinkClient, Grant } from "./contract.ts" +import type { Usage, Result } from "./contract.ts" + +/** A grant the link could not decode. `leaseId` and `attempt` are set when they alone decode, so the lease can be + * failed back to the floor as pre-start/invalid-spec; otherwise the floor's own expiry is the only way out. */ +export interface InvalidGrant { readonly leaseId?: string; readonly attempt?: number; readonly message: string } +/** Round 3: the reply as the link uses it, after the lenient envelope decode. Intervals and endpoint are kept only + * when they have the right JSON type; the link bounds the intervals (link.ts `boundedSeconds`). */ +export type LeaseReply = { + readonly grants: ReadonlyArray; readonly invalid?: ReadonlyArray + readonly stats?: unknown; readonly nextPollSeconds?: number; readonly heartbeatSeconds?: number; readonly endpoint?: string +} +/** Round 4: the Heartbeat reply after the lenient decode: each list keeps only its strings, a missing one is []. */ +type HeartbeatReply = { readonly renewed: ReadonlyArray; readonly lost: ReadonlyArray; readonly cancelRequested: ReadonlyArray; readonly stats?: unknown } +const strings = (v: unknown): Array => Array.isArray(v) ? v.filter((x): x is string => typeof x === "string") : [] +const heartbeatReply = (r: { readonly renewed?: unknown; readonly lost?: unknown; readonly cancelRequested?: unknown; readonly stats?: unknown }): HeartbeatReply => + ({ renewed: strings(r.renewed), lost: strings(r.lost), cancelRequested: strings(r.cancelRequested), stats: r.stats }) +type CompleteReply = { readonly duplicate: boolean; readonly withdrew?: string; readonly withdrewTransitions?: number; readonly stats?: unknown } +const num = (v: unknown) => typeof v === "number" ? v : undefined +const leaseReply = (r: { readonly grants: ReadonlyArray; readonly nextPollSeconds?: unknown; readonly heartbeatSeconds?: unknown; readonly endpoint?: unknown; readonly stats?: unknown }): LeaseReply => { + const s = splitGrants(r) + const poll = num(r.nextPollSeconds), hb = num(r.heartbeatSeconds) + return { grants: s.grants, invalid: s.invalid, stats: r.stats, ...(poll !== undefined ? { nextPollSeconds: poll } : {}), + ...(hb !== undefined ? { heartbeatSeconds: hb } : {}), ...(typeof r.endpoint === "string" ? { endpoint: r.endpoint } : {}) } +} +const completeReply = (r: { readonly duplicate?: unknown; readonly withdrew?: unknown; readonly withdrewTransitions?: unknown; readonly stats?: unknown }): CompleteReply => { + const wt = num(r.withdrewTransitions) + return { duplicate: r.duplicate === true, stats: r.stats, ...(typeof r.withdrew === "string" ? { withdrew: r.withdrew } : {}), + ...(wt !== undefined && Number.isInteger(wt) ? { withdrewTransitions: wt } : {}) } +} +const encodeComplete = Schema.encodeUnknownExit(Complete.payloadSchema) + +export type FloorErrorKind = "transient" | "auth" | "session-conflict" | "rate-limited" | "redirect" | "misrouted" | "rejected" | "server-error" +export class FloorError { + readonly _tag = "FloorError" + constructor(readonly kind: FloorErrorKind, readonly message: string, readonly retryAfterMs?: number, readonly status?: number) {} + get fatal() { return this.kind === "auth" || this.kind === "session-conflict" || this.kind === "redirect" || this.kind === "misrouted" } +} +/** The floor refused these bytes for good: one of its CompleteError codes (any string, round 2), or an HTTP 4xx. */ +export class CompleteRefused { + readonly _tag = "CompleteRefused" + constructor(readonly code: string) {} +} + +export interface CompletePayload { leaseId: string; attempt: number; result: Result; output?: unknown; usage?: Usage } +export interface FloorApi { + readonly lease: (p: { holderIdentity: string; capacity: number; requestKey: string }) => Effect.Effect + readonly heartbeat: (p: { holderIdentity: string; leaseIds: ReadonlyArray; pendingRequestKey?: string }) => Effect.Effect + readonly complete: (p: CompletePayload) => Effect.Effect +} + +export interface FloorOptions { + readonly url: string // e.g. https://substrate..workers.dev (no trailing /rpc) + readonly token: string // the per-link bearer, read from $CREDENTIALS_DIRECTORY; never logged + readonly sessionId: string // stable per state directory: a restart is the same session, a second replica is not + readonly fetch?: typeof globalThis.fetch // tests hand in an in-process handler; production uses globalThis.fetch + /** B9: a ceiling on every call (Buildkite's 330 s on acquire, shortened): Lease and Heartbeat 20 s, Complete 60 s. */ + readonly timeoutsMs?: { readonly lease: number; readonly heartbeat: number; readonly complete: number } +} +export const DEFAULT_FLOOR_TIMEOUTS_MS = { lease: 20_000, heartbeat: 20_000, complete: 60_000 } as const + +/** B9 (critique R9): a non-2xx answer is thrown by the fetch wrapper of the call that received it, so its status + * travels inside that call's own error chain; nothing is shared between concurrent calls. */ +class HttpStatus extends Error { + readonly linkHttpStatus = true + constructor(readonly status: number, readonly retryAfterMs: number | undefined) { super(`floor answered ${status}`) } +} +const statusIn = (e: unknown): HttpStatus | undefined => { + let x: any = e + for (let i = 0; i < 10 && x != null && typeof x === "object"; i++) { + if (x instanceof HttpStatus || x.linkHttpStatus === true) return x as HttpStatus + x = x.reason ?? x.cause ?? x.error + } + return undefined +} +const decodeGrant = Schema.decodeUnknownExit(Grant) +const Envelope = Schema.decodeUnknownExit(Schema.Struct({ leaseId: Schema.String, attempt: Schema.Int })) +/** Decode each grant on its own: the good ones are dispatched, the bad ones are refused one by one. */ +export const splitGrants = }>(r: R): Omit & { grants: Array; invalid: Array } => { + const grants: Array = [], invalid: Array = [] + for (const raw of r.grants) { + const g = decodeGrant(raw) + if (g._tag === "Success") { grants.push(g.value); continue } + const env = Envelope(raw) + const message = String(g.cause).slice(0, 500) + invalid.push(env._tag === "Success" ? { leaseId: env.value.leaseId, attempt: env.value.attempt, message } : { message }) + } + return { ...r, grants, invalid } +} + +export const classifyFloorError = (e: unknown): FloorError => { + const st = statusIn(e) + const s = st?.status ?? 0 + const msg = e instanceof Error ? e.message : String((e as any)?.message ?? e) + if (s >= 300 && s < 400) return new FloorError("redirect", `floor answered ${s}: redirects are never followed; fix LINK_FLOOR_URL`, undefined, s) + if (s === 401 || s === 403) return new FloorError("auth", `floor answered ${s}`, undefined, s) + if (s === 409) return new FloorError("session-conflict", "floor answered 409: another session holds this link identity", undefined, s) + if (s === 429) return new FloorError("rate-limited", "floor answered 429", st?.retryAfterMs, s) + // 404/405: the route is wrong for every call, not these bytes; fatal (exit 78), so no verdict is dropped for it + if (s === 404 || s === 405) return new FloorError("misrouted", `floor answered ${s}: fix LINK_FLOOR_URL or the Worker route`, undefined, s) + if (s >= 400 && s < 500 && s !== 408) return new FloorError("rejected", `floor answered ${s}`, undefined, s) + if (s >= 500 && s !== 502 && s !== 503 && s !== 504) return new FloorError("server-error", `floor answered ${s}`, undefined, s) + return new FloorError("transient", s ? `floor answered ${s}` : msg, undefined, s || undefined) +} +/** Round 3: a defect raised by a call is always the reply's: the link encodes the Complete payload itself before it + * sends (a failure there is `transient`, the link's own bug), and the envelopes it decodes are lenient. A defect is + * then the floor's RPC `Defect` reply (e.g. SQLITE_TOOBIG) or a reply that is not the protocol at all: the server + * refusing these bytes (`server-error`), counted per verdict on Complete and per requestKey on Lease. */ +export const rpcFloor = (o: FloorOptions): Effect.Effect => Effect.gen(function*() { + const base = o.fetch ?? globalThis.fetch + const t = o.timeoutsMs ?? DEFAULT_FLOOR_TIMEOUTS_MS + const observing: typeof globalThis.fetch = async (input, init) => { + // Round 2: never follow a redirect. A followed 307 lands on an origin the Nix list never declared, and the link + // would act on its replies (B18 bypass); an Access 302 to a login page would read as a transient decode error. + const res = await base(input, { ...init, redirect: "manual" }) + if (res.type === "opaqueredirect") throw new HttpStatus(307, undefined) + if (!res.ok) { + const ra = Number(res.headers.get("retry-after")) + throw new HttpStatus(res.status, Number.isFinite(ra) && ra > 0 ? ra * 1000 : undefined) + } + return res + } + const protocol = RpcClient.layerProtocolHttp({ + url: `${o.url.replace(/\/$/, "")}/rpc`, + transformClient: HttpClient.mapRequest((r) => r.pipe( + HttpClientRequest.bearerToken(o.token), + HttpClientRequest.setHeader("x-link-session", o.sessionId))) + }).pipe(Layer.provide([FetchHttpClient.layer, RpcSerialization.layerJson]), + Layer.provide(Layer.succeed(FetchHttpClient.Fetch, observing))) + const client = yield* RpcClient.make(FloorLinkClient).pipe(Effect.provide(protocol)) + // The timeout interrupts the call, which aborts its fetch (FetchHttpClient passes an AbortSignal). + const within = (ms: number, what: string) => (e: Effect.Effect) => + e.pipe(Effect.timeoutOrElse({ duration: ms, orElse: () => Effect.fail(new FloorError("transient", `${what} timed out after ${ms} ms`)) })) + return { + // Round 1: a reply that does not decode is a typed transient failure, never a defect that kills the link. + lease: (p) => client.Lease(p).pipe(Effect.catchDefect((d) => Effect.fail(new FloorError("server-error", `Lease reply undecodable: ${String(d).slice(0, 300)}`))), + Effect.mapError((e) => e instanceof FloorError ? e : classifyFloorError(e)), Effect.map(leaseReply), within(t.lease, "Lease")), + // Round 4: lenient like Lease and Complete; a reply that still does not decode is the server's (server-error). + heartbeat: (p) => client.Heartbeat({ holderIdentity: p.holderIdentity, leaseIds: p.leaseIds, ...(p.pendingRequestKey !== undefined ? { pendingRequestKey: p.pendingRequestKey } : {}) }).pipe( + Effect.catchDefect((d) => Effect.fail(new FloorError("server-error", `Heartbeat reply undecodable: ${String(d).slice(0, 300)}`))), + Effect.mapError((e) => e instanceof FloorError ? e : classifyFloorError(e)), Effect.map(heartbeatReply), within(t.heartbeat, "Heartbeat")), + // L9: omit absent optional keys; `usage: undefined` in an optionalKey field is an encode defect, not a failure. + complete: (p) => { + const payload = { leaseId: p.leaseId, attempt: p.attempt, result: p.result, + ...(p.output !== undefined ? { output: p.output } : {}), ...(p.usage !== undefined ? { usage: p.usage } : {}) } + const enc = encodeComplete(payload) + if (enc._tag === "Failure") return Effect.fail(new FloorError("transient", `encode defect: ${String(enc.cause).slice(0, 300)}`)) + return client.Complete(payload).pipe(Effect.map(completeReply), + Effect.catchDefect((d) => Effect.fail(new FloorError("server-error", `server defect: ${String((d as any)?.message ?? d).slice(0, 300)}`))), + Effect.mapError((e: any) => e instanceof FloorError ? e + : e && typeof e === "object" && "code" in e && !("_tag" in e) ? new CompleteRefused(String(e.code)) : classifyFloorError(e)), + // Round 2: a 4xx other than 401/403/408/409/429 refuses these bytes for good (Buildkite: not retryable) + Effect.mapError((e) => e instanceof FloorError && e.kind === "rejected" ? new CompleteRefused(`http-${e.status}`) : e), + within(t.complete, "Complete")) + } + } +}) diff --git a/pkgs/substrate-apps/src/apps/link/src/jobs.ts b/pkgs/substrate-apps/src/apps/link/src/jobs.ts new file mode 100644 index 00000000..830bfa8d --- /dev/null +++ b/pkgs/substrate-apps/src/apps/link/src/jobs.ts @@ -0,0 +1,120 @@ +// Pure mapping AgentJob -> ax Task JSON (FIELD-MAP-2026-09-23.md 5a keys). Ported from the option-A prototype +// (arc/proto/jobs.ts) with ARC-DECISION X1 applied: atespace `fleet`, image `ax-agent`, Gateway `halogen`, the names +// ax-fleet DESIGN.md 10.2 uses. No I/O: the link and any converter call the same function. +import { createHash } from "node:crypto" +import type { AgentJob, AxTask, Grant } from "./contract.ts" + +export const ENV_BUDGET = 16384 // FIELD-MAP 5a hard pre-dispatch check (stock v0.3.0, before P5) +const A = "ultracode.mecattaf.dev/" +const utf8Len = (s: string) => new TextEncoder().encode(s).length +/** FIELD-MAP 5a journal key: `:` (substrate taskspec.ts journalKey). */ +export const JOURNAL_KEY = /^[0-9a-f]{64}:\d+$/ + +export interface TaskShape { + readonly atespace: string // `fleet` + readonly image: string // `localhost:5000/ax-agent@sha256:...` (ax-fleet-image-ref) + readonly gateway: string // `halogen` + readonly command: (job: AgentJob) => ReadonlyArray | undefined // e.g. ["ax-agent", "pi"] for seat:halogen + readonly completeUrl?: string // set only in guest-completion mode (L7, pre-P1) + readonly holder?: string // round 1: AX_CONWIP_HOLDER, the ownership mark a journal-less link checks before a fence +} + +/** B7: no default seat. `undefined` unless `runs-on` names exactly one `seat:` label. */ +export const seatOf = (job: AgentJob): string | undefined => { + const seats = job.spec["runs-on"].filter((l) => l.startsWith("seat:")) + return seats.length === 1 ? seats[0]!.slice(5) : undefined +} + +/** B7: `runs-on` must be non-empty, name exactly one `seat:` label, and be wholly served by this link. */ +export const validRunsOn = (job: AgentJob, served: ReadonlyArray): { reason: string; message: string } | undefined => { + const on = job.spec["runs-on"] + if (on.length === 0) return { reason: "pre-start/runs-on-invalid", message: "runs-on is empty" } + const seats = on.filter((l) => l.startsWith("seat:")) + if (seats.length !== 1) return { reason: "pre-start/runs-on-invalid", message: `runs-on names ${seats.length} seat: labels, not 1` } + const unserved = on.filter((l) => !served.includes(l)) + if (unserved.length > 0) return { reason: "pre-start/label-not-served", message: unserved.join(",") } + return undefined +} + +export type Built = { readonly _tag: "ok"; readonly task: AxTask } | { readonly _tag: "refused"; readonly reason: string; readonly message: string } + +export function axTaskFromGrant(g: Grant, shape: TaskShape): Built { + const job = g.job + // Round 1 (fail closed): a guest Task without its per-lease token has no completion route; never create it. + if (shape.completeUrl !== undefined && (g.leaseToken === undefined || g.leaseToken === "")) + return { _tag: "refused", reason: "pre-start/lease-token-missing", message: "guest completion needs a leaseToken; the floor's token binding for this holder is not guest" } + const ann = job.metadata.annotations ?? {} + const lab = job.metadata.labels ?? {} + if (ann[A + "prompt-unresolved"] !== undefined) // L8: never dispatch an item whose prompt was not resolved + return { _tag: "refused", reason: "pre-start/prompt-unresolved", message: ann[A + "prompt-unresolved"]! } + // Round 4 (FIELD-MAP 5a): the journal key is sha256 of canonical [prompt, opts minus label] plus the occurrence, and + // only the floor-side converter can compute it (it holds opts and the occurrence). It rides the AgentJob as an + // annotation; without it the grant is refused, never replaced by the prompt digest (two calls sharing one prompt). + const journalKey = ann[A + "journal-key"] + if (journalKey === undefined || !JOURNAL_KEY.test(journalKey)) + return { _tag: "refused", reason: "pre-start/invalid-spec", message: journalKey === undefined ? `annotation ${A}journal-key is missing` : `annotation ${A}journal-key is not :` } + const w = job.spec.with + const seat = seatOf(job) + if (seat === undefined) return { _tag: "refused", reason: "pre-start/runs-on-invalid", message: "runs-on does not name exactly one seat: label" } + const command = shape.command(job) + if (command === undefined || command.length === 0) return { _tag: "refused", reason: "pre-start/seat-command-missing", message: `no command for seat ${seat}` } + const env: Array<{ name: string; value: string }> = [ + { name: "AX_CONWIP_RUN_ID", value: ann[A + "run-id-raw"] ?? "" }, + { name: "AX_CONWIP_LABEL", value: ann[A + "label"] ?? "" }, + { name: "AX_CONWIP_MODEL", value: w.model }, + { name: "AX_CONWIP_ITEM_KEY", value: ann[A + "item-key"] ?? "" }, + { name: "AX_CONWIP_JOURNAL_KEY", value: journalKey }, + { name: "AX_CONWIP_WORKFLOW", value: lab[A + "workflow"] ?? "" }, + { name: "AX_CONWIP_PHASE_INDEX", value: lab[A + "phase-index"] ?? "" }, + { name: "AX_CONWIP_PHASE_TITLE", value: ann[A + "phase-title"] ?? "" }, + { name: "AX_CONWIP_SEAT", value: seat }, + { name: "AX_CONWIP_ATTEMPT", value: String(g.attempt) }, + { name: "AX_CONWIP_LEASE_ID", value: g.leaseId }, + ...(shape.holder !== undefined ? [{ name: "AX_CONWIP_HOLDER", value: shape.holder }] : []), + { name: "AX_CONWIP_PROMPT_SHA256", value: w.prompt_ref.sha256 }, + { name: "AX_CONWIP_RESULT_PATH", value: "/workspace/.ax/result.json" }, + { name: "AX_CONWIP_USAGE_PATH", value: "/workspace/.ax/usage.json" } + ] + if (w.effort !== undefined) env.push({ name: "AX_CONWIP_EFFORT", value: w.effort }) + if (w.schema !== undefined) env.push({ name: "AX_CONWIP_SCHEMA_JSON", value: JSON.stringify(w.schema) }) + if (w.isolation !== undefined) env.push({ name: "AX_CONWIP_ISOLATION", value: w.isolation }) + if (w["agent-type"] !== undefined) env.push({ name: "AX_CONWIP_AGENT_TYPE", value: w["agent-type"] }) + if (job.spec["timeout-minutes"] !== undefined) + env.push({ name: "AX_CONWIP_TIMEOUT_MINUTES", value: String(job.spec["timeout-minutes"]) }) + if (shape.completeUrl !== undefined && g.leaseToken !== undefined) { // L7 (a missing token was refused above): a per-lease token, never the fleet bearer + env.push({ name: "AX_CONWIP_COMPLETE_URL", value: shape.completeUrl }) + env.push({ name: "AX_CONWIP_LEASE_TOKEN", value: g.leaseToken }) + } + const used = env.reduce((n, e) => n + utf8Len(e.value), 0) + // B12 (critique R2): the file route (AX_CONWIP_PROMPT_FILE) needs a Workspace that writes the file, and the Task + // carries none yet. Until that route exists a prompt that does not fit inline is refused, never sent to a dead path. + if (used > ENV_BUDGET) // FIELD-MAP D4 and probe test 4: over budget ax falls back silently, so refuse here + return { _tag: "refused", reason: "pre-start/env-over-budget", message: `env values ${used} B > ${ENV_BUDGET} B before the prompt` } + if (w.prompt === undefined || used + utf8Len(w.prompt) > ENV_BUDGET) + return { _tag: "refused", reason: "pre-start/prompt-file-route-missing", + message: w.prompt === undefined ? "no inline prompt and no Workspace route" : `prompt ${utf8Len(w.prompt)} B does not fit the ${ENV_BUDGET - used} B left inline` } + env.push({ name: "AX_CONWIP_PROMPT", value: w.prompt }) + const total = env.reduce((n, e) => n + utf8Len(e.value), 0) + if (total > ENV_BUDGET) // FIELD-MAP D4 and probe test 4: over budget ax falls back silently, so refuse here + return { _tag: "refused", reason: "pre-start/env-over-budget", message: `env values ${total} B > ${ENV_BUDGET} B` } + return { + _tag: "ok", + task: { + apiVersion: "ax.io/v1alpha1", + kind: "Task", + metadata: { name: g.leaseId, atespace: shape.atespace }, + spec: { image: shape.image, command: [...command], env, gateway: { name: shape.gateway } } + } + } +} + +// The spec digest used by create-only adoption (L4): found with the same digest = adopt, another = conflict. +// Only the fields the link writes are hashed, in a fixed order, so ax defaults added on the server do not count. +export const specDigest = (spec: { image?: string; command?: ReadonlyArray; env?: ReadonlyArray<{ name?: string; value?: string }>; gateway?: { name?: string } | null }) => + createHash("sha256").update(JSON.stringify([ + spec.image ?? "", [...(spec.command ?? [])], + [...(spec.env ?? [])].map((e) => [e.name ?? "", e.value ?? ""]), + spec.gateway?.name ?? "" + ])).digest("hex") + +export const envBytes = (t: AxTask) => t.spec.env.reduce((n, e) => n + utf8Len(e.value), 0) diff --git a/pkgs/substrate-apps/src/apps/link/src/journal.ts b/pkgs/substrate-apps/src/apps/link/src/journal.ts new file mode 100644 index 00000000..8ee3e03b --- /dev/null +++ b/pkgs/substrate-apps/src/apps/link/src/journal.ts @@ -0,0 +1,187 @@ +// The link's crash-safe local state: one append-only JSONL file, fsynced per line, written BEFORE the action it +// records (the tally uplink's "offline is a queue" outbox, kept verbatim in spirit: done / keep / drop). The ax +// store is the ledger of Tasks (Buildkite: the cluster is the ledger); this file only covers what ax cannot know: +// grants not yet created, and Complete reports not yet acknowledged by the floor. +import { appendFileSync, closeSync, existsSync, fsyncSync, mkdirSync, openSync, readFileSync, renameSync, writeSync } from "node:fs" +import { join } from "node:path" +import type { Grant, Result, Usage } from "./contract.ts" + +export type Entry = + | { ev: "grant"; leaseId: string; grant: Grant; regrant?: true } // round 2: `regrant` replaces a released record + | { ev: "creating"; leaseId: string; digest?: string } // round 1: written BEFORE the first UpdateTask; the Task may exist from here on. Round 3: the spec digest the link will write, so a cleanup can tell its own Task from a foreign one + | { ev: "not-mine"; leaseId: string } // round 3: a Task of this name exists and is not the link's; `creating` no longer holds + | { ev: "created"; leaseId: string; digest: string } + | { ev: "grant-invalid"; leaseId: string; attempt: number; message: string } // round 1: a grant that did not decode + | { ev: "report"; leaseId: string; attempt: number; result: Result; output?: unknown; usage?: Usage; replace?: true } // outbox; round 2: `replace` swaps an undeliverable verdict + | { ev: "reported"; leaseId: string; duplicate: boolean; withdrew?: string; withdrewGen?: Gen } // done: 2xx from the floor; round 2: what rule 4b withdrew; round 3: which generation of it + | { ev: "dropped"; leaseId: string; code: string } // drop: the floor refused these bytes for good + | { ev: "released"; leaseId: string; why: string } // no longer held (lost, cancelled, superseded) + | { ev: "deleting"; leaseId: string; why: string } // DeleteTask accepted; waiting for NotFound (ax deletes in two phases) + | { ev: "deleted"; leaseId: string } + | { ev: "lease-key"; requestKey: string } // B8: a Lease is in flight with this key; reuse it until its reply lands + | { ev: "lease-replied"; requestKey: string; abandoned?: true } // the reply's grants are journaled; the next Lease draws a new key + | { ev: "tomb"; leaseId: string; gen: Gen } // round 3: written by compaction for a finished lease; refuses its redelivery + +/** Round 3: a grant generation. `acquireTime` is absent when only the floor's `withdrewTransitions` is known. */ +export interface Gen { readonly leaseTransitions: number; readonly acquireTime?: number } +export const genOf = (g: Grant): Gen => ({ leaseTransitions: g.lease.leaseTransitions, acquireTime: g.lease.acquireTime }) +/** `g` is a strictly later generation than `than` (leaseTransitions first, then acquireTime). */ +export const laterGen = (g: Gen, than: Gen) => g.leaseTransitions > than.leaseTransitions || + (g.leaseTransitions === than.leaseTransitions && g.acquireTime !== undefined && than.acquireTime !== undefined && g.acquireTime > than.acquireTime) +/** `g` is the generation `w` names: equal leaseTransitions, and equal acquireTime when both are known. */ +export const sameGen = (g: Gen, w: Gen) => g.leaseTransitions === w.leaseTransitions && + (g.acquireTime === undefined || w.acquireTime === undefined || g.acquireTime === w.acquireTime) + +export interface LeaseRec { + grant: Grant + created?: string // spec digest + creating?: boolean // an UpdateTask may have landed (write-ahead); cleanup must DeleteTask by name and see NotFound + creatingDigest?: string // round 3: the spec digest the link was about to write + notMine?: boolean // round 3: the Task under this name is someone else's; never read, delete or report it + createdAt?: number + report?: Extract + reported?: boolean + duplicate?: boolean // round 2: the floor answered this verdict as a duplicate (rule 4c) + withdrew?: string // round 2: the leaseId the floor withdrew when it accepted this verdict (rule 4b), never inferred + withdrewGen?: Gen // round 3: the generation withdrawn; undefined when this link never held it (it then matches nothing) + dropped?: string + released?: string + deleting?: string // why: cancel, lost, dropped, janitor, pre-start, deadline, superseded + deleted?: boolean + at: number // when this record was last touched (ms) +} + +export class Journal { + readonly path: string + readonly recs = new Map() + /** Grants that did not decode: refused to the floor as pre-start/invalid-spec through the same outbox. */ + readonly invalid = new Map() + pendingLeaseKey: string | undefined + /** Round 3: finished leases, kept through compaction for TOMB_MS so a redelivered grant of one is never run again. */ + readonly tombs = new Map() + static TOMB_MS = 24 * 3600_000 // well past the floor's replay window (rule 2b: lease + grace, 390 s in production) + private readonly dir: string + private constructor(dir: string) { this.dir = dir; this.path = join(dir, "journal.jsonl") } + + static open(dir: string, now = Date.now()): Journal { + mkdirSync(dir, { recursive: true, mode: 0o700 }) + const j = new Journal(dir) + if (existsSync(j.path)) { + for (const line of readFileSync(j.path, "utf8").split("\n")) { + if (line.trim() === "") continue + let e: Entry & { t?: number } + try { e = JSON.parse(line) } catch { continue } // a torn last line from a crash mid-write is skipped + j.apply(e, e.t ?? now) + } + j.compact() + } + return j + } + + private apply(e: Entry, t: number) { + if (e.ev === "lease-key") { this.pendingLeaseKey = e.requestKey; return } + if (e.ev === "lease-replied") { if (this.pendingLeaseKey === e.requestKey) this.pendingLeaseKey = undefined; return } + if (e.ev === "tomb") { this.tombs.set(e.leaseId, { gen: e.gen, at: t }); return } + if (e.ev === "grant") { + if (this.recs.has(e.leaseId) && !e.regrant) return + this.recs.set(e.leaseId, { grant: e.grant, at: t }) + // round 2: the withdrawal recorded against the previous generation of this leaseId does not apply to the new one + if (e.regrant) for (const x of this.recs.values()) if (x.withdrew === e.leaseId) { delete x.withdrew; delete x.withdrewGen } + return + } + if (e.ev === "grant-invalid") { if (!this.recs.has(e.leaseId) && !this.invalid.has(e.leaseId)) this.invalid.set(e.leaseId, { attempt: e.attempt, message: e.message, at: t }); return } + const r = this.recs.get(e.leaseId) + if (!r) { + const x = this.invalid.get(e.leaseId) + if (x && e.ev === "reported") x.reported = true + if (x && e.ev === "dropped") x.dropped = e.code + if (x && e.ev === "released") x.released ??= e.why + return + } + r.at = t + switch (e.ev) { + case "creating": r.creating = true; delete r.notMine; if (e.digest !== undefined) r.creatingDigest = e.digest; break + case "not-mine": r.notMine = true; delete r.creating; break + case "created": r.created = e.digest; r.createdAt ??= t; break + case "report": if (!r.report || e.replace) r.report = e; break // first verdict wins (the floor dedupes as well), unless replaced + case "reported": r.reported = true; r.duplicate = e.duplicate; if (e.withdrew !== undefined) { r.withdrew = e.withdrew; if (e.withdrewGen !== undefined) r.withdrewGen = e.withdrewGen } break + case "dropped": r.dropped = e.code; break + case "released": r.released ??= e.why; break + case "deleting": r.deleting ??= e.why; break + case "deleted": r.deleted = true; break + } + } + + append(e: Entry, t = Date.now()) { + appendFileSync(this.path, JSON.stringify({ t, ...e }) + "\n", { mode: 0o600 }) + const fd = openSync(this.path, "r") + try { fsyncSync(fd) } finally { closeSync(fd) } + this.apply(e, t) + if (++this.appends >= Journal.COMPACT_EVERY) this.compact() // idle Lease keys must not grow the file for ever + } + static COMPACT_EVERY = 2000 + private appends = 0 + + /** Finished = nothing left to tell the floor and nothing left in ax. */ + static finished(r: LeaseRec) { + // Round 2: a verdict read after the lease was released (lost, superseded) is still owed to the floor + if (Journal.pending(r)) return false + const toldFloor = r.reported === true || r.dropped !== undefined || r.released !== undefined + const axClean = !Journal.maybeCreated(r) || r.deleted === true + return toldFloor && axClean + } + /** Round 1: a Task may exist in ax once `creating` is journaled, even if `created` never was (a crash, or a + * DEADLINE_EXCEEDED after the write landed). Every cleanup path treats such a record as created. */ + static maybeCreated(r: LeaseRec) { return r.created !== undefined || r.creating === true } + /** A verdict is journaled and neither acknowledged nor dropped. */ + static pending(r: LeaseRec) { return r.report !== undefined && !r.reported && r.dropped === undefined } + + /** Rewrite the file with only unfinished leases (atomic rename), so it never grows without bound. */ + compact() { + const tmp = this.path + ".tmp" + const fd = openSync(tmp, "w", 0o600) + try { + if (this.pendingLeaseKey !== undefined) writeSync(fd, JSON.stringify({ t: Date.now(), ev: "lease-key", requestKey: this.pendingLeaseKey }) + "\n") + for (const [id, r] of this.recs) { + if (Journal.finished(r)) { this.recs.delete(id); this.tombs.set(id, { gen: genOf(r.grant), at: r.at }); continue } + const lines: Array = [{ ev: "grant", leaseId: id, grant: r.grant }] + if (r.creating) lines.push({ ev: "creating", leaseId: id, ...(r.creatingDigest !== undefined ? { digest: r.creatingDigest } : {}) }) + if (r.notMine) lines.push({ ev: "not-mine", leaseId: id }) + if (r.created !== undefined) lines.push({ ev: "created", leaseId: id, digest: r.created }) + if (r.report) lines.push(r.report) + if (r.reported) lines.push({ ev: "reported", leaseId: id, duplicate: r.duplicate ?? false, ...(r.withdrew !== undefined ? { withdrew: r.withdrew } : {}), ...(r.withdrewGen !== undefined ? { withdrewGen: r.withdrewGen } : {}) }) + if (r.dropped !== undefined) lines.push({ ev: "dropped", leaseId: id, code: r.dropped }) + if (r.released !== undefined) lines.push({ ev: "released", leaseId: id, why: r.released }) + if (r.deleting !== undefined) lines.push({ ev: "deleting", leaseId: id, why: r.deleting }) + for (const l of lines) writeSync(fd, JSON.stringify({ t: l.ev === "created" || l.ev === "grant" ? (r.createdAt ?? r.at) : r.at, ...l }) + "\n") + } + const now = Date.now() + for (const [id, x] of this.tombs) { + if (now - x.at > Journal.TOMB_MS) { this.tombs.delete(id); continue } + writeSync(fd, JSON.stringify({ t: x.at, ev: "tomb", leaseId: id, gen: x.gen }) + "\n") + } + for (const [id, x] of this.invalid) { // never created, so finished once the floor is told + if (x.reported || x.dropped !== undefined || x.released !== undefined) { this.invalid.delete(id); continue } + writeSync(fd, JSON.stringify({ t: x.at, ev: "grant-invalid", leaseId: id, attempt: x.attempt, message: x.message }) + "\n") + } + fsyncSync(fd) + } finally { closeSync(fd) } + renameSync(tmp, this.path) + this.appends = 0 + const d = openSync(this.dir, "r") // B20: make the rename itself durable + try { fsyncSync(d) } finally { closeSync(d) } + } + + /** The complete set this holder believes it holds: sent in every Heartbeat (level-triggered). A lease whose + * verdict is still in the outbox stays held, so a heartbeat that races the outbox drain cannot orphan it. */ + held(): Array { + return [...[...this.recs].filter(([, r]) => !r.reported && r.dropped === undefined && r.released === undefined).map(([id]) => id), + ...[...this.invalid].filter(([, x]) => !x.reported && x.dropped === undefined && x.released === undefined).map(([id]) => id)] + } + /** Leases whose verdict is written but not yet acknowledged; they are still held until the floor says done. */ + outbox(): Array> { + return [...[...this.recs.values()].filter(Journal.pending).map((r) => r.report!), + ...[...this.invalid].filter(([, x]) => !x.reported && x.dropped === undefined && x.released === undefined) + .map(([leaseId, x]) => ({ ev: "report" as const, leaseId, attempt: x.attempt, result: "failure" as const, output: { reason: "pre-start/invalid-spec", message: x.message } }))] + } +} diff --git a/pkgs/substrate-apps/src/apps/link/src/link.ts b/pkgs/substrate-apps/src/apps/link/src/link.ts new file mode 100644 index 00000000..d8de5b99 --- /dev/null +++ b/pkgs/substrate-apps/src/apps/link/src/link.ts @@ -0,0 +1,921 @@ +// The NAS link (LINK-DESIGN.md section 5): one outbound-only process between the Cloudflare floor and ax. +// Lease (free capacity on every poll, ARC; the outbox drains first, B15; the requestKey is journaled, B8) +// -> journal the grant (write-ahead) -> fence every superseded attempt: DeleteTask and wait for NotFound (L3, B2) +// -> wait for a sandbox slot (B11) -> GetTask, UpdateTask only on NotFound (L4, create-only), read it back (B12) +// -> resync by paging ListTasks (Buildkite: the executor's list is the ledger; B1) and confirming every absence with +// GetTask -> read the outcome (P1: terminal phase + GetTaskResult, retried and digest-checked, B4; guest mode only +// when configured, L7, B6) -> Complete through the journal's outbox (L6) -> DeleteTask after the floor acknowledged. +// One batched Heartbeat per session carries the COMPLETE held set; its reply carries cancel and lost (Temporal). +// Nothing is leased while ax is unreachable, the completion mode is unproven (B6) or the egress Gateway is missing or +// open (B10). +import { createHash, randomUUID } from "node:crypto" +import { isIPv4, isIPv6 } from "node:net" +import { Cause, Deferred, Effect, Schedule, Semaphore } from "effect" +import { gatewayAllowsAll, listAll } from "./ax.ts" +import type { AxApi, AxError, AxObserved } from "./ax.ts" +import type { FailureOutput, Grant, Result, Usage } from "./contract.ts" +import type { FloorApi, FloorError, LeaseReply } from "./floor.ts" +import { axTaskFromGrant, specDigest, validRunsOn } from "./jobs.ts" +import type { TaskShape } from "./jobs.ts" +import { genOf, Journal, laterGen, sameGen } from "./journal.ts" +import type { Gen, LeaseRec } from "./journal.ts" + +export interface LinkConfig { + readonly holder: string // holderIdentity; bound to this link's token on the floor + readonly maxInFlight: number // sandbox cap: the WorkerPool replicas this link may fill + readonly servedLabels: ReadonlyArray // defence in depth: the floor already routes by holder + readonly shape: TaskShape + /** `p1` and `auto` both require P1 on the server, proven by a probe (B6); `auto` never selects guest. `guest` is + * the L7 reserve and needs a fleet-internal `shape.completeUrl` (critique A5). */ + readonly completion: "auto" | "p1" | "guest" + readonly secondMs: number // 1000 in production; tests shrink every server-set interval with it + readonly resyncMs: number // 15 s, the P1 --running-resync period + readonly pendingTimeoutMs: number // 15 min, agent-stack-k8s: a Pod stuck Pending fails the job + readonly deleteAfterMs: number // 10 min, agent-stack-k8s TTLSecondsAfterFinished + readonly deadlineBackstopMs: number // local stop past the floor's deadline, if the floor cannot say cancel + readonly createAttempts: number // UpdateTask tries while ax is unavailable before pre-start failure + readonly outboxBackoffMs: readonly [number, number] // uplink shut-door backoff: min, max + readonly fenceTimeoutMs?: number // B2: how long a superseded attempt may take to reach NotFound (default 120 s) + readonly resultReadTries?: number // B4: resyncs a Completed Task's result may fail to read (default 5) + readonly floorUrls?: ReadonlyArray // B18: the only URLs a Lease `endpoint` may move this link to + readonly initialPollSeconds?: number // until the first Lease reply sets them (defaults 15 and 30, section 3.7) + readonly initialHeartbeatSeconds?: number + /** Round 2: a result larger than this is refused `agent/output-too-large` before it is reported (the floor's DO row + * limit is 2 MB, MEASURED as SQLITE_TOOBIG at 2.2 MB; default 1 000 000 B). */ + readonly maxOutputBytes?: number + /** Round 2: server errors one verdict may collect before it is replaced by `infra/verdict-undeliverable`, and that + * replacement before it is dead-lettered (journaled `dropped`). Default 8. */ + readonly verdictAttempts?: number + /** Round 2: single-label hosts a guest Complete URL may name (fleet DNS names without a dot). Default none. */ + readonly internalHosts?: ReadonlyArray + /** Round 3: consecutive undecodable or server-error Lease replies for one journaled requestKey before the key is + * abandoned (its grants are then released by the floor's own expiry and requeued). Default 5. */ + readonly leaseKeyAttempts?: number +} + +export type Log = (ev: string, fields?: Record) => void +export interface LinkDeps { + readonly ax: AxApi; readonly floor: FloorApi; readonly journal: Journal; readonly log: Log; readonly stop: Deferred.Deferred + readonly floorUrl?: string // the URL in use, to tell a real endpoint switch from an echo + readonly onEndpoint?: (url: string) => void // B18: called once for an allowed switch; main persists it and restarts +} + +const TERMINAL = new Set(["Completed", "Failed"]) +const jitter = (ms: number) => Math.round(ms * (0.8 + 0.4 * Math.random())) +const NOT_FOUND = 5 // gRPC status: DeleteTask of a name that is already gone +const MAX_POLL_SECONDS = 300, MAX_HEARTBEAT_SECONDS = 300 +/** Round 3: a server-driven interval (Buildkite: the agent falls back to its defaults). Missing, non-finite or not + * positive gives `def`; anything else is clamped to [1, hi]. `Effect.sleep(0)` would otherwise be a tight loop. */ +export const boundedSeconds = (v: number | undefined, def: number, hi: number) => + v === undefined || !Number.isFinite(v) || v <= 0 ? Math.min(def, hi) : Math.max(1, Math.min(v, hi)) + +/** Critique A5 and N3: a guest's Complete URL must stay inside the fleet, never a public Workers host. + * Round 2: private ranges are matched only on IP literals (after WHATWG normalisation, so `10.evil.example` and + * `127.0.0.1.nip.io` are names, not addresses); a name must end in a non-public suffix; a single-label name only when + * it is listed. */ +export const isFleetInternalUrl = (raw: string | undefined, internalHosts: ReadonlyArray = []): boolean => { + if (raw === undefined) return false + let u: URL + try { u = new URL(raw) } catch { return false } + let h = u.hostname.toLowerCase() + if (h.endsWith(".workers.dev") || h.endsWith(".pages.dev")) return false + if (h.startsWith("[") && h.endsWith("]")) h = h.slice(1, -1) + if (isIPv4(h)) { + const [a, b] = h.split(".").map(Number) as [number, number] + return a === 127 || a === 10 || (a === 192 && b === 168) || (a === 172 && b >= 16 && b <= 31) || (a === 100 && b >= 64 && b <= 127) + } + if (isIPv6(h)) return h === "::1" || /^f[cd][0-9a-f]{2}:/.test(h) // loopback, ULA fc00::/7 (the tailnet's fd7a:115c:a1e0::/48) + if (h === "localhost" || internalHosts.map((x) => x.toLowerCase()).includes(h)) return true + return /^([a-z0-9-]+\.)+(internal|lan|local|home\.arpa)$/.test(h) +} + +export const runLink = (cfg: LinkConfig, deps: LinkDeps) => Effect.gen(function*() { + const { ax, floor, journal: j, log } = deps + const fatal = yield* Deferred.make() + const fenceTimeoutMs = cfg.fenceTimeoutMs ?? 120_000 + const resultReadTries = cfg.resultReadTries ?? 5 + const maxOutputBytes = cfg.maxOutputBytes ?? 1_000_000 + const verdictAttempts = cfg.verdictAttempts ?? 8 + const defaultPoll = cfg.initialPollSeconds ?? 15, defaultHb = cfg.initialHeartbeatSeconds ?? 30 + let pollSeconds = defaultPoll, hbAsked: number | undefined // round 3: the floor's raw heartbeatSeconds, bounded at use + let clampLogged = "" + const leaseKeyAttempts = cfg.leaseKeyAttempts ?? 5 + let leaseKeyFailures = 0 + let axUp = false, resynced = false + let serverP1: boolean | undefined // the probe's answer; cleared on every ax-up transition (B6) + let gatewayOk: boolean | undefined // B10 + const guestUrlOk = cfg.completion !== "guest" || isFleetInternalUrl(cfg.shape.completeUrl, cfg.internalHosts) + if (!guestUrlOk) log("guest-url-invalid", { completion: "guest" }) + let last = new Map() + let outboxWaitUntil = 0, outboxBackoff = cfg.outboxBackoffMs[0] + let outboxNotBefore = 0 // round 3: the floor's own Retry-After on Complete; Lease does not cut it short (B15 does not) + const dispatching = new Map() // round 2: keyed by leaseId, valued by the grant generation it runs + const admitted = new Set() // B11: grants that passed the slot gate in this process + const abandoned = new Map() // B3: set by the Heartbeat reply, read by dispatch + const absent = new Map() // B1: consecutive resyncs a held Task answered NotFound + const resultFailures = new Map() // B4 + const verdictFailures = new Map() // round 2: server errors per verdict (a restart grants a fresh budget) + const verdictNotBefore = new Map() // round 2: per-verdict backoff after a server error + const pendingResume = new Set() // B13: journaled, never created; resumed only once the floor renews them + /** Round 4 (Buildkite reserve-with-expiry): the local time of the last call the floor answered by renewing this + * grant generation (the Lease that granted it, then every Heartbeat listing it in `renewed`). The send time is taken, + * so it is never later than the floor's own renewTime. Keyed by the grant object: a regrant starts afresh. */ + const renewedAt = new WeakMap() + /** Round 4: a create may start only while the floor still holds the lease: the last renewal is less than 3/4 of a + * lease old. A grant never renewed in this process (journaled by a previous one) is not live until a Heartbeat says so. */ + const leaseLive = (g: Grant) => { + const at = renewedAt.get(g) + if (at === undefined) return false + const ms = g.lease.leaseDurationSeconds * cfg.secondMs + return Date.now() < at + ms - ms / 4 + } + const deadlinePassed = (g: Grant) => g.deadline !== undefined && Date.now() >= g.deadline + let wake = yield* Deferred.make() // B17: a verdict that frees a slot leases at once + let hbWake = yield* Deferred.make() // final pass (R3-1): a Lease reply re-times a heartbeat sleep already begun + const failFatal = (e: FloorError) => Deferred.fail(fatal, e).pipe(Effect.asVoid) + + /** The mode the link may create in, or undefined while it is unproven (capacity 0). */ + const mode = (): "p1" | "guest" | undefined => { + if (cfg.completion === "guest") return guestUrlOk ? "guest" : undefined + return serverP1 === true ? "p1" : undefined + } + const canCreate = () => axUp && resynced && gatewayOk === true && mode() !== undefined + + // ---------------------------------------------------------------- verdicts (write-ahead, then the outbox drains) + /** `salvage` (round 2): the verdict was read from a Task whose lease is already released (lost, superseded). It is + * still owed to the floor, which accepts it by rule 4b or answers stale-attempt (then it is dropped). */ + /** Round 4: `g`, when given, is the grant generation the verdict is about; a record that holds another generation + * by now (a regrant of the same leaseId) is not given it. */ + const report = (id: string, result: Result, output?: unknown, usage?: Usage, salvage = false, g?: Grant) => Effect.gen(function*() { + const r = j.recs.get(id) + if (g !== undefined && r?.grant !== g) return log("stale-generation-verdict", { leaseId: id, result }) + if (!r || r.report || (r.released !== undefined && !salvage)) return + j.append({ ev: "report", leaseId: id, attempt: r.grant.attempt, result, ...(output !== undefined ? { output } : {}), ...(usage !== undefined ? { usage } : {}) }) + log("verdict", { leaseId: id, result, ...(result === "failure" ? { reason: (output as FailureOutput | undefined)?.reason } : {}), ...(salvage ? { salvaged: r.released } : {}) }) + yield* Deferred.succeed(wake, undefined) + }) + const fail = (id: string, o: FailureOutput) => report(id, "failure", o) + const failG = (g: Grant, o: FailureOutput) => report(g.leaseId, "failure", o, undefined, false, g) // round 4 + + // Round 1: single-flight. The main loop, the resync fiber and a dispatch waiting on a superseded verdict all drain; + // two concurrent drains read the same outbox before either journals `reported` and send every verdict twice. + const outboxLock = Semaphore.makeUnsafe(1) + const drainOutbox = outboxLock.withPermit(Effect.gen(function*() { + if (Date.now() < outboxWaitUntil || Date.now() < outboxNotBefore) return + for (const p of j.outbox()) { + if ((verdictNotBefore.get(p.leaseId) ?? 0) > Date.now()) continue + // Round 4: the generation of attempt n+1 this link held when the verdict was SENT. A regrant of n+1 taken while + // the reply is in flight is later than anything this Complete can have withdrawn. + const succId = `${p.leaseId.replace(/-a\d+$/, "")}-a${p.attempt + 1}` + const succAtSend = j.recs.get(succId)?.grant + const r = yield* floor.complete({ leaseId: p.leaseId, attempt: p.attempt, result: p.result, ...(p.output !== undefined ? { output: p.output } : {}), ...(p.usage !== undefined ? { usage: p.usage } : {}) }).pipe(Effect.result) + if (r._tag === "Success") { + const withdrew = r.success.withdrew + // Round 3: a withdrawal names a GENERATION, not a leaseId. The floor may requeue the withdrawn attempt under the + // same leaseId (a retryable verdict of n), and that later grant is live. The generation is the floor's + // `withdrewTransitions` when it sends one, else the one this link holds; a generation this link never received + // (n+1 was still queued) matches no later grant. + const w = withdrew !== undefined ? j.recs.get(withdrew) : undefined + const withdrewGen: Gen | undefined = withdrew === undefined ? undefined + : r.success.withdrewTransitions !== undefined ? { leaseTransitions: r.success.withdrewTransitions } + : succAtSend !== undefined && withdrew === succId ? genOf(succAtSend) + : w !== undefined ? genOf(w.grant) : undefined + j.append({ ev: "reported", leaseId: p.leaseId, duplicate: r.success.duplicate, ...(withdrew !== undefined ? { withdrew } : {}), ...(withdrewGen !== undefined ? { withdrewGen } : {}) }) + log("complete", { leaseId: p.leaseId, result: p.result, duplicate: r.success.duplicate, ...(withdrew !== undefined ? { withdrew } : {}) }) + if (withdrew !== undefined && withdrewGen === undefined) log("withdrawn-unheld", { leaseId: withdrew, by: p.leaseId }) + if (w !== undefined && withdrewGen !== undefined && sameGen(genOf(w.grant), withdrewGen) && w.released === undefined && !Journal.maybeCreated(w)) { + j.append({ ev: "released", leaseId: withdrew!, why: "withdrawn" }) // round 2: released at once, so a regrant + log("withdrawn", { leaseId: withdrew, by: p.leaseId }) // of it is not taken for a duplicate + } + outboxBackoff = cfg.outboxBackoffMs[0] + verdictFailures.delete(p.leaseId); verdictNotBefore.delete(p.leaseId) + } else if (r.failure._tag === "CompleteRefused") { // done/keep/drop: the floor refused these bytes for good + j.append({ ev: "dropped", leaseId: p.leaseId, code: r.failure.code }) + log("complete-dropped", { leaseId: p.leaseId, code: r.failure.code }) + } else if (r.failure.kind === "server-error") { + // Round 2: the floor answered and refused THESE bytes (a 500, an RPC Defect such as SQLITE_TOOBIG). Counted per + // verdict; the rest of the outbox still goes out. At the ceiling the verdict is replaced by a small final + // failure the floor can store; if that fails as well it is dead-lettered, so one verdict never gates Lease. + const n = (verdictFailures.get(p.leaseId) ?? 0) + 1 + verdictFailures.set(p.leaseId, n) + if (n < verdictAttempts) { + const wait = jitter(Math.min(cfg.outboxBackoffMs[0] * 2 ** (n - 1), cfg.outboxBackoffMs[1])) + verdictNotBefore.set(p.leaseId, Date.now() + wait) + log("outbox-keep", { leaseId: p.leaseId, kind: r.failure.kind, try: n, waitMs: wait }) + continue + } + verdictFailures.delete(p.leaseId); verdictNotBefore.delete(p.leaseId) + const rec = j.recs.get(p.leaseId) + if (rec !== undefined && !p.replace) { + const digest = createHash("sha256").update(JSON.stringify(p.output ?? null)).digest("hex") + j.append({ ev: "report", leaseId: p.leaseId, attempt: p.attempt, result: "failure", replace: true, + output: { reason: "infra/verdict-undeliverable", message: `${p.result} verdict (output sha256 ${digest}) refused ${n} times: ${r.failure.message.slice(0, 200)}` } }) + log("verdict-replaced", { leaseId: p.leaseId, result: p.result, tries: n, outputSha256: digest }) + } else { + j.append({ ev: "dropped", leaseId: p.leaseId, code: "undeliverable" }) + log("verdict-dead-letter", { leaseId: p.leaseId, tries: n, message: r.failure.message.slice(0, 200) }) + } + } else { + if (r.failure.fatal) return yield* failFatal(r.failure) + outboxWaitUntil = Date.now() + Math.min(r.failure.retryAfterMs ?? jitter(outboxBackoff), MAX_POLL_SECONDS * cfg.secondMs) + if (r.failure.retryAfterMs !== undefined) outboxNotBefore = outboxWaitUntil + outboxBackoff = Math.min(outboxBackoff * 2, cfg.outboxBackoffMs[1]) + log("outbox-keep", { leaseId: p.leaseId, kind: r.failure.kind, waitMs: outboxWaitUntil - Date.now() }) + return // keep: oldest first, stop at the first transient failure + } + } + })) + + // ---------------------------------------------------------------- ax calls + // Round 3 (token scope): a record that is only `creating` names a Task the link MAY have written. The floor chose + // that name, so before anything reads or deletes it the Task must carry this link's write: the spec digest the + // link built for this grant, or this lease id and this holder in its env. Anything else is someone else's Task. + const envMarks = (id: string, t: AxObserved) => { + const env = new Map((t.spec.env ?? []).map((e) => [e.name ?? "", e.value ?? ""])) + return env.get("AX_CONWIP_LEASE_ID") === id && env.get("AX_CONWIP_HOLDER") === cfg.holder + } + const owns = (id: string, r: LeaseRec, t: AxObserved) => + r.created !== undefined || (r.creatingDigest !== undefined && specDigest(t.spec) === r.creatingDigest) || envMarks(id, t) + /** Journal that the Task under this name is not the link's: nothing reads, deletes or reports it any more. A cancel + * already asked for is answered `cancelled`, since nothing of this lease runs in ax. */ + const markNotMine = (id: string) => Effect.gen(function*() { + const r = j.recs.get(id) + if (!r || r.notMine) return + j.append({ ev: "not-mine", leaseId: id }) + log("not-mine", { leaseId: id }) + if (!r.report && r.released === undefined && (abandoned.get(id) === "cancel" || r.deleting === "cancel")) yield* report(id, "cancelled") + }) + /** GetTask a `creating`-only record's name: owned, absent, foreign (journaled not-mine) or unknown (ax did not answer). */ + const ownership = (id: string, r: LeaseRec) => Effect.gen(function*() { + const got = yield* ax.getTask(id).pipe(Effect.result) + if (got._tag === "Failure") return "unknown" as const + if (got.success === undefined) return "absent" as const + if (owns(id, r, got.success)) return "owned" as const + yield* markNotMine(id) + return "foreign" as const + }) + + // Round 3: the intent is journaled BEFORE DeleteTask (`deleting`), so a DeleteTask that lands and whose reply is lost + // (DEADLINE_EXCEEDED) still reads as a delete on the next resync (a cancel is reported `cancelled`, not task-lost), + // and one that did not land is sent again by reconcile. NotFound means already gone. + const del = (id: string, why: string): Effect.Effect => Effect.gen(function*() { + const rec = j.recs.get(id) + if (rec?.notMine) return true + if (rec !== undefined && rec.created === undefined && rec.creating) { + const o = yield* ownership(id, rec) + if (o === "unknown") { log("delete-deferred", { task: id, why, until: "ownership" }); return false } + if (o === "foreign") return true + } + if (rec !== undefined && rec.deleting === undefined) j.append({ ev: "deleting", leaseId: id, why }) + return yield* ax.deleteTask(id).pipe( + Effect.tap(() => Effect.sync(() => log("delete", { task: id, why }))), + Effect.as(true), + Effect.catch((e: AxError) => Effect.sync(() => { + if (e.code === NOT_FOUND) { log("delete", { task: id, why, absent: true }); return true } + log("delete-deferred", { task: id, why, code: e.code }); return false + }))) + }) + + /** B2: an old attempt is gone only when GetTask answers NotFound; until then attempt n+1 is not created. */ + const fence = (old: string) => Effect.gen(function*() { + const until = Date.now() + fenceTimeoutMs + let asked = false + for (;;) { + if (j.recs.get(old)?.notMine) return true // round 3: not this link's Task; nothing of attempt n runs under it + const r = yield* ax.getTask(old).pipe(Effect.result) + if (r._tag === "Success" && r.success === undefined) { + if (j.recs.has(old) && !j.recs.get(old)!.deleted) j.append({ ev: "deleted", leaseId: old }) + return true + } + if (r._tag === "Success" && (!asked || r.success!.phase !== "Terminating")) asked = (yield* del(old, "superseded")) || asked + if (Date.now() >= until) return false + yield* Effect.sleep(Math.min(cfg.resyncMs, 2 * cfg.secondMs)) + } + }) + + /** B11: never more live Tasks than maxInFlight. Busy = live Tasks in the atespace (anyone's) U admitted leases + * not known finished. Synchronous, so two dispatch fibers cannot both take the last slot. */ + const busy = () => { + const b = new Set() + for (const [name, t] of last) if (!TERMINAL.has(t.phase)) b.add(name) + for (const id of admitted) { + const r = j.recs.get(id), t = last.get(id) + if (!r || r.report || r.released !== undefined || r.dropped !== undefined || r.deleted || (t && TERMINAL.has(t.phase))) { admitted.delete(id); b.delete(id); continue } + b.add(id) + } + return b + } + const admit = (id: string) => Effect.sync(() => { + if (!canCreate()) return false + const b = busy() + if (b.has(id) || b.size < cfg.maxInFlight) { admitted.add(id); return true } + return false + }) + + /** A grant this link will not create any more (B3): a cancel is reported once any Task that may exist is gone. */ + const abandon = (id: string, why: "cancel" | "lost", where: string) => Effect.gen(function*() { + log(where, { leaseId: id, why }) + const rec = j.recs.get(id) + if (rec && Journal.maybeCreated(rec)) return yield* del(id, why) // round 1: reconcile reports cancelled on NotFound + if (why === "cancel") yield* report(id, "cancelled") + }) + + let guestTokenWarned = false + const createOnly = (g: Grant) => Effect.gen(function*() { + const id = g.leaseId + let waited = false + let unconfirmed = false + for (;;) { // round 1: the capacity gate is re-checked before every UpdateTask; a closed gate goes back to waiting + // Round 4: and the lease must still be live (renewed within the lease) before the slot is taken + while (!leaseLive(g) || !(yield* admit(id))) { // B11: journaled and heartbeated while it waits + if (abandoned.has(id) || deadlinePassed(g)) break + if (!leaseLive(g)) { if (!unconfirmed) { log("lease-unconfirmed", { leaseId: id }); unconfirmed = true } } + else if (!waited) { log("waiting-for-slot", { leaseId: id }); waited = true } + yield* Effect.sleep(cfg.resyncMs) + } + const early = abandoned.get(id) + if (early !== undefined) return yield* abandon(id, early, "abandoned-before-create") + const now = j.recs.get(id) // round 2: withdrawn, or replaced by a newer grant of the same leaseId: never create + if (now?.grant !== g || now.released !== undefined) return log("grant-superseded-before-create", { leaseId: id }) + if (deadlinePassed(g)) return yield* failG(g, { reason: "deadline-exceeded", message: "the grant's deadline passed before its Task was created" }) // round 4 + const m = mode() + const built = axTaskFromGrant(g, { ...cfg.shape, holder: cfg.holder, ...(m === "guest" ? {} : { completeUrl: undefined }) }) + if (built._tag === "refused") { + if (built.reason === "pre-start/lease-token-missing" && !guestTokenWarned) { + guestTokenWarned = true + log("guest-token-missing", { leaseId: id, hint: "floor and link disagree: the link completes as guest, the floor mints no lease token for this holder" }) + } + return yield* failG(g, { reason: built.reason, message: built.message }) + } + const digest = specDigest(built.task.spec) + for (let i = 1; ; i++) { + const why = abandoned.get(id) // B3: checked before every UpdateTask + if (why !== undefined) return yield* abandon(id, why, "abandoned-before-create") + const cur = j.recs.get(id) + if (cur?.grant !== g || cur.released !== undefined) return log("grant-superseded-before-create", { leaseId: id }) + if (!canCreate() || mode() !== m) { log("gate-closed-before-create", { leaseId: id, try: i }); break } // round 1 + if (deadlinePassed(g)) return yield* failG(g, { reason: "deadline-exceeded", message: "the grant's deadline passed before its Task was created" }) + if (!leaseLive(g)) { log("lease-unconfirmed-before-create", { leaseId: id, try: i }); break } // round 4: back to waiting + if (!j.recs.get(id)?.creating) j.append({ ev: "creating", leaseId: id, digest }) // round 1: write-ahead; round 3: with the digest + const r = yield* Effect.gen(function*() { + const found = yield* ax.getTask(id) + if (found === undefined) { yield* ax.createTask(built.task); return "created" as const } + if (specDigest(found.spec) === digest) return "adopted" as const + return envMarks(id, found) ? "conflict" as const : "foreign" as const + }).pipe(Effect.result) + if (r._tag === "Success") { + // L4: never overwrite a Task this link did not write. Round 3: a Task without this link's marks is journaled + // not-mine first, so no cleanup path (janitor, lost, cancel, salvage) ever reads or deletes it. + if (r.success === "foreign") yield* markNotMine(id) + if (r.success === "conflict" || r.success === "foreign") + return yield* failG(g, { reason: "pre-start/name-conflict", message: "a Task with this name and another spec exists" }) + j.append({ ev: "created", leaseId: id, digest }) + log(r.success, { leaseId: id, attempt: g.attempt }) + const after = abandoned.get(id) // B3: and again once UpdateTask returned + if (after !== undefined) { yield* del(id, after); return } + const back = yield* ax.getTask(id).pipe(Effect.result) // B12: FIELD-MAP 5a read-back + if (back._tag === "Success" && back.success !== undefined && specDigest(back.success.spec) !== digest) { + yield* del(id, "pre-start") + return yield* failG(g, { reason: "pre-start/readback-mismatch", message: "ax stored another spec than the link wrote" }) + } + return + } + const e = r.failure + if (e.resourceExhausted) return yield* failG(g, { reason: "pre-start/resource-exhausted", message: e.message }) // B5 + if (!e.unavailable) return yield* failG(g, { reason: "pre-start/invalid-spec", message: e.message }) + if (i >= cfg.createAttempts) return yield* failG(g, { reason: "pre-start/ax-unavailable", message: e.message }) + yield* Effect.sleep(jitter(2 ** i * 100)) + } + } + }) + + /** Round 1: an unjournaled name may be fenced only if it is `-a` with k < this attempt and the Task's own + * env says this lease id and this holder (AX_CONWIP_HOLDER, written by this link at create). */ + const ownEarlierAttempt = (old: string, g: Grant, t: AxObserved) => { + const base = g.leaseId.replace(/-a\d+$/, ""), m = /^(.*)-a(\d+)$/.exec(old) + if (m === null || m[1] !== base || Number(m[2]) >= g.attempt) return false + return envMarks(old, t) + } + + const dispatch = (g: Grant) => Effect.gen(function*() { + if (j.recs.get(g.leaseId)?.grant === g) dispatching.set(g.leaseId, g) // round 2: only the current generation + const bad = validRunsOn(g.job, cfg.servedLabels) // B7 + if (bad) return yield* failG(g, bad) + // Final pass (VERIFY-LINK V1): the fence set is `supersedes` plus every earlier attempt of the same job this link + // journaled, that may exist in ax and is not confirmed deleted. A floor that omits `supersedes` (or a Heartbeat + // whose `lost` arrives after this grant) no longer lets attempt n+1 start beside a running attempt n. + const olds = [...new Set([...(g.supersedes ?? []), ...[...j.recs.values()] + .filter((r) => r.grant.leaseId !== g.leaseId && r.grant.job.metadata.name === g.job.metadata.name && r.grant.attempt < g.attempt + && Journal.maybeCreated(r) && r.deleted !== true && !r.notMine) + .map((r) => r.grant.leaseId)])] + if (olds.length > (g.supersedes ?? []).length) log("fence-set-extended", { leaseId: g.leaseId, supersedes: g.supersedes ?? [], fence: olds }) + // Round 2: an earlier attempt that may exist and has no verdict yet is READ before anything fences it. A Task that + // finished while its lease expired carries a result the floor can still accept (rule 4b); deleting it first loses + // the success and runs the job again. + for (const o of olds) { + const until = Date.now() + fenceTimeoutMs + for (;;) { + const s = yield* salvage(o) + if (s !== "unknown") break + const why = abandoned.get(g.leaseId) + if (why !== undefined) return yield* abandon(g.leaseId, why, "abandoned-before-create") + if (Date.now() >= until) + return yield* failG(g, { reason: "pre-start/ax-unavailable", message: `superseded attempt ${o} could not be read before its fence` }) + yield* Effect.sleep(Math.min(cfg.resyncMs, 2 * cfg.secondMs)) + } + } + // Round 1 (B15 at dispatch): an earlier attempt whose verdict is still in the outbox is delivered BEFORE it is + // fenced. The floor accepts it (rule 4b: n+1 is queued, or leased to this holder and not created) and withdraws + // this grant; deleting the finished Task first would throw its result away and rerun the job. + const undelivered = () => olds.filter((o) => { const r = j.recs.get(o); return r?.report !== undefined && !r.reported && r.dropped === undefined }) + if (undelivered().length > 0) { + log("supersede-waits-for-verdict", { leaseId: g.leaseId, old: undelivered() }) + const until = Date.now() + fenceTimeoutMs + for (;;) { + if (undelivered().length === 0) break // delivered (maybe by another fiber): decided below + const mine = j.recs.get(g.leaseId) + if (mine?.grant !== g || mine.released !== undefined) return log("grant-superseded-before-create", { leaseId: g.leaseId }) + const why = abandoned.get(g.leaseId) + if (why !== undefined) return yield* abandon(g.leaseId, why, "abandoned-before-create") + yield* drainOutbox + if (undelivered().length === 0) break + if (Date.now() >= until) + return yield* failG(g, { reason: "pre-start/superseded-verdict-pending", message: `the verdict of ${undelivered().join(",")} is not delivered yet` }) + yield* Effect.sleep(Math.min(cfg.resyncMs, 2 * cfg.secondMs)) + } + } + // Round 2: withdrawn only when the floor SAID so (Complete.withdrew names this grant). A duplicate answer (rule 4c) + // or a delivered retryable failure withdrew nothing: this grant is the retry, so fence the old attempt and create. + // Round 3: and only for the generation it withdrew (see drainOutbox). + const by = olds.find((o) => { const x = j.recs.get(o); return x?.withdrew === g.leaseId && x.withdrewGen !== undefined && sameGen(genOf(g), x.withdrewGen) }) + if (by !== undefined) { + // Round 4: only this generation's record; a regrant taken meanwhile is the floor's newer word and stays live + const cur = j.recs.get(g.leaseId) + if (cur?.grant !== g) return log("grant-superseded-before-create", { leaseId: g.leaseId }) + if (cur.released === undefined) j.append({ ev: "released", leaseId: g.leaseId, why: "withdrawn" }) + return log("withdrawn", { leaseId: g.leaseId, by }) + } + const self = j.recs.get(g.leaseId) + if (self?.grant !== g || self.released !== undefined) return log("grant-superseded-before-create", { leaseId: g.leaseId }) + for (const old of olds) { // L3 and B2: the old attempt is gone before attempt n+1 is created + const rec = j.recs.get(old) + if (rec && rec.released === undefined) j.append({ ev: "released", leaseId: old, why: "superseded" }) + if (rec && Journal.maybeCreated(rec)) { + if (!(yield* fence(old))) { + log("fence-failed", { leaseId: g.leaseId, old }) + return yield* failG(g, { reason: "pre-start/ax-unavailable", message: `superseded attempt ${old} not confirmed deleted` }) + } + continue + } + // Round 1 (token scope): a name this link has no journal record for is never deleted on the floor's word alone. + // Absent is fine. Present is fenced only when it is an earlier attempt of THIS job and the Task itself carries + // this link's marks (the journal-lost case, F8); anything else is someone else's Task and this grant is refused. + const r = yield* ax.getTask(old).pipe(Effect.result) + if (r._tag === "Failure") + return yield* failG(g, { reason: "pre-start/ax-unavailable", message: `superseded attempt ${old} not confirmed absent` }) + if (r.success !== undefined && ownEarlierAttempt(old, g, r.success)) { + log("supersedes-unjournaled-own", { leaseId: g.leaseId, old }) + if (!(yield* fence(old))) { + log("fence-failed", { leaseId: g.leaseId, old }) + return yield* failG(g, { reason: "pre-start/ax-unavailable", message: `superseded attempt ${old} not confirmed deleted` }) + } + continue + } + if (r.success !== undefined) { + log("supersedes-foreign", { leaseId: g.leaseId, old }) + return yield* failG(g, { reason: "pre-start/supersedes-foreign", message: `supersedes names ${old}, a Task this link did not create` }) + } + } + yield* createOnly(g) + }).pipe( + Effect.ensuring(Effect.sync(() => { if (dispatching.get(g.leaseId) === g) dispatching.delete(g.leaseId) })), + // L9. Round 4: a defect is not swallowed while the lease is held. The grant is failed back (infra/link-defect, + // retryable: the floor requeues it and the next attempt fences anything this one may have created); if even that + // cannot be journaled, the process dies (exit 1) and a restart resumes from the journal (B13). + Effect.catchCause((c) => Cause.hasInterruptsOnly(c) ? Effect.interrupt : Effect.gen(function*() { + log("job-fiber-died", { leaseId: g.leaseId, cause: Cause.pretty(c) }) + const rec = j.recs.get(g.leaseId) + if (rec?.grant !== g || rec.report !== undefined || rec.released !== undefined || rec.notMine) return + yield* failG(g, { reason: "infra/link-defect", message: Cause.pretty(c).slice(0, 300) }) + }).pipe(Effect.catchCause((c2) => Cause.hasInterruptsOnly(c2) ? Effect.interrupt + : Effect.sync(() => log("link-defect-fatal", { leaseId: g.leaseId, cause: Cause.pretty(c2).slice(0, 300) })).pipe( + Effect.andThen(Deferred.die(fatal, Cause.squash(c2))), Effect.asVoid))))) + + // ---------------------------------------------------------------- resync: the executor's list is the ledger + const outcomeOf = (id: string, t: AxObserved, salvage = false) => Effect.gen(function*() { + const fail = (id: string, o: FailureOutput) => report(id, "failure", o, undefined, salvage) + if (t.phase === "Completed") { + const res = yield* ax.getTaskResult(id).pipe(Effect.result) + // Round 4: UNIMPLEMENTED is the server losing P1 (a rollout), not this Task's result: close the gate at once and + // keep the Task, uncharged, to read it once P1 is back. Guest mode does not read results through ax. + if (res._tag === "Success" && res.success === "unimplemented" && cfg.completion !== "guest") { + setServerP1(false) + return log("result-read-deferred", { leaseId: id, why: "server without P1" }) + } + const unreadable = res._tag === "Failure" ? `GetTaskResult: ${res.failure.message}` + : res.success === undefined ? "no result stored" + : res.success === "unimplemented" ? "server without P1" + : res.success.digestOk === false ? "sha256 mismatch" : undefined + if (unreadable !== undefined) { // B4: a transient read is retried at the next resync, bounded + const n = (resultFailures.get(id) ?? 0) + 1 + resultFailures.set(id, n) + const permanent = res._tag === "Failure" && !res.failure.unavailable + if (!permanent && n < resultReadTries) return log("result-read-retry", { leaseId: id, try: n, why: unreadable }) + return yield* fail(id, { reason: "infra/result-unreadable", phase: t.phase, message: unreadable }) + } + const ok = (res as { success: { content: string; sha256: string } }).success + const bytes = Buffer.byteLength(ok.content, "utf8") + if (bytes > maxOutputBytes) // round 2: refused here, typed, before the floor answers SQLITE_TOOBIG for ever + return yield* fail(id, { reason: "agent/output-too-large", phase: t.phase, + message: `result ${bytes} B > ${maxOutputBytes} B (sha256 ${ok.sha256 || createHash("sha256").update(ok.content).digest("hex")})` }) + let output: unknown + try { output = JSON.parse(ok.content) } catch { return yield* fail(id, { reason: "agent/result-not-json", phase: t.phase }) } + const u = t.usage + return yield* report(id, "success", output, u ? { prompt_tokens: u.promptTokens ?? 0, completion_tokens: u.completionTokens ?? 0, tool_calls: u.toolCalls ?? 0 } : undefined, salvage) + } + const ready = t.conditions.find((c) => c.type === "Ready") + const why = `${ready?.reason ?? ""} ${ready?.message ?? ""}` + const code = /ExitCode=(\d+)/.exec(why) + const reason = /ResourceExhausted/i.test(why) ? "pre-start/resource-exhausted" // B5: a capacity race, retryable (#367) + : /ActorTemplateRejected/.test(why) ? "pre-start/actor-template-rejected" + : /CRASH/i.test(why) ? "infra/actor-crashed" : code ? "agent/exit-code" : "agent/failed" + yield* fail(id, { reason, phase: t.phase, ...(ready?.message ? { message: ready.message } : {}), ...(code ? { exitCode: Number(code[1]) } : {}) }) + }) + + /** B1: a Task missing from the list is confirmed with GetTask. `gone` is true only on NotFound. */ + const confirm = (id: string) => ax.getTask(id).pipe(Effect.map((t) => ({ t, gone: t === undefined })), Effect.orElseSucceed(() => undefined)) + + /** Round 2: before any DeleteTask of a Task that may exist and whose verdict is not journaled (heartbeat `lost`, the + * janitor, a supersede fence), GetTask it; a terminal Task has its outcome read and journaled first. `unknown` means + * ax did not answer or the result read is being retried: delete nothing yet. */ + const salvage = (id: string) => Effect.gen(function*() { + const r = j.recs.get(id) + if (!r || r.report !== undefined || !Journal.maybeCreated(r) || r.deleting !== undefined || r.deleted) return "none" as const + const t = yield* ax.getTask(id).pipe(Effect.result) + if (t._tag === "Failure") return "unknown" as const + if (t.success === undefined) return "gone" as const + if (!owns(id, r, t.success)) { yield* markNotMine(id); return "foreign" as const } // round 3: never read another's result + if (!TERMINAL.has(t.success.phase)) return "live" as const + yield* outcomeOf(id, t.success, true) + return j.recs.get(id)?.report !== undefined ? "read" as const : "unknown" as const + }) + + const reconcile = (id: string, r: LeaseRec, listed: AxObserved | undefined, now: number) => Effect.gen(function*() { + if (!Journal.maybeCreated(r)) return // not created yet: dispatch owns it (or a restart resumes it) + let t = listed + let gone = false + if (t === undefined && !r.deleted) { + const c = yield* confirm(id) + if (c === undefined) return // ax did not answer; decide nothing this round + t = c.t; gone = c.gone + } + if (!gone) absent.delete(id) + if (t !== undefined && !gone && !owns(id, r, t)) return yield* markNotMine(id) // round 3: token scope + if (r.deleting !== undefined) { + if (gone) { + j.append({ ev: "deleted", leaseId: id }) + if (r.deleting === "cancel" && !r.report && r.released === undefined) yield* report(id, "cancelled") + } else if (t !== undefined && t.phase !== "Terminating") { // round 3: a journaled DeleteTask that did not land + if (r.released !== undefined && r.report === undefined && !r.reported && r.dropped === undefined && TERMINAL.has(t.phase)) { + yield* outcomeOf(id, t, true) // a released lease's finished Task is read before it is deleted + if (j.recs.get(id)?.report === undefined) return + } + yield* del(id, r.deleting) + } + return + } + if (r.reported || r.dropped !== undefined || r.released !== undefined) { // the floor is told: janitor only + if (gone) return j.append({ ev: "deleted", leaseId: id }) + if (t === undefined) return + if (r.released !== undefined && r.report === undefined && !r.reported && r.dropped === undefined && TERMINAL.has(t.phase)) { + yield* outcomeOf(id, t, true) // round 2: a released lease's finished Task is read before the janitor deletes it + if (j.recs.get(id)?.report === undefined) return // read retried at the next resync; nothing deleted before + } + const live = !TERMINAL.has(t.phase) // B3: a told lease's sandbox that still runs goes at once + if (live || r.released !== undefined || r.dropped !== undefined || now - r.at >= cfg.deleteAfterMs) + yield* del(id, r.released ? "released" : r.dropped ? "dropped" : live ? "told-still-running" : "janitor") + return + } + if (r.created === undefined) return // round 1: `creating` only: cleanup above, otherwise dispatch or a resume owns it + if (r.report) return // verdict written, waiting for the outbox + if (gone) { + const n = (absent.get(id) ?? 0) + 1 + absent.set(id, n) + if (n < 2) return log("task-absent", { leaseId: id, resyncs: n }) // B1: two NotFound resyncs in a row + return yield* fail(id, { reason: "infra/task-lost", message: "created by this link, NotFound on two resyncs" }) + } + if (t === undefined) return + if (TERMINAL.has(t.phase)) return yield* outcomeOf(id, t) + if ((t.phase === "" || t.phase === "Pending") && now - (r.createdAt ?? r.at) > cfg.pendingTimeoutMs) { + yield* del(id, "pre-start") + return yield* fail(id, { reason: "pre-start/pending-timeout", phase: t.phase || "Pending" }) + } + const deadline = r.grant.deadline + if (t.phase === "Running" && deadline !== undefined && now > deadline + cfg.deadlineBackstopMs) { + yield* del(id, "deadline") + return yield* fail(id, { reason: "deadline-exceeded", phase: t.phase }) + } + }) + + /** Round 4: "P1 proven" is a live condition. Logged on every change; p1-missing closes the gate (capacity 0). */ + const setServerP1 = (v: boolean) => { + if (v === serverP1) return + serverP1 = v + log("completion-probe", { serverP1, completion: cfg.completion, mode: mode() ?? "none" }) + if (!v && cfg.completion !== "guest") log("p1-missing", { completion: cfg.completion }) + } + const probe = Effect.gen(function*() { // P1 adds GetTaskResult; stock v0.3.0 answers UNIMPLEMENTED + const p = yield* ax.getTaskResult("conwip-link-capability-probe").pipe(Effect.result) + if (p._tag === "Failure") return // not answered: keep what is known (undefined after an outage: capacity 0) + setServerP1(p.success !== "unimplemented") + }) + + const checkGateway = Effect.gen(function*() { // B10 + const g = yield* ax.getGateway(cfg.shape.gateway).pipe(Effect.result) + const ok = g._tag === "Success" && g.success !== undefined && !gatewayAllowsAll(g.success) + if (ok !== gatewayOk) log(ok ? "gateway-ok" : "gateway-missing", { gateway: cfg.shape.gateway, ...(g._tag === "Failure" ? { code: g.failure.code } : g.success === undefined ? { found: false } : { allowsAll: true }) }) + gatewayOk = ok + }) + + const resync = Effect.gen(function*() { + const listed = yield* listAll(ax).pipe(Effect.result) + if (listed._tag === "Failure") { + if (axUp) log("ax-down", { code: listed.failure.code }) + axUp = false + // Round 2: nothing read before the outage opens the gate after it. Guest mode does not depend on serverP1, so + // the Gateway verdict and the resync flag are cleared too; canCreate() opens only after a full resync. + resynced = false + gatewayOk = undefined + return + } + const wasDown = !axUp + if (wasDown) serverP1 = undefined // B6: nothing proven before the outage holds after it + yield* probe // round 4: on every resync, since a rollout replaces ax-server with no failed ListTasks + yield* checkGateway + last = new Map(listed.success.map((t) => [t.name, t])) + if (wasDown) log("ax-up", { tasks: listed.success.length }) + axUp = true // round 2: only once probe, Gateway and the occupancy snapshot are all from this side of the outage + const now = Date.now() + for (const [id, r] of [...j.recs]) { + if (dispatching.has(id)) continue + yield* reconcile(id, r, last.get(id), now) + } + resynced = true + }) + + // Occupancy = |live Tasks in the atespace (ours or not) U leases held and not known terminal| (limiter rebuilt from + // the executor's list, agent-stack-k8s limiter.go). A P1-terminal Task is suspended and holds no worker. + const occupancy = () => { + const b = new Set() + for (const [name, t] of last) if (!TERMINAL.has(t.phase)) b.add(name) + for (const id of j.held()) { const t = last.get(id); if (!(t && TERMINAL.has(t.phase))) b.add(id) } + return b.size + } + + // ---------------------------------------------------------------- heartbeat: level-triggered, complete held set + const heartbeat = Effect.gen(function*() { + const ids = j.held() + // Round 4: the reply is about the generations this call vouched for. A reply that lands after a regrant of the + // same leaseId (gen n+1) says nothing about it: `lost`, `cancelRequested` and `renewed` apply only while the record + // still holds the grant object that was sent. + const sent = new Map(ids.map((id) => [id, j.recs.get(id)?.grant])) + const current = (id: string) => sent.has(id) && j.recs.get(id)?.grant === sent.get(id) + const sentAt = Date.now() + const pendingRequestKey = j.pendingLeaseKey // round 4: vouch for grants under a key whose reply never landed (B8) + const r = yield* floor.heartbeat({ holderIdentity: cfg.holder, leaseIds: ids, ...(pendingRequestKey !== undefined ? { pendingRequestKey } : {}) }).pipe(Effect.result) + if (r._tag === "Failure") { + if (r.failure.fatal) { yield* failFatal(r.failure); return undefined } + log("heartbeat-error", { kind: r.failure.kind }) + return undefined + } + for (const id of r.success.renewed) { const g = sent.get(id); if (g !== undefined && current(id)) renewedAt.set(g, Math.max(renewedAt.get(g) ?? 0, sentAt)) } + for (const id of r.success.lost) { // fencing: the lease is gone, so the Task goes; no report + if (!current(id)) { log("stale-heartbeat-reply", { leaseId: id, about: "lost" }); continue } + const rec = j.recs.get(id) + pendingResume.delete(id) + if (!rec) { const x = j.invalid.get(id); if (x && x.released === undefined) j.append({ ev: "released", leaseId: id, why: "lost" }); continue } + if (rec.released !== undefined) continue + j.append({ ev: "released", leaseId: id, why: "lost" }) + abandoned.set(id, "lost") + log("lost", { leaseId: id }) + if (Journal.maybeCreated(rec)) { // round 1: `creating` counts; the janitor confirms NotFound + const s = yield* salvage(id) // round 2: a finished Task's verdict is read and journaled before the delete + if (s === "unknown") log("delete-deferred", { task: id, why: "lost", until: "verdict-read" }) // the janitor reads, then deletes + else if (s !== "gone" && s !== "foreign") yield* del(id, "lost") + } + } + for (const id of r.success.cancelRequested) { // Temporal cancel_requested; ax cancel is DeleteTask, two-phase + if (!current(id)) { log("stale-heartbeat-reply", { leaseId: id, about: "cancel" }); continue } + const rec = j.recs.get(id) + if (!rec || rec.report || rec.deleting !== undefined) continue + log("cancel", { leaseId: id }) + pendingResume.delete(id) + abandoned.set(id, "cancel") // B3: a running dispatch reports or deletes; no later resume creates it + if (dispatching.has(id)) continue + // Round 1: a Task that may exist (`creating`) is deleted and seen NotFound before `cancelled` is reported + if (Journal.maybeCreated(rec)) yield* del(id, "cancel") + else yield* report(id, "cancelled") + } + const renewed = new Set(r.success.renewed) + for (const id of [...pendingResume]) { // B13: resume only what the floor still says is ours + if (!renewed.has(id) || !current(id)) continue + pendingResume.delete(id) + const rec = j.recs.get(id) + if (rec && rec.created === undefined && !rec.report && rec.released === undefined && rec.dropped === undefined) { + log("resume", { leaseId: id }) + yield* Effect.forkScoped(dispatch(rec.grant)) + } + } + return renewed + }) + + // ---------------------------------------------------------------- the loops + const every = (name: string, ms: () => number, body: Effect.Effect) => Effect.forever( + Effect.suspend(() => Effect.sleep(ms())).pipe(Effect.andThen(body), // re-read: the floor may change the interval + Effect.catchCause((c) => Cause.hasInterruptsOnly(c) ? Effect.interrupt : Effect.sync(() => log(`${name}-died`, { cause: Cause.pretty(c) }))))) + + /** Round 3: the heartbeat interval, at most a third of the shortest lease held, so a large floor value can never let + * a held lease expire between two heartbeats. */ + const hbSeconds = () => { + let hi = MAX_HEARTBEAT_SECONDS + for (const id of j.held()) { + const d = j.recs.get(id)?.grant.lease.leaseDurationSeconds + if (d !== undefined && Number.isFinite(d) && d > 0) hi = Math.min(hi, d / 3) + } + return boundedSeconds(hbAsked, defaultHb, Math.max(1, hi)) + } + + /** Final pass (VERIFY-LINK R3-1). The heartbeat sleep is measured from the last heartbeat and is cut short when a + * Lease reply shrinks the interval or journals a grant with a shorter lease: a sleep begun at the 30 s default while + * nothing was held no longer outlives a 12 s lease granted during it. */ + let lastBeatAt = Date.now() + const heartbeatLoop = Effect.forever(Effect.gen(function*() { + let target = jitter(hbSeconds() * cfg.secondMs) + for (;;) { + const left = lastBeatAt + target - Date.now() + if (left <= 0) break + const w = hbWake + yield* Effect.raceFirst(Effect.sleep(left), Deferred.await(w)) + if (yield* Deferred.isDone(w)) { + hbWake = yield* Deferred.make() + target = Math.min(target, jitter(hbSeconds() * cfg.secondMs)) + } + } + lastBeatAt = Date.now() + yield* heartbeat + }).pipe(Effect.catchCause((c) => Cause.hasInterruptsOnly(c) ? Effect.interrupt : Effect.sync(() => log("heartbeat-died", { cause: Cause.pretty(c) }))))) + + let endpointSeen: string | undefined + const leaseOnce = Effect.gen(function*() { + outboxWaitUntil = Math.min(outboxWaitUntil, Date.now()) // B15: a verdict goes out before new work comes in + yield* drainOutbox + // Round 1 (B15): while any verdict is still undelivered, ask for nothing. A grant that arrived now could supersede + // that verdict's attempt, and once n+1 is leased the floor answers n's verdict stale-attempt. + // Round 2: a verdict the floor has answered with a server error is not an outage; it no longer gates Lease (its + // own budget replaces or dead-letters it, and a grant that supersedes it waits at dispatch, bounded). + const undelivered = j.outbox().filter((p) => !verdictFailures.has(p.leaseId)).length + if (undelivered > 0) log("lease-gated-by-outbox", { undelivered }) + const free = undelivered === 0 && canCreate() ? Math.max(0, cfg.maxInFlight - occupancy()) : 0 + // B8: a key is journaled before the call and reused until a reply is journaled, so a lost reply is replayed, + // not stranded. A capacity-0 call can grant nothing, so it is not journaled. + let requestKey = j.pendingLeaseKey + if (requestKey === undefined) { + requestKey = randomUUID() + if (free > 0) j.append({ ev: "lease-key", requestKey }) + } + const sentAt = Date.now() + const r = yield* callLease(free, requestKey) + if (r._tag === "Failure") { + if (r.failure.fatal) return yield* failFatal(r.failure) + leaseFailed(r.failure, requestKey) + // Round 3: bounded, and only this loop waits; the heartbeat and resync fibers run on their own intervals + return yield* Effect.sleep(Math.min(r.failure.retryAfterMs ?? jitter(pollSeconds * cfg.secondMs), MAX_POLL_SECONDS * cfg.secondMs)) + } + yield* applyLease(r.success, requestKey, sentAt) + }) + + const callLease = (capacity: number, requestKey: string) => floor.lease({ holderIdentity: cfg.holder, capacity, requestKey }).pipe( + Effect.retry({ times: 2, while: (e: FloorError) => e.kind === "transient", schedule: Schedule.exponential(Math.max(1, cfg.secondMs / 5)).pipe(Schedule.jittered) }), Effect.result) + const leaseFailed = (e: FloorError, requestKey: string) => { + log("lease-error", { kind: e.kind, pendingKey: j.pendingLeaseKey !== undefined, message: e.message.slice(0, 200) }) + // Round 3: a journaled key is replayed through outages (B8), but not through replies the floor keeps answering + // with a server error or that do not decode: those would strand the key's grants for ever. After + // `leaseKeyAttempts` in a row the key is abandoned; the floor's expiry requeues what it granted under it. + if (e.kind === "server-error" && j.pendingLeaseKey === requestKey) { + if (++leaseKeyFailures >= leaseKeyAttempts) { + j.append({ ev: "lease-replied", requestKey, abandoned: true }) + log("lease-key-abandoned", { tries: leaseKeyFailures, message: e.message.slice(0, 200) }) + leaseKeyFailures = 0 + } + } + } + /** Round 4 (B8 at startup): a key journaled by a previous process whose reply never landed is replayed BEFORE the + * first Heartbeat, so its grants are journaled and in the held set when the Heartbeat re-adopts orphans; replayed + * after it, an orphan granted under the key is omitted and released (L3 a). Capacity 0: a key the floor never saw + * grants nothing new. A failed replay leaves the key pending; the Heartbeat then vouches for it. */ + const replayPendingLease = Effect.gen(function*() { + const requestKey = j.pendingLeaseKey + if (requestKey === undefined) return + const sentAt = Date.now() + const r = yield* callLease(0, requestKey) + if (r._tag === "Failure") { + if (r.failure.fatal) return yield* failFatal(r.failure) + return leaseFailed(r.failure, requestKey) + } + log("lease-key-replayed", { grants: r.success.grants.length }) + yield* applyLease(r.success, requestKey, sentAt) + }) + + const applyLease = (reply: LeaseReply, requestKey: string, sentAt: number) => Effect.gen(function*() { + leaseKeyFailures = 0 + // Round 3: the server-driven intervals are bounded (Buildkite falls back to its defaults): 0, a negative or a + // missing value would make both loops tight loops of billed requests. + pollSeconds = boundedSeconds(reply.nextPollSeconds, defaultPoll, MAX_POLL_SECONDS) + hbAsked = reply.heartbeatSeconds + const hbNow = hbSeconds() + const clamp = `${reply.nextPollSeconds}->${pollSeconds} ${reply.heartbeatSeconds}->${hbNow}` + if ((pollSeconds !== reply.nextPollSeconds || hbNow !== reply.heartbeatSeconds) && clamp !== clampLogged) { + clampLogged = clamp + log("interval-clamped", { nextPollSeconds: reply.nextPollSeconds ?? null, usedPoll: pollSeconds, heartbeatSeconds: reply.heartbeatSeconds ?? null, usedHeartbeat: hbNow }) + } + for (const g of reply.grants) { + const prior = j.recs.get(g.leaseId) + // Round 3: a grant of a lease this link finished (compacted away since) is at-least-once redelivery, not work. + const tomb = j.tombs.get(g.leaseId) + if (prior === undefined && tomb !== undefined && !laterGen(genOf(g), tomb.gen)) { log("duplicate-grant", { leaseId: g.leaseId, finished: true }); continue } + if (prior !== undefined) { + // The floor already said it withdrew this generation (Complete.withdrew journaled on an earlier attempt) but its + // dispatch has not run yet: release it now, so its regrant is not taken for a duplicate. + const pg = genOf(prior.grant) + if (prior.released === undefined && !Journal.maybeCreated(prior) && [...j.recs.values()].some((x) => x.withdrew === g.leaseId && x.withdrewGen !== undefined && sameGen(pg, x.withdrewGen))) { + j.append({ ev: "released", leaseId: g.leaseId, why: "withdrawn" }) + log("withdrawn", { leaseId: g.leaseId, by: "journal" }) + } + // Round 2: dedupe on the grant's identity, not its leaseId. A floor that withdrew n+1 (rule 4b) and then requeued + // it for n's retryable verdict grants the same leaseId again with a new acquireTime and leaseTransitions; that + // is a new grant generation, dispatched, when the old record is released and holds nothing in ax or the outbox. + const newer = laterGen(genOf(g), pg) // round 3: strictly later, so a stale redelivery is never a regrant + // Round 4: the floor's generation is authoritative. A strictly later grant means the floor already withdrew + // the one held (its Complete reply may still be in flight, or lost). If the held one never reached ax and owes + // no verdict, it is released now and the regrant taken; the floor does not redeliver it. Its dispatch sees the + // record change and stops (grant-superseded-before-create). + if (newer && prior.released === undefined && !Journal.maybeCreated(prior) && !Journal.pending(prior)) { + j.append({ ev: "released", leaseId: g.leaseId, why: "superseded-by-regrant" }) + log("superseded-by-regrant", { leaseId: g.leaseId, was: pg.leaseTransitions, now: g.lease.leaseTransitions }) + } + const reusable = prior.released !== undefined && !Journal.pending(prior) && (!Journal.maybeCreated(prior) || prior.deleted === true) + if (!(newer && reusable)) { log("duplicate-grant", { leaseId: g.leaseId }); continue } // at-least-once delivery + for (const m of [abandoned, absent, resultFailures, verdictFailures, verdictNotBefore]) m.delete(g.leaseId) + admitted.delete(g.leaseId); pendingResume.delete(g.leaseId) + j.append({ ev: "grant", leaseId: g.leaseId, grant: g, regrant: true }) + log("regrant", { leaseId: g.leaseId, was: prior.released }) + } else j.append({ ev: "grant", leaseId: g.leaseId, grant: g }) + renewedAt.set(g, sentAt) // round 4: the floor acquired it no earlier than this call was sent + log("leased", { leaseId: g.leaseId, attempt: g.attempt, supersedes: g.supersedes ?? [] }) + yield* Effect.forkScoped(dispatch(g)) + } + for (const b of reply.invalid ?? []) { // round 1: refuse one bad grant, keep the rest of the reply + if (b.leaseId === undefined || b.attempt === undefined) { log("grant-undecodable", { message: b.message }); continue } // the floor's expiry frees it + if (j.recs.has(b.leaseId) || j.invalid.has(b.leaseId)) continue + j.append({ ev: "grant-invalid", leaseId: b.leaseId, attempt: b.attempt, message: b.message }) + log("grant-invalid", { leaseId: b.leaseId, attempt: b.attempt }) + yield* Deferred.succeed(wake, undefined) + } + if (j.pendingLeaseKey === requestKey) j.append({ ev: "lease-replied", requestKey }) + yield* Deferred.succeed(hbWake, undefined) // final pass (R3-1): the interval or the held set may have shrunk + const ep = reply.endpoint // B18: only an exact match in the Nix-declared list, and only once + if (ep !== undefined && ep !== deps.floorUrl && ep !== endpointSeen) { + endpointSeen = ep + if ((cfg.floorUrls ?? []).includes(ep) && ep.startsWith("https://")) { log("endpoint-accepted", { endpoint: ep }); deps.onEndpoint?.(ep) } + else log("endpoint-ignored", { endpoint: ep }) + } + }) + + const body = Effect.gen(function*() { + yield* resync // limiter and outcomes rebuilt before the first heartbeat or lease + for (const [id, r] of j.recs) // grants journaled by a previous process but never created (B13: resumed on renewal) + if (r.created === undefined && !r.report && r.released === undefined && r.dropped === undefined && !r.notMine) pendingResume.add(id) + yield* replayPendingLease // round 4: a key left pending by a kill is answered before the first Heartbeat + yield* heartbeat // vouch first: re-adopt or release orphans (L3), then resume only renewed grants + lastBeatAt = Date.now() + // Round 3: renewal and resync start now, before any Complete or Lease can stall (a 429 Retry-After, a timeout + // chain): held leases keep being renewed whatever the first Lease does. Both re-read their interval every time. + yield* Effect.forkScoped(heartbeatLoop) + yield* Effect.forkScoped(every("resync", () => cfg.resyncMs, resync.pipe(Effect.andThen(drainOutbox)))) + yield* drainOutbox // replay before new work (uplink) + yield* Effect.raceFirst(leaseOnce, Deferred.await(deps.stop)) // its reply sets the server-driven intervals + while (!(yield* Deferred.isDone(deps.stop))) { + const w = wake + yield* Effect.raceFirst(Effect.raceFirst(Effect.suspend(() => Effect.sleep(jitter(pollSeconds * cfg.secondMs))), Deferred.await(w)), Deferred.await(deps.stop)) + if (yield* Deferred.isDone(deps.stop)) break + if (yield* Deferred.isDone(w)) wake = yield* Deferred.make() + yield* Effect.raceFirst(leaseOnce, Deferred.await(deps.stop)) + } + // drain (Buildkite stop before disconnect): no more leases; ax Tasks keep running and are re-adopted on restart + outboxWaitUntil = 0 + yield* drainOutbox + log("drained", { held: j.held().length }) + }) + + return yield* Effect.raceFirst(Effect.scoped(body), Deferred.await(fatal)) +}) diff --git a/pkgs/substrate-apps/src/apps/link/src/main.ts b/pkgs/substrate-apps/src/apps/link/src/main.ts new file mode 100644 index 00000000..fa4306e5 --- /dev/null +++ b/pkgs/substrate-apps/src/apps/link/src/main.ts @@ -0,0 +1,105 @@ +// conwip-link: the NAS-side process. Configuration is environment only (the NixOS module sets it); the bearer is a +// file under $CREDENTIALS_DIRECTORY (systemd LoadCredential from the agenix secret), read once and never logged. +// Exit codes: 0 drained on SIGTERM; 75 another session holds this identity (L5), or an allowed endpoint switch (B18) +// was persisted and the unit should restart onto it; 78 the floor refused the token, answered a redirect (round 2), or +// the configuration is invalid. +import { randomUUID } from "node:crypto" +import { existsSync, readFileSync, writeFileSync } from "node:fs" +import { join } from "node:path" +import { Deferred, Effect } from "effect" +import { grpcAx } from "./ax.ts" +import { ConfigInvalid, readLinkEnv } from "./config.ts" +import type { LinkEnv } from "./config.ts" +import { rpcFloor } from "./floor.ts" +import { seatOf } from "./jobs.ts" +import { Journal } from "./journal.ts" +import { runLink } from "./link.ts" + +const env = (k: string, d?: string) => { + const v = process.env[k] + if (v !== undefined && v !== "") return v + if (d !== undefined) return d + console.error(JSON.stringify({ ev: "config-missing", key: k })) + process.exit(78) +} +const log = (ev: string, f: Record = {}) => console.log(JSON.stringify({ t: new Date().toISOString(), ev, ...f })) +// Round 1: every numeric and enum key is validated up front; a bad one is exit 78 naming the key, never the value. +let c: LinkEnv +try { c = readLinkEnv(process.env) } catch (e) { + if (e instanceof ConfigInvalid) { console.error(JSON.stringify({ ev: "config-invalid", key: e.key, why: e.why })); process.exit(78) } + throw e +} + +const stateDir = env("LINK_STATE_DIR", process.env.STATE_DIRECTORY ?? "/var/lib/conwip-link") +const tokenPath = env("LINK_TOKEN_FILE", join(process.env.CREDENTIALS_DIRECTORY ?? "/nonexistent", "floor-link-token")) +if (!existsSync(tokenPath)) { log("token-missing", { path: tokenPath }); process.exit(78) } +const token = readFileSync(tokenPath, "utf8").trim() +const sessionPath = join(stateDir, "session-id") // stable across restarts of this unit, distinct for a second replica +const journal = Journal.open(stateDir) +if (!existsSync(sessionPath)) writeFileSync(sessionPath, randomUUID() + "\n", { mode: 0o600 }) +const sessionId = readFileSync(sessionPath, "utf8").trim() + +// B18: the floor URLs this link may use, declared in Nix. A switch the floor asked for is persisted here and used on +// the next start only while it is still in the declared list. +const declaredUrl = env("LINK_FLOOR_URL") +const floorUrls = (process.env.LINK_FLOOR_URLS ?? declaredUrl).split(",").map((u) => u.trim()).filter((u) => u !== "") +const endpointPath = join(stateDir, "floor-endpoint") +const persisted = existsSync(endpointPath) ? readFileSync(endpointPath, "utf8").trim() : "" +const floorUrl = persisted !== "" && floorUrls.includes(persisted) ? persisted : declaredUrl +let restartForEndpoint = false + +// Round 2: the link loads its own P1 proto (apps/link/proto/ax-p1.proto), never the stock one: with the stock proto +// GetTaskResult is answered "unimplemented" locally and completion auto/p1 would lease nothing for ever. +let ax: ReturnType +try { ax = grpcAx(c.axServer, c.atespace, undefined, c.axProtoPath) } catch (e) { + console.error(JSON.stringify({ ev: "config-invalid", key: "LINK_AX_PROTO_PATH", why: String((e as Error)?.message ?? e).slice(0, 200) })) + process.exit(78) +} +const program = Effect.gen(function*() { + const stop = yield* Deferred.make() + for (const sig of ["SIGTERM", "SIGINT"] as const) + process.once(sig, () => { log("stop-requested", { signal: sig }); Effect.runFork(Deferred.succeed(stop, undefined)) }) + const floor = yield* rpcFloor({ url: floorUrl, token, sessionId }) + yield* runLink({ + holder: c.holder, + maxInFlight: c.maxInFlight, + servedLabels: c.servedLabels, + shape: { + atespace: c.atespace, + image: c.image, + gateway: c.gateway, + command: (job) => { const seat = seatOf(job); return seat === undefined ? undefined : c.seatCommands[seat] }, // B7: no default seat + ...(c.guestCompleteUrl !== undefined ? { completeUrl: c.guestCompleteUrl } : {}) + }, + completion: c.completion, + secondMs: 1000, + resyncMs: c.resyncSeconds * 1000, + pendingTimeoutMs: c.pendingTimeoutSeconds * 1000, + deleteAfterMs: c.deleteAfterSeconds * 1000, + deadlineBackstopMs: c.deadlineBackstopSeconds * 1000, + createAttempts: c.createAttempts, + outboxBackoffMs: [c.outboxBackoffSeconds[0] * 1000, c.outboxBackoffSeconds[1] * 1000], + fenceTimeoutMs: c.fenceTimeoutSeconds * 1000, + resultReadTries: c.resultReadTries, + verdictAttempts: c.verdictAttempts, + maxOutputBytes: c.maxOutputBytes, + internalHosts: c.internalHosts, + floorUrls + }, { + ax, floor, journal, log, stop, floorUrl, + onEndpoint: (url) => { // B18: persist, drain, exit 75 so systemd restarts onto the new URL + writeFileSync(endpointPath, url + "\n", { mode: 0o600 }) + restartForEndpoint = true + Effect.runFork(Deferred.succeed(stop, undefined)) + } + }) +}) + +Effect.runPromise(Effect.scoped(program)).then( + () => { ax.close(); process.exit(restartForEndpoint ? 75 : 0) }, + (e) => { + ax.close() + const kind = (e as { kind?: string })?.kind + log("fatal", { kind: kind ?? "defect", message: String((e as { message?: string })?.message ?? e) }) + process.exit(kind === "session-conflict" ? 75 : kind === "auth" || kind === "redirect" || kind === "misrouted" ? 78 : 1) + }) diff --git a/pkgs/substrate-apps/src/apps/link/test/ax-grpc.test.ts b/pkgs/substrate-apps/src/apps/link/test/ax-grpc.test.ts new file mode 100644 index 00000000..32a137f3 --- /dev/null +++ b/pkgs/substrate-apps/src/apps/link/test/ax-grpc.test.ts @@ -0,0 +1,91 @@ +// The production AxApi against a real gRPC server loaded from the vendored ax.proto (v0.3.0 d8ed0fe), on loopback. +// Proves the wire mapping the doubles skip: protojson field names survive UpdateTask, NotFound maps to undefined, +// DeleteTask NotFound is success. Round 2: the link loads its own P1 proto (apps/link/proto/ax-p1.proto), so the +// capability probe is decided by the SERVER: a stock server answers UNIMPLEMENTED over the wire (pre-P1), a P1 server +// answers; a proto without GetTaskResult is refused at construction instead of answering "unimplemented" locally. +import { createHash } from "node:crypto" +import * as grpc from "@grpc/grpc-js" +import * as protoLoader from "@grpc/proto-loader" +import { Effect } from "effect" +import { afterAll, beforeAll, expect, it } from "vitest" +import { resolveProtoPath, VENDORED_PROTO_PATH } from "../../../src/axclient.ts" +import { gatewayAllowsAll, grpcAx, LINK_PROTO_PATH, listAll } from "../src/ax.ts" +import { axTaskFromGrant } from "../src/jobs.ts" + +const stored = new Map() +const gateways = new Map([ + ["halogen", { apiVersion: "ax.io/v1alpha1", kind: "Gateway", metadata: { name: "halogen", atespace: "fleet" }, spec: { egress: { allowlist: { hosts: [{ host: "worker", port: 8731 }] } } } }], + ["open", { apiVersion: "ax.io/v1alpha1", kind: "Gateway", metadata: { name: "open", atespace: "fleet" }, spec: {} }] +]) +let server: grpc.Server, port = 0 +const nf = (name: string) => ({ code: grpc.status.NOT_FOUND, details: `task "${name}" not found` }) + +beforeAll(async () => { + const def = protoLoader.loadSync(resolveProtoPath(), { keepCase: false, longs: String, enums: String, defaults: true, oneofs: true }) + const svc = (grpc.loadPackageDefinition(def) as any).ax.v1alpha1.AX.service + server = new grpc.Server() + const impl: Record> = { + GetTask: (c, cb) => { const t = stored.get(c.request.name); t ? cb(null, t) : cb(nf(c.request.name)) }, + UpdateTask: (c, cb) => { stored.set(c.request.task.metadata.name, { ...c.request.task, status: { phase: "Pending" } }); cb(null, c.request.task) }, + // upstream server.go:91-104: limit <= 0 means 50, newest first (store.go ZRevRange) + ListTasks: (c, cb) => { const n = Number(c.request.limit) > 0 ? Number(c.request.limit) : 50, o = Number(c.request.offset) || 0; cb(null, { tasks: [...stored.values()].reverse().slice(o, o + n) }) }, + GetGateway: (c, cb) => { const g = gateways.get(c.request.name); g ? cb(null, g) : cb({ code: grpc.status.NOT_FOUND, details: "gateway not found" }) }, + DeleteTask: (c, cb) => stored.delete(c.request.name) ? cb(null, {}) : cb(nf(c.request.name)) + } + server.addService(svc, impl) + port = await new Promise((res, rej) => server.bindAsync("127.0.0.1:0", grpc.ServerCredentials.createInsecure(), (e, p) => e ? rej(e) : res(p))) +}) +afterAll(() => { server.forceShutdown() }) + +it("G1 grpcAx speaks ax.proto: create-only round trip, NotFound as undefined, a stock server answers GetTaskResult UNIMPLEMENTED", async () => { + const ax = grpcAx(`127.0.0.1:${port}`, "fleet", 3000) + const job = { + apiVersion: "ultracode.mecattaf.dev/v1alpha1" as const, kind: "AgentJob" as const, + metadata: { name: "wf-g-1-abababab", annotations: { "ultracode.mecattaf.dev/item-key": "wf_g#1", "ultracode.mecattaf.dev/journal-key": `${"cd".repeat(32)}:1` } }, + spec: { "runs-on": ["seat:halogen"], with: { prompt: "hi", prompt_ref: { sha256: "ab".repeat(32), bytes: 2, uri: "journal://wf_g/1/prompt.md" }, model: "halogen-qwen3.8-flash-next" } } + } + const built = axTaskFromGrant({ leaseId: "wf-g-1-abababab-a1", attempt: 1, job, lease: { holderIdentity: "nas-link-1", leaseDurationSeconds: 90, acquireTime: 0, renewTime: 0, leaseTransitions: 0 } }, + { atespace: "fleet", image: "localhost:5000/ax-agent@sha256:00", gateway: "halogen", command: () => ["ax-agent", "pi"] }) + if (built._tag !== "ok") throw new Error(built.reason) + const run = (e: Effect.Effect) => Effect.runPromise(e) + expect(await run(ax.getTask("wf-g-1-abababab-a1"))).toBeUndefined() + await run(ax.createTask(built.task)) + const back = stored.get("wf-g-1-abababab-a1") + expect([back.apiVersion, back.metadata.atespace, back.spec.gateway.name, back.spec.command]).toEqual(["ax.io/v1alpha1", "fleet", "halogen", ["ax-agent", "pi"]]) + expect(back.spec.env.find((e: any) => e.name === "AX_CONWIP_ITEM_KEY").value).toBe("wf_g#1") + const seen = await run(ax.getTask("wf-g-1-abababab-a1")) + expect(seen?.phase).toBe("Pending") + expect((await run(ax.listTasks(0, 0))).map((t) => t.name)).toEqual(["wf-g-1-abababab-a1"]) + for (let i = 0; i < 120; i++) stored.set(`bulk-${i}`, { apiVersion: "ax.io/v1alpha1", kind: "Task", metadata: { name: `bulk-${i}`, atespace: "fleet" }, spec: {}, status: { phase: "Completed" } }) + expect((await run(ax.listTasks(0, 0))).length).toBe(50) // B1: one call is one 50-row page + const all = await run(listAll(ax)) + expect([all.length, all.some((t) => t.name === "wf-g-1-abababab-a1")]).toEqual([121, true]) + for (let i = 0; i < 120; i++) stored.delete(`bulk-${i}`) + const gw = await run(ax.getGateway("halogen")) // B10 wire mapping + expect([gw?.hasAllowlist, gw?.hosts, gatewayAllowsAll(gw!)]).toEqual([true, [{ host: "worker", port: 8731 }], false]) + const open = await run(ax.getGateway("open")) + expect([open?.hasAllowlist, gatewayAllowsAll(open!)]).toEqual([false, true]) // no egress: ax applies *:443 + expect(await run(ax.getGateway("missing"))).toBeUndefined() + await run(ax.deleteTask("wf-g-1-abababab-a1")) + await run(ax.deleteTask("wf-g-1-abababab-a1")) // NotFound is success: delete is idempotent for the janitor + expect(await run(ax.getTaskResult("anything"))).toBe("unimplemented") // v0.3.0 server has no GetTaskResult: pre-P1 mode + ax.close() +}) + +it("R2-10 the packaged link (default proto) proves P1 against a P1 server and reads a result; a stock proto is refused", async () => { + let calls = 0 + const def = protoLoader.loadSync(LINK_PROTO_PATH, { keepCase: false, longs: String, enums: String, defaults: true, oneofs: true }) + const svc = (grpc.loadPackageDefinition(def) as any).ax.v1alpha1.AX.service + const p1 = new grpc.Server() + const body = Buffer.from('{"ok":true}') + p1.addService(svc, { + GetTaskResult: (c: any, cb: any) => { calls++; c.request.name === "t1" ? cb(null, { content: body, sha256: createHash("sha256").update(body).digest("hex") }) : cb({ code: grpc.status.NOT_FOUND, details: "nf" }) } + }) + const p1Port = await new Promise((res, rej) => p1.bindAsync("127.0.0.1:0", grpc.ServerCredentials.createInsecure(), (e, p) => e ? rej(e) : res(p))) + const ax = grpcAx(`127.0.0.1:${p1Port}`, "fleet", 3000) // exactly what main.ts builds when LINK_AX_PROTO_PATH is unset + expect(await Effect.runPromise(ax.getTaskResult("conwip-link-capability-probe"))).toBeUndefined() // NotFound: P1 present + expect(await Effect.runPromise(ax.getTaskResult("t1"))).toEqual({ content: '{"ok":true}', sha256: createHash("sha256").update(body).digest("hex"), digestOk: true }) + expect(calls).toBe(2) + ax.close(); p1.forceShutdown() + expect(() => grpcAx("127.0.0.1:1", "fleet", 1000, resolveProtoPath({ AX_CONWIP_PROTO_PATH: VENDORED_PROTO_PATH }))).toThrow(/GetTaskResult/) +}) diff --git a/pkgs/substrate-apps/src/apps/link/test/critique-fixed.test.ts b/pkgs/substrate-apps/src/apps/link/test/critique-fixed.test.ts new file mode 100644 index 00000000..12a7799a --- /dev/null +++ b/pkgs/substrate-apps/src/apps/link/test/critique-fixed.test.ts @@ -0,0 +1,406 @@ +// The nine defects the 2026-09-23 critique reproduced (LINK-DESIGN.md "Critique applied", section B), each INVERTED +// into the fixed behaviour now that its build item is in (commit history: the characterization versions are in +// 6342476 and b900624). Plus the new cases the build list names for B1, B6, B9 to B13, B15, B17 and B18. +// Same doubles as link.test.ts; the floor double implements section H's amended rules (B16). +import { mkdtempSync, rmSync } from "node:fs" +import { tmpdir } from "node:os" +import { join } from "node:path" +import { Deferred, Effect, Fiber } from "effect" +import { afterEach, beforeEach, describe, expect, it } from "vitest" +import { AxError } from "../src/ax.ts" +import type { AgentJob, AxTask } from "../src/contract.ts" +import { rpcFloor } from "../src/floor.ts" +import type { FloorOptions } from "../src/floor.ts" +import { Journal } from "../src/journal.ts" +import { isFleetInternalUrl, runLink } from "../src/link.ts" +import type { LinkConfig } from "../src/link.ts" +import { FakeAx } from "./fake-ax.ts" +import { FakeFloor } from "./fake-floor.ts" +import type { FloorConfig } from "./fake-floor.ts" + +const SEC = 20 +const UNAVAILABLE = 14 +const A = "ultracode.mecattaf.dev/" +const JK = `${"cd".repeat(32)}:1` // round 4: FIELD-MAP 5a journal key (jobs.ts refuses a grant without it) +const job = (n: string, spec: Partial = {}): AgentJob => ({ + apiVersion: "ultracode.mecattaf.dev/v1alpha1", kind: "AgentJob", + metadata: { + name: `wf-test-${n}`, + labels: { [A + "run-id"]: "wf-test", [A + "workflow"]: "link-test", [A + "phase-index"]: "1" }, + annotations: { [A + "run-id-raw"]: "wf_test", [A + "label"]: `probe:${n}`, [A + "item-key"]: `wf_test#${n}`, [A + "journal-key"]: JK, [A + "phase-title"]: "Probe" } + }, + spec: { + "runs-on": ["seat:halogen", "runtime:gvisor"], + with: { prompt: `say ${n}`, prompt_ref: { sha256: "ab".repeat(32), bytes: 5, uri: `journal://wf_test/${n}/prompt.md` }, model: "halogen-qwen3.8-flash-next" }, + ...spec + } +}) +const world = (o: Partial = {}) => ({ + floor: new FakeFloor({ cap: 2, leaseSeconds: 6, graceSeconds: 15, pollSeconds: 1, heartbeatSeconds: 2, maxAttempts: 3, secondMs: SEC, tokens: { "tok-nas": "nas-link-1" }, ...o }), + ax: new FakeAx() +}) +type World = ReturnType +const foreign = (name: string): AxTask => ({ apiVersion: "ax.io/v1alpha1", kind: "Task", metadata: { name, atespace: "fleet" }, spec: { image: "someone-else", command: ["true"], env: [] } }) + +interface Extra { fetch?: typeof globalThis.fetch; timeoutsMs?: FloorOptions["timeoutsMs"]; onEndpoint?: (u: string) => void } +function start(w: World, dir: string, over: Partial = {}, x: Extra = {}) { + const logs: Array> = [] + const stop = Effect.runSync(Deferred.make()) + const cfg: LinkConfig = { + holder: "nas-link-1", maxInFlight: 2, servedLabels: ["seat:halogen", "runtime:gvisor"], + shape: { atespace: "fleet", image: "ax-agent", gateway: "halogen", command: () => ["ax-agent", "pi"] }, + completion: "auto", secondMs: SEC, resyncMs: 30, pendingTimeoutMs: 1500, deleteAfterMs: 0, deadlineBackstopMs: 300, + createAttempts: 3, outboxBackoffMs: [20, 100], initialPollSeconds: 1, initialHeartbeatSeconds: 2, fenceTimeoutMs: 300, resultReadTries: 3, ...over + } + const fiber = Effect.runFork(Effect.scoped(Effect.gen(function*() { + const floor = yield* rpcFloor({ url: "http://floor.test", token: "tok-nas", sessionId: `s:${dir}`, fetch: x.fetch ?? w.floor.fetch, ...(x.timeoutsMs ? { timeoutsMs: x.timeoutsMs } : {}) }) + return yield* runLink(cfg, { ax: w.ax, floor, journal: Journal.open(dir), log: (ev, f) => logs.push({ ev, ...f }), stop, floorUrl: "http://floor.test", ...(x.onEndpoint ? { onEndpoint: x.onEndpoint } : {}) }) + }))) + return { + logs, + has: (ev: string, leaseId?: string) => logs.some((l) => l.ev === ev && (leaseId === undefined || l.leaseId === leaseId)), + verdict: (reason: string) => logs.some((l) => l.ev === "verdict" && l.reason === reason), + crash: () => Effect.runPromise(Fiber.interrupt(fiber)), + drain: () => { Effect.runSync(Deferred.succeed(stop, undefined)); return Promise.race([Effect.runPromise(Fiber.await(fiber)), sleep(1500)]) } + } +} +const until = async (what: string, pred: () => boolean, ms = 5000) => { + const t0 = Date.now() + while (!pred()) { if (Date.now() - t0 > ms) throw new Error(`timeout waiting for: ${what}`); await new Promise((r) => setTimeout(r, 5)) } +} +const sleep = (ms: number) => new Promise((r) => setTimeout(r, ms)) +const J = (w: World, n = "1") => w.floor.jobs.get(`wf-test-${n}`)! +const liveOwn = (w: World) => [...w.ax.tasks.keys()].filter((k) => k.startsWith("wf-test-") && w.ax.live(k) && !["Completed", "Failed"].includes(w.ax.tasks.get(k)!.phase)).length + +describe("critique 2026-09-23: the defects, fixed", () => { + let dir: string + let worlds: Array = [] + const W = (o: Partial = {}) => { const w = world(o); worlds.push(w); return w } + beforeEach(() => { dir = mkdtempSync(join(tmpdir(), "conwip-link-critique-")) }) + afterEach(() => { for (const w of worlds) w.floor.close(); worlds = []; rmSync(dir, { recursive: true, force: true }) }) + + it("C1 (B1) a live Task off ListTasks' first 50-row page is neither lost nor rerun", async () => { + const w = W(); w.floor.enqueue(job("1")) + const l = start(w, dir) + await until("running", () => w.ax.tasks.get("wf-test-1-a1")?.phase === "Running") + for (let i = 0; i < 50; i++) w.ax.put(foreign(`foreign-${i}`), "Completed") // newer than ours: ours is on page 2 + const calls = w.ax.listCalls + await until("ten resyncs", () => w.ax.listCalls >= calls + 10) + expect([l.verdict("infra/task-lost"), l.has("task-absent"), w.ax.tasks.has("wf-test-1-a2"), w.ax.deletes]).toEqual([false, false, false, []]) + w.ax.finish("wf-test-1-a1", 0, { answer: 42 }) + await until("floor done", () => J(w).state === "done") + expect([J(w).result, J(w).output, J(w).attempt]).toEqual(["success", { answer: 42 }, 1]) + await l.drain() + }) + + it("C1b (B1) 120 Tasks: occupancy counts a live foreign Task on page 3, and a truly gone Task needs two NotFound resyncs", async () => { + const w = W() + for (let i = 0; i < 2; i++) w.ax.put(foreign(`busy-${i}`), "Running") // oldest: on the last page + for (let i = 0; i < 118; i++) w.ax.put(foreign(`done-${i}`), "Completed") + w.floor.enqueue(job("1")) + const l = start(w, dir) + await until("a Lease was sent", () => w.floor.calls.some((c) => c.rpc === "Lease")) + await sleep(10 * SEC) + expect(w.floor.calls.filter((c) => c.rpc === "Lease").every((c) => c.capacity === 0)).toBe(true) // 2 busy of 2 + w.ax.tasks.delete("busy-0"); w.ax.tasks.delete("busy-1") + await until("created once the sandboxes free", () => w.ax.tasks.get("wf-test-1-a1")?.phase === "Running") + w.ax.tasks.delete("wf-test-1-a1") // the store forgets it (F19) + await until("task-lost after two NotFound resyncs", () => l.verdict("infra/task-lost")) + const absent = l.logs.filter((x) => x.ev === "task-absent" && x.leaseId === "wf-test-1-a1") + expect(absent.length).toBe(1) + await until("attempt 2 created", () => w.ax.tasks.has("wf-test-1-a2")) + await l.drain() + }) + + it("C2 (B2) a superseded attempt that will not delete blocks attempt n+1: retryable pre-start/ax-unavailable, never two at once", async () => { + const w = W(); w.floor.enqueue(job("1")); w.ax.holdPending.add("wf-test-1-a1") + const del = w.ax.deleteTask + w.ax.deleteTask = (name) => name === "wf-test-1-a1" ? Effect.fail(new AxError(UNAVAILABLE, "unavailable")) : del(name) + const l = start(w, dir, { pendingTimeoutMs: 200 }) + await until("fence failed", () => l.has("fence-failed", "wf-test-1-a2"), 8000) + expect(l.verdict("pre-start/ax-unavailable")).toBe(true) + expect(w.ax.updates.has("wf-test-1-a2")).toBe(false) + await until("the infra budget is spent", () => J(w).state === "done", 10000) + expect([J(w).result, (J(w).output as { reason: string }).reason]).toEqual(["failure", "infra/retry-budget-spent"]) + expect([...w.ax.updates.keys()]).toEqual(["wf-test-1-a1"]) // only ever one attempt in ax + expect(J(w).supersedes.length).toBeGreaterThan(1) // rule 7 amended: every unconfirmed old attempt is listed + await l.drain() + }) + + it("C3 (B3) a cancel that lands while UpdateTask is retrying stops the create; nothing runs on", async () => { + const w = W(); w.floor.enqueue(job("1")) + let refusals = 2 + const create = w.ax.createTask + w.ax.createTask = (t) => refusals-- > 0 ? Effect.fail(new AxError(UNAVAILABLE, "unavailable")) : create(t) + const l = start(w, dir, { deleteAfterMs: 60_000, createAttempts: 5 }) + await until("leased", () => l.has("leased", "wf-test-1-a1")) + w.floor.cancel("wf-test-1", "tom") + await until("floor done", () => J(w).state === "done") + expect(J(w).result).toBe("cancelled") + await sleep(600) + expect([w.ax.live("wf-test-1-a1"), l.has("abandoned-before-create", "wf-test-1-a1")]).toEqual([false, true]) + await l.drain() + }) + + it("C3b (B3) a cancel that lands while UpdateTask is in flight deletes the Task as soon as it returns, then reports cancelled", async () => { + const w = W(); w.floor.enqueue(job("1")) + let gate!: () => void + const create = w.ax.createTask + w.ax.createTask = (t) => create(t).pipe(Effect.andThen(Effect.callback((resume) => { gate = () => resume(Effect.void) }))) + const l = start(w, dir, { deleteAfterMs: 60_000 }) + await until("create in flight", () => w.ax.tasks.has("wf-test-1-a1") && gate !== undefined) + w.floor.cancel("wf-test-1", "tom") + await until("cancel seen", () => l.has("cancel", "wf-test-1-a1")) + gate() + await until("floor done", () => J(w).state === "done") + expect([J(w).result, w.ax.live("wf-test-1-a1")]).toEqual(["cancelled", false]) + await l.drain() + }) + + it("C4 (B4) one transient GetTaskResult error is retried at the next resync: the success lands, nothing reruns", async () => { + const w = W(); w.floor.enqueue(job("1")) + let blips = 1 + const res = w.ax.getTaskResult + w.ax.getTaskResult = (n) => n === "wf-test-1-a1" && blips-- > 0 ? Effect.fail(new AxError(UNAVAILABLE, "unavailable")) : res(n) + const l = start(w, dir) + await until("running", () => w.ax.tasks.get("wf-test-1-a1")?.phase === "Running") + w.ax.finish("wf-test-1-a1", 0, { answer: 42 }) + await until("floor done", () => J(w).state === "done") + expect([J(w).result, J(w).output, J(w).attempt, l.has("result-read-retry", "wf-test-1-a1")]).toEqual(["success", { answer: 42 }, 1, true]) + expect(J(w).history.some((h) => h.startsWith("requeue"))).toBe(false) + await l.drain() + }) + + it("C4b (B4) a result whose sha256 never matches is infra/result-unreadable after the bounded tries", async () => { + const w = W(); w.floor.enqueue(job("1")) + const res = w.ax.getTaskResult + w.ax.getTaskResult = (n) => res(n).pipe(Effect.map((r) => r && r !== "unimplemented" && n === "wf-test-1-a1" ? { ...r, digestOk: false } : r)) + const l = start(w, dir) + await until("running", () => w.ax.tasks.get("wf-test-1-a1")?.phase === "Running") + w.ax.finish("wf-test-1-a1", 0, { answer: 42 }) + await until("attempt 2 created", () => w.ax.tasks.has("wf-test-1-a2")) + expect(J(w).history).toContain("requeue:infra/result-unreadable") + expect(l.logs.filter((x) => x.ev === "result-read-retry" && x.leaseId === "wf-test-1-a1").length).toBe(2) + await l.drain() + }) + + it("C5 (B5) Failed with ResourceExhausted is the retryable pre-start/resource-exhausted", async () => { + const w = W(); w.floor.enqueue(job("1")) + const l = start(w, dir) + await until("running", () => w.ax.tasks.get("wf-test-1-a1")?.phase === "Running") + const t = w.ax.tasks.get("wf-test-1-a1")! + t.phase = "Failed" + t.conditions = [{ type: "Ready", status: "False", reason: "ActorCreateFailed", message: "ResourceExhausted: no free workers available" }] + await until("attempt 2 created", () => w.ax.tasks.has("wf-test-1-a2")) + expect(J(w).history).toContain("requeue:pre-start/resource-exhausted") + await l.drain() + }) + + it("C6 (B6) configured p1 on a server without P1 leases nothing and says p1-missing", async () => { + const w = W(); w.floor.enqueue(job("1")); w.ax.p1 = false + const l = start(w, dir, { completion: "p1" }) + await until("probe", () => l.has("p1-missing")) + await sleep(10 * SEC) + expect(w.floor.calls.filter((c) => c.rpc === "Lease").every((c) => c.capacity === 0)).toBe(true) + expect([w.ax.updates.size, J(w).state]).toEqual([0, "queued"]) + await l.drain() + }) + + it("C6b (B6) auto never selects guest, and re-probes when ax comes back: P1 appears, work starts", async () => { + const w = W(); w.floor.enqueue(job("1")); w.ax.p1 = false + const l = start(w, dir, { completion: "auto" }) + await until("probe", () => l.has("p1-missing")) + await sleep(5 * SEC) + expect(w.ax.updates.size).toBe(0) + w.ax.up = false + await until("ax down", () => l.has("ax-down")) + w.ax.p1 = true; w.ax.up = true // the operator rolled out P1 + await until("created in p1 mode", () => w.ax.tasks.has("wf-test-1-a1")) + const env = Object.fromEntries(w.ax.tasks.get("wf-test-1-a1")!.task.spec.env.map((e) => [e.name, e.value])) + expect(env.AX_CONWIP_COMPLETE_URL).toBeUndefined() + await l.drain() + }) + + it("C6c (B6, A5) guest mode with a workers.dev Complete URL leases nothing", async () => { + expect([isFleetInternalUrl("https://conwip-floor.x.workers.dev/guest"), isFleetInternalUrl("http://link.fleet.internal/guest"), isFleetInternalUrl("http://10.201.0.9:8732/g"), isFleetInternalUrl("https://example.com/")]).toEqual([false, true, true, false]) + const w = W(); w.floor.enqueue(job("1")); w.ax.p1 = false + const l = start(w, dir, { completion: "guest", shape: { atespace: "fleet", image: "ax-agent", gateway: "halogen", command: () => ["ax-agent", "pi"], completeUrl: "https://conwip-floor.x.workers.dev/guest" } }) + await until("refused", () => l.has("guest-url-invalid")) + await sleep(5 * SEC) + expect(w.floor.calls.filter((c) => c.rpc === "Lease").every((c) => c.capacity === 0)).toBe(true) + await l.drain() + }) + + it("C7 (B7) an empty runs-on, or two seats, is refused by the link (and by the floor's enqueue)", async () => { + const w = W({ cap: 3 }) + expect(w.floor.enqueue(job("1", { "runs-on": [] }))).toBe(false) + w.floor.enqueueUnchecked(job("1", { "runs-on": [] })) + w.floor.o.tokens["tok-nas"] = { holder: "nas-link-1", labels: ["seat:halogen", "seat:cc", "runtime:gvisor"] } + w.floor.enqueueUnchecked(job("2", { "runs-on": ["seat:halogen", "seat:cc"] })) + const l = start(w, dir, { maxInFlight: 3, servedLabels: ["seat:halogen", "seat:cc", "runtime:gvisor"] }) + await until("both done", () => J(w).state === "done" && J(w, "2").state === "done") + expect([(J(w).output as { reason: string }).reason, (J(w, "2").output as { reason: string }).reason]).toEqual(["pre-start/runs-on-invalid", "pre-start/runs-on-invalid"]) + expect(w.ax.updates.size).toBe(0) + await l.drain() + }) + + it("C8 (B8) a Lease reply lost on all three tries is replayed by the journaled requestKey: attempt 1 runs", async () => { + const w = W(); w.floor.enqueue(job("1")) + let lose = 3 + const lossy: typeof globalThis.fetch = async (input, init) => { + const res = await w.floor.fetch(input, init) + if (lose > 0 && (await res.clone().text()).includes("\"leaseId\":\"wf-test-1-a1\"")) { lose--; throw new TypeError("fetch failed: reply lost") } + return res + } + const l = start(w, dir, {}, { fetch: lossy }) + await until("attempt 1 created", () => w.ax.tasks.has("wf-test-1-a1"), 8000) + expect(J(w).history.some((h) => h.startsWith("requeue"))).toBe(false) + expect(w.ax.tasks.has("wf-test-1-a2")).toBe(false) + await l.drain() + }) + + it("C9 (B9) a hung Heartbeat times out; the next one renews; the running attempt survives", async () => { + const w = W(); w.floor.enqueue(job("1")) + let hang = false + const hanging: typeof globalThis.fetch = async (input, init) => { + const req = new Request(input as string | URL | Request, init) + if (hang && (await req.clone().text()).includes("\"Heartbeat\"")) { hang = false; return new Promise(() => {}) } + return w.floor.fetch(req) + } + const l = start(w, dir, {}, { fetch: hanging, timeoutsMs: { lease: 150, heartbeat: 150, complete: 300 } }) + await until("running", () => w.ax.tasks.get("wf-test-1-a1")?.phase === "Running") + hang = true + await until("the hang timed out", () => l.logs.some((x) => x.ev === "heartbeat-error")) + await sleep(25 * SEC) // past lease plus grace + expect([J(w).history.includes("requeue:grace"), w.ax.deletes.includes("wf-test-1-a1"), J(w).attempt]).toEqual([false, false, 1]) + w.ax.finish("wf-test-1-a1", 0, { ok: 1 }) + await until("floor done", () => J(w).state === "done") + expect([J(w).result, J(w).attempt]).toEqual(["success", 1]) + await l.drain() + }) + + it("C9b (B9) concurrent 401 and 503 are each charged to their own call", async () => { + const w = W() + const f: typeof globalThis.fetch = async (input, init) => { + const body = await new Request(input as string | URL | Request, init).text() + if (body.includes("\"Lease\"")) { await sleep(30); return new Response("busy", { status: 503 }) } + return new Response("no", { status: 401 }) + } + const out = await Effect.runPromise(Effect.scoped(Effect.gen(function*() { + const floor = yield* rpcFloor({ url: "http://floor.test", token: "tok-nas", sessionId: "s", fetch: f }) + return yield* Effect.all([ + floor.lease({ holderIdentity: "nas-link-1", capacity: 1, requestKey: "k" }).pipe(Effect.flip), + floor.heartbeat({ holderIdentity: "nas-link-1", leaseIds: [] }).pipe(Effect.flip) + ], { concurrency: "unbounded" }) + }))) + expect(out.map((e) => e.kind)).toEqual(["transient", "auth"]) + void w + }) + + it("B10 a missing Gateway, or one that allows everything, leases nothing; a restricted one does", async () => { + const w = W(); w.floor.enqueue(job("1")) + const halogen = w.ax.gateways.get("halogen")! + w.ax.gateways.delete("halogen") + const l = start(w, dir) + await until("gateway-missing", () => l.has("gateway-missing")) + w.ax.gateways.set("halogen", { hasAllowlist: true, hosts: [{ host: "*", port: 443 }] }) + await sleep(5 * SEC) + expect([w.ax.updates.size, w.floor.calls.filter((c) => c.rpc === "Lease").every((c) => c.capacity === 0)]).toEqual([0, true]) + w.ax.gateways.set("halogen", halogen) + await until("created", () => w.ax.tasks.has("wf-test-1-a1")) + expect(l.has("gateway-ok")).toBe(true) + await l.drain() + }) + + it("B11 a floor that over-grants never makes the link over-create: extra grants wait, journaled and heartbeated", async () => { + const w = W({ cap: 10, overGrant: 2 }) + for (const n of ["1", "2", "3"]) w.floor.enqueue(job(n)) + const l = start(w, dir, { maxInFlight: 1 }) + let peak = 0 + const sample = setInterval(() => { peak = Math.max(peak, liveOwn(w)) }, 2) + await until("three grants", () => ["1", "2", "3"].every((n) => J(w, n).state === "leased")) + await until("waiting", () => l.has("waiting-for-slot")) + for (const n of ["1", "2", "3"]) { + await until(`job ${n} running`, () => w.ax.tasks.get(`wf-test-${n}-a1`)?.phase === "Running") + w.ax.finish(`wf-test-${n}-a1`, 0, { n }) + await until(`job ${n} done`, () => J(w, n).state === "done") + } + clearInterval(sample) + expect(peak).toBe(1) + expect(["1", "2", "3"].map((n) => [J(w, n).result, J(w, n).attempt])).toEqual([["success", 1], ["success", 1], ["success", 1]]) + await l.drain() + }) + + it("B12 a Task that reads back with another spec is deleted and refused", async () => { + const w = W(); w.floor.enqueue(job("1")) + const create = w.ax.createTask + w.ax.createTask = (t) => create({ ...t, spec: { ...t.spec, env: t.spec.env.slice(0, 3) } }) // the store dropped env rows + const l = start(w, dir) + await until("floor done", () => J(w).state === "done") + expect([(J(w).output as { reason: string }).reason, w.ax.live("wf-test-1-a1")]).toEqual(["pre-start/readback-mismatch", false]) + await l.drain() + }) + + it("B13 a journaled grant the floor released while the link was dead is never created on restart", async () => { + const w = W(); w.floor.enqueue(job("1")); w.ax.blockCreate = true + const l1 = start(w, dir) + await until("grant journaled", () => l1.has("leased", "wf-test-1-a1")) + await l1.crash() + w.ax.blockCreate = false + await until("requeued after the grace", () => J(w).history.includes("requeue:grace"), 3000) + const l2 = start(w, dir) + await until("attempt 2 created", () => w.ax.tasks.has("wf-test-1-a2")) + expect([w.ax.updates.has("wf-test-1-a1"), l2.has("resume", "wf-test-1-a1")]).toEqual([false, false]) + w.ax.tick(); w.ax.finish("wf-test-1-a2", 0, { ok: 2 }) + await until("floor done", () => J(w).state === "done") + expect([J(w).result, J(w).attempt]).toEqual(["success", 2]) + await l2.drain() + }) + + it("B15 floor down past lease plus grace: the queued success goes out before the next Lease and is accepted (rule 4b)", async () => { + const w = W(); w.floor.enqueue(job("1")) + const l = start(w, dir) + await until("running", () => w.ax.tasks.get("wf-test-1-a1")?.phase === "Running") + w.floor.down = true + w.ax.finish("wf-test-1-a1", 0, { v: 1 }) + await until("outbox keeps", () => l.has("outbox-keep", "wf-test-1-a1")) + await until("the alarm requeued it while the floor was unreachable", () => J(w).history.includes("requeue:grace"), 3000) + w.floor.down = false + await until("floor done", () => J(w).state === "done") + expect([J(w).result, J(w).output, J(w).attempt]).toEqual(["success", { v: 1 }, 1]) + expect(J(w).history).toContain("withdrawn:a2") + expect(w.ax.tasks.has("wf-test-1-a2")).toBe(false) + await l.drain() + }) + + it("B17 a verdict that frees a slot leases at once, not at the next poll", async () => { + const w = W({ pollSeconds: 100 }) // 2 s between polls here + w.floor.enqueue(job("1")); w.floor.enqueue(job("2")) + const l = start(w, dir, { maxInFlight: 1 }) + await until("running", () => w.ax.tasks.get("wf-test-1-a1")?.phase === "Running", 4000) + await sleep(200) + const t0 = Date.now() + w.ax.finish("wf-test-1-a1", 0, { ok: 1 }) + await until("job 2 created", () => w.ax.tasks.has("wf-test-2-a1"), 1500) + expect(Date.now() - t0).toBeLessThan(1000) + await l.drain() + }) + + it("B18 a Lease endpoint is followed only when it is in the declared list", async () => { + const w = W() + let ep = "https://evil.example/floor" + const f: typeof globalThis.fetch = async (input, init) => { + const res = await w.floor.fetch(input, init) + const text = await res.text() + const h = new Headers(res.headers); h.delete("content-length") + return new Response(text.replace("\"nextPollSeconds\":", `"endpoint":"${ep}","nextPollSeconds":`), { status: res.status, headers: h }) + } + const seen: Array = [] + const l = start(w, dir, { floorUrls: ["https://conwip-floor-2.example/"] }, { fetch: f, onEndpoint: (u) => seen.push(u) }) + await until("ignored", () => l.has("endpoint-ignored")) + ep = "https://conwip-floor-2.example/" + await until("accepted", () => l.has("endpoint-accepted")) + expect(seen).toEqual(["https://conwip-floor-2.example/"]) + await l.drain() + }) +}) diff --git a/pkgs/substrate-apps/src/apps/link/test/fake-ax.ts b/pkgs/substrate-apps/src/apps/link/test/fake-ax.ts new file mode 100644 index 00000000..5cc096e8 --- /dev/null +++ b/pkgs/substrate-apps/src/apps/link/test/fake-ax.ts @@ -0,0 +1,84 @@ +// Test double for ax-server, with stock v0.3.0 semantics where they matter to the link (MEASURED in ax d8ed0fe: +// internal/server/server.go UpdateTask is a blind upsert that also overwrites status; DeleteTask is two-phase; +// FIELD-MAP D3: a save re-runs the reconcile and resumes the actor). `p1` switches on the carried patch P1: +// a terminal phase is written by the controller and GetTaskResult exists. ListTasks pages like ax's Redis store +// (ZRevRange: newest first, limit 0 means 50; MEASURED upstream server.go:91-104, store.go:209-212). GetGateway +// answers from `gateways`; the default `halogen` Gateway allows one host, as ax-fleet's does. +import { createHash } from "node:crypto" +import { Effect } from "effect" +import { AxError } from "../src/ax.ts" +import type { AxApi, AxGateway, AxObserved, AxResult } from "../src/ax.ts" +import type { AxTask } from "../src/contract.ts" + +const NOT_FOUND = 5, UNAVAILABLE = 14 +type T = { task: AxTask; phase: string; conditions: Array<{ type: string; status: string; reason: string; message: string }>; result?: string; usage?: { promptTokens: number; completionTokens: number; toolCalls: number }; exited?: boolean } + +export class FakeAx implements AxApi { + readonly tasks = new Map() + readonly updates = new Map() // UpdateTask calls per name + upsertsOnExisting = 0 // the hazard: an UpdateTask on a name that already existed (re-runs the actor on v0.3.0) + resumedAfterExit = 0 // how many times that hazard restarted a finished agent + deletes: Array = [] + events: Array = [] // update: and delete:, in call order + up = true + p1 = true + blockCreate = false // createTask never returns (a crash window before the Task exists) + hangAfterCreate = false // createTask writes the Task, then never returns (a crash window after it exists) + holdPending = new Set() // names the controller never starts + gateways = new Map([["halogen", { hasAllowlist: true, hosts: [{ host: "worker", port: 8731 }] }]]) + listCalls = 0 + + private guard = (): Effect.Effect => this.up ? Effect.void : Effect.fail(new AxError(UNAVAILABLE, "connection refused")) + private obs = (name: string, t: T): AxObserved => ({ name, phase: t.phase, spec: t.task.spec, conditions: t.conditions, usage: t.usage ?? null }) + + /** The controller's work between two resyncs: start Pending Tasks, finish two-phase deletes. */ + tick() { + for (const [name, t] of this.tasks) { + if (t.phase === "Terminating") this.tasks.delete(name) + else if (t.phase === "Pending" && !this.holdPending.has(name)) t.phase = "Running" + } + } + /** The agent exits. With P1 the controller writes the terminal phase; without it the Task stays Running. */ + finish(name: string, exitCode: number, result?: unknown, usage = { promptTokens: 11, completionTokens: 7, toolCalls: 3 }) { + const t = this.tasks.get(name) + if (!t) throw new Error(`no task ${name}`) + t.exited = true + if (!this.p1) return + t.phase = exitCode === 0 ? "Completed" : "Failed" + t.conditions = [{ type: "Ready", status: "False", reason: "CommandExited", message: `ExitCode=${exitCode}` }] + if (result !== undefined) t.result = JSON.stringify(result) + t.usage = usage + } + put(task: AxTask, phase = "Running") { this.tasks.set(task.metadata.name, { task, phase, conditions: [] }) } + live(name: string) { const t = this.tasks.get(name); return t !== undefined && t.phase !== "Terminating" } + + getTask = (name: string) => this.guard().pipe(Effect.map(() => { const t = this.tasks.get(name); return t ? this.obs(name, t) : undefined })) + createTask = (task: AxTask) => this.guard().pipe(Effect.andThen(() => { + if (this.blockCreate) return Effect.never + const name = task.metadata.name + this.updates.set(name, (this.updates.get(name) ?? 0) + 1) + this.events.push(`update:${name}`) + const prior = this.tasks.get(name) + if (prior) { this.upsertsOnExisting++; if (prior.exited) this.resumedAfterExit++ } // v0.3.0: status reset, actor resumed + this.tasks.set(name, { task, phase: "Pending", conditions: [] }) + return this.hangAfterCreate ? Effect.never : Effect.void + })) + listTasks = (limit: number, offset: number) => this.guard().pipe(Effect.map(() => { + if (offset === 0) { this.tick(); this.listCalls++ } + const n = limit > 0 ? limit : 50 + return [...this.tasks].reverse().slice(offset, offset + n).map(([name, t]) => this.obs(name, t)) + })) + getGateway = (name: string) => this.guard().pipe(Effect.map(() => this.gateways.get(name))) + deleteTask = (name: string) => this.guard().pipe(Effect.map(() => { + const t = this.tasks.get(name) + if (t) { t.phase = "Terminating"; this.deletes.push(name); this.events.push(`delete:${name}`) } + })) + getTaskResult = (name: string) => this.guard().pipe(Effect.andThen((): Effect.Effect => { + if (!this.p1) return Effect.succeed("unimplemented" as const) + const t = this.tasks.get(name) + if (!t) return Effect.succeed(undefined) + const content = t.result ?? "null" + return Effect.succeed({ content, sha256: createHash("sha256").update(content).digest("hex"), digestOk: true }) + })) + static notFound = NOT_FOUND +} diff --git a/pkgs/substrate-apps/src/apps/link/test/fake-floor.ts b/pkgs/substrate-apps/src/apps/link/test/fake-floor.ts new file mode 100644 index 00000000..179354ea --- /dev/null +++ b/pkgs/substrate-apps/src/apps/link/test/fake-floor.ts @@ -0,0 +1,304 @@ +// Test double for the Cloudflare floor: the FloorLink group served by Effect's stock HTTP RPC server through a web +// handler (the same `RpcServer.layerHttp` + `HttpRouter.toWebHandler` pair the prototype DO runs), called in-process +// through the link's `fetch`. State is in memory. It implements the floor rules of LINK-DESIGN.md section 4 as amended +// by "Critique applied", section H (B16): +// rule 1 states queued, leased, orphaned, done; WIP = leased + orphaned +// rule 2 route by runs-on against the holder's labels, cap per label, an admission stub, refuse at enqueue a job +// whose runs-on is empty or does not name exactly one seat: label +// rule 2b a replayed requestKey returns the same grants (kept for the life of the double, at least lease + grace) +// rule 4 stale-attempt, unknown-lease, duplicate; rule 4b: a verdict for attempt n while n+1 is still queued is +// accepted and n+1 withdrawn +// round 1 rule 4c: the verdict applied is recorded per leaseId, and a repeat for the same (leaseId, attempt) answers +// duplicate: true before rule 4b is considered (a lost reply or a second sender never spends a second +// release); rule 4b also accepts n's verdict while n+1 is leased to the SAME holder, which does not create +// n+1 while n's verdict is undelivered (link.ts dispatch) +// round 2 rule 4b names the grant it withdrew in the answer (`withdrew`); the link never infers a withdrawal. Rule +// 4c's duplicate answer replays the original answer's `withdrew`, so a lost reply is answered the same way +// round 3 rule 4b also names the generation withdrawn (`withdrewTransitions`: the leaseTransitions a grant of it +// carried or would have carried); `sendWithdrewGen: false` is a floor that omits it +// rule 5 a timer (the DO alarm): leased past renew + lease becomes orphaned, even with a cancel pending +// rule 6 the holder's heartbeat renews and re-adopts; lost for anything else; cancelRequested +// rule 7 an omitted orphan is released, so is one past the grace; supersedes lists EVERY earlier attempt +// round 4 rule 6b: a Heartbeat's `pendingRequestKey` vouches for every lease granted under that key (renewed and +// re-adopted, never released as omitted) while the holder has not seen that Lease answered +// rule 8 two budgets: infrastructure releases and agent retries, each final reason naming its budget +// rule 9 a per-lease token, minted only for guest-mode holders (A3), single use +// rule 10 at the deadline the floor sets cancel with reason deadline +// rule 11 holder = f(bearer): a payload holderIdentity that differs answers 403; Complete only for that holder's +// leases; one live session per holder (409); pause per holder +// It is NOT the production floor. +import { randomUUID } from "node:crypto" +import { Effect, Layer } from "effect" +import { HttpRouter } from "effect/unstable/http" +import { RpcSerialization, RpcServer } from "effect/unstable/rpc" +import { FloorLink, FloorLinkClient } from "../src/contract.ts" +import type { AgentJob, FailureOutput } from "../src/contract.ts" + +type Job = { + name: string; job: AgentJob; state: "queued" | "leased" | "orphaned" | "done"; attempt: number + leaseId?: string; holder?: string; acquire?: number; renew?: number; orphanedAt?: number; transitions: number + cancel?: string; result?: string; output?: unknown; usage?: unknown; supersedes: Array; token?: string; deadline?: number + history: Array; infraSpent: number; agentSpent: number; holders: Record +} +export type TokenBinding = string | { holder: string; labels?: Array; guest?: boolean } +export interface FloorConfig { + cap: number; leaseSeconds: number; graceSeconds: number; pollSeconds: number; heartbeatSeconds: number + maxAttempts: number // rule 8: agent retries (actor-crashed, result-unreadable) + infraAttempts?: number // rule 8: infrastructure releases (default 5) + secondMs: number + tokens: Record + labelCaps?: Record // rule 2: cap per label + timer?: boolean // rule 5: the DO alarm as a real timer (default true) + overGrant?: number // a broken floor that grants this many more than asked (B11's test) + skewedEncode?: boolean // round 1: a floor whose schema drifted from the link's; grants go out without validation + sendWithdrewGen?: boolean // round 3: Complete names the generation withdrawn (default true) +} +const DEFAULT_LABELS = ["seat:halogen", "runtime:gvisor"] +const INFRA = new Set(["omitted", "grace", "infra/task-lost", "pre-start/pending-timeout", "pre-start/ax-unavailable", "pre-start/resource-exhausted", "pre-start/superseded-verdict-pending", "infra/link-defect"]) +const AGENT = new Set(["infra/actor-crashed", "infra/result-unreadable"]) + +export class FakeFloor { + readonly jobs = new Map() + readonly grants = new Map>() + readonly sessions = new Map() + readonly calls: Array<{ rpc: string; holder?: string; capacity?: number; granted?: number; leaseIds?: ReadonlyArray; status?: number }> = [] + readonly paused = new Set() + readonly verdicts = new Map() // rule 4c + admit: (job: AgentJob) => boolean = () => true // rule 3 stub (seat readings) + seq = 0 + down = false + redeliver: Array = [] // at-least-once: the next Lease hands these leases out again + private readonly timer: ReturnType | undefined + constructor(readonly o: FloorConfig) { + if (o.timer !== false) { this.timer = setInterval(() => this.sweep(), o.secondMs); this.timer.unref?.() } + } + close() { if (this.timer) clearInterval(this.timer) } + private ms = (s: number) => s * this.o.secondMs + private binding(token: string) { + const b = this.o.tokens[token] + if (b === undefined) return undefined + return typeof b === "string" ? { holder: b, labels: DEFAULT_LABELS, guest: false } : { holder: b.holder, labels: b.labels ?? DEFAULT_LABELS, guest: b.guest ?? false } + } + private bindingOf(holder: string) { + for (const t of Object.keys(this.o.tokens)) { const b = this.binding(t)!; if (b.holder === holder) return b } + return undefined + } + + /** Rule 2 (amended): refuses a job whose runs-on is empty or does not name exactly one seat: label. */ + enqueue(job: AgentJob): boolean { + const seats = job.spec["runs-on"].filter((l) => l.startsWith("seat:")) + if (job.spec["runs-on"].length === 0 || seats.length !== 1) return false + this.enqueueUnchecked(job) + return true + } + /** A floor without rule 2's enqueue check, to test the link's own defence (B7). */ + enqueueUnchecked(job: AgentJob) { + if (!this.jobs.has(job.metadata.name)) this.jobs.set(job.metadata.name, { name: job.metadata.name, job, state: "queued", attempt: 1, transitions: 0, supersedes: [], history: [], infraSpent: 0, agentSpent: 0, holders: {} }) + } + cancel(name: string, why = "tom") { + const j = this.jobs.get(name)! + if (j.state === "queued") this.finish(j, "cancelled", { reason: why }) + else if (j.state !== "done") j.cancel = why + } + wip(label?: string) { return [...this.jobs.values()].filter((j) => (j.state === "leased" || j.state === "orphaned") && (label === undefined || j.job.spec["runs-on"].includes(label))).length } + stats() { return { seq: ++this.seq, cap: this.o.cap, wip: this.wip(), queued: [...this.jobs.values()].filter((j) => j.state === "queued").length, done: [...this.jobs.values()].filter((j) => j.state === "done").length } } + private finish(j: Job, result: string, output?: unknown, usage?: unknown) { + Object.assign(j, { state: "done", result, output, usage, token: undefined }); j.history.push(`done:${result}`) + } + /** An orphan the fleet side no longer vouches for: a cancelled job ends cancelled, any other goes back to the queue. */ + private release(j: Job, why: string) { + if (j.cancel) { j.history.push(`released:${why}`); return this.finish(j, "cancelled", { reason: j.cancel }) } + this.requeue(j, why) + } + private requeue(j: Job, why: string) { + j.history.push(`requeue:${why}`) + const infra = INFRA.has(why) || !AGENT.has(why) + if (infra) j.infraSpent++; else j.agentSpent++ + if (infra && j.infraSpent >= (this.o.infraAttempts ?? 5)) return this.finish(j, "failure", { reason: "infra/retry-budget-spent", message: why }) + if (!infra && j.agentSpent >= this.o.maxAttempts) return this.finish(j, "failure", { reason: "agent/retry-budget-spent", message: why }) + const old = j.leaseId! + Object.assign(j, { state: "queued", attempt: j.attempt + 1, transitions: j.transitions + 1, supersedes: [...j.supersedes, old], leaseId: undefined, holder: undefined, token: undefined, orphanedAt: undefined }) + } + /** The DO storage alarm's work (rules 5, 7 b and 10). Runs on the timer and on every request. */ + sweep(now = Date.now()) { + for (const j of this.jobs.values()) { + if (j.state === "leased" && j.deadline !== undefined && now >= j.deadline && !j.cancel) j.cancel = "deadline" + if (j.state === "leased" && j.renew! + this.ms(this.o.leaseSeconds) <= now) { // L3: expiry never frees a live slot, + j.state = "orphaned"; j.orphanedAt = now; j.history.push("orphaned") // even when a cancel is pending + } + if (j.state === "orphaned" && j.orphanedAt! + this.ms(this.o.graceSeconds) <= now) this.release(j, "grace") // L3 b + } + } + private grantable(j: Job, labels: Array, extra: Map) { + if (j.state !== "queued") return false + if (!j.job.spec["runs-on"].every((l) => labels.includes(l))) return false // rule 2: routed by label + for (const l of j.job.spec["runs-on"]) { + const cap = this.o.labelCaps?.[l] + if (cap !== undefined && this.wip(l) + (extra.get(l) ?? 0) >= cap) return false + } + return this.admit(j.job) + } + lease(p: { holderIdentity: string; capacity: number; requestKey: string }) { + this.sweep() + const b = this.bindingOf(p.holderIdentity) + let rows: Array + const prior = this.grants.get(p.requestKey) + if (prior) rows = prior.map((id) => [...this.jobs.values()].find((j) => j.leaseId === id && j.state !== "done")).filter((j): j is Job => j !== undefined) + else if (this.paused.has(p.holderIdentity) || b === undefined) rows = [] + else { + const free = Math.max(0, Math.min(p.capacity, this.o.cap - this.wip())) + (this.o.overGrant ?? 0) + const extra = new Map() + rows = [] + for (const j of this.jobs.values()) { + if (rows.length >= free) break + if (!this.grantable(j, b.labels, extra)) continue + rows.push(j) + for (const l of j.job.spec["runs-on"]) extra.set(l, (extra.get(l) ?? 0) + 1) + } + const now = Date.now() + for (const j of rows) { + const tm = j.job.spec["timeout-minutes"] + const leaseId = `${j.name}-a${j.attempt}` + Object.assign(j, { state: "leased", leaseId, holder: p.holderIdentity, acquire: now, renew: now, token: b.guest ? randomUUID() : undefined, + deadline: tm === undefined ? undefined : now + this.ms(tm * 60) }) + j.holders[leaseId] = p.holderIdentity + j.history.push(`leased:a${j.attempt}`) + } + this.grants.set(p.requestKey, rows.map((j) => j.leaseId!)) + } + if (this.redeliver.length) { + rows = [...rows, ...this.redeliver.map((id) => [...this.jobs.values()].find((j) => j.leaseId === id)).filter((j): j is Job => j !== undefined)] + this.redeliver = [] + } + this.calls.push({ rpc: "Lease", holder: p.holderIdentity, capacity: p.capacity, granted: rows.length }) + return { + grants: rows.map((j) => ({ + leaseId: j.leaseId!, attempt: j.attempt, job: j.job, + lease: { holderIdentity: j.holder!, leaseDurationSeconds: this.o.leaseSeconds, acquireTime: j.acquire!, renewTime: j.renew!, leaseTransitions: j.transitions }, + ...(j.supersedes.length ? { supersedes: [...j.supersedes] } : {}), ...(j.token ? { leaseToken: j.token } : {}), ...(j.deadline !== undefined ? { deadline: j.deadline } : {}) + })), + stats: this.stats(), nextPollSeconds: this.paused.has(p.holderIdentity) ? this.o.pollSeconds * 10 : this.o.pollSeconds, heartbeatSeconds: this.o.heartbeatSeconds + } + } + vouchByKey = true // round 4 rule 6b; false is an L1-only floor that ignores pendingRequestKey + heartbeat(p: { holderIdentity: string; leaseIds: ReadonlyArray; pendingRequestKey?: string }) { + this.sweep() + this.calls.push({ rpc: "Heartbeat", holder: p.holderIdentity, leaseIds: p.leaseIds }) + const byKey = new Set(this.vouchByKey && p.pendingRequestKey !== undefined ? this.grants.get(p.pendingRequestKey) ?? [] : []) + for (const id of byKey) { // rule 6b: vouched for by key, not listed (the holder never saw these ids) + if (p.leaseIds.includes(id)) continue + const j = [...this.jobs.values()].find((x) => x.leaseId === id && (x.state === "leased" || x.state === "orphaned") && x.holder === p.holderIdentity) + if (!j) continue + if (j.state === "orphaned") { j.state = "leased"; j.history.push("re-adopted") } + j.renew = Date.now() + } + const renewed: Array = [], lost: Array = [], cancelRequested: Array = [] + for (const id of p.leaseIds) { + const j = [...this.jobs.values()].find((x) => x.leaseId === id && (x.state === "leased" || x.state === "orphaned") && x.holder === p.holderIdentity) + if (!j) { lost.push(id); continue } + if (j.state === "orphaned") { j.state = "leased"; j.history.push("re-adopted") } // the holder vouches again + j.renew = Date.now(); renewed.push(id) + if (j.cancel) cancelRequested.push(id) + } + for (const j of this.jobs.values()) // L3 a: an orphaned lease its holder no longer lists is gone on the fleet side + if (j.state === "orphaned" && j.holder === p.holderIdentity && !p.leaseIds.includes(j.leaseId!) && !byKey.has(j.leaseId!)) this.release(j, "omitted") + return { renewed, lost, cancelRequested, stats: this.stats() } + } + /** `holder` is the bearer's holder (rule 11); undefined only for the guest path, which its lease token authorises. */ + complete(p: { leaseId: string; attempt: number; result: string; output?: unknown; usage?: unknown }, holder?: string) { + this.sweep() + this.calls.push({ rpc: "Complete", leaseIds: [p.leaseId] }) + const seen = this.verdicts.get(p.leaseId) // rule 4c: at most one verdict per (leaseId, attempt) + if (seen !== undefined && seen.attempt === p.attempt) return { duplicate: true, stats: this.stats(), ...this.w(seen.withdrew, seen.withdrewTransitions) } + const name = p.leaseId.replace(/-a\d+$/, "") + let j = [...this.jobs.values()].find((x) => x.leaseId === p.leaseId) + let withdrew: string | undefined, withdrewTransitions: number | undefined + if (!j) { + const q = this.jobs.get(name) + if (!q) return { code: "unknown-lease" as const } + // rule 4b: attempt n's verdict while n+1 is still queued: accept it and withdraw n+1 + // rule 4b (round 1): n+1 queued, or leased to the holder of n (that holder has not created it) + const nHolder = q.holders[p.leaseId] + const withdrawable = q.state === "queued" || (q.state === "leased" && q.holder === nHolder) + if (withdrawable && q.attempt === p.attempt + 1 && q.supersedes.includes(p.leaseId) && (holder === undefined || nHolder === holder)) { + q.history.push(`withdrawn:a${q.attempt}`) + withdrew = `${q.name}-a${q.attempt}`; withdrewTransitions = q.transitions + Object.assign(q, { state: "leased", attempt: p.attempt, leaseId: p.leaseId, holder: nHolder, token: undefined, renew: Date.now(), supersedes: q.supersedes.filter((x) => x !== p.leaseId) }) + j = q + } else return { code: "stale-attempt" as const } + } + if (holder !== undefined && j.holders[p.leaseId] !== holder) return { code: "stale-attempt" as const } // rule 11 + if (j.state === "done") return { duplicate: true, stats: this.stats() } + if ((j.state !== "leased" && j.state !== "orphaned") || j.attempt !== p.attempt) return { code: "stale-attempt" as const } + const reason = (p.output as FailureOutput | undefined)?.reason + this.verdicts.set(p.leaseId, { attempt: p.attempt, result: p.result, ...(reason !== undefined ? { reason } : {}), ...(withdrew !== undefined ? { withdrew, withdrewTransitions } : {}) }) + const w = this.w(withdrew, withdrewTransitions) + if (p.result === "failure" && reason !== undefined && (INFRA.has(reason) || AGENT.has(reason))) { this.requeue(j, reason); return { duplicate: false, stats: this.stats(), ...w } } + this.finish(j, p.result, p.output ?? (j.cancel ? { reason: j.cancel } : null), p.usage ?? null) + return { duplicate: false, stats: this.stats(), ...w } + } + private w(withdrew: string | undefined, t: number | undefined) { + if (withdrew === undefined) return {} + return this.o.sendWithdrewGen === false || t === undefined ? { withdrew } : { withdrew, withdrewTransitions: t } + } + /** L7: the guest's own Complete, authorised by its per-lease token alone (single use, bound to one lease). */ + guestComplete(leaseId: string, token: string, result: string, output?: unknown) { + const j = [...this.jobs.values()].find((x) => x.leaseId === leaseId) + if (!j || j.token === undefined || j.token !== token) return { refused: "token" as const } + j.token = undefined // single use + return this.complete({ leaseId, attempt: j.attempt, result, output }) + } + + private handlerCache: ((r: Request) => Promise) | undefined + get handler(): (r: Request) => Promise { return this.handlerCache ??= this.skewed() ? this.lenientHandler() : this.strictHandler() } + private skewed() { return this.o.skewedEncode === true } + /** Round 1: the same handlers served through the link's lenient view of Lease, so a grant is sent as the floor + * holds it, without the strict encode (a floor whose contract drifted). */ + private lenientHandler() { + return HttpRouter.toWebHandler( + RpcServer.layerHttp({ group: FloorLinkClient, path: "/rpc", protocol: "http" }).pipe( + Layer.provide(FloorLinkClient.toLayer({ + Lease: (p) => Effect.sync(() => this.lease(p)), + Heartbeat: (p) => Effect.sync(() => this.heartbeat(p)), + Complete: (p, o) => Effect.suspend(() => { + const r = this.complete(p, String((o.headers as Record)["x-link-holder"] ?? "")) + return r.code !== undefined ? Effect.fail({ code: r.code }) : Effect.succeed({ duplicate: r.duplicate!, stats: r.stats!, ...(r.withdrew !== undefined ? { withdrew: r.withdrew } : {}), ...(r.withdrewTransitions !== undefined ? { withdrewTransitions: r.withdrewTransitions } : {}) }) + }) + })), + Layer.provide(RpcSerialization.layerJson)), { disableLogger: true }).handler as (r: Request) => Promise + } + private strictHandler() { return HttpRouter.toWebHandler( + RpcServer.layerHttp({ group: FloorLink, path: "/rpc", protocol: "http" }).pipe( + Layer.provide(FloorLink.toLayer({ + Lease: (p) => Effect.sync(() => this.lease(p)), + Heartbeat: (p) => Effect.sync(() => this.heartbeat(p)), + Complete: (p, o) => Effect.suspend(() => { + const r = this.complete(p, String((o.headers as Record)["x-link-holder"] ?? "")) + return r.code !== undefined ? Effect.fail({ code: r.code }) : Effect.succeed({ duplicate: r.duplicate!, stats: r.stats!, ...(r.withdrew !== undefined ? { withdrew: r.withdrew } : {}), ...(r.withdrewTransitions !== undefined ? { withdrewTransitions: r.withdrewTransitions } : {}) }) + }) + })), + Layer.provide(RpcSerialization.layerJson)), { disableLogger: true }).handler as (r: Request) => Promise } + + /** The Worker entry: network, bearer (per-link token bound to a holder), holder binding (403), then one session per + * holder (409, L5). The resolved holder reaches the handlers as `x-link-holder`; a client's own copy is dropped. */ + readonly fetch: typeof globalThis.fetch = async (input, init) => { + if (this.down) throw new TypeError("fetch failed") + const req = new Request(input as string | URL | Request, init) + const b = this.binding((req.headers.get("authorization") ?? "").replace(/^Bearer /, "")) + const answer = (status: number, body: string) => { this.calls.push({ rpc: "entry", status }); return new Response(body, { status }) } + if (b === undefined) return answer(401, "unauthorized") + const body = await req.clone().text() + let msgs: Array = [] + try { const m = JSON.parse(body); msgs = Array.isArray(m) ? m : [m] } catch { /* not JSON: the RPC server answers */ } + if (msgs.some((m) => m?.payload && typeof m.payload.holderIdentity === "string" && m.payload.holderIdentity !== b.holder)) return answer(403, "holder mismatch") + const sid = req.headers.get("x-link-session") ?? "" + const s = this.sessions.get(b.holder), now = Date.now() + if (s && s.id !== sid && s.until > now) return answer(409, "session held") + this.sessions.set(b.holder, { id: sid, until: now + this.ms(3 * this.o.heartbeatSeconds) }) + const u = new URL(req.url) + if (u.pathname === "/rpc/") u.pathname = "/rpc" // Effect's client posts to "/" (PROTO.md section 4) + const h = new Headers(req.headers) + h.set("x-link-holder", b.holder) + return this.handler(new Request(u, { method: req.method, headers: h, body })) + } +} diff --git a/pkgs/substrate-apps/src/apps/link/test/final-pass.test.ts b/pkgs/substrate-apps/src/apps/link/test/final-pass.test.ts new file mode 100644 index 00000000..2530486b --- /dev/null +++ b/pkgs/substrate-apps/src/apps/link/test/final-pass.test.ts @@ -0,0 +1,70 @@ +// Final pass (2026-09-23): regression tests for the two link-side findings of wave/VERIFY-LINK.md that were still open at +// 3b036da, both reproduced there by the chaos harness against the prototype floor under wrangler dev --local +// (link/final-verify/chaos-fresh-3b036da: f1 ran as a1 and a2, OVERLAP 8 s). +// FP-1 (R3-1): a heartbeat sleep begun at the 30 s default while nothing was held outlived a short lease granted +// during it; the floor released the orphan and the job ran twice. +// FP-2 (V1): the link fenced only what Grant.supersedes named; a floor that omits it got attempt n+1 created beside +// a still Running attempt n. +import { expect, it } from "vitest" +import { bodyOf, job, sleep, start, until, world } from "./review-r4-harness.ts" + +it("FP-1 a short lease granted during the first long heartbeat sleep is renewed in time", async () => { + // Lease 6 s and grace 2 s at SEC = 20 ms; the link starts with the production default heartbeat (30 s = 600 ms). + const w = world({ leaseSeconds: 6, graceSeconds: 2, heartbeatSeconds: 2 }) + // Every Lease reply is held 30 ms, as a real HTTPS round trip is (the chaos run: wrangler dev --local), so the heartbeat + // fiber sizes its first sleep before the first reply has set the server's interval. With an in-process reply the fiber + // happened to start after it, which hid the defect from every earlier test. + const slowLease: typeof globalThis.fetch = async (input, init) => { + const body = await bodyOf(input, init) + if (body.includes('"Lease"')) await sleep(30) + return w.floor.fetch(input as any, { ...init, body }) + } + const l = start(w, undefined, { initialHeartbeatSeconds: 30 }, slowLease) + await until("first Lease answered", () => w.floor.jobs.size === 0 && l.logs.some((x) => x.ev === "ax-up"), 2000).catch(() => {}) + await sleep(60) // the first heartbeat sleep has begun, sized from the 30 s default + w.floor.enqueue(job("1")) + const J = () => w.floor.jobs.get("wf-test-1")! + await until("a1 running", () => w.ax.tasks.get("wf-test-1-a1") !== undefined, 3000) + await sleep(1200) // twice the default heartbeat: without the fix the lease expires and the grace runs out first + const out = { history: J().history, attempt: J().attempt, a2: w.ax.updates.get("wf-test-1-a2") ?? 0 } + console.log(JSON.stringify(out)) + await l.drain(); w.floor.close() + expect(out.attempt).toBe(1) + expect(out.a2).toBe(0) + expect(out.history).not.toContain("requeue:omitted") + expect(out.history.filter((h: string) => h.startsWith("requeue")).length).toBe(0) +}) + +it("FP-2 attempt n+1 is never created beside a live attempt n, even when the floor omits supersedes", async () => { + const w = world({ leaseSeconds: 6, graceSeconds: 2, heartbeatSeconds: 2 }) + let failHeartbeat = false + const strip = (s: string) => s.replace(/,"supersedes":\[[^\]]*\]/g, "").replace(/"supersedes":\[[^\]]*\],/g, "") + const fetch: typeof globalThis.fetch = async (input, init) => { + const body = await bodyOf(input, init) + if (failHeartbeat && body.includes('"Heartbeat"')) throw new TypeError("fetch failed") + const res = await w.floor.fetch(input as any, { ...init, body }) + const text = await res.text() + return new Response(strip(text), { status: res.status, headers: res.headers }) + } + let a1LiveAtA2Create: boolean | undefined + const create = w.ax.createTask + ;(w.ax as any).createTask = (task: any) => { + if (task.metadata.name === "wf-test-1-a2" && a1LiveAtA2Create === undefined) a1LiveAtA2Create = w.ax.live("wf-test-1-a1") + return create(task) + } + w.floor.enqueue(job("1")) + const l = start(w, undefined, {}, fetch) + const J = () => w.floor.jobs.get("wf-test-1")! + await until("a1 running", () => w.ax.live("wf-test-1-a1") && l.has("created", "wf-test-1-a1"), 3000) + failHeartbeat = true // a1's lease expires and the grace runs out; the floor requeues attempt 2 and grants it on Lease + await until("floor requeued attempt 2", () => J().attempt === 2, 4000) + await until("a2 leased", () => l.has("leased", "wf-test-1-a2"), 4000) + failHeartbeat = false + await until("a2 created", () => (w.ax.updates.get("wf-test-1-a2") ?? 0) > 0, 4000) + const out = { a1LiveAtA2Create, extended: l.has("fence-set-extended", "wf-test-1-a2"), events: (w.ax as any).events?.filter((e: string) => e.includes("wf-test-1")) } + console.log(JSON.stringify(out)) + await l.drain(); w.floor.close() + expect(out.extended).toBe(true) + expect(out.a1LiveAtA2Create).toBe(false) + expect(w.ax.upsertsOnExisting).toBe(0) +}) diff --git a/pkgs/substrate-apps/src/apps/link/test/fixtures/ax-p1.proto b/pkgs/substrate-apps/src/apps/link/test/fixtures/ax-p1.proto new file mode 100644 index 00000000..a18e4a43 --- /dev/null +++ b/pkgs/substrate-apps/src/apps/link/test/fixtures/ax-p1.proto @@ -0,0 +1,405 @@ +// TEST FIXTURE: the vendored ax.proto (v0.3.0 d8ed0fe) plus the three hunks of carried patch P1 that the link reads: +// rpc GetTaskResult, messages GetTaskResultRequest and TaskResult, UsageStats.tool_calls. Copied from dotfiles branch +// ax/fleet-bringup pkgs/ax/patches/p1-completion.patch (lines 3240-3300) on 2026-09-23. Used only by the workerd +// integration test; production vendors the P1 proto when ax-fleet ships it (LINK-DESIGN section 9). +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +syntax = "proto3"; + +package ax.v1alpha1; + +import "google/protobuf/struct.proto"; +import "google/protobuf/timestamp.proto"; + +option go_package = "github.com/google/ax/pkg/apis/v1alpha1"; + +// AX defines the core control plane gRPC service for autonomous agent orchestration. +service AX { + // Manifests are parsed by the client (see `ax apply`) and submitted through the + // typed Update* RPCs below; the server never receives raw YAML. + + // Tasks + rpc GetTask(GetTaskRequest) returns (Task); + rpc ListTasks(ListTasksRequest) returns (ListTasksResponse); + rpc UpdateTask(UpdateTaskRequest) returns (Task); + rpc DeleteTask(DeleteTaskRequest) returns (DeleteTaskResponse); + rpc SuspendTask(SuspendTaskRequest) returns (Task); + rpc ResumeTask(ResumeTaskRequest) returns (Task); + rpc WatchTask(WatchTaskRequest) returns (stream WatchTaskResponse); + // GetTaskResult returns the result file the task's command wrote, as copied + // by the controller when the command exited. NotFound until then. + rpc GetTaskResult(GetTaskResultRequest) returns (TaskResult); + + // Gateways + rpc GetGateway(GetGatewayRequest) returns (Gateway); + rpc ListGateways(ListGatewaysRequest) returns (ListGatewaysResponse); + rpc UpdateGateway(UpdateGatewayRequest) returns (Gateway); + rpc DeleteGateway(DeleteGatewayRequest) returns (DeleteGatewayResponse); + + // Workspaces + rpc GetWorkspace(GetWorkspaceRequest) returns (Workspace); + rpc ListWorkspaces(ListWorkspacesRequest) returns (ListWorkspacesResponse); + rpc UpdateWorkspace(UpdateWorkspaceRequest) returns (Workspace); + rpc DeleteWorkspace(DeleteWorkspaceRequest) returns (DeleteWorkspaceResponse); + + // Models + rpc GetModel(GetModelRequest) returns (Model); + rpc ListModels(ListModelsRequest) returns (ListModelsResponse); + rpc UpdateModel(UpdateModelRequest) returns (Model); + rpc DeleteModel(DeleteModelRequest) returns (DeleteModelResponse); + +} + +// ObjectMeta is metadata for AX resources. +message ObjectMeta { + string name = 1; + string atespace = 2; + google.protobuf.Timestamp creation_timestamp = 3; +} + +// --- Task --- + +message Task { + string api_version = 1; + string kind = 2; + ObjectMeta metadata = 3; + TaskSpec spec = 4; + TaskStatus status = 5; +} + +message TaskSpec { + // Field 1 was `goal`, removed; the workspace goal lives on WorkspaceRef. + reserved 1; + reserved "goal"; + bool suspend = 2; + string image = 3; + repeated string command = 4; + repeated EnvVar env = 5; + ResourceReqs resources = 6; + // workspaces binds one or more Workspaces, each mounted at its own path + // under /workspace. The first entry is the task command's working directory. + repeated WorkspaceRef workspaces = 7; + GatewayRef gateway = 8; + // Field 9 was `policies` (budget and approval config), removed for now. + reserved 9; + reserved "policies"; + // debug enables the in-container guest services (process execution and file + // access) that back `ax ssh`. Off by default. + bool debug = 10; +} + +message EnvVar { + string name = 1; + string value = 2; +} + +message ResourceReqs { + ResourceList requests = 1; + ResourceList limits = 2; +} + +message ResourceList { + string cpu = 1; + string memory = 2; +} + +message WorkspaceRef { + string name = 1; + string path = 2; + string goal = 3; +} + +message GatewayRef { + string name = 1; +} + +message TaskStatus { + string phase = 1; + string id = 2; + string actor = 3; + // json_name keeps the established `workerIP` spelling in YAML and JSON. + string worker_ip = 4 [json_name = "workerIP"]; + PendingApproval pending_approval = 5; + UsageStats usage = 6; + repeated Condition conditions = 7; +} + +message PendingApproval { + string id = 1; + string action = 2; + google.protobuf.Timestamp requested_at = 3; +} + +message UsageStats { + int32 prompt_tokens = 1; + int32 completion_tokens = 2; + int32 tool_calls = 3; +} + +message Condition { + string type = 1; + string status = 2; + google.protobuf.Timestamp last_transition_time = 3; + string reason = 4; + string message = 5; +} + +// --- Gateway --- + +message Gateway { + string api_version = 1; + string kind = 2; + ObjectMeta metadata = 3; + GatewaySpec spec = 4; +} + +message GatewaySpec { + repeated Listener listeners = 1; + EgressConfig egress = 2; +} + +message Listener { + string name = 1; + int32 port = 2; + string protocol = 3; +} + +message EgressConfig { + EgressAllowlist allowlist = 1; +} + +message EgressAllowlist { + repeated HostRule hosts = 1; +} + +message HostRule { + string host = 1; + int32 port = 2; +} + +// --- Workspace --- + +message Workspace { + string api_version = 1; + string kind = 2; + ObjectMeta metadata = 3; + WorkspaceSpec spec = 4; +} + +message WorkspaceSpec { + repeated GitRepo git = 1; + MCPConfig mcp = 2; + SkillsConfig skills = 3; +} + +message GitRepo { + string name = 1; + string repo = 2; + string branch = 3; + string dir = 4; + int32 depth = 5; +} + +message MCPConfig { + repeated MCPRegistry registries = 1; + repeated MCPServer servers = 2; +} + +message MCPRegistry { + string provider = 1; + string project = 2; + string query = 3; + repeated MCPServer servers = 4; +} + +message MCPServer { + string name = 1; + string endpoint = 2; + string command = 3; + repeated string args = 4; +} + +message SkillsConfig { + repeated SkillRegistry registries = 1; + string path = 2; +} + +message SkillRegistry { + string provider = 1; + string project = 2; + string query = 3; +} + +// --- Model --- + +message Model { + string api_version = 1; + string kind = 2; + ObjectMeta metadata = 3; + ModelSpec spec = 4; +} + +message ModelSpec { + string provider = 1; + string model = 2; + // Fields 3-5 were typed temperature, max_tokens, and system_instruction; + // provider settings now live in the free-form `parameters` map. + reserved 3, 4, 5; + reserved "temperature", "max_tokens", "system_instruction"; + SecretKeyRef secret_key = 6; + // parameters are provider-specific generation settings passed through to the + // model API as-is, for example temperature or maxOutputTokens for Gemini. + google.protobuf.Struct parameters = 7; +} + +message SecretKeyRef { + string name = 1; + string key = 2; +} + +// --- RPC Request & Response Messages --- + +// Tasks +message GetTaskRequest { + string atespace = 1; + string name = 2; +} + +message GetTaskResultRequest { + string atespace = 1; + string name = 2; +} + +message TaskResult { + bytes content = 1; + string sha256 = 2; +} + +message ListTasksRequest { + string atespace = 1; + int64 limit = 2; + int64 offset = 3; +} + +message ListTasksResponse { + repeated Task tasks = 1; +} + +message UpdateTaskRequest { + Task task = 1; +} + +message DeleteTaskRequest { + string atespace = 1; + string name = 2; +} + +message DeleteTaskResponse {} + +message SuspendTaskRequest { + string atespace = 1; + string name = 2; +} + +message ResumeTaskRequest { + string atespace = 1; + string name = 2; +} + +message WatchTaskRequest { + string atespace = 1; + string name = 2; +} + +message WatchTaskResponse { + Task task = 1; + string action = 2; +} + +// Gateways +message GetGatewayRequest { + string atespace = 1; + string name = 2; +} + +message ListGatewaysRequest { + string atespace = 1; +} + +message ListGatewaysResponse { + repeated Gateway gateways = 1; +} + +message UpdateGatewayRequest { + Gateway gateway = 1; +} + +message DeleteGatewayRequest { + string atespace = 1; + string name = 2; +} + +message DeleteGatewayResponse {} + +// Workspaces +message GetWorkspaceRequest { + string atespace = 1; + string name = 2; +} + +message ListWorkspacesRequest { + string atespace = 1; +} + +message ListWorkspacesResponse { + repeated Workspace workspaces = 1; +} + +message UpdateWorkspaceRequest { + Workspace workspace = 1; +} + +message DeleteWorkspaceRequest { + string atespace = 1; + string name = 2; +} + +message DeleteWorkspaceResponse {} + +// Models +message GetModelRequest { + string atespace = 1; + string name = 2; +} + +message ListModelsRequest { + string atespace = 1; +} + +message ListModelsResponse { + repeated Model models = 1; +} + +message UpdateModelRequest { + Model model = 1; +} + +message DeleteModelRequest { + string atespace = 1; + string name = 2; +} + +message DeleteModelResponse {} + diff --git a/pkgs/substrate-apps/src/apps/link/test/link.test.ts b/pkgs/substrate-apps/src/apps/link/test/link.test.ts new file mode 100644 index 00000000..d095d6fb --- /dev/null +++ b/pkgs/substrate-apps/src/apps/link/test/link.test.ts @@ -0,0 +1,357 @@ +// The failure matrix of LINK-DESIGN.md section 8, one test per row, against the two doubles (fake ax with v0.3.0 +// upsert semantics and an optional P1; fake floor served through Effect's HTTP RPC server). Every server-set +// interval is scaled so one "second" is SEC ms. A crash is a fiber interrupt: nothing after it runs, and only what the +// journal fsynced survives, which is what kill -9 leaves. +import { mkdtempSync, rmSync } from "node:fs" +import { tmpdir } from "node:os" +import { join } from "node:path" +import { Deferred, Effect, Exit, Fiber } from "effect" +import { afterEach, beforeEach, describe, expect, it } from "vitest" +import type { AgentJob } from "../src/contract.ts" +import { rpcFloor } from "../src/floor.ts" +import { Journal } from "../src/journal.ts" +import { runLink } from "../src/link.ts" +import type { LinkConfig } from "../src/link.ts" +import { FakeAx } from "./fake-ax.ts" +import { FakeFloor } from "./fake-floor.ts" +import type { FloorConfig } from "./fake-floor.ts" + +const SEC = 20 +const A = "ultracode.mecattaf.dev/" +const JK = `${"cd".repeat(32)}:1` // round 4: FIELD-MAP 5a journal key (jobs.ts refuses a grant without it) +const job = (n: string, spec: Partial = {}, ann: Record = {}, w: Partial = {}): AgentJob => ({ + apiVersion: "ultracode.mecattaf.dev/v1alpha1", kind: "AgentJob", + metadata: { + name: `wf-test-${n}`, + labels: { [A + "run-id"]: "wf-test", [A + "workflow"]: "link-test", [A + "phase-index"]: "1" }, + annotations: { [A + "run-id-raw"]: "wf_test", [A + "label"]: `probe:${n}`, [A + "item-key"]: `wf_test#${n}`, [A + "journal-key"]: JK, [A + "phase-title"]: "Probe", ...ann } + }, + spec: { + "runs-on": ["seat:halogen", "runtime:gvisor"], + with: { prompt: `say ${n}`, prompt_ref: { sha256: "ab".repeat(32), bytes: 5, uri: `journal://wf_test/${n}/prompt.md` }, model: "halogen-qwen3.8-flash-next", ...w }, + ...spec + } +}) +const world = (o: Partial = {}) => ({ + floor: new FakeFloor({ cap: 2, leaseSeconds: 6, graceSeconds: 15, pollSeconds: 1, heartbeatSeconds: 2, maxAttempts: 3, secondMs: SEC, tokens: { "tok-nas": "nas-link-1" }, ...o }), + ax: new FakeAx() +}) +type World = ReturnType + +const GUEST = { completion: "guest" as const, shape: { atespace: "fleet", image: "ax-agent", gateway: "halogen", command: () => ["ax-agent", "pi"], completeUrl: "http://link.fleet.internal/guest" } } +const guestWorld = () => { const w = world({ tokens: { "tok-nas": { holder: "nas-link-1", guest: true } } }); w.ax.p1 = false; return w } + +function start(w: World, dir: string, over: Partial = {}, session = `s:${dir}`) { + const logs: Array> = [] + const stop = Effect.runSync(Deferred.make()) + const cfg: LinkConfig = { + holder: "nas-link-1", maxInFlight: 2, servedLabels: ["seat:halogen", "runtime:gvisor"], + shape: { atespace: "fleet", image: "ax-agent", gateway: "halogen", command: () => ["ax-agent", "pi"] }, + completion: "auto", secondMs: SEC, resyncMs: 30, pendingTimeoutMs: 1500, deleteAfterMs: 0, deadlineBackstopMs: 300, + createAttempts: 3, outboxBackoffMs: [20, 100], initialPollSeconds: 1, initialHeartbeatSeconds: 2, ...over + } + const fiber = Effect.runFork(Effect.scoped(Effect.gen(function*() { + const floor = yield* rpcFloor({ url: "http://floor.test", token: "tok-nas", sessionId: session, fetch: w.floor.fetch }) + return yield* runLink(cfg, { ax: w.ax, floor, journal: Journal.open(dir), log: (ev, f) => logs.push({ ev, ...f }), stop }) + }))) + return { + logs, + has: (ev: string, leaseId?: string) => logs.some((l) => l.ev === ev && (leaseId === undefined || l.leaseId === leaseId)), + crash: () => Effect.runPromise(Fiber.interrupt(fiber)), + drain: () => { Effect.runSync(Deferred.succeed(stop, undefined)); return Effect.runPromise(Fiber.await(fiber)) }, + exit: () => Effect.runPromise(Fiber.await(fiber)) + } +} +const until = async (what: string, pred: () => boolean, ms = 5000) => { + const t0 = Date.now() + while (!pred()) { if (Date.now() - t0 > ms) throw new Error(`timeout waiting for: ${what}`); await new Promise((r) => setTimeout(r, 5)) } +} +const sleep = (ms: number) => new Promise((r) => setTimeout(r, ms)) +const J = (w: World, n = "1") => w.floor.jobs.get(`wf-test-${n}`)! + +describe("conwip link failure matrix", () => { + let dir: string + beforeEach(() => { dir = mkdtempSync(join(tmpdir(), "conwip-link-")) }) + afterEach(() => rmSync(dir, { recursive: true, force: true })) + + it("F0 happy path on P1: one create, typed verdict with usage, Task deleted after the floor acknowledged", async () => { + const w = world(); w.floor.enqueue(job("1")) + const l = start(w, dir) + await until("running", () => w.ax.tasks.get("wf-test-1-a1")?.phase === "Running") + const env = Object.fromEntries(w.ax.tasks.get("wf-test-1-a1")!.task.spec.env.map((e) => [e.name, e.value])) + expect(env.AX_CONWIP_ITEM_KEY).toBe("wf_test#1"); expect(env.AX_CONWIP_ATTEMPT).toBe("1"); expect(env.AX_CONWIP_PROMPT).toBe("say 1") + expect(env.AX_CONWIP_LEASE_TOKEN).toBeUndefined() // P1 present: the guest carries no token at all + expect(w.ax.tasks.get("wf-test-1-a1")!.task.apiVersion).toBe("ax.io/v1alpha1") + w.ax.finish("wf-test-1-a1", 0, { answer: 42 }) + await until("floor done", () => J(w).state === "done") + expect([J(w).result, J(w).output, J(w).usage]).toEqual(["success", { answer: 42 }, { prompt_tokens: 11, completion_tokens: 7, tool_calls: 3 }]) + await until("janitor deleted the Task", () => !w.ax.tasks.has("wf-test-1-a1")) + expect(w.ax.updates.get("wf-test-1-a1")).toBe(1); expect(w.ax.upsertsOnExisting).toBe(0) + await l.drain() + }) + + it("F1 duplicate delivery: a redelivered grant creates nothing twice", async () => { + const w = world(); w.floor.enqueue(job("1")) + const l = start(w, dir) + await until("running", () => w.ax.tasks.get("wf-test-1-a1")?.phase === "Running") + w.floor.redeliver = ["wf-test-1-a1"] + await until("duplicate seen", () => l.has("duplicate-grant", "wf-test-1-a1")) + w.ax.finish("wf-test-1-a1", 0, { ok: true }) + await until("floor done", () => J(w).state === "done") + expect(w.ax.updates.get("wf-test-1-a1")).toBe(1); expect(w.ax.upsertsOnExisting).toBe(0) + await l.drain() + }) + + it("F2 crash after Lease, before UpdateTask: the restart creates exactly once", async () => { + const w = world(); w.floor.enqueue(job("1")); w.ax.blockCreate = true + const l1 = start(w, dir) + await until("grant journaled", () => l1.has("leased", "wf-test-1-a1")) + await l1.crash() + w.ax.blockCreate = false + const l2 = start(w, dir) + await until("running", () => w.ax.tasks.get("wf-test-1-a1")?.phase === "Running") + w.ax.finish("wf-test-1-a1", 0, { ok: 1 }) + await until("floor done", () => J(w).state === "done") + expect([J(w).result, J(w).attempt, w.ax.updates.get("wf-test-1-a1")]).toEqual(["success", 1, 1]) + await l2.drain() + }) + + it("F3 crash after UpdateTask, before the journal said created: the restart adopts, no second UpdateTask", async () => { + const w = world(); w.floor.enqueue(job("1")); w.ax.hangAfterCreate = true + const l1 = start(w, dir) + await until("task exists", () => w.ax.tasks.has("wf-test-1-a1")) + await l1.crash() + w.ax.hangAfterCreate = false + const l2 = start(w, dir) + await until("adopted", () => l2.has("adopted", "wf-test-1-a1")) + w.ax.finish("wf-test-1-a1", 0, { ok: 1 }) + await until("floor done", () => J(w).state === "done") + expect([J(w).result, w.ax.updates.get("wf-test-1-a1"), w.ax.upsertsOnExisting]).toEqual(["success", 1, 0]) + await l2.drain() + }) + + it("F4 crash while running; the agent finishes while the link is down; the restart reports once", async () => { + const w = world(); w.floor.enqueue(job("1")) + const l1 = start(w, dir) + await until("running", () => w.ax.tasks.get("wf-test-1-a1")?.phase === "Running") + await l1.crash() + w.ax.finish("wf-test-1-a1", 0, { late: true }) + const l2 = start(w, dir) + await until("floor done", () => J(w).state === "done") + expect([J(w).result, J(w).output, J(w).attempt]).toEqual(["success", { late: true }, 1]) + expect(w.floor.calls.filter((c) => c.rpc === "Complete").length).toBe(1) + await l2.drain() + }) + + it("F5 floor unreachable: the verdict waits in the outbox across a crash, the lease is orphaned not requeued, then lands", async () => { + const w = world(); w.floor.enqueue(job("1")) + const l1 = start(w, dir) + await until("running", () => w.ax.tasks.get("wf-test-1-a1")?.phase === "Running") + w.floor.down = true + w.ax.finish("wf-test-1-a1", 0, { v: 1 }) + await until("outbox keeps", () => l1.has("outbox-keep", "wf-test-1-a1")) + await l1.crash() + await sleep(7 * SEC) // past the 6 s lease, well inside the 15 s grace + const l2 = start(w, dir) + await sleep(3 * SEC) + w.floor.down = false + w.floor.sweep() // the DO alarm fires + await until("floor done", () => J(w).state === "done") + expect([J(w).result, J(w).attempt]).toEqual(["success", 1]) + expect(J(w).history.some((h) => h.startsWith("requeue"))).toBe(false) + await l2.drain() + }) + + it("F6 ax unreachable: capacity 0 on every poll, held leases still renewed, work resumes when ax returns", async () => { + const w = world(); w.floor.enqueue(job("1")) + const l = start(w, dir) + await until("running", () => w.ax.tasks.get("wf-test-1-a1")?.phase === "Running") + w.ax.up = false + await until("link saw ax down", () => l.has("ax-down")) + const mark = w.floor.calls.length + w.floor.enqueue(job("2")) + await sleep(10 * SEC) // longer than the 6 s lease + if (process.env.LINK_DEBUG) console.log(JSON.stringify({ history: J(w).history, calls: w.floor.calls.slice(mark).map((c) => [c.rpc, c.capacity, c.leaseIds]), logs: l.logs.slice(-15) })) + const during = w.floor.calls.slice(mark).filter((c) => c.rpc === "Lease") + expect(during.length).toBeGreaterThan(0) + expect(during.every((c) => c.capacity === 0 && c.granted === 0)).toBe(true) + expect([J(w).state, J(w, "2").state]).toEqual(["leased", "queued"]) + w.ax.up = true + await until("second job created", () => w.ax.tasks.has("wf-test-2-a1")) + await l.drain() + }) + + it("F7 lease expiry, link back inside the grace: the orphaned lease is re-adopted, same attempt, no second Task", async () => { + const w = world(); w.floor.enqueue(job("1")) + const l1 = start(w, dir) + await until("running", () => w.ax.tasks.get("wf-test-1-a1")?.phase === "Running") + await l1.crash() + await sleep(7 * SEC); w.floor.sweep() + expect([J(w).state, w.floor.wip()]).toEqual(["orphaned", 1]) // L3: never reclaim a live slot + const l2 = start(w, dir) + await until("re-adopted", () => J(w).history.includes("re-adopted")) + w.ax.finish("wf-test-1-a1", 0, { ok: 1 }) + await until("floor done", () => J(w).state === "done") + expect([J(w).result, J(w).attempt, w.ax.updates.get("wf-test-1-a1")]).toEqual(["success", 1, 1]) + await l2.drain() + }) + + it("F8 journal lost: the holder's complete heartbeat omits the orphan, attempt 2 supersedes it, old Task deleted first", async () => { + const w = world(); w.floor.enqueue(job("1")) + const l1 = start(w, dir) + await until("running", () => w.ax.tasks.get("wf-test-1-a1")?.phase === "Running") + await l1.crash() + rmSync(dir, { recursive: true, force: true }) + await sleep(7 * SEC); w.floor.sweep() + const l2 = start(w, dir, {}, "s:fresh-state") + await until("attempt 2 created", () => w.ax.tasks.has("wf-test-1-a2")) + expect(J(w).history).toContain("requeue:omitted") + expect(w.ax.events.indexOf("delete:wf-test-1-a1")).toBeGreaterThanOrEqual(0) + expect(w.ax.events.indexOf("delete:wf-test-1-a1")).toBeLessThan(w.ax.events.indexOf("update:wf-test-1-a2")) + w.ax.tick(); w.ax.finish("wf-test-1-a2", 0, { ok: 2 }) + await until("floor done", () => J(w).state === "done") + expect([J(w).result, J(w).attempt]).toEqual(["success", 2]) + await l2.drain() + }) + + it("F9 link dead past the grace: the alarm requeues; the returning link drops attempt 1 (lost) and runs attempt 2", async () => { + const w = world(); w.floor.enqueue(job("1")) + const l1 = start(w, dir) + await until("running", () => w.ax.tasks.get("wf-test-1-a1")?.phase === "Running") + await l1.crash() + await sleep(7 * SEC); w.floor.sweep() + await sleep(16 * SEC); w.floor.sweep() + expect(J(w).history).toContain("requeue:grace") + const l2 = start(w, dir) + await until("attempt 2 created", () => w.ax.tasks.has("wf-test-1-a2")) + expect(w.ax.deletes).toContain("wf-test-1-a1") + w.ax.tick(); w.ax.finish("wf-test-1-a2", 0, { ok: 2 }) + await until("floor done", () => J(w).state === "done") + expect([J(w).result, J(w).attempt]).toEqual(["success", 2]) + // the late verdict of attempt 1 can never be recorded: the floor fences on the attempt + expect(w.floor.complete({ leaseId: "wf-test-1-a1", attempt: 1, result: "success" })).toEqual({ code: "stale-attempt" }) + await l2.drain() + }) + + it("F10 cancel: carried in the heartbeat reply, DeleteTask, then Complete(cancelled) once the Task is gone", async () => { + const w = world(); w.floor.enqueue(job("1")) + const l = start(w, dir) + await until("running", () => w.ax.tasks.get("wf-test-1-a1")?.phase === "Running") + w.floor.cancel("wf-test-1") + await until("floor done", () => J(w).state === "done") + if (process.env.LINK_DEBUG) console.log(JSON.stringify({ history: J(w).history, events: w.ax.events, phase: w.ax.tasks.get("wf-test-1-a1")?.phase, logs: l.logs })) + expect([J(w).result, J(w).attempt, w.ax.tasks.has("wf-test-1-a1")]).toEqual(["cancelled", 1, false]) + await l.drain() + }) + + it("F11 stock v0.3.0 (no P1), guest mode configured: the Task stays Running after exit; the guest completes with its lease token; the link deletes on lost", async () => { + const w = guestWorld(); w.floor.enqueue(job("1")) + const l = start(w, dir, GUEST) + await until("running", () => w.ax.tasks.get("wf-test-1-a1")?.phase === "Running") + const env = Object.fromEntries(w.ax.tasks.get("wf-test-1-a1")!.task.spec.env.map((e) => [e.name, e.value])) + expect(env.AX_CONWIP_COMPLETE_URL).toBe("http://link.fleet.internal/guest") + expect(Object.values(env).some((v) => v.includes("tok-nas"))).toBe(false) // the fleet bearer never enters the sandbox + w.ax.finish("wf-test-1-a1", 0) + await sleep(5 * SEC) + expect(w.ax.tasks.get("wf-test-1-a1")!.phase).toBe("Running") // FIELD-MAP D3: no terminal phase without P1 + expect(w.floor.guestComplete("wf-test-1-a1", env.AX_CONWIP_LEASE_TOKEN!, "success", { guest: true })).toMatchObject({ duplicate: false }) + expect(w.floor.guestComplete("wf-test-1-a1", env.AX_CONWIP_LEASE_TOKEN!, "success", { guest: true })).toEqual({ refused: "token" }) + await until("link deleted the still-Running Task", () => !w.ax.tasks.has("wf-test-1-a1")) + expect([J(w).result, J(w).output]).toEqual(["success", { guest: true }]) + await l.drain() + }) + + it("F12 stuck Running (no P1, the guest never reports): the floor's deadline cancels it and ax frees the worker", async () => { + const w = guestWorld(); w.floor.enqueue(job("1", { "timeout-minutes": 1 })) // 60 s = 1200 ms here + const l = start(w, dir, GUEST) + await until("running", () => w.ax.tasks.get("wf-test-1-a1")?.phase === "Running") + w.ax.finish("wf-test-1-a1", 0) // exited, invisible to ax without P1 + await until("floor done", () => J(w).state === "done", 8000) + expect([J(w).result, J(w).output]).toEqual(["cancelled", { reason: "deadline" }]) + await until("Task gone", () => !w.ax.tasks.has("wf-test-1-a1")) + await l.drain() + }) + + it("F13 deadline passes while the floor is unreachable: the local backstop deletes the Task and queues the verdict", async () => { + const w = guestWorld(); w.floor.enqueue(job("1", { "timeout-minutes": 1 })) + const l = start(w, dir, GUEST) + await until("running", () => w.ax.tasks.get("wf-test-1-a1")?.phase === "Running") + w.floor.down = true + await until("backstop verdict", () => l.logs.some((x) => x.ev === "verdict" && x.reason === "deadline-exceeded"), 8000) + await until("Task deleted", () => !w.ax.tasks.has("wf-test-1-a1") || w.ax.tasks.get("wf-test-1-a1")!.phase === "Terminating") + await l.crash() + }) + + it("F14 stuck Pending: pre-start failure at the timeout, retried by the floor as attempt 2 superseding attempt 1", async () => { + const w = world(); w.floor.enqueue(job("1")); w.ax.holdPending.add("wf-test-1-a1") + const l = start(w, dir, { pendingTimeoutMs: 200 }) + await until("attempt 2 created", () => w.ax.tasks.has("wf-test-1-a2"), 8000) + expect(J(w).history).toContain("requeue:pre-start/pending-timeout") + expect(w.ax.deletes).toContain("wf-test-1-a1") + w.ax.tick(); w.ax.finish("wf-test-1-a2", 0, { ok: 2 }) + await until("floor done", () => J(w).state === "done") + expect([J(w).result, J(w).attempt]).toEqual(["success", 2]) + await l.drain() + }) + + it("F15 name conflict: a foreign Task with the lease's name is never overwritten (create-only, L4)", async () => { + const w = world(); w.floor.enqueue(job("1")) + w.ax.put({ apiVersion: "ax.io/v1alpha1", kind: "Task", metadata: { name: "wf-test-1-a1", atespace: "fleet" }, spec: { image: "someone-else", command: ["sleep"], env: [] } }) + const l = start(w, dir) + await until("floor done", () => J(w).state === "done") + expect([J(w).result, (J(w).output as { reason: string }).reason]).toEqual(["failure", "pre-start/name-conflict"]) + expect([w.ax.updates.get("wf-test-1-a1"), w.ax.tasks.get("wf-test-1-a1")?.task.spec.image]).toEqual([undefined, "someone-else"]) + await l.drain() + }) + + it("F16 deterministic refusals are final and touch nothing in ax: env over budget, unresolved prompt, prompt too big to inline (B12)", async () => { + const w = world({ cap: 3 }) + w.floor.enqueue(job("1", {}, {}, { schema: { description: "x".repeat(20000) } })) + w.floor.enqueue(job("2", {}, { [A + "prompt-unresolved"]: "no agent() call carries label" })) + w.floor.enqueue(job("3", {}, {}, { prompt: "p".repeat(17000) })) + const l = start(w, dir, { maxInFlight: 3 }) + await until("all done", () => J(w).state === "done" && J(w, "2").state === "done" && J(w, "3").state === "done") + expect([(J(w, "3").output as { reason: string }).reason, J(w, "3").attempt]).toEqual(["pre-start/prompt-file-route-missing", 1]) + expect([(J(w).output as { reason: string }).reason, J(w).attempt]).toEqual(["pre-start/env-over-budget", 1]) + expect([(J(w, "2").output as { reason: string }).reason, J(w, "2").attempt]).toEqual(["pre-start/prompt-unresolved", 1]) + expect(w.ax.updates.size).toBe(0) + await l.drain() + }) + + it("F17 a second replica with the same identity is refused (409) and exits; the first keeps its session", async () => { + const w = world(); w.floor.enqueue(job("1")) + const l1 = start(w, dir) + await until("running", () => w.ax.tasks.get("wf-test-1-a1")?.phase === "Running") + const dir2 = mkdtempSync(join(tmpdir(), "conwip-link-b-")) + const l2 = start(w, dir2, {}, "s:second-replica") + const exit = await l2.exit() + rmSync(dir2, { recursive: true, force: true }) + expect(Exit.isFailure(exit)).toBe(true) + expect(JSON.stringify(exit)).toContain("session-conflict") + w.ax.finish("wf-test-1-a1", 0, { ok: 1 }) + await until("floor done", () => J(w).state === "done") + await l1.drain() + }) + + it("F19 ax lost a Task it had created (Redis loss): infra/task-lost, the floor retries as attempt 2", async () => { + const w = world(); w.floor.enqueue(job("1")) + const l = start(w, dir) + await until("running", () => w.ax.tasks.get("wf-test-1-a1")?.phase === "Running") + w.ax.tasks.delete("wf-test-1-a1") // gone without a DeleteTask: the store forgot it + await until("attempt 2 created", () => w.ax.tasks.has("wf-test-1-a2")) + expect(J(w).history).toContain("requeue:infra/task-lost") + w.ax.tick(); w.ax.finish("wf-test-1-a2", 0, { ok: 2 }) + await until("floor done", () => J(w).state === "done") + expect([J(w).result, J(w).attempt]).toEqual(["success", 2]) + await l.drain() + }) + + it("F18 drain on SIGTERM: stop leasing, flush the outbox, leave running Tasks to be re-adopted", async () => { + const w = world(); w.floor.enqueue(job("1")) + const l = start(w, dir) + await until("running", () => w.ax.tasks.get("wf-test-1-a1")?.phase === "Running") + const exit = await l.drain() + expect(Exit.isSuccess(exit)).toBe(true) + expect([w.ax.tasks.get("wf-test-1-a1")?.phase, J(w).state]).toEqual(["Running", "leased"]) + }) +}) diff --git a/pkgs/substrate-apps/src/apps/link/test/review-r1.test.ts b/pkgs/substrate-apps/src/apps/link/test/review-r1.test.ts new file mode 100644 index 00000000..5e49eeb7 --- /dev/null +++ b/pkgs/substrate-apps/src/apps/link/test/review-r1.test.ts @@ -0,0 +1,391 @@ +// Review round 1 (2026-09-23): the ten findings, each as a regression test that failed before its fix. The scratch +// repros are in /home/tom/today/evals-2026-09-23/link/scratch-r1-{0,1,2}; the log is link/REVIEW-LOG.md. +import { mkdtempSync, rmSync } from "node:fs" +import { tmpdir } from "node:os" +import { join } from "node:path" +import { Deferred, Effect, Fiber } from "effect" +import { afterEach, beforeEach, describe, expect, it } from "vitest" +import { AxError } from "../src/ax.ts" +import type { AgentJob, AxTask } from "../src/contract.ts" +import { rpcFloor } from "../src/floor.ts" +import type { FloorOptions } from "../src/floor.ts" +import { Journal } from "../src/journal.ts" +import { runLink } from "../src/link.ts" +import type { LinkConfig } from "../src/link.ts" +import { FakeAx } from "./fake-ax.ts" +import { FakeFloor } from "./fake-floor.ts" +import type { FloorConfig } from "./fake-floor.ts" +import { ConfigInvalid, readLinkEnv } from "../src/config.ts" +import type { FloorApi } from "../src/floor.ts" +import { axTaskFromGrant } from "../src/jobs.ts" + +const SEC = 20 +const UNAVAILABLE = 14 +const A = "ultracode.mecattaf.dev/" +const JK = `${"cd".repeat(32)}:1` // round 4: FIELD-MAP 5a journal key (jobs.ts refuses a grant without it) +const job = (n: string, spec: Partial = {}): AgentJob => ({ + apiVersion: "ultracode.mecattaf.dev/v1alpha1", kind: "AgentJob", + metadata: { + name: `wf-test-${n}`, + labels: { [A + "run-id"]: "wf-test", [A + "workflow"]: "link-test", [A + "phase-index"]: "1" }, + annotations: { [A + "run-id-raw"]: "wf_test", [A + "label"]: `probe:${n}`, [A + "item-key"]: `wf_test#${n}`, [A + "journal-key"]: JK, [A + "phase-title"]: "Probe" } + }, + spec: { + "runs-on": ["seat:halogen", "runtime:gvisor"], + with: { prompt: `say ${n}`, prompt_ref: { sha256: "ab".repeat(32), bytes: 5, uri: `journal://wf_test/${n}/prompt.md` }, model: "halogen-qwen3.8-flash-next" }, + ...spec + } +}) +const world = (o: Partial = {}) => ({ + floor: new FakeFloor({ cap: 2, leaseSeconds: 6, graceSeconds: 15, pollSeconds: 1, heartbeatSeconds: 2, maxAttempts: 3, secondMs: SEC, tokens: { "tok-nas": "nas-link-1" }, ...o }), + ax: new FakeAx() +}) +type World = ReturnType +const foreign = (name: string): AxTask => ({ apiVersion: "ax.io/v1alpha1", kind: "Task", metadata: { name, atespace: "fleet" }, spec: { image: "someone-else", command: ["true"], env: [] } }) + +interface Extra { fetch?: typeof globalThis.fetch; timeoutsMs?: FloorOptions["timeoutsMs"]; onEndpoint?: (u: string) => void } +function start(w: World, dir: string, over: Partial = {}, x: Extra = {}) { + const logs: Array> = [] + const stop = Effect.runSync(Deferred.make()) + const cfg: LinkConfig = { + holder: "nas-link-1", maxInFlight: 2, servedLabels: ["seat:halogen", "runtime:gvisor"], + shape: { atespace: "fleet", image: "ax-agent", gateway: "halogen", command: () => ["ax-agent", "pi"] }, + completion: "auto", secondMs: SEC, resyncMs: 30, pendingTimeoutMs: 1500, deleteAfterMs: 0, deadlineBackstopMs: 300, + createAttempts: 3, outboxBackoffMs: [20, 100], initialPollSeconds: 1, initialHeartbeatSeconds: 2, fenceTimeoutMs: 300, resultReadTries: 3, ...over + } + const fiber = Effect.runFork(Effect.scoped(Effect.gen(function*() { + const floor = yield* rpcFloor({ url: "http://floor.test", token: "tok-nas", sessionId: `s:${dir}`, fetch: x.fetch ?? w.floor.fetch, ...(x.timeoutsMs ? { timeoutsMs: x.timeoutsMs } : {}) }) + return yield* runLink(cfg, { ax: w.ax, floor, journal: Journal.open(dir), log: (ev, f) => logs.push({ ev, ...f }), stop, floorUrl: "http://floor.test", ...(x.onEndpoint ? { onEndpoint: x.onEndpoint } : {}) }) + }))) + return { + logs, + has: (ev: string, leaseId?: string) => logs.some((l) => l.ev === ev && (leaseId === undefined || l.leaseId === leaseId)), + verdict: (reason: string) => logs.some((l) => l.ev === "verdict" && l.reason === reason), + crash: () => Effect.runPromise(Fiber.interrupt(fiber)), + drain: () => { Effect.runSync(Deferred.succeed(stop, undefined)); return Promise.race([Effect.runPromise(Fiber.await(fiber)), sleep(1500)]) } + } +} +const until = async (what: string, pred: () => boolean, ms = 5000) => { + const t0 = Date.now() + while (!pred()) { if (Date.now() - t0 > ms) throw new Error(`timeout waiting for: ${what}`); await new Promise((r) => setTimeout(r, 5)) } +} +const sleep = (ms: number) => new Promise((r) => setTimeout(r, ms)) +const J = (w: World, n = "1") => w.floor.jobs.get(`wf-test-${n}`)! +const liveOwn = (w: World) => [...w.ax.tasks.keys()].filter((k) => k.startsWith("wf-test-") && w.ax.live(k) && !["Completed", "Failed"].includes(w.ax.tasks.get(k)!.phase)).length + + +const bodyOf = async (input: unknown, init?: RequestInit) => { + const b = init?.body + if (typeof b === "string") return b + if (b instanceof Uint8Array) return new TextDecoder().decode(b) + if (b) return await new Response(b as ConstructorParameters[0]).text() + return (input instanceof Request) ? await input.clone().text() : "" +} +const isComplete = async (input: unknown, init?: RequestInit) => (await bodyOf(input, init)).includes('"Complete"') +const stats = { seq: 1, cap: 2, wip: 0, queued: 0, done: 0 } +const grantOf = (n: string, extra: Record = {}) => ({ + leaseId: `wf-test-${n}-a1`, attempt: 1, job: job(n), + lease: { holderIdentity: "nas-link-1", leaseDurationSeconds: 30, acquireTime: Date.now(), renewTime: Date.now(), leaseTransitions: 0 }, ...extra +}) as any +/** A floor that hands out `grants` once and renews everything: for link-side properties the double cannot stage. */ +const stubFloor = (grants: Array) => { + let sent = false + const completes: Array = [] + const api: FloorApi = { + lease: (p) => Effect.sync(() => { const g = !sent && p.capacity > 0 ? grants : []; if (g.length) sent = true; return { grants: g, stats, nextPollSeconds: 1, heartbeatSeconds: 2 } }), + heartbeat: (p) => Effect.succeed({ renewed: [...p.leaseIds], lost: [], cancelRequested: [], stats }), + complete: (p) => Effect.sync(() => { completes.push(p); return { duplicate: false, stats } }) + } + return { api, completes } +} +const cfgOf = (over: Partial = {}): LinkConfig => ({ + holder: "nas-link-1", maxInFlight: 2, servedLabels: ["seat:halogen", "runtime:gvisor"], + shape: { atespace: "fleet", image: "ax-agent", gateway: "halogen", command: () => ["ax-agent", "pi"] }, + completion: "auto", secondMs: SEC, resyncMs: 30, pendingTimeoutMs: 60_000, deleteAfterMs: 60_000, deadlineBackstopMs: 300, + createAttempts: 3, outboxBackoffMs: [20, 100], initialPollSeconds: 1, initialHeartbeatSeconds: 2, fenceTimeoutMs: 300, resultReadTries: 3, ...over +}) +const runStub = async (cfg: LinkConfig, ax: FakeAx, floor: FloorApi, ms: number) => { + const d = mkdtempSync(join(tmpdir(), "conwip-link-r1s-")) + const logs: Array> = [] + const stop = Effect.runSync(Deferred.make()) + const f = Effect.runFork(runLink(cfg, { ax, floor, journal: Journal.open(d), log: (ev, x) => logs.push({ ev, ...x }), stop })) + await sleep(ms) + Effect.runSync(Deferred.succeed(stop, undefined)); await Effect.runPromise(Fiber.await(f)); rmSync(d, { recursive: true, force: true }) + return logs +} + +describe("review round 1: fixed", () => { + let dir: string + let worlds: Array = [] + const W = (o: Partial = {}) => { const w = world(o); worlds.push(w); return w } + beforeEach(() => { dir = mkdtempSync(join(tmpdir(), "conwip-link-r1-")) }) + afterEach(() => { for (const w of worlds) w.floor.close(); worlds = []; rmSync(dir, { recursive: true, force: true }) }) + + // ------------------------------------------------------------------ findings 1 and 2: one verdict, one application + it("R1-1 no fault: every verdict costs exactly one Complete (single-flight outbox)", async () => { + const w = W({ cap: 8 }) + for (let i = 1; i <= 8; i++) w.floor.enqueue(job(String(i))) + const l = start(w, dir, { maxInFlight: 8, resyncMs: 5 }) + await until("all running", () => [1, 2, 3, 4, 5, 6, 7, 8].every((i) => w.ax.tasks.get(`wf-test-${i}-a1`)?.phase === "Running")) + for (let i = 1; i <= 8; i++) w.ax.finish(`wf-test-${i}-a1`, 0, { i }) + await until("all done", () => [1, 2, 3, 4, 5, 6, 7, 8].every((i) => J(w, String(i)).state === "done")) + await sleep(100) + const per = new Map() + for (const c of w.floor.calls.filter((c) => c.rpc === "Complete")) per.set(c.leaseIds![0]!, (per.get(c.leaseIds![0]!) ?? 0) + 1) + expect([...per.values()]).toEqual([1, 1, 1, 1, 1, 1, 1, 1]) + expect(l.logs.filter((x) => x.ev === "complete").length).toBe(8) + await l.drain() + }) + + it("R1-1 (D5) one retryable failure with a budget of two releases leases attempt 2", async () => { + const w = W({ infraAttempts: 2 }); w.floor.enqueue(job("1")); w.ax.holdPending.add("wf-test-1-a1") + const l = start(w, dir, { pendingTimeoutMs: 150 }) + await until("attempt 2 leased", () => J(w).state === "leased" && J(w).attempt === 2, 4000) + expect(J(w).infraSpent).toBe(1) + expect(J(w).history).not.toContain("withdrawn:a2") + await l.drain() + }) + + it("R1-2 (D2) a Complete whose reply is lost after the floor committed is resent and answered duplicate, not applied twice", async () => { + const w = W({ infraAttempts: 2 }); w.floor.enqueue(job("1")); w.ax.holdPending.add("wf-test-1-a1") + let dropped = 0 + const fetch: typeof globalThis.fetch = async (input, init) => { + const c = await isComplete(input, init) + const res = await w.floor.fetch(input, init) + if (c && dropped === 0) { dropped++; throw new TypeError("fetch failed: reply lost after the floor committed") } + return res + } + const l = start(w, dir, { pendingTimeoutMs: 150 }, { fetch }) + await until("attempt 2 leased", () => J(w).state === "leased" && J(w).attempt === 2, 4000) + await until("the resend is acknowledged", () => l.logs.some((x) => x.ev === "complete" && x.leaseId === "wf-test-1-a1")) + expect(dropped).toBe(1) + expect(J(w).infraSpent).toBe(1) + expect(l.logs.find((x) => x.ev === "complete" && x.leaseId === "wf-test-1-a1")!.duplicate).toBe(true) + await l.drain() + }) + + // ------------------------------------------------------------------ finding 3: the create window is write-ahead + it("R1-3 (D1) crash after UpdateTask landed, before `created`, then a cancel: the Task is deleted and NotFound precedes `cancelled`", async () => { + const w = W(); w.floor.enqueue(job("1")); w.ax.hangAfterCreate = true + const l1 = start(w, dir) + await until("Task written to ax", () => w.ax.tasks.has("wf-test-1-a1")) + await l1.crash() + w.ax.hangAfterCreate = false + w.floor.cancel("wf-test-1") + const l2 = start(w, dir) + await until("floor done", () => J(w).state === "done") + expect(J(w).result).toBe("cancelled") + expect(w.ax.deletes).toContain("wf-test-1-a1") + expect(w.ax.tasks.has("wf-test-1-a1")).toBe(false) // NotFound before the verdict (reconcile reports on gone) + const evs = l2.logs.filter((x) => x.leaseId === "wf-test-1-a1" || x.task === "wf-test-1-a1").map((x) => x.ev) + expect(evs.indexOf("delete")).toBeLessThan(evs.indexOf("verdict")) + await l2.drain() + expect(Journal.open(dir).recs.has("wf-test-1-a1")).toBe(false) // compacted only once ax is clean + }) + + it("R1-3 (D1 lost variant) crash in the create window, the floor gives up on the job: `lost` deletes the maybe-created Task", async () => { + const w = W({ infraAttempts: 1 }); w.floor.enqueue(job("1")); w.ax.hangAfterCreate = true + const l1 = start(w, dir) + await until("Task written to ax", () => w.ax.tasks.has("wf-test-1-a1")) + await l1.crash() + w.ax.hangAfterCreate = false + await sleep(7 * SEC); w.floor.sweep(); await sleep(16 * SEC); w.floor.sweep() + expect(J(w).state).toBe("done") // budget spent at the first release: no re-grant + const l2 = start(w, dir) + await until("lost", () => l2.has("lost", "wf-test-1-a1")) + await until("Task gone", () => !w.ax.tasks.has("wf-test-1-a1")) + expect(w.ax.deletes).toContain("wf-test-1-a1") + await l2.drain() + }) + + // ------------------------------------------------------------------ findings 4 and 8: B15 + it("R1-8 (B15 deterministic) the first Complete after the floor returns is lost in transit: no Lease runs before it lands", async () => { + const w = W(); w.floor.enqueue(job("1")) + let failCompletes = 0 + const fetch: typeof globalThis.fetch = async (input, init) => { + if (failCompletes > 0 && await isComplete(input, init)) { failCompletes--; throw new TypeError("fetch failed") } + return w.floor.fetch(input, init) + } + const l = start(w, dir, {}, { fetch }) + await until("running", () => w.ax.tasks.get("wf-test-1-a1")?.phase === "Running") + w.floor.down = true + w.ax.finish("wf-test-1-a1", 0, { v: 1 }) + await until("outbox keeps", () => l.has("outbox-keep", "wf-test-1-a1")) + await until("requeued", () => J(w).history.includes("requeue:grace"), 3000) + failCompletes = 1 + const mark = w.floor.calls.length + w.floor.down = false + await until("floor done", () => J(w).state === "done", 4000) + expect([J(w).result, J(w).output, J(w).attempt]).toEqual(["success", { v: 1 }, 1]) + // round 3 (finding 12): the gate is asserted on the floor's side, not on the log line + const after = w.floor.calls.slice(mark), landed = after.findIndex((c) => c.rpc === "Complete") + expect(after.slice(0, landed).filter((c) => c.rpc === "Lease").map((c) => c.capacity).filter((c) => c !== 0)).toEqual([]) + expect(J(w).history).not.toContain("leased:a2") + expect(w.ax.updates.has("wf-test-1-a2")).toBe(false) + expect(l.has("complete-dropped", "wf-test-1-a1")).toBe(false) + await l.drain() + }) + + it("R1-4 (D3) Complete answers 503 while Lease answers: the gate asks for nothing until the success lands", async () => { + const w = W(); w.floor.enqueue(job("1")) + let failComplete = false + const fetch: typeof globalThis.fetch = async (input, init) => { + if (failComplete && await isComplete(input, init)) return new Response("upstream timeout", { status: 503 }) + return w.floor.fetch(input, init) + } + const l = start(w, dir, {}, { fetch }) + await until("running", () => w.ax.tasks.get("wf-test-1-a1")?.phase === "Running") + w.floor.down = true + w.ax.finish("wf-test-1-a1", 0, { answer: 42 }) + await until("verdict written", () => l.has("verdict", "wf-test-1-a1")) + await until("floor requeued attempt 2", () => J(w).state === "queued" && J(w).attempt === 2, 4000) + failComplete = true + const mark = w.floor.calls.length + w.floor.down = false + await sleep(400) // several polls while only Complete fails + expect(l.has("lease-gated-by-outbox")).toBe(true) + // round 3 (finding 12): every Lease while the success is undelivered asks for nothing, and nothing is leased + const leases = w.floor.calls.slice(mark).filter((c) => c.rpc === "Lease") + expect([leases.length > 0, leases.filter((c) => c.capacity !== 0).length, J(w).history.includes("leased:a2")]).toEqual([true, 0, false]) + expect(w.ax.tasks.has("wf-test-1-a2")).toBe(false) + // a1 may be deleted as `lost` (its verdict is already journaled); it is never fenced as `superseded` + expect(l.logs.some((x) => x.ev === "delete" && x.task === "wf-test-1-a1" && x.why === "superseded")).toBe(false) + failComplete = false + await until("floor done", () => J(w).state === "done", 4000) + expect([J(w).result, J(w).output, J(w).attempt]).toEqual(["success", { answer: 42 }, 1]) + await l.drain() + }) + + it("R1-4 (D3 at dispatch) a grant that supersedes an undelivered success is withdrawn, never fenced or created", async () => { + const w = W(); w.floor.enqueue(job("1")) + let failComplete = false + const fetch: typeof globalThis.fetch = async (input, init) => { + if (failComplete && await isComplete(input, init)) return new Response("upstream timeout", { status: 503 }) + return w.floor.fetch(input, init) + } + const l = start(w, dir, {}, { fetch, }) + await until("running", () => w.ax.tasks.get("wf-test-1-a1")?.phase === "Running") + w.floor.down = true + w.ax.finish("wf-test-1-a1", 0, { answer: 42 }) + await until("verdict written", () => l.has("verdict", "wf-test-1-a1")) + await until("floor requeued attempt 2", () => J(w).state === "queued" && J(w).attempt === 2, 4000) + failComplete = true + w.floor.o.overGrant = 1 // a floor that grants on a capacity-0 Lease: the link's own defence must hold + w.floor.down = false + await until("a2 granted", () => l.has("leased", "wf-test-1-a2"), 4000) + w.floor.o.overGrant = 0 + await until("dispatch waits for the verdict", () => l.has("supersede-waits-for-verdict", "wf-test-1-a2")) + failComplete = false + await until("floor done", () => J(w).state === "done", 4000) + expect([J(w).result, J(w).output, J(w).attempt]).toEqual(["success", { answer: 42 }, 1]) + expect(J(w).history).toContain("withdrawn:a2") + await until("a2 withdrawn locally", () => l.has("withdrawn", "wf-test-1-a2")) + expect(w.ax.updates.has("wf-test-1-a2")).toBe(false) + // the finished Task was not fenced before its verdict (a `lost` delete is fine: the verdict is already journaled) + expect(l.logs.some((x) => x.ev === "delete" && x.task === "wf-test-1-a1" && x.why === "superseded")).toBe(false) + await l.drain() + }) + + // ------------------------------------------------------------------ finding 5: supersedes is scoped to own Tasks + it("R1-5 supersedes naming a Task this link never created: no DeleteTask, the grant is refused", async () => { + const ax = new FakeAx() + ax.put(foreign("someone-elses-task"), "Running") + const f = stubFloor([grantOf("1", { attempt: 2, leaseId: "wf-test-1-a2", supersedes: ["someone-elses-task"] })]) + const logs = await runStub(cfgOf(), ax, f.api, 600) + expect(ax.deletes).toEqual([]) + expect(ax.live("someone-elses-task")).toBe(true) + expect(ax.tasks.has("wf-test-1-a2")).toBe(false) + expect(f.completes.map((c) => [c.leaseId, c.output?.reason])).toEqual([["wf-test-1-a2", "pre-start/supersedes-foreign"]]) + expect(logs.some((x) => x.ev === "supersedes-foreign")).toBe(true) + }) + + it("R1-5 an earlier attempt of the same job that carries another holder's mark is not deleted either", async () => { + const ax = new FakeAx() + const built = axTaskFromGrant(grantOf("1"), { atespace: "fleet", image: "ax-agent", gateway: "halogen", command: () => ["ax-agent", "pi"], holder: "other-link" }) + if (built._tag !== "ok") throw new Error("build") + ax.put(built.task, "Running") + const f = stubFloor([grantOf("1", { attempt: 2, leaseId: "wf-test-1-a2", supersedes: ["wf-test-1-a1"] })]) + await runStub(cfgOf(), ax, f.api, 600) + expect(ax.deletes).toEqual([]) + expect(f.completes.map((c) => c.output?.reason)).toEqual(["pre-start/supersedes-foreign"]) + }) + + // ------------------------------------------------------------------ finding 6: the gate is checked per UpdateTask + it("R1-6 the Gateway disappears during the UpdateTask retry window: nothing is created while it is missing", async () => { + const ax = new FakeAx() + let failed = false + const orig = ax.createTask + ax.createTask = (task) => { if (!failed) { failed = true; ax.gateways.delete("halogen"); return Effect.fail(new AxError(UNAVAILABLE, "transient")) } return orig(task) } + const f = stubFloor([grantOf("3")]) + const logs = await runStub(cfgOf(), ax, f.api, 1000) + expect(logs.some((x) => x.ev === "gateway-missing")).toBe(true) + expect(ax.tasks.has("wf-test-3-a1")).toBe(false) + expect(logs.some((x) => x.ev === "gate-closed-before-create")).toBe(true) + }) + + it("R1-6 ax returns without P1 during the retry window: nothing is created in p1 mode", async () => { + const ax = new FakeAx() + let failed = false + const orig = ax.createTask + ax.createTask = (task) => { + if (!failed) { failed = true; ax.up = false; setTimeout(() => { ax.p1 = false; ax.up = true }, 60); return Effect.fail(new AxError(UNAVAILABLE, "transient")) } + return orig(task) + } + const f = stubFloor([grantOf("4")]) + const logs = await runStub(cfgOf(), ax, f.api, 1200) + expect(logs.some((x) => x.ev === "p1-missing")).toBe(true) + expect(ax.tasks.has("wf-test-4-a1")).toBe(false) + }) + + // ------------------------------------------------------------------ finding 7: guest mode fails closed + it("R1-7 guest mode with no leaseToken in the grant: refused before UpdateTask, logged once", async () => { + const ax = new FakeAx(); ax.p1 = false + const f = stubFloor([grantOf("2")]) + const logs = await runStub(cfgOf({ completion: "guest", shape: { atespace: "fleet", image: "ax-agent", gateway: "halogen", command: () => ["ax-agent", "pi"], completeUrl: "http://link.fleet.internal/guest" } }), ax, f.api, 600) + expect(ax.tasks.has("wf-test-2-a1")).toBe(false) + expect(f.completes.map((c) => c.output?.reason)).toEqual(["pre-start/lease-token-missing"]) + expect(logs.filter((x) => x.ev === "guest-token-missing").length).toBe(1) + }) + + // ------------------------------------------------------------------ finding 9: one bad grant is refused alone + it("R1-9 one undecodable grant next to a good one: the good one runs, the bad one is refused, no crash loop", async () => { + const w = W({ skewedEncode: true }) // the floor sends what the link's contract calls malformed + w.floor.enqueue(job("good")) + w.floor.enqueue(job("bad", { "timeout-minutes": 1.5 } as any)) // a float where the contract says Schema.Int + const l = start(w, dir) + await until("good created", () => w.ax.tasks.has("wf-test-good-a1")) + await until("bad refused", () => w.floor.jobs.get("wf-test-bad")!.state === "done") + expect((w.floor.jobs.get("wf-test-bad")!.output as any).reason).toBe("pre-start/invalid-spec") + expect(l.has("grant-invalid", "wf-test-bad-a1")).toBe(true) + await l.drain() + const j = Journal.open(dir) + expect(j.pendingLeaseKey).toBeUndefined() + expect(j.invalid.size).toBe(0) // told and compacted + }) + + it("R1-9 a Lease reply the floor cannot even encode is a typed transient error: the link stays up and heartbeats", async () => { + const w = W() // strict floor: its own encode of the float fails + w.floor.enqueue(job("bad", { "timeout-minutes": 1.5 } as any)) + const l = start(w, dir) + await until("lease errors", () => l.logs.filter((x) => x.ev === "lease-error").length >= 2) + await until("still heartbeating", () => w.floor.calls.filter((c) => c.rpc === "Heartbeat").length >= 2) + expect(l.logs.some((x) => String(x.ev).endsWith("-died"))).toBe(false) + expect(await l.drain()).toBeDefined() // exits on the stop signal, not on a defect + }) + + // ------------------------------------------------------------------ finding 10: configuration fails closed + it("R1-10 numeric and enum configuration is validated: a typo is ConfigInvalid naming the key, never a NaN interval", () => { + const bad = (env: Record) => { try { readLinkEnv(env); return undefined } catch (e) { return e instanceof ConfigInvalid ? e.key : "other" } } + expect(bad({ LINK_RESYNC_SECONDS: "15s" })).toBe("LINK_RESYNC_SECONDS") + expect(bad({ LINK_MAX_IN_FLIGHT: "0" })).toBe("LINK_MAX_IN_FLIGHT") + expect(bad({ LINK_CREATE_ATTEMPTS: "NaN" })).toBe("LINK_CREATE_ATTEMPTS") + expect(bad({ LINK_PENDING_TIMEOUT_SECONDS: "1.5" })).toBe("LINK_PENDING_TIMEOUT_SECONDS") + expect(bad({ LINK_COMPLETION: "gues" })).toBe("LINK_COMPLETION") + expect(bad({ LINK_SEAT_COMMANDS: "{halogen:[ax-agent]}" })).toBe("LINK_SEAT_COMMANDS") + expect(bad({ LINK_SEAT_COMMANDS: '{"halogen":[]}' })).toBe("LINK_SEAT_COMMANDS") + expect(bad({ LINK_OUTBOX_BACKOFF_MIN_SECONDS: "900", LINK_OUTBOX_BACKOFF_MAX_SECONDS: "60" })).toBe("LINK_OUTBOX_BACKOFF_MAX_SECONDS") + const ok = readLinkEnv({}) + expect([ok.resyncSeconds, ok.maxInFlight, ok.createAttempts, ok.completion, ok.seatCommands]).toEqual([15, 2, 7, "auto", { halogen: ["ax-agent", "pi"] }]) + try { readLinkEnv({ LINK_RESYNC_SECONDS: "secret-looking-value" }) } catch (e) { expect(String((e as Error).message)).not.toContain("secret-looking-value") } + }) +}) diff --git a/pkgs/substrate-apps/src/apps/link/test/review-r2.test.ts b/pkgs/substrate-apps/src/apps/link/test/review-r2.test.ts new file mode 100644 index 00000000..710d332c --- /dev/null +++ b/pkgs/substrate-apps/src/apps/link/test/review-r2.test.ts @@ -0,0 +1,419 @@ +// Review round 2 (2026-09-23): the twelve findings, each as a regression test. The scratch repros they restate are in +// /home/tom/today/evals-2026-09-23/link/scratch-r2-{0,1,2}; they failed against c5ed453. The log is link/REVIEW-LOG.md. +import { mkdtempSync, rmSync } from "node:fs" +import { tmpdir } from "node:os" +import { join } from "node:path" +import { Deferred, Effect, Fiber } from "effect" +import { afterEach, beforeEach, describe, expect, it } from "vitest" +import { cidrIsOpen, gatewayAllowsAll } from "../src/ax.ts" +import type { AgentJob } from "../src/contract.ts" +import { rpcFloor } from "../src/floor.ts" +import { Journal } from "../src/journal.ts" +import { isFleetInternalUrl, runLink } from "../src/link.ts" +import type { LinkConfig } from "../src/link.ts" +import { FakeAx } from "./fake-ax.ts" +import { FakeFloor } from "./fake-floor.ts" +import type { FloorConfig } from "./fake-floor.ts" + +const SEC = 20 +const A = "ultracode.mecattaf.dev/" +const JK = `${"cd".repeat(32)}:1` // round 4: FIELD-MAP 5a journal key (jobs.ts refuses a grant without it) +const job = (n: string): AgentJob => ({ + apiVersion: "ultracode.mecattaf.dev/v1alpha1", kind: "AgentJob", + metadata: { name: `wf-test-${n}`, labels: { [A + "run-id"]: "wf-test", [A + "workflow"]: "link-test", [A + "phase-index"]: "1" }, + annotations: { [A + "run-id-raw"]: "wf_test", [A + "label"]: `probe:${n}`, [A + "item-key"]: `wf_test#${n}`, [A + "journal-key"]: JK, [A + "phase-title"]: "Probe" } }, + spec: { "runs-on": ["seat:halogen", "runtime:gvisor"], + with: { prompt: `say ${n}`, prompt_ref: { sha256: "ab".repeat(32), bytes: 5, uri: `journal://wf_test/${n}/prompt.md` }, model: "halogen-qwen3.8-flash-next" } } +}) +const world = (o: Partial = {}) => ({ + floor: new FakeFloor({ cap: 2, leaseSeconds: 6, graceSeconds: 15, pollSeconds: 1, heartbeatSeconds: 2, maxAttempts: 3, secondMs: SEC, tokens: { "tok-nas": "nas-link-1" }, ...o }), + ax: new FakeAx() +}) +type World = ReturnType +function start(w: World, dir: string, over: Partial = {}, fetch?: typeof globalThis.fetch) { + const logs: Array> = [] + const stop = Effect.runSync(Deferred.make()) + const cfg: LinkConfig = { + holder: "nas-link-1", maxInFlight: 2, servedLabels: ["seat:halogen", "runtime:gvisor"], + shape: { atespace: "fleet", image: "ax-agent", gateway: "halogen", command: () => ["ax-agent", "pi"] }, + completion: "auto", secondMs: SEC, resyncMs: 30, pendingTimeoutMs: 1500, deleteAfterMs: 0, deadlineBackstopMs: 300, + createAttempts: 3, outboxBackoffMs: [20, 100], initialPollSeconds: 1, initialHeartbeatSeconds: 2, fenceTimeoutMs: 300, resultReadTries: 3, ...over + } + const fiber = Effect.runFork(Effect.scoped(Effect.gen(function*() { + const floor = yield* rpcFloor({ url: "http://floor.test", token: "tok-nas", sessionId: `s:${dir}`, fetch: fetch ?? w.floor.fetch }) + return yield* runLink(cfg, { ax: w.ax, floor, journal: Journal.open(dir), log: (ev, f) => logs.push({ t: Date.now(), ev, ...f }), stop, floorUrl: "http://floor.test" }) + }))) + return { + logs, fiber, + has: (ev: string, leaseId?: string) => logs.some((l) => l.ev === ev && (leaseId === undefined || l.leaseId === leaseId)), + trace: () => logs.filter((x) => typeof x.leaseId === "string" || typeof x.task === "string").map((x) => `${x.ev}:${x.leaseId ?? x.task}${x.why ? ":" + x.why : ""}`), + crash: () => Effect.runPromise(Fiber.interrupt(fiber)), + drain: () => { Effect.runSync(Deferred.succeed(stop, undefined)); return Promise.race([Effect.runPromise(Fiber.await(fiber)), sleep(1500)]) } + } +} +const until = async (what: string, pred: () => boolean, ms = 5000) => { + const t0 = Date.now() + while (!pred()) { if (Date.now() - t0 > ms) throw new Error(`timeout waiting for: ${what}`); await new Promise((r) => setTimeout(r, 5)) } +} +const sleep = (ms: number) => new Promise((r) => setTimeout(r, ms)) +const J = (w: World, n = "1") => w.floor.jobs.get(`wf-test-${n}`)! +const bodyOf = async (input: unknown, init?: RequestInit) => { + const b = init?.body + if (typeof b === "string") return b + if (b instanceof Uint8Array) return new TextDecoder().decode(b) + if (b) return await new Response(b as ConstructorParameters[0]).text() + return (input instanceof Request) ? await input.clone().text() : "" +} +const rpcOf = async (input: unknown, init?: RequestInit) => { const b = await bodyOf(input, init); return b.includes('"Complete"') ? "Complete" : b.includes('"Lease"') ? "Lease" : b.includes('"Heartbeat"') ? "Heartbeat" : "?" } +/** A capacity race: Failed with ResourceExhausted, which the link maps to retryable pre-start/resource-exhausted. */ +const failRetryable = (w: World, name: string) => { + const t = w.ax.tasks.get(name)! + t.exited = true; t.phase = "Failed" + t.conditions = [{ type: "Ready", status: "False", reason: "ResourceExhausted", message: "ResourceExhausted: no free worker" }] +} +const why = (w: World, l: ReturnType) => JSON.stringify({ floor: J(w).history, link: l.trace() }) + +// A floor stub for the fail-closed tests (scratch-r2-1/repro.mts): hands out its grants once, renews everything. +const stats = { seq: 1, cap: 2, wip: 0, queued: 0, done: 0 } +const grantOf = (n: string, extra: Record = {}) => ({ leaseId: `wf-test-${n}-a1`, attempt: 1, job: job(n), + lease: { holderIdentity: "nas-link-1", leaseDurationSeconds: 30, acquireTime: Date.now(), renewTime: Date.now(), leaseTransitions: 0 }, ...extra }) +const stubFloor = (grants: Array, gate: () => boolean = () => true) => { + let sent = false + return { + lease: (p: any) => Effect.sync(() => { const g = !sent && p.capacity > 0 && gate() ? grants : []; if (g.length) sent = true; return { grants: g, invalid: [], stats, nextPollSeconds: 1, heartbeatSeconds: 2 } }), + heartbeat: (p: any) => Effect.succeed({ renewed: [...p.leaseIds], lost: [], cancelRequested: [], stats }), + complete: (_p: any) => Effect.succeed({ duplicate: false, stats }) + } +} +const runStub = async (over: Partial, ax: FakeAx, floor: any, ms: number) => { + const dir = mkdtempSync(join(tmpdir(), "conwip-link-r2s-")) + const logs: Array> = [] + const stop = Effect.runSync(Deferred.make()) + const cfg: LinkConfig = { holder: "nas-link-1", maxInFlight: 2, servedLabels: ["seat:halogen", "runtime:gvisor"], + shape: { atespace: "fleet", image: "ax-agent", gateway: "halogen", command: () => ["ax-agent", "pi"] }, + completion: "auto", secondMs: SEC, resyncMs: 30, pendingTimeoutMs: 60_000, deleteAfterMs: 60_000, deadlineBackstopMs: 300, + createAttempts: 3, outboxBackoffMs: [20, 100], initialPollSeconds: 1, initialHeartbeatSeconds: 2, ...over } + const f = Effect.runFork(runLink(cfg, { ax, floor, journal: Journal.open(dir), log: (ev, x) => logs.push({ t: Date.now(), ev, ...x }), stop })) + await sleep(ms) + Effect.runSync(Deferred.succeed(stop, undefined)); await Effect.runPromise(Fiber.await(f)); rmSync(dir, { recursive: true, force: true }) + return logs +} + +describe("review round 2: fixed", () => { + let dir: string + let worlds: Array = [] + const W = (o: Partial = {}) => { const w = world(o); worlds.push(w); return w } + beforeEach(() => { dir = mkdtempSync(join(tmpdir(), "conwip-link-r2-")) }) + afterEach(() => { for (const w of worlds) w.floor.close(); worlds = []; rmSync(dir, { recursive: true, force: true }) }) + + // ---------------------------------------------------------------- R2-1: heartbeat `lost` and the janitor read first + it("R2-1a kill -9, the agent finishes, the floor requeues past grace, restart while ax is down: the success is delivered, a2 never runs", async () => { + const w = W(); w.floor.enqueue(job("1")) + const l1 = start(w, dir) + await until("running", () => w.ax.tasks.get("wf-test-1-a1")?.phase === "Running") + await l1.crash() + w.ax.finish("wf-test-1-a1", 0, { answer: 42 }) + await until("requeued by the alarm", () => J(w).history.includes("requeue:grace"), 3000) + w.ax.up = false + const l2 = start(w, dir) + await until("heartbeat says lost", () => l2.has("lost", "wf-test-1-a1")) + w.ax.up = true + await until("floor settles", () => J(w).state === "done" || w.ax.updates.has("wf-test-1-a2"), 4000) + await sleep(200) + expect(w.ax.updates.has("wf-test-1-a2"), why(w, l2)).toBe(false) + expect([J(w).result, J(w).output, J(w).attempt]).toEqual(["success", { answer: 42 }, 1]) + expect(l2.logs.findIndex((x) => x.ev === "verdict")).toBeLessThan(l2.logs.findIndex((x) => x.ev === "delete" && x.task === "wf-test-1-a1")) + await l2.drain() + }) + + it("R2-1b no restart: the agent finishes as the partition heals and the heartbeat's lost beats the resync: the success is delivered", async () => { + const w = W(); w.floor.enqueue(job("1")) + const l = start(w, dir, { resyncMs: 300 }) + await until("running", () => w.ax.tasks.get("wf-test-1-a1")?.phase === "Running") + w.floor.down = true + await until("requeued by the alarm", () => J(w).history.includes("requeue:grace"), 3000) + w.ax.finish("wf-test-1-a1", 0, { answer: 42 }) + w.floor.down = false + await until("floor settles", () => J(w).state === "done" || w.ax.updates.has("wf-test-1-a2"), 4000) + await sleep(200) + expect(w.ax.updates.has("wf-test-1-a2"), why(w, l)).toBe(false) + expect([J(w).result, J(w).output, J(w).attempt]).toEqual(["success", { answer: 42 }, 1]) + await l.drain() + }) + + // ---------------------------------------------------------------- R2-2: the supersede fence reads first + it("R2-2 the Lease wins the race (heartbeats failing): a2's fence reads a1's Completed result, the floor withdraws a2", async () => { + const w = W(); w.floor.enqueue(job("1")) + let failHeartbeat = false + const fetch: typeof globalThis.fetch = async (input, init) => { + if (failHeartbeat && await rpcOf(input, init) === "Heartbeat") throw new TypeError("fetch failed") + return w.floor.fetch(input, init) + } + const l = start(w, dir, { resyncMs: 300 }, fetch) + await until("running", () => w.ax.tasks.get("wf-test-1-a1")?.phase === "Running") + w.floor.down = true + await until("requeued by the alarm", () => J(w).history.includes("requeue:grace"), 3000) + failHeartbeat = true + w.ax.finish("wf-test-1-a1", 0, { answer: 42 }) + w.floor.down = false + await until("floor settles", () => J(w).state === "done" || w.ax.updates.has("wf-test-1-a2"), 4000) + failHeartbeat = false + await sleep(200) + expect(w.ax.updates.has("wf-test-1-a2"), why(w, l)).toBe(false) + expect([J(w).result, J(w).output, J(w).attempt]).toEqual(["success", { answer: 42 }, 1]) + await l.drain() + }) + + // ---------------------------------------------------------------- R2-3: one verdict never wedges the link + it("R2-3a a result over the link's output cap is refused agent/output-too-large; later jobs are leased", async () => { + const w = W({ cap: 4 }); w.floor.enqueue(job("1")) + const l = start(w, dir, { maxInFlight: 2, outboxBackoffMs: [20, 60] }) + await until("a1 running", () => w.ax.tasks.get("wf-test-1-a1")?.phase === "Running") + w.ax.finish("wf-test-1-a1", 0, { report: "x".repeat(2 * 1024 * 1024 + 1000) }) + await until("verdict written", () => l.has("verdict", "wf-test-1-a1")) + w.floor.enqueue(job("2")); w.floor.enqueue(job("3")) + await until("job 2 leased", () => J(w, "2").state !== "queued", 3000) + expect([J(w).result, (J(w).output as any)?.reason]).toEqual(["failure", "agent/output-too-large"]) + await l.drain() + }) + + it("R2-3b a verdict the floor answers 500 every time is replaced by infra/verdict-undeliverable, and never gates Lease", async () => { + const w = W({ cap: 4 }); w.floor.enqueue(job("1")) + let refused = 0 + const fetch: typeof globalThis.fetch = async (input, init) => { + const b = await bodyOf(input, init) + if (b.includes('"Complete"') && b.includes('"success"')) { refused++; return new Response("internal error", { status: 500 }) } + return w.floor.fetch(input, init) + } + const l = start(w, dir, { outboxBackoffMs: [20, 60], verdictAttempts: 4 }, fetch) + await until("a1 running", () => w.ax.tasks.get("wf-test-1-a1")?.phase === "Running") + w.ax.finish("wf-test-1-a1", 0, { answer: 42 }) + await until("verdict written", () => l.has("verdict", "wf-test-1-a1")) + w.floor.enqueue(job("2")) + await until("job 2 leased", () => J(w, "2").state !== "queued", 3000) + await until("job 1 closed", () => J(w).state === "done", 3000) + expect([J(w).result, (J(w).output as any)?.reason, refused]).toEqual(["failure", "infra/verdict-undeliverable", 4]) + expect(l.has("verdict-replaced", "wf-test-1-a1")).toBe(true) + await l.drain() + }) + + it("R2-3c an RPC Defect reply (SQLITE_TOOBIG) is a server-error, not an encode defect", async () => { + const fetch: typeof globalThis.fetch = async (input, init) => { + const req = JSON.parse(await bodyOf(input, init)), id = (Array.isArray(req) ? req[0] : req).id + return new Response(JSON.stringify([{ _tag: "Exit", requestId: id, exit: { _tag: "Failure", cause: [{ _tag: "Die", defect: { message: "string or blob too big: SQLITE_TOOBIG" } }] } }]), { status: 200, headers: { "content-type": "application/json" } }) + } + const e = await Effect.runPromise(Effect.scoped(Effect.gen(function*() { + const floor = yield* rpcFloor({ url: "http://floor.test", token: "t", sessionId: "s", fetch }) + return yield* floor.complete({ leaseId: "x-a1", attempt: 1, result: "success", output: {} }).pipe(Effect.flip) + }))) + expect([e._tag, (e as any).kind, /SQLITE_TOOBIG/.test((e as any).message ?? "")]).toEqual(["FloorError", "server-error", true]) + }) + + it("R2-3d Complete classification: 413 and 422 drop the verdict, 404 is fatal misrouted (verdict kept), 500 is server-error, 503 transient", async () => { + const out: Record = {} + for (const status of [413, 422, 404, 500, 503, 429]) { + const e = await Effect.runPromise(Effect.scoped(Effect.gen(function*() { + const floor = yield* rpcFloor({ url: "http://floor.test", token: "t", sessionId: "s", fetch: async () => new Response("x", { status, headers: { "retry-after": "7" } }) }) + return yield* floor.complete({ leaseId: "x-a1", attempt: 1, result: "success" }).pipe(Effect.flip) + }))) + out[status] = e._tag === "CompleteRefused" ? `refused:${e.code}` : `${e.kind}:${e.fatal}${e.kind === "rate-limited" ? `:${e.retryAfterMs}` : ""}` + } + // round 3 (finding 13): 429 is rate-limited with its Retry-After, the verdict kept (not `rejected`, which drops it) + expect(out).toEqual({ 413: "refused:http-413", 422: "refused:http-422", 404: "misrouted:true", 500: "server-error:false", 503: "transient:false", 429: "rate-limited:false:7000" }) + }) + + // ---------------------------------------------------------------- R2-4: withdrawal is explicit on the wire + it("R2-4a a Lease crosses a1's retryable verdict, whose reply is lost; the resend answers duplicate: a2 is created", async () => { + const w = W(); w.floor.enqueue(job("1")) + let holdLease = false, loseCompleteReply = false, failComplete = false + const fetch: typeof globalThis.fetch = async (input, init) => { + const rpc = await rpcOf(input, init) + if (rpc === "Lease") while (holdLease) await sleep(2) + if (rpc === "Complete" && failComplete) return new Response("upstream timeout", { status: 503 }) + if (rpc === "Complete" && loseCompleteReply) { await w.floor.fetch(input, init); loseCompleteReply = false; failComplete = true; throw new TypeError("fetch failed") } + return w.floor.fetch(input, init) + } + const l = start(w, dir, {}, fetch) + await until("running", () => w.ax.tasks.get("wf-test-1-a1")?.phase === "Running") + holdLease = true + await sleep(60) + loseCompleteReply = true + failRetryable(w, "wf-test-1-a1") + await until("floor applied a1 and requeued a2", () => J(w).state === "queued" && J(w).attempt === 2, 4000) + holdLease = false + await until("a2 granted", () => l.has("leased", "wf-test-1-a2"), 4000) + await until("dispatch waits for the verdict", () => l.has("supersede-waits-for-verdict", "wf-test-1-a2")) + failComplete = false + await until("a2 created or withdrawn", () => w.ax.updates.has("wf-test-1-a2") || l.has("withdrawn", "wf-test-1-a2"), 4000) + await sleep(300) + expect(l.has("withdrawn", "wf-test-1-a2"), why(w, l)).toBe(false) + expect(w.ax.updates.get("wf-test-1-a2")).toBe(1) + expect(J(w).history).not.toContain("requeue:omitted") + await l.drain() + }) + + it("R2-4b the same race with an infrastructure budget of 2: the retry runs once, the job is not ended retry-budget-spent", async () => { + const w = W({ infraAttempts: 2 }); w.floor.enqueue(job("1")) + let armed = false, dropped = 0, committed!: () => void + const done = new Promise((r) => { committed = r }) + const fetch: typeof globalThis.fetch = async (input, init) => { + const body = await bodyOf(input, init) + if (armed && body.includes('"Lease"') && /"capacity":[1-9]/.test(body)) { armed = false; await done } + if (body.includes('"Complete"') && body.includes("wf-test-1-a1") && dropped > 0) await sleep(200) + const res = await w.floor.fetch(input, init) + if (body.includes('"Complete"') && body.includes("wf-test-1-a1") && dropped === 0) { dropped++; committed(); throw new TypeError("fetch failed: reply lost") } + return res + } + const l = start(w, dir, { outboxBackoffMs: [150, 150], fenceTimeoutMs: 3000 }, fetch) + await until("a1 running", () => w.ax.tasks.get("wf-test-1-a1")?.phase === "Running") + armed = true + await until("a Lease with capacity is held", () => !armed, 3000) + failRetryable(w, "wf-test-1-a1") + await until("a2 leased", () => l.has("leased", "wf-test-1-a2"), 4000) + await until("a2 created", () => w.ax.updates.has("wf-test-1-a2"), 4000).catch(() => {}) + expect(w.ax.updates.get("wf-test-1-a2") ?? 0, why(w, l)).toBe(1) + expect(l.has("withdrawn", "wf-test-1-a2")).toBe(false) + expect((J(w).output as any)?.reason).not.toBe("infra/retry-budget-spent") + await l.drain() + }) + + // ---------------------------------------------------------------- R2-5: a regrant of a withdrawn leaseId is new work + it("R2-5a rule 4b withdraws a2 for a1's retryable verdict and the floor grants a2 again: the new generation is created", async () => { + const w = W(); w.floor.enqueue(job("1")) + let holdLease = false, failHeartbeat = false, failComplete = false + const fetch: typeof globalThis.fetch = async (input, init) => { + const rpc = await rpcOf(input, init) + if (rpc === "Heartbeat" && failHeartbeat) throw new TypeError("fetch failed") + if (rpc === "Complete" && failComplete) return new Response("upstream timeout", { status: 503 }) + if (rpc === "Lease") while (holdLease) await sleep(2) + return w.floor.fetch(input, init) + } + const l = start(w, dir, {}, fetch) + await until("running", () => w.ax.tasks.get("wf-test-1-a1")?.phase === "Running") + w.floor.down = true + await until("floor requeued attempt 2", () => J(w).state === "queued" && J(w).attempt === 2, 4000) + failHeartbeat = true; holdLease = true; failComplete = true + w.floor.down = false + await sleep(80) + failRetryable(w, "wf-test-1-a1") + await until("verdict written", () => l.has("verdict", "wf-test-1-a1")) + holdLease = false + await until("a2 received", () => l.has("leased", "wf-test-1-a2"), 4000) + await until("dispatch waits for the verdict", () => l.has("supersede-waits-for-verdict", "wf-test-1-a2")) + failComplete = false; failHeartbeat = false + await until("a2 created, or the floor gives up on a2", () => w.ax.updates.has("wf-test-1-a2") || J(w).attempt >= 3 || J(w).state === "done", 4000).catch(() => {}) + expect(w.ax.updates.get("wf-test-1-a2") ?? 0, why(w, l)).toBe(1) + expect(l.has("duplicate-grant", "wf-test-1-a2")).toBe(false) + await l.drain() + }) + + it("R2-5b the Lease reply carrying a2 is slow; a1's retryable verdict lands first and withdraws it; the regrant is created", async () => { + const w = W(); w.floor.enqueue(job("1")) + let holdLeaseReply = false, failHeartbeat = false + const fetch: typeof globalThis.fetch = async (input, init) => { + const rpc = await rpcOf(input, init) + if (rpc === "Heartbeat" && failHeartbeat) throw new TypeError("fetch failed") + if (rpc === "Lease" && holdLeaseReply) { const r = await w.floor.fetch(input, init); while (holdLeaseReply) await sleep(2); return r } + return w.floor.fetch(input, init) + } + const l = start(w, dir, {}, fetch) + await until("running", () => w.ax.tasks.get("wf-test-1-a1")?.phase === "Running") + w.floor.down = true + await until("floor requeued attempt 2", () => J(w).state === "queued" && J(w).attempt === 2, 4000) + failHeartbeat = true; holdLeaseReply = true + w.floor.down = false + await until("floor leased a2 to this link", () => J(w).state === "leased" && J(w).attempt === 2, 4000) + failRetryable(w, "wf-test-1-a1") + await until("verdict written", () => l.has("verdict", "wf-test-1-a1")) + holdLeaseReply = false + await until("a2 received", () => l.has("leased", "wf-test-1-a2"), 4000) + failHeartbeat = false + await until("a2 created, or the floor gives up", () => w.ax.updates.has("wf-test-1-a2") || J(w).attempt >= 3 || J(w).state === "done", 4000).catch(() => {}) + expect(w.ax.updates.get("wf-test-1-a2") ?? 0, why(w, l)).toBe(1) + await l.drain() + }) + + // ---------------------------------------------------------------- R2-6: redirects are never followed + it("R2-6 the floor answers 307 to another origin: nothing is leased or created and the link stops fatal (redirect)", async () => { + const w = W(); w.floor.enqueue(job("1")) + let rogue = 0 + const fetch: typeof globalThis.fetch = async (input, init) => { + if ((init as RequestInit | undefined)?.redirect !== "manual") { rogue++; return w.floor.fetch(input, init) } // a follower would land on the rogue origin + return new Response(null, { status: 307, headers: { location: "http://127.0.0.1:9/rpc" } }) + } + const l = start(w, dir, {}, fetch) + const exit = await Promise.race([Effect.runPromise(Fiber.await(l.fiber)), sleep(1500).then(() => undefined)]) + expect([rogue, w.ax.tasks.size, l.has("leased")]).toEqual([0, 0, false]) + expect(exit?._tag).toBe("Failure") + expect(JSON.stringify(exit)).toContain("redirect") + }) + + // ---------------------------------------------------------------- R2-7, R2-8: fail-closed checks + it("R2-7 isFleetInternalUrl matches private ranges on IP literals only", () => { + for (const u of ["https://10.evil.example/guest", "https://127.0.0.1.nip.io/guest", "https://192.168.0.1.attacker.com/c", "https://link/guest", "https://x.workers.dev/", "https://10.internal.example.com/"]) + expect([u, isFleetInternalUrl(u)]).toEqual([u, false]) + for (const u of ["http://10.201.0.80:8080/c", "http://127.0.0.1/c", "http://100.64.0.7/c", "http://[fd7a:115c:a1e0::1]/c", "http://[::1]/c", "http://link.fleet.internal/guest", "http://nas.lan/c", "http://localhost:1/c", "http://0x7f.1/c"]) + expect([u, isFleetInternalUrl(u)]).toEqual([u, true]) + expect(isFleetInternalUrl("http://link/guest", ["link"])).toBe(true) + }) + + it("R2-7b guest mode with a public name that starts with a private prefix: no Task, no token handed out", async () => { + const ax = new FakeAx(); ax.p1 = false + const logs = await runStub({ completion: "guest", shape: { atespace: "fleet", image: "ax-agent", gateway: "halogen", command: () => ["ax-agent", "pi"], completeUrl: "https://10.evil.example/guest" } }, + ax, stubFloor([grantOf("g", { leaseToken: "tok" })]), 400) + expect([logs.some((l) => l.ev === "guest-url-invalid"), ax.tasks.has("wf-test-g-a1")]).toEqual([true, false]) + }) + + it("R2-8 a Gateway open by CIDR arithmetic is refused: ::/0, 0.0.0.0/1 + 128.0.0.0/1, and unparseable CIDRs", async () => { + const gw = (hosts: Array) => ({ hasAllowlist: true, hosts: hosts.map((host) => ({ host, port: 443 })) }) + expect(gatewayAllowsAll(gw(["worker", "::/0"]))).toBe(true) + expect(gatewayAllowsAll(gw(["0.0.0.0/1", "128.0.0.0/1"]))).toBe(true) + expect(gatewayAllowsAll(gw([" * "]))).toBe(true) + expect(gatewayAllowsAll(gw(["10.0.0.0/x"]))).toBe(true) + expect(gatewayAllowsAll(gw(["worker", "10.201.0.0/24", "fd7a:115c:a1e0::/48"]))).toBe(false) + expect([cidrIsOpen("8.0.0.0/8"), cidrIsOpen("10.1.0.0/16"), cidrIsOpen("2000::/3"), cidrIsOpen("fd00::/64")]).toEqual([true, false, true, false]) + const ax = new FakeAx() + ax.gateways.set("halogen", gw(["worker", "::/0"])) + const logs = await runStub({}, ax, stubFloor([grantOf("w")]), 400) + expect([logs.some((l) => l.ev === "gateway-missing"), ax.tasks.has("wf-test-w-a1")]).toEqual([true, false]) + }) + + // ---------------------------------------------------------------- R2-9: nothing from before an ax outage opens the gate + it("R2-9 guest mode: ax returns without the Gateway and GetGateway is slow; nothing is created on pre-outage state", async () => { + const ax = new FakeAx(); ax.p1 = false + const origGw = ax.getGateway + let outageOver = false, release = false + ax.getGateway = (name: string) => outageOver ? Effect.sleep(300).pipe(Effect.andThen(origGw(name))) : origGw(name) + setTimeout(() => { ax.up = false }, 150) + setTimeout(() => { release = true }, 170) + setTimeout(() => { ax.gateways.delete("halogen"); outageOver = true; ax.up = true }, 400) + const logs = await runStub({ completion: "guest", shape: { atespace: "fleet", image: "ax-agent", gateway: "halogen", command: () => ["ax-agent", "pi"], completeUrl: "http://link.fleet.internal/guest" } }, + ax, stubFloor([grantOf("u", { leaseToken: "tok" })], () => release), 1200) + expect(ax.tasks.has("wf-test-u-a1"), JSON.stringify(logs.map((l) => l.ev))).toBe(false) + expect(logs.some((l) => l.ev === "gateway-missing")).toBe(true) + }) + + // ---------------------------------------------------------------- R2-11: a permanent Complete refusal is dropped + for (const variant of ["http-413", "unknown-error-code"] as const) + it(`R2-11 one Complete refused for good (${variant}) is dropped and job 2 is leased`, async () => { + const w = W(); w.floor.enqueue(job("1")) + const fetch: typeof globalThis.fetch = async (input, init) => { + const body = await bodyOf(input, init) + if (body.includes('"Complete"') && body.includes("wf-test-1-a1")) { + if (variant === "http-413") return new Response("payload too large", { status: 413 }) + const req = JSON.parse(body), id = (Array.isArray(req) ? req[0] : req).id + return new Response(JSON.stringify([{ _tag: "Exit", requestId: id, exit: { _tag: "Failure", cause: [{ _tag: "Fail", error: { code: "output-too-large" } }] } }]), { status: 200, headers: { "content-type": "application/json" } }) + } + return w.floor.fetch(input, init) + } + const l = start(w, dir, { outboxBackoffMs: [20, 40] }, fetch) + await until("a1 running", () => w.ax.tasks.get("wf-test-1-a1")?.phase === "Running") + w.ax.finish("wf-test-1-a1", 0, { big: "x" }) + await until("verdict", () => l.has("verdict", "wf-test-1-a1")) + w.floor.enqueue(job("2")) + await until("job 2 leased", () => J(w, "2").state !== "queued", 3000) + const d = l.logs.find((x) => x.ev === "complete-dropped" && x.leaseId === "wf-test-1-a1") + expect(d?.code).toBe(variant === "http-413" ? "http-413" : "output-too-large") + await l.drain() + }) +}) diff --git a/pkgs/substrate-apps/src/apps/link/test/review-r3.test.ts b/pkgs/substrate-apps/src/apps/link/test/review-r3.test.ts new file mode 100644 index 00000000..057b6faa --- /dev/null +++ b/pkgs/substrate-apps/src/apps/link/test/review-r3.test.ts @@ -0,0 +1,484 @@ +// Review round 3 (2026-09-23): the thirteen findings, each as a regression test. The scratch repros they restate are +// in /home/tom/today/evals-2026-09-23/link/scratch-r3-{0,1,2}; they failed against d582ead. The log is +// link/REVIEW-LOG.md. Test adequacy findings 11 to 13 also strengthen R1-4, R1-8 and R2-3d in place. +import { appendFileSync, mkdtempSync, rmSync } from "node:fs" +import { tmpdir } from "node:os" +import { join } from "node:path" +import { Deferred, Effect, Fiber } from "effect" +import { afterEach, beforeEach, describe, expect, it } from "vitest" +import { AxError } from "../src/ax.ts" +import type { AgentJob } from "../src/contract.ts" +import { rpcFloor } from "../src/floor.ts" +import { Journal } from "../src/journal.ts" +import { boundedSeconds, isFleetInternalUrl, runLink } from "../src/link.ts" +import type { LinkConfig } from "../src/link.ts" +import { FakeAx } from "./fake-ax.ts" +import { FakeFloor } from "./fake-floor.ts" +import type { FloorConfig } from "./fake-floor.ts" + +const SEC = 20 +const A = "ultracode.mecattaf.dev/" +const JK = `${"cd".repeat(32)}:1` // round 4: FIELD-MAP 5a journal key (jobs.ts refuses a grant without it) +const job = (n: string, spec: Record = {}): AgentJob => ({ + apiVersion: "ultracode.mecattaf.dev/v1alpha1", kind: "AgentJob", + metadata: { name: `wf-test-${n}`, labels: { [A + "run-id"]: "wf-test", [A + "workflow"]: "link-test", [A + "phase-index"]: "1" }, + annotations: { [A + "run-id-raw"]: "wf_test", [A + "label"]: `probe:${n}`, [A + "item-key"]: `wf_test#${n}`, [A + "journal-key"]: JK, [A + "phase-title"]: "Probe" } }, + spec: { "runs-on": ["seat:halogen", "runtime:gvisor"], + with: { prompt: `say ${n}`, prompt_ref: { sha256: "ab".repeat(32), bytes: 5, uri: `journal://wf_test/${n}/prompt.md` }, model: "halogen-qwen3.8-flash-next" }, ...spec } as AgentJob["spec"] +}) +const world = (o: Partial = {}) => ({ + floor: new FakeFloor({ cap: 2, leaseSeconds: 6, graceSeconds: 15, pollSeconds: 1, heartbeatSeconds: 2, maxAttempts: 3, secondMs: SEC, tokens: { "tok-nas": "nas-link-1" }, ...o }), + ax: new FakeAx() +}) +type World = ReturnType +function start(w: World, dir: string, over: Partial = {}, fetch?: typeof globalThis.fetch) { + const logs: Array> = [] + const stop = Effect.runSync(Deferred.make()) + const cfg: LinkConfig = { + holder: "nas-link-1", maxInFlight: 2, servedLabels: ["seat:halogen", "runtime:gvisor"], + shape: { atespace: "fleet", image: "ax-agent", gateway: "halogen", command: () => ["ax-agent", "pi"] }, + completion: "auto", secondMs: SEC, resyncMs: 30, pendingTimeoutMs: 1500, deleteAfterMs: 0, deadlineBackstopMs: 300, + createAttempts: 3, outboxBackoffMs: [20, 100], initialPollSeconds: 1, initialHeartbeatSeconds: 2, fenceTimeoutMs: 300, resultReadTries: 3, ...over + } + const fiber = Effect.runFork(Effect.scoped(Effect.gen(function*() { + const floor = yield* rpcFloor({ url: "http://floor.test", token: "tok-nas", sessionId: `s:${dir}`, fetch: fetch ?? w.floor.fetch }) + return yield* runLink(cfg, { ax: w.ax, floor, journal: Journal.open(dir), log: (ev, f) => logs.push({ t: Date.now(), ev, ...f }), stop, floorUrl: "http://floor.test" }) + }))) + return { + logs, fiber, + has: (ev: string, leaseId?: string) => logs.some((l) => l.ev === ev && (leaseId === undefined || l.leaseId === leaseId)), + trace: () => logs.filter((x) => typeof x.leaseId === "string" || typeof x.task === "string").map((x) => `${x.ev}:${x.leaseId ?? x.task}${x.why ? ":" + x.why : ""}`), + crash: () => Effect.runPromise(Fiber.interrupt(fiber)), + drain: () => { Effect.runSync(Deferred.succeed(stop, undefined)); return Promise.race([Effect.runPromise(Fiber.await(fiber)), sleep(1500)]) } + } +} +const until = async (what: string, pred: () => boolean, ms = 5000) => { + const t0 = Date.now() + while (!pred()) { if (Date.now() - t0 > ms) throw new Error(`timeout waiting for: ${what}`); await new Promise((r) => setTimeout(r, 5)) } +} +const sleep = (ms: number) => new Promise((r) => setTimeout(r, ms)) +const J = (w: World, n = "1") => w.floor.jobs.get(`wf-test-${n}`)! +const bodyOf = async (input: unknown, init?: RequestInit) => { + const b = init?.body + if (typeof b === "string") return b + if (b instanceof Uint8Array) return new TextDecoder().decode(b) + if (b) return await new Response(b as ConstructorParameters[0]).text() + return (input instanceof Request) ? await input.clone().text() : "" +} +const rpcOf = async (input: unknown, init?: RequestInit) => { const b = await bodyOf(input, init); return b.includes('"Complete"') ? "Complete" : b.includes('"Lease"') ? "Lease" : b.includes('"Heartbeat"') ? "Heartbeat" : "?" } +const failRetryable = (w: World, name: string) => { + const t = w.ax.tasks.get(name)! + t.exited = true; t.phase = "Failed" + t.conditions = [{ type: "Ready", status: "False", reason: "ResourceExhausted", message: "ResourceExhausted: no free worker" }] +} +const why = (w: World, l: ReturnType) => JSON.stringify({ floor: J(w).history, link: l.trace() }) +/** Rewrite every JSON reply of one RPC with `patch` (a floor whose wire drifted from the link's). */ +const rewriting = (w: World, rpc: string, patch: (x: any) => any): typeof globalThis.fetch => async (input, init) => { + const b = await bodyOf(input, init) + const res = await w.floor.fetch(input, init) + if (!b.includes(`"${rpc}"`)) return res + const txt = await res.text() + let j: unknown; try { j = JSON.parse(txt) } catch { return new Response(txt, { status: res.status, headers: res.headers }) } + return new Response(JSON.stringify(patch(j)), { status: res.status, headers: { "content-type": "application/json" } }) +} +const deep = (f: (o: Record) => void) => { const walk = (x: any): any => { + if (Array.isArray(x)) return x.map(walk) + if (x && typeof x === "object") { const o: Record = {}; for (const [k, v] of Object.entries(x)) o[k] = walk(v); f(o); return o } + return x +}; return walk } + +// A floor stub that grants one lease named after a Task someone else owns (scratch-r3-1). +const stats = { seq: 1, cap: 2, wip: 0, queued: 0, done: 0 } +const foreignTask = (name: string) => ({ apiVersion: "ax.io/v1alpha1" as const, kind: "Task" as const, metadata: { name, atespace: "fleet" }, + spec: { image: "someone-else", command: ["long-job"], env: [{ name: "AX_CONWIP_HOLDER", value: "nas-link-2" }], gateway: { name: "halogen" } } }) +const foreignGrant = (name: string) => ({ leaseId: name, attempt: 1, job: job("x"), + lease: { holderIdentity: "nas-link-1", leaseDurationSeconds: 30, acquireTime: Date.now(), renewTime: Date.now(), leaseTransitions: 0 } }) +const runStub = async (dir: string, ax: FakeAx, floor: any, ms: number, over: Partial = {}) => { + const logs: Array> = [] + const stop = Effect.runSync(Deferred.make()) + const cfg: LinkConfig = { holder: "nas-link-1", maxInFlight: 2, servedLabels: ["seat:halogen", "runtime:gvisor"], + shape: { atespace: "fleet", image: "ax-agent", gateway: "halogen", command: () => ["ax-agent", "pi"] }, + completion: "auto", secondMs: SEC, resyncMs: 30, pendingTimeoutMs: 60_000, deleteAfterMs: 600_000, deadlineBackstopMs: 300, + createAttempts: 3, outboxBackoffMs: [20, 100], initialPollSeconds: 1, initialHeartbeatSeconds: 2, ...over } + const f = Effect.runFork(runLink(cfg, { ax, floor, journal: Journal.open(dir), log: (ev, x) => logs.push({ t: Date.now(), ev, ...x }), stop })) + await sleep(ms) + Effect.runSync(Deferred.succeed(stop, undefined)); await Effect.runPromise(Fiber.await(f)) + return logs +} + +describe("review round 3", () => { + let dir: string + let worlds: Array = [] + const W = (o: Partial = {}) => { const w = world(o); worlds.push(w); return w } + beforeEach(() => { dir = mkdtempSync(join(tmpdir(), "conwip-link-r3-")) }) + afterEach(() => { for (const w of worlds) w.floor.close(); worlds = []; rmSync(dir, { recursive: true, force: true }) }) + + // ---------------------------------------------------------------- R3-1: a withdrawal names a generation + for (const sendWithdrewGen of [true, false]) { + it(`R3-1a (E) rule 4b withdraws a QUEUED a2 for a1's retryable verdict; the requeued a2 is created (floor names the generation: ${sendWithdrewGen})`, async () => { + const w = W({ sendWithdrewGen }); w.floor.enqueue(job("1")) + // Deterministic E: no Lease reaches the floor between its return and a1's verdict landing, so a2 gen 1 is withdrawn + // while still QUEUED and this link never receives it (a Lease retried across the return could carry a capacity + // computed before the verdict and receive gen 1 first: that is R2-5a's interleaving, not this one). + let gate = false, landed = false + const fetch: typeof globalThis.fetch = async (input, init) => { + const rpc = await rpcOf(input, init) + if (rpc === "Lease" && gate && !landed) throw new TypeError("fetch failed") + const r = await w.floor.fetch(input, init) + if (rpc === "Complete" && gate) landed = true + return r + } + const l = start(w, dir, {}, fetch) + await until("running", () => w.ax.tasks.get("wf-test-1-a1")?.phase === "Running") + gate = true + w.floor.down = true + await until("floor requeued attempt 2", () => J(w).state === "queued" && J(w).attempt === 2, 4000) + failRetryable(w, "wf-test-1-a1") + await until("verdict written", () => l.has("verdict", "wf-test-1-a1")) + w.floor.down = false + await until("a2 created", () => w.ax.updates.has("wf-test-1-a2"), 4000).catch(() => { throw new Error(why(w, l)) }) + expect(J(w).history).toContain("withdrawn:a2") + expect(l.logs.some((x) => x.ev === "leased" && x.leaseId === "wf-test-1-a2" && l.logs.indexOf(x) < l.logs.findIndex((y) => y.ev === "complete"))).toBe(false) + w.ax.finish("wf-test-1-a2", 0, { ok: 2 }) + await until("floor done", () => J(w).state === "done", 4000) + expect([J(w).result, J(w).attempt, w.ax.updates.get("wf-test-1-a2")]).toEqual(["success", 2, 1]) + await l.drain() + }) + + it(`R3-1b (C) kill -9 between the withdrawal and the regrant: the regrant of a2 is created after restart (floor names the generation: ${sendWithdrewGen})`, async () => { + const w = W({ sendWithdrewGen }); w.floor.enqueue(job("1")) + let holdLease = false, failHeartbeat = false, failComplete = false, holdAfterComplete = false + const fetch: typeof globalThis.fetch = async (input, init) => { + const rpc = await rpcOf(input, init) + if (rpc === "Heartbeat" && failHeartbeat) throw new TypeError("fetch failed") + if (rpc === "Complete" && failComplete) return new Response("upstream timeout", { status: 503 }) + if (rpc === "Lease") while (holdLease) await sleep(2) + const r = await w.floor.fetch(input, init) + if (rpc === "Complete" && holdAfterComplete) holdLease = true // the regrant never reaches this process + return r + } + const over = { deleteAfterMs: 60_000 } // a1's record survives the restart's compaction; a2 gen 1 does not + const l1 = start(w, dir, over, fetch) + await until("running", () => w.ax.tasks.get("wf-test-1-a1")?.phase === "Running") + w.floor.down = true + await until("floor requeued attempt 2", () => J(w).state === "queued" && J(w).attempt === 2, 4000) + failHeartbeat = true; holdLease = true; failComplete = true + w.floor.down = false + await sleep(80) + failRetryable(w, "wf-test-1-a1") + await until("verdict written", () => l1.has("verdict", "wf-test-1-a1")) + holdLease = false + await until("a2 received", () => l1.has("leased", "wf-test-1-a2"), 4000) + await until("dispatch waits for the verdict", () => l1.has("supersede-waits-for-verdict", "wf-test-1-a2")) + holdAfterComplete = true; failComplete = false; failHeartbeat = false + await until("floor withdrew a2 and the link journaled it", () => l1.has("withdrawn", "wf-test-1-a2"), 4000) + await l1.crash() + const l2 = start(w, dir, over) + await until("a2 created", () => w.ax.updates.has("wf-test-1-a2"), 4000).catch(() => { throw new Error(why(w, l2)) }) + expect(w.ax.updates.get("wf-test-1-a2")).toBe(1) + expect(J(w).history.filter((h) => h === "leased:a2").length).toBe(2) // gen 1 withdrawn, gen 2 run + await l2.drain() + }) + } + + // ---------------------------------------------------------------- R3-2: the cancel intent is journaled first + it("R3-2 (A) a cancel whose DeleteTask lands but whose reply is lost is reported cancelled; no attempt 2 is started", async () => { + const w = W({ leaseSeconds: 20, heartbeatSeconds: 6, graceSeconds: 40 }); w.floor.enqueue(job("1")) + const l = start(w, dir, { initialHeartbeatSeconds: 6 }) + await until("running", () => w.ax.tasks.get("wf-test-1-a1")?.phase === "Running") + const real = w.ax.deleteTask + let lostReplies = 0 + w.ax.deleteTask = (name: string) => real(name).pipe(Effect.andThen(() => lostReplies++ === 0 ? Effect.fail(new AxError(4, "deadline exceeded")) : Effect.void)) + w.floor.cancel("wf-test-1") + await until("floor done", () => J(w).state === "done", 4000).catch(() => { throw new Error(why(w, l)) }) + await sleep(200) + const v = l.logs.find((x) => x.ev === "verdict" && x.leaseId === "wf-test-1-a1")! + expect([v.result, J(w).result, J(w).attempt, J(w).infraSpent]).toEqual(["cancelled", "cancelled", 1, 0]) + expect(w.ax.updates.has("wf-test-1-a2")).toBe(false) + expect(l.logs.some((x) => x.ev === "delete-deferred" && x.task === "wf-test-1-a1")).toBe(true) + await l.drain() + }) + + it("R3-2b a journaled DeleteTask that did not land is sent again by the resync", async () => { + const w = W({ leaseSeconds: 20, heartbeatSeconds: 6, graceSeconds: 40 }); w.floor.enqueue(job("1")) + const l = start(w, dir, { initialHeartbeatSeconds: 6 }) + await until("running", () => w.ax.tasks.get("wf-test-1-a1")?.phase === "Running") + const real = w.ax.deleteTask + let refused = 0 + w.ax.deleteTask = (name: string) => refused++ === 0 ? Effect.fail(new AxError(14, "unavailable")) : real(name) + w.floor.cancel("wf-test-1") + await until("floor done", () => J(w).state === "done", 4000).catch(() => { throw new Error(why(w, l)) }) + expect([J(w).result, w.ax.deletes.filter((d) => d === "wf-test-1-a1").length]).toEqual(["cancelled", 1]) + await l.drain() + }) + + // ---------------------------------------------------------------- R3-3: finished leases are tombstoned + for (const restart of [false, true]) it(`R3-3 (D) a redelivered grant of a finished lease is never run again (restart=${restart})`, async () => { + const w = W(); w.floor.enqueue(job("1")) + const l1 = start(w, dir) + await until("running", () => w.ax.tasks.get("wf-test-1-a1")?.phase === "Running") + w.ax.finish("wf-test-1-a1", 0, { answer: 42 }) + await until("done", () => J(w).state === "done") + await until("janitor deleted a1", () => !w.ax.tasks.has("wf-test-1-a1"), 3000) + await sleep(100) + let l = l1 + if (restart) { await l1.crash(); l = start(w, dir); await sleep(100) } + w.floor.redeliver = ["wf-test-1-a1"] + await until("duplicate-grant", () => l.has("duplicate-grant", "wf-test-1-a1"), 2000) + await sleep(200) + expect(w.ax.updates.get("wf-test-1-a1")).toBe(1) + await l.drain() + }) + + it("R3-3b the tombstone survives compaction and expires after TOMB_MS; a later generation of the same leaseId is not refused", () => { + const j = Journal.open(dir) + const g = { leaseId: "x-a1", attempt: 1, job: job("x"), lease: { holderIdentity: "h", leaseDurationSeconds: 30, acquireTime: 1000, renewTime: 1000, leaseTransitions: 0 } } + j.append({ ev: "grant", leaseId: "x-a1", grant: g }) + j.append({ ev: "report", leaseId: "x-a1", attempt: 1, result: "success" }) + j.append({ ev: "reported", leaseId: "x-a1", duplicate: false }) + j.compact() + expect(j.recs.has("x-a1")).toBe(false) + expect(Journal.open(dir).tombs.get("x-a1")?.gen).toEqual({ leaseTransitions: 0, acquireTime: 1000 }) + const keep = Journal.TOMB_MS + try { Journal.TOMB_MS = -1; Journal.open(dir).compact(); expect(Journal.open(dir).tombs.has("x-a1")).toBe(false) } finally { Journal.TOMB_MS = keep } + }) + + // ---------------------------------------------------------------- R3-4: renewal does not wait for the first Lease + it("R3-4 (B) a restart whose first Lease is rate-limited keeps heartbeating; the running a1 is not requeued", async () => { + const w = W(); w.floor.enqueue(job("1")) + const l1 = start(w, dir) + await until("running", () => w.ax.tasks.get("wf-test-1-a1")?.phase === "Running") + await l1.crash() + let leases = 0 + const hbAt: Array = [] + const f: typeof globalThis.fetch = async (input, init) => { + const rpc = await rpcOf(input, init) + if (rpc === "Heartbeat") hbAt.push(Date.now()) + if (rpc === "Lease" && leases++ === 0) return new Response("slow down", { status: 429, headers: { "retry-after": "2" } }) + return w.floor.fetch(input, init) + } + const t0 = Date.now() + const l2 = start(w, dir, {}, f) + await sleep(2300) + expect(hbAt.filter((t) => t - t0 > 200 && t - t0 < 2000).length).toBeGreaterThan(5) + expect(J(w).history).not.toContain("requeue:grace") + expect([J(w).attempt, w.ax.tasks.get("wf-test-1-a1")?.phase]).toEqual([1, "Running"]) + await l2.drain() + }) + + // ---------------------------------------------------------------- R3-5: a Task the link did not write is never touched + for (const variant of ["janitor", "lost", "cancel"] as const) for (const phase of ["Running", "Completed"] as const) + it(`R3-5a a grant named after a foreign ${phase} Task: refused name-conflict, never deleted or read (${variant})`, async () => { + const ax = new FakeAx() + ax.put(foreignTask("other-team-build"), "Running") + if (phase === "Completed") ax.finish("other-team-build", 0, { secret: "other tenant's output" }) + let sent = false + const completes: Array = [] + const floor = { + lease: (p: any) => Effect.sync(() => { const g = !sent && p.capacity > 0 ? [foreignGrant("other-team-build")] : []; if (g.length) sent = true; return { grants: g, invalid: [], stats, nextPollSeconds: 1, heartbeatSeconds: 2 } }), + heartbeat: (p: any) => Effect.sync(() => { + const lost = variant === "lost" ? p.leaseIds.filter((x: string) => x === "other-team-build") : [] + const cancel = variant === "cancel" ? p.leaseIds.filter((x: string) => x === "other-team-build") : [] + return { renewed: p.leaseIds.filter((x: string) => !lost.includes(x)), lost, cancelRequested: cancel, stats } + }), + complete: (p: any) => Effect.sync(() => { completes.push(p); return { duplicate: false, stats } }) + } + const logs = await runStub(dir, ax, floor, 800) + expect(ax.deletes).toEqual([]) + expect(ax.tasks.get("other-team-build")?.phase).toBe(phase) + expect(completes.every((c) => c.result !== "success" && JSON.stringify(c).includes("secret") === false)).toBe(true) + expect(logs.some((x) => x.ev === "not-mine" && x.leaseId === "other-team-build")).toBe(true) + }) + + it("R3-5b (salvage) a crash in the create window, then `lost`: a foreign finished Task's result is never reported, the Task never deleted", async () => { + const g = foreignGrant("other-team-build") + const t = Date.now() + appendFileSync(join(dir, "journal.jsonl"), JSON.stringify({ t, ev: "grant", leaseId: g.leaseId, grant: g }) + "\n" + JSON.stringify({ t, ev: "creating", leaseId: g.leaseId }) + "\n") + const ax = new FakeAx() + ax.put(foreignTask("other-team-build"), "Running") + ax.finish("other-team-build", 0, { secret: "other tenant's output" }) + const completes: Array = [] + const floor = { + lease: (_p: any) => Effect.succeed({ grants: [], invalid: [], stats, nextPollSeconds: 1, heartbeatSeconds: 2 }), + heartbeat: (p: any) => Effect.succeed({ renewed: [], lost: p.leaseIds.filter((x: string) => x === "other-team-build"), cancelRequested: [], stats }), + complete: (p: any) => Effect.sync(() => { completes.push(p); return { duplicate: false, stats } }) + } + const logs = await runStub(dir, ax, floor, 600) + expect([completes, ax.deletes]).toEqual([[], []]) + expect(logs.some((x) => x.ev === "not-mine")).toBe(true) + }) + + it("R3-5c the link's own Task under a `creating`-only record (crash before `created`) is still salvaged and deleted", async () => { + const w = W(); w.floor.enqueue(job("1")) + w.ax.hangAfterCreate = true + const l1 = start(w, dir) + await until("created in ax", () => w.ax.tasks.has("wf-test-1-a1")) + await l1.crash() + w.ax.hangAfterCreate = false + w.ax.tick() + w.ax.finish("wf-test-1-a1", 0, { mine: true }) + const l2 = start(w, dir) + await until("floor done", () => J(w).state === "done", 4000).catch(() => { throw new Error(why(w, l2)) }) + expect([J(w).result, J(w).output]).toEqual(["success", { mine: true }]) + await until("deleted", () => !w.ax.tasks.has("wf-test-1-a1"), 3000) + expect(l2.has("not-mine")).toBe(false) + await l2.drain() + }) + + // ---------------------------------------------------------------- R3-6 and R3-7: server-driven intervals are bounded + for (const v of [0, -5]) it(`R3-6 a floor answering nextPollSeconds=${v} and heartbeatSeconds=${v} gets a bounded call rate (production secondMs)`, async () => { + const w = W({ pollSeconds: v, heartbeatSeconds: v, secondMs: 1000, leaseSeconds: 60, graceSeconds: 60 }) + w.floor.enqueue(job("1")) + const l = start(w, dir, { secondMs: 1000, resyncMs: 1000, initialPollSeconds: 15, initialHeartbeatSeconds: 30 }) + await sleep(1500) + const n = (rpc: string) => w.floor.calls.filter((c) => c.rpc === rpc).length + expect(n("Lease")).toBeLessThanOrEqual(4) + expect(n("Heartbeat")).toBeLessThanOrEqual(4) + expect(l.has("interval-clamped")).toBe(true) + await l.drain() + }) + + it("R3-7 boundedSeconds: default for missing, non-finite and not positive; clamped to [1, hi]", () => { + expect([boundedSeconds(undefined, 15, 300), boundedSeconds(0, 15, 300), boundedSeconds(-5, 15, 300), boundedSeconds(Number.NaN, 15, 300), + boundedSeconds(0.2, 15, 300), boundedSeconds(10 / 3, 15, 300), boundedSeconds(1e9, 15, 300), boundedSeconds(60, 30, 10)]).toEqual([15, 15, 15, 15, 1, 10 / 3, 300, 10]) + }) + + // ---------------------------------------------------------------- R3-8: the Lease envelope decodes leniently + it("R3-8 a Lease reply with heartbeatSeconds 3.333, endpoint null and no stats is used; the job runs", async () => { + const w = W(); w.floor.enqueue(job("1")) + const fetch = rewriting(w, "Lease", deep((o) => { if ("heartbeatSeconds" in o) { o.heartbeatSeconds = 10 / 3; o.endpoint = null; delete o.stats } })) + const l = start(w, dir, {}, fetch) + await until("created", () => w.ax.updates.has("wf-test-1-a1"), 4000).catch(() => { throw new Error(why(w, l)) }) + w.ax.finish("wf-test-1-a1", 0, { ok: 1 }) + await until("floor done", () => J(w).state === "done", 4000) + expect([J(w).result, J(w).attempt, l.has("lease-error")]).toEqual(["success", 1, false]) + await l.drain() + }) + + it("R3-8b a Lease key whose replies never decode is abandoned after leaseKeyAttempts, not replayed for ever; the job then runs", async () => { + const w = W(); w.floor.enqueue(job("1")) + let broken = true + const keys: Array = [] + const inner = rewriting(w, "Lease", deep((o) => { if (broken && "grants" in o && "nextPollSeconds" in o) o.grants = "not-an-array" })) + const fetch: typeof globalThis.fetch = async (input, init) => { + const b = await bodyOf(input, init) + const k = /"requestKey":"([^"]+)"/.exec(b)?.[1] + if (k && b.includes('"Lease"') && broken) keys.push(k) + return inner(input, init) + } + const l = start(w, dir, { leaseKeyAttempts: 3 }, fetch) + await until("key abandoned twice", () => l.logs.filter((x) => x.ev === "lease-key-abandoned").length >= 2, 4000).catch(() => { throw new Error(why(w, l)) }) + broken = false + expect(new Set(keys).size).toBeGreaterThan(1) // before round 3: one key, `transient`, for as long as the floor answers so + expect(l.logs.filter((x) => x.ev === "lease-error").every((x) => x.kind === "server-error")).toBe(true) + await until("created", () => w.ax.updates.size > 0, 4000).catch(() => { throw new Error(why(w, l)) }) + await l.drain() + }) + + it("R3-8c (R1-9b scenario) a floor that cannot encode one job: the link stays up and a job enqueued later runs", async () => { + // The good job co-granted with the bad one is only ever sent inside replies the strict floor cannot encode, so no + // link-side change can deliver it (REFUTED for the link; the floor port must encode per grant, round 1 H). + const w = W() + w.floor.enqueue(job("bad", { "timeout-minutes": 1.5 })) + w.floor.enqueue(job("good")) + const l = start(w, dir) + await sleep(500) + w.floor.enqueue(job("later")) + await until("later created", () => w.ax.updates.has("wf-test-later-a1"), 8000).catch(() => { throw new Error(why(w, l)) }) + expect(l.logs.filter((x) => x.ev === "lease-error").every((x) => x.kind === "server-error")).toBe(true) + await l.drain() + }) + + // ---------------------------------------------------------------- R3-9: Complete's success decodes leniently + it("R3-9 a Complete success reply with `withdrew: null` is reported once; the next jobs run", async () => { + const w = W(); w.floor.enqueue(job("1")); w.floor.enqueue(job("2")) + let completes = 0 + const patch = deep((o) => { if ("duplicate" in o && "stats" in o && !("withdrew" in o)) o.withdrew = null }) + const inner = rewriting(w, "Complete", patch) + const fetch: typeof globalThis.fetch = async (input, init) => { if (await rpcOf(input, init) === "Complete") completes++; return inner(input, init) } + const l = start(w, dir, { maxInFlight: 1 }, fetch) + await until("a1", () => w.ax.tasks.has("wf-test-1-a1")) + w.ax.finish("wf-test-1-a1", 0, { ok: true }) + await until("job 2 created", () => w.ax.updates.has("wf-test-2-a1"), 4000).catch(() => { throw new Error(why(w, l)) }) + expect([J(w).state, J(w).result, completes, l.has("outbox-keep")]).toEqual(["done", "success", 1, false]) + await l.drain() + }) + + it("R3-9b a Complete success reply that is not the protocol at all is a server-error: bounded by verdictAttempts, never gates Lease", async () => { + const w = W(); w.floor.enqueue(job("1")); w.floor.enqueue(job("2")) + // the floor applies the verdict; its reply's success value is replaced by a number + const fetch = rewriting(w, "Complete", deep((o) => { if (o.value && typeof o.value === "object" && "duplicate" in (o.value as object)) o.value = 7 })) + const l = start(w, dir, { maxInFlight: 1, verdictAttempts: 2 }, fetch) + await until("a1", () => w.ax.tasks.has("wf-test-1-a1")) + w.ax.finish("wf-test-1-a1", 0, { ok: true }) + await until("job 2 created", () => w.ax.updates.has("wf-test-2-a1"), 4000).catch(() => { throw new Error(why(w, l)) }) + expect(l.logs.filter((x) => x.ev === "outbox-keep").every((x) => x.kind === "server-error")).toBe(true) + await l.drain() + }) + + // ---------------------------------------------------------------- R3-10: B2 (n+1 only after NotFound for n) + it("R3-10 (F8 shape) attempt n+1 is written only once attempt n is absent from ax; while n stays Terminating it is never written", async () => { + const w = W() + const create = w.ax.createTask + const seenAtCreate: Array = [] + w.ax.createTask = (t) => { if (t.metadata.name === "wf-test-1-a2") seenAtCreate.push(...[...w.ax.tasks].map(([n, x]) => `${n}:${x.phase}`)); return create(t) } + w.floor.enqueue(job("1")) + const l1 = start(w, dir) + await until("running", () => w.ax.tasks.get("wf-test-1-a1")?.phase === "Running") + await l1.crash() + rmSync(dir, { recursive: true, force: true }) // the journal is lost: a2 is fenced on the Task's own marks + await sleep(7 * SEC); w.floor.sweep() + const tick = w.ax.tick.bind(w.ax) + let hold = true + w.ax.tick = () => { if (hold) { for (const t of w.ax.tasks.values()) if (t.phase === "Pending") t.phase = "Running" } else tick() } + const l2 = start(w, mkdtempSync(join(tmpdir(), "conwip-link-r3-")), { fenceTimeoutMs: 4000 }) + await until("a1 Terminating", () => w.ax.tasks.get("wf-test-1-a1")?.phase === "Terminating", 4000).catch(() => { throw new Error(why(w, l2)) }) + await sleep(250) + expect(w.ax.updates.has("wf-test-1-a2")).toBe(false) // the delete was asked, NotFound not yet seen + hold = false + await until("a2 written", () => w.ax.updates.has("wf-test-1-a2"), 6000).catch(() => { throw new Error(why(w, l2)) }) + expect(seenAtCreate.filter((x) => x.startsWith("wf-test-1-a1"))).toEqual([]) + await l2.drain() + }) + + // ---------------------------------------------------------------- R3-11: 429, Retry-After, endpoint and address rows + it("R3-11 a Complete answered 429 Retry-After: 1 keeps the verdict and waits at least Retry-After before the next try", async () => { + const w = W(); w.floor.enqueue(job("1")) + const at: Array = [] + const fetch: typeof globalThis.fetch = async (input, init) => { + if (await rpcOf(input, init) !== "Complete") return w.floor.fetch(input, init) + at.push(Date.now()) + if (at.length === 1) return new Response("slow down", { status: 429, headers: { "retry-after": "1" } }) + return w.floor.fetch(input, init) + } + const l = start(w, dir, {}, fetch) + await until("a1", () => w.ax.tasks.get("wf-test-1-a1")?.phase === "Running") + w.ax.finish("wf-test-1-a1", 0, { v: 1 }) + await until("floor done", () => J(w).state === "done", 5000).catch(() => { throw new Error(why(w, l)) }) + expect([J(w).result, J(w).attempt, l.has("complete-dropped")]).toEqual(["success", 1, false]) + expect(l.logs.some((x) => x.ev === "outbox-keep" && x.kind === "rate-limited")).toBe(true) + expect(at[1]! - at[0]!).toBeGreaterThanOrEqual(950) + await l.drain() + }) + + it("R3-11b an http:// endpoint is ignored even when it is in the declared list; 100.x outside 100.64/10 is not fleet-internal", async () => { + const w = W() + const fetch = rewriting(w, "Lease", deep((o) => { if ("nextPollSeconds" in o) o.endpoint = "http://floor2.test" })) + const switched: Array = [] + const logs: Array> = [] + const stop = Effect.runSync(Deferred.make()) + const cfg: LinkConfig = { holder: "nas-link-1", maxInFlight: 2, servedLabels: ["seat:halogen", "runtime:gvisor"], + shape: { atespace: "fleet", image: "ax-agent", gateway: "halogen", command: () => ["ax-agent", "pi"] }, + completion: "auto", secondMs: SEC, resyncMs: 30, pendingTimeoutMs: 1500, deleteAfterMs: 0, deadlineBackstopMs: 300, + createAttempts: 3, outboxBackoffMs: [20, 100], initialPollSeconds: 1, initialHeartbeatSeconds: 2, floorUrls: ["http://floor.test", "http://floor2.test"] } + const fiber = Effect.runFork(Effect.scoped(Effect.gen(function*() { + const floor = yield* rpcFloor({ url: "http://floor.test", token: "tok-nas", sessionId: "s", fetch }) + return yield* runLink(cfg, { ax: w.ax, floor, journal: Journal.open(dir), log: (ev, f) => logs.push({ ev, ...f }), stop, floorUrl: "http://floor.test", onEndpoint: (u) => switched.push(u) }) + }))) + await until("endpoint seen", () => logs.some((x) => x.ev === "endpoint-ignored" || x.ev === "endpoint-accepted"), 3000) + Effect.runSync(Deferred.succeed(stop, undefined)); await Promise.race([Effect.runPromise(Fiber.await(fiber)), sleep(1500)]) + expect([switched, logs.some((x) => x.ev === "endpoint-ignored")]).toEqual([[], true]) + expect(["http://100.1.2.3/c", "http://100.128.0.1/c", "http://100.63.255.255/c", "http://100.64.0.1/c", "http://100.127.255.254/c"].map((u) => isFleetInternalUrl(u))) + .toEqual([false, false, false, true, true]) + }) +}) diff --git a/pkgs/substrate-apps/src/apps/link/test/review-r4-expired.test.ts b/pkgs/substrate-apps/src/apps/link/test/review-r4-expired.test.ts new file mode 100644 index 00000000..c2b499f5 --- /dev/null +++ b/pkgs/substrate-apps/src/apps/link/test/review-r4-expired.test.ts @@ -0,0 +1,86 @@ +// Round 4 regression test, ported from /home/tom/today/evals-2026-09-23/link/scratch-r4-1/expired.repro.ts (assertions kept; additions marked). +// Round 4, lens fail-closed and capacity gate: a grant that waits at the slot gate (B11: gate closed, e.g. the Gateway +// check failing) is created whenever the gate reopens, with no check that its lease is still live. The floor, which +// the link could not reach (heartbeats failing), has already expired it, requeued it, and may grant attempt 2. +// Buildkite's pattern the design adopts (row 8, "reserve with expiry, finish -1 before start") would refuse it. +import { expect, it } from "vitest" +import { bodyOf, job, sleep, start, until, world } from "./review-r4-harness.ts" + +it("a grant whose lease expired while it waited at the gate is still created", async () => { + const w = world({ leaseSeconds: 6, graceSeconds: 15 }) // at secondMs 20: lease 120 ms, grace 300 ms + w.floor.enqueue(job("1")) + const gw = w.ax.gateways.get("halogen")! + let holdOnce = true + let l: ReturnType + const fetch: typeof globalThis.fetch = async (input, init) => { + const b = await bodyOf(input, init) + const res = await w.floor.fetch(input, init) + if (holdOnce && b.includes('"Lease"')) { + const txt = await res.clone().text() + if (txt.includes("wf-test-1-a1")) { + holdOnce = false + // the Gateway is edited away while the reply is in flight; the next resync closes the gate (B10) + w.ax.gateways.delete("halogen") + await until("gateway-missing", () => l.has("gateway-missing"), 3000) + } + } + return res + } + l = start(w, undefined, { maxInFlight: 1 }, fetch) + await until("waiting-for-slot", () => l.has("waiting-for-slot", "wf-test-1-a1"), 3000) + // the NAS loses its route to Cloudflare; ax stays reachable + w.floor.down = true + const J = w.floor.jobs.get("wf-test-1")! + await until("floor requeued a1 after lease + grace", () => J.state === "queued" && J.attempt === 2, 3000) + const floorStateAtReopen = { state: J.state, attempt: J.attempt, history: [...J.history] } + w.ax.gateways.set("halogen", gw) // the Gateway is back; the floor is still unreachable + await until("a1 created", () => (w.ax.updates.get("wf-test-1-a1") ?? 0) > 0, 3000).catch(() => {}) + const createdA1WhileExpired = (w.ax.updates.get("wf-test-1-a1") ?? 0) > 0 + await sleep(200) + const a1BeforeFloorReturned = w.ax.updates.get("wf-test-1-a1") ?? 0 // round 4 addition + w.floor.down = false + await until("a2 created", () => w.ax.tasks.has("wf-test-1-a2"), 3000).catch(() => {}) + await sleep(300) + const out = { + floorStateAtReopen, createdA1WhileExpired, axEvents: w.ax.events, + linkTrace: l!.logs.filter((x) => typeof x.leaseId === "string" || typeof x.task === "string").map((x) => `${x.ev}:${x.leaseId ?? x.task}${x.why ? ":" + x.why : ""}`), + heartbeatErrors: l!.logs.filter((x) => x.ev === "heartbeat-error").length, floorHistory: J.history + } + console.log(JSON.stringify(out)) + await l!.drain(); w.floor.close() + expect(createdA1WhileExpired).toBe(false) // fail closed: a lease known to be past renewTime + duration is not started + expect(a1BeforeFloorReturned).toBe(0) // round 4 addition: nothing of a1 before the floor returns + expect([...w.ax.updates.keys()]).toEqual(["wf-test-1-a2"]) // exactly one created attempt, a2, afterwards + expect(l!.has("lease-unconfirmed", "wf-test-1-a1")).toBe(true) +}) + +// Round 4 addition (finding 5, deadline half): a grant whose deadline has passed is refused with no create. The gate +// closes while the grant is in flight and the Heartbeats fail, so the floor cannot cancel at its deadline (the lease is +// orphaned, grace is long); only the link's own check can end it. +it("R4-5b: a grant past its deadline is failed deadline-exceeded before any UpdateTask", async () => { + const w = world({ graceSeconds: 400 }) + w.floor.enqueue(job("d", { "timeout-minutes": 1 })) // 60 s x 20 ms = 1.2 s deadline + const gw = w.ax.gateways.get("halogen")! + let hbDown = false + let l: ReturnType + const fetch: typeof globalThis.fetch = async (input, init) => { + const b = await bodyOf(input, init) + if (hbDown && b.includes('"Heartbeat"')) throw new TypeError("fetch failed") + const res = await w.floor.fetch(input, init) + if (!hbDown && b.includes('"Lease"') && (await res.clone().text()).includes("wf-test-d-a1")) { + hbDown = true + w.ax.gateways.delete("halogen") + await until("gateway-missing", () => l.has("gateway-missing"), 3000) + } + return res + } + l = start(w, undefined, { maxInFlight: 1 }, fetch) + await until("waiting", () => l.has("waiting-for-slot", "wf-test-d-a1") || l.has("lease-unconfirmed", "wf-test-d-a1"), 3000) + await sleep(1400) + w.ax.gateways.set("halogen", gw) + await until("verdict", () => l.has("verdict", "wf-test-d-a1"), 3000) + const v = l.logs.find((x) => x.ev === "verdict" && x.leaseId === "wf-test-d-a1")! + await l.drain(); w.floor.close() + expect(v.reason).toBe("deadline-exceeded") + expect(w.ax.updates.get("wf-test-d-a1") ?? 0).toBe(0) +}) diff --git a/pkgs/substrate-apps/src/apps/link/test/review-r4-fiber-defect.test.ts b/pkgs/substrate-apps/src/apps/link/test/review-r4-fiber-defect.test.ts new file mode 100644 index 00000000..f68fcaa3 --- /dev/null +++ b/pkgs/substrate-apps/src/apps/link/test/review-r4-fiber-defect.test.ts @@ -0,0 +1,96 @@ +// Round 4 regression test, ported from /home/tom/today/evals-2026-09-23/link/scratch-r4-2/fiber-defect.repro.ts (assertions kept; additions marked). +// Effect idiom: dispatch ends in `Effect.catchCause(... log("job-fiber-died"))`, so a defect in one job fiber is logged +// and swallowed while the process lives on. Here one journal write fails once (ENOSPC, then the disk recovers). The +// grant stays in `held()`, every Heartbeat renews it on the floor, nothing re-dispatches it, and nothing reports it: +// the job never runs and never ends, and it holds a floor WIP slot and a local slot until the process restarts. +import { mkdtempSync } from "node:fs" +import { tmpdir } from "node:os" +import { join } from "node:path" +import { Deferred, Effect, Fiber } from "effect" +import { expect, it } from "vitest" +import { rpcFloor } from "../src/floor.ts" +import { Journal } from "../src/journal.ts" +import { runLink } from "../src/link.ts" +import type { LinkConfig } from "../src/link.ts" +import { job, sleep, until, world, SEC } from "./review-r4-harness.ts" + +it("one failed journal write in a dispatch fiber strands the lease for the life of the process", async () => { + const w = world({ cap: 1 }) + w.floor.enqueue(job("1")) + w.floor.enqueue(job("2")) + const dir = mkdtempSync(join(tmpdir(), "r4-2-defect-")) + const j = Journal.open(dir) + const orig = j.append.bind(j) + let failed = 0 + ;(j as any).append = (e: any, t?: number) => { + if (e.ev === "creating" && failed === 0) { failed++; throw Object.assign(new Error("ENOSPC: no space left on device, write"), { code: "ENOSPC" }) } + return orig(e, t) + } + const logs: Array = [] + const stop = Effect.runSync(Deferred.make()) + const cfg: LinkConfig = { + holder: "nas-link-1", maxInFlight: 1, servedLabels: ["seat:halogen", "runtime:gvisor"], + shape: { atespace: "fleet", image: "ax-agent", gateway: "halogen", command: () => ["ax-agent", "pi"] }, + completion: "auto", secondMs: SEC, resyncMs: 30, pendingTimeoutMs: 1500, deleteAfterMs: 0, deadlineBackstopMs: 300, + createAttempts: 3, outboxBackoffMs: [20, 100], initialPollSeconds: 1, initialHeartbeatSeconds: 2, fenceTimeoutMs: 300, resultReadTries: 3 + } + const fiber = Effect.runFork(Effect.scoped(Effect.gen(function*() { + const floor = yield* rpcFloor({ url: "http://floor.test", token: "tok-nas", sessionId: "s:defect", fetch: w.floor.fetch }) + return yield* runLink(cfg, { ax: w.ax, floor, journal: j, log: (ev, f) => logs.push({ ev, ...f }), stop, floorUrl: "http://floor.test" }) + }))) + try { + await until("fiber died", () => logs.some((x) => x.ev === "job-fiber-died")) + await sleep(3000) // 25 lease durations (6 s x 20 ms); the disk is fine again + const J1 = w.floor.jobs.get("wf-test-1")!, J2 = w.floor.jobs.get("wf-test-2")! + const out = { + linkAlive: fiber.pollUnsafe() === undefined, failedWrites: failed, + job1: { state: J1.state, attempt: J1.attempt, history: J1.history, task: w.ax.tasks.get("wf-test-1-a1")?.phase ?? "absent" }, + job2: { state: J2.state, history: J2.history }, + held: j.held(), heartbeats: w.floor.calls.filter((c) => c.rpc === "Heartbeat").length, + completes: w.floor.calls.filter((c) => c.rpc === "Complete").length, + events: logs.map((x) => x.ev).filter((e, i, a) => a.indexOf(e) === i) + } + console.log(JSON.stringify(out)) + // Expected: the lease is failed back (retryable) or the process crashes and systemd restarts it; either frees job 1. + expect(J1.state === "done" || J1.attempt > 1 || w.ax.tasks.has("wf-test-1-a1") || !out.linkAlive).toBe(true) + } finally { + Effect.runSync(Deferred.succeed(stop, undefined)) + await Promise.race([Effect.runPromise(Fiber.await(fiber)), sleep(1500)]) + w.floor.close() + } +}) + +// Round 4 addition (finding 7): when even the fail-back cannot be journaled, the process dies (exit 1 under systemd) +// instead of renewing a lease nothing will ever run. +it("R4-7b: a dispatch defect whose fail-back also fails ends the link", async () => { + const w = world({ cap: 1 }) + w.floor.enqueue(job("1")) + const dir = mkdtempSync(join(tmpdir(), "r4-fix-defect-")) + const j = Journal.open(dir) + const orig = j.append.bind(j) + ;(j as any).append = (e: any, t?: number) => { + if (e.ev === "creating" || e.ev === "report") throw Object.assign(new Error("ENOSPC: no space left on device, write"), { code: "ENOSPC" }) + return orig(e, t) + } + const logs: Array = [] + const stop = Effect.runSync(Deferred.make()) + const cfg: LinkConfig = { + holder: "nas-link-1", maxInFlight: 1, servedLabels: ["seat:halogen", "runtime:gvisor"], + shape: { atespace: "fleet", image: "ax-agent", gateway: "halogen", command: () => ["ax-agent", "pi"] }, + completion: "auto", secondMs: SEC, resyncMs: 30, pendingTimeoutMs: 1500, deleteAfterMs: 0, deadlineBackstopMs: 300, + createAttempts: 3, outboxBackoffMs: [20, 100], initialPollSeconds: 1, initialHeartbeatSeconds: 2, fenceTimeoutMs: 300, resultReadTries: 3 + } + const fiber = Effect.runFork(Effect.scoped(Effect.gen(function*() { + const floor = yield* rpcFloor({ url: "http://floor.test", token: "tok-nas", sessionId: "s:defect2", fetch: w.floor.fetch }) + return yield* runLink(cfg, { ax: w.ax, floor, journal: j, log: (ev, f) => logs.push({ ev, ...f }), stop, floorUrl: "http://floor.test" }) + }))) + try { + const exit = await Promise.race([Effect.runPromise(Fiber.await(fiber)), sleep(3000).then(() => undefined)]) + expect(exit).toBeDefined() + expect(exit!._tag).toBe("Failure") + expect(logs.some((x) => x.ev === "link-defect-fatal")).toBe(true) + } finally { + Effect.runSync(Deferred.succeed(stop, undefined)) + w.floor.close() + } +}) diff --git a/pkgs/substrate-apps/src/apps/link/test/review-r4-harness.ts b/pkgs/substrate-apps/src/apps/link/test/review-r4-harness.ts new file mode 100644 index 00000000..94e35fc0 --- /dev/null +++ b/pkgs/substrate-apps/src/apps/link/test/review-r4-harness.ts @@ -0,0 +1,59 @@ +// Round 4 fix pass: the reviewers' scratch harness (scratch-r4-0/1/2, identical but for the temp prefix), moved into +// the worktree. The only change: the job carries the FIELD-MAP 5a journal-key annotation jobs.ts now requires. +import { mkdtempSync } from "node:fs" +import { tmpdir } from "node:os" +import { join } from "node:path" +import { Deferred, Effect, Fiber } from "effect" +import type { AgentJob } from "../src/contract.ts" +import { rpcFloor } from "../src/floor.ts" +import { Journal } from "../src/journal.ts" +import { runLink } from "../src/link.ts" +import type { LinkConfig } from "../src/link.ts" +import { FakeAx } from "./fake-ax.ts" +import { FakeFloor } from "./fake-floor.ts" +import type { FloorConfig } from "./fake-floor.ts" +export const SEC = 20 +const A = "ultracode.mecattaf.dev/" +export const job = (n: string, spec: Record = {}): AgentJob => ({ + apiVersion: "ultracode.mecattaf.dev/v1alpha1", kind: "AgentJob", + metadata: { name: `wf-test-${n}`, labels: { [A + "run-id"]: "wf-test", [A + "workflow"]: "link-test", [A + "phase-index"]: "1" }, + annotations: { [A + "run-id-raw"]: "wf_test", [A + "label"]: `probe:${n}`, [A + "item-key"]: `wf_test#${n}`, [A + "journal-key"]: `${"cd".repeat(32)}:1`, [A + "phase-title"]: "Probe" } }, + spec: { "runs-on": ["seat:halogen", "runtime:gvisor"], + with: { prompt: `say ${n}`, prompt_ref: { sha256: "ab".repeat(32), bytes: 5, uri: `journal://wf_test/${n}/prompt.md` }, model: "halogen-qwen3.8-flash-next" }, ...spec } as any +}) +export const world = (o: Partial = {}) => ({ + floor: new FakeFloor({ cap: 2, leaseSeconds: 6, graceSeconds: 15, pollSeconds: 1, heartbeatSeconds: 2, maxAttempts: 3, secondMs: SEC, tokens: { "tok-nas": "nas-link-1" }, ...o }), + ax: new FakeAx() +}) +export type World = ReturnType +export function start(w: World, dir = mkdtempSync(join(tmpdir(), "r4-fix-")), over: Partial = {}, fetch?: typeof globalThis.fetch) { + const logs: Array> = [] + const stop = Effect.runSync(Deferred.make()) + const cfg: LinkConfig = { + holder: "nas-link-1", maxInFlight: 2, servedLabels: ["seat:halogen", "runtime:gvisor"], + shape: { atespace: "fleet", image: "ax-agent", gateway: "halogen", command: () => ["ax-agent", "pi"] }, + completion: "auto", secondMs: SEC, resyncMs: 30, pendingTimeoutMs: 1500, deleteAfterMs: 0, deadlineBackstopMs: 300, + createAttempts: 3, outboxBackoffMs: [20, 100], initialPollSeconds: 1, initialHeartbeatSeconds: 2, fenceTimeoutMs: 300, resultReadTries: 3, ...over + } + const fiber = Effect.runFork(Effect.scoped(Effect.gen(function*() { + const floor = yield* rpcFloor({ url: "http://floor.test", token: "tok-nas", sessionId: `s:${dir}`, fetch: fetch ?? w.floor.fetch }) + return yield* runLink(cfg, { ax: w.ax, floor, journal: Journal.open(dir), log: (ev, f) => logs.push({ t: Date.now(), ev, ...f }), stop, floorUrl: "http://floor.test" }) + }))) + return { + dir, logs, fiber, + has: (ev: string, leaseId?: string) => logs.some((l) => l.ev === ev && (leaseId === undefined || l.leaseId === leaseId)), + drain: () => { Effect.runSync(Deferred.succeed(stop, undefined)); return Promise.race([Effect.runPromise(Fiber.await(fiber)), sleep(1500)]) } + } +} +export const sleep = (ms: number) => new Promise((r) => setTimeout(r, ms)) +export const until = async (what: string, pred: () => boolean, ms = 5000) => { + const t0 = Date.now() + while (!pred()) { if (Date.now() - t0 > ms) throw new Error(`timeout waiting for: ${what}`); await new Promise((r) => setTimeout(r, 5)) } +} +export const bodyOf = async (input: unknown, init?: RequestInit) => { + const b = init?.body + if (typeof b === "string") return b + if (b instanceof Uint8Array) return new TextDecoder().decode(b) + if (b) return await new Response(b as any).text() + return (input instanceof Request) ? await input.clone().text() : "" +} diff --git a/pkgs/substrate-apps/src/apps/link/test/review-r4-heartbeat-envelope.test.ts b/pkgs/substrate-apps/src/apps/link/test/review-r4-heartbeat-envelope.test.ts new file mode 100644 index 00000000..81b5de39 --- /dev/null +++ b/pkgs/substrate-apps/src/apps/link/test/review-r4-heartbeat-envelope.test.ts @@ -0,0 +1,134 @@ +// Round 4 regression test, ported from /home/tom/today/evals-2026-09-23/link/scratch-r4-2/heartbeat-envelope.repro.ts (assertions kept; additions marked). +// Schema compatibility: the round-3 fix made the Lease and Complete success envelopes lenient ("a changed stats shape +// would otherwise fail the decode"), but Heartbeat still decodes with the strict L1 schema. The same drift on the +// Heartbeat reply (here `stats: null`, which Lease tolerates) makes every heartbeat a transient error: held leases are +// never renewed, the floor requeues a healthy running attempt as a2, and the link fences a1 and runs the job again. +import { expect, it } from "vitest" +import { job, start, until, world, bodyOf, sleep } from "./review-r4-harness.ts" + +const drift = (rpc: string, floorFetch: typeof globalThis.fetch): typeof globalThis.fetch => async (input, init) => { + const body = await bodyOf(input, init) + const res = await floorFetch(input, init) + if (!body.includes(`"${rpc}"`)) return res + const text = await res.text() + let m: any + try { m = JSON.parse(text) } catch { return new Response(text, { status: res.status, headers: res.headers }) } + const walk = (x: any) => { if (x && typeof x === "object") { if ("stats" in x && "seq" in (x.stats ?? {})) x.stats = null; for (const v of Object.values(x)) walk(v) } } + walk(m) + return new Response(JSON.stringify(m), { status: res.status, headers: res.headers }) +} + +for (const rpc of ["Lease", "Heartbeat"]) { + it(`stats: null on every ${rpc} reply`, async () => { + const w = world() + w.floor.enqueue(job("1")) + const l = start(w, undefined, {}, drift(rpc, w.floor.fetch)) + try { + await until("a1 running", () => w.ax.tasks.get("wf-test-1-a1")?.phase === "Running") + await sleep(1500) // about 12 lease durations (6 s x 20 ms) with a1 healthy and running + const J = w.floor.jobs.get("wf-test-1")! + const hbErr = l.logs.filter((x) => x.ev === "heartbeat-error").length + const out = { rpc, floorState: J.state, attempt: J.attempt, history: J.history, heartbeatErrors: hbErr, + a1: w.ax.tasks.get("wf-test-1-a1")?.phase ?? "deleted", a2: w.ax.tasks.get("wf-test-1-a2")?.phase ?? "absent", + events: l.logs.map((x) => x.ev).filter((e, i, a) => a.indexOf(e) === i) } + console.log(JSON.stringify(out)) + // The expected behaviour, as for the Lease drift: a healthy a1 keeps its lease and nothing reruns. + expect(J.attempt).toBe(1) + expect(w.ax.tasks.get("wf-test-1-a2")).toBeUndefined() + } finally { await l.drain(); w.floor.close() } + }) +} + +// The consequence: the floor renews on its side, but the link never reads `cancelRequested`, `lost` or `renewed`. +for (const rpc of ["Lease", "Heartbeat"]) { + it(`cancel with stats: null on every ${rpc} reply`, async () => { + const w = world() + w.floor.enqueue(job("1")) + const l = start(w, undefined, {}, drift(rpc, w.floor.fetch)) + try { + await until("a1 running", () => w.ax.tasks.get("wf-test-1-a1")?.phase === "Running") + w.floor.cancel("wf-test-1") + await sleep(1500) + const J = w.floor.jobs.get("wf-test-1")! + const out = { rpc, floorState: J.state, result: J.result ?? null, cancel: J.cancel ?? null, history: J.history, + a1: w.ax.tasks.get("wf-test-1-a1")?.phase ?? "deleted", cancelLogged: l.has("cancel") } + console.log(JSON.stringify(out)) + expect(J.state).toBe("done") + expect(J.result).toBe("cancelled") + } finally { await l.drain(); w.floor.close() } + }) +} + +// B13: a grant journaled by a previous process and never created is resumed only once a Heartbeat reply renews it. +for (const rpc of ["Lease", "Heartbeat"]) { + it(`restart resume with stats: null on every ${rpc} reply`, async () => { + const { Effect } = await import("effect") + const w = world() + w.floor.enqueue(job("1")) + const orig = w.ax.createTask + ;(w.ax as any).createTask = () => Effect.never // first process: UpdateTask never answers, then the process stops + const l1 = start(w) + const dir = l1.dir + await until("a1 creating", () => l1.has("leased")) + await sleep(100) + await l1.drain() + ;(w.ax as any).createTask = orig + const J = w.floor.jobs.get("wf-test-1")! + const l2 = start(w, dir, {}, drift(rpc, w.floor.fetch)) + try { + await sleep(2000) + const out = { rpc, floorState: J.state, attempt: J.attempt, history: J.history, result: J.result ?? null, + a1: w.ax.tasks.get("wf-test-1-a1")?.phase ?? "absent", resumed: l2.has("resume"), + heartbeatErrors: l2.logs.filter((x) => x.ev === "heartbeat-error").length, + events: l2.logs.map((x) => x.ev).filter((e, i, a) => a.indexOf(e) === i) } + console.log(JSON.stringify(out)) + expect(J.state === "done" || w.ax.tasks.get("wf-test-1-a1") !== undefined).toBe(true) + } finally { await l2.drain(); w.floor.close() } + }) +} + +// Round 4 addition (finding 6): a Heartbeat reply that omits `cancelRequested` (or has it null) is read for the rest. +const omit = (key: string, value: "omit" | null, floorFetch: typeof globalThis.fetch): typeof globalThis.fetch => async (input, init) => { + const body = await bodyOf(input, init) + const res = await floorFetch(input, init) + if (!body.includes('"Heartbeat"')) return res + const text = await res.text() + let m: any + try { m = JSON.parse(text) } catch { return new Response(text, { status: res.status, headers: res.headers }) } + const walk = (x: any) => { if (x && typeof x === "object") { if (key in x && "renewed" in x) { if (value === "omit") delete x[key]; else x[key] = value } for (const v of Object.values(x)) walk(v) } } + walk(m) + return new Response(JSON.stringify(m), { status: res.status, headers: res.headers }) +} +for (const value of ["omit", null] as const) { + it(`R4-6b: restart resume with cancelRequested ${value === "omit" ? "omitted" : "null"} on every Heartbeat reply`, async () => { + const { Effect } = await import("effect") + const w = world() + w.floor.enqueue(job("1")) + const orig = w.ax.createTask + ;(w.ax as any).createTask = () => Effect.never + const l1 = start(w) + await until("a1 creating", () => l1.has("leased")) + await sleep(100) + await l1.drain() + ;(w.ax as any).createTask = orig + const l2 = start(w, l1.dir, {}, omit("cancelRequested", value, w.floor.fetch)) + try { + await until("a1 resumed and created", () => w.ax.tasks.get("wf-test-1-a1") !== undefined, 3000) + expect(l2.has("resume", "wf-test-1-a1")).toBe(true) + expect(l2.logs.filter((x) => x.ev === "heartbeat-error").length).toBe(0) + } finally { await l2.drain(); w.floor.close() } + }) +} +it("R4-6c: a lost lease is still acted on when the Heartbeat reply omits stats and cancelRequested", async () => { + const w = world() + w.floor.enqueue(job("1")) + const both: typeof globalThis.fetch = omit("cancelRequested", "omit", drift("Heartbeat", w.floor.fetch)) + const l = start(w, undefined, {}, both) + try { + await until("a1 running", () => w.ax.tasks.get("wf-test-1-a1")?.phase === "Running") + const J = w.floor.jobs.get("wf-test-1")! + J.holder = "someone-else" // the floor answers lost:[a1] from now on + await until("lost", () => l.has("lost", "wf-test-1-a1"), 2000) + await until("a1 deleted", () => w.ax.deletes.includes("wf-test-1-a1"), 2000) + } finally { await l.drain(); w.floor.close() } +}) diff --git a/pkgs/substrate-apps/src/apps/link/test/review-r4-journal-key.test.ts b/pkgs/substrate-apps/src/apps/link/test/review-r4-journal-key.test.ts new file mode 100644 index 00000000..0b101904 --- /dev/null +++ b/pkgs/substrate-apps/src/apps/link/test/review-r4-journal-key.test.ts @@ -0,0 +1,48 @@ +// Round 4 regression test for finding 8 (repro: /home/tom/today/evals-2026-09-23/link/scratch-r4-2/journal-key.repro.ts). +// AX_CONWIP_JOURNAL_KEY must be the FIELD-MAP 5a journal key ("sha256 of canonical [prompt, opts minus label] plus the +// occurrence"), carried on the AgentJob as the `ultracode.mecattaf.dev/journal-key` annotation, never the prompt +// digest. The expected values are pinned from the substrate taskspec (journalKey in +// /home/tom/mecattaf/ax-conwip-wt-integration/src/taskspec.ts at b42b310, run 2026-09-23), so this test does not +// import another worktree. +import { createHash } from "node:crypto" +import { expect, it } from "vitest" +import { axTaskFromGrant } from "../src/jobs.ts" +import { job } from "./review-r4-harness.ts" + +const TASKSPEC = { + halogen1: "29a9c6cb8956c07073345109161d0e7eaa08708f4c2039f4e4af7a09d212ee07:1", // journalKey("same prompt", {model: halogen}, 1) + opus1: "bd381a2d66f7a1d93c7df296954218de4c9749a0feb9d94719b9c484f8c06718:1", // journalKey("same prompt", {model: opus}, 1) + halogen2: "29a9c6cb8956c07073345109161d0e7eaa08708f4c2039f4e4af7a09d212ee07:2" // the same call, second occurrence +} +const sha = (s: string) => createHash("sha256").update(s).digest("hex") +const A = "ultracode.mecattaf.dev/" +const grant = (n: string, model: string, key: string | undefined) => { + const j: any = job(n) + j.spec.with.prompt = "same prompt" + j.spec.with.prompt_ref = { sha256: sha("same prompt"), bytes: 11, uri: `journal://wf_test/${n}/prompt.md` } + j.spec.with.model = model + if (key === undefined) delete j.metadata.annotations[A + "journal-key"] + else j.metadata.annotations[A + "journal-key"] = key + return { leaseId: `wf-test-${n}-a1`, attempt: 1, job: j, lease: { holderIdentity: "h", leaseDurationSeconds: 90, acquireTime: 1, renewTime: 1, leaseTransitions: 0 } } +} +const shape = { atespace: "fleet", image: "ax-agent", gateway: "halogen", command: () => ["ax-agent", "pi"] } +const env = (g: any) => { const b: any = axTaskFromGrant(g, shape); expect(b._tag).toBe("ok"); return Object.fromEntries(b.task.spec.env.map((e: any) => [e.name, e.value])) } + +it("R4-8a: AX_CONWIP_JOURNAL_KEY is the taskspec journal key for two calls that share a prompt", () => { + const a = env(grant("1", "halogen-qwen3.8-flash-next", TASKSPEC.halogen1)) + const b = env(grant("2", "claude-opus-5-5", TASKSPEC.opus1)) + const c = env(grant("3", "halogen-qwen3.8-flash-next", TASKSPEC.halogen2)) + expect(a.AX_CONWIP_JOURNAL_KEY).toBe(TASKSPEC.halogen1) + expect(b.AX_CONWIP_JOURNAL_KEY).toBe(TASKSPEC.opus1) + expect(c.AX_CONWIP_JOURNAL_KEY).toBe(TASKSPEC.halogen2) + expect(a.AX_CONWIP_JOURNAL_KEY).not.toBe(a.AX_CONWIP_PROMPT_SHA256) + expect(a.AX_CONWIP_PROMPT_SHA256).toBe(b.AX_CONWIP_PROMPT_SHA256) // the prompt digest is shared; the journal key is not +}) + +it("R4-8b: a grant without the annotation, or with a malformed one, is refused pre-start/invalid-spec", () => { + for (const key of [undefined, "", sha("same prompt"), `${sha("x")}:`, `${sha("x").toUpperCase()}:1`]) { + const b = axTaskFromGrant(grant("4", "halogen-qwen3.8-flash-next", key) as any, shape) + expect(b._tag).toBe("refused") + if (b._tag === "refused") expect(b.reason).toBe("pre-start/invalid-spec") + } +}) diff --git a/pkgs/substrate-apps/src/apps/link/test/review-r4-overtake.test.ts b/pkgs/substrate-apps/src/apps/link/test/review-r4-overtake.test.ts new file mode 100644 index 00000000..e1274ca3 --- /dev/null +++ b/pkgs/substrate-apps/src/apps/link/test/review-r4-overtake.test.ts @@ -0,0 +1,78 @@ +// Round 4 regression test, ported from /home/tom/today/evals-2026-09-23/link/scratch-r4-0/overtake.repro.ts (assertions kept; additions marked). +// Reply ordering: the regrant of a2 (gen 2) overtakes the Complete reply that withdrew a2 (gen 1). Set-up is R2-5a +// (review-r2.test.ts): a1 fails retryable while a2 gen 1 is leased to this link and waits for a1's verdict. a1's first +// Complete answers 500 once (server-error), which by design stops gating Lease (round 2), so a Lease with capacity 1 is +// in flight while the retried Complete is applied. The floor withdraws gen 1, requeues gen 2 and grants it on that +// Lease; the Lease reply lands before the Complete reply. The link still holds gen 1 unreleased, so gen 2 is dropped as +// `duplicate-grant`; the Complete reply then releases gen 1. Nothing of a2 is ever created, although the floor has +// gen 2 leased to this link: the job waits for lease expiry and is released `omitted` (an infra release). +import { expect, it } from "vitest" +import { bodyOf, job, sleep, start, until, world } from "./review-r4-harness.ts" +import type { World } from "./review-r4-harness.ts" + +const rpcOf = async (input: unknown, init?: RequestInit) => { const b = await bodyOf(input, init); return b.includes('"Complete"') ? "Complete" : b.includes('"Lease"') ? "Lease" : b.includes('"Heartbeat"') ? "Heartbeat" : "?" } +const failRetryable = (w: World, name: string) => { + const t = w.ax.tasks.get(name)! + t.exited = true; t.phase = "Failed" + t.conditions = [{ type: "Ready", status: "False", reason: "ResourceExhausted", message: "ResourceExhausted: no free worker" }] +} + +// Round 4 addition: also against a floor that omits withdrewTransitions (the link then uses the generation of a2 it +// held when it SENT a1's verdict, not the regrant it holds when the reply lands). +for (const sendWithdrewGen of [true, false]) +it(`the regrant of a2 is created even when its Lease reply lands before the Complete reply that withdrew gen 1 (withdrewTransitions ${sendWithdrewGen ? "sent" : "omitted"})`, async () => { + const w = world({ sendWithdrewGen }); w.floor.enqueue(job("1")) + const J = () => w.floor.jobs.get("wf-test-1")! + let holdLease = false, failHeartbeat = false, failComplete = false, fiveHundredOnce = false + let l: ReturnType + let armed = false, leaseHeld = false + const fetch: typeof globalThis.fetch = async (input, init) => { + const body = await bodyOf(input, init) + const rpc = await rpcOf(input, init) + if (rpc === "Heartbeat" && (failHeartbeat || armed)) throw new TypeError("fetch failed") // no Heartbeat in the window + if (rpc === "Complete" && failComplete) return new Response("upstream timeout", { status: 503 }) + if (rpc === "Complete" && fiveHundredOnce) { fiveHundredOnce = false; armed = true; return new Response("internal error", { status: 500 }) } + if (rpc === "Lease") while (holdLease) await sleep(2) + if (rpc === "Lease" && armed && /"capacity":[1-9]/.test(body) && !leaseHeld) { + leaseHeld = true // a Lease sent before the retried Complete, delivered after it (network reordering) + await until("the retried Complete was applied", () => J().history.includes("withdrawn:a2"), 3000).catch(() => {}) + // Round 4 addition: longer than the dispatch wait period (at most 2 x SEC), so the gen-1 dispatch is inside, or + // waiting on, the outbox lock when the Complete reply lands, and reaches its withdrawn branch deterministically + await sleep(80) + } + const res = await w.floor.fetch(input as any, { ...init, body }) + if (rpc === "Complete" && armed) { + await until("the Lease reply reached the link", () => l.has("duplicate-grant", "wf-test-1-a2") || l.has("regrant", "wf-test-1-a2"), 1500).catch(() => {}) + armed = false + } + return res + } + l = start(w, undefined, {}, fetch) + await until("running", () => w.ax.tasks.get("wf-test-1-a1")?.phase === "Running") + w.floor.down = true + await until("floor requeued attempt 2", () => J().state === "queued" && J().attempt === 2, 4000) + failHeartbeat = true; holdLease = true; failComplete = true + w.floor.down = false + await sleep(80) + failRetryable(w, "wf-test-1-a1") + await until("verdict written", () => l.has("verdict", "wf-test-1-a1")) + holdLease = false + await until("a2 received", () => l.has("leased", "wf-test-1-a2"), 4000) + await until("dispatch waits for the verdict", () => l.has("supersede-waits-for-verdict", "wf-test-1-a2")) + fiveHundredOnce = true + failComplete = false; failHeartbeat = false + await until("a2 created, or 1.5 s", () => w.ax.updates.has("wf-test-1-a2"), 1500).catch(() => {}) + await sleep(300) + const trace = l.logs.filter((x) => x.leaseId === "wf-test-1-a2" || x.task === "wf-test-1-a2" || x.ev === "complete" || x.ev === "outbox-keep" || x.ev === "lease-gated-by-outbox").map((x) => `${x.ev}:${x.leaseId ?? x.task}${x.why ? ":" + x.why : ""}${x.withdrew ? ":withdrew=" + x.withdrew : ""}`) + const out = { leaseHeld, floor: J().history, state: J().state, attempt: J().attempt, infraSpent: J().infraSpent, a2Updates: w.ax.updates.get("wf-test-1-a2") ?? 0, trace } + console.log(JSON.stringify(out)) + if (out.a2Updates !== 1 || out.floor.includes("requeue:omitted")) console.log("FULL", JSON.stringify(l.logs.map((x) => ({ ...x, cause: undefined }))), JSON.stringify(w.floor.calls.slice(-40))) + await l.drain(); w.floor.close() + expect(out.a2Updates).toBe(1) + expect(out.floor).not.toContain("requeue:omitted") // round 4 addition: no infra release spent on the stall + expect(out.leaseHeld).toBe(true) + // Round 4 addition: the gen-1 dispatch never releases the regrant (the withdrawn branch is bound to its own grant) + const a2 = l.logs.filter((x) => x.leaseId === "wf-test-1-a2").map((x) => String(x.ev)) + expect(a2.slice(a2.indexOf("regrant"))).not.toContain("withdrawn") + expect(w.ax.deletes).not.toContain("wf-test-1-a2") +}) diff --git a/pkgs/substrate-apps/src/apps/link/test/review-r4-p1stale.test.ts b/pkgs/substrate-apps/src/apps/link/test/review-r4-p1stale.test.ts new file mode 100644 index 00000000..17e77600 --- /dev/null +++ b/pkgs/substrate-apps/src/apps/link/test/review-r4-p1stale.test.ts @@ -0,0 +1,74 @@ +// Round 4 regression test, ported from /home/tom/today/evals-2026-09-23/link/scratch-r4-1/p1stale.repro.ts (assertions kept; additions marked). +// Round 4, lens capacity gate: canCreate() needs "P1 proven", but the probe runs only on an ax-up transition +// (link.ts resync: `if (wasDown) serverP1 = undefined`). ax-server is a replicas:1 Deployment with the default +// RollingUpdate strategy (upstream deploy/ax-server.yaml), so a rollout to a server without P1 happens with no failed +// ListTasks, and the link keeps creating Tasks it can never read a verdict from. +import { Effect } from "effect" +import { expect, it } from "vitest" +import { job, sleep, start, until, world } from "./review-r4-harness.ts" + +it("A: ax rolled to stock v0.3.0 (server and controller) with no ListTasks failure", async () => { + const w = world() + w.floor.enqueue(job("1")) + const l = start(w) + await until("a1 running", () => w.ax.tasks.get("wf-test-1-a1")?.phase === "Running") + const probesBefore = l.logs.filter((x) => x.ev === "completion-probe").length + w.ax.p1 = false // the rollout: GetTaskResult is now UNIMPLEMENTED and no terminal phase is ever written + await until("p1-missing logged", () => l.has("p1-missing"), 1000) // round 4 fix: the probe runs on every resync + const leasesAtFlip = w.floor.calls.filter((c) => c.rpc === "Lease").length + await sleep(100) // several resyncs (30 ms) and Lease polls pass; none fails + w.floor.enqueue(job("2")) + await sleep(1500) // round 4: job 2 is never created on the stock server (the repro waited for its create here) + const J2 = w.floor.jobs.get("wf-test-2")! + const out = { + probesBefore, probesAfter: l.logs.filter((x) => x.ev === "completion-probe").length, + p1MissingLogged: l.has("p1-missing"), + job2: { state: J2.state, attempt: J2.attempt, history: J2.history }, + task2Phase: w.ax.tasks.get("wf-test-2-a1")?.phase, task2Exited: (w.ax.tasks.get("wf-test-2-a1") as any)?.exited, + verdictJob2: l.has("verdict", "wf-test-2-a1"), + leaseCapacitiesAfterRollout: [...new Set(w.floor.calls.filter((c) => c.rpc === "Lease").slice(-10).map((c) => c.capacity))] + } + console.log("A", JSON.stringify(out)) + const leasesAfter = w.floor.calls.filter((c) => c.rpc === "Lease").slice(leasesAtFlip + 1) // round 4 addition + await l.drain(); w.floor.close() + // Fail-closed expectation: after the P1 route disappears the link leases and creates nothing new. + expect(w.ax.updates.get("wf-test-2-a1") ?? 0).toBe(0) + expect(out.p1MissingLogged).toBe(true) + expect(leasesAfter.length).toBeGreaterThan(0) + expect(leasesAfter.every((c) => c.capacity === 0)).toBe(true) + expect(J2.state).toBe("queued") +}) + +it("B: ax-server rolled to stock, the P1 controller still writes Completed", async () => { + const w = world() + w.floor.enqueue(job("1")) + const l = start(w) + await until("a1 running", () => w.ax.tasks.get("wf-test-1-a1")?.phase === "Running") + let stockServer = false + const orig = w.ax.getTaskResult + ;(w.ax as any).getTaskResult = (n: string) => stockServer ? Effect.succeed("unimplemented" as const) : orig(n) + stockServer = true + w.ax.finish("wf-test-1-a1", 0, { answer: 1 }) // Completed, result stored by the controller, unreadable via ax-server + for (let i = 0; i < 40; i++) { // every attempt the floor grants finishes with a success the link cannot read + for (const [name, t] of w.ax.tasks) if (t.phase === "Running" && !t.exited) w.ax.finish(name, 0, { answer: name }) + await sleep(50) + } + const J = w.floor.jobs.get("wf-test-1")! + const out = { + probes: l.logs.filter((x) => x.ev === "completion-probe").length, p1MissingLogged: l.has("p1-missing"), + creates: [...w.ax.updates.keys()], job1: { state: J.state, result: (J as any).result, output: (J as any).output, history: J.history }, + verdicts: l.logs.filter((x) => x.ev === "verdict").map((x) => `${x.leaseId}:${x.result}:${x.reason ?? ""}`) + } + console.log("B", JSON.stringify(out)) + // Round 4 addition: the Task is kept, uncharged, and read once P1 is back: the success is reported, not thrown away + stockServer = false + await until("job 1 done", () => J.state === "done", 3000) + const leasesUnimpl = w.floor.calls.filter((c) => c.rpc === "Lease") + await l.drain(); w.floor.close() + expect(out.creates.length).toBe(1) // fail closed: nothing created once the server answered UNIMPLEMENTED + expect(out.p1MissingLogged).toBe(true) + expect(out.verdicts.some((v) => v.includes("result-unreadable"))).toBe(false) + expect((J as any).result).toBe("success") + expect((J as any).output).toEqual({ answer: 1 }) + expect(leasesUnimpl.length).toBeGreaterThan(0) +}) diff --git a/pkgs/substrate-apps/src/apps/link/test/review-r4-restart-replay.test.ts b/pkgs/substrate-apps/src/apps/link/test/review-r4-restart-replay.test.ts new file mode 100644 index 00000000..9307a75a --- /dev/null +++ b/pkgs/substrate-apps/src/apps/link/test/review-r4-restart-replay.test.ts @@ -0,0 +1,143 @@ +// Round 4 regression test, ported from /home/tom/today/evals-2026-09-23/link/scratch-r4-0/restart-replay.repro.ts (assertions kept; additions marked). +// Kill -9 mid-Lease (B8): the floor applied the Lease and granted a1, the link journaled the requestKey but died +// before the reply was journaled. The restart takes longer than the lease (orphaned) and less than lease + grace. +// B8 says the journaled key is replayed so the grant is recovered. At startup the link sends its first Heartbeat +// (held set without a1) BEFORE it replays the key, so the floor releases the omitted orphan (rule 7 / L3 a), and the +// replay then returns nothing: one infra release is spent and the job reruns as a2. +import { readFileSync } from "node:fs" +import { Effect, Fiber } from "effect" +import { expect, it } from "vitest" +import { job, SEC, sleep, start, until, world, bodyOf } from "./review-r4-harness.ts" + +it("a journaled requestKey whose grant outlived the lease is recovered on restart", async () => { + const w = world() + w.floor.enqueue(job("r1")) + // link #1: the first capacity>0 Lease reaches the floor, then the process "dies" before the reply lands + let leased = false + const dying: typeof globalThis.fetch = async (input, init) => { + const body = await bodyOf(input, init) + if (/"Lease"/.test(body) && /"capacity":[1-9]/.test(body)) { + await w.floor.fetch(input as any, { ...init, body }) + leased = true + return new Promise(() => {}) // the reply never reaches the journal + } + return w.floor.fetch(input as any, { ...init, body }) + } + const l1 = start(w, undefined, {}, dying) + await until("floor granted a1", () => leased) + Effect.runFork(Fiber.interrupt(l1.fiber)) // kill -9: nothing after this point runs in link #1 + await sleep(50) + const journal1 = readFileSync(l1.dir + "/journal.jsonl", "utf8") + const keyJournaled = /"ev":"lease-key"/.test(journal1) && !/"ev":"lease-replied"/.test(journal1) + // downtime: past the lease (6 s * SEC = 120 ms) and well inside lease + grace (21 s * SEC = 420 ms) + await sleep(8 * SEC) + const stateAtRestart = w.floor.jobs.get("wf-test-r1")!.state + const l2 = start(w, l1.dir) + await until("a Task exists", () => w.ax.tasks.size > 0, 3000).catch(() => {}) + await sleep(200) + const j = w.floor.jobs.get("wf-test-r1")! + const out = { keyJournaled, stateAtRestart, history: j.history, infraSpent: j.infraSpent, tasks: [...w.ax.tasks.keys()], + l2: l2.logs.filter((l) => ["leased", "duplicate-grant", "created", "resume"].includes(String(l.ev))).map((l) => `${l.ev}:${l.leaseId}`) } + console.log(JSON.stringify(out)) + await l2.drain() + w.floor.close() + expect(keyJournaled).toBe(true) + expect(stateAtRestart).toBe("orphaned") + expect(j.history).not.toContain("requeue:omitted") // the grant under the journaled key was released by the first heartbeat + expect(out.tasks).toContain("wf-test-r1-a1") + expect(j.infraSpent).toBe(0) // round 4 addition + expect(out.tasks).not.toContain("wf-test-r1-a2") +}) + +it("control: the same kill with a restart shorter than the lease recovers a1", async () => { + const w = world() + w.floor.enqueue(job("r2")) + let leased = false + const dying: typeof globalThis.fetch = async (input, init) => { + const body = await bodyOf(input, init) + if (/"Lease"/.test(body) && /"capacity":[1-9]/.test(body)) { await w.floor.fetch(input as any, { ...init, body }); leased = true; return new Promise(() => {}) } + return w.floor.fetch(input as any, { ...init, body }) + } + const l1 = start(w, undefined, {}, dying) + await until("floor granted a1", () => leased) + Effect.runFork(Fiber.interrupt(l1.fiber)) + await sleep(2 * SEC) + const l2 = start(w, l1.dir) + await until("a Task exists", () => w.ax.tasks.size > 0, 3000).catch(() => {}) + const j = w.floor.jobs.get("wf-test-r2")! + console.log(JSON.stringify({ control: true, history: j.history, tasks: [...w.ax.tasks.keys()] })) + await l2.drain(); w.floor.close() + expect([...w.ax.tasks.keys()]).toContain("wf-test-r2-a1") +}) + +it("running process: the reply is lost and Lease stays unanswered past the lease while Heartbeat works", async () => { + const w = world() + w.floor.enqueue(job("r3")) + let first = true, blockUntil = 0 + const f: typeof globalThis.fetch = async (input, init) => { + const body = await bodyOf(input, init) + if (/"Lease"/.test(body)) { + if (first && /"capacity":[1-9]/.test(body)) { first = false; await w.floor.fetch(input as any, { ...init, body }); blockUntil = Date.now() + 10 * SEC; throw new TypeError("fetch failed") } + if (Date.now() < blockUntil) throw new TypeError("fetch failed") + } + return w.floor.fetch(input as any, { ...init, body }) + } + const l = start(w, undefined, {}, f) + await until("a Task exists", () => w.ax.tasks.size > 0, 4000).catch(() => {}) + await sleep(100) + const j = w.floor.jobs.get("wf-test-r3")! + console.log(JSON.stringify({ running: true, history: j.history, infraSpent: j.infraSpent, tasks: [...w.ax.tasks.keys()] })) + await l.drain(); w.floor.close() + expect(j.history).not.toContain("requeue:omitted") + expect([...w.ax.tasks.keys()]).toEqual(["wf-test-r3-a1"]) // round 4 addition +}) + +// Round 4 addition: the running-process half needs the floor's rule 6b. Against an L1-only floor (vouchByKey false) +// the residual is still there; this pins that the link-side fix alone does not claim it. +it("control: an L1-only floor that ignores pendingRequestKey still releases the omitted orphan", async () => { + const w = world() + w.floor.vouchByKey = false + w.floor.enqueue(job("r4")) + let first = true, blockUntil = 0 + const f: typeof globalThis.fetch = async (input, init) => { + const body = await bodyOf(input, init) + if (/"Lease"/.test(body)) { + if (first && /"capacity":[1-9]/.test(body)) { first = false; await w.floor.fetch(input as any, { ...init, body }); blockUntil = Date.now() + 10 * SEC; throw new TypeError("fetch failed") } + if (Date.now() < blockUntil) throw new TypeError("fetch failed") + } + return w.floor.fetch(input as any, { ...init, body }) + } + const l = start(w, undefined, {}, f) + await until("a Task exists", () => w.ax.tasks.size > 0, 4000).catch(() => {}) + const j = w.floor.jobs.get("wf-test-r4")! + await l.drain(); w.floor.close() + expect(j.history).toContain("requeue:omitted") +}) + +// Round 4 addition: the startup half does not depend on the floor. Against an L1-only floor (no rule 6b) the journaled +// key is still answered before the first Heartbeat, so the orphan granted under it is listed and re-adopted. +it("restart past the lease against an L1-only floor: the startup replay recovers a1", async () => { + const w = world() + w.floor.vouchByKey = false + w.floor.enqueue(job("r5")) + let leased = false + const dying: typeof globalThis.fetch = async (input, init) => { + const body = await bodyOf(input, init) + if (/"Lease"/.test(body) && /"capacity":[1-9]/.test(body)) { await w.floor.fetch(input as any, { ...init, body }); leased = true; return new Promise(() => {}) } + return w.floor.fetch(input as any, { ...init, body }) + } + const l1 = start(w, undefined, {}, dying) + await until("floor granted a1", () => leased) + Effect.runFork(Fiber.interrupt(l1.fiber)) + await sleep(8 * SEC) + const stateAtRestart = w.floor.jobs.get("wf-test-r5")!.state + const l2 = start(w, l1.dir) + await until("a Task exists", () => w.ax.tasks.size > 0, 3000).catch(() => {}) + await sleep(100) + const j = w.floor.jobs.get("wf-test-r5")! + await l2.drain(); w.floor.close() + expect(stateAtRestart).toBe("orphaned") + expect(l2.has("lease-key-replayed")).toBe(true) + expect(j.history).not.toContain("requeue:omitted") + expect([...w.ax.tasks.keys()]).toEqual(["wf-test-r5-a1"]) +}) diff --git a/pkgs/substrate-apps/src/apps/link/test/review-r4-stale-lost.test.ts b/pkgs/substrate-apps/src/apps/link/test/review-r4-stale-lost.test.ts new file mode 100644 index 00000000..a999b9fd --- /dev/null +++ b/pkgs/substrate-apps/src/apps/link/test/review-r4-stale-lost.test.ts @@ -0,0 +1,67 @@ +// Round 4 regression test, ported from /home/tom/today/evals-2026-09-23/link/scratch-r4-0/stale-lost.repro.ts (assertions kept; additions marked). +// Duplicate/stale delivery: a Heartbeat reply is applied to whatever record holds that leaseId when the reply lands, +// not to the generation the Heartbeat vouched for. Set-up is R2-5a (review-r2.test.ts): a1 fails retryable while a2 +// gen 1 is leased to this link and waits for a1's verdict. One Heartbeat carrying [a2] (gen 1) reaches the floor +// after rule 4b withdrew a2 gen 1 and requeued a2 gen 2, so the floor answers lost:[a2]. Its reply lands after the +// link took the regrant of a2 (gen 2). The link then releases gen 2 as `lost` and never creates it, although the floor +// has gen 2 leased to it: the job is stranded until the lease expires, then released `omitted` (an infra release). +import { Effect } from "effect" +import { expect, it } from "vitest" +import { bodyOf, job, sleep, start, until, world } from "./review-r4-harness.ts" +import type { World } from "./review-r4-harness.ts" + +const rpcOf = async (input: unknown, init?: RequestInit) => { const b = await bodyOf(input, init); return b.includes('"Complete"') ? "Complete" : b.includes('"Lease"') ? "Lease" : b.includes('"Heartbeat"') ? "Heartbeat" : "?" } +const failRetryable = (w: World, name: string) => { + const t = w.ax.tasks.get(name)! + t.exited = true; t.phase = "Failed" + t.conditions = [{ type: "Ready", status: "False", reason: "ResourceExhausted", message: "ResourceExhausted: no free worker" }] +} + +it("a Heartbeat reply computed for a2 gen 1 does not release a2 gen 2", async () => { + const w = world(); w.floor.enqueue(job("1")) + const J = () => w.floor.jobs.get("wf-test-1")! + let holdLease = false, failHeartbeat = false, failComplete = false + let armed = false, hbForwarded = false, hbIntercepted = false + let l: ReturnType + const fetch: typeof globalThis.fetch = async (input, init) => { + const body = await bodyOf(input, init) + const rpc = await rpcOf(input, init) + if (rpc === "Heartbeat" && failHeartbeat) throw new TypeError("fetch failed") + if (rpc === "Complete" && failComplete) return new Response("upstream timeout", { status: 503 }) + if (rpc === "Complete" && armed) await until("a Heartbeat vouching for a2 gen 1 is in flight", () => hbIntercepted, 4000) + if (rpc === "Lease") { while (holdLease) await sleep(2); while (armed && !hbForwarded) await sleep(2) } + if (rpc === "Heartbeat" && armed && !hbIntercepted && body.includes("wf-test-1-a2")) { + hbIntercepted = true + await until("floor withdrew a2 gen 1 and requeued a2 gen 2", () => J().history.includes("withdrawn:a2") && J().state === "queued", 4000) + const res = await w.floor.fetch(input as any, { ...init, body }) + hbForwarded = true // Lease may go now: the floor regrants a2 (gen 2) + await until("link took the regrant", () => l.has("regrant", "wf-test-1-a2"), 4000).catch(() => {}) + return res // the reply (lost:[a2]) lands after the regrant + } + return w.floor.fetch(input as any, { ...init, body }) + } + l = start(w, undefined, {}, fetch) + await until("running", () => w.ax.tasks.get("wf-test-1-a1")?.phase === "Running") + w.floor.down = true + await until("floor requeued attempt 2", () => J().state === "queued" && J().attempt === 2, 4000) + failHeartbeat = true; holdLease = true; failComplete = true + w.floor.down = false + await sleep(80) + failRetryable(w, "wf-test-1-a1") + await until("verdict written", () => l.has("verdict", "wf-test-1-a1")) + holdLease = false + await until("a2 received", () => l.has("leased", "wf-test-1-a2"), 4000) + await until("dispatch waits for the verdict", () => l.has("supersede-waits-for-verdict", "wf-test-1-a2")) + armed = true + failComplete = false; failHeartbeat = false + await until("a2 created, or 1 s", () => w.ax.updates.has("wf-test-1-a2"), 1000).catch(() => {}) + await sleep(300) + const trace = l.logs.filter((x) => x.leaseId === "wf-test-1-a2" || x.task === "wf-test-1-a2").map((x) => `${x.ev}${x.why ? ":" + x.why : ""}`) + const out = { hbIntercepted, floor: J().history, state: J().state, attempt: J().attempt, infraSpent: J().infraSpent, a2Updates: w.ax.updates.get("wf-test-1-a2") ?? 0, a2trace: trace } + console.log(JSON.stringify(out)) + await l.drain(); w.floor.close() + expect(hbIntercepted).toBe(true) + expect(out.a2Updates).toBe(1) + expect(out.floor).not.toContain("requeue:omitted") // round 4 addition + expect(out.a2trace).toContain("stale-heartbeat-reply") +}) diff --git a/pkgs/substrate-apps/src/apps/link/test/workerd.integration.test.ts b/pkgs/substrate-apps/src/apps/link/test/workerd.integration.test.ts new file mode 100644 index 00000000..05142b89 --- /dev/null +++ b/pkgs/substrate-apps/src/apps/link/test/workerd.integration.test.ts @@ -0,0 +1,191 @@ +// Integration: the link against a REAL floor process and a REAL gRPC ax wire. +// floor the option-A prototype Durable Object (../arc/proto/floor, Effect's RpcServer.layerHttp inside a DO, DO +// SQLite, a storage alarm) served by `wrangler dev --local` on loopback (workerd). Copied to a temp dir first, +// so nothing is written into the prototype's own tree. It is NOT the floor port (FL1 to FL8 are not in it). +// ax a gRPC server on loopback built from the vendored ax.proto plus P1's GetTaskResult (fixtures/ax-p1.proto), +// with ax's own ListTasks paging (50 rows, newest first) and a Gateway. The link uses its production grpcAx. +// Opt-in: LINK_WORKERD=1, run inside ~/.local/bin/runtime-test (it starts workerd). Skipped otherwise. +import { spawn } from "node:child_process" +import type { ChildProcess } from "node:child_process" +import { createHash } from "node:crypto" +import { cpSync, mkdtempSync, rmSync, symlinkSync } from "node:fs" +import { tmpdir } from "node:os" +import { join } from "node:path" +import * as grpc from "@grpc/grpc-js" +import * as protoLoader from "@grpc/proto-loader" +import { Deferred, Effect, Exit, Fiber } from "effect" +import { afterAll, beforeAll, describe, expect, it } from "vitest" +import { grpcAx } from "../src/ax.ts" +import type { AgentJob } from "../src/contract.ts" +import { rpcFloor } from "../src/floor.ts" +import { Journal } from "../src/journal.ts" +import { runLink } from "../src/link.ts" +import type { LinkConfig } from "../src/link.ts" + +const RUN = process.env.LINK_WORKERD === "1" && (process.env.LINK_PROTO_DIR ?? "") !== "" +// The ARC prototype floor directory (LINK_PROTO_DIR); the test is skipped without it. +const PROTO_DIR = process.env.LINK_PROTO_DIR ?? "" +const P1_PROTO = join(import.meta.dirname, "fixtures", "ax-p1.proto") +const TOKEN = "local-dev-only-not-a-secret" // the prototype's wrangler.jsonc var, a public test value +const PORT = 18000 + Math.floor(Math.random() * 900) +const F = `http://127.0.0.1:${PORT}` +const A = "ultracode.mecattaf.dev/" + +const job = (n: string): AgentJob => ({ + apiVersion: "ultracode.mecattaf.dev/v1alpha1", kind: "AgentJob", + metadata: { name: `wf-int-${n}`, labels: { [A + "workflow"]: "link-int", [A + "phase-index"]: "1" }, annotations: { [A + "run-id-raw"]: "wf_int", [A + "label"]: `int:${n}`, [A + "item-key"]: `wf_int#${n}`, [A + "journal-key"]: `${"cd".repeat(32)}:1`, [A + "phase-title"]: "Integration" } }, + spec: { "runs-on": ["seat:halogen", "runtime:gvisor"], with: { prompt: `say ${n}`, prompt_ref: { sha256: "cd".repeat(32), bytes: 5, uri: `journal://wf_int/${n}/prompt.md` }, model: "halogen-qwen3.8-flash-next" } } +}) + +// ---------------------------------------------------------------- the ax gRPC server (P1 wire) +const tasks = new Map() +const results = new Map() +let axUnavailable = false +let axServer: grpc.Server +let axPort = 0 +const unavailable = { code: grpc.status.UNAVAILABLE, details: "ax-server unavailable" } +const nf = (what: string) => ({ code: grpc.status.NOT_FOUND, details: `${what} not found` }) +const guard = (cb: grpc.sendUnaryData, f: () => void) => axUnavailable ? cb(unavailable) : f() + +async function startAx() { + const def = protoLoader.loadSync(P1_PROTO, { keepCase: false, longs: String, enums: String, defaults: true, oneofs: true }) + const svc = (grpc.loadPackageDefinition(def) as any).ax.v1alpha1.AX.service + axServer = new grpc.Server() + axServer.addService(svc, { + GetTask: (c: any, cb: any) => guard(cb, () => { const t = tasks.get(c.request.name); t ? cb(null, t) : cb(nf(c.request.name)) }), + UpdateTask: (c: any, cb: any) => guard(cb, () => { tasks.set(c.request.task.metadata.name, { ...c.request.task, status: { phase: "Pending" } }); cb(null, c.request.task) }), + ListTasks: (c: any, cb: any) => guard(cb, () => { const n = Number(c.request.limit) > 0 ? Number(c.request.limit) : 50, o = Number(c.request.offset) || 0; cb(null, { tasks: [...tasks.values()].reverse().slice(o, o + n) }) }), + DeleteTask: (c: any, cb: any) => guard(cb, () => { tasks.delete(c.request.name); results.delete(c.request.name); cb(null, {}) }), + GetGateway: (c: any, cb: any) => guard(cb, () => c.request.name === "halogen" + ? cb(null, { apiVersion: "ax.io/v1alpha1", kind: "Gateway", metadata: { name: "halogen", atespace: "fleet" }, spec: { egress: { allowlist: { hosts: [{ host: "worker", port: 8731 }] } } } }) + : cb(nf("gateway"))), + GetTaskResult: (c: any, cb: any) => guard(cb, () => { const b = results.get(c.request.name); b ? cb(null, { content: b, sha256: createHash("sha256").update(b).digest("hex") }) : cb(nf("result")) }) + }) + axPort = await new Promise((res, rej) => axServer.bindAsync("127.0.0.1:0", grpc.ServerCredentials.createInsecure(), (e, p) => e ? rej(e) : res(p))) +} +const run = (name: string) => { tasks.get(name).status = { phase: "Running" } } +const finish = (name: string, out: unknown) => { + results.set(name, Buffer.from(JSON.stringify(out))) + tasks.get(name).status = { phase: "Completed", usage: { promptTokens: 5, completionTokens: 3, toolCalls: 1 }, conditions: [{ type: "Ready", status: "False", reason: "CommandExited", message: "ExitCode=0" }] } +} + +// ---------------------------------------------------------------- the floor under wrangler dev --local +let work = "", wrangler: ChildProcess | undefined +const floorLog: Array = [] +async function floorUp() { + wrangler = spawn("wrangler", ["dev", "--local", "--ip", "127.0.0.1", "--port", String(PORT), "--persist-to", join(work, "state"), + "--var", "LEASE_SECONDS:30", "--show-interactive-dev-session=false"], + { cwd: join(work, "floor"), env: { ...process.env, WRANGLER_SEND_METRICS: "false", CI: "1" }, detached: true, stdio: ["ignore", "pipe", "pipe"] }) + wrangler.stdout!.on("data", (d) => floorLog.push(String(d))) + wrangler.stderr!.on("data", (d) => floorLog.push(String(d))) + await until("floor up", async () => (await admin("GET", "/admin/state").catch(() => undefined)) !== undefined, 60_000) +} +async function floorDown() { + if (wrangler?.pid) { try { process.kill(-wrangler.pid, "SIGTERM") } catch { /* gone */ } } + await until("floor down", async () => (await admin("GET", "/admin/state").catch(() => undefined)) === undefined, 20_000) +} +async function admin(method: string, path: string, body?: unknown): Promise { + const r = await fetch(F + path, { method, headers: { authorization: `Bearer ${TOKEN}`, "content-type": "application/json" }, ...(body ? { body: JSON.stringify(body) } : {}), signal: AbortSignal.timeout(3000) }) + if (!r.ok) throw new Error(`admin ${path}: ${r.status}`) + return r.json() +} +const jobRow = async (n: string) => ((await admin("GET", "/admin/state")).jobs as Array).find((j) => j.name === `wf-int-${n}`) + +async function until(what: string, pred: () => boolean | Promise, ms = 30_000) { + const t0 = Date.now() + while (!(await pred())) { if (Date.now() - t0 > ms) throw new Error(`timeout waiting for: ${what}`); await new Promise((r) => setTimeout(r, 100)) } +} +const sleep = (ms: number) => new Promise((r) => setTimeout(r, ms)) + +function startLink(token = TOKEN) { + const logs: Array> = [] + const dir = mkdtempSync(join(tmpdir(), "conwip-link-int-")) + const stop = Effect.runSync(Deferred.make()) + const ax = grpcAx(`127.0.0.1:${axPort}`, "fleet", 5_000, P1_PROTO) + const cfg: LinkConfig = { + holder: "nas-link-1", maxInFlight: 2, servedLabels: ["seat:halogen", "runtime:gvisor"], + shape: { atespace: "fleet", image: "ax-agent", gateway: "halogen", command: () => ["ax-agent", "pi"] }, + completion: "p1", secondMs: 1000, resyncMs: 500, pendingTimeoutMs: 60_000, deleteAfterMs: 0, deadlineBackstopMs: 60_000, + createAttempts: 3, outboxBackoffMs: [200, 1000], fenceTimeoutMs: 5_000, initialPollSeconds: 1, initialHeartbeatSeconds: 2 + } + const fiber = Effect.runFork(Effect.scoped(Effect.gen(function*() { + const floor = yield* rpcFloor({ url: F, token, sessionId: `s:${dir}`, timeoutsMs: { lease: 5_000, heartbeat: 5_000, complete: 10_000 } }) + return yield* runLink(cfg, { ax, floor, journal: Journal.open(dir), log: (ev, f) => logs.push({ ev, ...f }), stop, floorUrl: F }) + }))) + return { + logs, + has: (ev: string) => logs.some((l) => l.ev === ev), + exit: () => Effect.runPromise(Fiber.await(fiber)), + drain: async () => { Effect.runSync(Deferred.succeed(stop, undefined)); const e = await Effect.runPromise(Fiber.await(fiber)); ax.close(); rmSync(dir, { recursive: true, force: true }); return e } + } +} + +describe.skipIf(!RUN)("link against the prototype floor under wrangler dev --local and a P1 gRPC ax", () => { + beforeAll(async () => { + work = mkdtempSync(join(tmpdir(), "conwip-floor-int-")) + cpSync(join(PROTO_DIR, "floor"), join(work, "floor"), { recursive: true, filter: (s) => !s.includes(".wrangler") }) + cpSync(join(PROTO_DIR, "contract.ts"), join(work, "contract.ts")) + symlinkSync(join(PROTO_DIR, "node_modules"), join(work, "node_modules")) + await startAx() + await floorUp() + }, 90_000) + afterAll(async () => { + await floorDown().catch(() => undefined) + axServer?.forceShutdown() + if (process.env.LINK_DEBUG) console.log(floorLog.join("").slice(-4000)) + if (work) rmSync(work, { recursive: true, force: true }) + }, 30_000) + + it("I1 happy path over real HTTP RPC and real gRPC: lease, create-only, P1 result with sha256, Complete, janitor delete", async () => { + await admin("POST", "/admin/enqueue", job("1")) + const l = startLink() + await until("created", () => tasks.has("wf-int-1-a1")) + expect(tasks.get("wf-int-1-a1").spec.env.find((e: any) => e.name === "AX_CONWIP_PROMPT").value).toBe("say 1") + run("wf-int-1-a1"); finish("wf-int-1-a1", { answer: 42 }) + await until("floor done", async () => (await jobRow("1"))?.state === "done") + const r = await jobRow("1") + expect([r.result, JSON.parse(r.output), JSON.parse(r.usage), r.attempt]).toEqual(["success", { answer: 42 }, { prompt_tokens: 5, completion_tokens: 3, tool_calls: 1 }, 1]) + await until("janitor deleted the Task", () => !tasks.has("wf-int-1-a1")) + expect(l.has("completion-probe") && l.has("gateway-ok")).toBe(true) + expect(Exit.isSuccess(await l.drain())).toBe(true) + }, 60_000) + + it("I2 floor process restarted while a verdict is pending: the outbox keeps it and it lands once, attempt 1", async () => { + await admin("POST", "/admin/enqueue", job("2")) + const l = startLink() + await until("created", () => tasks.has("wf-int-2-a1")) + run("wf-int-2-a1") + await floorDown() + finish("wf-int-2-a1", { restarted: true }) + await until("outbox keeps", () => l.has("outbox-keep")) + await floorUp() // same --persist-to: the DO's SQLite and alarm survive + await until("floor done", async () => (await jobRow("2"))?.state === "done", 45_000) + const r = await jobRow("2") + expect([r.result, JSON.parse(r.output), r.attempt, r.transitions]).toEqual(["success", { restarted: true }, 1, 0]) + expect(Exit.isSuccess(await l.drain())).toBe(true) + }, 120_000) + + it("I3 ax answers UNAVAILABLE: every Lease asks for 0, the job stays queued; ax returns and the job runs", async () => { + axUnavailable = true + const l = startLink() + await sleep(1_500) // the start-up resync fails; nothing is probed, capacity stays 0 + expect(l.has("completion-probe")).toBe(false) + await admin("POST", "/admin/enqueue", job("3")) + await sleep(6_000) // three polls at the floor's 2 s + expect((await jobRow("3")).state).toBe("queued") + axUnavailable = false + await until("created", () => tasks.has("wf-int-3-a1")) + run("wf-int-3-a1"); finish("wf-int-3-a1", { back: true }) + await until("floor done", async () => (await jobRow("3"))?.state === "done") + expect((await jobRow("3")).result).toBe("success") + expect(Exit.isSuccess(await l.drain())).toBe(true) + }, 60_000) + + it("I4 the real Worker entry answers 401 to a wrong bearer: the link stops with kind auth (exit 78 in main)", async () => { + const l = startLink("not-the-token") + const exit = await l.exit() + expect(Exit.isFailure(exit)).toBe(true) + expect(JSON.stringify(exit)).toContain("\"kind\":\"auth\"") + await l.drain() + }, 30_000) +}) diff --git a/pkgs/substrate-apps/src/apps/link/tsconfig.json b/pkgs/substrate-apps/src/apps/link/tsconfig.json new file mode 100644 index 00000000..eb9a0298 --- /dev/null +++ b/pkgs/substrate-apps/src/apps/link/tsconfig.json @@ -0,0 +1,16 @@ +{ + "compilerOptions": { + "target": "ES2022", + "module": "NodeNext", + "moduleResolution": "NodeNext", + "lib": ["ES2022"], + "strict": true, + "exactOptionalPropertyTypes": false, + "noUncheckedIndexedAccess": true, + "allowImportingTsExtensions": true, + "noEmit": true, + "skipLibCheck": true, + "types": ["node"] + }, + "include": ["src", "test"] +} diff --git a/pkgs/substrate-apps/src/apps/link/vitest.config.ts b/pkgs/substrate-apps/src/apps/link/vitest.config.ts new file mode 100644 index 00000000..7ef18d4f --- /dev/null +++ b/pkgs/substrate-apps/src/apps/link/vitest.config.ts @@ -0,0 +1,8 @@ +import { defineConfig } from "vitest/config" + +export default defineConfig({ + test: { + include: ["test/**/*.test.ts"], + environment: "node" + } +}) diff --git a/pkgs/substrate-apps/src/apps/pusher/bin/seat-pusher.mjs b/pkgs/substrate-apps/src/apps/pusher/bin/seat-pusher.mjs new file mode 100644 index 00000000..afdf7a62 --- /dev/null +++ b/pkgs/substrate-apps/src/apps/pusher/bin/seat-pusher.mjs @@ -0,0 +1,182 @@ +#!/usr/bin/env node +// apps/pusher/bin/seat-pusher.mjs: LEGACY. Kept for its tests; not the package bin. The pusher to run is +// bin/substrate-pusher.mjs (docs/pusher.md), which reads `seats --json` and never calls a usage endpoint itself. +// +// pushes seat capacity to the CONWIP floor. +// +// node apps/pusher/bin/seat-pusher.mjs --url URL --token-file PATH [options] +// node apps/pusher/bin/seat-pusher.mjs --dry-run [options] +// +// Each seat named under `usage` in the config is read through its own source +// (src/usageSource.mjs): "oauth" reads the usage endpoint directly with the +// seat's credentials file, under the gentle refresh policy +// (src/refreshPolicy.mjs: 120 s hard minimum, 900 s idle, 300 s active, 180 s +// near cap, a read at each reset + 90 s, 429 backoff, no call after an auth +// failure until the credentials change); "cache-file" reads a file another +// reader wrote. A legacy meters directory (--meters) is read only when named. +// The snapshot (`seat-capacity/2`) goes OUTBOUND to `/capacity/seats`. +// +// Push policy: push when any seat's reading changed (a Claude seat's provider +// observed_at, grade, windows, dispatchability, slots, plan) and at least every +// --heartbeat seconds (default 1800), or every --slot-heartbeat seconds +// (default 600) while a dispatchable Halogen slot row is published, so the +// floor never grades a live slot row STALE; never twice within --min-interval +// seconds (default 60). A restamp with the same reading is not a change. +// +// Options: +// --url URL the floor's base URL (required unless --dry-run) +// --token-file PATH the floor's bearer, read from a file and never printed +// --config PATH default ~/.config/substrate/seat-capacity.json (optional; the +// dry run reads the same path, and the ruled seats cc3 +// and gpu-coordinator are never dispatchable either way) +// --state PATH default ~/.local/state/substrate/pusher/state.json +// --usage-dir DIR the direct reader's cache and refresh state +// (default ~/.local/state/substrate/usage) +// --demand-dir DIR demand markers (default ~/.local/state/substrate/demand) +// --meters DIR a legacy meters directory; none by default +// --capacity-dir DIR a directory holding a feeder-written seats.json; none by default +// --host NAME overrides the config's host and the hostname +// --watch SECONDS loop, one pass every SECONDS (minimum 30); default: one pass +// --heartbeat SECONDS --slot-heartbeat SECONDS --min-interval SECONDS +// --force --instance NAME +// --pre-dispatch treat this pass as a pre-dispatch read (reading older than 300 s) +// --no-network never call a usage endpoint (cached readings only) +// --dry-run print the snapshot and the decision; no network, no state +// +// Exit: 0 on a pass that pushed or had nothing due; 1 on a failed push; 2 usage. + +import { readdirSync } from "node:fs" +import { join } from "node:path" +import { parseArgs } from "node:util" + +import { tokenFromFile } from "../src/token.mjs" +import { + DEFAULT_POLICY, + defaultConfigPath, + defaultPusherDir, + readConfig, + runOnce +} from "../src/seatCapacity.mjs" + +const usage = (message) => { + process.stderr.write(`seat-pusher: ${message}\n`) + process.exit(2) +} + +let args +try { + args = parseArgs({ + options: { + url: { type: "string" }, + "token-file": { type: "string" }, + meters: { type: "string" }, + "capacity-dir": { type: "string" }, + state: { type: "string" }, + config: { type: "string" }, + host: { type: "string" }, + watch: { type: "string" }, + heartbeat: { type: "string" }, + "slot-heartbeat": { type: "string" }, + "min-interval": { type: "string" }, + instance: { type: "string" }, + "usage-dir": { type: "string" }, + "demand-dir": { type: "string" }, + "pre-dispatch": { type: "boolean", default: false }, + "no-network": { type: "boolean", default: false }, + force: { type: "boolean", default: false }, + "dry-run": { type: "boolean", default: false } + }, + strict: true + }).values +} catch (error) { + usage(error.message) +} + +const seconds = (name, fallback, minimum = 0) => { + const raw = args[name] + if (raw === undefined) return fallback + const value = Number(raw) + if (!Number.isFinite(value) || value < minimum) usage(`--${name} must be a number of seconds >= ${minimum}`) + return value +} + +const dryRun = args["dry-run"] +if (!dryRun && (args.url === undefined || args["token-file"] === undefined)) { + usage("--url and --token-file are required (or --dry-run)") +} +const metersDir = args.meters === undefined || args.meters === "" ? null : args.meters +const capacityDir = args["capacity-dir"] === undefined || args["capacity-dir"] === "" ? null : args["capacity-dir"] +const statePath = args.state ?? join(defaultPusherDir(), "state.json") +// The dry run reads the same config as the real run, so it previews what a +// real run would push. +const config = readConfig(args.config ?? defaultConfigPath()) +if (!config.present) { + process.stderr.write( + "seat-pusher: no seat-capacity config; using built-in rulings only (cc3, gpu-coordinator not dispatchable)\n" + ) +} +if (args.host !== undefined) config.host = args.host +const policy = { + heartbeatSeconds: seconds("heartbeat", DEFAULT_POLICY.heartbeatSeconds, 60), + slotHeartbeatSeconds: seconds("slot-heartbeat", DEFAULT_POLICY.slotHeartbeatSeconds, 60), + minIntervalSeconds: seconds("min-interval", DEFAULT_POLICY.minIntervalSeconds, 0) +} +const watch = args.watch === undefined ? null : seconds("watch", 300, 30) +let token = "" +if (!dryRun) { + try { + token = tokenFromFile(args["token-file"]) + } catch (error) { + process.stderr.write(`seat-pusher: ${error.message}\n`) + process.exit(1) + } +} + +const listMeters = () => { + if (metersDir === null) return [] + try { + return readdirSync(metersDir) + } catch (error) { + process.stderr.write(`seat-pusher: cannot list ${metersDir}: ${error.code ?? error.name}\n`) + return [] + } +} + +const pass = async () => { + const result = await runOnce({ + metersDir, + capacityDir, + statePath, + config, + base: args.url ?? "", + token, + list: listMeters(), + policy, + force: args.force, + dryRun, + instance: args.instance ?? "", + allowNetwork: !args["no-network"], + preDispatch: args["pre-dispatch"], + ...(args["usage-dir"] === undefined ? {} : { usageDir: args["usage-dir"] }), + ...(args["demand-dir"] === undefined ? {} : { demandDir: args["demand-dir"] }) + }) + process.stdout.write(`${JSON.stringify(result)}\n`) +} + +if (watch === null) { + try { + await pass() + } catch (error) { + process.stderr.write(`seat-pusher: ${error.message}\n`) + process.exit(1) + } +} else { + for (;;) { + try { + await pass() + } catch (error) { + process.stderr.write(`seat-pusher: ${error.message}\n`) + } + await new Promise((resolve) => setTimeout(resolve, watch * 1000)) + } +} diff --git a/pkgs/substrate-apps/src/apps/pusher/bin/substrate-pusher.mjs b/pkgs/substrate-apps/src/apps/pusher/bin/substrate-pusher.mjs new file mode 100755 index 00000000..3d1ebc9a --- /dev/null +++ b/pkgs/substrate-apps/src/apps/pusher/bin/substrate-pusher.mjs @@ -0,0 +1,202 @@ +#!/usr/bin/env node +// apps/pusher/bin/substrate-pusher.mjs: the gentle capacity pusher, a plain +// long-lived process. It asks the box's capacity oracle (`seats --json`, +// seat-capacity/1) under the repo's refresh policy, converts the answer to +// seat-capacity/2 and posts it to the floor's POST /capacity/snapshots. It +// never calls a provider's usage endpoint itself (src/seatsOracle.mjs, +// src/gentle.mjs). +// +// substrate-pusher [--config PATH] [--url URL] [--token-file PATH] [options] +// substrate-pusher --dry-run --once [--seats-json FILE] +// +// Options (each overrides the config file; see apps/pusher/pusher.example.json): +// --config PATH default ~/.config/substrate/pusher.json (optional) +// --url URL the floor's base URL (config floorUrl) +// --token-file PATH the floor's bearer, read from a file, never printed +// (config tokenFile; default ~/.local/state/substrate/pusher-token) +// --seats-bin PATH the oracle (config seatsBin; default ~/.local/bin/seats) +// --seats-json FILE read the oracle's answer from a file instead of running it +// --state PATH default ~/.local/state/substrate/pusher/gentle-state.json +// --pidfile PATH default ~/.local/state/substrate/pusher.pid +// --demand-dir DIR default ~/.local/state/substrate/demand; a file touched here marks the box active +// --once one tick, then exit (the pidfile is still taken and released) +// --dry-run read and convert, print the snapshot, post nothing, keep no state +// +// Signals: SIGUSR1 asks for a pre-dispatch read (still never within 120 s of the +// last one); SIGTERM and SIGINT stop the loop and remove the pidfile. +// Exit: 0 stopped cleanly or --once done; 1 a failed --once tick; 2 usage; +// 3 another pusher holds the pidfile. + +import { execFile } from "node:child_process" +import { closeSync, existsSync, mkdirSync, openSync, readFileSync, readdirSync, renameSync, statSync, unlinkSync, writeFileSync, writeSync } from "node:fs" +import { homedir } from "node:os" +import { dirname, join } from "node:path" +import { parseArgs } from "node:util" + +import { initialGentleState, postSnapshot, tick } from "../src/gentle.mjs" +import { tokenFromFile } from "../src/token.mjs" + +const home = homedir() +const expand = (p) => (typeof p !== "string" ? p : p === "~" ? home : p.startsWith("~/") ? join(home, p.slice(2)) : p) +const stateDir = join(home, ".local", "state", "substrate") +const fail = (code, message) => { + process.stderr.write(`substrate-pusher: ${message}\n`) + process.exit(code) +} + +let args +try { + args = parseArgs({ + options: { + config: { type: "string" }, url: { type: "string" }, "token-file": { type: "string" }, + "seats-bin": { type: "string" }, "seats-json": { type: "string" }, state: { type: "string" }, + pidfile: { type: "string" }, "demand-dir": { type: "string" }, + once: { type: "boolean", default: false }, "dry-run": { type: "boolean", default: false } + }, + strict: true + }).values +} catch (error) { + fail(2, error.message) +} + +const configPath = expand(args.config ?? join(home, ".config", "substrate", "pusher.json")) +let config = {} +if (existsSync(configPath)) { + try { + config = JSON.parse(readFileSync(configPath, "utf8")) + } catch (error) { + fail(2, `cannot parse the pusher config ${configPath}: ${error.name}`) + } +} else if (args.config !== undefined) { + fail(2, `no pusher config at ${configPath}`) +} + +const dryRun = args["dry-run"] +const url = args.url ?? config.floorUrl +const tokenFile = expand(args["token-file"] ?? config.tokenFile ?? join(stateDir, "pusher-token")) +const seatsBin = expand(args["seats-bin"] ?? config.seatsBin ?? join(home, ".local", "bin", "seats")) +const seatsJson = args["seats-json"] === undefined ? null : expand(args["seats-json"]) +const statePath = expand(args.state ?? join(stateDir, "pusher", "gentle-state.json")) +const pidfile = expand(args.pidfile ?? join(stateDir, "pusher.pid")) +const demandDir = expand(args["demand-dir"] ?? join(stateDir, "demand")) +if (!dryRun && (typeof url !== "string" || !/^https?:\/\/[^/]/.test(url))) fail(2, "--url (or floorUrl in the config) must be http(s)://host") + +let token = "" +if (!dryRun) { + try { + token = tokenFromFile(tokenFile) + } catch (error) { + fail(1, error.message) + } +} + +// ---- the pidfile: one pusher per box. A pidfile whose process is gone, or is not a pusher, is stale and replaced. +const alivePusher = (pid) => { + try { + process.kill(pid, 0) + } catch { + return false + } + try { + return readFileSync(`/proc/${pid}/cmdline`, "utf8").includes("substrate-pusher") + } catch { + return true + } +} +const takePidfile = () => { + mkdirSync(dirname(pidfile), { recursive: true, mode: 0o700 }) + for (let tries = 0; tries < 2; tries++) { + try { + const fd = openSync(pidfile, "wx", 0o600) + writeSync(fd, `${process.pid}\n`) + closeSync(fd) + return + } catch (error) { + if (error.code !== "EEXIST") fail(1, `cannot write the pidfile: ${error.code}`) + const held = Number.parseInt(readFileSync(pidfile, "utf8").trim(), 10) + if (Number.isInteger(held) && held !== process.pid && alivePusher(held)) fail(3, `another pusher (pid ${held}) holds ${pidfile}`) + try { unlinkSync(pidfile) } catch { /* raced with its owner's exit */ } + } + } + fail(3, `cannot take ${pidfile}`) +} +const releasePidfile = () => { + try { + if (readFileSync(pidfile, "utf8").trim() === String(process.pid)) unlinkSync(pidfile) + } catch { /* already gone */ } +} + +// ---- state, 0600 by atomic rename +const readState = () => { + try { + return { ...initialGentleState(), ...JSON.parse(readFileSync(statePath, "utf8")) } + } catch { + return initialGentleState() + } +} +const writeState = (state) => { + mkdirSync(dirname(statePath), { recursive: true, mode: 0o700 }) + const temporary = `${statePath}.tmp-${process.pid}` + writeFileSync(temporary, `${JSON.stringify(state)}\n`, { mode: 0o600 }) + renameSync(temporary, statePath) +} + +const newestDemandMs = () => { + try { + const times = readdirSync(demandDir).map((name) => statSync(join(demandDir, name)).mtimeMs) + return times.length === 0 ? null : Math.max(...times) + } catch { + return null + } +} + +// ---- the oracle: `seats --json --no-spend` (no transcript scan); its own cache and 429 handling stand. +const runOracle = () => + seatsJson !== null + ? Promise.resolve(JSON.parse(readFileSync(seatsJson, "utf8"))) + : new Promise((resolve, reject) => { + execFile(seatsBin, ["--json", "--no-spend"], { timeout: 90_000, maxBuffer: 16 * 1024 * 1024 }, (error, stdout) => { + if (error) return reject(new Error(`the oracle failed: ${error.code ?? error.signal ?? error.name}`)) + try { + resolve(JSON.parse(stdout)) + } catch { + reject(new Error("the oracle answered no JSON")) + } + }) + }) + +const post = (snapshot) => postSnapshot({ base: url, token, snapshot }) + +let stopping = false +let preDispatch = false +let wake = null +const sleep = (ms) => new Promise((resolve) => { const t = setTimeout(resolve, ms); wake = () => { clearTimeout(t); resolve() } }) +const stop = () => { stopping = true; wake?.() } +process.on("SIGTERM", stop) +process.on("SIGINT", stop) +process.on("SIGUSR1", () => { preDispatch = true; wake?.() }) +process.on("exit", releasePidfile) + +takePidfile() +let state = dryRun ? initialGentleState() : readState() +let failedOnce = false +for (;;) { + const nowMs = Date.now() + const asked = preDispatch + preDispatch = false + const result = await tick(state, { nowMs, demandAtMs: newestDemandMs(), preDispatch: asked, config, runOracle, post, dryRun }) + state = result.state + for (const event of result.events) { + if (event.event === "wait" && !args.once) continue + process.stdout.write(`${JSON.stringify(event)}\n`) + if (event.event === "read-failed" || event.event === "push-failed") failedOnce = true + } + if (!dryRun) writeState(state) + if (args.once || stopping) break + // Wake at the next due instant, at most every 30 s (a demand marker or a signal can make a read due sooner). + const next = result.events.map((event) => (event.event === "wait" && event.next_at ? Date.parse(event.next_at) : Number.NaN)).find(Number.isFinite) + await sleep(Math.max(1_000, Math.min(30_000, Number.isFinite(next) ? next - Date.now() : 30_000))) + if (stopping) break +} +releasePidfile() +process.exit(args.once && failedOnce ? 1 : 0) diff --git a/pkgs/substrate-apps/src/apps/pusher/package.json b/pkgs/substrate-apps/src/apps/pusher/package.json new file mode 100644 index 00000000..863db745 --- /dev/null +++ b/pkgs/substrate-apps/src/apps/pusher/package.json @@ -0,0 +1,28 @@ +{ + "name": "@substrate/pusher", + "private": true, + "type": "module", + "version": "0.1.0", + "description": "The box-side gentle capacity pusher: reads the box's capacity oracle (seats --json, seat-capacity/1) under the refresh policy, converts to seat-capacity/2 and posts to the floor's POST /capacity/snapshots. The legacy seat-pusher (meters and direct usage reads) is kept for its tests only.", + "license": "UNLICENSED", + "bin": { + "substrate-pusher": "./bin/substrate-pusher.mjs" + }, + "exports": { + ".": "./src/seatCapacity.mjs", + "./*": "./src/*" + }, + "scripts": { + "test": "vitest run" + }, + "peerDependencies": { + "effect": "4.0.0-rc.112" + }, + "devDependencies": { + "fast-check": "^4.9.0", + "@substrate/factory": "workspace:*", + "@substrate/planning": "workspace:*", + "effect": "4.0.0-rc.112", + "vitest": "^3.2.4" + } +} diff --git a/pkgs/substrate-apps/src/apps/pusher/pusher.example.json b/pkgs/substrate-apps/src/apps/pusher/pusher.example.json new file mode 100644 index 00000000..a71490a4 --- /dev/null +++ b/pkgs/substrate-apps/src/apps/pusher/pusher.example.json @@ -0,0 +1,28 @@ +{ + "comment": "Example gentle-pusher config. Install as ~/.config/substrate/pusher.json (a dotfiles module will own it). Values here are placeholders, not a deployment.", + "floorUrl": "https://substrate.example.dev", + "tokenFile": "~/.local/state/substrate/pusher-token", + "seatsBin": "~/.local/bin/seats", + "host": "coordinator", + "seats": [ + "cc", + "cc2", + "codex", + "pi-qwencloud", + "halogen" + ], + "seatIds": { + "gpu-worker": "halogen" + }, + "owners": {}, + "not_dispatchable": { + "cc3": "evicted" + }, + "plans": {}, + "slots": { + "halogen": 1 + }, + "estimatedProviders": [ + "qwen" + ] +} diff --git a/pkgs/substrate-apps/src/apps/pusher/seat-capacity.example.json b/pkgs/substrate-apps/src/apps/pusher/seat-capacity.example.json new file mode 100644 index 00000000..7b536559 --- /dev/null +++ b/pkgs/substrate-apps/src/apps/pusher/seat-capacity.example.json @@ -0,0 +1,10 @@ +{ + "host": "coordinator", + "not_dispatchable": { + "cc3": "evicted", + "gpu-coordinator": "halogen is declared but not resident on the coordinator" + }, + "plans": { + "pi-qwencloud": "2026-11-07T00:00:00Z" + } +} diff --git a/pkgs/substrate-apps/src/apps/pusher/seat-capacity.usage.example.json b/pkgs/substrate-apps/src/apps/pusher/seat-capacity.usage.example.json new file mode 100644 index 00000000..a45f37d1 --- /dev/null +++ b/pkgs/substrate-apps/src/apps/pusher/seat-capacity.usage.example.json @@ -0,0 +1,9 @@ +{ + "host": "coordinator", + "usage": { + "cc": { "kind": "oauth", "credentials": "/.credentials.json", "config_dir": "" }, + "cc2": { "kind": "cache-file", "path": "" } + }, + "not_dispatchable": { "cc3": "evicted" }, + "plans": { "pi-qwencloud": "2026-11-07T00:00:00Z" } +} diff --git a/pkgs/substrate-apps/src/apps/pusher/src/gentle.mjs b/pkgs/substrate-apps/src/apps/pusher/src/gentle.mjs new file mode 100644 index 00000000..dfd3e0c5 --- /dev/null +++ b/pkgs/substrate-apps/src/apps/pusher/src/gentle.mjs @@ -0,0 +1,145 @@ +// apps/pusher/src/gentle.mjs: the gentle pusher's loop, one tick at a time. +// Pure decisions plus one `tick` that takes every effect as a parameter +// (run the oracle, post, read the clock), so the tests drive it on a fake clock. +// +// WHEN THE ORACLE IS ASKED (refreshPolicy.mjs, the policy already in the repo): +// never twice within 120 s; every 900 s when idle; every 300 s when active; +// every 180 s when active and a binding window is at or above 85 percent; +// once at each reset + 90 s; a pre-dispatch ask (SIGUSR1) when the reading is +// older than 300 s. "Active" means a demand marker younger than 300 s, or the +// last two readings differed in any utilization (someone is spending). +// WHEN THE FLOOR IS TOLD (seatCapacity.mjs decidePush): after a read, when a +// seat's reading changed, or as a heartbeat (1800 s; 600 s while a Halogen +// slot row is published); never twice within 60 s. A failed post is retried +// on the next tick after 60 s with the same snapshot, without a new read. +// A failed oracle run (non-zero exit, timeout, not seat-capacity/1) counts as +// an attempt: the 120 s spacing holds, nothing is posted, and the floor's copy +// ages to STALE on its own, which refuses admission (fail closed). + +import { REFRESH_POLICY, afterAttempt, initialRefreshState, refreshDue } from "./refreshPolicy.mjs" +import { DEFAULT_POLICY, decidePush, stateAfterPush } from "./seatCapacity.mjs" +import { snapshotFromSeatsV1 } from "./seatsOracle.mjs" + +export const PUSH_RETRY_SECONDS = 60 + +export const initialGentleState = () => ({ refresh: initialRefreshState(), push: {}, last: null, previous_utilization: null, pending: null }) + +/** Utilization per (seat, window) of a snapshot, for "is anyone spending". */ +export const utilizationKey = (snapshot) => + snapshot === null + ? null + : JSON.stringify( + snapshot.seats.map((seat) => [seat.seat, seat.windows.map((w) => [w.kind, w.model, w.utilization_pct])]) + ) + +/** The highest utilization over binding seat-wide windows of dispatchable seats; null when none is known. */ +export const bindingMaxPct = (snapshot) => { + if (snapshot === null) return null + const known = snapshot.seats + .filter((seat) => seat.dispatchable) + .flatMap((seat) => seat.windows) + .filter((w) => w.binding && w.kind !== "model_scoped" && typeof w.utilization_pct === "number") + .map((w) => w.utilization_pct) + return known.length === 0 ? null : Math.max(...known) +} + +/** Every reset instant the last snapshot states. */ +export const resetsOf = (snapshot) => + snapshot === null ? [] : snapshot.seats.flatMap((seat) => seat.windows.map((w) => w.resets_at)).filter((r) => r !== null) + +/** + * Whether the oracle is due now. + * @param input `{ nowMs, demandAtMs, preDispatch }`; demandAtMs is the newest demand marker's mtime or null. + */ +export const planRead = (state, input, policy = REFRESH_POLICY) => { + const s = state ?? initialGentleState() + const demand = typeof input.demandAtMs === "number" && input.nowMs - input.demandAtMs < policy.activeSeconds * 1000 + const moving = s.previous_utilization !== null && s.last !== null && utilizationKey(s.last) !== s.previous_utilization + return refreshDue(s.refresh, { + nowMs: input.nowMs, + active: demand || moving, + preDispatch: input.preDispatch === true, + bindingMaxPct: bindingMaxPct(s.last), + resets: resetsOf(s.last) + }, policy) +} + +/** + * One tick. Effects: + * runOracle() -> Promise the parsed `seats --json` answer (throws on failure) + * post(snapshot) -> Promise the floor's answer (throws on failure) + * @returns `{ state, events }`; events are log lines with no secret in them. + */ +export const tick = async (state, { nowMs, demandAtMs = null, preDispatch = false, config, runOracle, post, dryRun = false, pushPolicy = DEFAULT_POLICY, refreshPolicy = REFRESH_POLICY }) => { + let s = { ...initialGentleState(), ...(state ?? {}) } + const now = new Date(nowMs).toISOString() + const events = [] + // A post that failed is retried with the same snapshot before any new read. + if (s.pending !== null && !dryRun) { + if (nowMs < Date.parse(s.pending.retry_at)) return { state: s, events: [{ at: now, event: "wait", reason: "push-retry", next_at: s.pending.retry_at }] } + try { + await post(s.pending.snapshot) + s = { ...s, push: stateAfterPush(s.pending.snapshot, now), pending: null } + events.push({ at: now, event: "pushed", reason: "retry", seats: s.last?.seats.map((seat) => seat.seat) ?? [] }) + } catch (error) { + s = { ...s, pending: { ...s.pending, retry_at: new Date(nowMs + PUSH_RETRY_SECONDS * 1000).toISOString() } } + return { state: s, events: [{ at: now, event: "push-failed", error: String(error?.message ?? error).slice(0, 200) }] } + } + } + const due = planRead(s, { nowMs, demandAtMs, preDispatch }, refreshPolicy) + if (!due.due) return { state: s, events: [...events, { at: now, event: "wait", reason: due.reason, next_at: due.next_at }] } + let built + try { + const doc = await runOracle() + built = snapshotFromSeatsV1(doc, config, now) + } catch (error) { + s = { ...s, refresh: afterAttempt(s.refresh, { kind: "error" }, nowMs, null, refreshPolicy) } + return { state: s, events: [...events, { at: now, event: "read-failed", reason: due.reason, error: String(error?.message ?? error).slice(0, 200) }] } + } + s = { + ...s, + refresh: afterAttempt(s.refresh, { kind: "success" }, nowMs, null, refreshPolicy), + previous_utilization: utilizationKey(s.last), + last: built.snapshot + } + events.push({ at: now, event: "read", reason: due.reason, seats: built.snapshot.seats.map((seat) => seat.seat), skipped: built.skipped }) + const decision = decidePush(s.push, built.snapshot, nowMs, pushPolicy) + if (dryRun) return { state: s, events: [...events, { at: now, event: "dry-run", decision, snapshot: built.snapshot }] } + if (!decision.push) return { state: s, events: [...events, { at: now, event: "not-pushed", reason: decision.reason }] } + try { + const answer = await post(built.snapshot) + s = { ...s, push: stateAfterPush(built.snapshot, now) } + const results = answer?.report?.results ?? answer?.results ?? null + events.push({ at: now, event: "pushed", reason: decision.reason, changed: decision.changed, results }) + } catch (error) { + s = { ...s, pending: { snapshot: built.snapshot, retry_at: new Date(nowMs + PUSH_RETRY_SECONDS * 1000).toISOString() } } + events.push({ at: now, event: "push-failed", error: String(error?.message ?? error).slice(0, 200) }) + } + return { state: s, events } +} + +/** + * Posts a snapshot to `/capacity/snapshots` with the bearer. The token is + * sent and never echoed: an error names the status, not the header. + */ +export const postSnapshot = async ({ base, token, snapshot, fetchImpl = fetch }) => { + const url = `${base.replace(/\/+$/, "")}/capacity/snapshots` + let response + try { + response = await fetchImpl(url, { + method: "POST", + headers: { authorization: `Bearer ${token}`, "content-type": "application/json" }, + body: JSON.stringify(snapshot), + signal: AbortSignal.timeout(20_000) + }) + } catch (error) { + throw new Error(`POST /capacity/snapshots did not complete: ${error?.name ?? "Error"}`) + } + const body = await response.text() + if (response.status !== 200) throw new Error(`POST /capacity/snapshots answered ${response.status}: ${body.slice(0, 200)}`) + try { + return JSON.parse(body) + } catch { + throw new Error("POST /capacity/snapshots answered 200 with a body that is not JSON") + } +} diff --git a/pkgs/substrate-apps/src/apps/pusher/src/refreshPolicy.mjs b/pkgs/substrate-apps/src/apps/pusher/src/refreshPolicy.mjs new file mode 100644 index 00000000..f5f2fab6 --- /dev/null +++ b/pkgs/substrate-apps/src/apps/pusher/src/refreshPolicy.mjs @@ -0,0 +1,113 @@ +// apps/pusher/src/refreshPolicy.mjs: when a direct usage read is worth a +// network call. Pure: every instant is a parameter, nothing reads a clock. +// +// The values are the capacity scout's refresh policy (evals-2026-09-23 +// capacity/SCOUT.md section 5.5), one enforcer per credential: +// +// hard minimum between network calls 120 s (sustained success spacing was +// MEASURED at 123 s; faster is 429s) +// idle floor 900 s +// active cadence 300 s +// active and a binding window >= 85 % 180 s +// pre-dispatch read if the reading is older than 300 s +// reset boundary one read at resets_at + 90 s +// 429 one attempt, no in-tick retry; next +// allowed = now + max(Retry-After, +// min(120 s x 2^k, 960 s)); k resets on success +// auth failure no call until the credentials file's +// mtime changes + +export const REFRESH_POLICY = Object.freeze({ + minSpacingSeconds: 120, + idleSeconds: 900, + activeSeconds: 300, + nearCapSeconds: 180, + nearCapPct: 85, + preDispatchAgeSeconds: 300, + resetGraceSeconds: 90, + backoffBaseSeconds: 120, + backoffMaxSeconds: 960 +}) + +/** The state a fresh credential starts from. */ +export const initialRefreshState = () => ({ + last_attempt_at: null, + last_success_at: null, + backoff_k: 0, + next_allowed_at: null, + auth_failed_mtime_ms: null +}) + +const ms = (iso) => { + if (typeof iso !== "string") return null + const value = Date.parse(iso) + return Number.isFinite(value) ? value : null +} + +const iso = (value) => new Date(value).toISOString() + +/** + * Decides whether a network read is due. + * + * @param state the credential's refresh state (initialRefreshState() shape). + * @param input `{ nowMs, active, preDispatch, bindingMaxPct, resets, credentialsMtimeMs }`: + * `resets` are the binding windows' `resets_at` instants from the last reading; + * `bindingMaxPct` the highest binding utilization, or null when unknown. + * @returns `{ due, reason, next_at }`, `next_at` an ISO instant or null. + */ +export const refreshDue = (state, input, policy = REFRESH_POLICY) => { + const s = { ...initialRefreshState(), ...(state ?? {}) } + const now = input.nowMs + if (s.auth_failed_mtime_ms !== null && input.credentialsMtimeMs === s.auth_failed_mtime_ms) { + return { due: false, reason: "auth-failed", next_at: null } + } + const allowed = ms(s.next_allowed_at) + if (allowed !== null && now < allowed) return { due: false, reason: "backoff", next_at: iso(allowed) } + const attempted = ms(s.last_attempt_at) + const earliest = attempted === null ? null : attempted + policy.minSpacingSeconds * 1000 + if (earliest !== null && now < earliest) return { due: false, reason: "min-spacing", next_at: iso(earliest) } + const succeeded = ms(s.last_success_at) + if (succeeded === null) return { due: true, reason: "first-read", next_at: null } + // A reset boundary read turns a PROJECTED window into a MEASURED one. + const boundaries = (input.resets ?? []) + .map(ms) + .filter((value) => value !== null) + .map((value) => value + policy.resetGraceSeconds * 1000) + if (boundaries.some((at) => at > succeeded && at <= now)) { + return { due: true, reason: "reset-boundary", next_at: null } + } + const ageSeconds = (now - succeeded) / 1000 + if (input.preDispatch === true && ageSeconds > policy.preDispatchAgeSeconds) { + return { due: true, reason: "pre-dispatch", next_at: null } + } + const nearCap = typeof input.bindingMaxPct === "number" && input.bindingMaxPct >= policy.nearCapPct + const cadence = input.active === true ? (nearCap ? policy.nearCapSeconds : policy.activeSeconds) : policy.idleSeconds + if (ageSeconds >= cadence) return { due: true, reason: input.active === true ? "active" : "idle", next_at: null } + const candidates = [succeeded + cadence * 1000, ...boundaries.filter((at) => at > now)] + const next = Math.max(Math.min(...candidates), earliest ?? 0) + return { due: false, reason: "fresh", next_at: iso(next) } +} + +/** + * The state after one attempt. + * + * @param outcome `{ kind: "success" }`, `{ kind: "rate-limited", retryAfterSeconds }`, + * `{ kind: "auth-failed" }` or `{ kind: "error" }`. + */ +export const afterAttempt = (state, outcome, nowMs, credentialsMtimeMs, policy = REFRESH_POLICY) => { + const s = { ...initialRefreshState(), ...(state ?? {}), last_attempt_at: iso(nowMs) } + if (outcome.kind === "success") { + return { ...s, last_success_at: iso(nowMs), backoff_k: 0, next_allowed_at: null, auth_failed_mtime_ms: null } + } + if (outcome.kind === "rate-limited") { + const k = Number.isInteger(s.backoff_k) && s.backoff_k >= 0 ? s.backoff_k : 0 + const exponential = Math.min(policy.backoffBaseSeconds * 2 ** k, policy.backoffMaxSeconds) + const retryAfter = + typeof outcome.retryAfterSeconds === "number" && Number.isFinite(outcome.retryAfterSeconds) + ? outcome.retryAfterSeconds + : 0 + return { ...s, backoff_k: k + 1, next_allowed_at: iso(nowMs + Math.max(retryAfter, exponential) * 1000) } + } + if (outcome.kind === "auth-failed") return { ...s, auth_failed_mtime_ms: credentialsMtimeMs ?? -1 } + return s +} diff --git a/pkgs/substrate-apps/src/apps/pusher/src/seatCapacity.mjs b/pkgs/substrate-apps/src/apps/pusher/src/seatCapacity.mjs new file mode 100644 index 00000000..2d7862a8 --- /dev/null +++ b/pkgs/substrate-apps/src/apps/pusher/src/seatCapacity.mjs @@ -0,0 +1,593 @@ +// apps/uplink/src/seatCapacity.mjs: EVAL-CAPACITY. The seat-capacity pusher's +// library: build a `seat-capacity/2` snapshot from the feeders' files, decide +// whether it is worth sending, and send it outbound. +// +// THREE CLOCKS, KEPT APART (SCOUT.md, evals-2026-09-23/capacity). The feeder +// restamps meter rows every 30 s for the kernel; the provider is read far less +// often; the lake projects from the last reading. This module sits between the +// first two and the third: it reads LOCAL FILES ONLY and never calls a +// provider, so running it often costs nothing upstream. What it carries is the +// instant the PROVIDER answered (`observed_at`), never the restamp. +// +// SOURCES, IN ORDER OF AUTHORITY: +// 1. `/seats.json`, when the feeder writes the snapshot itself +// (the proposed feeder change, not yet applied): carried as it is. +// 2. The meters directory: `.json` rows, and for a Claude seat the raw +// usage cache `.window-cache-.json` beside it, which is the only +// place the model-scoped rows (`weekly_scoped`, e.g. Fable) appear. Only +// the `limits[]` rows and `observed_at` are read from the cache; nothing +// else in it is copied anywhere. +// +// WHAT IT REFUSES TO INVENT. A cell the file does not state stays null +// (UNKNOWN, never headroom); a meter this module cannot classify is skipped +// with a named reason rather than guessed into a provider. + +import { existsSync, mkdirSync, readFileSync, renameSync, writeFileSync } from "node:fs" +import { homedir, hostname } from "node:os" +import { dirname, join } from "node:path" + +import { limitsComplete, readSeatUsage, windowsFromLimits, windowsFromUsage } from "./usageSource.mjs" + +const SCHEMA_VERSION = "seat-capacity/2" + +const MINUTES = { five_hour: 300, seven_day: 10080, model_scoped: 10080 } + +/** The pusher's own state; nothing under a tally path is a default any more. */ +export const defaultPusherDir = () => join(homedir(), ".local", "state", "substrate", "pusher") +export const defaultConfigPath = () => join(homedir(), ".config", "substrate", "seat-capacity.json") + +/** + * Refresh policy defaults, in seconds. + * + * `slotHeartbeatSeconds` is the beat for a dispatchable Halogen slot row. Its + * `observed_at` is the kernel's own observation, and the lake grades a reading + * STALE after 1200 s (MEASURED_MAX_AGE_S in packages/planning), so a slot row + * re-sent only every 1800 s would be refused for half of every beat. + */ +export const DEFAULT_POLICY = { heartbeatSeconds: 1800, slotHeartbeatSeconds: 600, minIntervalSeconds: 60 } + +/** + * Seats Tom has ruled out of dispatch. Built in, so a missing or partial config + * can never publish them as dispatchable; a config may add rulings and reword + * these, never lift them. cc3: evicted. gpu-coordinator: Halogen is declared on + * the coordinator but never resident there (ruling in ~/today/CLAUDE.md). + */ +const RULED_NOT_DISPATCHABLE = Object.freeze({ + cc3: "evicted", + "gpu-coordinator": "halogen is declared but not resident on the coordinator" +}) + +const ZONED = /^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}(:\d{2}(\.\d+)?)?(Z|[+-]\d{2}:\d{2})$/ + +/** An instant the lake's schema will accept, or null. */ +const instant = (value) => + typeof value === "string" && ZONED.test(value) && Number.isFinite(Date.parse(value)) ? value : null + +const percent = (value) => (typeof value === "number" && Number.isFinite(value) && value >= 0 ? value : null) + +const readJson = (path) => { + if (!existsSync(path)) return { missing: true } + try { + return { value: JSON.parse(readFileSync(path, "utf8")) } + } catch (error) { + return { error: `${error.name}` } + } +} + +/** + * Reads the pusher's config; a missing file is the defaults, a broken one is an + * error. The built-in rulings always apply (`RULED_NOT_DISPATCHABLE`): a config + * entry that is not a reason string is ignored, so it cannot lift one. + */ +export const readConfig = (path) => { + const read = path === null ? { missing: true } : readJson(path) + if (read.error) throw new Error(`cannot parse the seat-capacity config ${path}: ${read.error}`) + const stated = read.value ?? {} + const statedRulings = + stated.not_dispatchable !== null && typeof stated.not_dispatchable === "object" + ? Object.fromEntries( + Object.entries(stated.not_dispatchable).filter(([, reason]) => typeof reason === "string" && reason !== "") + ) + : {} + return { + present: read.value !== undefined, + host: typeof stated.host === "string" ? stated.host : null, + seats: Array.isArray(stated.seats) ? stated.seats.filter((seat) => typeof seat === "string") : null, + notDispatchable: { ...RULED_NOT_DISPATCHABLE, ...statedRulings }, + plans: stated.plans !== null && typeof stated.plans === "object" ? stated.plans : {}, + // seat -> { kind: "oauth", credentials, token_pointer?, config_dir? } | { kind: "cache-file", path } + usage: stated.usage !== null && typeof stated.usage === "object" ? stated.usage : {}, + owners: stated.owners !== null && typeof stated.owners === "object" ? stated.owners : {}, + providers: stated.providers !== null && typeof stated.providers === "object" ? stated.providers : {} + } +} + +/** + * A reason as it leaves the box: local absolute paths are replaced by ``. + * The lake is on the public internet (behind a bearer); where a file lives on + * the coordinator is none of its business. + */ +const scrubReason = (value) => + typeof value === "string" ? value.replace(/(?:~|\/(?:home|root|Users|nix|run|tmp|var|etc))\/[^\s,;:'"`)]+/g, "") : null + +const AUTH_FAILURE = /token expired|unauthori[sz]ed|\b401\b|invalid[_ ]grant|credentials/i + +const dispatchOf = (seat, owner, staleReason, config) => { + const ruled = config.notDispatchable[seat] + if (typeof ruled === "string") return { dispatchable: false, dispatchable_reason: ruled } + if (owner !== "tom") return { dispatchable: false, dispatchable_reason: "third-party" } + if (typeof staleReason === "string" && AUTH_FAILURE.test(staleReason)) { + return { dispatchable: false, dispatchable_reason: "auth-failed" } + } + return { dispatchable: true, dispatchable_reason: null } +} + +const planOf = (seat, config) => { + const expires = instant(config.plans[seat]) + return expires === null ? null : { expires_at: expires } +} + +const later = (left, right) => { + if (left === null) return right + if (right === null) return left + return Date.parse(right) > Date.parse(left) ? right : left +} + +/** The pusher's one limits[] mapping (usageSource.mjs windowsFromLimits): an unplaceable row is a binding UNKNOWN seven_day window. */ +const claudeWindowsFromLimits = (limits) => windowsFromLimits(limits).windows + +const claudeWindowsFromMeter = (window) => { + if (window === null || typeof window !== "object") return [] + return [ + ["five_hour", window.primary], + ["seven_day", window.secondary] + ].flatMap(([kind, entry]) => + entry === null || typeof entry !== "object" + ? [] + : [ + { + kind, + model: null, + binding: true, + minutes: MINUTES[kind], + utilization_pct: percent(entry.utilization_pct), + resets_at: instant(entry.resets_at), + severity: null, + grade: "MEASURED" + } + ] + ) +} + +/** + * The model-scoped rows a cache last showed, carried forward as UNKNOWN when the + * reading itself comes from the meter row: the model limit exists and was not + * read with this reading, so its utilization and severity are not known. The + * reset instant is kept (a passed one projects forward at the lake). The lake + * refuses a job for that model `window-unknown`, never admits it as headroom. + */ +const carriedModelWindows = (limits) => + Array.isArray(limits) + ? claudeWindowsFromLimits(limits) + // An unplaceable row (UNKNOWN seven_day) is carried too: dropping it read as headroom. + .filter((window) => window.kind === "model_scoped" || window.grade === "UNKNOWN") + .map((window) => ({ ...window, utilization_pct: null, severity: null, grade: "UNKNOWN" })) + : [] + +const claudeSeat = (seat, meter, cache, config) => { + const meterObserved = later(instant(meter.reading_observed_at), instant(meter.source?.source_observed_at)) + const cacheObserved = instant(cache?.observed_at) + const observed = later(meterObserved, cacheObserved) + if (observed === null) return { skip: "the meter states no reading instant" } + const limits = cache?.usage?.limits + // The cache's limits[] are the reading only when the cache IS the latest read. + // Otherwise (cache missing, torn mid-write, or behind the meter row) the + // meter row gives the seat-wide windows and the model limits stay unread. + const fromCache = + Array.isArray(limits) && cacheObserved !== null && Date.parse(cacheObserved) === Date.parse(observed) + const graded = meter.grade === "MEASURED" || meter.grade === "STALE-MEASURED" + const staleReason = scrubReason(meter.stale_reason) + return { + seat: { + seat, + provider: "claude", + owner: meter.owner === "tom" ? "tom" : "third-party", + ...dispatchOf(seat, meter.owner, staleReason, config), + plan: planOf(seat, config), + slots: null, + windows: fromCache + ? claudeWindowsFromLimits(limits) + : [...claudeWindowsFromMeter(meter.window), ...carriedModelWindows(limits)], + observed_at: observed, + source: { + kind: "oauth-usage-endpoint", + detail: fromCache ? "usage cache limits[]" : "meter row window" + }, + // The reading's age is the lake's to judge from observed_at: a row the + // feeder marked STALE-MEASURED is a MEASURED reading that has aged. + grade: graded ? "MEASURED" : "UNKNOWN", + stale_reason: staleReason, + // Only limits[] read with this very reading, every row placed, lists every model limit. + model_windows_complete: fromCache && windowsFromLimits(limits).unmapped === 0 + } + } +} + +const codexSeat = (seat, meter, config) => { + const observed = instant(meter.observed_at) + if (observed === null) return { skip: "the meter states no observed_at" } + const window = meter.window + const windows = + window !== null && typeof window === "object" && window.minutes === 10080 + ? [ + { + kind: "seven_day", + model: null, + binding: true, + minutes: 10080, + utilization_pct: percent(window.utilization_pct), + resets_at: instant(window.resets_at), + severity: null, + grade: "ESTIMATED" + } + ] + : [] + return { + seat: { + seat, + provider: "codex", + owner: meter.owner === "tom" ? "tom" : "third-party", + ...dispatchOf(seat, meter.owner, null, config), + plan: planOf(seat, config), + slots: null, + windows, + observed_at: observed, + // The meter row states no provider or rollout-event instant; observed_at + // is the feeder's read of a local rollout. Said so, rather than implied. + source: { kind: "codex-rollout-rate-limits", detail: "observed_at is the feeder's read instant" }, + // Read from a local rollout, not from the provider at a known instant. + grade: "ESTIMATED", + stale_reason: null + } + } +} + +const qwenSeat = (seat, meter, config) => { + const observed = instant(meter.observed_at) + if (observed === null) return { skip: "the meter states no observed_at" } + const window = meter.window + const windows = + window !== null && typeof window === "object" && instant(window.resets_at) !== null + ? [ + { + kind: "seven_day", + model: null, + binding: true, + minutes: 10080, + utilization_pct: null, + resets_at: instant(window.resets_at), + severity: null, + grade: "UNKNOWN" + } + ] + : [] + return { + seat: { + seat, + provider: "qwen", + owner: meter.owner === "tom" ? "tom" : "third-party", + ...dispatchOf(seat, meter.owner, null, config), + plan: planOf(seat, config), + slots: null, + windows, + observed_at: observed, + source: { kind: "hold-record", detail: "observed_at is the feeder's read instant" }, + grade: "UNKNOWN", + stale_reason: scrubReason(meter.window_reason) + } + } +} + +const halogenSeat = (seat, meter, config) => { + const observed = instant(meter.observed_at) + if (observed === null) return { skip: "the meter states no observed_at" } + const capacity = Number.isInteger(meter.capacity) && meter.capacity >= 0 ? meter.capacity : null + const holders = Number.isInteger(meter.holders) && meter.holders >= 0 ? meter.holders : null + return { + seat: { + seat, + provider: "halogen", + owner: "tom", + ...dispatchOf(seat, "tom", null, config), + plan: planOf(seat, config), + slots: capacity === null || holders === null ? null : { capacity, holders }, + windows: [], + observed_at: observed, + source: { kind: "kernel-row", detail: null }, + grade: meter.running_grade === "MEASURED" ? "MEASURED" : "UNKNOWN", + stale_reason: null + } + } +} + +/** + * A seat read through its own usage source (usageSource.mjs), no meter row + * needed. `read` is `{ reading, error }` from readSeatUsage. + */ +const seatFromUsage = (seat, read, sourceKind, config) => { + const reading = read.reading + if (reading === null) return { skip: read.error ?? "no usage reading yet" } + const owner = config.owners[seat] === "third-party" ? "third-party" : "tom" + const staleReason = scrubReason(read.error) + return { + seat: { + seat, + provider: typeof config.providers[seat] === "string" ? config.providers[seat] : "claude", + owner, + ...dispatchOf(seat, owner, staleReason, config), + plan: planOf(seat, config), + slots: null, + windows: windowsFromUsage(reading.usage), + // The reading lists every model limit only when limits[] is the reading; + // without the claim, admission refuses a model job with no window. + // An unplaceable row withdraws the claim (successor review r4). + model_windows_complete: limitsComplete(reading.usage), + observed_at: reading.observed_at, + source: { + kind: "oauth-usage-endpoint", + detail: sourceKind === "oauth" ? "direct read" : "usage cache file" + }, + grade: "MEASURED", + stale_reason: staleReason + } + } +} + +/** Classifies one meter file into a seat, or a named skip. */ +const seatFromMeter = (seat, meter, cache, config) => { + if (meter === null || typeof meter !== "object") return { skip: "the meter is not an object" } + const kind = meter.source?.kind + if (kind === "oauth-usage-endpoint") return claudeSeat(seat, meter, cache, config) + if (kind === "codex-rollout-rate-limits") return codexSeat(seat, meter, config) + if (typeof meter.source?.hold_record === "string" || seat.startsWith("pi-qwen")) { + return qwenSeat(seat, meter, config) + } + if (seat.startsWith("gpu-") && meter.window?.kind === "none") return halogenSeat(seat, meter, config) + return { skip: "not a compute seat this pusher knows" } +} + +const METER_FILE = /^([a-z0-9][a-z0-9-]{0,62})\.json$/ + +/** + * The feeder's own snapshot, held to the same local rules as one built here: + * the config's seat filter, the rulings (which only ever take dispatch away), + * and the path scrub on every reason that leaves the box. + */ +const governWritten = (snapshot, config) => ({ + ...snapshot, + seats: snapshot.seats + .filter((seat) => config.seats === null || config.seats.includes(seat?.seat)) + .map((seat) => { + if (seat === null || typeof seat !== "object") return seat + const staleReason = scrubReason(seat.stale_reason) + const ruled = dispatchOf(seat.seat, seat.owner, staleReason, config) + const dispatch = + seat.dispatchable === true && ruled.dispatchable + ? { dispatchable: true, dispatchable_reason: null } + : { + dispatchable: false, + dispatchable_reason: ruled.dispatchable + ? scrubReason(seat.dispatchable_reason) + : ruled.dispatchable_reason + } + return { ...seat, ...dispatch, stale_reason: staleReason } + }) +}) + +/** + * Builds the snapshot the lake ingests. + * + * @returns `{ snapshot, skipped }`: skipped names every meter file left out and why. + */ +export const buildSnapshot = ({ metersDir = null, capacityDir = null, config, now, list, usage = {} }) => { + const host = config.host ?? hostname().split(".")[0] + if (capacityDir !== null) { + const written = readJson(join(capacityDir, "seats.json")) + if (written.value?.schema_version === SCHEMA_VERSION && Array.isArray(written.value.seats)) { + return { snapshot: governWritten(written.value, config), skipped: [], source: "seats.json" } + } + } + const wanted = (name) => config.seats === null || config.seats.includes(name) + const seats = [] + const skipped = [] + // Seats with their own usage source come first and win over a meter row. + const direct = Object.keys(usage).filter(wanted).sort() + for (const name of direct) { + const built = seatFromUsage(name, usage[name], config.usage[name]?.kind ?? null, config) + if (built.skip !== undefined) skipped.push({ seat: name, reason: scrubReason(built.skip) }) + else seats.push(built.seat) + } + const names = (metersDir === null ? [] : (list ?? [])) + .map((name) => METER_FILE.exec(name)?.[1]) + .filter((name) => name !== undefined) + .filter((name) => wanted(name) && !direct.includes(name)) + .sort() + for (const name of names) { + const meter = readJson(join(metersDir, `${name}.json`)) + if (meter.value === undefined) { + skipped.push({ seat: name, reason: meter.error ? `unreadable: ${meter.error}` : "missing" }) + continue + } + const cache = readJson(join(metersDir, `.window-cache-${name}.json`)).value ?? null + const built = seatFromMeter(name, meter.value, cache, config) + if (built.skip !== undefined) skipped.push({ seat: name, reason: built.skip }) + else seats.push(built.seat) + } + return { + snapshot: { schema_version: SCHEMA_VERSION, host, published_at: now, seats }, + skipped, + source: direct.length > 0 ? (metersDir === null ? "usage" : "usage+meters") : "meters" + } +} + +/** Key-sorted JSON. */ +const canonical = (value) => + JSON.stringify(value, (_key, entry) => + entry !== null && typeof entry === "object" && !Array.isArray(entry) + ? Object.fromEntries(Object.entries(entry).sort(([left], [right]) => (left < right ? -1 : left > right ? 1 : 0))) + : entry + ) + +/** + * Providers whose `observed_at` is a provider's answer. For the others it is a + * feeder or kernel restamp (codex: the feeder's re-read of a local rollout; + * qwen: a hold record; halogen: the kernel's slot row), which moves every tick + * without the reading changing, so it is left out of the fingerprint and the + * heartbeats carry it instead. + */ +const PROVIDER_INSTANT = new Set(["claude"]) + +/** What decides a push: the reading's identity, not the instant it was restamped. */ +export const fingerprint = (seat) => + canonical({ + observed_at: PROVIDER_INSTANT.has(seat.provider) ? Date.parse(seat.observed_at) : null, + grade: seat.grade, + windows: seat.windows, + dispatchable: seat.dispatchable, + dispatchable_reason: seat.dispatchable_reason, + slots: seat.slots, + plan: seat.plan + }) + +/** + * The refresh policy: push when any seat's reading changed, and at least every + * `heartbeatSeconds` as a liveness beat (every `slotHeartbeatSeconds` while a + * dispatchable Halogen slot row is published); never twice within + * `minIntervalSeconds`. + */ +export const decidePush = (state, snapshot, nowMs, policy = DEFAULT_POLICY) => { + const last = state?.last_push_at === undefined ? null : Date.parse(state.last_push_at) + const since = last === null || !Number.isFinite(last) ? null : (nowMs - last) / 1000 + if (since !== null && since >= 0 && since < policy.minIntervalSeconds) { + return { push: false, reason: "min-interval", changed: [] } + } + const held = state?.fingerprints ?? {} + const current = Object.fromEntries(snapshot.seats.map((seat) => [seat.seat, fingerprint(seat)])) + const changed = Object.keys(current).filter((seat) => held[seat] !== current[seat]) + const removed = Object.keys(held).filter((seat) => !(seat in current)) + if (changed.length > 0 || removed.length > 0) { + return { push: true, reason: "changed", changed: [...changed, ...removed].sort() } + } + if (since === null || since < 0 || since >= policy.heartbeatSeconds) { + return { push: true, reason: "heartbeat", changed: [] } + } + const slotBeat = policy.slotHeartbeatSeconds ?? DEFAULT_POLICY.slotHeartbeatSeconds + const liveSlots = snapshot.seats.some((seat) => seat.provider === "halogen" && seat.dispatchable) + if (liveSlots && since >= slotBeat) { + return { push: true, reason: "heartbeat", changed: [] } + } + return { push: false, reason: "unchanged", changed: [] } +} + +/** The state to keep after a successful push. */ +export const stateAfterPush = (snapshot, now) => ({ + last_push_at: now, + fingerprints: Object.fromEntries(snapshot.seats.map((seat) => [seat.seat, fingerprint(seat)])) +}) + +export const readState = (path) => { + const read = readJson(path) + return read.value !== null && typeof read.value === "object" ? read.value : {} +} + +/** Atomic, 0600: a partial state file must never make the next run skip a push. */ +const writeState = (path, state) => { + mkdirSync(dirname(path), { recursive: true, mode: 0o700 }) + const temporary = `${path}.tmp-${process.pid}` + writeFileSync(temporary, `${JSON.stringify(state, null, 2)}\n`, { mode: 0o600 }) + renameSync(temporary, path) +} + +class PushError extends Error {} + +/** + * Sends the snapshot to `POST /capacity/seats`. The token is sent and + * never echoed: an error names the status and the route, not the header. + */ +const pushSnapshot = async ({ base, token, snapshot, fetchImpl = fetch, instance = "" }) => { + const url = `${base.replace(/\/+$/, "")}/capacity/seats` + let response + try { + response = await fetchImpl(url, { + method: "POST", + headers: { + authorization: `Bearer ${token}`, + "content-type": "application/json", + ...(instance === "" ? {} : { "x-conwip-factory-instance": instance }) + }, + body: JSON.stringify(snapshot) + }) + } catch (error) { + throw new PushError(`POST /capacity/seats did not complete: ${error?.name ?? "Error"}`) + } + const text = await response.text() + if (response.status !== 200) { + throw new PushError(`POST /capacity/seats answered ${response.status}: ${text.slice(0, 200)}`) + } + try { + return JSON.parse(text) + } catch { + throw new PushError("POST /capacity/seats answered 200 with a body that is not JSON") + } +} + +/** + * One pass: build, decide, push when due, and keep the state only after the + * lake has answered 200. + */ +export const runOnce = async ({ + metersDir, + capacityDir, + statePath, + config, + base, + token, + now = () => new Date().toISOString(), + list, + fetchImpl = fetch, + policy = DEFAULT_POLICY, + force = false, + dryRun = false, + instance = "", + allowNetwork = true, + preDispatch = false, + usageDir, + demandDir +}) => { + const at = now() + // Each seat's usage source first. A dry run never calls the network. + const usage = {} + for (const seat of Object.keys(config.usage ?? {}).sort()) { + usage[seat] = await readSeatUsage({ + seat, + source: config.usage[seat], + nowMs: Date.parse(at), + allowNetwork: allowNetwork && !dryRun, + preDispatch, + fetchImpl, + ...(usageDir === undefined ? {} : { usageDir }), + ...(demandDir === undefined ? {} : { demandDir }) + }) + } + const { snapshot, skipped, source } = buildSnapshot({ metersDir, capacityDir, config, now: at, list, usage }) + const state = readState(statePath) + const decision = force + ? { push: true, reason: "forced", changed: [] } + : decidePush(state, snapshot, Date.parse(at), policy) + const summary = { at, source, seats: snapshot.seats.map((seat) => seat.seat), skipped, decision } + if (dryRun) return { ...summary, pushed: false, snapshot } + if (!decision.push) return { ...summary, pushed: false } + const answer = await pushSnapshot({ base, token, snapshot, fetchImpl, instance }) + writeState(statePath, stateAfterPush(snapshot, at)) + return { ...summary, pushed: true, results: answer.results ?? null } +} diff --git a/pkgs/substrate-apps/src/apps/pusher/src/seatsOracle.mjs b/pkgs/substrate-apps/src/apps/pusher/src/seatsOracle.mjs new file mode 100644 index 00000000..7194ad8c --- /dev/null +++ b/pkgs/substrate-apps/src/apps/pusher/src/seatsOracle.mjs @@ -0,0 +1,201 @@ +// apps/pusher/src/seatsOracle.mjs: the gentle pusher's one source. It reads the +// box's capacity oracle (`seats --json`, schema seat-capacity/1) and converts +// the answer to seat-capacity/2 for the floor. Pure: no clock, no file, no +// network; the process (bin/substrate-pusher.mjs) runs the oracle and posts. +// +// The pusher never calls a provider's usage endpoint itself. The oracle owns +// its own cache and its own 429 handling; the pusher only decides how often it +// asks the oracle (gentle.mjs, the repo's refresh policy). +// +// WHAT THE CONVERSION REFUSES TO INVENT (fail closed, as the /2 schema says): +// - a window the /2 kinds cannot place (a length other than 300 or 10080 +// minutes, a scope naming no model) becomes a binding seven_day window +// graded UNKNOWN with its reset kept, and the seat withdraws +// model_windows_complete; +// - a null or missing percentage stays null (UNKNOWN, never headroom); +// - a seat the oracle did not report is named in `skipped`, never carried +// forward; the floor's copy then ages to STALE and refuses on its own; +// - local paths in any reason are replaced by `` before they leave the +// box, and no oracle field other than the few named here is copied. + +import { hostname } from "node:os" + +export const SOURCE_SCHEMA = "seat-capacity/1" +export const TARGET_SCHEMA = "seat-capacity/2" + +/** The seats the pusher publishes unless its config says otherwise (the runs-on seat names). */ +export const DEFAULT_SEATS = Object.freeze(["cc", "cc2", "codex", "pi-qwencloud", "halogen"]) +/** Oracle seat ids onto runs-on seat names: the oracle calls the worker's Halogen server `gpu-worker`. */ +export const DEFAULT_SEAT_IDS = Object.freeze({ "gpu-worker": "halogen" }) +/** Providers whose percentages are the oracle's estimate, not a provider answer (Qwen Cloud: a credit count from local logs). */ +export const DEFAULT_ESTIMATED_PROVIDERS = Object.freeze(["qwen"]) +/** Seats ruled out of dispatch whatever a config says (a config may add rulings, never lift these). */ +export const RULED_NOT_DISPATCHABLE = Object.freeze({ + cc3: "evicted", + "gpu-coordinator": "halogen is declared but not resident on the coordinator" +}) + +export class SeatsFormatError extends Error {} + +const ZONED = /^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}(:\d{2}(\.\d+)?)?(Z|[+-]\d{2}:\d{2})$/ +const SEAT_ID = /^[a-z0-9][a-z0-9-]{0,62}$/ +const PROVIDER = /^[a-z][a-z0-9-]{0,31}$/ + +export const instant = (value) => + typeof value === "string" && ZONED.test(value) && Number.isFinite(Date.parse(value)) ? value : null +const percent = (value) => (typeof value === "number" && Number.isFinite(value) && value >= 0 ? value : null) +const text = (value) => (typeof value === "string" && value.trim() !== "" ? value.trim() : null) + +/** A reason as it leaves the box: local absolute paths become ``. */ +export const scrub = (value) => + typeof value === "string" + ? value.replace(/(?:~|\/(?:home|root|Users|nix|run|tmp|var|etc|mnt|srv))\/[^\s,;:'"`)]+/g, "").slice(0, 300) + : null + +/** + * The conversion's settings, from the pusher config (defaults filled in). + * `owners` overrides the oracle's owner field per seat (the oracle's own seat + * table can lag a ruling); `slots` is the slot count of a slot-counted seat. + */ +export const conversionSettings = (config = {}) => { + const object = (value) => (value !== null && typeof value === "object" && !Array.isArray(value) ? value : {}) + const stated = Object.fromEntries( + Object.entries(object(config.not_dispatchable)).filter(([, reason]) => typeof reason === "string" && reason !== "") + ) + return { + host: typeof config.host === "string" && config.host !== "" ? config.host : null, + seats: Array.isArray(config.seats) ? config.seats.filter((seat) => typeof seat === "string") : [...DEFAULT_SEATS], + seatIds: { ...DEFAULT_SEAT_IDS, ...object(config.seatIds) }, + owners: object(config.owners), + plans: object(config.plans), + slots: object(config.slots), + estimatedProviders: Array.isArray(config.estimatedProviders) ? config.estimatedProviders : [...DEFAULT_ESTIMATED_PROVIDERS], + notDispatchable: { ...RULED_NOT_DISPATCHABLE, ...stated } + } +} + +/** When the provider (or the local artefact) answered, as best the oracle states it. */ +const observedOf = (seat, generatedMs) => { + const stated = instant(seat?.source?.observed_at) + if (stated !== null) return new Date(Date.parse(stated)).toISOString() + const age = seat?.source?.reading_age_seconds + const ageS = typeof age === "number" && Number.isFinite(age) && age >= 0 ? age : 0 + return new Date(generatedMs - ageS * 1000).toISOString() +} + +/** One oracle window as a /2 window, or an UNKNOWN binding weekly window when it cannot be placed. */ +const windowOf = (entry, grade) => { + const minutes = entry?.minutes + const scope = text(entry?.scope) + const base = { + utilization_pct: percent(entry?.used_pct), + resets_at: instant(entry?.resets_at), + severity: text(entry?.severity), + grade + } + if (scope !== null && minutes === 10080) return { placed: true, window: { kind: "model_scoped", model: scope, binding: true, minutes: 10080, ...base } } + if (scope === null && minutes === 300) return { placed: true, window: { kind: "five_hour", model: null, binding: true, minutes: 300, ...base } } + if (scope === null && minutes === 10080) return { placed: true, window: { kind: "seven_day", model: null, binding: true, minutes: 10080, ...base } } + return { + placed: false, + window: { kind: "seven_day", model: null, binding: true, minutes: 10080, utilization_pct: null, resets_at: instant(entry?.resets_at), severity: null, grade: "UNKNOWN" } + } +} + +/** The /2 grade of an oracle seat. CACHED is a provider answer that has aged: MEASURED, dated by its age. */ +const gradeOf = (seat, provider, settings) => { + if (seat.grade === "UNKNOWN" || seat.grade === undefined) return "UNKNOWN" + if (settings.estimatedProviders.includes(provider)) return "ESTIMATED" + if (seat.grade === "MEASURED" || seat.grade === "CACHED") return "MEASURED" + return "UNKNOWN" +} + +const dispatchOf = (id, owner, seat, provider, settings) => { + const ruled = settings.notDispatchable[id] + if (typeof ruled === "string") return { dispatchable: false, dispatchable_reason: ruled } + if (owner !== "tom") return { dispatchable: false, dispatchable_reason: "third-party" } + if (seat.state === "unauth") return { dispatchable: false, dispatchable_reason: "auth-failed" } + if (seat.state === "n/a") return { dispatchable: false, dispatchable_reason: scrub(text(seat.detail) ?? "not applicable") } + if (provider === "halogen" && seat.state !== "open") return { dispatchable: false, dispatchable_reason: `down: ${scrub(text(seat.detail) ?? seat.state ?? "unknown")}` } + return { dispatchable: true, dispatchable_reason: null } +} + +/** + * Converts one `seats --json` document to a seat-capacity/2 snapshot. + * + * @param doc the parsed oracle answer (schema seat-capacity/1). + * @param config the pusher config (see conversionSettings). + * @param now the publishing instant (ISO), the pusher's clock. + * @returns `{ snapshot, skipped }`; skipped names every configured seat left out and why. + * @throws SeatsFormatError when the document is not seat-capacity/1. + */ +export const snapshotFromSeatsV1 = (doc, config, now) => { + if (doc === null || typeof doc !== "object" || doc.schema_version !== SOURCE_SCHEMA || !Array.isArray(doc.seats)) { + throw new SeatsFormatError(`the oracle's answer is not ${SOURCE_SCHEMA}`) + } + if (instant(now) === null) throw new RangeError(`now is not an instant: ${now}`) + const settings = conversionSettings(config) + const generated = instant(doc.generated_at) + const generatedMs = generated === null ? Date.parse(now) : Math.min(Date.parse(generated), Date.parse(now)) + const seats = [] + const skipped = [] + const seen = new Set() + for (const raw of doc.seats) { + if (raw === null || typeof raw !== "object" || typeof raw.id !== "string") continue + const id = settings.seatIds[raw.id] ?? raw.id + if (!settings.seats.includes(id) || seen.has(id)) continue + seen.add(id) + const provider = typeof raw.provider === "string" ? raw.provider : "" + if (!SEAT_ID.test(id)) { skipped.push({ seat: id, reason: "not a seat id the floor accepts" }); continue } + if (!PROVIDER.test(provider) || provider === "none") { skipped.push({ seat: id, reason: "the oracle names no provider" }); continue } + const configured = settings.owners[id] + const owner = configured === "tom" || configured === "third-party" ? configured : raw.owner === "tom" || raw.owner === "kernel" ? "tom" : "third-party" + const grade = gradeOf(raw, provider, settings) + const observed = observedOf(raw, generatedMs) + let unmapped = 0 + const windows = (Array.isArray(raw.windows) ? raw.windows : []).map((entry) => { + const placed = windowOf(entry, grade) + if (!placed.placed) unmapped++ + return placed.window + }) + const ageS = Math.round((generatedMs - Date.parse(observed)) / 1000) + const staleReason = + grade === "UNKNOWN" + ? scrub(text(raw.detail) ?? "the oracle could not read this seat") + : raw.grade === "CACHED" + ? `the oracle served a cached reading ${ageS} s old` + : null + const slotCount = settings.slots[id] + const slots = + provider === "halogen" && raw.state === "open" + ? { capacity: Number.isInteger(slotCount) && slotCount >= 0 ? slotCount : 1, holders: 0 } + : null + const plan = instant(settings.plans[id]) + seats.push({ + seat: id, + provider, + owner, + ...dispatchOf(id, owner, raw, provider, settings), + plan: plan === null ? null : { expires_at: plan }, + slots, + windows, + observed_at: observed, + source: { kind: "seats-oracle", detail: scrub(`${SOURCE_SCHEMA} ${text(raw.source?.kind) ?? "no-source"}`) }, + grade, + stale_reason: staleReason, + model_windows_complete: grade !== "UNKNOWN" && unmapped === 0 + }) + } + for (const id of settings.seats) { + if (!seen.has(id)) skipped.push({ seat: id, reason: "the oracle did not report this seat" }) + } + return { + snapshot: { + schema_version: TARGET_SCHEMA, + host: settings.host ?? (typeof doc.host === "string" && doc.host !== "" ? doc.host : hostname().split(".")[0]), + published_at: now, + seats: seats.sort((a, b) => (a.seat < b.seat ? -1 : a.seat > b.seat ? 1 : 0)) + }, + skipped + } +} diff --git a/pkgs/substrate-apps/src/apps/pusher/src/token.mjs b/pkgs/substrate-apps/src/apps/pusher/src/token.mjs new file mode 100644 index 00000000..2a61bd70 --- /dev/null +++ b/pkgs/substrate-apps/src/apps/pusher/src/token.mjs @@ -0,0 +1,13 @@ +// apps/pusher/src/token.mjs: reads the floor's bearer from a file. The value is +// returned to the caller and never printed; an error names the path only. +import { readFileSync } from "node:fs" + +export class TokenError extends Error {} + +export const tokenFromFile = (path) => { + try { + return readFileSync(path, "utf8").trim() + } catch (error) { + throw new TokenError(`cannot read the floor token file ${path}: ${error.code ?? error.message}`) + } +} diff --git a/pkgs/substrate-apps/src/apps/pusher/src/usageSource.mjs b/pkgs/substrate-apps/src/apps/pusher/src/usageSource.mjs new file mode 100644 index 00000000..1c13cb3f --- /dev/null +++ b/pkgs/substrate-apps/src/apps/pusher/src/usageSource.mjs @@ -0,0 +1,352 @@ +// apps/pusher/src/usageSource.mjs: the Claude usage source, read directly or +// through an adapter, so the pusher needs no tally-rewrite feeder. +// +// TWO ADAPTERS, ONE READING SHAPE `{ observed_at, usage }`: +// +// "oauth" the direct reader. It calls the usage endpoint Claude Code's +// own /usage view calls, with the seat's OAuth bearer read from +// the seat's credentials file at call time. The bearer is held +// in memory for the one request and never written, logged or +// put in an error. Calls obey refreshPolicy.mjs. +// (Endpoint, headers and the token's place in the file are +// REPORTED from research-methods bin/stamp-receipt.py:103-126.) +// "cache-file" a file some other reader already wrote in the endpoint's +// answer shape: the tally feeder's `.window-cache-.json` +// while it still exists, or this module's own cache. +// +// THE CACHE KEEPS ONLY WHAT IS READ. The direct reader writes `observed_at` +// and `usage.{five_hour, seven_day, limits}`; spend and every other field of +// the answer are dropped before anything touches the disk. +import { existsSync, mkdirSync, readdirSync, readFileSync, renameSync, statSync, writeFileSync } from "node:fs" +import { homedir } from "node:os" +import { dirname, join } from "node:path" + +import { afterAttempt, initialRefreshState, REFRESH_POLICY, refreshDue } from "./refreshPolicy.mjs" + +export const USAGE_ENDPOINT = "https://api.anthropic.com/api/oauth/usage" +const USAGE_HEADERS = { "content-type": "application/json", "anthropic-beta": "oauth-2025-04-20" } +/** Where the bearer sits in a Claude credentials file. */ +export const DEFAULT_TOKEN_POINTER = ["claudeAiOauth", "accessToken"] + +export const defaultStateDir = () => join(homedir(), ".local", "state", "substrate") +export const defaultUsageDir = () => join(defaultStateDir(), "usage") +export const defaultDemandDir = () => join(defaultStateDir(), "demand") + +/** The usage endpoint's `limits[].kind`, mapped to the floor's window kinds. */ +const LIMIT_KINDS = { session: "five_hour", weekly_all: "seven_day", weekly_scoped: "model_scoped" } +const MINUTES = { five_hour: 300, seven_day: 10080, model_scoped: 10080 } +const ZONED = /^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}(:\d{2}(\.\d+)?)?(Z|[+-]\d{2}:\d{2})$/ + +export const instant = (value) => + typeof value === "string" && ZONED.test(value) && Number.isFinite(Date.parse(value)) ? value : null +const percent = (value) => (typeof value === "number" && Number.isFinite(value) && value >= 0 ? value : null) + +export class CredentialError extends Error {} + +/** + * The one mapping from the endpoint's `limits[]` to windows, for every pusher + * path (capacity review round 3, ported here by successor review r4: the + * pusher dropped rows it could not place and still claimed + * model_windows_complete, so the gate admitted past a 100 percent limit the + * tally adapter refused). A row it cannot place (a kind it does not know, or + * a scoped row naming no model, such as one scoped by surface only) is never + * dropped: it becomes a binding seven_day window graded UNKNOWN with its reset + * kept, and `unmapped` counts it, so the reading withdraws + * model_windows_complete. src/capacity/tally-meters.ts maps the same way; a + * root test pins the two together. + */ +export const windowsFromLimits = (limits) => { + let unmapped = 0 + const windows = limits.flatMap((limit) => { + const kind = LIMIT_KINDS[limit?.kind] + const model = + kind === "model_scoped" ? (limit.scope?.model?.display_name ?? limit.scope?.model?.id ?? null) : null + if (kind === undefined || (kind === "model_scoped" && typeof model !== "string")) { + unmapped++ + return [ + { + kind: "seven_day", + model: null, + binding: true, + minutes: MINUTES.seven_day, + utilization_pct: null, + resets_at: instant(limit?.resets_at), + severity: null, + grade: "UNKNOWN" + } + ] + } + return [ + { + kind, + model, + binding: true, + minutes: MINUTES[kind], + utilization_pct: percent(limit.percent), + resets_at: instant(limit.resets_at), + severity: typeof limit.severity === "string" ? limit.severity : null, + grade: "MEASURED" + } + ] + }) + return { windows, unmapped } +} + +/** A usage answer lists every model limit: limits[] is present and every row was placed. */ +export const limitsComplete = (usage) => + usage !== null && typeof usage === "object" && Array.isArray(usage.limits) && windowsFromLimits(usage.limits).unmapped === 0 + +/** + * The windows a usage answer states: `limits[]` when present (the only place + * model-scoped rows appear), else the flat `five_hour` / `seven_day` pair. + */ +export const windowsFromUsage = (usage) => { + if (usage === null || typeof usage !== "object") return [] + if (Array.isArray(usage.limits)) return windowsFromLimits(usage.limits).windows + return [ + ["five_hour", usage.five_hour], + ["seven_day", usage.seven_day] + ].flatMap(([kind, entry]) => + entry === null || typeof entry !== "object" + ? [] + : [ + { + kind, + model: null, + binding: true, + minutes: MINUTES[kind], + utilization_pct: percent(entry.utilization), + resets_at: instant(entry.resets_at), + severity: null, + grade: "MEASURED" + } + ] + ) +} + +/** Only the fields the pusher reads; spend and the rest never reach the disk. */ +export const minimalUsage = (usage) => { + const pick = (entry) => + entry !== null && typeof entry === "object" + ? { utilization: percent(entry.utilization), resets_at: instant(entry.resets_at) } + : null + const limits = Array.isArray(usage?.limits) + ? usage.limits.map((limit) => ({ + kind: typeof limit?.kind === "string" ? limit.kind : null, + percent: percent(limit?.percent), + severity: typeof limit?.severity === "string" ? limit.severity : null, + resets_at: instant(limit?.resets_at), + scope: + typeof limit?.scope?.model?.display_name === "string" || typeof limit?.scope?.model?.id === "string" + ? { + model: { + display_name: limit.scope.model.display_name ?? null, + id: limit.scope.model.id ?? null + } + } + : null + })) + : undefined + return { + five_hour: pick(usage?.five_hour), + seven_day: pick(usage?.seven_day), + ...(limits === undefined ? {} : { limits }) + } +} + +const readJson = (path) => { + if (!existsSync(path)) return { missing: true } + try { + return { value: JSON.parse(readFileSync(path, "utf8")) } + } catch (error) { + return { error: `${error.name}` } + } +} + +/** Atomic, 0600, directory 0700. */ +export const writePrivateJson = (path, value) => { + mkdirSync(dirname(path), { recursive: true, mode: 0o700 }) + const temporary = `${path}.tmp-${process.pid}` + writeFileSync(temporary, `${JSON.stringify(value, null, 2)}\n`, { mode: 0o600 }) + renameSync(temporary, path) +} + +/** A reading `{ observed_at, usage }` from a file in the endpoint's answer shape, or null. */ +export const readUsageFile = (path) => { + const read = readJson(path) + const value = read.value + if (value === null || typeof value !== "object") return null + const observed = instant(value.observed_at) + if (observed === null || value.usage === null || typeof value.usage !== "object") return null + return { observed_at: observed, usage: value.usage } +} + +/** + * The bearer from a credentials file. The value is returned and never shown: + * every error names the path and the pointer only. + */ +export const bearerFromCredentials = (path, pointer = DEFAULT_TOKEN_POINTER) => { + let parsed + try { + parsed = JSON.parse(readFileSync(path, "utf8")) + } catch (error) { + throw new CredentialError(`cannot read the credentials file ${path}: ${error.code ?? error.name}`) + } + let node = parsed + for (const key of pointer) node = node !== null && typeof node === "object" ? node[key] : undefined + if (typeof node !== "string" || node.length === 0) { + throw new CredentialError(`the credentials file ${path} has no string at ${pointer.join(".")}`) + } + return node +} + +const mtimeMs = (path) => { + try { + return statSync(path).mtimeMs + } catch { + return null + } +} + +/** + * Whether a seat is in active use: a demand marker touched within the window + * (dispatch touches it), or a transcript under the seat's config directory + * written within the window (interactive use). Looks at the 5 most recently + * touched project directories only, to keep a pass cheap. + */ +export const seatActive = ({ seat, demandDir, configDir, nowMs, windowSeconds = 600 }) => { + const recent = (value) => value !== null && nowMs - value <= windowSeconds * 1000 + if (demandDir !== null && demandDir !== undefined && recent(mtimeMs(join(demandDir, seat)))) return true + if (configDir === null || configDir === undefined) return false + const projects = join(configDir, "projects") + let dirs + try { + dirs = readdirSync(projects, { withFileTypes: true }).filter((entry) => entry.isDirectory()) + } catch { + return false + } + const newest = dirs + .map((entry) => ({ path: join(projects, entry.name), at: mtimeMs(join(projects, entry.name)) ?? 0 })) + .sort((left, right) => right.at - left.at) + .slice(0, 5) + for (const dir of newest) { + if (recent(dir.at)) return true + try { + for (const name of readdirSync(dir.path)) { + if (name.endsWith(".jsonl") && recent(mtimeMs(join(dir.path, name)))) return true + } + } catch { + // unreadable directory: not evidence of activity + } + } + return false +} + +const retryAfterSeconds = (response) => { + const raw = response.headers?.get?.("retry-after") + if (raw === null || raw === undefined) return null + const seconds = Number(raw) + if (Number.isFinite(seconds)) return seconds + const at = Date.parse(raw) + return Number.isFinite(at) ? Math.max(0, (at - Date.now()) / 1000) : null +} + +/** One GET of the usage endpoint. The bearer is only in the request header. */ +export const fetchUsage = async ({ token, fetchImpl = fetch, timeoutMs = 12000 }) => { + let response + try { + response = await fetchImpl(USAGE_ENDPOINT, { + method: "GET", + headers: { ...USAGE_HEADERS, authorization: `Bearer ${token}` }, + signal: AbortSignal.timeout(timeoutMs) + }) + } catch (error) { + return { kind: "error", detail: `the usage read did not complete: ${error?.name ?? "Error"}` } + } + if (response.status === 429) { + return { kind: "rate-limited", retryAfterSeconds: retryAfterSeconds(response), detail: "usage endpoint 429" } + } + if (response.status === 401 || response.status === 403) { + return { kind: "auth-failed", detail: `usage endpoint ${response.status} unauthorized` } + } + if (response.status !== 200) return { kind: "error", detail: `usage endpoint answered ${response.status}` } + try { + return { kind: "success", usage: JSON.parse(await response.text()) } + } catch { + return { kind: "error", detail: "usage endpoint answered 200 with a body that is not JSON" } + } +} + +const bindingFacts = (reading) => { + const windows = reading === null ? [] : windowsFromUsage(reading.usage).filter((w) => w.kind !== "model_scoped") + const known = windows.map((w) => w.utilization_pct).filter((value) => value !== null) + return { + bindingMaxPct: known.length === 0 ? null : Math.max(...known), + resets: windows.map((w) => w.resets_at).filter((value) => value !== null) + } +} + +/** + * Reads one seat's usage through its configured source. + * + * @param source `{ kind: "oauth", credentials, token_pointer?, config_dir? }` or + * `{ kind: "cache-file", path }`. + * @returns `{ reading, attempted, decision, error }`; `reading` is + * `{ observed_at, usage }` or null, `error` a line safe to publish or null. + */ +export const readSeatUsage = async ({ + seat, + source, + usageDir = defaultUsageDir(), + demandDir = defaultDemandDir(), + nowMs = Date.now(), + allowNetwork = true, + preDispatch = false, + fetchImpl = fetch, + policy = REFRESH_POLICY +}) => { + if (source?.kind === "cache-file") { + const reading = typeof source.path === "string" ? readUsageFile(source.path) : null + return { reading, attempted: false, decision: null, error: reading === null ? "no usage cache file reading" : null } + } + if (source?.kind !== "oauth" || typeof source.credentials !== "string") { + return { reading: null, attempted: false, decision: null, error: "no usage source configured" } + } + const cachePath = join(usageDir, `${seat}.json`) + const statePath = join(usageDir, `${seat}.state.json`) + const cached = readUsageFile(cachePath) + const stored = readJson(statePath).value + const state = stored !== null && typeof stored === "object" ? stored : initialRefreshState() + const credentialsMtimeMs = mtimeMs(source.credentials) + const decision = refreshDue( + state, + { + nowMs, + active: seatActive({ seat, demandDir, configDir: source.config_dir ?? null, nowMs }), + preDispatch, + credentialsMtimeMs, + ...bindingFacts(cached) + }, + policy + ) + const lastError = typeof state.last_error === "string" ? state.last_error : null + if (!decision.due || !allowNetwork) { + return { reading: cached, attempted: false, decision, error: lastError } + } + let outcome + try { + const token = bearerFromCredentials(source.credentials, source.token_pointer ?? DEFAULT_TOKEN_POINTER) + outcome = await fetchUsage({ token, fetchImpl }) + } catch (error) { + outcome = { kind: "auth-failed", detail: error instanceof CredentialError ? "credentials unreadable" : "credentials error" } + } + const next = { ...afterAttempt(state, outcome, nowMs, credentialsMtimeMs, policy), last_error: outcome.kind === "success" ? null : outcome.detail } + if (outcome.kind === "success") { + const reading = { observed_at: new Date(nowMs).toISOString(), usage: minimalUsage(outcome.usage) } + writePrivateJson(cachePath, reading) + writePrivateJson(statePath, next) + return { reading, attempted: true, decision, error: null } + } + writePrivateJson(statePath, next) + return { reading: cached, attempted: true, decision, error: outcome.detail } +} diff --git a/pkgs/substrate-apps/src/apps/pusher/test/fixtures/seats-v1.json b/pkgs/substrate-apps/src/apps/pusher/test/fixtures/seats-v1.json new file mode 100644 index 00000000..672b511a --- /dev/null +++ b/pkgs/substrate-apps/src/apps/pusher/test/fixtures/seats-v1.json @@ -0,0 +1,269 @@ +{ + "schema_version": "seat-capacity/1", + "generated_at": "2026-09-23T20:00:00Z", + "host": "example-box", + "recommend": "cc", + "thresholds": { + "wall_pct": 95.0, + "warn_pct": 80.0 + }, + "allowance_changes": [], + "seats": [ + { + "id": "cc", + "provider": "claude", + "label": "Claude (personal)", + "owner": "tom", + "state": "open", + "usable": true, + "grade": "MEASURED", + "detail": "", + "note": "", + "source": { + "kind": "seat-feeder-cache", + "config_dir": "/home/example/.claude", + "reading_age_seconds": 40 + }, + "windows": [ + { + "binding": true, + "id": "five_hour", + "is_active": false, + "label": "session (5h)", + "minutes": 300, + "pace_pct_per_hour": null, + "projected_pct_at_reset": null, + "remaining_pct": 100.0, + "resets_at": "", + "resets_in_seconds": null, + "scope": null, + "severity": "normal", + "used_pct": 0.0, + "wall_eta": "" + }, + { + "binding": true, + "id": "seven_day", + "is_active": false, + "label": "weekly (7d)", + "minutes": 10080, + "pace_pct_per_hour": null, + "projected_pct_at_reset": null, + "remaining_pct": 94.0, + "resets_at": "2026-09-30T09:59:59Z", + "resets_in_seconds": null, + "scope": null, + "severity": "normal", + "used_pct": 6.0, + "wall_eta": "" + }, + { + "binding": false, + "id": "weekly:fable", + "is_active": false, + "label": "weekly \u00b7 Fable", + "minutes": 10080, + "pace_pct_per_hour": null, + "projected_pct_at_reset": null, + "remaining_pct": 98.0, + "resets_at": "2026-09-30T09:59:59Z", + "resets_in_seconds": null, + "scope": "Fable", + "severity": "normal", + "used_pct": 2.0, + "wall_eta": "" + } + ] + }, + { + "id": "cc2", + "provider": "claude", + "label": "Claude (work)", + "owner": "tom", + "state": "tight", + "usable": true, + "grade": "CACHED", + "detail": "", + "note": "", + "source": { + "kind": "own-cache", + "reading_age_seconds": 1500 + }, + "windows": [ + { + "binding": true, + "id": "five_hour", + "is_active": false, + "label": "session (5h)", + "minutes": 300, + "pace_pct_per_hour": null, + "projected_pct_at_reset": null, + "remaining_pct": 86.0, + "resets_at": "2026-09-23T23:40:00Z", + "resets_in_seconds": null, + "scope": null, + "severity": "normal", + "used_pct": 14.0, + "wall_eta": "" + }, + { + "binding": true, + "id": "seven_day", + "is_active": false, + "label": "weekly (7d)", + "minutes": 10080, + "pace_pct_per_hour": null, + "projected_pct_at_reset": null, + "remaining_pct": 24.0, + "resets_at": "2026-09-24T06:00:00Z", + "resets_in_seconds": null, + "scope": null, + "severity": "warning", + "used_pct": 76.0, + "wall_eta": "" + }, + { + "binding": false, + "id": "weekly:fable", + "is_active": false, + "label": "weekly \u00b7 Fable", + "minutes": 10080, + "pace_pct_per_hour": null, + "projected_pct_at_reset": null, + "remaining_pct": 0.0, + "resets_at": "2026-09-24T06:00:00Z", + "resets_in_seconds": null, + "scope": "Fable", + "severity": "critical", + "used_pct": 100.0, + "wall_eta": "" + } + ] + }, + { + "id": "cc3", + "provider": "claude", + "label": "Claude (third)", + "owner": "tom", + "state": "unauth", + "usable": false, + "grade": "UNKNOWN", + "detail": "sign-in expired, see /home/example/.claude-3/.credentials.json", + "note": "", + "source": {}, + "windows": [] + }, + { + "id": "codex", + "provider": "codex", + "label": "Codex CLI", + "owner": "third-party", + "state": "open", + "usable": true, + "grade": "MEASURED", + "detail": "", + "note": "", + "source": { + "kind": "codex-rollout-rate-limits", + "path": "/home/example/.codex/sessions/x.jsonl", + "observed_at": "2026-09-23T19:50:00Z" + }, + "windows": [ + { + "binding": true, + "id": "codex:primary", + "is_active": false, + "label": "weekly (7d)", + "minutes": 10080, + "pace_pct_per_hour": null, + "projected_pct_at_reset": null, + "remaining_pct": 96.0, + "resets_at": "2026-09-28T19:07:31Z", + "resets_in_seconds": null, + "scope": null, + "severity": "", + "used_pct": 4.0, + "wall_eta": "" + }, + { + "binding": true, + "id": "codex:odd", + "is_active": false, + "label": "odd (1d)", + "minutes": 1440, + "pace_pct_per_hour": null, + "projected_pct_at_reset": null, + "remaining_pct": 90.0, + "resets_at": "2026-09-24T19:00:00Z", + "resets_in_seconds": null, + "scope": null, + "severity": "", + "used_pct": 10.0, + "wall_eta": "" + } + ] + }, + { + "id": "pi-qwencloud", + "provider": "qwen", + "label": "Qwen Cloud", + "owner": "tom", + "state": "open", + "usable": true, + "grade": "MEASURED-FROM-REFUSAL", + "detail": "", + "note": "", + "source": { + "kind": "429-refusal-text", + "key_source": "/run/agenix/example-token" + }, + "windows": [ + { + "binding": true, + "id": "qwen:weekly", + "is_active": false, + "label": "weekly (7d)", + "minutes": 10080, + "pace_pct_per_hour": null, + "projected_pct_at_reset": null, + "remaining_pct": null, + "resets_at": "", + "resets_in_seconds": null, + "scope": null, + "severity": "", + "used_pct": null, + "wall_eta": "" + } + ] + }, + { + "id": "gpu-coordinator", + "provider": "none", + "label": "Local GPU", + "owner": "kernel", + "state": "n/a", + "usable": false, + "grade": "UNKNOWN", + "detail": "serves nothing", + "note": "", + "source": {}, + "windows": [] + }, + { + "id": "gpu-worker", + "provider": "halogen", + "label": "Halogen", + "owner": "kernel", + "state": "open", + "usable": true, + "grade": "MEASURED", + "detail": "up", + "note": "", + "source": { + "kind": "halogen-health", + "endpoint": "http://worker.example:8731/health" + }, + "windows": [] + } + ] +} \ No newline at end of file diff --git a/pkgs/substrate-apps/src/apps/pusher/test/gentle.test.mjs b/pkgs/substrate-apps/src/apps/pusher/test/gentle.test.mjs new file mode 100644 index 00000000..27c8d8fa --- /dev/null +++ b/pkgs/substrate-apps/src/apps/pusher/test/gentle.test.mjs @@ -0,0 +1,257 @@ +// The gentle pusher: seat-capacity/1 (the box's oracle, `seats --json`) to +// seat-capacity/2, the refresh policy it reads under, and the stale handling. +import { spawn } from "node:child_process" +import { existsSync, mkdtempSync, readFileSync, readdirSync, rmSync, statSync, writeFileSync } from "node:fs" +import { createServer } from "node:http" +import { tmpdir } from "node:os" +import { join } from "node:path" +import { fileURLToPath } from "node:url" +import { Schema } from "effect" +import { afterEach, describe, expect, it } from "vitest" + +import { SeatCapacitySnapshot } from "@substrate/planning/schema/seatCapacity.ts" +import { admitSeat } from "@substrate/planning/capacity/project.ts" +import { SeatsFormatError, snapshotFromSeatsV1 } from "../src/seatsOracle.mjs" +import { initialGentleState, planRead, tick } from "../src/gentle.mjs" + +const FIXTURE = fileURLToPath(new URL("./fixtures/seats-v1.json", import.meta.url)) +const BIN = fileURLToPath(new URL("../bin/substrate-pusher.mjs", import.meta.url)) +const SRC = fileURLToPath(new URL("../src/", import.meta.url)) +const v1 = () => JSON.parse(readFileSync(FIXTURE, "utf8")) +const NOW = "2026-09-23T20:00:05Z" +const decode = Schema.decodeUnknownSync(SeatCapacitySnapshot) +const convert = (config = {}, doc = v1(), now = NOW) => snapshotFromSeatsV1(doc, config, now) +const bySeat = (snapshot) => Object.fromEntries(snapshot.seats.map((seat) => [seat.seat, seat])) + +describe("seat-capacity/1 to seat-capacity/2", () => { + it("covers cc, cc2, codex, pi-qwencloud and halogen, and decodes as seat-capacity/2", () => { + const { snapshot, skipped } = convert() + expect(decode(snapshot)).toBeTruthy() + expect(snapshot.seats.map((seat) => seat.seat)).toEqual(["cc", "cc2", "codex", "halogen", "pi-qwencloud"]) + expect(skipped).toEqual([]) + expect(snapshot.published_at).toBe(NOW) + expect(snapshot.host).toBe("example-box") + }) + + it("places the 5h, 7d and model-scoped windows, with the reset instants and severities", () => { + const cc2 = bySeat(convert().snapshot).cc2 + expect(cc2.windows.map((w) => [w.kind, w.model, w.minutes, w.utilization_pct, w.resets_at, w.severity])).toEqual([ + ["five_hour", null, 300, 14, "2026-09-23T23:40:00Z", "normal"], + ["seven_day", null, 10080, 76, "2026-09-24T06:00:00Z", "warning"], + ["model_scoped", "Fable", 10080, 100, "2026-09-24T06:00:00Z", "critical"] + ]) + expect(cc2.model_windows_complete).toBe(true) + // An empty reset is "opens on first use"; an empty severity is none. + expect(bySeat(convert().snapshot).cc.windows[0]).toMatchObject({ kind: "five_hour", resets_at: null, utilization_pct: 0 }) + }) + + it("dates each reading by the provider's answer, not by the oracle's run", () => { + const seats = bySeat(convert().snapshot) + expect(seats.cc.observed_at).toBe("2026-09-23T19:59:20.000Z") // generated_at - 40 s + expect(seats.cc2.observed_at).toBe("2026-09-23T19:35:00.000Z") // a CACHED reading 1500 s old + expect(seats.cc2.stale_reason).toMatch(/cached reading 1500 s old/) + expect(seats.codex.observed_at).toBe("2026-09-23T19:50:00.000Z") // the rollout's own instant + }) + + it("a window it cannot place becomes a binding UNKNOWN weekly window and withdraws model_windows_complete", () => { + const codex = bySeat(convert().snapshot).codex + expect(codex.windows[1]).toMatchObject({ kind: "seven_day", binding: true, utilization_pct: null, grade: "UNKNOWN", resets_at: "2026-09-24T19:00:00Z" }) + expect(codex.model_windows_complete).toBe(false) + }) + + it("owners, rulings, slots and estimated providers follow the config; the oracle's owner field can be overridden", () => { + const plain = bySeat(convert().snapshot) + expect([plain.codex.owner, plain.codex.dispatchable, plain.codex.dispatchable_reason]).toEqual(["third-party", false, "third-party"]) + const ruled = bySeat(convert({ owners: { codex: "tom" }, slots: { halogen: 2 }, plans: { "pi-qwencloud": "2026-11-07T00:00:00Z" } }).snapshot) + expect([ruled.codex.owner, ruled.codex.dispatchable]).toEqual(["tom", true]) + expect(ruled.halogen).toMatchObject({ provider: "halogen", owner: "tom", slots: { capacity: 2, holders: 0 }, grade: "MEASURED", dispatchable: true }) + expect(ruled["pi-qwencloud"]).toMatchObject({ grade: "ESTIMATED", plan: { expires_at: "2026-11-07T00:00:00Z" } }) + const cc3 = bySeat(convert({ seats: ["cc3"] }).snapshot).cc3 + expect([cc3.dispatchable, cc3.dispatchable_reason, cc3.grade]).toEqual([false, "evicted", "UNKNOWN"]) + const unauth = bySeat(convert({ seats: ["cc3"], not_dispatchable: {} }).snapshot).cc3 + expect(unauth.dispatchable_reason).toBe("evicted") // a config cannot lift a built-in ruling + const skip = convert({ seats: ["gpu-coordinator", "cc"] }) + expect(skip.skipped).toEqual([{ seat: "gpu-coordinator", reason: "the oracle names no provider" }]) + }) + + it("no local path, endpoint or credential location leaves the box", () => { + const out = JSON.stringify(convert({ seats: ["cc", "cc2", "cc3", "codex", "pi-qwencloud", "halogen"] }).snapshot) + expect(out).not.toMatch(/\/home\/|\/run\/|agenix|credentials|worker\.example|config_dir|key_source/) + expect(out).toContain("") + }) + + it("refuses a document that is not seat-capacity/1", () => { + expect(() => convert({}, { schema_version: "seat-capacity/2", seats: [] })).toThrow(SeatsFormatError) + expect(() => convert({}, null)).toThrow(SeatsFormatError) + }) + + it("the admit rule on the converted seats: model-scoped refusal, stale refusal, estimated refusal, slot admission", () => { + const seats = bySeat(convert({ owners: { codex: "tom" }, slots: { halogen: 1 } }).snapshot) + const at = NOW + expect(admitSeat("cc", seats.cc, { model: "claude-fable-5-1", min_headroom_pct: 0 }, at)).toMatchObject({ admit: true }) + // cc2's reading is 1505 s old at NOW: STALE, which refuses and asks for a fresh read. + expect(admitSeat("cc2", seats.cc2, { model: "claude-opus-5", min_headroom_pct: 0 }, at)).toMatchObject({ admit: false, reason: "stale", raise_demand: true }) + // The same reading when fresh: Fable refused by its own window, Opus admitted. + const fresh = { ...seats.cc2, observed_at: "2026-09-23T20:00:00Z" } + expect(admitSeat("cc2", fresh, { model: "claude-fable-5-1", min_headroom_pct: 0 }, at).admit).toBe(false) + expect(admitSeat("cc2", fresh, { model: "claude-opus-5", min_headroom_pct: 0 }, at)).toMatchObject({ admit: true }) + expect(admitSeat("pi-qwencloud", seats["pi-qwencloud"], { model: null, min_headroom_pct: 0 }, at)).toMatchObject({ admit: false, reason: "estimated" }) + expect(admitSeat("halogen", seats.halogen, { model: null, min_headroom_pct: 0 }, at)).toMatchObject({ admit: true }) + }) +}) + +describe("the refresh policy the pusher reads under", () => { + const T0 = Date.parse("2026-09-23T20:00:00Z") + const S = 1000 + const read = (state, nowMs, extra = {}) => tick(state, { nowMs, config: {}, runOracle: async () => v1(), post: async () => ({ report: { results: [] } }), ...extra }) + + it("first read at once; never again within 120 s; idle every 900 s", async () => { + expect(planRead(initialGentleState(), { nowMs: T0 })).toMatchObject({ due: true, reason: "first-read" }) + const first = await read(initialGentleState(), T0) + expect(first.events.map((e) => e.event)).toEqual(["read", "pushed"]) + expect(planRead(first.state, { nowMs: T0 + 60 * S })).toMatchObject({ due: false, reason: "min-spacing" }) + expect(planRead(first.state, { nowMs: T0 + 60 * S, preDispatch: true })).toMatchObject({ due: false, reason: "min-spacing" }) + expect(planRead(first.state, { nowMs: T0 + 899 * S })).toMatchObject({ due: false, reason: "fresh" }) + expect(planRead(first.state, { nowMs: T0 + 900 * S })).toMatchObject({ due: true, reason: "idle" }) + }) + + it("active every 300 s (a demand marker, or readings that move); 180 s near a cap; pre-dispatch after 300 s", async () => { + const first = await read(initialGentleState(), T0) + expect(planRead(first.state, { nowMs: T0 + 300 * S, demandAtMs: T0 + 250 * S })).toMatchObject({ due: true, reason: "active" }) + expect(planRead(first.state, { nowMs: T0 + 299 * S, demandAtMs: T0 + 250 * S }).due).toBe(false) + expect(planRead(first.state, { nowMs: T0 + 301 * S, preDispatch: true })).toMatchObject({ due: true, reason: "pre-dispatch" }) + // Two reads whose utilization differs: the box is active without a marker. cc2's 7d is 76 %, below 85. + const moved = v1() + moved.seats[0].windows[1].used_pct = 7.0 + const second = await read(first.state, T0 + 900 * S, { runOracle: async () => moved }) + expect(planRead(second.state, { nowMs: T0 + 1200 * S })).toMatchObject({ due: true, reason: "active" }) + const hot = v1() + hot.seats[1].windows[1].used_pct = 90.0 + const third = await read(second.state, T0 + 1200 * S, { runOracle: async () => hot }) + expect(planRead(third.state, { nowMs: T0 + 1380 * S })).toMatchObject({ due: true, reason: "active" }) + expect(planRead(third.state, { nowMs: T0 + 1379 * S }).due).toBe(false) + }) + + it("an unchanged reading is not re-posted until the heartbeat", async () => { + const claudeOnly = { config: { seats: ["cc", "cc2"] } } + const first = await read(initialGentleState(), T0, claudeOnly) + const again = await read(first.state, T0 + 900 * S, claudeOnly) + expect(again.events.map((e) => e.event)).toEqual(["read", "not-pushed"]) + const beat = await read(again.state, T0 + 1800 * S, claudeOnly) + expect(beat.events.at(-1)).toMatchObject({ event: "pushed", reason: "heartbeat" }) + // A published Halogen slot row beats every 600 s, so the floor never grades it STALE (1200 s). + const slots = await read((await read(initialGentleState(), T0)).state, T0 + 900 * S) + expect(slots.events.at(-1)).toMatchObject({ event: "pushed", reason: "heartbeat" }) + }) +}) + +describe("stale and failed reads", () => { + const T0 = Date.parse("2026-09-23T20:00:00Z") + const S = 1000 + + it("a failed oracle run posts nothing, keeps the 120 s spacing, and the next due read recovers", async () => { + const posts = [] + const failed = await tick(initialGentleState(), { nowMs: T0, config: {}, runOracle: async () => { throw new Error("the oracle failed: 1") }, post: async (s) => posts.push(s) }) + expect(failed.events.at(-1)).toMatchObject({ event: "read-failed" }) + expect(posts).toEqual([]) + expect(planRead(failed.state, { nowMs: T0 + 119 * S })).toMatchObject({ due: false, reason: "min-spacing" }) + const wrong = await tick(failed.state, { nowMs: T0 + 120 * S, config: {}, runOracle: async () => ({ schema_version: "nope" }), post: async (s) => posts.push(s) }) + expect(wrong.events.at(-1)).toMatchObject({ event: "read-failed" }) + const ok = await tick(wrong.state, { nowMs: T0 + 240 * S, config: {}, runOracle: async () => v1(), post: async (s) => { posts.push(s); return {} } }) + expect(ok.events.map((e) => e.event)).toEqual(["read", "pushed"]) + expect(posts).toHaveLength(1) + }) + + it("a seat the oracle stops reporting is named, not carried forward", async () => { + const doc = v1() + doc.seats = doc.seats.filter((seat) => seat.id !== "cc") + const r = await tick(initialGentleState(), { nowMs: T0, config: {}, runOracle: async () => doc, post: async () => ({}) }) + expect(r.events[0].skipped).toEqual([{ seat: "cc", reason: "the oracle did not report this seat" }]) + expect(r.state.last.seats.map((seat) => seat.seat)).not.toContain("cc") + }) + + it("a failed post is retried with the same snapshot after 60 s, without a new read", async () => { + let reads = 0 + const runOracle = async () => { reads++; return v1() } + const down = await tick(initialGentleState(), { nowMs: T0, config: {}, runOracle, post: async () => { throw new Error("answered 503") } }) + expect(down.events.at(-1)).toMatchObject({ event: "push-failed" }) + const early = await tick(down.state, { nowMs: T0 + 30 * S, config: {}, runOracle, post: async () => ({}) }) + expect(early.events).toEqual([expect.objectContaining({ event: "wait", reason: "push-retry" })]) + const sent = [] + const retry = await tick(down.state, { nowMs: T0 + 60 * S, config: {}, runOracle, post: async (s) => { sent.push(s); return {} } }) + expect(retry.events[0]).toMatchObject({ event: "pushed", reason: "retry" }) + expect(sent[0]).toEqual(down.state.pending.snapshot) + expect(reads).toBe(1) + }) +}) + +describe("the pusher never calls a usage endpoint itself", () => { + it("its modules name no provider endpoint and do not import the legacy direct reader", () => { + for (const file of ["seatsOracle.mjs", "gentle.mjs"]) { + const text = readFileSync(join(SRC, file), "utf8") + expect(text, file).not.toMatch(/api\.anthropic\.com|oauth\/usage|usageSource/) + } + const bin = readFileSync(BIN, "utf8") + expect(bin).not.toMatch(/api\.anthropic\.com|oauth\/usage|usageSource|seatCapacity\.mjs"/) + }) +}) + +describe("the process", () => { + const dirs = [] + afterEach(() => { for (const d of dirs.splice(0)) rmSync(d, { recursive: true, force: true }) }) + const run = (args, env = {}) => new Promise((resolve) => { + const child = spawn(process.execPath, [BIN, ...args], { env: { ...process.env, ...env }, stdio: ["ignore", "pipe", "pipe"] }) + let out = "", err = "" + child.stdout.on("data", (b) => { out += b }) + child.stderr.on("data", (b) => { err += b }) + child.on("exit", (code) => resolve({ code, out, err, child })) + }) + + it("--once posts to /capacity/snapshots with the bearer, keeps 0600 state, and releases its pidfile", async () => { + const dir = mkdtempSync(join(tmpdir(), "gentle-")); dirs.push(dir) + writeFileSync(join(dir, "token"), "test-token-value\n", { mode: 0o600 }) + const seen = [] + const server = createServer((req, res) => { + let body = "" + req.on("data", (b) => { body += b }) + req.on("end", () => { + seen.push({ method: req.method, url: req.url, auth: req.headers.authorization, body: JSON.parse(body) }) + res.writeHead(200, { "content-type": "application/json" }).end(JSON.stringify({ report: { results: [], changed: true } })) + }) + }) + await new Promise((r) => server.listen(0, "127.0.0.1", r)) + const port = server.address().port + const pid = join(dir, "pusher.pid") + const r = await run(["--once", "--url", `http://127.0.0.1:${port}`, "--token-file", join(dir, "token"), "--seats-json", FIXTURE, + "--state", join(dir, "state.json"), "--pidfile", pid, "--demand-dir", join(dir, "demand"), "--config", join(dir, "absent.json")].slice(0, -2)) + server.close() + expect(r.code, r.err).toBe(0) + expect(seen).toHaveLength(1) + expect([seen[0].method, seen[0].url, seen[0].auth]).toEqual(["POST", "/capacity/snapshots", "Bearer test-token-value"]) + expect(decode(seen[0].body).seats.length).toBe(5) + expect(r.out).not.toContain("test-token-value") + expect(existsSync(pid)).toBe(false) + expect(statSync(join(dir, "state.json")).mode & 0o777).toBe(0o600) + }) + + it("a second pusher refuses while the first holds the pidfile; a stale pidfile is replaced", async () => { + const dir = mkdtempSync(join(tmpdir(), "gentle-")); dirs.push(dir) + const pid = join(dir, "pusher.pid") + const common = ["--dry-run", "--seats-json", FIXTURE, "--pidfile", pid, "--demand-dir", join(dir, "demand")] + const first = spawn(process.execPath, [BIN, ...common], { stdio: "ignore" }) + for (let i = 0; i < 100 && !existsSync(pid); i++) await new Promise((r) => setTimeout(r, 50)) + expect(readFileSync(pid, "utf8").trim()).toBe(String(first.pid)) + const second = await run([...common, "--once"]) + expect(second.code).toBe(3) + expect(second.err).toMatch(/another pusher/) + const exited = new Promise((r) => first.on("exit", r)) + first.kill("SIGTERM") + await exited + expect(existsSync(pid)).toBe(false) + writeFileSync(pid, "999999999\n") + const third = await run([...common, "--once"]) + expect(third.code, third.err).toBe(0) + expect(JSON.parse(third.out.trim().split("\n").at(-1))).toMatchObject({ event: "dry-run" }) + expect(readdirSync(dir)).not.toContain("pusher.pid") + }) +}) diff --git a/pkgs/substrate-apps/src/apps/pusher/test/refreshPolicy.test.mjs b/pkgs/substrate-apps/src/apps/pusher/test/refreshPolicy.test.mjs new file mode 100644 index 00000000..be3385cb --- /dev/null +++ b/pkgs/substrate-apps/src/apps/pusher/test/refreshPolicy.test.mjs @@ -0,0 +1,117 @@ +// The gentle refresh policy (SCOUT.md 5.5), pinned case by case and by property. +import fc from "fast-check" +import { describe, expect, it } from "vitest" + +import { afterAttempt, initialRefreshState, REFRESH_POLICY, refreshDue } from "../src/refreshPolicy.mjs" + +const T0 = Date.parse("2026-09-23T06:00:00Z") +const s = (seconds) => T0 + seconds * 1000 +const ok = (at) => afterAttempt(initialRefreshState(), { kind: "success" }, at, 1) +const due = (state, input) => refreshDue(state, { active: false, preDispatch: false, bindingMaxPct: null, resets: [], credentialsMtimeMs: 1, ...input }) + +describe("refreshDue", () => { + it("P1 the first read is due", () => { + expect(due(initialRefreshState(), { nowMs: T0 })).toMatchObject({ due: true, reason: "first-read" }) + }) + + it("P2 idle: 900 s between reads", () => { + const state = ok(T0) + expect(due(state, { nowMs: s(899) }).due).toBe(false) + expect(due(state, { nowMs: s(900) })).toMatchObject({ due: true, reason: "idle" }) + expect(due(state, { nowMs: s(600) }).next_at).toBe(new Date(s(900)).toISOString()) + }) + + it("P3 active: 300 s, and 180 s once a binding window is at 85 percent", () => { + const state = ok(T0) + expect(due(state, { nowMs: s(299), active: true }).due).toBe(false) + expect(due(state, { nowMs: s(300), active: true }).due).toBe(true) + expect(due(state, { nowMs: s(179), active: true, bindingMaxPct: 85 }).due).toBe(false) + expect(due(state, { nowMs: s(180), active: true, bindingMaxPct: 85 }).due).toBe(true) + expect(due(state, { nowMs: s(180), active: false, bindingMaxPct: 99 }).due).toBe(false) + }) + + it("P4 the hard 120 s minimum holds over every other reason", () => { + const failed = afterAttempt(ok(T0 - 3600_000), { kind: "error" }, T0, 1) + expect(due(failed, { nowMs: s(119), active: true, preDispatch: true }).reason).toBe("min-spacing") + expect(due(failed, { nowMs: s(120), active: true }).due).toBe(true) + }) + + it("P5 a reset boundary is read at resets_at + 90 s", () => { + const state = ok(T0) + const resets = [new Date(s(200)).toISOString()] + expect(due(state, { nowMs: s(289), resets }).due).toBe(false) + expect(due(state, { nowMs: s(290), resets })).toMatchObject({ due: true, reason: "reset-boundary" }) + expect(due(state, { nowMs: s(150), resets }).next_at).toBe(new Date(s(290)).toISOString()) + // a boundary already covered by a later read is not read again + expect(due(ok(s(300)), { nowMs: s(500), resets }).due).toBe(false) + }) + + it("P6 pre-dispatch reads a reading older than 300 s", () => { + const state = ok(T0) + expect(due(state, { nowMs: s(300), preDispatch: true }).due).toBe(false) + expect(due(state, { nowMs: s(301), preDispatch: true })).toMatchObject({ due: true, reason: "pre-dispatch" }) + }) + + it("P7 429: Retry-After or 120 s x 2^k capped at 960 s, reset on success", () => { + let state = initialRefreshState() + const waits = [] + let at = T0 + for (let k = 0; k < 5; k += 1) { + state = afterAttempt(state, { kind: "rate-limited", retryAfterSeconds: null }, at, 1) + const next = Date.parse(state.next_allowed_at) + waits.push((next - at) / 1000) + expect(due(state, { nowMs: next - 1 }).reason).toBe("backoff") + at = next + } + expect(waits).toEqual([120, 240, 480, 960, 960]) + const honoured = afterAttempt(initialRefreshState(), { kind: "rate-limited", retryAfterSeconds: 700 }, T0, 1) + expect(Date.parse(honoured.next_allowed_at) - T0).toBe(700_000) + expect(afterAttempt(state, { kind: "success" }, at, 1).backoff_k).toBe(0) + }) + + it("P8 an auth failure makes no call until the credentials file changes", () => { + const state = afterAttempt(ok(T0 - 3600_000), { kind: "auth-failed" }, T0, 42) + expect(due(state, { nowMs: s(100000), credentialsMtimeMs: 42 }).reason).toBe("auth-failed") + expect(due(state, { nowMs: s(100000), credentialsMtimeMs: 43 }).due).toBe(true) + }) + + it("property: two network attempts are never closer than the hard minimum", () => { + fc.assert( + fc.property( + fc.array( + fc.record({ + step: fc.integer({ min: 1, max: 400 }), + active: fc.boolean(), + pre: fc.boolean(), + pct: fc.option(fc.integer({ min: 0, max: 100 })), + outcome: fc.constantFrom("success", "rate-limited", "error") + }), + { maxLength: 60 } + ), + (steps) => { + let state = initialRefreshState() + let now = T0 + let last = null + for (const step of steps) { + now += step.step * 1000 + const d = due(state, { nowMs: now, active: step.active, preDispatch: step.pre, bindingMaxPct: step.pct }) + if (!d.due) continue + if (last !== null && now - last < REFRESH_POLICY.minSpacingSeconds * 1000) return false + last = now + state = afterAttempt(state, { kind: step.outcome, retryAfterSeconds: null }, now, 1) + } + return true + } + ) + ) + }) + + it("property: next_at is never at or before now when a read is not due", () => { + fc.assert( + fc.property(fc.integer({ min: 0, max: 5000 }), fc.boolean(), fc.integer({ min: -2000, max: 5000 }), (after, active, reset) => { + const d = due(ok(T0), { nowMs: s(after), active, resets: [new Date(s(reset)).toISOString()] }) + return d.due || d.next_at === null || Date.parse(d.next_at) > s(after) + }) + ) + }) +}) diff --git a/pkgs/substrate-apps/src/apps/pusher/test/seatCapacity.test.mjs b/pkgs/substrate-apps/src/apps/pusher/test/seatCapacity.test.mjs new file mode 100644 index 00000000..b53cf5ee --- /dev/null +++ b/pkgs/substrate-apps/src/apps/pusher/test/seatCapacity.test.mjs @@ -0,0 +1,494 @@ +// The seat-capacity pusher: snapshot building from today's meter shapes, the +// refresh policy, the push, and a round trip into the real Factory handler. +import { spawnSync } from "node:child_process" +import { mkdtempSync, readdirSync, readFileSync, rmSync, statSync, writeFileSync, mkdirSync } from "node:fs" +import { tmpdir } from "node:os" +import { join } from "node:path" +import { fileURLToPath } from "node:url" +import { Schema } from "effect" +import { describe, expect, it } from "vitest" + +import { + buildSnapshot, + decidePush, + fingerprint, + readConfig, + readState, + runOnce, + stateAfterPush +} from "../src/seatCapacity.mjs" +import { SeatCapacitySnapshot } from "@substrate/planning/schema/seatCapacity.ts" +import { admitSeat } from "@substrate/planning/capacity/project.ts" +import { + emptyFactory, + artifactBytes, + artifactDigest +} from "../../../packages/factory/test/fixtures.ts" +import { makeFactoryHttpHandler, makeTableArtifactHasher } from "@substrate/factory" + +const decodeSnapshot = Schema.decodeUnknownSync(SeatCapacitySnapshot) +const PUSHER = fileURLToPath(new URL("../bin/seat-pusher.mjs", import.meta.url)) +const EXAMPLE_CONFIG = fileURLToPath(new URL("../seat-capacity.example.json", import.meta.url)) + +const OBSERVED = "2026-09-23T04:52:18.363166+00:00" +const NOW = "2026-09-23T04:53:18.000Z" + +// Shapes as MEASURED on 2026-09-23 (SCOUT.md section 2); values synthetic. +const ccCache = (observed = OBSERVED, fable = 100) => ({ + observed_at: observed, + usage: { + five_hour: { utilization: 8, resets_at: "2026-09-23T08:49:59.526442+00:00", used_dollars: 1.5 }, + seven_day: { utilization: 72, resets_at: "2026-09-23T09:59:59.526461+00:00" }, + limits: [ + { kind: "session", group: "session", percent: 8, severity: "normal", resets_at: "2026-09-23T08:49:59.526442+00:00", scope: null, is_active: false }, + { kind: "weekly_all", group: "weekly", percent: 72, severity: "normal", resets_at: "2026-09-23T09:59:59.526461+00:00", scope: null, is_active: false }, + { kind: "weekly_scoped", group: "weekly", percent: fable, severity: fable >= 100 ? "critical" : "warning", resets_at: "2026-09-23T09:59:59.526615+00:00", scope: { model: { id: null, display_name: "Fable" }, surface: null }, is_active: true } + // An unknown kind ("something_new") is no longer dropped: it withdraws + // model_windows_complete and refuses (successor review r4); that case + // lives in unmappedLimits.test.mjs. + ], + spend: { limit_dollars: 999, used_dollars: 123 } + } +}) + +const claudeMeter = (seat, overrides = {}) => ({ + schema_version: "seat-meter/1", + seat, + owner: "tom", + grade: "MEASURED", + observed_at: "2026-09-23T04:53:00Z", + reading_observed_at: "2026-09-23T04:52:18Z", + reading_age_seconds: 42, + source: { kind: "oauth-usage-endpoint", reader: "stamp-receipt.py window", seat, source_observed_at: "2026-09-23T04:52:18Z" }, + utilization_pct: 8, + weekly_utilization_pct: 72, + window: { + kind: "nested", + primary: { minutes: 300, resets_at: "2026-09-23T08:49:59.526442+00:00", utilization_pct: 8 }, + secondary: { minutes: 10080, resets_at: "2026-09-23T09:59:59.526461+00:00", utilization_pct: 72 } + }, + ...overrides +}) + +const fleet = () => { + const dir = mkdtempSync(join(tmpdir(), "seat-pusher-")) + const meters = join(dir, "meters") + mkdirSync(meters) + const put = (name, value) => writeFileSync(join(meters, name), `${JSON.stringify(value)}\n`) + put("cc.json", claudeMeter("cc")) + put(".window-cache-cc.json", ccCache()) + put("cc2.json", claudeMeter("cc2", { grade: "STALE-MEASURED", stale_reason: "the reader could not be run: TimeoutExpired" })) + put(".window-cache-cc2.json", ccCache(OBSERVED, 81)) + put("cc3.json", claudeMeter("cc3", { + stale_reason: "token expired", + reading_observed_at: "2026-09-11T15:15:00Z", + source: { kind: "oauth-usage-endpoint", seat: "cc3", source_observed_at: "2026-09-11T15:15:00Z" }, + window: { kind: "nested", primary: { minutes: 300, resets_at: "2026-09-11T19:00:00Z", utilization_pct: 12 }, secondary: { minutes: 10080, resets_at: "2026-09-12T11:00:00.199042+00:00", utilization_pct: 80 } } + })) + put("codex.json", { + schema_version: "seat-meter/1", seat: "codex", owner: "third-party", grade: "MEASURED", observed_at: "2026-09-23T04:52:18Z", + source: { kind: "codex-rollout-rate-limits", path: "/elsewhere/rollout.jsonl" }, + window: { kind: "rolling", minutes: 10080, resets_at: "2026-09-28T19:07:31Z", utilization_pct: 3 } + }) + put("pi-qwencloud.json", { + schema_version: "exp002-meter/1", seat: "pi-qwencloud", owner: "tom", observed_at: "2026-09-23T04:52:18Z", + capacity: { grade: "UNKNOWN", reason: "TL-17 is unset" }, + source: { hold_record: "/elsewhere/pi-hold.json", kind: "unavailable" }, + window: "UNKNOWN", window_reason: "the hold record at /home/someone/plan/pi-hold.json is not held" + }) + put("gpu-worker.json", { + schema_version: "seat-meter/1", seat: "gpu-worker", owner: "kernel", observed_at: "2026-09-23T04:47:32.509Z", + holders: 0, capacity: 1, running_grade: "MEASURED", window: { kind: "none" } + }) + put("mechanical.json", { schema_version: "seat-meter/1", seat: "mechanical", owner: "kernel", observed_at: "2026-09-23T04:47:32.509Z", window: { kind: "none" } }) + writeFileSync(join(meters, "broken.json"), "{") + return { dir, meters, capacity: join(dir, "capacity"), state: join(dir, "capacity", "pusher-state.json") } +} + +const config = () => readConfig(EXAMPLE_CONFIG) +const build = (paths, now = NOW) => + buildSnapshot({ metersDir: paths.meters, capacityDir: paths.capacity, config: config(), now, list: readdirSync(paths.meters) }) + +describe("building the snapshot from today's meter shapes", () => { + it("decodes against the lake's schema and classifies every seat", () => { + const paths = fleet() + const { snapshot, skipped } = build(paths) + const decoded = decodeSnapshot(snapshot) + expect(decoded.seats.map((seat) => [seat.seat, seat.provider, seat.dispatchable, seat.dispatchable_reason])).toEqual([ + ["cc", "claude", true, null], + ["cc2", "claude", true, null], + ["cc3", "claude", false, "evicted"], + ["codex", "codex", false, "third-party"], + ["gpu-worker", "halogen", true, null], + ["pi-qwencloud", "qwen", true, null] + ]) + expect(skipped).toEqual([ + { seat: "broken", reason: "unreadable: SyntaxError" }, + { seat: "mechanical", reason: "not a compute seat this pusher knows" } + ]) + }) + + it("carries the model-scoped Fable row from the usage cache, and the provider's instant", () => { + const cc = build(fleet()).snapshot.seats.find((seat) => seat.seat === "cc") + expect(cc.observed_at).toBe(OBSERVED) + expect(cc.windows.map((window) => [window.kind, window.model, window.utilization_pct, window.severity])).toEqual([ + ["five_hour", null, 8, "normal"], + ["seven_day", null, 72, "normal"], + ["model_scoped", "Fable", 100, "critical"] + ]) + }) + + it("never copies spend, or anything else outside limits[], from the cache", () => { + const text = JSON.stringify(build(fleet()).snapshot) + expect(text).not.toMatch(/spend|dollars|123|999/) + expect(text).not.toContain("/elsewhere/") + }) + + it("a STALE-MEASURED row is a MEASURED reading that has aged; the lake judges its age", () => { + const cc2 = build(fleet()).snapshot.seats.find((seat) => seat.seat === "cc2") + expect(cc2.grade).toBe("MEASURED") + expect(cc2.stale_reason).toBe("the reader could not be run: TimeoutExpired") + }) + + it("falls back to the meter row when the cache is older than the reading", () => { + const paths = fleet() + writeFileSync(join(paths.meters, ".window-cache-cc.json"), JSON.stringify(ccCache("2026-09-23T03:00:00Z"))) + const cc = build(paths).snapshot.seats.find((seat) => seat.seat === "cc") + expect(cc.source.detail).toBe("meter row window") + expect(cc.observed_at).toBe("2026-09-23T04:52:18Z") + // The cache's model rows are carried forward as UNKNOWN (review round 1). + expect(cc.windows.map((window) => window.kind)).toEqual(["five_hour", "seven_day", "model_scoped"]) + expect(cc.model_windows_complete).toBe(false) + }) + + it("an auth failure makes a seat not dispatchable even without a config ruling", () => { + const paths = fleet() + // cc3 is ruled out by the built-in rulings; cc2 carries the auth failure here. + writeFileSync(join(paths.meters, "cc2.json"), JSON.stringify(claudeMeter("cc2", { stale_reason: "token expired" }))) + const { snapshot } = buildSnapshot({ + metersDir: paths.meters, capacityDir: null, config: readConfig(null), now: NOW, list: readdirSync(paths.meters) + }) + const cc2 = snapshot.seats.find((seat) => seat.seat === "cc2") + expect([cc2.dispatchable, cc2.dispatchable_reason]).toEqual([false, "auth-failed"]) + expect(snapshot.seats.find((seat) => seat.seat === "pi-qwencloud").plan).toBeNull() + }) + + it("prefers the feeder's own seats.json when it writes one", () => { + const paths = fleet() + mkdirSync(paths.capacity, { recursive: true }) + const written = { ...build(paths).snapshot, host: "from-feeder", seats: [] } + writeFileSync(join(paths.capacity, "seats.json"), JSON.stringify(written)) + expect(build(paths)).toMatchObject({ source: "seats.json", snapshot: { host: "from-feeder" } }) + }) + + it("carries the qwen plan end from the config file", () => { + const qwen = build(fleet()).snapshot.seats.find((seat) => seat.seat === "pi-qwencloud") + expect(qwen.plan).toEqual({ expires_at: "2026-11-07T00:00:00Z" }) + expect(qwen.windows).toEqual([]) + expect(qwen.stale_reason).toBe("the hold record at is not held") + }) +}) + +describe("the refresh policy", () => { + const minute = 60_000 + const at = Date.parse(NOW) + + it("pushes the first time, then only on a changed reading or the 30 minute heartbeat", () => { + const paths = fleet() + const first = build(paths).snapshot + expect(decidePush({}, first, at)).toMatchObject({ push: true, reason: "changed" }) + const state = stateAfterPush(first, NOW) + // A restamp: the meter row's observed_at moves, the reading does not. + writeFileSync(join(paths.meters, "cc.json"), JSON.stringify(claudeMeter("cc", { observed_at: "2026-09-23T04:55:00Z" }))) + const restamped = build(paths, new Date(at + 5 * minute).toISOString()).snapshot + expect(decidePush(state, restamped, at + 5 * minute)).toMatchObject({ push: false, reason: "unchanged" }) + expect(decidePush(state, restamped, at + 30 * minute)).toMatchObject({ push: true, reason: "heartbeat" }) + // A new reading on one seat. + writeFileSync(join(paths.meters, ".window-cache-cc.json"), JSON.stringify(ccCache("2026-09-23T04:57:18Z"))) + const read = build(paths, new Date(at + 5 * minute).toISOString()).snapshot + expect(decidePush(state, read, at + 5 * minute)).toMatchObject({ push: true, reason: "changed", changed: ["cc"] }) + }) + + it("never pushes twice within the minimum interval, and notices a removed seat", () => { + const snapshot = build(fleet()).snapshot + const state = stateAfterPush(snapshot, NOW) + const fewer = { ...snapshot, seats: snapshot.seats.slice(1) } + expect(decidePush(state, fewer, at + 30_000)).toMatchObject({ push: false, reason: "min-interval" }) + expect(decidePush(state, fewer, at + 2 * minute)).toMatchObject({ push: true, changed: ["cc"] }) + }) + + it("the fingerprint ignores how an instant is spelled", () => { + const cc = build(fleet()).snapshot.seats[0] + expect(fingerprint({ ...cc, observed_at: "2026-09-23T06:52:18.363+02:00" })).toBe( + fingerprint({ ...cc, observed_at: "2026-09-23T04:52:18.363Z" }) + ) + }) +}) + +describe("pushing", () => { + const recorder = (status = 200, body = { results: [], changed: true }) => { + const calls = [] + const fetchImpl = async (url, init) => { + calls.push({ url, init }) + return new Response(JSON.stringify(body), { status }) + } + return { calls, fetchImpl } + } + const pass = (paths, fetchImpl, now = NOW) => + runOnce({ + metersDir: paths.meters, capacityDir: paths.capacity, statePath: paths.state, config: config(), + base: "https://lake.invalid/", token: "secret-token-value", now: () => now, + list: readdirSync(paths.meters), fetchImpl + }) + + it("posts once, keeps a 0600 state, and posts nothing on the next pass", async () => { + const paths = fleet() + const { calls, fetchImpl } = recorder() + expect((await pass(paths, fetchImpl)).pushed).toBe(true) + expect(calls).toHaveLength(1) + expect(calls[0].url).toBe("https://lake.invalid/capacity/seats") + expect(calls[0].init.headers.authorization).toBe("Bearer secret-token-value") + expect(statSync(paths.state).mode & 0o777).toBe(0o600) + const second = await pass(paths, fetchImpl, "2026-09-23T04:58:18.000Z") + expect(second).toMatchObject({ pushed: false, decision: { reason: "unchanged" } }) + expect(calls).toHaveLength(1) + }) + + it("a refused push keeps no state and never prints the token", async () => { + const paths = fleet() + const { fetchImpl } = recorder(401, { error: "Unauthorized" }) + const failure = await pass(paths, fetchImpl).then(() => null, (error) => error) + expect(failure.message).toContain("401") + expect(failure.message).not.toContain("secret-token-value") + expect(readState(paths.state)).toEqual({}) + }) + + it("round trip: the pusher's snapshot lands in the real Factory handler and answers admission", async () => { + const paths = fleet() + const factory = await emptyFactory() + const handler = makeFactoryHttpHandler({ + factory, + token: "secret-token-value", + artifactHasher: makeTableArtifactHasher([[artifactBytes, artifactDigest]]), + now: () => NOW + }) + const fetchImpl = (url, init) => handler(new Request(url, init)) + const result = await pass(paths, fetchImpl) + expect(result.results.every((entry) => entry.outcome === "accepted")).toBe(true) + const ask = async (query) => (await handler(new Request(`https://lake.invalid/capacity/admit?${query}`))).json() + expect(await ask("seat=cc&model=Opus")).toMatchObject({ admit: true, headroom_pct: 28 }) + expect(await ask("seat=cc&model=Fable")).toMatchObject({ admit: false, reason: "severity-critical" }) + expect(await ask("seat=cc2&model=Fable")).toMatchObject({ admit: true, signal: "SLOW" }) + expect(await ask("seat=cc3&model=Opus")).toMatchObject({ admit: false, reason: "not-dispatchable" }) + expect(await ask("seat=codex")).toMatchObject({ admit: false, reason: "not-dispatchable" }) + expect(await ask("seat=pi-qwencloud")).toMatchObject({ admit: false, reason: "unknown" }) + expect(await ask("seat=gpu-worker")).toMatchObject({ admit: true }) + // A repeat pass with --force is idempotent at the lake. + const again = await runOnce({ + metersDir: paths.meters, capacityDir: paths.capacity, statePath: paths.state, config: config(), + base: "https://lake.invalid", token: "secret-token-value", now: () => NOW, + list: readdirSync(paths.meters), fetchImpl, force: true + }) + expect(again.results.every((entry) => entry.outcome === "duplicate")).toBe(true) + }) +}) + +describe("the CLI", () => { + it("--dry-run prints the snapshot and touches no network and no state", () => { + const paths = fleet() + const run = spawnSync(process.execPath, [PUSHER, "--dry-run", "--meters", paths.meters, "--capacity-dir", "", "--config", EXAMPLE_CONFIG], { encoding: "utf8" }) + expect(run.status).toBe(0) + const out = JSON.parse(run.stdout) + expect(out.pushed).toBe(false) + expect(out.snapshot.schema_version).toBe("seat-capacity/2") + expect(out.snapshot.host).toBe("coordinator") + expect(() => statSync(paths.state)).toThrow() + }) + + it("refuses to run without a url and a token file", () => { + const run = spawnSync(process.execPath, [PUSHER], { encoding: "utf8" }) + expect(run.status).toBe(2) + expect(run.stderr).toContain("--url and --token-file are required") + }) +}) + +describe("review round 1: the pusher never drops a model limit silently", () => { + const FABLE = { model: "Fable", min_headroom_pct: 0 } + const ANY = { model: null, min_headroom_pct: 0 } + const AT = "2026-09-23T04:53:30Z" + const ccOf = (paths) => decodeSnapshot(build(paths).snapshot).seats.find((seat) => seat.seat === "cc") + + it("the usage cache read from the same provider call states every model limit", () => { + const cc = ccOf(fleet()) + expect(cc.model_windows_complete).toBe(true) + expect(admitSeat("cc", cc, { model: "Opus", min_headroom_pct: 0 }, AT).admit).toBe(true) + }) + + it("a meter row newer than the cache carries the Fable row forward as UNKNOWN and refuses Fable", () => { + const paths = fleet() + // The cache (and its Fable row at 100 percent critical) is from 04:40; the + // meter row's provider read is 04:52:18 (the feeder tests' shape). + writeFileSync(join(paths.meters, ".window-cache-cc.json"), JSON.stringify(ccCache("2026-09-23T04:40:00Z"))) + const cc = ccOf(paths) + expect(cc.model_windows_complete).toBe(false) + expect(cc.windows.find((window) => window.kind === "model_scoped")).toMatchObject({ + model: "Fable", + utilization_pct: null, + severity: null, + grade: "UNKNOWN" + }) + expect(admitSeat("cc", cc, FABLE, AT)).toMatchObject({ admit: false, reason: "window-unknown", raise_demand: true }) + expect(admitSeat("cc", cc, ANY, AT).admit).toBe(true) + }) + + it("a cache 1 s behind the meter row refuses Fable", () => { + const paths = fleet() + writeFileSync(join(paths.meters, ".window-cache-cc.json"), JSON.stringify(ccCache("2026-09-23T04:52:17Z"))) + writeFileSync(join(paths.meters, "cc.json"), JSON.stringify(claudeMeter("cc", { + reading_observed_at: "2026-09-23T04:52:18Z", + source: { kind: "oauth-usage-endpoint", source_observed_at: "2026-09-23T04:52:18Z" } + }))) + expect(admitSeat("cc", ccOf(paths), FABLE, AT)).toMatchObject({ admit: false, reason: "window-unknown" }) + }) + + it("a torn (unparseable) or missing cache refuses every model job and still admits a job for no model", () => { + for (const cache of ['{"observed_at": "2026-09-23T04:52:18.365', null]) { + const paths = fleet() + const cachePath = join(paths.meters, ".window-cache-cc.json") + if (cache === null) rmSync(cachePath) + else writeFileSync(cachePath, cache) + const cc = ccOf(paths) + expect(cc.model_windows_complete).toBe(false) + expect(cc.windows.map((window) => window.kind)).toEqual(["five_hour", "seven_day"]) + expect(admitSeat("cc", cc, FABLE, AT)).toMatchObject({ admit: false, reason: "window-unknown" }) + expect(admitSeat("cc", cc, { model: "Opus", min_headroom_pct: 0 }, AT)).toMatchObject({ admit: false, reason: "window-unknown" }) + expect(admitSeat("cc", cc, ANY, AT).admit).toBe(true) + } + }) +}) + +describe("review round 1: the ruled seats are never dispatchable, config or not", () => { + const withCoordinatorGpu = () => { + const paths = fleet() + writeFileSync(join(paths.meters, "gpu-coordinator.json"), JSON.stringify({ + schema_version: "seat-meter/1", seat: "gpu-coordinator", owner: "kernel", observed_at: "2026-09-23T04:52:18Z", + holders: 0, capacity: 1, running_grade: "MEASURED", window: { kind: "none" } + })) + // cc3 after a re-login: no auth failure left to catch it. + writeFileSync(join(paths.meters, "cc3.json"), JSON.stringify(claudeMeter("cc3"))) + writeFileSync(join(paths.meters, ".window-cache-cc3.json"), JSON.stringify(ccCache())) + return paths + } + const dispatch = (snapshot) => + Object.fromEntries(snapshot.seats.map((seat) => [seat.seat, [seat.dispatchable, seat.dispatchable_reason]])) + + it("with no config file, cc3 and gpu-coordinator are published not dispatchable and refused", () => { + const paths = withCoordinatorGpu() + const { snapshot } = buildSnapshot({ + metersDir: paths.meters, capacityDir: null, config: readConfig(null), now: NOW, list: readdirSync(paths.meters) + }) + const seats = decodeSnapshot(snapshot).seats + expect(dispatch(snapshot)["cc3"]).toEqual([false, "evicted"]) + expect(dispatch(snapshot)["gpu-coordinator"][0]).toBe(false) + for (const id of ["cc3", "gpu-coordinator"]) { + expect(admitSeat(id, seats.find((seat) => seat.seat === id), { model: null, min_headroom_pct: 0 }, NOW)).toMatchObject({ + admit: false, + reason: "not-dispatchable" + }) + } + }) + + it("a config can extend the rulings but never lift one", () => { + const dir = mkdtempSync(join(tmpdir(), "seat-config-")) + const path = join(dir, "seat-capacity.json") + writeFileSync(path, JSON.stringify({ not_dispatchable: { cc3: null, "gpu-coordinator": false, cc2: "paused" } })) + const ruled = readConfig(path).notDispatchable + expect(typeof ruled.cc3).toBe("string") + expect(typeof ruled["gpu-coordinator"]).toBe("string") + expect(ruled.cc2).toBe("paused") + }) + + it("the feeder's seats.json is held to the same rulings, seat filter and path scrub", () => { + const paths = fleet() + mkdirSync(paths.capacity, { recursive: true }) + const built = build(paths).snapshot + const written = { + ...built, + seats: built.seats.map((seat) => + seat.seat === "cc3" + ? { ...seat, dispatchable: true, dispatchable_reason: null, stale_reason: "read /home/tom/.claude/x failed" } + : seat + ) + } + writeFileSync(join(paths.capacity, "seats.json"), JSON.stringify(written)) + const cfg = { ...readConfig(EXAMPLE_CONFIG), seats: ["cc", "cc3"] } + const { snapshot, source } = buildSnapshot({ + metersDir: paths.meters, capacityDir: paths.capacity, config: cfg, now: NOW, list: readdirSync(paths.meters) + }) + expect(source).toBe("seats.json") + expect(snapshot.seats.map((seat) => seat.seat)).toEqual(["cc", "cc3"]) + const cc3 = snapshot.seats.find((seat) => seat.seat === "cc3") + expect([cc3.dispatchable, cc3.dispatchable_reason]).toEqual([false, "evicted"]) + expect(cc3.stale_reason).toBe("read failed") + expect(() => decodeSnapshot(snapshot)).not.toThrow() + }) + + it("--dry-run reads the same default config path as a real run", () => { + const paths = fleet() + const home = mkdtempSync(join(tmpdir(), "seat-home-")) + mkdirSync(join(home, ".config", "substrate"), { recursive: true }) + writeFileSync(join(home, ".config", "substrate", "seat-capacity.json"), JSON.stringify({ host: "from-default-config" })) + const run = spawnSync(process.execPath, [PUSHER, "--dry-run", "--meters", paths.meters, "--capacity-dir", ""], { + encoding: "utf8", + env: { ...process.env, HOME: home } + }) + expect(run.status).toBe(0) + expect(JSON.parse(run.stdout).snapshot.host).toBe("from-default-config") + }) +}) + +describe("review round 1: a restamp is not a change for any provider", () => { + const minute = 60_000 + const at = Date.parse(NOW) + const restampAll = (paths, instant) => { + for (const name of ["codex.json", "pi-qwencloud.json", "gpu-worker.json"]) { + const path = join(paths.meters, name) + const row = JSON.parse(readFileSync(path, "utf8")) + writeFileSync(path, JSON.stringify({ ...row, observed_at: instant })) + } + } + + it("moving only the codex, qwen and halogen rows' observed_at does not push", () => { + const paths = fleet() + const first = build(paths).snapshot + const state = stateAfterPush(first, NOW) + restampAll(paths, "2026-09-23T04:57:48Z") + const later = build(paths, new Date(at + 5 * minute).toISOString()).snapshot + expect(decidePush(state, later, at + 5 * minute)).toMatchObject({ push: false, reason: "unchanged" }) + // No provider instant exists in those rows; the snapshot says so. + for (const id of ["codex", "pi-qwencloud"]) { + expect(later.seats.find((seat) => seat.seat === id).source.detail).toBe("observed_at is the feeder's read instant") + } + }) + + it("a changed codex utilization or halogen holder count still pushes", () => { + const paths = fleet() + const state = stateAfterPush(build(paths).snapshot, NOW) + const path = join(paths.meters, "gpu-worker.json") + writeFileSync(path, JSON.stringify({ ...JSON.parse(readFileSync(path, "utf8")), holders: 1 })) + expect(decidePush(state, build(paths).snapshot, at + 2 * minute)).toMatchObject({ push: true, changed: ["gpu-worker"] }) + }) + + it("a dispatchable slot row is re-sent before the lake would grade it STALE (1200 s)", () => { + const paths = fleet() + const state = stateAfterPush(build(paths).snapshot, NOW) + restampAll(paths, "2026-09-23T05:03:18Z") + const snapshot = build(paths).snapshot + expect(decidePush(state, snapshot, at + 9 * minute)).toMatchObject({ push: false }) + expect(decidePush(state, snapshot, at + 10 * minute)).toMatchObject({ push: true, reason: "heartbeat" }) + // Without a dispatchable slot row, the 1800 s heartbeat stands. + const noSlots = { ...snapshot, seats: snapshot.seats.filter((seat) => seat.provider !== "halogen") } + const stateNoSlots = stateAfterPush({ ...build(paths).snapshot, seats: noSlots.seats }, NOW) + expect(decidePush(stateNoSlots, noSlots, at + 10 * minute)).toMatchObject({ push: false, reason: "unchanged" }) + }) +}) diff --git a/pkgs/substrate-apps/src/apps/pusher/test/unmappedLimits.test.mjs b/pkgs/substrate-apps/src/apps/pusher/test/unmappedLimits.test.mjs new file mode 100644 index 00000000..b7bd1007 --- /dev/null +++ b/pkgs/substrate-apps/src/apps/pusher/test/unmappedLimits.test.mjs @@ -0,0 +1,32 @@ +// Successor review r4: the pusher placed limit rows the way capacity round 3 +// placed them in src/capacity/tally-meters.ts. A row it cannot place (an +// unknown kind, a scoped row naming no model) is a binding seven_day window +// graded UNKNOWN with its reset kept, and model_windows_complete is withdrawn. +import { describe, expect, it } from "vitest" +import { windowsFromLimits, windowsFromUsage, limitsComplete } from "../src/usageSource.mjs" + +const reset = "2026-09-23T09:59:59Z" +const limits = [ + { kind: "session", percent: 10, resets_at: reset, severity: "normal", scope: null }, + { kind: "weekly_all", percent: 20, resets_at: reset, severity: "normal", scope: null }, + { kind: "weekly_scoped", percent: 100, resets_at: reset, severity: "critical", scope: { model: null, surface: "claude_code" } }, + { kind: "something_new", percent: 100, resets_at: reset, severity: "critical", scope: null } +] + +describe("r4: the pusher never drops a limit row it cannot place", () => { + it("unknown kind and a model-less scoped row become UNKNOWN seven_day windows with their resets", () => { + const { windows, unmapped } = windowsFromLimits(limits) + expect(unmapped).toBe(2) + const unknown = windows.filter((w) => w.grade === "UNKNOWN") + expect(unknown).toHaveLength(2) + for (const w of unknown) { + expect(w).toMatchObject({ kind: "seven_day", model: null, binding: true, utilization_pct: null, resets_at: reset }) + } + expect(windowsFromUsage({ limits })).toEqual(windows) + }) + it("model_windows_complete is claimed only when every row was placed", () => { + expect(limitsComplete({ limits })).toBe(false) + expect(limitsComplete({ limits: limits.slice(0, 2) })).toBe(true) + expect(limitsComplete({ five_hour: {}, seven_day: {} })).toBe(false) + }) +}) diff --git a/pkgs/substrate-apps/src/apps/pusher/test/usageSource.test.mjs b/pkgs/substrate-apps/src/apps/pusher/test/usageSource.test.mjs new file mode 100644 index 00000000..e9fad6b9 --- /dev/null +++ b/pkgs/substrate-apps/src/apps/pusher/test/usageSource.test.mjs @@ -0,0 +1,298 @@ +// The Claude usage source: the direct reader under the refresh policy, the +// cache-file adapter, and a snapshot built with no tally path at all. +import { spawnSync } from "node:child_process" +import { mkdirSync, mkdtempSync, readFileSync, statSync, utimesSync, writeFileSync } from "node:fs" +import { tmpdir } from "node:os" +import { join } from "node:path" +import { fileURLToPath } from "node:url" +import { Schema } from "effect" +import { describe, expect, it } from "vitest" + +import { SeatCapacitySnapshot } from "@substrate/planning/schema/seatCapacity.ts" +import { admitSeat } from "@substrate/planning/capacity/project.ts" +import { buildSnapshot, defaultConfigPath, defaultPusherDir, readConfig, runOnce } from "../src/seatCapacity.mjs" +import { + bearerFromCredentials, + defaultUsageDir, + readSeatUsage, + USAGE_ENDPOINT, + windowsFromUsage +} from "../src/usageSource.mjs" + +const decode = Schema.decodeUnknownSync(SeatCapacitySnapshot) +const PUSHER = fileURLToPath(new URL("../bin/seat-pusher.mjs", import.meta.url)) +const FAKE = "fake-bearer-for-tests-only" +const T0 = Date.parse("2026-09-23T06:00:00Z") + +// The endpoint's answer shape as MEASURED in the feeder cache (SCOUT.md section 2); values synthetic. +const answer = (fable = 100) => ({ + five_hour: { utilization: 8, resets_at: "2026-09-23T08:49:59.526442+00:00" }, + seven_day: { utilization: 72, resets_at: "2026-09-23T09:59:59.526461+00:00" }, + limits: [ + { kind: "session", percent: 8, severity: "normal", resets_at: "2026-09-23T08:49:59.526442+00:00", scope: null }, + { kind: "weekly_all", percent: 72, severity: "normal", resets_at: "2026-09-23T09:59:59.526461+00:00", scope: null }, + { kind: "weekly_scoped", percent: fable, severity: "critical", resets_at: "2026-09-23T09:59:59.526615+00:00", scope: { model: { id: null, display_name: "Fable" } } } + ], + spend: { limit_dollars: 999, used_dollars: 123 } +}) + +const box = () => { + const dir = mkdtempSync(join(tmpdir(), "usage-source-")) + const credentials = join(dir, "credentials.json") + writeFileSync(credentials, JSON.stringify({ claudeAiOauth: { accessToken: FAKE } })) + return { dir, credentials, usageDir: join(dir, "usage"), demandDir: join(dir, "demand") } +} + +const fakeFetch = (responses) => { + const calls = [] + const impl = async (url, init) => { + calls.push({ url, init }) + const next = responses.shift() ?? { status: 200, body: answer() } + return { + status: next.status, + headers: { get: (name) => (name === "retry-after" ? (next.retryAfter ?? null) : null) }, + text: async () => JSON.stringify(next.body ?? {}) + } + } + return { impl, calls } +} + +const read = (b, fetchImpl, nowMs, extra = {}) => + readSeatUsage({ + seat: "cc", + source: { kind: "oauth", credentials: b.credentials }, + usageDir: b.usageDir, + demandDir: b.demandDir, + nowMs, + fetchImpl, + ...extra + }) + +describe("windowsFromUsage", () => { + it("maps limits[] to five_hour, seven_day and a model-scoped row", () => { + const windows = windowsFromUsage(answer()) + expect(windows.map((w) => [w.kind, w.model, w.utilization_pct])).toEqual([ + ["five_hour", null, 8], + ["seven_day", null, 72], + ["model_scoped", "Fable", 100] + ]) + }) + it("falls back to the flat pair when limits[] is absent", () => { + const { limits, ...flat } = answer() + expect(windowsFromUsage(flat).map((w) => [w.kind, w.utilization_pct])).toEqual([ + ["five_hour", 8], + ["seven_day", 72] + ]) + }) +}) + +describe("bearerFromCredentials", () => { + it("reads the pointer and never puts the value in an error", () => { + const b = box() + expect(bearerFromCredentials(b.credentials)).toBe(FAKE) + let message = "" + try { + bearerFromCredentials(b.credentials, ["claudeAiOauth", "refreshToken"]) + } catch (error) { + message = error.message + } + expect(message).toMatch(/refreshToken/) + expect(message).not.toContain(FAKE) + }) +}) + +describe("the direct reader", () => { + it("reads once, sends the bearer only in the header, caches 0600 without spend", async () => { + const b = box() + const f = fakeFetch([{ status: 200, body: answer() }]) + const r = await read(b, f.impl, T0) + expect(r.attempted).toBe(true) + expect(f.calls).toHaveLength(1) + expect(f.calls[0].url).toBe(USAGE_ENDPOINT) + expect(f.calls[0].init.headers.authorization).toBe(`Bearer ${FAKE}`) + expect(f.calls[0].init.headers["anthropic-beta"]).toBe("oauth-2025-04-20") + const cache = readFileSync(join(b.usageDir, "cc.json"), "utf8") + expect(cache).not.toContain("spend") + expect(cache).not.toContain(FAKE) + expect(readFileSync(join(b.usageDir, "cc.state.json"), "utf8")).not.toContain(FAKE) + expect(statSync(join(b.usageDir, "cc.json")).mode & 0o777).toBe(0o600) + expect(r.reading.observed_at).toBe(new Date(T0).toISOString()) + }) + + it("is gentle: no second call inside the idle floor, one at 900 s, 300 s when active", async () => { + const b = box() + const f = fakeFetch([]) + await read(b, f.impl, T0) + for (const after of [30, 119, 120, 600, 899]) await read(b, f.impl, T0 + after * 1000) + expect(f.calls).toHaveLength(1) + const idle = await read(b, f.impl, T0 + 900_000) + expect(idle.decision.reason).toBe("idle") + expect(f.calls).toHaveLength(2) + mkdirSync(b.demandDir, { recursive: true }) + writeFileSync(join(b.demandDir, "cc"), "") + const markerAt = (T0 + 1_200_000) / 1000 + utimesSync(join(b.demandDir, "cc"), markerAt, markerAt) + const active = await read(b, f.impl, T0 + 1_200_000) + expect(active.decision.reason).toBe("active") + expect(f.calls).toHaveLength(3) + }) + + it("backs off after a 429, honouring Retry-After, and keeps serving the cached reading", async () => { + const b = box() + const f = fakeFetch([{ status: 200, body: answer() }, { status: 429, retryAfter: "1500" }]) + await read(b, f.impl, T0) + const limited = await read(b, f.impl, T0 + 900_000) + expect(limited.attempted).toBe(true) + expect(limited.error).toMatch(/429/) + expect(limited.reading.observed_at).toBe(new Date(T0).toISOString()) + await read(b, f.impl, T0 + (900 + 1499) * 1000) + expect(f.calls).toHaveLength(2) + await read(b, f.impl, T0 + (900 + 1500) * 1000) + expect(f.calls).toHaveLength(3) + }) + + it("after an auth failure, calls again only once the credentials file changes", async () => { + const b = box() + const f = fakeFetch([{ status: 401 }]) + const failed = await read(b, f.impl, T0) + expect(failed.error).toMatch(/401/) + await read(b, f.impl, T0 + 86_400_000) + expect(f.calls).toHaveLength(1) + const later = (T0 + 90_000_000) / 1000 + utimesSync(b.credentials, later, later) + await read(b, f.impl, T0 + 90_000_000) + expect(f.calls).toHaveLength(2) + }) + + it("a dry run never calls the network, even with a first read due", async () => { + const b = box() + const f = fakeFetch([]) + const config = readConfig(null) + config.usage = { cc: { kind: "oauth", credentials: b.credentials } } + const dry = await runOnce({ + metersDir: null, + capacityDir: null, + statePath: join(b.dir, "state.json"), + config, + base: "", + token: "", + now: () => new Date(T0).toISOString(), + fetchImpl: f.impl, + dryRun: true, + usageDir: b.usageDir, + demandDir: b.demandDir + }) + expect(f.calls).toHaveLength(0) + expect(dry.skipped.map((entry) => entry.seat)).toEqual(["cc"]) + }) + + it("never calls the network when told not to", async () => { + const b = box() + const f = fakeFetch([]) + const r = await read(b, f.impl, T0, { allowNetwork: false }) + expect(f.calls).toHaveLength(0) + expect(r.reading).toBeNull() + }) +}) + +describe("the snapshot without tally-rewrite", () => { + it("builds from usage sources alone, decodes, and admission honours the model-scoped row", async () => { + const b = box() + const cacheFile = join(b.dir, "feeder-cache.json") + writeFileSync(cacheFile, JSON.stringify({ observed_at: "2026-09-23T05:59:00Z", usage: answer(81) })) + const config = readConfig(null) + config.host = "coordinator" + config.usage = { cc: { kind: "oauth", credentials: b.credentials }, cc2: { kind: "cache-file", path: cacheFile } } + const f = fakeFetch([{ status: 200, body: answer(100) }]) + const result = await runOnce({ + metersDir: null, + capacityDir: null, + statePath: join(b.dir, "state.json"), + config, + base: "http://floor.invalid", + token: "floor-token", + now: () => new Date(T0).toISOString(), + fetchImpl: f.impl, + dryRun: false, + usageDir: b.usageDir, + demandDir: b.demandDir + }).catch((error) => ({ error: error.message })) + // The push itself fails against the fake (it answers usage JSON, not the floor's); + // what matters here is the usage read, below, and the dry-run snapshot. + expect(f.calls[0].url).toBe(USAGE_ENDPOINT) + const dry = await runOnce({ + metersDir: null, + capacityDir: null, + statePath: join(b.dir, "state.json"), + config, + base: "", + token: "", + now: () => new Date(T0 + 1000).toISOString(), + fetchImpl: f.impl, + dryRun: true, + usageDir: b.usageDir, + demandDir: b.demandDir + }) + expect(result).toBeDefined() + expect(f.calls).toHaveLength(2) // the failed push, never a second usage read + expect(dry.source).toBe("usage") + const snapshot = decode(dry.snapshot) + expect(snapshot.seats.map((seat) => [seat.seat, seat.source.detail])).toEqual([ + ["cc", "direct read"], + ["cc2", "usage cache file"] + ]) + expect(JSON.stringify(snapshot)).not.toContain("spend") + const asOf = new Date(T0 + 2000).toISOString() + const cc = snapshot.seats[0] + expect(admitSeat("cc", cc, { model: "opus-class", min_headroom_pct: 0 }, asOf).admit).toBe(true) + expect(admitSeat("cc", cc, { model: "Fable", min_headroom_pct: 0 }, asOf).admit).toBe(false) + expect(cc.model_windows_complete).toBe(true) + }) + + it("a reading without limits[] does not claim every model limit, so a model job is refused", () => { + const b = box() + const { limits, ...flat } = answer() + const cacheFile = join(b.dir, "flat.json") + writeFileSync(cacheFile, JSON.stringify({ observed_at: new Date(T0).toISOString(), usage: flat })) + const config = readConfig(null) + config.usage = { cc: { kind: "cache-file", path: cacheFile } } + const usage = { cc: { reading: { observed_at: new Date(T0).toISOString(), usage: flat }, error: null } } + const seat = decode(buildSnapshot({ config, now: new Date(T0).toISOString(), usage }).snapshot).seats[0] + expect(seat.model_windows_complete).toBe(false) + const asOf = new Date(T0 + 1000).toISOString() + const refused = admitSeat("cc", seat, { model: "Fable", min_headroom_pct: 0 }, asOf) + expect(refused.admit).toBe(false) + expect(refused.reason).toBe("window-unknown") + expect(admitSeat("cc", seat, { model: null, min_headroom_pct: 0 }, asOf).admit).toBe(true) + }) + + it("the example configs name no real credentials path (tests load the first one)", () => { + const example = fileURLToPath(new URL("../seat-capacity.example.json", import.meta.url)) + expect(readConfig(example).usage).toEqual({}) + const usageExample = readConfig(fileURLToPath(new URL("../seat-capacity.usage.example.json", import.meta.url))) + for (const source of Object.values(usageExample.usage)) { + for (const value of Object.values(source)) expect(String(value).startsWith("/")).toBe(false) + } + }) + + it("names no tally path in any default", () => { + for (const path of [defaultConfigPath(), defaultPusherDir(), defaultUsageDir()]) { + expect(path).not.toMatch(/tally/) + } + const built = buildSnapshot({ config: readConfig(null), now: new Date(T0).toISOString(), list: ["cc.json"] }) + expect(built.snapshot.seats).toEqual([]) + }) + + it("the CLI dry run reads a cache-file source and calls no network", () => { + const b = box() + const cacheFile = join(b.dir, "cache.json") + writeFileSync(cacheFile, JSON.stringify({ observed_at: "2026-09-23T05:59:00Z", usage: answer() })) + const configPath = join(b.dir, "config.json") + writeFileSync(configPath, JSON.stringify({ host: "coordinator", usage: { cc: { kind: "cache-file", path: cacheFile } } })) + const run = spawnSync(process.execPath, [PUSHER, "--dry-run", "--config", configPath], { encoding: "utf8" }) + expect(run.status).toBe(0) + const out = JSON.parse(run.stdout) + expect(out.pushed).toBe(false) + expect(decode(out.snapshot).seats.map((seat) => seat.seat)).toEqual(["cc"]) + }) +}) diff --git a/pkgs/substrate-apps/src/apps/pusher/vitest.config.ts b/pkgs/substrate-apps/src/apps/pusher/vitest.config.ts new file mode 100644 index 00000000..1d909a98 --- /dev/null +++ b/pkgs/substrate-apps/src/apps/pusher/vitest.config.ts @@ -0,0 +1,8 @@ +import { defineConfig } from "vitest/config" + +export default defineConfig({ + test: { + include: ["test/**/*.test.mjs", "test/**/*.test.ts"], + environment: "node" + } +}) diff --git a/pkgs/substrate-apps/src/package.json b/pkgs/substrate-apps/src/package.json new file mode 100644 index 00000000..34a14c37 --- /dev/null +++ b/pkgs/substrate-apps/src/package.json @@ -0,0 +1,31 @@ +{ + "name": "substrate", + "version": "0.1.0", + "private": true, + "type": "module", + "description": "CONWIP factory: turns Claude ultracode workflow run records into ax Tasks under a WIP cap, plus the Effect workspace (schema, serializer, planning, factory) and the Cloudflare floor.", + "scripts": { + "test": "vitest run", + "typecheck": "tsc --noEmit -p . && tsc --noEmit -p packages/schema && tsc --noEmit -p packages/serializer && tsc --noEmit -p packages/planning && tsc --noEmit -p packages/factory && tsc --noEmit -p apps/floor && tsc --noEmit -p packages/interpreter && tsc --noEmit -p packages/runners && tsc --noEmit -p apps/link", + "check:fences": "node tools/check-fences.mjs", + "check": "pnpm typecheck && pnpm check:fences && pnpm test", + "conwip": "tsx src/cli.ts", + "conwip-run": "tsx src/run-cli.ts" + }, + "dependencies": { + "@grpc/grpc-js": "^1.12.6", + "@grpc/proto-loader": "^0.7.13", + "effect": "4.0.0-rc.112", + "@substrate/planning": "workspace:*", + "@substrate/schema": "workspace:*", + "@substrate/interpreter": "workspace:*", + "@substrate/runners": "workspace:*" + }, + "devDependencies": { + "@types/node": "^22.10.2", + "tsx": "^4.19.2", + "typescript": "^5.9.3", + "vitest": "^3.2.4", + "fast-check": "^4.9.0" + } +} diff --git a/pkgs/substrate-apps/src/pnpm-lock.yaml b/pkgs/substrate-apps/src/pnpm-lock.yaml new file mode 100644 index 00000000..05608238 --- /dev/null +++ b/pkgs/substrate-apps/src/pnpm-lock.yaml @@ -0,0 +1,2551 @@ +lockfileVersion: '9.0' + +settings: + autoInstallPeers: true + excludeLinksFromLockfile: false + +importers: + + .: + dependencies: + '@grpc/grpc-js': + specifier: ^1.12.6 + version: 1.14.5 + '@grpc/proto-loader': + specifier: ^0.7.13 + version: 0.7.15 + '@substrate/interpreter': + specifier: workspace:* + version: link:packages/interpreter + '@substrate/planning': + specifier: workspace:* + version: link:packages/planning + '@substrate/runners': + specifier: workspace:* + version: link:packages/runners + '@substrate/schema': + specifier: workspace:* + version: link:packages/schema + effect: + specifier: 4.0.0-rc.112 + version: 4.0.0-rc.112 + devDependencies: + '@types/node': + specifier: ^22.10.2 + version: 22.20.4 + fast-check: + specifier: ^4.9.0 + version: 4.10.2 + tsx: + specifier: ^4.19.2 + version: 4.23.15 + typescript: + specifier: ^5.9.3 + version: 5.9.3 + vitest: + specifier: ^3.2.4 + version: 3.2.7(@types/node@22.20.4)(supports-color@10.2.2)(tsx@4.23.15) + + apps/floor: + dependencies: + '@substrate/factory': + specifier: workspace:* + version: link:../../packages/factory + '@substrate/interpreter': + specifier: workspace:* + version: link:../../packages/interpreter + '@substrate/link': + specifier: workspace:* + version: link:../link + '@substrate/planning': + specifier: workspace:* + version: link:../../packages/planning + devDependencies: + '@cloudflare/vitest-pool-workers': + specifier: 0.9.14 + version: 0.9.14(@vitest/runner@3.2.7)(@vitest/snapshot@3.2.7)(vitest@3.2.7(@types/node@22.20.4)(supports-color@10.2.2)(tsx@4.23.15)) + '@types/node': + specifier: ^22.10.2 + version: 22.20.4 + effect: + specifier: 4.0.0-rc.112 + version: 4.0.0-rc.112 + fast-check: + specifier: ^4.9.0 + version: 4.10.2 + typescript: + specifier: ^5.9.3 + version: 5.9.3 + vitest: + specifier: ^3.2.4 + version: 3.2.7(@types/node@22.20.4)(supports-color@10.2.2)(tsx@4.23.15) + + apps/link: + dependencies: + '@grpc/grpc-js': + specifier: ^1.12.6 + version: 1.14.5 + '@grpc/proto-loader': + specifier: ^0.7.13 + version: 0.7.15 + devDependencies: + '@types/node': + specifier: ^22.10.2 + version: 22.20.4 + effect: + specifier: 4.0.0-rc.112 + version: 4.0.0-rc.112 + typescript: + specifier: ^5.9.3 + version: 5.9.3 + vitest: + specifier: ^3.2.4 + version: 3.2.7(@types/node@22.20.4)(supports-color@10.2.2)(tsx@4.23.15) + + apps/pusher: + devDependencies: + '@substrate/factory': + specifier: workspace:* + version: link:../../packages/factory + '@substrate/planning': + specifier: workspace:* + version: link:../../packages/planning + effect: + specifier: 4.0.0-rc.112 + version: 4.0.0-rc.112 + fast-check: + specifier: ^4.9.0 + version: 4.10.2 + vitest: + specifier: ^3.2.4 + version: 3.2.7(@types/node@22.20.4)(supports-color@10.2.2)(tsx@4.23.15) + + packages/factory: + dependencies: + '@substrate/planning': + specifier: workspace:* + version: link:../planning + '@substrate/schema': + specifier: workspace:* + version: link:../schema + '@substrate/serializer': + specifier: workspace:* + version: link:../serializer + devDependencies: + effect: + specifier: 4.0.0-rc.112 + version: 4.0.0-rc.112 + fast-check: + specifier: ^4.9.0 + version: 4.10.2 + typescript: + specifier: ^5.9.3 + version: 5.9.3 + vitest: + specifier: ^3.2.4 + version: 3.2.7(@types/node@22.20.4)(supports-color@10.2.2)(tsx@4.23.15) + + packages/interpreter: + dependencies: + ajv: + specifier: ^8.17.1 + version: 8.20.0 + devDependencies: + '@types/node': + specifier: ^22.10.2 + version: 22.20.4 + effect: + specifier: 4.0.0-rc.112 + version: 4.0.0-rc.112 + tsx: + specifier: ^4.19.2 + version: 4.23.15 + typescript: + specifier: ^5.9.3 + version: 5.9.3 + vitest: + specifier: ^3.2.4 + version: 3.2.7(@types/node@22.20.4)(supports-color@10.2.2)(tsx@4.23.15) + + packages/planning: + devDependencies: + effect: + specifier: 4.0.0-rc.112 + version: 4.0.0-rc.112 + fast-check: + specifier: ^4.9.0 + version: 4.10.2 + typescript: + specifier: ^5.9.3 + version: 5.9.3 + vitest: + specifier: ^3.2.4 + version: 3.2.7(@types/node@22.20.4)(supports-color@10.2.2)(tsx@4.23.15) + + packages/runners: + dependencies: + smol-toml: + specifier: ^1.4.2 + version: 1.8.0 + devDependencies: + '@types/node': + specifier: ^22.10.2 + version: 22.20.4 + effect: + specifier: 4.0.0-rc.112 + version: 4.0.0-rc.112 + tsx: + specifier: ^4.19.2 + version: 4.23.15 + typescript: + specifier: ^5.9.3 + version: 5.9.3 + vitest: + specifier: ^3.2.4 + version: 3.2.7(@types/node@22.20.4)(supports-color@10.2.2)(tsx@4.23.15) + + packages/schema: + devDependencies: + '@substrate/serializer': + specifier: workspace:* + version: link:../serializer + '@types/node': + specifier: ^22.10.2 + version: 22.20.4 + effect: + specifier: 4.0.0-rc.112 + version: 4.0.0-rc.112 + fast-check: + specifier: ^4.9.0 + version: 4.10.2 + typescript: + specifier: ^5.9.3 + version: 5.9.3 + vitest: + specifier: ^3.2.4 + version: 3.2.7(@types/node@22.20.4)(supports-color@10.2.2)(tsx@4.23.15) + + packages/serializer: + devDependencies: + effect: + specifier: 4.0.0-rc.112 + version: 4.0.0-rc.112 + fast-check: + specifier: ^4.9.0 + version: 4.10.2 + typescript: + specifier: ^5.9.3 + version: 5.9.3 + vitest: + specifier: ^3.2.4 + version: 3.2.7(@types/node@22.20.4)(supports-color@10.2.2)(tsx@4.23.15) + +packages: + + '@cloudflare/kv-asset-handler@0.4.0': + resolution: {integrity: sha512-+tv3z+SPp+gqTIcImN9o0hqE9xyfQjI1XD9pL6NuKjua9B1y7mNYv0S9cP+QEbA4ppVgGZEmKOvHX5G5Ei1CVA==} + engines: {node: '>=18.0.0'} + + '@cloudflare/unenv-preset@2.7.8': + resolution: {integrity: sha512-Ky929MfHh+qPhwCapYrRPwPVHtA2Ioex/DbGZyskGyNRDe9Ru3WThYZivyNVaPy5ergQSgMs9OKrM9Ajtz9F6w==} + peerDependencies: + unenv: 2.0.0-rc.21 + workerd: ^1.20250927.0 + peerDependenciesMeta: + workerd: + optional: true + + '@cloudflare/vitest-pool-workers@0.9.14': + resolution: {integrity: sha512-1D/1KFa32EymaYDojWcrmUyMbHJwewn/mZSxsAqXdybvKMrHVaoaqGpLoH9bgWSfbrGs4af4RrmIXMINOqc4Lw==} + peerDependencies: + '@vitest/runner': 2.0.x - 3.2.x + '@vitest/snapshot': 2.0.x - 3.2.x + vitest: 2.0.x - 3.2.x + + '@cloudflare/workerd-darwin-64@1.20251011.0': + resolution: {integrity: sha512-0DirVP+Z82RtZLlK2B+VhLOkk+ShBqDYO/jhcRw4oVlp0TOvk3cOVZChrt3+y3NV8Y/PYgTEywzLKFSziK4wCg==} + engines: {node: '>=16'} + cpu: [x64] + os: [darwin] + + '@cloudflare/workerd-darwin-arm64@1.20251011.0': + resolution: {integrity: sha512-1WuFBGwZd15p4xssGN/48OE2oqokIuc51YvHvyNivyV8IYnAs3G9bJNGWth1X7iMDPe4g44pZrKhRnISS2+5dA==} + engines: {node: '>=16'} + cpu: [arm64] + os: [darwin] + + '@cloudflare/workerd-linux-64@1.20251011.0': + resolution: {integrity: sha512-BccMiBzFlWZyFghIw2szanmYJrJGBGHomw2y/GV6pYXChFzMGZkeCEMfmCyJj29xczZXxcZmUVJxNy4eJxO8QA==} + engines: {node: '>=16'} + cpu: [x64] + os: [linux] + + '@cloudflare/workerd-linux-arm64@1.20251011.0': + resolution: {integrity: sha512-79o/216lsbAbKEVDZYXR24ivEIE2ysDL9jvo0rDTkViLWju9dAp3CpyetglpJatbSi3uWBPKZBEOqN68zIjVsQ==} + engines: {node: '>=16'} + cpu: [arm64] + os: [linux] + + '@cloudflare/workerd-windows-64@1.20251011.0': + resolution: {integrity: sha512-RIXUQRchFdqEvaUqn1cXZXSKjpqMaSaVAkI5jNZ8XzAw/bw2bcdOVUtakrflgxDprltjFb0PTNtuss1FKtH9Jg==} + engines: {node: '>=16'} + cpu: [x64] + os: [win32] + + '@cspotcode/source-map-support@0.8.1': + resolution: {integrity: sha512-IchNf6dN4tHoMFIn/7OE8LWZ19Y6q/67Bmf6vnGREv8RSbBVb9LPJxEcnwrcwX6ixSvaiGoomAUvu4YSxXrVgw==} + engines: {node: '>=12'} + + '@emnapi/runtime@1.11.3': + resolution: {integrity: sha512-Xz4Tpyki7XyrpbUK1jR1AhdAdaXyhhY4lZ3neLodmhpuWfy2PAQN5B46sAiU4liOXGLkHypn/qU+jvfWSCYYLA==} + + '@esbuild/aix-ppc64@0.25.4': + resolution: {integrity: sha512-1VCICWypeQKhVbE9oW/sJaAmjLxhVqacdkvPLEjwlttjfwENRSClS8EjBz0KzRyFSCPDIkuXW34Je/vk7zdB7Q==} + engines: {node: '>=18'} + cpu: [ppc64] + os: [aix] + + '@esbuild/aix-ppc64@0.28.2': + resolution: {integrity: sha512-XExcO+dvLKvVtNTibSTBej1NCAbaGhWn9Ww1ZPx80qsahhPFe/8jgWP0IchNe0F3HwkU7n8ejhH8bjonqht8mQ==} + engines: {node: '>=18'} + cpu: [ppc64] + os: [aix] + + '@esbuild/android-arm64@0.25.4': + resolution: {integrity: sha512-bBy69pgfhMGtCnwpC/x5QhfxAz/cBgQ9enbtwjf6V9lnPI/hMyT9iWpR1arm0l3kttTr4L0KSLpKmLp/ilKS9A==} + engines: {node: '>=18'} + cpu: [arm64] + os: [android] + + '@esbuild/android-arm64@0.28.2': + resolution: {integrity: sha512-5YfKeeI8qWfBZIX+u2xZC3Zlb3Os/gLS2sbEKM+I4ZOcsWmHS2WLysCcQZDAFRslDUU5Oiq44gf6PYN1vGwG5A==} + engines: {node: '>=18'} + cpu: [arm64] + os: [android] + + '@esbuild/android-arm@0.25.4': + resolution: {integrity: sha512-QNdQEps7DfFwE3hXiU4BZeOV68HHzYwGd0Nthhd3uCkkEKK7/R6MTgM0P7H7FAs5pU/DIWsviMmEGxEoxIZ+ZQ==} + engines: {node: '>=18'} + cpu: [arm] + os: [android] + + '@esbuild/android-arm@0.28.2': + resolution: {integrity: sha512-kXXoiPVVGQcnIYGOeaovwOURpniDBpSq4A03qkQ+BMQqtGG6HYap3xne9C1O1yo4TR3qxlCX5IqqmX6fFo2Lqg==} + engines: {node: '>=18'} + cpu: [arm] + os: [android] + + '@esbuild/android-x64@0.25.4': + resolution: {integrity: sha512-TVhdVtQIFuVpIIR282btcGC2oGQoSfZfmBdTip2anCaVYcqWlZXGcdcKIUklfX2wj0JklNYgz39OBqh2cqXvcQ==} + engines: {node: '>=18'} + cpu: [x64] + os: [android] + + '@esbuild/android-x64@0.28.2': + resolution: {integrity: sha512-O387ite7SzUyCcy3JQX4P4bLtEA7bLLkx+esve5JHnyYfNTxcVpXZo9jhdB0lTKN44gztELTdU7nS8Nr16Fs1Q==} + engines: {node: '>=18'} + cpu: [x64] + os: [android] + + '@esbuild/darwin-arm64@0.25.4': + resolution: {integrity: sha512-Y1giCfM4nlHDWEfSckMzeWNdQS31BQGs9/rouw6Ub91tkK79aIMTH3q9xHvzH8d0wDru5Ci0kWB8b3up/nl16g==} + engines: {node: '>=18'} + cpu: [arm64] + os: [darwin] + + '@esbuild/darwin-arm64@0.28.2': + resolution: {integrity: sha512-n4KqkOQrraxHJcgjM1RvwbigfQKIKJVpM7xp+KsxiyUSrRdIXnt73VhrPAx0fV44hgfmIVKjxMN9J1t5jySVkw==} + engines: {node: '>=18'} + cpu: [arm64] + os: [darwin] + + '@esbuild/darwin-x64@0.25.4': + resolution: {integrity: sha512-CJsry8ZGM5VFVeyUYB3cdKpd/H69PYez4eJh1W/t38vzutdjEjtP7hB6eLKBoOdxcAlCtEYHzQ/PJ/oU9I4u0A==} + engines: {node: '>=18'} + cpu: [x64] + os: [darwin] + + '@esbuild/darwin-x64@0.28.2': + resolution: {integrity: sha512-uq6suIWYP37qzGddBKPw5QEQPi6HiLGsO7UmkpfyaYNQ3D+rN6w6WfwH+nuqcGXWvawGwxOEroO4YGnFh95azw==} + engines: {node: '>=18'} + cpu: [x64] + os: [darwin] + + '@esbuild/freebsd-arm64@0.25.4': + resolution: {integrity: sha512-yYq+39NlTRzU2XmoPW4l5Ifpl9fqSk0nAJYM/V/WUGPEFfek1epLHJIkTQM6bBs1swApjO5nWgvr843g6TjxuQ==} + engines: {node: '>=18'} + cpu: [arm64] + os: [freebsd] + + '@esbuild/freebsd-arm64@0.28.2': + resolution: {integrity: sha512-n+I0BTSRIoy+d6RPKnEVwql5UwBJolytvY4mAOIEJorKlqgPII8ix6slVVrfZ5Tnj7glIZvloylbB/EJPMWEXw==} + engines: {node: '>=18'} + cpu: [arm64] + os: [freebsd] + + '@esbuild/freebsd-x64@0.25.4': + resolution: {integrity: sha512-0FgvOJ6UUMflsHSPLzdfDnnBBVoCDtBTVyn/MrWloUNvq/5SFmh13l3dvgRPkDihRxb77Y17MbqbCAa2strMQQ==} + engines: {node: '>=18'} + cpu: [x64] + os: [freebsd] + + '@esbuild/freebsd-x64@0.28.2': + resolution: {integrity: sha512-78XJTJkvPs0kz2w61301PJjXl4g7q3JqiYMZ/M/yVI73EHBrCRTgkhu9oqG7vPqq+a/yadEW8aD+agKlk5xrmg==} + engines: {node: '>=18'} + cpu: [x64] + os: [freebsd] + + '@esbuild/linux-arm64@0.25.4': + resolution: {integrity: sha512-+89UsQTfXdmjIvZS6nUnOOLoXnkUTB9hR5QAeLrQdzOSWZvNSAXAtcRDHWtqAUtAmv7ZM1WPOOeSxDzzzMogiQ==} + engines: {node: '>=18'} + cpu: [arm64] + os: [linux] + + '@esbuild/linux-arm64@0.28.2': + resolution: {integrity: sha512-pW4AC0P3it8c7do9MVM4p51FzHzdM/TZrerurgRcHJ2WTa1VQ1CIq18xncfpBJw4ojkiZZrKW2yIBWBP92j6Ug==} + engines: {node: '>=18'} + cpu: [arm64] + os: [linux] + + '@esbuild/linux-arm@0.25.4': + resolution: {integrity: sha512-kro4c0P85GMfFYqW4TWOpvmF8rFShbWGnrLqlzp4X1TNWjRY3JMYUfDCtOxPKOIY8B0WC8HN51hGP4I4hz4AaQ==} + engines: {node: '>=18'} + cpu: [arm] + os: [linux] + + '@esbuild/linux-arm@0.28.2': + resolution: {integrity: sha512-XlDnu2q5yoqems+xay6wSAcg9DDD7K9RLKZEBOMZm3ckNpJBvOX20tSfby8KfrrhINDyv9V2YVZKY/SpoGJI8w==} + engines: {node: '>=18'} + cpu: [arm] + os: [linux] + + '@esbuild/linux-ia32@0.25.4': + resolution: {integrity: sha512-yTEjoapy8UP3rv8dB0ip3AfMpRbyhSN3+hY8mo/i4QXFeDxmiYbEKp3ZRjBKcOP862Ua4b1PDfwlvbuwY7hIGQ==} + engines: {node: '>=18'} + cpu: [ia32] + os: [linux] + + '@esbuild/linux-ia32@0.28.2': + resolution: {integrity: sha512-CYbnj78HsIeA+DhgUKgFCfvNsTHFhMMrinUrMZpDXJXKN8T3XViTZ/+wtHeVxEWY8ewSzTFN+nRmSwO2tZaLUQ==} + engines: {node: '>=18'} + cpu: [ia32] + os: [linux] + + '@esbuild/linux-loong64@0.25.4': + resolution: {integrity: sha512-NeqqYkrcGzFwi6CGRGNMOjWGGSYOpqwCjS9fvaUlX5s3zwOtn1qwg1s2iE2svBe4Q/YOG1q6875lcAoQK/F4VA==} + engines: {node: '>=18'} + cpu: [loong64] + os: [linux] + + '@esbuild/linux-loong64@0.28.2': + resolution: {integrity: sha512-buwkd8nsph4R+ajRvw0qM5Hja/TXQow3ptzWO2EbG/cqcIkHloRrdlBtQlshyYGTNFvfkfJ5tpPLVkY4DtsPfQ==} + engines: {node: '>=18'} + cpu: [loong64] + os: [linux] + + '@esbuild/linux-mips64el@0.25.4': + resolution: {integrity: sha512-IcvTlF9dtLrfL/M8WgNI/qJYBENP3ekgsHbYUIzEzq5XJzzVEV/fXY9WFPfEEXmu3ck2qJP8LG/p3Q8f7Zc2Xg==} + engines: {node: '>=18'} + cpu: [mips64el] + os: [linux] + + '@esbuild/linux-mips64el@0.28.2': + resolution: {integrity: sha512-ZVykbDyk7519VwiNb9Lcj9m8XM6v5V9uKPvrEMkkEedVewf+0itkhahp4HDpgERXhwLRpWFypsGbG/J8s0QjJA==} + engines: {node: '>=18'} + cpu: [mips64el] + os: [linux] + + '@esbuild/linux-ppc64@0.25.4': + resolution: {integrity: sha512-HOy0aLTJTVtoTeGZh4HSXaO6M95qu4k5lJcH4gxv56iaycfz1S8GO/5Jh6X4Y1YiI0h7cRyLi+HixMR+88swag==} + engines: {node: '>=18'} + cpu: [ppc64] + os: [linux] + + '@esbuild/linux-ppc64@0.28.2': + resolution: {integrity: sha512-CAXl+Dtd9UUuJd8pKKdwh6MLm3MUMiqMPmhZ3tTSXPqfyQ3vDl6R5hZdZ/kYojK4ofXtdfSv1tFq8XzWx3heNQ==} + engines: {node: '>=18'} + cpu: [ppc64] + os: [linux] + + '@esbuild/linux-riscv64@0.25.4': + resolution: {integrity: sha512-i8JUDAufpz9jOzo4yIShCTcXzS07vEgWzyX3NH2G7LEFVgrLEhjwL3ajFE4fZI3I4ZgiM7JH3GQ7ReObROvSUA==} + engines: {node: '>=18'} + cpu: [riscv64] + os: [linux] + + '@esbuild/linux-riscv64@0.28.2': + resolution: {integrity: sha512-GeXCej4IQtU1B+QlDV8W/RRvbzI3O/Stss+/bCXv4lZls5WGRtu2a+3JkA3i4qIUlMXpcHebWpF8AkJhATowuA==} + engines: {node: '>=18'} + cpu: [riscv64] + os: [linux] + + '@esbuild/linux-s390x@0.25.4': + resolution: {integrity: sha512-jFnu+6UbLlzIjPQpWCNh5QtrcNfMLjgIavnwPQAfoGx4q17ocOU9MsQ2QVvFxwQoWpZT8DvTLooTvmOQXkO51g==} + engines: {node: '>=18'} + cpu: [s390x] + os: [linux] + + '@esbuild/linux-s390x@0.28.2': + resolution: {integrity: sha512-3H1weTYZPxt/WOhByszQZybS9w5lKzUn1FDMsgEChbHWQwHYQQRfBxgCcZvPhjHfKyJjIievvMmEUawJrdY9Dg==} + engines: {node: '>=18'} + cpu: [s390x] + os: [linux] + + '@esbuild/linux-x64@0.25.4': + resolution: {integrity: sha512-6e0cvXwzOnVWJHq+mskP8DNSrKBr1bULBvnFLpc1KY+d+irZSgZ02TGse5FsafKS5jg2e4pbvK6TPXaF/A6+CA==} + engines: {node: '>=18'} + cpu: [x64] + os: [linux] + + '@esbuild/linux-x64@0.28.2': + resolution: {integrity: sha512-4xTZr1FUmSoQW4XIWmit3tzQrUTZM+N3P0XV8xROKYF50XfI7xeO90+1bZvNwxIufQ9hDQVRJH5YhgPVF8A/HQ==} + engines: {node: '>=18'} + cpu: [x64] + os: [linux] + + '@esbuild/netbsd-arm64@0.25.4': + resolution: {integrity: sha512-vUnkBYxZW4hL/ie91hSqaSNjulOnYXE1VSLusnvHg2u3jewJBz3YzB9+oCw8DABeVqZGg94t9tyZFoHma8gWZQ==} + engines: {node: '>=18'} + cpu: [arm64] + os: [netbsd] + + '@esbuild/netbsd-arm64@0.28.2': + resolution: {integrity: sha512-sSATRjPeDBg3pdgHoQfoYBob11Kk1FGa9lui5RIHZCoCkJa9QKlvl3/vKz2usCmYYjs7ymJR/2Nnsqe+Hjt5nw==} + engines: {node: '>=18'} + cpu: [arm64] + os: [netbsd] + + '@esbuild/netbsd-x64@0.25.4': + resolution: {integrity: sha512-XAg8pIQn5CzhOB8odIcAm42QsOfa98SBeKUdo4xa8OvX8LbMZqEtgeWE9P/Wxt7MlG2QqvjGths+nq48TrUiKw==} + engines: {node: '>=18'} + cpu: [x64] + os: [netbsd] + + '@esbuild/netbsd-x64@0.28.2': + resolution: {integrity: sha512-lqnzCV+mM0gIADaKihiCg6ifgfU2L3h5E33rNQBN1Y4MaVGnzryzmvvf7UHxprpQdE8hpqLolJ9Rl+SkIRDpyw==} + engines: {node: '>=18'} + cpu: [x64] + os: [netbsd] + + '@esbuild/openbsd-arm64@0.25.4': + resolution: {integrity: sha512-Ct2WcFEANlFDtp1nVAXSNBPDxyU+j7+tId//iHXU2f/lN5AmO4zLyhDcpR5Cz1r08mVxzt3Jpyt4PmXQ1O6+7A==} + engines: {node: '>=18'} + cpu: [arm64] + os: [openbsd] + + '@esbuild/openbsd-arm64@0.28.2': + resolution: {integrity: sha512-AL2qJILH7lNjrDmCQDvdxMfAUIv8KMNZOvrwAQ8i8//ntL9FflhOyMJ8OZSMBb8/AWXe3/5v5S20y3zCoZWKoQ==} + engines: {node: '>=18'} + cpu: [arm64] + os: [openbsd] + + '@esbuild/openbsd-x64@0.25.4': + resolution: {integrity: sha512-xAGGhyOQ9Otm1Xu8NT1ifGLnA6M3sJxZ6ixylb+vIUVzvvd6GOALpwQrYrtlPouMqd/vSbgehz6HaVk4+7Afhw==} + engines: {node: '>=18'} + cpu: [x64] + os: [openbsd] + + '@esbuild/openbsd-x64@0.28.2': + resolution: {integrity: sha512-QtiuPytchRyC4rwUKhexJdQKvDuZ6hWloi3igqPQNUJCS1/v9EiO3UTOXR6A3FoMo4fnAKbWJdqaIwhOzh8qEw==} + engines: {node: '>=18'} + cpu: [x64] + os: [openbsd] + + '@esbuild/openharmony-arm64@0.28.2': + resolution: {integrity: sha512-WkhYDmpTjLvGlScA1rwjRUmhl4k8oXR3cIbtqWmELgU/dFeHHlEllxDvdWcNJV9rbzCexB5vz8gtNewWLgCT7Q==} + engines: {node: '>=18'} + cpu: [arm64] + os: [openharmony] + + '@esbuild/sunos-x64@0.25.4': + resolution: {integrity: sha512-Mw+tzy4pp6wZEK0+Lwr76pWLjrtjmJyUB23tHKqEDP74R3q95luY/bXqXZeYl4NYlvwOqoRKlInQialgCKy67Q==} + engines: {node: '>=18'} + cpu: [x64] + os: [sunos] + + '@esbuild/sunos-x64@0.28.2': + resolution: {integrity: sha512-GPMSkTOtMnv2U2F8gxe4Io6qmVs+YKyp832Etqqxr0hFngmXQ3rzwytelm3GIn7T4VviRUlf3sOgBOiTdvaf7g==} + engines: {node: '>=18'} + cpu: [x64] + os: [sunos] + + '@esbuild/win32-arm64@0.25.4': + resolution: {integrity: sha512-AVUP428VQTSddguz9dO9ngb+E5aScyg7nOeJDrF1HPYu555gmza3bDGMPhmVXL8svDSoqPCsCPjb265yG/kLKQ==} + engines: {node: '>=18'} + cpu: [arm64] + os: [win32] + + '@esbuild/win32-arm64@0.28.2': + resolution: {integrity: sha512-PIhhEkE9uPBleRBrQEJpUn7MBnibZzbGzYWPmY3x+YoVg/95zbjB4CxPPOQ8l5tYYM4mMaCthF8/1DIfBQQyWQ==} + engines: {node: '>=18'} + cpu: [arm64] + os: [win32] + + '@esbuild/win32-ia32@0.25.4': + resolution: {integrity: sha512-i1sW+1i+oWvQzSgfRcxxG2k4I9n3O9NRqy8U+uugaT2Dy7kLO9Y7wI72haOahxceMX8hZAzgGou1FhndRldxRg==} + engines: {node: '>=18'} + cpu: [ia32] + os: [win32] + + '@esbuild/win32-ia32@0.28.2': + resolution: {integrity: sha512-YmJbfTlvU7Sdn9BB+4PRES4oB6pxgS37MAONj+hBr/cpXS1aBPKXxNnDbu+QCWPj0o9dgyxeq79g6c5P8KeuYA==} + engines: {node: '>=18'} + cpu: [ia32] + os: [win32] + + '@esbuild/win32-x64@0.25.4': + resolution: {integrity: sha512-nOT2vZNw6hJ+z43oP1SPea/G/6AbN6X+bGNhNuq8NtRHy4wsMhw765IKLNmnjek7GvjWBYQ8Q5VBoYTFg9y1UQ==} + engines: {node: '>=18'} + cpu: [x64] + os: [win32] + + '@esbuild/win32-x64@0.28.2': + resolution: {integrity: sha512-5ebpxr3nWMzrL/rnUI755Jkuee0bHL/Gq0WTF9lvcpv73wAp5eu8MfBUgWK9bhWvZjj7yX8etf/8tI8Ney695g==} + engines: {node: '>=18'} + cpu: [x64] + os: [win32] + + '@grpc/grpc-js@1.14.5': + resolution: {integrity: sha512-7VZM+SVdEcUUqSQeNI3zM8Qs/BhQKZndPo2h5VkYkAM8Iz0wJIa8mKV5ekQGqG8UUsnkQ0NMxIxwkIHYvj0qOw==} + engines: {node: '>=12.10.0'} + + '@grpc/proto-loader@0.7.15': + resolution: {integrity: sha512-tMXdRCfYVixjuFK+Hk0Q1s38gV9zDiDJfWL3h1rv4Qc39oILCu1TRTDt7+fGUI8K4G1Fj125Hx/ru3azECWTyQ==} + engines: {node: '>=6'} + hasBin: true + + '@grpc/proto-loader@0.8.1': + resolution: {integrity: sha512-wtF6h+DY6M3YaDBPAmvuuA6jV8Sif9MjtOI5euKFWRgCDl5PeDpPsHR9u2l6St5ceY8AZgoNDww5+HvEsXFsGg==} + engines: {node: '>=6'} + hasBin: true + + '@img/sharp-darwin-arm64@0.33.5': + resolution: {integrity: sha512-UT4p+iz/2H4twwAoLCqfA9UH5pI6DggwKEGuaPy7nCVQ8ZsiY5PIcrRvD1DzuY3qYL07NtIQcWnBSY/heikIFQ==} + engines: {node: ^18.17.0 || ^20.3.0 || >=21.0.0} + cpu: [arm64] + os: [darwin] + + '@img/sharp-darwin-x64@0.33.5': + resolution: {integrity: sha512-fyHac4jIc1ANYGRDxtiqelIbdWkIuQaI84Mv45KvGRRxSAa7o7d1ZKAOBaYbnepLC1WqxfpimdeWfvqqSGwR2Q==} + engines: {node: ^18.17.0 || ^20.3.0 || >=21.0.0} + cpu: [x64] + os: [darwin] + + '@img/sharp-libvips-darwin-arm64@1.0.4': + resolution: {integrity: sha512-XblONe153h0O2zuFfTAbQYAX2JhYmDHeWikp1LM9Hul9gVPjFY427k6dFEcOL72O01QxQsWi761svJ/ev9xEDg==} + cpu: [arm64] + os: [darwin] + + '@img/sharp-libvips-darwin-x64@1.0.4': + resolution: {integrity: sha512-xnGR8YuZYfJGmWPvmlunFaWJsb9T/AO2ykoP3Fz/0X5XV2aoYBPkX6xqCQvUTKKiLddarLaxpzNe+b1hjeWHAQ==} + cpu: [x64] + os: [darwin] + + '@img/sharp-libvips-linux-arm64@1.0.4': + resolution: {integrity: sha512-9B+taZ8DlyyqzZQnoeIvDVR/2F4EbMepXMc/NdVbkzsJbzkUjhXv/70GQJ7tdLA4YJgNP25zukcxpX2/SueNrA==} + cpu: [arm64] + os: [linux] + libc: [glibc] + + '@img/sharp-libvips-linux-arm@1.0.5': + resolution: {integrity: sha512-gvcC4ACAOPRNATg/ov8/MnbxFDJqf/pDePbBnuBDcjsI8PssmjoKMAz4LtLaVi+OnSb5FK/yIOamqDwGmXW32g==} + cpu: [arm] + os: [linux] + libc: [glibc] + + '@img/sharp-libvips-linux-s390x@1.0.4': + resolution: {integrity: sha512-u7Wz6ntiSSgGSGcjZ55im6uvTrOxSIS8/dgoVMoiGE9I6JAfU50yH5BoDlYA1tcuGS7g/QNtetJnxA6QEsCVTA==} + cpu: [s390x] + os: [linux] + libc: [glibc] + + '@img/sharp-libvips-linux-x64@1.0.4': + resolution: {integrity: sha512-MmWmQ3iPFZr0Iev+BAgVMb3ZyC4KeFc3jFxnNbEPas60e1cIfevbtuyf9nDGIzOaW9PdnDciJm+wFFaTlj5xYw==} + cpu: [x64] + os: [linux] + libc: [glibc] + + '@img/sharp-libvips-linuxmusl-arm64@1.0.4': + resolution: {integrity: sha512-9Ti+BbTYDcsbp4wfYib8Ctm1ilkugkA/uscUn6UXK1ldpC1JjiXbLfFZtRlBhjPZ5o1NCLiDbg8fhUPKStHoTA==} + cpu: [arm64] + os: [linux] + libc: [musl] + + '@img/sharp-libvips-linuxmusl-x64@1.0.4': + resolution: {integrity: sha512-viYN1KX9m+/hGkJtvYYp+CCLgnJXwiQB39damAO7WMdKWlIhmYTfHjwSbQeUK/20vY154mwezd9HflVFM1wVSw==} + cpu: [x64] + os: [linux] + libc: [musl] + + '@img/sharp-linux-arm64@0.33.5': + resolution: {integrity: sha512-JMVv+AMRyGOHtO1RFBiJy/MBsgz0x4AWrT6QoEVVTyh1E39TrCUpTRI7mx9VksGX4awWASxqCYLCV4wBZHAYxA==} + engines: {node: ^18.17.0 || ^20.3.0 || >=21.0.0} + cpu: [arm64] + os: [linux] + libc: [glibc] + + '@img/sharp-linux-arm@0.33.5': + resolution: {integrity: sha512-JTS1eldqZbJxjvKaAkxhZmBqPRGmxgu+qFKSInv8moZ2AmT5Yib3EQ1c6gp493HvrvV8QgdOXdyaIBrhvFhBMQ==} + engines: {node: ^18.17.0 || ^20.3.0 || >=21.0.0} + cpu: [arm] + os: [linux] + libc: [glibc] + + '@img/sharp-linux-s390x@0.33.5': + resolution: {integrity: sha512-y/5PCd+mP4CA/sPDKl2961b+C9d+vPAveS33s6Z3zfASk2j5upL6fXVPZi7ztePZ5CuH+1kW8JtvxgbuXHRa4Q==} + engines: {node: ^18.17.0 || ^20.3.0 || >=21.0.0} + cpu: [s390x] + os: [linux] + libc: [glibc] + + '@img/sharp-linux-x64@0.33.5': + resolution: {integrity: sha512-opC+Ok5pRNAzuvq1AG0ar+1owsu842/Ab+4qvU879ippJBHvyY5n2mxF1izXqkPYlGuP/M556uh53jRLJmzTWA==} + engines: {node: ^18.17.0 || ^20.3.0 || >=21.0.0} + cpu: [x64] + os: [linux] + libc: [glibc] + + '@img/sharp-linuxmusl-arm64@0.33.5': + resolution: {integrity: sha512-XrHMZwGQGvJg2V/oRSUfSAfjfPxO+4DkiRh6p2AFjLQztWUuY/o8Mq0eMQVIY7HJ1CDQUJlxGGZRw1a5bqmd1g==} + engines: {node: ^18.17.0 || ^20.3.0 || >=21.0.0} + cpu: [arm64] + os: [linux] + libc: [musl] + + '@img/sharp-linuxmusl-x64@0.33.5': + resolution: {integrity: sha512-WT+d/cgqKkkKySYmqoZ8y3pxx7lx9vVejxW/W4DOFMYVSkErR+w7mf2u8m/y4+xHe7yY9DAXQMWQhpnMuFfScw==} + engines: {node: ^18.17.0 || ^20.3.0 || >=21.0.0} + cpu: [x64] + os: [linux] + libc: [musl] + + '@img/sharp-wasm32@0.33.5': + resolution: {integrity: sha512-ykUW4LVGaMcU9lu9thv85CbRMAwfeadCJHRsg2GmeRa/cJxsVY9Rbd57JcMxBkKHag5U/x7TSBpScF4U8ElVzg==} + engines: {node: ^18.17.0 || ^20.3.0 || >=21.0.0} + cpu: [wasm32] + + '@img/sharp-win32-ia32@0.33.5': + resolution: {integrity: sha512-T36PblLaTwuVJ/zw/LaH0PdZkRz5rd3SmMHX8GSmR7vtNSP5Z6bQkExdSK7xGWyxLw4sUknBuugTelgw2faBbQ==} + engines: {node: ^18.17.0 || ^20.3.0 || >=21.0.0} + cpu: [ia32] + os: [win32] + + '@img/sharp-win32-x64@0.33.5': + resolution: {integrity: sha512-MpY/o8/8kj+EcnxwvrP4aTJSWw/aZ7JIGR4aBeZkZw5B7/Jn+tY9/VNwtcoGmdT7GfggGIU4kygOMSbYnOrAbg==} + engines: {node: ^18.17.0 || ^20.3.0 || >=21.0.0} + cpu: [x64] + os: [win32] + + '@jridgewell/resolve-uri@3.1.2': + resolution: {integrity: sha512-bRISgCIjP20/tbWSPWMEi54QVPRZExkuD9lJL+UIxUKtwVJA8wW1Trb1jMs1RFXo1CBTNZ/5hpC9QvmKWdopKw==} + engines: {node: '>=6.0.0'} + + '@jridgewell/sourcemap-codec@1.6.0': + resolution: {integrity: sha512-T7jf+5zgsZHwNJ4lvQ7/aezbyk0nNX+zJVWpmHA7VYsEx7a7qr5Rg5IbtJFqkgze5Y2sruq1RUY8Q837Od7iFw==} + + '@jridgewell/trace-mapping@0.3.9': + resolution: {integrity: sha512-3Belt6tdc8bPgAtbcmdtNJlirVoTmEb5e2gC94PnkwEW9jI6CAHUeoG85tjWP5WquqfavoMtMwiG4P926ZKKuQ==} + + '@js-sdsl/ordered-map@4.4.2': + resolution: {integrity: sha512-iUKgm52T8HOE/makSxjqoWhe95ZJA1/G1sYsGev2JDKUSS14KAgg1LHb+Ba+IPow0xflbnSkOsZcO08C7w1gYw==} + + '@msgpackr-extract/msgpackr-extract-darwin-arm64@3.0.4': + resolution: {integrity: sha512-LCkGo6JDfaBhgST7UpPWgNgLINpcpabaHfyz5OBx75nUYxBsaEPxjnyNjWpeb/xBup/682QnBfRBy2/LvPutZQ==} + cpu: [arm64] + os: [darwin] + + '@msgpackr-extract/msgpackr-extract-darwin-x64@3.0.4': + resolution: {integrity: sha512-zExlW9zUJKZH/tOtVMttwjKa4Xm/3KcNjnE3dPN92uCktwavMxpgCA3MoJK/DOnTWsQgo224OaST27/mPNAf+w==} + cpu: [x64] + os: [darwin] + + '@msgpackr-extract/msgpackr-extract-linux-arm64@3.0.4': + resolution: {integrity: sha512-dgX0P/9wGPJeHFBG+ZmhgE6bmtMt7NP5CRBGyyktpopdk/mW4POnrpQsSLtKI1dwpc+pPLuXHDh6vvskyQE/sw==} + cpu: [arm64] + os: [linux] + + '@msgpackr-extract/msgpackr-extract-linux-arm@3.0.4': + resolution: {integrity: sha512-Tg3yX65f5GbtXLkrYEHE5oibZG9epyYWas7FogTTEJeDEF9JlXJzKgXaNhT3UXlTOeA+AfZpYZYZ0uPj7Cfquw==} + cpu: [arm] + os: [linux] + + '@msgpackr-extract/msgpackr-extract-linux-x64@3.0.4': + resolution: {integrity: sha512-8TNXMEjJc3QEy7R/x1INhgiU+XakDAFUzBhaz7+Rbrs8NH5UQeHQxxmzsSBJGyV6I1jW79undiQm8tOI+D+8FQ==} + cpu: [x64] + os: [linux] + + '@msgpackr-extract/msgpackr-extract-win32-x64@3.0.4': + resolution: {integrity: sha512-CmCXPQrkbwExx3j946/PtHWHbYJiCRBRDl4BlkRQcJB/YOwQxJRTpoo7aTsortjgoJ1x7opzTSxn7C+ASSLVjQ==} + cpu: [x64] + os: [win32] + + '@napi-rs/lzma-linux-x64-gnu@1.5.1': + resolution: {integrity: sha512-oTXEIha4SsuXdTA4Iyskj0kpdx2yVXdhd75c2v3xGrHFfVMsbhTPZU/nMPL4sWKo4pBHm3aucLaqGlF696dTyQ==} + engines: {node: ^22.20 || ^24.12 || >=25} + cpu: [x64] + os: [linux] + libc: [glibc] + + '@poppinss/colors@4.1.6': + resolution: {integrity: sha512-H9xkIdFswbS8n1d6vmRd8+c10t2Qe+rZITbbDHHkQixH5+2x1FDGmi/0K+WgWiqQFKPSlIYB7jlH6Kpfn6Fleg==} + + '@poppinss/dumper@0.6.5': + resolution: {integrity: sha512-NBdYIb90J7LfOI32dOewKI1r7wnkiH6m920puQ3qHUeZkxNkQiFnXVWoE6YtFSv6QOiPPf7ys6i+HWWecDz7sw==} + + '@poppinss/exception@1.2.3': + resolution: {integrity: sha512-dCED+QRChTVatE9ibtoaxc+WkdzOSjYTKi/+uacHWIsfodVfpsueo3+DKpgU5Px8qXjgmXkSvhXvSCz3fnP9lw==} + + '@protobufjs/aspromise@1.1.2': + resolution: {integrity: sha512-j+gKExEuLmKwvz3OgROXtrJ2UG2x8Ch2YZUxahh+s1F2HZ+wAceUNLkvy6zKCPVRkU++ZWQrdxsUeQXmcg4uoQ==} + + '@protobufjs/base64@1.1.2': + resolution: {integrity: sha512-AZkcAA5vnN/v4PDqKyMR5lx7hZttPDgClv83E//FMNhR2TMcLUhfRUBHCmSl0oi9zMgDDqRUJkSxO3wm85+XLg==} + + '@protobufjs/codegen@2.0.5': + resolution: {integrity: sha512-zgXFLzW3Ap33e6d0Wlj4MGIm6Ce8O89n/apUaGNB/jx+hw+ruWEp7EwGUshdLKVRCxZW12fp9r40E1mQrf/34g==} + + '@protobufjs/eventemitter@1.1.1': + resolution: {integrity: sha512-vW1GmwMZNnL+gMRaovlh9yZX74kc+TTU3FObkkurpMaRtBfLP3ldjS9KQWlwZgraRE0+dheEEoAxdzcJQ8eXZg==} + + '@protobufjs/fetch@1.1.1': + resolution: {integrity: sha512-GpptLrs57adMSuHi3VNj0mAF8dwh36LMaYF6XyJ6JMWlVsc+t42tm1HSEDmOs3A8fC9yyeisgLhsTVQokOZ0zw==} + + '@protobufjs/float@1.0.2': + resolution: {integrity: sha512-Ddb+kVXlXst9d+R9PfTIxh1EdNkgoRe5tOX6t01f1lYWOvJnSPDBlG241QLzcyPdoNTsblLUdujGSE4RzrTZGQ==} + + '@protobufjs/path@1.1.2': + resolution: {integrity: sha512-6JOcJ5Tm08dOHAbdR3GrvP+yUUfkjG5ePsHYczMFLq3ZmMkAD98cDgcT2iA1lJ9NVwFd4tH/iSSoe44YWkltEA==} + + '@protobufjs/pool@1.1.0': + resolution: {integrity: sha512-0kELaGSIDBKvcgS4zkjz1PeddatrjYcmMWOlAuAPwAeccUrPHdUqo/J6LiymHHEiJT5NrF1UVwxY14f+fy4WQw==} + + '@protobufjs/utf8@1.1.2': + resolution: {integrity: sha512-b1UQwcEZ4yCnMCD8DAL1VlbvBJE9/IX4FTIp7BG1xYpf29SLazLSrqUkj4w7Y5y7cCVP6E5tcqqcI0xemPkHug==} + + '@rollup/rollup-android-arm-eabi@4.63.4': + resolution: {integrity: sha512-I+BSHzTAhKN2n7ZwGZsegGcZjDpLqFOMAtJz/u6uFGe0pUFbq56dEHjqJV/ZUdRJtNXNxA+hREUatZBvMR3Oiw==} + cpu: [arm] + os: [android] + + '@rollup/rollup-android-arm64@4.63.4': + resolution: {integrity: sha512-pu3BdjS2LtEzRu2elmGzS3fIeWSZy4BMDIaLNwjorO76+k2d0LMluijhsDx3KQyQBQ/lLUZCQA9/s6csvUfuhw==} + cpu: [arm64] + os: [android] + + '@rollup/rollup-darwin-arm64@4.63.4': + resolution: {integrity: sha512-xfSrj9MHnWK9GaSqT9U0ImHtH/N8WZlHLx4cZHiuLcqs640hvZ3hLPd5UR2AZS57FaE8HrRUSpltbZdWRxHiDA==} + cpu: [arm64] + os: [darwin] + + '@rollup/rollup-darwin-x64@4.63.4': + resolution: {integrity: sha512-bqU99PLJb/dqb3S0GIMdeuyAEETSUgZBoqXYd3Sd+WCsV+MmPhnN6JrotWyir31+QgH7EvvE5/mwGJlEoci8Fw==} + cpu: [x64] + os: [darwin] + + '@rollup/rollup-freebsd-arm64@4.63.4': + resolution: {integrity: sha512-JinsFZ5G40oXQb+sUuiA5x689vhr6dDYK0H0NL+rwKdL6CqnmYN8PE4ZwfRSoIjrCxqTQG/SLfTtSvHeGxoVlw==} + cpu: [arm64] + os: [freebsd] + + '@rollup/rollup-freebsd-x64@4.63.4': + resolution: {integrity: sha512-GAdA4UxpiNm27cLHr2GqXBpAD0x9FqwYBY7/YSP0Ss0/PNi4k8gbviqpIpYbVSRBaS2ZcegXEzgTQMbRNCwxCw==} + cpu: [x64] + os: [freebsd] + + '@rollup/rollup-linux-arm-gnueabihf@4.63.4': + resolution: {integrity: sha512-qDd6NoA1znaLjp4jR5U/KWCdLAKDJNB8W9ChbbDaKbo0xA+Atln5HK6LFCZ4oJQpemtRZA288DCirFRjrspptw==} + cpu: [arm] + os: [linux] + libc: [glibc] + + '@rollup/rollup-linux-arm-musleabihf@4.63.4': + resolution: {integrity: sha512-WtB5Tz5KTNINb8ZA+8sQ7bmjuS1JrRT7YverYIhUGdWWDlpzVWmIwuZE+jidkEXUn1l0zrEkaIMa8dHF3NGcsA==} + cpu: [arm] + os: [linux] + libc: [musl] + + '@rollup/rollup-linux-arm64-gnu@4.63.4': + resolution: {integrity: sha512-VcQ3L1tjnkKzWjryAVaFhHEWcqOfICX9uxVVoDzm2t0DpgKRHd2zOpVrJc0xsWeBZcBFyYROCIBdyR/fS174pg==} + cpu: [arm64] + os: [linux] + libc: [glibc] + + '@rollup/rollup-linux-arm64-musl@4.63.4': + resolution: {integrity: sha512-6+ZQX6P5s0cMDN2Ypb8Lbm2+/sZYmZjdaYny992ujUU9UKi/4CWoJWsl1pNvjWJHNHGK51m+jKGLlh1ylb2ifQ==} + cpu: [arm64] + os: [linux] + libc: [musl] + + '@rollup/rollup-linux-loong64-gnu@4.63.4': + resolution: {integrity: sha512-D72ZnvkFkBXOfzMMQLcwfPLyGkKb7HZ9/mf97B7v6/P5Lbv4oFOtSY/uHbS8lH6uKUOxoKiuokdb50XZSzzbJw==} + cpu: [loong64] + os: [linux] + libc: [glibc] + + '@rollup/rollup-linux-loong64-musl@4.63.4': + resolution: {integrity: sha512-piU6BxeqA3O9KSu3kRCIQQtNqFFaTu21SEV4FwaRZowpnj3bLaWPZHw+xFqCs0XlJ+aOH3PTRWGoglH+mKA/OA==} + cpu: [loong64] + os: [linux] + libc: [musl] + + '@rollup/rollup-linux-ppc64-gnu@4.63.4': + resolution: {integrity: sha512-/5PGpHwqt2EEEOUs1XwzubE/ucr0dWDQ+to3zqi4Ds7EWpwtQ79wXc4JBoxqj/OwpawTsKWzJxHfSuBOq3DrWA==} + cpu: [ppc64] + os: [linux] + libc: [glibc] + + '@rollup/rollup-linux-ppc64-musl@4.63.4': + resolution: {integrity: sha512-cX3beZDLWt7G2oJF+nhChiT+qtaihs+S2xi7ziGmVB+2pwPng6D0Ed0HmElQOgv2UsUmSJJLGwpBao/3TDx3VA==} + cpu: [ppc64] + os: [linux] + libc: [musl] + + '@rollup/rollup-linux-riscv64-gnu@4.63.4': + resolution: {integrity: sha512-1uz2mGWHyptR7DgHHrlbdRAjXK7v7elGZ9lMja910/RP+ZYbX6xAmCiU9UZSX4hqmgtHMv6lr5l3kq1HIOpcag==} + cpu: [riscv64] + os: [linux] + libc: [glibc] + + '@rollup/rollup-linux-riscv64-musl@4.63.4': + resolution: {integrity: sha512-nLS8topojxyz7SRpKR2IODRpQ0XPZ+xaOXvT3+hqK/Uy8Lo5HFgkkIBiIrCu5tL5YqzTvgovGw55PwpahTAGig==} + cpu: [riscv64] + os: [linux] + libc: [musl] + + '@rollup/rollup-linux-s390x-gnu@4.63.4': + resolution: {integrity: sha512-gs7DRKotr3l3q+jGPQBjH0ng1FjlEDm5ueQrkw5JtQvtLyEIcLASqAEaor56BhkKRzk+IcQzrcanBdb/bBQn8g==} + cpu: [s390x] + os: [linux] + libc: [glibc] + + '@rollup/rollup-linux-x64-gnu@4.63.4': + resolution: {integrity: sha512-791ET7W17NnScOZM7h4dX5hYspxE28htPFsb1awY/NRR8+PRNkS53e475rDdxXXDrP+kwnCcNWg9CX5ztn/Aqw==} + cpu: [x64] + os: [linux] + libc: [glibc] + + '@rollup/rollup-linux-x64-musl@4.63.4': + resolution: {integrity: sha512-iwZQRcmj7g88g3tzefIrQY7qvmuA/cfYwhrDtTBhsmukO4U2huVO5W+86XacUMRvdSFVAc6kZUZy21JaRwiB9w==} + cpu: [x64] + os: [linux] + libc: [musl] + + '@rollup/rollup-openbsd-x64@4.63.4': + resolution: {integrity: sha512-dVHFp9gRWrdTpnqQuGfCwd7hOQDatK1VCP2iWhLY/cGrOQs/ucFzJ6A5SRqbXX12ZDI8EUuejSM5kwg+ja7Png==} + cpu: [x64] + os: [openbsd] + + '@rollup/rollup-openharmony-arm64@4.63.4': + resolution: {integrity: sha512-t3NlauOW6gxZVVFcBEnO62Cb4wbyDFL416gTg1uFI/2tgqYQlf69FbSE115Ajre9I+c26Lk4mcmdFUsS/DGifQ==} + cpu: [arm64] + os: [openharmony] + + '@rollup/rollup-win32-arm64-msvc@4.63.4': + resolution: {integrity: sha512-xWuIaSye5FWZF8+UYtVEcHtRJDN5kN9Kfgxx3Kq8XIov9KSKbc1fiqQCm90SKrgQbUXZelbnUhnlUJmfSE7P9A==} + cpu: [arm64] + os: [win32] + + '@rollup/rollup-win32-ia32-msvc@4.63.4': + resolution: {integrity: sha512-9ALJJUOg/ZflMJepVo2PlgsGxSaxN7SQ4Z8GoZfVlarWr6r3rkHUNsd/zAio7p4YMtChSMXPionxej4Hkf6CXQ==} + cpu: [ia32] + os: [win32] + + '@rollup/rollup-win32-x64-gnu@4.63.4': + resolution: {integrity: sha512-blj9z5qx/Pv4WU0W1NMFDB97e0JH5ed+aZGywW8WCvp/NhWX/4PFAq5uu6Q0AebNn+Vo6KzUYDT++JzTT5ojlQ==} + cpu: [x64] + os: [win32] + + '@rollup/rollup-win32-x64-msvc@4.63.4': + resolution: {integrity: sha512-Erx822VRBwLa124shbj+wNXe//BOgMEctDV0m1aqTQdNO1S69DgNUCFKC1RCeZfixs1J31l6igk1ziyXErbigQ==} + cpu: [x64] + os: [win32] + + '@sindresorhus/is@7.2.0': + resolution: {integrity: sha512-P1Cz1dWaFfR4IR+U13mqqiGsLFf1KbayybWwdd2vfctdV6hDpUkgCY0nKOLLTMSoRd/jJNjtbqzf13K8DCCXQw==} + engines: {node: '>=18'} + + '@speed-highlight/core@1.2.24': + resolution: {integrity: sha512-qeW2e1l78afw8VhRPfPQ1Gjj+KU5XFQ/OFV5ti6eTa9bruO7mJyZtA4vw0ofqmA3tKCkROE9xLk3VZoeRc98nw==} + + '@types/chai@5.2.3': + resolution: {integrity: sha512-Mw558oeA9fFbv65/y4mHtXDs9bPnFMZAL/jxdPFUpOHHIXX91mcgEHbS5Lahr+pwZFR8A7GQleRWeI6cGFC2UA==} + + '@types/deep-eql@4.0.2': + resolution: {integrity: sha512-c9h9dVVMigMPc4bwTvC5dxqtqJZwQPePsWjPlpSOnojbor6pGqdk541lfA7AqFQr5pB1BRdq0juY9db81BwyFw==} + + '@types/estree@1.0.9': + resolution: {integrity: sha512-GhdPgy1el4/ImP05X05Uw4cw2/M93BCUmnEvWZNStlCzEKME4Fkk+YpoA5OiHNQmoS7Cafb8Xa3Pya8m1Qrzeg==} + + '@types/node@22.20.4': + resolution: {integrity: sha512-zJRE40jpHtKqE/C4fgHrAKQLJuSpzEnP9ff9Y7YtoR3Wd2pwqzlekDeEuUQXjRd+QCYnVnNwuJYmhdk9XV8gvA==} + + '@vitest/expect@3.2.7': + resolution: {integrity: sha512-E8eBXaKibuvH2pSZErOjdVb5vF4PbKYcrnluBTYxEk1l/VhhwZg1kZQsdtjq+CsF5CFydf2Rdkz7jDHKSisi3w==} + + '@vitest/mocker@3.2.7': + resolution: {integrity: sha512-Trr0hYO9CM3Wj6ksWHRhK9IZpIY6wTMO5u/MqXurMxT57sWBaOPEtP3Oq60ihZuh5JsiagKfz95OcxdEP6dBrA==} + peerDependencies: + msw: ^2.4.9 + vite: ^5.0.0 || ^6.0.0 || ^7.0.0-0 + peerDependenciesMeta: + msw: + optional: true + vite: + optional: true + + '@vitest/pretty-format@3.2.7': + resolution: {integrity: sha512-KUHlwqVu0sRlhCdyPdQ/wBoTfRahjUky1MubOmYw9fWfIZy1gNoHpuaaQBPAaMaVYdQYHJLurzj8ECCj5OwTqA==} + + '@vitest/runner@3.2.7': + resolution: {integrity: sha512-sB9y4ovltoQP+WaUPwmSxO9WIg9Ig694Di5PalVPsYHklAdE027mehpWF2SQSVq+k6sFgaivbTjTJwZLSHbedA==} + + '@vitest/snapshot@3.2.7': + resolution: {integrity: sha512-7C+MwShwtBSI5Buwoyg3s/iY1eHL9PKAf+O1wVh/TdnjXUtkoL/9YQtre90i4MtNXM6edP1wJ2zOBpfCyhIS7g==} + + '@vitest/spy@3.2.7': + resolution: {integrity: sha512-Q2eQGI6d2L/hBtZ0qNuKcAGid68XK6cv1xsoaIma6PaJhHPoqcEJhYpXZ/5myCMqkNgtP6UKuBhbc0nHKnrkuQ==} + + '@vitest/utils@3.2.7': + resolution: {integrity: sha512-x6BDOd7dyo3PFLY3I9/HJ25X/6OurhGXk2/B9gOZNPF7XDVjeBK4k01lQE5uvDpbuheErh91qYuE1E2OEjK3Rw==} + + acorn-walk@8.3.2: + resolution: {integrity: sha512-cjkyv4OtNCIeqhHrfS81QWXoCBPExR/J62oyEqepVw8WaQeSqpW2uhuLPh1m9eWhDuOo/jUXVTlifvesOWp/4A==} + engines: {node: '>=0.4.0'} + + acorn@8.14.0: + resolution: {integrity: sha512-cl669nCJTZBsL97OF4kUQm5g5hC2uihk0NxY3WENAC0TYdILVkAyHymAntgxGkl7K+t0cXIrH5siy5S4XkFycA==} + engines: {node: '>=0.4.0'} + hasBin: true + + ajv@8.20.0: + resolution: {integrity: sha512-Thbli+OlOj+iMPYFBVBfJ3OmCAnaSyNn4M1vz9T6Gka5Jt9ba/HIR56joy65tY6kx/FCF5VXNB819Y7/GUrBGA==} + + ansi-regex@5.0.1: + resolution: {integrity: sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==} + engines: {node: '>=8'} + + ansi-styles@4.3.0: + resolution: {integrity: sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==} + engines: {node: '>=8'} + + assertion-error@2.0.1: + resolution: {integrity: sha512-Izi8RQcffqCeNVgFigKli1ssklIbpHnCYc6AknXGYoB6grJqyeby7jv12JUQgmTAnIDnbck1uxksT4dzN3PWBA==} + engines: {node: '>=12'} + + birpc@0.2.14: + resolution: {integrity: sha512-37FHE8rqsYM5JEKCnXFyHpBCzvgHEExwVVTq+nUmloInU7l8ezD1TpOhKpS8oe1DTYFqEK27rFZVKG43oTqXRA==} + + blake3-wasm@2.1.5: + resolution: {integrity: sha512-F1+K8EbfOZE49dtoPtmxUQrpXaBIl3ICvasLh+nJta0xkz+9kF/7uet9fLnwKqhDrmj6g+6K3Tw9yQPUg2ka5g==} + + cac@6.7.14: + resolution: {integrity: sha512-b6Ilus+c3RrdDk+JhLKUAQfzzgLEPy6wcXqS7f/xe1EETvsDP6GORG7SFuOs6cID5YkqchW/LXZbX5bc8j7ZcQ==} + engines: {node: '>=8'} + + chai@5.3.3: + resolution: {integrity: sha512-4zNhdJD/iOjSH0A05ea+Ke6MU5mmpQcbQsSOkgdaUMJ9zTlDTD/GYlwohmIE2u0gaxHYiVHEn1Fw9mZ/ktJWgw==} + engines: {node: '>=18'} + + check-error@2.1.3: + resolution: {integrity: sha512-PAJdDJusoxnwm1VwW07VWwUN1sl7smmC3OKggvndJFadxxDRyFJBX/ggnu/KE4kQAB7a3Dp8f/YXC1FlUprWmA==} + engines: {node: '>= 16'} + + cjs-module-lexer@1.4.3: + resolution: {integrity: sha512-9z8TZaGM1pfswYeXrUpzPrkx8UnWYdhJclsiYMm6x/w5+nN+8Tf/LnAgfLGQCm59qAOxU8WwHEq2vNwF6i4j+Q==} + + cliui@8.0.1: + resolution: {integrity: sha512-BSeNnyus75C4//NQ9gQt1/csTXyo/8Sb+afLAkzAptFuMsod9HFokGNudZpi/oQV73hnVK+sR+5PVRMd+Dr7YQ==} + engines: {node: '>=12'} + + color-convert@2.0.1: + resolution: {integrity: sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==} + engines: {node: '>=7.0.0'} + + color-name@1.1.4: + resolution: {integrity: sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==} + + color-string@1.9.1: + resolution: {integrity: sha512-shrVawQFojnZv6xM40anx4CkoDP+fZsw/ZerEMsW/pyzsRbElpsL/DBVW7q3ExxwusdNXI3lXpuhEZkzs8p5Eg==} + + color@4.2.3: + resolution: {integrity: sha512-1rXeuUUiGGrykh+CeBdu5Ie7OJwinCgQY0bc7GCRxy5xVHy+moaqkpL/jqQq0MtQOeYcrqEz4abc5f0KtU7W4A==} + engines: {node: '>=12.5.0'} + + cookie@1.1.1: + resolution: {integrity: sha512-ei8Aos7ja0weRpFzJnEA9UHJ/7XQmqglbRwnf2ATjcB9Wq874VKH9kfjjirM6UhU2/E5fFYadylyhFldcqSidQ==} + engines: {node: '>=18'} + + debug@4.4.3: + resolution: {integrity: sha512-RGwwWnwQvkVfavKVt22FGLw+xYSdzARwm0ru6DhTVA3umU5hZc28V3kO4stgYryrTlLpuvgI9GiijltAjNbcqA==} + engines: {node: '>=6.0'} + peerDependencies: + supports-color: '*' + peerDependenciesMeta: + supports-color: + optional: true + + deep-eql@5.0.2: + resolution: {integrity: sha512-h5k/5U50IJJFpzfL6nO9jaaumfjO/f2NjK/oYB2Djzm4p9L+3T9qWpZqZ2hAbLPuuYq9wrU08WQyBTL5GbPk5Q==} + engines: {node: '>=6'} + + defu@6.1.7: + resolution: {integrity: sha512-7z22QmUWiQ/2d0KkdYmANbRUVABpZ9SNYyH5vx6PZ+nE5bcC0l7uFvEfHlyld/HcGBFTL536ClDt3DEcSlEJAQ==} + + detect-libc@2.1.2: + resolution: {integrity: sha512-Btj2BOOO83o3WyH59e8MgXsxEQVcarkUOpEYrubB0urwnN10yQ364rsiByU11nZlqWYZm05i/of7io4mzihBtQ==} + engines: {node: '>=8'} + + devalue@5.9.4: + resolution: {integrity: sha512-sPAT4pztbu6586/hrhOnMKS17IJrvg12mXiSPSS3W5qDeN2RGgvZ0diZCm31dBbnevfVmujNO3IM2wrS4Y2Rhg==} + + effect@4.0.0-rc.112: + resolution: {integrity: sha512-wXxwuh1Ywnv4cPRM3Wfa0vDwuOHnZ1TsTgHJkG9XgzND6inhBH9n1vBxhg3iIXOia/OrpmvVmd3lrD4vq6bF3A==} + + emoji-regex@8.0.0: + resolution: {integrity: sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==} + + error-stack-parser-es@1.0.5: + resolution: {integrity: sha512-5qucVt2XcuGMcEGgWI7i+yZpmpByQ8J1lHhcL7PwqCwu9FPP3VUXzT4ltHe5i2z9dePwEHcDVOAfSnHsOlCXRA==} + + es-module-lexer@1.7.0: + resolution: {integrity: sha512-jEQoCwk8hyb2AZziIOLhDqpm5+2ww5uIE6lkO/6jcOCusfk6LhMHpXXfBLXTZ7Ydyt0j4VoUQv6uGNYbdW+kBA==} + + esbuild@0.25.4: + resolution: {integrity: sha512-8pgjLUcUjcgDg+2Q4NYXnPbo/vncAY4UmyaCm0jZevERqCHZIaWwdJHkf8XQtu4AxSKCdvrUbT0XUr1IdZzI8Q==} + engines: {node: '>=18'} + hasBin: true + + esbuild@0.28.2: + resolution: {integrity: sha512-HKVLS8dvII+xoKW9kmqxbRKrnWEXfJJr/FZhhJmiqIB0e053QNYFqOBouTMO/k5sID4MvCiUCvv8b9M4h32wIA==} + engines: {node: '>=18'} + hasBin: true + + escalade@3.2.0: + resolution: {integrity: sha512-WUj2qlxaQtO4g6Pq5c29GTcWGDyd8itL8zTlipgECz3JesAiiOKotd8JU6otB3PACgG6xkJUyVhboMS+bje/jA==} + engines: {node: '>=6'} + + estree-walker@3.0.3: + resolution: {integrity: sha512-7RUKfXgSMMkzt6ZuXmqapOurLGPPfgj6l9uRZ7lRGolvk0y2yocc35LdcxKC5PQZdn2DMqioAQ2NoWcrTKmm6g==} + + exit-hook@2.2.1: + resolution: {integrity: sha512-eNTPlAD67BmP31LDINZ3U7HSF8l57TxOY2PmBJ1shpCvpnxBF93mWCE8YHBnXs8qiUZJc9WDcWIeC3a2HIAMfw==} + engines: {node: '>=6'} + + expect-type@1.4.0: + resolution: {integrity: sha512-KfYbmpRm0VbLjEvVa9yGwCi9GI34xvi7A/HXYWQO65CSD2u3MczUJSuwXKFIxlGsgBQizV9q5J9NHj4VG0n+pA==} + engines: {node: '>=12.0.0'} + + exsolve@1.1.1: + resolution: {integrity: sha512-9U/jZUgjnSGyntRr6y5Muu1MJcwFl6kPu7k8qLF0IMNfLqvw0NZ4nnVDq0RVoZ0RvCyumib4Ez3KYrVfilrw+g==} + + fast-check@4.10.2: + resolution: {integrity: sha512-iK2f+YrcmoeGqk6fA0ea2bptcu/itMIm4NfEozq6N25+aG6h7s5HZbB/k1aV7b5w5sFLMCbbtRUsTVR+BgC3xw==} + engines: {node: '>=12.17.0'} + + fast-deep-equal@3.1.3: + resolution: {integrity: sha512-f3qQ9oQy9j2AhBe/H9VC91wLmKBCCU/gDOnKNAYG5hswO7BLKj09Hc5HYNz9cGI++xlpDCIgDaitVs03ATR84Q==} + + fast-uri@3.1.8: + resolution: {integrity: sha512-GZMtZUTNRpOVIECoXwLNZS5xUGE+mVNbTB8h/7Rwh2TFWcBQiPzTgyZi05BF9UMZKkLJv8XBRJTlU7zg8+ZfMg==} + + fdir@6.5.0: + resolution: {integrity: sha512-tIbYtZbucOs0BRGqPJkshJUYdL+SDH7dVM8gjy+ERp3WAUjLEFJE+02kanyHtwjWOnwrKYBiwAmM0p4kLJAnXg==} + engines: {node: '>=12.0.0'} + peerDependencies: + picomatch: ^3 || ^4 + peerDependenciesMeta: + picomatch: + optional: true + + fsevents@2.3.3: + resolution: {integrity: sha512-5xoDfX+fL7faATnagmWPpbFtwh/R77WmMMqqHGS65C3vvB0YHrgF+B1YmZ3441tMj5n63k0212XNoJwzlhffQw==} + engines: {node: ^8.16.0 || ^10.6.0 || >=11.0.0} + os: [darwin] + + get-caller-file@2.0.5: + resolution: {integrity: sha512-DyFP3BM/3YHTQOCUL/w0OZHR0lpKeGrxotcHWcqNEdnltqFwXVfhEBQ94eIo34AfQpo0rGki4cyIiftY06h2Fg==} + engines: {node: 6.* || 8.* || >= 10.*} + + glob-to-regexp@0.4.1: + resolution: {integrity: sha512-lkX1HJXwyMcprw/5YUZc2s7DrpAiHB21/V+E1rHUrVNokkvB6bqMzT0VfV6/86ZNabt1k14YOIaT7nDvOX3Iiw==} + + is-arrayish@0.3.4: + resolution: {integrity: sha512-m6UrgzFVUYawGBh1dUsWR5M2Clqic9RVXC/9f8ceNlv2IcO9j9J/z8UoCLPqtsPBFNzEpfR3xftohbfqDx8EQA==} + + is-fullwidth-code-point@3.0.0: + resolution: {integrity: sha512-zymm5+u+sCsSWyD9qNaejV3DFvhCKclKdizYaJUuHA83RLjb7nSuGnddCHGv0hk+KY7BMAlsWeK4Ueg6EV6XQg==} + engines: {node: '>=8'} + + js-tokens@9.0.1: + resolution: {integrity: sha512-mxa9E9ITFOt0ban3j6L5MpjwegGz6lBQmM1IJkWeBZGcMxto50+eWdjC/52xDbS2vy0k7vIMK0Fe2wfL9OQSpQ==} + + json-schema-traverse@1.0.0: + resolution: {integrity: sha512-NM8/P9n3XjXhIZn1lLhkFaACTOURQXjWhV4BA/RnOv8xvgqtqpAX9IO4mRQxSx1Rlo4tqzeqb0sOlruaOy3dug==} + + kleur@4.1.5: + resolution: {integrity: sha512-o+NO+8WrRiQEE4/7nwRJhN1HWpVmJm511pBHUxPLtp0BUISzlBplORYSmTclCnJvQq2tKu/sgl3xVpkc7ZWuQQ==} + engines: {node: '>=6'} + + lodash.camelcase@4.3.0: + resolution: {integrity: sha512-TwuEnCnxbc3rAvhf/LbG7tJUDzhqXyFnv3dtzLOPgCG/hODL7WFnsbwktkD7yUV0RrreP/l1PALq/YSg6VvjlA==} + + long@5.3.2: + resolution: {integrity: sha512-mNAgZ1GmyNhD7AuqnTG3/VQ26o760+ZYBPKjPvugO8+nLbYfX6TVpJPseBvopbdY+qpZ/lKUnmEc1LeZYS3QAA==} + + loupe@3.2.1: + resolution: {integrity: sha512-CdzqowRJCeLU72bHvWqwRBBlLcMEtIvGrlvef74kMnV2AolS9Y8xUv1I0U/MNAWMhBlKIoyuEgoJ0t/bbwHbLQ==} + + magic-string@0.30.21: + resolution: {integrity: sha512-vd2F4YUyEXKGcLHoq+TEyCjxueSeHnFxyyjNp80yg0XV4vUhnDer/lvvlqM/arB5bXQN5K2/3oinyCRyx8T2CQ==} + + mime@3.0.0: + resolution: {integrity: sha512-jSCU7/VB1loIWBZe14aEYHU/+1UMEHoaO7qxCOVJOw9GgH72VAWppxNcjU+x9a2k3GSIBXNKxXQFqRvvZ7vr3A==} + engines: {node: '>=10.0.0'} + hasBin: true + + miniflare@4.20251011.0: + resolution: {integrity: sha512-DlZ7vR5q/RE9eLsxsrXzfSZIF2f6O5k0YsFrSKhWUtdefyGtJt4sSpR6V+Af/waaZ6+zIFy9lsknHBCm49sEYA==} + engines: {node: '>=18.0.0'} + hasBin: true + + ms@2.1.3: + resolution: {integrity: sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==} + + msgpackr-extract@3.0.4: + resolution: {integrity: sha512-4kmO/MdyUIkLIvTPr8VHLil4AtoKIoniWPIEk5+CDy0xnWC84azhSFmuJ7PxZdsYtiP5kEeQsORAVIeMgxT+Hw==} + hasBin: true + + msgpackr@2.1.0: + resolution: {integrity: sha512-p/pBCVO63CsvvpkomUnNNag6+n38rULuDA6HHe70o2gtC8ODI52foF/4ko2qQcp6OiErJXTmrZeXmsGGHsIQNQ==} + + nanoid@3.3.19: + resolution: {integrity: sha512-Y2tUNy4ouw6tq5oDSKeQYGOyhkUBhNOcGV/02KC+6kd9eDGqdZd++mjMiIDilrBYvjEnCYvVtsuHCuP+okSfug==} + engines: {node: ^10 || ^12 || ^13.7 || ^14 || >=15.0.1} + hasBin: true + + node-gyp-build-optional-packages@5.2.2: + resolution: {integrity: sha512-s+w+rBWnpTMwSFbaE0UXsRlg7hU4FjekKU4eyAih5T8nJuNZT1nNsskXpxmeqSK9UzkBl6UgRlnKc8hz8IEqOw==} + hasBin: true + + ohash@2.0.12: + resolution: {integrity: sha512-65S/5gk9YSsaRjcyf7Nfa6h/d3E8/1gslpXfI4W7Dxn/oap8IKRuNT5VXkLQ1YFKIEg4apRY4Pj6aiwFzrDdmw==} + + path-to-regexp@6.3.0: + resolution: {integrity: sha512-Yhpw4T9C6hPpgPeA28us07OJeqZ5EzQTkbfwuhsUg0c237RomFoETJgmp2sa3F/41gfLE6G5cqcYwznmeEeOlQ==} + + pathe@2.0.3: + resolution: {integrity: sha512-WUjGcAqP1gQacoQe+OBJsFA7Ld4DyXuUIjZ5cc75cLHvJ7dtNsTugphxIADwspS+AraAUePCKrSVtPLFj/F88w==} + + pathval@2.0.1: + resolution: {integrity: sha512-//nshmD55c46FuFw26xV/xFAaB5HF9Xdap7HJBBnrKdAd6/GxDBaNA1870O79+9ueg61cZLSVc+OaFlfmObYVQ==} + engines: {node: '>= 14.16'} + + picocolors@1.1.1: + resolution: {integrity: sha512-xceH2snhtb5M9liqDsmEw56le376mTZkEX/jEb/RxNFyegNul7eNslCXP9FDj/Lcu0X8KEyMceP2ntpaHrDEVA==} + + picomatch@4.0.7: + resolution: {integrity: sha512-qcJu88Q2IWqJsDD529JKMdwGm/dvInW4HvQnRwiH9JtihJvzGOscDtHE3x1pBKeUOTysQ8kVmLnJ2kJu7yhcGA==} + engines: {node: '>=12'} + + postcss@8.5.28: + resolution: {integrity: sha512-RRuzqDtt5Y9h3quz5hWhK+TPnsmVs6WwSU6LkJMeY4HstUEDuYTG8UJSdawMRzmzAtV+KEoG8N3Qg2qLy5vM/A==} + engines: {node: ^10 || ^12 || >=14} + + protobufjs@7.6.6: + resolution: {integrity: sha512-dYDWdjSl5RNb7SgPxGQcRU+GtvP7s2fpkrY0r432PcOIaZ0/rBcxEZnQN67iJhFuQiVw754JDoPruPCNdGsbjg==} + engines: {node: '>=12.0.0'} + + pure-rand@8.4.2: + resolution: {integrity: sha512-vvuOGgcuPJAirlHvuQw1TrOiw7ptaIXXmIbNuiNOY6lNGJJH49PQ1Kj4nd783nPdQhQdicgOjVI2yI/9BD6/Ng==} + + require-directory@2.1.1: + resolution: {integrity: sha512-fGxEI7+wsG9xrvdjsrlmL22OMTTiHRwAMroiEeMgq8gzoLC/PQr7RsRDSTLUg/bZAZtF+TVIkHc6/4RIKrui+Q==} + engines: {node: '>=0.10.0'} + + require-from-string@2.0.2: + resolution: {integrity: sha512-Xf0nWe6RseziFMu+Ap9biiUbmplq6S9/p+7w7YXP/JBHhrUDDUhwa+vANyubuqfZWTveU//DYVGsDG7RKL/vEw==} + engines: {node: '>=0.10.0'} + + rollup@4.63.4: + resolution: {integrity: sha512-4U0liVayNIoLp3GFl1FcI8561WepLnZ1rqfraGh7S9B3Ur5F9S283y8Futii7RUU2C/97tOBmBy7nYvhoiOpbQ==} + engines: {node: '>=18.0.0', npm: '>=8.0.0'} + hasBin: true + + semver@7.8.5: + resolution: {integrity: sha512-Y7/KDsb8LjooZpwaqGyulO6DQlksgCncchHGk+sZIY4SBvUocMBEFH5Ur1fI4dV+Jvl0w6cjvucaIi40puRioA==} + engines: {node: '>=10'} + hasBin: true + + sharp@0.33.5: + resolution: {integrity: sha512-haPVm1EkS9pgvHrQ/F3Xy+hgcuMV0Wm9vfIBSiwZ05k+xgb0PkBQpGsAA/oWdDobNaZTH5ppvHtzCFbnSEwHVw==} + engines: {node: ^18.17.0 || ^20.3.0 || >=21.0.0} + + siginfo@2.0.0: + resolution: {integrity: sha512-ybx0WO1/8bSBLEWXZvEd7gMW3Sn3JFlW3TvX1nREbDLRNQNaeNN8WK0meBwPdAaOI7TtRRRJn/Es1zhrrCHu7g==} + + simple-swizzle@0.2.4: + resolution: {integrity: sha512-nAu1WFPQSMNr2Zn9PGSZK9AGn4t/y97lEm+MXTtUDwfP0ksAIX4nO+6ruD9Jwut4C49SB1Ws+fbXsm/yScWOHw==} + + smol-toml@1.8.0: + resolution: {integrity: sha512-kCZr2V3ch9i00x8zXRhjUNVcjG9ijES5dDudkXvUVCT5QlJNQWElSJdZqyPemffHoLNUYwOcou0Fy+ojN0uHSQ==} + engines: {node: '>= 18'} + + source-map-js@1.2.1: + resolution: {integrity: sha512-UXWMKhLOwVKb728IUtQPXxfYU+usdybtUrK/8uGE8CQMvrhOpwvzDBwj0QhSL7MQc7vIsISBG8VQ8+IDQxpfQA==} + engines: {node: '>=0.10.0'} + + stackback@0.0.2: + resolution: {integrity: sha512-1XMJE5fQo1jGH6Y/7ebnwPOBEkIEnT4QF32d5R1+VXdXveM0IBMJt8zfaxX1P3QhVwrYe+576+jkANtSS2mBbw==} + + std-env@3.10.0: + resolution: {integrity: sha512-5GS12FdOZNliM5mAOxFRg7Ir0pWz8MdpYm6AY6VPkGpbA7ZzmbzNcBJQ0GPvvyWgcY7QAhCgf9Uy89I03faLkg==} + + stoppable@1.1.0: + resolution: {integrity: sha512-KXDYZ9dszj6bzvnEMRYvxgeTHU74QBFL54XKtP3nyMuJ81CFYtABZ3bAzL2EdFUaEwJOBOgENyFj3R7oTzDyyw==} + engines: {node: '>=4', npm: '>=6'} + + string-width@4.2.3: + resolution: {integrity: sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==} + engines: {node: '>=8'} + + strip-ansi@6.0.1: + resolution: {integrity: sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==} + engines: {node: '>=8'} + + strip-literal@3.1.0: + resolution: {integrity: sha512-8r3mkIM/2+PpjHoOtiAW8Rg3jJLHaV7xPwG+YRGrv6FP0wwk/toTpATxWYOW0BKdWwl82VT2tFYi5DlROa0Mxg==} + + supports-color@10.2.2: + resolution: {integrity: sha512-SS+jx45GF1QjgEXQx4NJZV9ImqmO2NPz5FNsIHrsDjh2YsHnawpan7SNQ1o8NuhrbHZy9AZhIoCUiCeaW/C80g==} + engines: {node: '>=18'} + + tinybench@2.9.0: + resolution: {integrity: sha512-0+DUvqWMValLmha6lr4kD8iAMK1HzV0/aKnCtWb9v9641TnP/MFb7Pc2bxoxQjTXAErryXVgUOfv2YqNllqGeg==} + + tinyexec@0.3.2: + resolution: {integrity: sha512-KQQR9yN7R5+OSwaK0XQoj22pwHoTlgYqmUscPYoknOoWCWfj/5/ABTMRi69FrKU5ffPVh5QcFikpWJI/P1ocHA==} + + tinyglobby@0.2.17: + resolution: {integrity: sha512-wXR/dYpcqKmfWpEdZjiKJOwCNFndD0DMnrW/cYjVGttEkBfVgcLFHoNrlj47mjOVic9yyNu65alsgF4NQyTa2g==} + engines: {node: '>=12.0.0'} + + tinypool@1.1.1: + resolution: {integrity: sha512-Zba82s87IFq9A9XmjiX5uZA/ARWDrB03OHlq+Vw1fSdt0I+4/Kutwy8BP4Y/y/aORMo61FQ0vIb5j44vSo5Pkg==} + engines: {node: ^18.0.0 || >=20.0.0} + + tinyrainbow@2.0.0: + resolution: {integrity: sha512-op4nsTR47R6p0vMUUoYl/a+ljLFVtlfaXkLQmqfLR1qHma1h/ysYk4hEXZ880bf2CYgTskvTa/e196Vd5dDQXw==} + engines: {node: '>=14.0.0'} + + tinyspy@4.0.6: + resolution: {integrity: sha512-u8KszXvGfU68hVcZpRHKG28T0krMuv2G5nDhiHaMLen/gIuFEgIJhaJuO69qjnXg5paSrbPMFfx3brNuN8eVSg==} + engines: {node: '>=14.0.0'} + + tslib@2.8.1: + resolution: {integrity: sha512-oJFu94HQb+KVduSUQL7wnpmqnfmLsOA/nAh6b6EH0wCEoK0/mPeXU6c3wKDV83MkOuHPRHtSXKKU99IBazS/2w==} + + tsx@4.23.15: + resolution: {integrity: sha512-Yiex1Ovn8z2xPpOWckIiysV1SSyRMY9BkLF++q0yKiDxCqRhosKfMg3janKkiLBwZ5c/YryloKwGZcrEmtwxKw==} + engines: {node: '>=18.0.0'} + hasBin: true + + typescript@5.9.3: + resolution: {integrity: sha512-jl1vZzPDinLr9eUt3J/t7V6FgNEw9QjvBPdysz9KfQDD41fQrC2Y4vKQdiaUpFT4bXlb1RHhLpp8wtm6M5TgSw==} + engines: {node: '>=14.17'} + hasBin: true + + ufo@1.6.4: + resolution: {integrity: sha512-JFNbkD1Svwe0KvGi8GOeLcP4kAWQ609twvCdcHxq1oSL8svv39ZuSvajcD8B+5D0eL4+s1Is2D/O6KN3qcTeRA==} + + undici-types@6.21.0: + resolution: {integrity: sha512-iwDZqg0QAGrg9Rav5H4n0M64c3mkR59cJ6wQp+7C4nI0gsmExaedaYLNO44eT4AtBBwjbTiGPMlt2Md0T9H9JQ==} + + undici@7.14.0: + resolution: {integrity: sha512-Vqs8HTzjpQXZeXdpsfChQTlafcMQaaIwnGwLam1wudSSjlJeQ3bw1j+TLPePgrCnCpUXx7Ba5Pdpf5OBih62NQ==} + engines: {node: '>=20.18.1'} + + unenv@2.0.0-rc.21: + resolution: {integrity: sha512-Wj7/AMtE9MRnAXa6Su3Lk0LNCfqDYgfwVjwRFVum9U7wsto1imuHqk4kTm7Jni+5A0Hn7dttL6O/zjvUvoo+8A==} + + vite-node@3.2.4: + resolution: {integrity: sha512-EbKSKh+bh1E1IFxeO0pg1n4dvoOTt0UDiXMd/qn++r98+jPO1xtJilvXldeuQ8giIB5IkpjCgMleHMNEsGH6pg==} + engines: {node: ^18.0.0 || ^20.0.0 || >=22.0.0} + hasBin: true + + vite@7.3.6: + resolution: {integrity: sha512-4XP60spRGjSZFf1qYH+dJIkK2znL3zQfl9KkOV9MkkRR/3Dls0dxaBsQPTloEc5BLXWPL9vsOxopxyKoMmDueg==} + engines: {node: ^20.19.0 || >=22.12.0} + hasBin: true + peerDependencies: + '@types/node': ^20.19.0 || >=22.12.0 + jiti: '>=1.21.0' + less: ^4.0.0 + lightningcss: ^1.21.0 + sass: ^1.70.0 + sass-embedded: ^1.70.0 + stylus: '>=0.54.8' + sugarss: ^5.0.0 + terser: ^5.16.0 + tsx: ^4.8.1 + yaml: ^2.4.2 + peerDependenciesMeta: + '@types/node': + optional: true + jiti: + optional: true + less: + optional: true + lightningcss: + optional: true + sass: + optional: true + sass-embedded: + optional: true + stylus: + optional: true + sugarss: + optional: true + terser: + optional: true + tsx: + optional: true + yaml: + optional: true + + vitest@3.2.7: + resolution: {integrity: sha512-KrxIJ62Fd89gfysR4WotlgZABiz2dqFPgqGzX7s+CwsqLFomRH7777ZcrOD6+WVAh7khPQP41A+BKbpcJFrdEg==} + engines: {node: ^18.0.0 || ^20.0.0 || >=22.0.0} + hasBin: true + peerDependencies: + '@edge-runtime/vm': '*' + '@types/debug': ^4.1.12 + '@types/node': ^18.0.0 || ^20.0.0 || >=22.0.0 + '@vitest/browser': 3.2.7 + '@vitest/ui': 3.2.7 + happy-dom: '*' + jsdom: '*' + peerDependenciesMeta: + '@edge-runtime/vm': + optional: true + '@types/debug': + optional: true + '@types/node': + optional: true + '@vitest/browser': + optional: true + '@vitest/ui': + optional: true + happy-dom: + optional: true + jsdom: + optional: true + + why-is-node-running@2.3.0: + resolution: {integrity: sha512-hUrmaWBdVDcxvYqnyh09zunKzROWjbZTiNy8dBEjkS7ehEDQibXJ7XvlmtbwuTclUiIyN+CyXQD4Vmko8fNm8w==} + engines: {node: '>=8'} + hasBin: true + + workerd@1.20251011.0: + resolution: {integrity: sha512-Dq35TLPEJAw7BuYQMkN3p9rge34zWMU2Gnd4DSJFeVqld4+DAO2aPG7+We2dNIAyM97S8Y9BmHulbQ00E0HC7Q==} + engines: {node: '>=16'} + hasBin: true + + wrangler@4.44.0: + resolution: {integrity: sha512-BLOUigckcWZ0r4rm7b5PuaTpb9KP9as0XeCRSJ8kqcNgXcKoUD3Ij8FlPvN25KybLnFnetaO0ZdfRYUPWle4qw==} + engines: {node: '>=18.0.0'} + hasBin: true + peerDependencies: + '@cloudflare/workers-types': ^4.20251011.0 + peerDependenciesMeta: + '@cloudflare/workers-types': + optional: true + + wrap-ansi@7.0.0: + resolution: {integrity: sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q==} + engines: {node: '>=10'} + + ws@8.18.0: + resolution: {integrity: sha512-8VbfWfHLbbwu3+N6OKsOMpBdT4kXPDDB9cJk2bJ6mh9ucxdlnNvH1e+roYkKmN9Nxw2yjz7VzeO9oOz2zJ04Pw==} + engines: {node: '>=10.0.0'} + peerDependencies: + bufferutil: ^4.0.1 + utf-8-validate: '>=5.0.2' + peerDependenciesMeta: + bufferutil: + optional: true + utf-8-validate: + optional: true + + y18n@5.0.8: + resolution: {integrity: sha512-0pfFzegeDWJHJIAmTLRP2DwHjdF5s7jo9tuztdQxAhINCdvS+3nGINqPd00AphqJR/0LhANUS6/+7SCb98YOfA==} + engines: {node: '>=10'} + + yargs-parser@21.1.1: + resolution: {integrity: sha512-tVpsJW7DdjecAiFpbIB1e3qxIQsE6NoPc5/eTdrbbIC4h0LVsWhnoa3g+m2HclBIujHzsxZ4VJVA+GUuc2/LBw==} + engines: {node: '>=12'} + + yargs@17.7.3: + resolution: {integrity: sha512-GZtjxm/J/4TSxuL3FNYjCmLktBTnIw/rVmKSIyKeYAZpmJB2ig9VauCC5xsa82GNKVKDAqpOn3KVzNt0zmrU0g==} + engines: {node: '>=12'} + + youch-core@0.3.3: + resolution: {integrity: sha512-ho7XuGjLaJ2hWHoK8yFnsUGy2Y5uDpqSTq1FkHLK4/oqKtyUU1AFbOOxY4IpC9f0fTLjwYbslUz0Po5BpD1wrA==} + + youch@4.1.0-beta.10: + resolution: {integrity: sha512-rLfVLB4FgQneDr0dv1oddCVZmKjcJ6yX6mS4pU82Mq/Dt9a3cLZQ62pDBL4AUO+uVrCvtWz3ZFUL2HFAFJ/BXQ==} + + zod@3.22.3: + resolution: {integrity: sha512-EjIevzuJRiRPbVH4mGc8nApb/lVLKVpmUhAaR5R5doKGfAnGJ6Gr3CViAVjP+4FWSxCsybeWQdcgCtbX+7oZug==} + + zod@3.25.76: + resolution: {integrity: sha512-gzUt/qt81nXsFGKIFcC3YnfEAx5NkunCfnDlvuBSSFS02bcXu4Lmea0AFIUwbLWxWPx3d9p8S5QoaujKcNQxcQ==} + +snapshots: + + '@cloudflare/kv-asset-handler@0.4.0': + dependencies: + mime: 3.0.0 + + '@cloudflare/unenv-preset@2.7.8(unenv@2.0.0-rc.21)(workerd@1.20251011.0)': + dependencies: + unenv: 2.0.0-rc.21 + optionalDependencies: + workerd: 1.20251011.0 + + '@cloudflare/vitest-pool-workers@0.9.14(@vitest/runner@3.2.7)(@vitest/snapshot@3.2.7)(vitest@3.2.7(@types/node@22.20.4)(supports-color@10.2.2)(tsx@4.23.15))': + dependencies: + '@vitest/runner': 3.2.7 + '@vitest/snapshot': 3.2.7 + birpc: 0.2.14 + cjs-module-lexer: 1.4.3 + devalue: 5.9.4 + miniflare: 4.20251011.0 + semver: 7.8.5 + vitest: 3.2.7(@types/node@22.20.4)(supports-color@10.2.2)(tsx@4.23.15) + wrangler: 4.44.0 + zod: 3.25.76 + transitivePeerDependencies: + - '@cloudflare/workers-types' + - bufferutil + - utf-8-validate + + '@cloudflare/workerd-darwin-64@1.20251011.0': + optional: true + + '@cloudflare/workerd-darwin-arm64@1.20251011.0': + optional: true + + '@cloudflare/workerd-linux-64@1.20251011.0': + optional: true + + '@cloudflare/workerd-linux-arm64@1.20251011.0': + optional: true + + '@cloudflare/workerd-windows-64@1.20251011.0': + optional: true + + '@cspotcode/source-map-support@0.8.1': + dependencies: + '@jridgewell/trace-mapping': 0.3.9 + + '@emnapi/runtime@1.11.3': + dependencies: + tslib: 2.8.1 + optional: true + + '@esbuild/aix-ppc64@0.25.4': + optional: true + + '@esbuild/aix-ppc64@0.28.2': + optional: true + + '@esbuild/android-arm64@0.25.4': + optional: true + + '@esbuild/android-arm64@0.28.2': + optional: true + + '@esbuild/android-arm@0.25.4': + optional: true + + '@esbuild/android-arm@0.28.2': + optional: true + + '@esbuild/android-x64@0.25.4': + optional: true + + '@esbuild/android-x64@0.28.2': + optional: true + + '@esbuild/darwin-arm64@0.25.4': + optional: true + + '@esbuild/darwin-arm64@0.28.2': + optional: true + + '@esbuild/darwin-x64@0.25.4': + optional: true + + '@esbuild/darwin-x64@0.28.2': + optional: true + + '@esbuild/freebsd-arm64@0.25.4': + optional: true + + '@esbuild/freebsd-arm64@0.28.2': + optional: true + + '@esbuild/freebsd-x64@0.25.4': + optional: true + + '@esbuild/freebsd-x64@0.28.2': + optional: true + + '@esbuild/linux-arm64@0.25.4': + optional: true + + '@esbuild/linux-arm64@0.28.2': + optional: true + + '@esbuild/linux-arm@0.25.4': + optional: true + + '@esbuild/linux-arm@0.28.2': + optional: true + + '@esbuild/linux-ia32@0.25.4': + optional: true + + '@esbuild/linux-ia32@0.28.2': + optional: true + + '@esbuild/linux-loong64@0.25.4': + optional: true + + '@esbuild/linux-loong64@0.28.2': + optional: true + + '@esbuild/linux-mips64el@0.25.4': + optional: true + + '@esbuild/linux-mips64el@0.28.2': + optional: true + + '@esbuild/linux-ppc64@0.25.4': + optional: true + + '@esbuild/linux-ppc64@0.28.2': + optional: true + + '@esbuild/linux-riscv64@0.25.4': + optional: true + + '@esbuild/linux-riscv64@0.28.2': + optional: true + + '@esbuild/linux-s390x@0.25.4': + optional: true + + '@esbuild/linux-s390x@0.28.2': + optional: true + + '@esbuild/linux-x64@0.25.4': + optional: true + + '@esbuild/linux-x64@0.28.2': + optional: true + + '@esbuild/netbsd-arm64@0.25.4': + optional: true + + '@esbuild/netbsd-arm64@0.28.2': + optional: true + + '@esbuild/netbsd-x64@0.25.4': + optional: true + + '@esbuild/netbsd-x64@0.28.2': + optional: true + + '@esbuild/openbsd-arm64@0.25.4': + optional: true + + '@esbuild/openbsd-arm64@0.28.2': + optional: true + + '@esbuild/openbsd-x64@0.25.4': + optional: true + + '@esbuild/openbsd-x64@0.28.2': + optional: true + + '@esbuild/openharmony-arm64@0.28.2': + optional: true + + '@esbuild/sunos-x64@0.25.4': + optional: true + + '@esbuild/sunos-x64@0.28.2': + optional: true + + '@esbuild/win32-arm64@0.25.4': + optional: true + + '@esbuild/win32-arm64@0.28.2': + optional: true + + '@esbuild/win32-ia32@0.25.4': + optional: true + + '@esbuild/win32-ia32@0.28.2': + optional: true + + '@esbuild/win32-x64@0.25.4': + optional: true + + '@esbuild/win32-x64@0.28.2': + optional: true + + '@grpc/grpc-js@1.14.5': + dependencies: + '@grpc/proto-loader': 0.8.1 + '@js-sdsl/ordered-map': 4.4.2 + + '@grpc/proto-loader@0.7.15': + dependencies: + lodash.camelcase: 4.3.0 + long: 5.3.2 + protobufjs: 7.6.6 + yargs: 17.7.3 + + '@grpc/proto-loader@0.8.1': + dependencies: + lodash.camelcase: 4.3.0 + long: 5.3.2 + protobufjs: 7.6.6 + yargs: 17.7.3 + + '@img/sharp-darwin-arm64@0.33.5': + optionalDependencies: + '@img/sharp-libvips-darwin-arm64': 1.0.4 + optional: true + + '@img/sharp-darwin-x64@0.33.5': + optionalDependencies: + '@img/sharp-libvips-darwin-x64': 1.0.4 + optional: true + + '@img/sharp-libvips-darwin-arm64@1.0.4': + optional: true + + '@img/sharp-libvips-darwin-x64@1.0.4': + optional: true + + '@img/sharp-libvips-linux-arm64@1.0.4': + optional: true + + '@img/sharp-libvips-linux-arm@1.0.5': + optional: true + + '@img/sharp-libvips-linux-s390x@1.0.4': + optional: true + + '@img/sharp-libvips-linux-x64@1.0.4': + optional: true + + '@img/sharp-libvips-linuxmusl-arm64@1.0.4': + optional: true + + '@img/sharp-libvips-linuxmusl-x64@1.0.4': + optional: true + + '@img/sharp-linux-arm64@0.33.5': + optionalDependencies: + '@img/sharp-libvips-linux-arm64': 1.0.4 + optional: true + + '@img/sharp-linux-arm@0.33.5': + optionalDependencies: + '@img/sharp-libvips-linux-arm': 1.0.5 + optional: true + + '@img/sharp-linux-s390x@0.33.5': + optionalDependencies: + '@img/sharp-libvips-linux-s390x': 1.0.4 + optional: true + + '@img/sharp-linux-x64@0.33.5': + optionalDependencies: + '@img/sharp-libvips-linux-x64': 1.0.4 + optional: true + + '@img/sharp-linuxmusl-arm64@0.33.5': + optionalDependencies: + '@img/sharp-libvips-linuxmusl-arm64': 1.0.4 + optional: true + + '@img/sharp-linuxmusl-x64@0.33.5': + optionalDependencies: + '@img/sharp-libvips-linuxmusl-x64': 1.0.4 + optional: true + + '@img/sharp-wasm32@0.33.5': + dependencies: + '@emnapi/runtime': 1.11.3 + optional: true + + '@img/sharp-win32-ia32@0.33.5': + optional: true + + '@img/sharp-win32-x64@0.33.5': + optional: true + + '@jridgewell/resolve-uri@3.1.2': {} + + '@jridgewell/sourcemap-codec@1.6.0': {} + + '@jridgewell/trace-mapping@0.3.9': + dependencies: + '@jridgewell/resolve-uri': 3.1.2 + '@jridgewell/sourcemap-codec': 1.6.0 + + '@js-sdsl/ordered-map@4.4.2': {} + + '@msgpackr-extract/msgpackr-extract-darwin-arm64@3.0.4': + optional: true + + '@msgpackr-extract/msgpackr-extract-darwin-x64@3.0.4': + optional: true + + '@msgpackr-extract/msgpackr-extract-linux-arm64@3.0.4': + optional: true + + '@msgpackr-extract/msgpackr-extract-linux-arm@3.0.4': + optional: true + + '@msgpackr-extract/msgpackr-extract-linux-x64@3.0.4': + optional: true + + '@msgpackr-extract/msgpackr-extract-win32-x64@3.0.4': + optional: true + + '@napi-rs/lzma-linux-x64-gnu@1.5.1': + optional: true + + '@poppinss/colors@4.1.6': + dependencies: + kleur: 4.1.5 + + '@poppinss/dumper@0.6.5': + dependencies: + '@poppinss/colors': 4.1.6 + '@sindresorhus/is': 7.2.0 + supports-color: 10.2.2 + + '@poppinss/exception@1.2.3': {} + + '@protobufjs/aspromise@1.1.2': {} + + '@protobufjs/base64@1.1.2': {} + + '@protobufjs/codegen@2.0.5': {} + + '@protobufjs/eventemitter@1.1.1': {} + + '@protobufjs/fetch@1.1.1': + dependencies: + '@protobufjs/aspromise': 1.1.2 + + '@protobufjs/float@1.0.2': {} + + '@protobufjs/path@1.1.2': {} + + '@protobufjs/pool@1.1.0': {} + + '@protobufjs/utf8@1.1.2': {} + + '@rollup/rollup-android-arm-eabi@4.63.4': + optional: true + + '@rollup/rollup-android-arm64@4.63.4': + optional: true + + '@rollup/rollup-darwin-arm64@4.63.4': + optional: true + + '@rollup/rollup-darwin-x64@4.63.4': + optional: true + + '@rollup/rollup-freebsd-arm64@4.63.4': + optional: true + + '@rollup/rollup-freebsd-x64@4.63.4': + optional: true + + '@rollup/rollup-linux-arm-gnueabihf@4.63.4': + optional: true + + '@rollup/rollup-linux-arm-musleabihf@4.63.4': + optional: true + + '@rollup/rollup-linux-arm64-gnu@4.63.4': + optional: true + + '@rollup/rollup-linux-arm64-musl@4.63.4': + optional: true + + '@rollup/rollup-linux-loong64-gnu@4.63.4': + optional: true + + '@rollup/rollup-linux-loong64-musl@4.63.4': + optional: true + + '@rollup/rollup-linux-ppc64-gnu@4.63.4': + optional: true + + '@rollup/rollup-linux-ppc64-musl@4.63.4': + optional: true + + '@rollup/rollup-linux-riscv64-gnu@4.63.4': + optional: true + + '@rollup/rollup-linux-riscv64-musl@4.63.4': + optional: true + + '@rollup/rollup-linux-s390x-gnu@4.63.4': + optional: true + + '@rollup/rollup-linux-x64-gnu@4.63.4': + optional: true + + '@rollup/rollup-linux-x64-musl@4.63.4': + optional: true + + '@rollup/rollup-openbsd-x64@4.63.4': + optional: true + + '@rollup/rollup-openharmony-arm64@4.63.4': + optional: true + + '@rollup/rollup-win32-arm64-msvc@4.63.4': + optional: true + + '@rollup/rollup-win32-ia32-msvc@4.63.4': + optional: true + + '@rollup/rollup-win32-x64-gnu@4.63.4': + optional: true + + '@rollup/rollup-win32-x64-msvc@4.63.4': + optional: true + + '@sindresorhus/is@7.2.0': {} + + '@speed-highlight/core@1.2.24': {} + + '@types/chai@5.2.3': + dependencies: + '@types/deep-eql': 4.0.2 + assertion-error: 2.0.1 + + '@types/deep-eql@4.0.2': {} + + '@types/estree@1.0.9': {} + + '@types/node@22.20.4': + dependencies: + undici-types: 6.21.0 + + '@vitest/expect@3.2.7': + dependencies: + '@types/chai': 5.2.3 + '@vitest/spy': 3.2.7 + '@vitest/utils': 3.2.7 + chai: 5.3.3 + tinyrainbow: 2.0.0 + + '@vitest/mocker@3.2.7(vite@7.3.6(@types/node@22.20.4)(tsx@4.23.15))': + dependencies: + '@vitest/spy': 3.2.7 + estree-walker: 3.0.3 + magic-string: 0.30.21 + optionalDependencies: + vite: 7.3.6(@types/node@22.20.4)(tsx@4.23.15) + + '@vitest/pretty-format@3.2.7': + dependencies: + tinyrainbow: 2.0.0 + + '@vitest/runner@3.2.7': + dependencies: + '@vitest/utils': 3.2.7 + pathe: 2.0.3 + strip-literal: 3.1.0 + + '@vitest/snapshot@3.2.7': + dependencies: + '@vitest/pretty-format': 3.2.7 + magic-string: 0.30.21 + pathe: 2.0.3 + + '@vitest/spy@3.2.7': + dependencies: + tinyspy: 4.0.6 + + '@vitest/utils@3.2.7': + dependencies: + '@vitest/pretty-format': 3.2.7 + loupe: 3.2.1 + tinyrainbow: 2.0.0 + + acorn-walk@8.3.2: {} + + acorn@8.14.0: {} + + ajv@8.20.0: + dependencies: + fast-deep-equal: 3.1.3 + fast-uri: 3.1.8 + json-schema-traverse: 1.0.0 + require-from-string: 2.0.2 + + ansi-regex@5.0.1: {} + + ansi-styles@4.3.0: + dependencies: + color-convert: 2.0.1 + + assertion-error@2.0.1: {} + + birpc@0.2.14: {} + + blake3-wasm@2.1.5: {} + + cac@6.7.14: {} + + chai@5.3.3: + dependencies: + assertion-error: 2.0.1 + check-error: 2.1.3 + deep-eql: 5.0.2 + loupe: 3.2.1 + pathval: 2.0.1 + + check-error@2.1.3: {} + + cjs-module-lexer@1.4.3: {} + + cliui@8.0.1: + dependencies: + string-width: 4.2.3 + strip-ansi: 6.0.1 + wrap-ansi: 7.0.0 + + color-convert@2.0.1: + dependencies: + color-name: 1.1.4 + + color-name@1.1.4: {} + + color-string@1.9.1: + dependencies: + color-name: 1.1.4 + simple-swizzle: 0.2.4 + + color@4.2.3: + dependencies: + color-convert: 2.0.1 + color-string: 1.9.1 + + cookie@1.1.1: {} + + debug@4.4.3(supports-color@10.2.2): + dependencies: + ms: 2.1.3 + optionalDependencies: + supports-color: 10.2.2 + + deep-eql@5.0.2: {} + + defu@6.1.7: {} + + detect-libc@2.1.2: {} + + devalue@5.9.4: {} + + effect@4.0.0-rc.112: + dependencies: + fast-check: 4.10.2 + msgpackr: 2.1.0 + + emoji-regex@8.0.0: {} + + error-stack-parser-es@1.0.5: {} + + es-module-lexer@1.7.0: {} + + esbuild@0.25.4: + optionalDependencies: + '@esbuild/aix-ppc64': 0.25.4 + '@esbuild/android-arm': 0.25.4 + '@esbuild/android-arm64': 0.25.4 + '@esbuild/android-x64': 0.25.4 + '@esbuild/darwin-arm64': 0.25.4 + '@esbuild/darwin-x64': 0.25.4 + '@esbuild/freebsd-arm64': 0.25.4 + '@esbuild/freebsd-x64': 0.25.4 + '@esbuild/linux-arm': 0.25.4 + '@esbuild/linux-arm64': 0.25.4 + '@esbuild/linux-ia32': 0.25.4 + '@esbuild/linux-loong64': 0.25.4 + '@esbuild/linux-mips64el': 0.25.4 + '@esbuild/linux-ppc64': 0.25.4 + '@esbuild/linux-riscv64': 0.25.4 + '@esbuild/linux-s390x': 0.25.4 + '@esbuild/linux-x64': 0.25.4 + '@esbuild/netbsd-arm64': 0.25.4 + '@esbuild/netbsd-x64': 0.25.4 + '@esbuild/openbsd-arm64': 0.25.4 + '@esbuild/openbsd-x64': 0.25.4 + '@esbuild/sunos-x64': 0.25.4 + '@esbuild/win32-arm64': 0.25.4 + '@esbuild/win32-ia32': 0.25.4 + '@esbuild/win32-x64': 0.25.4 + + esbuild@0.28.2: + optionalDependencies: + '@esbuild/aix-ppc64': 0.28.2 + '@esbuild/android-arm': 0.28.2 + '@esbuild/android-arm64': 0.28.2 + '@esbuild/android-x64': 0.28.2 + '@esbuild/darwin-arm64': 0.28.2 + '@esbuild/darwin-x64': 0.28.2 + '@esbuild/freebsd-arm64': 0.28.2 + '@esbuild/freebsd-x64': 0.28.2 + '@esbuild/linux-arm': 0.28.2 + '@esbuild/linux-arm64': 0.28.2 + '@esbuild/linux-ia32': 0.28.2 + '@esbuild/linux-loong64': 0.28.2 + '@esbuild/linux-mips64el': 0.28.2 + '@esbuild/linux-ppc64': 0.28.2 + '@esbuild/linux-riscv64': 0.28.2 + '@esbuild/linux-s390x': 0.28.2 + '@esbuild/linux-x64': 0.28.2 + '@esbuild/netbsd-arm64': 0.28.2 + '@esbuild/netbsd-x64': 0.28.2 + '@esbuild/openbsd-arm64': 0.28.2 + '@esbuild/openbsd-x64': 0.28.2 + '@esbuild/openharmony-arm64': 0.28.2 + '@esbuild/sunos-x64': 0.28.2 + '@esbuild/win32-arm64': 0.28.2 + '@esbuild/win32-ia32': 0.28.2 + '@esbuild/win32-x64': 0.28.2 + + escalade@3.2.0: {} + + estree-walker@3.0.3: + dependencies: + '@types/estree': 1.0.9 + + exit-hook@2.2.1: {} + + expect-type@1.4.0: {} + + exsolve@1.1.1: {} + + fast-check@4.10.2: + dependencies: + pure-rand: 8.4.2 + + fast-deep-equal@3.1.3: {} + + fast-uri@3.1.8: {} + + fdir@6.5.0(picomatch@4.0.7): + optionalDependencies: + picomatch: 4.0.7 + + fsevents@2.3.3: + optional: true + + get-caller-file@2.0.5: {} + + glob-to-regexp@0.4.1: {} + + is-arrayish@0.3.4: {} + + is-fullwidth-code-point@3.0.0: {} + + js-tokens@9.0.1: {} + + json-schema-traverse@1.0.0: {} + + kleur@4.1.5: {} + + lodash.camelcase@4.3.0: {} + + long@5.3.2: {} + + loupe@3.2.1: {} + + magic-string@0.30.21: + dependencies: + '@jridgewell/sourcemap-codec': 1.6.0 + + mime@3.0.0: {} + + miniflare@4.20251011.0: + dependencies: + '@cspotcode/source-map-support': 0.8.1 + acorn: 8.14.0 + acorn-walk: 8.3.2 + exit-hook: 2.2.1 + glob-to-regexp: 0.4.1 + sharp: 0.33.5 + stoppable: 1.1.0 + undici: 7.14.0 + workerd: 1.20251011.0 + ws: 8.18.0 + youch: 4.1.0-beta.10 + zod: 3.22.3 + transitivePeerDependencies: + - bufferutil + - utf-8-validate + + ms@2.1.3: {} + + msgpackr-extract@3.0.4: + dependencies: + node-gyp-build-optional-packages: 5.2.2 + optionalDependencies: + '@msgpackr-extract/msgpackr-extract-darwin-arm64': 3.0.4 + '@msgpackr-extract/msgpackr-extract-darwin-x64': 3.0.4 + '@msgpackr-extract/msgpackr-extract-linux-arm': 3.0.4 + '@msgpackr-extract/msgpackr-extract-linux-arm64': 3.0.4 + '@msgpackr-extract/msgpackr-extract-linux-x64': 3.0.4 + '@msgpackr-extract/msgpackr-extract-win32-x64': 3.0.4 + optional: true + + msgpackr@2.1.0: + optionalDependencies: + msgpackr-extract: 3.0.4 + + nanoid@3.3.19: {} + + node-gyp-build-optional-packages@5.2.2: + dependencies: + detect-libc: 2.1.2 + optional: true + + ohash@2.0.12: {} + + path-to-regexp@6.3.0: {} + + pathe@2.0.3: {} + + pathval@2.0.1: {} + + picocolors@1.1.1: {} + + picomatch@4.0.7: {} + + postcss@8.5.28: + dependencies: + nanoid: 3.3.19 + picocolors: 1.1.1 + source-map-js: 1.2.1 + + protobufjs@7.6.6: + dependencies: + '@protobufjs/aspromise': 1.1.2 + '@protobufjs/base64': 1.1.2 + '@protobufjs/codegen': 2.0.5 + '@protobufjs/eventemitter': 1.1.1 + '@protobufjs/fetch': 1.1.1 + '@protobufjs/float': 1.0.2 + '@protobufjs/path': 1.1.2 + '@protobufjs/pool': 1.1.0 + '@protobufjs/utf8': 1.1.2 + '@types/node': 22.20.4 + long: 5.3.2 + + pure-rand@8.4.2: {} + + require-directory@2.1.1: {} + + require-from-string@2.0.2: {} + + rollup@4.63.4: + dependencies: + '@types/estree': 1.0.9 + optionalDependencies: + '@napi-rs/lzma-linux-x64-gnu': 1.5.1 + '@rollup/rollup-android-arm-eabi': 4.63.4 + '@rollup/rollup-android-arm64': 4.63.4 + '@rollup/rollup-darwin-arm64': 4.63.4 + '@rollup/rollup-darwin-x64': 4.63.4 + '@rollup/rollup-freebsd-arm64': 4.63.4 + '@rollup/rollup-freebsd-x64': 4.63.4 + '@rollup/rollup-linux-arm-gnueabihf': 4.63.4 + '@rollup/rollup-linux-arm-musleabihf': 4.63.4 + '@rollup/rollup-linux-arm64-gnu': 4.63.4 + '@rollup/rollup-linux-arm64-musl': 4.63.4 + '@rollup/rollup-linux-loong64-gnu': 4.63.4 + '@rollup/rollup-linux-loong64-musl': 4.63.4 + '@rollup/rollup-linux-ppc64-gnu': 4.63.4 + '@rollup/rollup-linux-ppc64-musl': 4.63.4 + '@rollup/rollup-linux-riscv64-gnu': 4.63.4 + '@rollup/rollup-linux-riscv64-musl': 4.63.4 + '@rollup/rollup-linux-s390x-gnu': 4.63.4 + '@rollup/rollup-linux-x64-gnu': 4.63.4 + '@rollup/rollup-linux-x64-musl': 4.63.4 + '@rollup/rollup-openbsd-x64': 4.63.4 + '@rollup/rollup-openharmony-arm64': 4.63.4 + '@rollup/rollup-win32-arm64-msvc': 4.63.4 + '@rollup/rollup-win32-ia32-msvc': 4.63.4 + '@rollup/rollup-win32-x64-gnu': 4.63.4 + '@rollup/rollup-win32-x64-msvc': 4.63.4 + fsevents: 2.3.3 + + semver@7.8.5: {} + + sharp@0.33.5: + dependencies: + color: 4.2.3 + detect-libc: 2.1.2 + semver: 7.8.5 + optionalDependencies: + '@img/sharp-darwin-arm64': 0.33.5 + '@img/sharp-darwin-x64': 0.33.5 + '@img/sharp-libvips-darwin-arm64': 1.0.4 + '@img/sharp-libvips-darwin-x64': 1.0.4 + '@img/sharp-libvips-linux-arm': 1.0.5 + '@img/sharp-libvips-linux-arm64': 1.0.4 + '@img/sharp-libvips-linux-s390x': 1.0.4 + '@img/sharp-libvips-linux-x64': 1.0.4 + '@img/sharp-libvips-linuxmusl-arm64': 1.0.4 + '@img/sharp-libvips-linuxmusl-x64': 1.0.4 + '@img/sharp-linux-arm': 0.33.5 + '@img/sharp-linux-arm64': 0.33.5 + '@img/sharp-linux-s390x': 0.33.5 + '@img/sharp-linux-x64': 0.33.5 + '@img/sharp-linuxmusl-arm64': 0.33.5 + '@img/sharp-linuxmusl-x64': 0.33.5 + '@img/sharp-wasm32': 0.33.5 + '@img/sharp-win32-ia32': 0.33.5 + '@img/sharp-win32-x64': 0.33.5 + + siginfo@2.0.0: {} + + simple-swizzle@0.2.4: + dependencies: + is-arrayish: 0.3.4 + + smol-toml@1.8.0: {} + + source-map-js@1.2.1: {} + + stackback@0.0.2: {} + + std-env@3.10.0: {} + + stoppable@1.1.0: {} + + string-width@4.2.3: + dependencies: + emoji-regex: 8.0.0 + is-fullwidth-code-point: 3.0.0 + strip-ansi: 6.0.1 + + strip-ansi@6.0.1: + dependencies: + ansi-regex: 5.0.1 + + strip-literal@3.1.0: + dependencies: + js-tokens: 9.0.1 + + supports-color@10.2.2: {} + + tinybench@2.9.0: {} + + tinyexec@0.3.2: {} + + tinyglobby@0.2.17: + dependencies: + fdir: 6.5.0(picomatch@4.0.7) + picomatch: 4.0.7 + + tinypool@1.1.1: {} + + tinyrainbow@2.0.0: {} + + tinyspy@4.0.6: {} + + tslib@2.8.1: + optional: true + + tsx@4.23.15: + dependencies: + esbuild: 0.28.2 + optionalDependencies: + fsevents: 2.3.3 + + typescript@5.9.3: {} + + ufo@1.6.4: {} + + undici-types@6.21.0: {} + + undici@7.14.0: {} + + unenv@2.0.0-rc.21: + dependencies: + defu: 6.1.7 + exsolve: 1.1.1 + ohash: 2.0.12 + pathe: 2.0.3 + ufo: 1.6.4 + + vite-node@3.2.4(@types/node@22.20.4)(supports-color@10.2.2)(tsx@4.23.15): + dependencies: + cac: 6.7.14 + debug: 4.4.3(supports-color@10.2.2) + es-module-lexer: 1.7.0 + pathe: 2.0.3 + vite: 7.3.6(@types/node@22.20.4)(tsx@4.23.15) + transitivePeerDependencies: + - '@types/node' + - jiti + - less + - lightningcss + - sass + - sass-embedded + - stylus + - sugarss + - supports-color + - terser + - tsx + - yaml + + vite@7.3.6(@types/node@22.20.4)(tsx@4.23.15): + dependencies: + esbuild: 0.28.2 + fdir: 6.5.0(picomatch@4.0.7) + picomatch: 4.0.7 + postcss: 8.5.28 + rollup: 4.63.4 + tinyglobby: 0.2.17 + optionalDependencies: + '@types/node': 22.20.4 + fsevents: 2.3.3 + tsx: 4.23.15 + + vitest@3.2.7(@types/node@22.20.4)(supports-color@10.2.2)(tsx@4.23.15): + dependencies: + '@types/chai': 5.2.3 + '@vitest/expect': 3.2.7 + '@vitest/mocker': 3.2.7(vite@7.3.6(@types/node@22.20.4)(tsx@4.23.15)) + '@vitest/pretty-format': 3.2.7 + '@vitest/runner': 3.2.7 + '@vitest/snapshot': 3.2.7 + '@vitest/spy': 3.2.7 + '@vitest/utils': 3.2.7 + chai: 5.3.3 + debug: 4.4.3(supports-color@10.2.2) + expect-type: 1.4.0 + magic-string: 0.30.21 + pathe: 2.0.3 + picomatch: 4.0.7 + std-env: 3.10.0 + tinybench: 2.9.0 + tinyexec: 0.3.2 + tinyglobby: 0.2.17 + tinypool: 1.1.1 + tinyrainbow: 2.0.0 + vite: 7.3.6(@types/node@22.20.4)(tsx@4.23.15) + vite-node: 3.2.4(@types/node@22.20.4)(supports-color@10.2.2)(tsx@4.23.15) + why-is-node-running: 2.3.0 + optionalDependencies: + '@types/node': 22.20.4 + transitivePeerDependencies: + - jiti + - less + - lightningcss + - msw + - sass + - sass-embedded + - stylus + - sugarss + - supports-color + - terser + - tsx + - yaml + + why-is-node-running@2.3.0: + dependencies: + siginfo: 2.0.0 + stackback: 0.0.2 + + workerd@1.20251011.0: + optionalDependencies: + '@cloudflare/workerd-darwin-64': 1.20251011.0 + '@cloudflare/workerd-darwin-arm64': 1.20251011.0 + '@cloudflare/workerd-linux-64': 1.20251011.0 + '@cloudflare/workerd-linux-arm64': 1.20251011.0 + '@cloudflare/workerd-windows-64': 1.20251011.0 + + wrangler@4.44.0: + dependencies: + '@cloudflare/kv-asset-handler': 0.4.0 + '@cloudflare/unenv-preset': 2.7.8(unenv@2.0.0-rc.21)(workerd@1.20251011.0) + blake3-wasm: 2.1.5 + esbuild: 0.25.4 + miniflare: 4.20251011.0 + path-to-regexp: 6.3.0 + unenv: 2.0.0-rc.21 + workerd: 1.20251011.0 + optionalDependencies: + fsevents: 2.3.3 + transitivePeerDependencies: + - bufferutil + - utf-8-validate + + wrap-ansi@7.0.0: + dependencies: + ansi-styles: 4.3.0 + string-width: 4.2.3 + strip-ansi: 6.0.1 + + ws@8.18.0: {} + + y18n@5.0.8: {} + + yargs-parser@21.1.1: {} + + yargs@17.7.3: + dependencies: + cliui: 8.0.1 + escalade: 3.2.0 + get-caller-file: 2.0.5 + require-directory: 2.1.1 + string-width: 4.2.3 + y18n: 5.0.8 + yargs-parser: 21.1.1 + + youch-core@0.3.3: + dependencies: + '@poppinss/exception': 1.2.3 + error-stack-parser-es: 1.0.5 + + youch@4.1.0-beta.10: + dependencies: + '@poppinss/colors': 4.1.6 + '@poppinss/dumper': 0.6.5 + '@speed-highlight/core': 1.2.24 + cookie: 1.1.1 + youch-core: 0.3.3 + + zod@3.22.3: {} + + zod@3.25.76: {} diff --git a/pkgs/substrate-apps/src/pnpm-workspace.yaml b/pkgs/substrate-apps/src/pnpm-workspace.yaml new file mode 100644 index 00000000..d890b395 --- /dev/null +++ b/pkgs/substrate-apps/src/pnpm-workspace.yaml @@ -0,0 +1,12 @@ +# One workspace, one effect instance (ES-3): every package declares effect as a +# peerDependency at the same exact version as the root, so pnpm links a single +# physical copy. test/workspace-single-effect.test.ts is the oracle. +packages: + - "packages/*" + - "apps/*" +allowBuilds: + esbuild: true + msgpackr-extract: false + protobufjs: false + workerd: false + sharp: false diff --git a/pkgs/substrate-apps/src/tsconfig.base.json b/pkgs/substrate-apps/src/tsconfig.base.json new file mode 100644 index 00000000..7dc1891f --- /dev/null +++ b/pkgs/substrate-apps/src/tsconfig.base.json @@ -0,0 +1,20 @@ +{ + "$schema": "https://json.schemastore.org/tsconfig", + "compilerOptions": { + "target": "ES2022", + "lib": ["ES2022"], + "module": "ESNext", + "moduleResolution": "bundler", + "moduleDetection": "force", + "verbatimModuleSyntax": true, + "isolatedModules": true, + "strict": true, + "exactOptionalPropertyTypes": true, + "noUncheckedIndexedAccess": true, + "noImplicitOverride": true, + "noFallthroughCasesInSwitch": true, + "noEmit": true, + "skipLibCheck": true, + "forceConsistentCasingInFileNames": true + } +} diff --git a/pkgs/substrate-apps/src/tsconfig.json b/pkgs/substrate-apps/src/tsconfig.json new file mode 100644 index 00000000..eb9a0298 --- /dev/null +++ b/pkgs/substrate-apps/src/tsconfig.json @@ -0,0 +1,16 @@ +{ + "compilerOptions": { + "target": "ES2022", + "module": "NodeNext", + "moduleResolution": "NodeNext", + "lib": ["ES2022"], + "strict": true, + "exactOptionalPropertyTypes": false, + "noUncheckedIndexedAccess": true, + "allowImportingTsExtensions": true, + "noEmit": true, + "skipLibCheck": true, + "types": ["node"] + }, + "include": ["src", "test"] +} diff --git a/pkgs/substrate-apps/sync.sh b/pkgs/substrate-apps/sync.sh new file mode 100755 index 00000000..0d7b5ecf --- /dev/null +++ b/pkgs/substrate-apps/sync.sh @@ -0,0 +1,23 @@ +#!/usr/bin/env bash +# pkgs/substrate-apps/sync.sh +# Re-vendors ./src from one commit of agency-agency/substrate: the workspace files plus apps/link, +# apps/pusher and, once it exists upstream, apps/puller. Then updates the sha line in SYNC.md and the +# sourceSha in default.nix. The pnpm hash is refreshed by hand (a failing build), never guessed. +set -euo pipefail +checkout=${1:?usage: sync.sh } +sha=${2:?usage: sync.sh } +here=$(cd "$(dirname "$0")" && pwd) +full=$(git -C "$checkout" rev-parse --verify "$sha^{commit}") +paths=(package.json pnpm-lock.yaml pnpm-workspace.yaml tsconfig.base.json tsconfig.json apps/link apps/pusher) +puller=absent +if git -C "$checkout" cat-file -e "$full:apps/puller" 2>/dev/null; then + paths+=(apps/puller) + puller=present +fi +rm -rf "$here/src" +mkdir -p "$here/src" +git -C "$checkout" archive "$full" "${paths[@]}" | tar -x -C "$here/src" +sed -i "s/^- Source sha: .*/- Source sha: \`$full\` (synced $(date -u +%FT%TZ), apps\/puller $puller)/" "$here/SYNC.md" +sed -i "s/^ sourceSha = \".*\";/ sourceSha = \"$full\";/" "$here/default.nix" +echo "vendored $full into $here/src: ${paths[*]} (apps/puller $puller)" +echo "next: set pnpmDeps.hash to lib.fakeHash in default.nix, nix build .#substrate-apps-link, paste the got: value, rebuild, commit" From 0861cebac46d3dcfa0a271b23c4de3ce0493815a Mon Sep 17 00:00:00 2001 From: mecattaf Date: Wed, 23 Sep 2026 23:23:28 +0200 Subject: [PATCH 2/9] substrate: services.substrate.pusher and .puller on the coordinator, declared OFF modules/substrate.nix: the coordinator side of the Cloudflare Substrate. Two systemd user units for Tom's manager (ConditionUser), both gated off: pusher substrate-pusher --config --token-file %d/floor-token. seats --json --no-spend under the gentle policy, posted to POST /capacity/snapshots. Seats cc, cc2, codex, pi-qwencloud, halogen; gpu-worker published as halogen; cc3 not dispatchable; owners.codex = tom on the coordinator (E6). puller the interpreter host. PENDING its package (apps/puller is not upstream yet), so package defaults to null and arming asserts. The unit renders runtimes.toml (host, herdr, gvisor with nix runsc and pasta, ssh:worker as pi on halogen; default host; [seats] claude = cc2) into AX_CONWIP_RUNTIMES, a substrate.json into its RuntimeDirectory with capacityFloorTokenFile pointing at the credential, and CLAUDE_CONFIG_DIR=/home/tom/.claude-work (cc2, by path). An optional runtime-test wrapper around ExecStart. The bearer is an agenix secret NAME (substrate-floor-token, owner tom, 0400) handed to each unit by LoadCredential; no rendered file or unit line carries a value. hosts/coordinator imports the module and sets floorUrl https://substrate.mecattaf.dev with both enables false. tests/substrate-modules (checks.substrate-modules): eval-time assertions that the gates are OFF and no unit or secret is rendered on the real host, then the same host armed through extendModules with fixtures, whose pusher.json and runtimes.toml the check parses (TOML decodes, runtime types known, no token-shaped key). Co-Authored-By: Claude Fable 5.1 --- flake.nix | 6 + hosts/coordinator/default.nix | 17 ++ modules/substrate.nix | 455 ++++++++++++++++++++++++++++ tests/substrate-modules/default.nix | 84 +++++ 4 files changed, 562 insertions(+) create mode 100644 modules/substrate.nix create mode 100644 tests/substrate-modules/default.nix diff --git a/flake.nix b/flake.nix index ec00ec4c..afcef388 100644 --- a/flake.nix +++ b/flake.nix @@ -787,6 +787,12 @@ inherit pkgs self; inherit (nixpkgs) lib; }; + # modules/substrate.nix rendered OFF on the coordinator and armed with + # fixtures (tests/substrate-modules); every assertion is eval-time. + substrate-modules = import ./tests/substrate-modules { + inherit pkgs self; + inherit (nixpkgs) lib; + }; qwen-speech = pkgs.runCommand "qwen-speech-tests" diff --git a/hosts/coordinator/default.nix b/hosts/coordinator/default.nix index f0a7f558..88d7d875 100644 --- a/hosts/coordinator/default.nix +++ b/hosts/coordinator/default.nix @@ -85,10 +85,27 @@ # all three interactive hosts, OFF on all three; read that module's header # for the runbook and for what it deliberately does not declare. ../../modules/ax-client.nix + # The Cloudflare Substrate's coordinator side (2026-09-23, E1/A2): the + # gentle capacity pusher and the interpreter-host puller, both declared + # OFF below. The NAS side is hosts/nas/substrate-link.nix. + ../../modules/substrate.nix ]; networking.hostName = "coordinator"; + # ── Substrate on this box: declared, OFF ─────────────────────────────── + # modules/substrate.nix. Arming either needs the floor live at floorUrl, + # the sealed FLOOR_TOKEN (secrets/substrate-floor-token.age, owner tom) and, + # for the puller, its package (pkgs/substrate-apps/SYNC.md: pending). + services.substrate = { + floorUrl = "https://substrate.mecattaf.dev"; + pusher = { + enable = false; + owners.codex = "tom"; # E6 (2026-09-23): the codex login is Tom's + }; + puller.enable = false; + }; + # ── ax on the fleet: THE kill switch for this host ───────────────────── # The HARNESS node (modules/ax-fleet/harness.nix): a k3s agent tainted # ate.dev/sandboxClass=gvisor:NoSchedule, so only atelet and the gVisor diff --git a/modules/substrate.nix b/modules/substrate.nix new file mode 100644 index 00000000..0ec5bc2e --- /dev/null +++ b/modules/substrate.nix @@ -0,0 +1,455 @@ +{ + config, + lib, + pkgs, + ... +}: +# services.substrate: the coordinator's two box-side programs of the Cloudflare Substrate, declared and OFF. +# +# CODE: pkgs/substrate-apps (vendored from agency-agency/substrate at the sha in its SYNC.md). ARCHITECTURE: +# ~/today/wednesday-prep-2026-09-23/03-substrate-ax-conwip.md section 2. RULING: E1 (2026-09-23), the code that +# wraps ax lives in dotfiles; A2, the NAS link, the coordinator puller and the capacity pusher are dotfiles +# packages and modules. The NAS side is hosts/nas/substrate-link.nix (Lane A); this file is the coordinator side. +# +# pusher reads `seats --json --no-spend` (the box's one capacity oracle, ~/.local/bin/seats) under the gentle +# policy (120 s floor, 900 s idle, 300 s active, 180 s above 85 percent, one read at reset + 90 s) and +# posts seat-capacity/2 snapshots to the floor's POST /capacity/snapshots. A dead pusher fails closed +# at the floor: readings grade STALE after 1200 s and admission stops. +# puller the interpreter host: leases runtime:interpreter work from the floor and runs each workflow's agent() +# calls on the runtimes this module renders into runtimes.toml (host, herdr, gvisor, ssh:worker) with +# the cc2 seat's config dir by path. PENDING upstream: apps/puller is not in the pinned source yet +# (pkgs/substrate-apps/SYNC.md), so `package` defaults to null and arming asserts. +# +# WHY USER UNITS. Both programs speak for Tom's logins: `seats` reads each Claude seat's OAuth usage from its +# config dir, and the puller runs claude, herdr and ssh as Tom. A system unit with User=tom would carry none of +# the session (herdr socket, ssh agent, XDG dirs). ConditionUser pins each unit to that one user's manager. +# +# THE TOKEN IS A PATH, NEVER A VALUE. The floor's operator bearer (FLOOR_TOKEN) is sealed as an agenix secret, +# decrypted at /run/agenix/ for the user (owner = user, 0400), and handed to each unit as a private +# copy by LoadCredential; the programs get `%d/floor-token` (the pusher's --token-file; the puller's +# capacityFloorTokenFile through a runtime-rendered substrate.json). Nothing here reads, prints or interpolates +# the value; the rendered pusher.json and substrate.json carry only paths. Minting the token (wrangler secret put +# FLOOR_TOKEN and this age file) is Tom's hand. +# +# THE GATE. Both `enable`s default to false and hosts/coordinator sets them false explicitly. Arming needs the +# floor deployed at floorUrl, the sealed token, and for the puller its package. tests/substrate-modules renders +# both units armed with fixtures so the shape is proven at evaluation time without a token on disk. +let + cfg = config.services.substrate; + inherit (lib) + mkEnableOption + mkIf + mkOption + types + ; + substrateApps = pkgs.callPackage ../pkgs/substrate-apps { }; + home = config.users.users.${cfg.user}.home; + tomlFormat = pkgs.formats.toml { }; + anyEnabled = cfg.pusher.enable || cfg.puller.enable; + credentialPath = config.age.secrets.${cfg.tokenSecret}.path; + + # The pusher's config file: apps/pusher/pusher.example.json shape, minus tokenFile (given as --token-file). + pusherConfig = pkgs.writeText "substrate-pusher.json" ( + builtins.toJSON ( + { + floorUrl = cfg.floorUrl; + seatsBin = cfg.pusher.seatsBin; + host = cfg.pusher.host; + seats = cfg.pusher.seats; + seatIds = cfg.pusher.seatIds; + owners = cfg.pusher.owners; + not_dispatchable = cfg.pusher.notDispatchable; + plans = cfg.pusher.plans; + slots = cfg.pusher.slots; + estimatedProviders = cfg.pusher.estimatedProviders; + } + // cfg.pusher.extraConfig + ) + ); + + runtimesToml = tomlFormat.generate "substrate-runtimes.toml" cfg.puller.runtimes; + + # The deploy config (src/deploy-config.ts) with the bearer path left as a placeholder; the unit fills it from + # $CREDENTIALS_DIRECTORY at start, into its RuntimeDirectory, so no path to a credential is guessed at eval. + deployConfigTemplate = pkgs.writeText "substrate.json.in" ( + builtins.toJSON ( + cfg.puller.deployConfig + // { + capacityFloorUrl = cfg.floorUrl; + capacityFloorTokenFile = "@FLOOR_TOKEN_FILE@"; + } + ) + ); + + pullerLabels = lib.concatStringsSep "," cfg.puller.servedLabels; + pullerExec = lib.escapeShellArgs ( + lib.optionals (cfg.puller.runtimeTestWrapper != null) [ + cfg.puller.runtimeTestWrapper + "--" + ] + ++ [ + (lib.getExe cfg.puller.package) + "--floor" + cfg.floorUrl + "--holder" + cfg.puller.holder + "--labels" + pullerLabels + "--max-in-flight" + (toString cfg.puller.maxInFlight) + ] + ++ cfg.puller.extraArgs + ); + + pullerStart = pkgs.writeShellScript "substrate-puller-start" '' + set -euo pipefail + : "''${CREDENTIALS_DIRECTORY:?substrate-puller needs LoadCredential (no CREDENTIALS_DIRECTORY)}" + : "''${RUNTIME_DIRECTORY:?substrate-puller needs RuntimeDirectory}" + token="$CREDENTIALS_DIRECTORY/floor-token" + test -r "$token" || { echo "substrate-puller: no floor-token credential" >&2; exit 78; } + umask 077 + ${pkgs.gnused}/bin/sed "s|@FLOOR_TOKEN_FILE@|$token|" ${deployConfigTemplate} > "$RUNTIME_DIRECTORY/substrate.json" + export SUBSTRATE_CONFIG="$RUNTIME_DIRECTORY/substrate.json" + export AX_CONWIP_RUNTIMES=${runtimesToml} + exec ${pullerExec} --token-file "$token" + ''; + + commonService = { + Restart = "on-failure"; + RestartSec = "30s"; + TimeoutStopSec = "30s"; + UMask = "0077"; + NoNewPrivileges = true; + LoadCredential = [ "floor-token:${credentialPath}" ]; + }; +in +{ + options.services.substrate = { + user = mkOption { + type = types.str; + default = "tom"; + description = "The login whose user manager runs both units and whose seats they speak for."; + }; + + floorUrl = mkOption { + type = types.str; + default = ""; + example = "https://substrate.example.dev"; + description = "The floor Worker's base URL. HTTPS only; the bearer never travels in clear."; + }; + + tokenSecret = mkOption { + type = types.str; + default = "substrate-floor-token"; + description = "The agenix secret NAME holding the floor's operator bearer (FLOOR_TOKEN). Declared by this module when a unit is enabled, decrypted for `user`, handed to each unit as the credential `floor-token`. A name, never a value."; + }; + + tokenAgeFile = mkOption { + type = types.path; + default = ../secrets/substrate-floor-token.age; + defaultText = lib.literalExpression "../secrets/substrate-floor-token.age"; + description = "The sealed bearer (secrets.nix: editors ++ the coordinator key). Evaluated only when a unit is enabled."; + }; + + pusher = { + enable = mkEnableOption "the gentle capacity pusher (seats --json to the floor). OFF; see this file's header"; + + package = mkOption { + type = types.package; + default = substrateApps.pusher; + defaultText = lib.literalExpression "(pkgs.callPackage ../pkgs/substrate-apps { }).pusher"; + description = "The pusher, built from pkgs/substrate-apps/src/apps/pusher."; + }; + + configFile = mkOption { + type = types.path; + readOnly = true; + default = pusherConfig; + description = "The rendered pusher.json (read-only; for tests)."; + }; + + seatsBin = mkOption { + type = types.str; + default = "${home}/.local/bin/seats"; + defaultText = lib.literalExpression ''"''${home}/.local/bin/seats"''; + description = "The capacity oracle. A chezmoi-managed script (home/dot_local/bin/seats), stdlib python."; + }; + + host = mkOption { + type = types.str; + default = config.networking.hostName; + defaultText = lib.literalExpression "config.networking.hostName"; + description = "The host name the snapshot carries."; + }; + + seats = mkOption { + type = types.listOf types.str; + default = [ + "cc" + "cc2" + "codex" + "pi-qwencloud" + "halogen" + ]; + description = "Seats published to the floor (runs-on seat ids)."; + }; + + seatIds = mkOption { + type = types.attrsOf types.str; + default = { + gpu-worker = "halogen"; + }; + description = "Oracle seat id to runs-on seat id."; + }; + + owners = mkOption { + type = types.attrsOf types.str; + default = { }; + example = { + codex = "tom"; + }; + description = "Overrides the oracle's third-party marks (E6: codex is Tom's login tonight)."; + }; + + notDispatchable = mkOption { + type = types.attrsOf types.str; + default = { + cc3 = "evicted"; + }; + description = "Seats never published, with the reason."; + }; + + plans = mkOption { + type = types.attrsOf types.str; + default = { }; + description = "Plan expiry per seat (ISO instant)."; + }; + + slots = mkOption { + type = types.attrsOf types.int; + default = { + halogen = 1; + }; + description = "Slot capacity for seats the oracle reports no holders for (rule 15: the floor subtracts its own WIP)."; + }; + + estimatedProviders = mkOption { + type = types.listOf types.str; + default = [ "qwen" ]; + description = "Providers whose readings grade ESTIMATED (never admitted unless the floor ungates them)."; + }; + + extraConfig = mkOption { + type = types.attrs; + default = { }; + description = "Merged last into pusher.json. Never a token."; + }; + }; + + puller = { + enable = mkEnableOption "the coordinator puller (the interpreter host). OFF; PENDING its package, see pkgs/substrate-apps/SYNC.md"; + + package = mkOption { + type = types.nullOr types.package; + default = substrateApps.puller or null; + defaultText = lib.literalExpression "(pkgs.callPackage ../pkgs/substrate-apps { }).puller or null"; + description = "The puller. null until apps/puller is vendored; arming asserts on it."; + }; + + holder = mkOption { + type = types.str; + default = "coordinator-puller-1"; + description = "holderIdentity at the floor; one session per identity."; + }; + + servedLabels = mkOption { + type = types.listOf types.str; + default = [ "runtime:interpreter" ]; + description = "runs-on labels this puller leases (ARC W6: the coordinator serves the interpreter and the Claude seats; the NAS link serves seat:halogen runtime:gvisor)."; + }; + + maxInFlight = mkOption { + type = types.ints.positive; + default = 1; + description = "Runs interpreted at once on this host."; + }; + + claudeSeat = mkOption { + type = types.str; + default = "cc2"; + description = "The Claude seat this host's claude harness spends ([seats].claude and TALLY_SEAT)."; + }; + + claudeConfigDir = mkOption { + type = types.str; + default = "${home}/.claude-work"; + defaultText = lib.literalExpression ''"''${home}/.claude-work"''; + description = "CLAUDE_CONFIG_DIR for that seat, by path (cc2 = ~/.claude-work, the seats oracle's table). Mounted, never copied, into sandboxed runtimes."; + }; + + runtimeTestWrapper = mkOption { + type = types.nullOr types.str; + default = null; + example = "/home/tom/.local/bin/runtime-test"; + description = "When set, ExecStart runs under this wrapper (` -- ...`): a private /run/user tree and PID/IPC namespaces, which the microvm runtime requires (packages/runners/src/microvm.ts). Off by default: herdr and ssh want the live session."; + }; + + runtimes = mkOption { + type = tomlFormat.type; + default = { + default = "host"; + allow = [ + "host" + "herdr" + "gvisor" + "ssh:worker" + ]; + seats = { + claude = cfg.puller.claudeSeat; + pi = "halogen"; + codex = "codex"; + }; + credentials = { + claude = cfg.puller.claudeConfigDir; + mode = "rw"; + scope = "credential"; + }; + runtime = { + host = { + type = "host"; + harness = "claude"; + }; + herdr = { + type = "herdr"; + harness = "claude"; + mode = "action"; + }; + gvisor = { + type = "gvisor"; + harness = "claude"; + runsc = "${pkgs.gvisor}/bin/runsc"; + pasta = "${pkgs.passt}/bin/pasta"; + state = "${home}/.local/state/substrate/runsc"; + network = "isolated"; + }; + "ssh:worker" = { + type = "ssh"; + host = "worker"; + harness = "pi"; + seat = "halogen"; + }; + }; + }; + description = "The runtimes file (packages/runners/src/config.ts), rendered to TOML and passed as AX_CONWIP_RUNTIMES. Defaults: host, herdr, gvisor (nix runsc and pasta) and ssh:worker (pi on Halogen), default host, cc2's config dir as the claude credential."; + }; + + runtimesFile = mkOption { + type = types.path; + readOnly = true; + default = runtimesToml; + description = "The rendered runtimes.toml (read-only; for tests)."; + }; + + deployConfig = mkOption { + type = types.attrs; + default = { + forbiddenReadPrefixes = [ + "~/rawa" + "~/.local/state/secrets-holdout-2026-09-18" + ]; + forbiddenLedgerPrefixes = [ + "/run/user" + "~/.local/state/tally-rewrite" + ]; + metersDir = "~/.local/state/tally-rewrite/meters"; + axProtoFallbackPath = null; + realSeats = [ + "cc" + "cc2" + "cc3" + "halogen" + "codex" + ]; + codexWorkdir = "~/mecattaf/substrate"; + }; + description = "src/deploy-config.ts keys (SUBSTRATE_CONFIG). capacityFloorUrl and capacityFloorTokenFile are set by the unit from floorUrl and the credential."; + }; + + extraArgs = mkOption { + type = types.listOf types.str; + default = [ ]; + description = "Appended to the puller's command line."; + }; + + extraPath = mkOption { + type = types.listOf types.package; + default = [ ]; + description = "Packages on the unit's PATH besides openssh and coreutils (claude, pi, herdr come from the user profile)."; + }; + }; + }; + + config = mkIf anyEnabled { + assertions = [ + { + assertion = lib.hasPrefix "https://" cfg.floorUrl; + message = "services.substrate.floorUrl must be an https:// URL (the bearer never travels in clear)."; + } + { + assertion = !cfg.puller.enable || cfg.puller.package != null; + message = "services.substrate.puller has no package: apps/puller is not vendored yet (pkgs/substrate-apps/SYNC.md, Pending)."; + } + { + assertion = !cfg.pusher.enable || !(cfg.pusher.extraConfig ? tokenFile); + message = "services.substrate.pusher.extraConfig must not name a tokenFile: the bearer arrives as the credential floor-token."; + } + ]; + + age.secrets.${cfg.tokenSecret} = { + file = cfg.tokenAgeFile; + owner = cfg.user; + mode = "0400"; + }; + + systemd.user.services.substrate-pusher = mkIf cfg.pusher.enable { + description = "substrate-pusher: seats --json to the Substrate floor, gently"; + wantedBy = [ "default.target" ]; + unitConfig.ConditionUser = cfg.user; + path = [ + pkgs.python3 + pkgs.coreutils + ]; + serviceConfig = commonService // { + ExecStart = "${lib.getExe cfg.pusher.package} --config ${pusherConfig} --token-file %d/floor-token"; + # 2 usage, 3 another pusher holds the pidfile: neither is cured by a restart. + RestartPreventExitStatus = [ + 2 + 3 + ]; + MemoryMax = "256M"; + }; + }; + + systemd.user.services.substrate-puller = mkIf cfg.puller.enable { + description = "substrate-puller: the Substrate interpreter host on this box"; + wantedBy = [ "default.target" ]; + unitConfig.ConditionUser = cfg.user; + path = [ + pkgs.openssh + pkgs.coreutils + ] + ++ cfg.puller.extraPath; + environment = { + CLAUDE_CONFIG_DIR = cfg.puller.claudeConfigDir; + TALLY_SEAT = cfg.puller.claudeSeat; + }; + serviceConfig = commonService // { + ExecStart = pullerStart; + RuntimeDirectory = "substrate-puller"; + RuntimeDirectoryMode = "0700"; + RestartPreventExitStatus = [ 78 ]; + MemoryMax = "2G"; + }; + }; + }; +} diff --git a/tests/substrate-modules/default.nix b/tests/substrate-modules/default.nix new file mode 100644 index 00000000..8d98406d --- /dev/null +++ b/tests/substrate-modules/default.nix @@ -0,0 +1,84 @@ +{ + pkgs, + self, + lib, +}: +# substrate-modules: the rendered shape of modules/substrate.nix, proven at evaluation time. +# +# `nix flake check --no-build` evaluates this derivation's inputs, which forces the coordinator's +# services.substrate config twice: as declared (both gates OFF, no unit rendered, no secret declared) and +# armed through extendModules with fixtures (a stub puller, an empty file standing in for the age file), which +# renders both user units, the pusher.json and the runtimes.toml. `nix build .#checks..substrate-modules` +# then parses both files: the TOML decodes, every runtime table has a type the runners know, ssh:worker and the +# cc2 config dir are present, and no rendered file carries a token-shaped key. +let + coord = self.nixosConfigurations.coordinator; + declared = coord.config; + armed = + (coord.extendModules { + modules = [ + { + services.substrate = { + pusher.enable = lib.mkForce true; + puller.enable = lib.mkForce true; + puller.package = pkgs.writeShellScriptBin "substrate-puller" "exit 78"; + tokenAgeFile = pkgs.writeText "substrate-floor-token.age" "eval fixture, not a secret"; + }; + } + ]; + }).config; + pusherUnit = armed.systemd.user.services.substrate-pusher; + pullerUnit = armed.systemd.user.services.substrate-puller; + hasInfix = lib.hasInfix; +in +# Declared OFF on the real host, and the import survived (the option exists). +assert declared.services ? substrate; +assert declared.services.substrate.pusher.enable == false; +assert declared.services.substrate.puller.enable == false; +assert declared.services.substrate.floorUrl == "https://substrate.mecattaf.dev"; +assert !(declared.systemd.user.services ? substrate-pusher); +assert !(declared.systemd.user.services ? substrate-puller); +assert !(declared.age.secrets ? substrate-floor-token); +# Armed: both units exist, both take the bearer as a credential, never as a value. +assert armed.age.secrets.substrate-floor-token.owner == "tom"; +assert + pusherUnit.serviceConfig.LoadCredential == [ "floor-token:/run/agenix/substrate-floor-token" ]; +assert + pullerUnit.serviceConfig.LoadCredential == [ "floor-token:/run/agenix/substrate-floor-token" ]; +assert hasInfix "--token-file %d/floor-token" pusherUnit.serviceConfig.ExecStart; +assert pusherUnit.unitConfig.ConditionUser == "tom"; +assert pullerUnit.environment.CLAUDE_CONFIG_DIR == "/home/tom/.claude-work"; +assert pullerUnit.environment.TALLY_SEAT == "cc2"; +assert pullerUnit.serviceConfig.RuntimeDirectory == "substrate-puller"; +pkgs.runCommand "substrate-modules" + { + nativeBuildInputs = [ pkgs.python3 ]; + pusherJson = armed.services.substrate.pusher.configFile; + runtimesToml = armed.services.substrate.puller.runtimesFile; + pullerStart = pullerUnit.serviceConfig.ExecStart; + } + '' + python3 - "$pusherJson" "$runtimesToml" "$pullerStart" <<'PY' + import json, sys, tomllib + pusher = json.load(open(sys.argv[1])) + assert pusher["floorUrl"].startswith("https://"), pusher + assert "tokenFile" not in pusher and not any("token" in k.lower() for k in pusher), list(pusher) + assert pusher["seats"] == ["cc", "cc2", "codex", "pi-qwencloud", "halogen"], pusher["seats"] + assert pusher["seatIds"] == {"gpu-worker": "halogen"} + assert pusher["seatsBin"] == "/home/tom/.local/bin/seats" + rt = tomllib.load(open(sys.argv[2], "rb")) + known = {"host", "herdr", "gvisor", "microvm", "ssh", "workerd", "ax"} + for name, table in rt["runtime"].items(): + assert table["type"] in known, (name, table) + assert rt["default"] == "host" + assert rt["runtime"]["ssh:worker"] == {"type": "ssh", "host": "worker", "harness": "pi", "seat": "halogen"} + assert rt["runtime"]["gvisor"]["runsc"].endswith("/bin/runsc") and rt["runtime"]["gvisor"]["pasta"].endswith("/bin/pasta") + assert rt["credentials"]["claude"] == "/home/tom/.claude-work" + assert rt["seats"] == {"claude": "cc2", "pi": "halogen", "codex": "codex"} + assert set(rt["allow"]) == {"host", "herdr", "gvisor", "ssh:worker"} + start = open(sys.argv[3]).read() + assert "@FLOOR_TOKEN_FILE@" in start and "CREDENTIALS_DIRECTORY" in start and "AX_CONWIP_RUNTIMES=" in start + print("substrate-modules: pusher.json and runtimes.toml render as expected") + PY + touch "$out" + '' From 2d97f40cf0114c1dd911d83ec5ec576dfd4edc63 Mon Sep 17 00:00:00 2001 From: mecattaf Date: Wed, 23 Sep 2026 23:33:25 +0200 Subject: [PATCH 3/9] substrate-apps: resync to b105179, vendor and build apps/puller; puller unit on the real [puller] config Re-vendored from agency-agency/substrate origin/main b1051790f376c103ba4e901619ba11efeb78cef6 (017893d added apps/puller, the coordinator-side interpreter host; b105179 is docs). The vendored set now carries src/, proto/, packages/* and deploy/ as well, because the puller runs through tsx's ESM loader over the workspace (substrate root, @substrate/{api,link,interpreter,runners}); apps/floor, apps/cli, apps/mcp, test/, tools/ and docs/ stay out. sync.sh's path list grew accordingly. The pnpm hash did not change (MEASURED). pkgs/substrate-apps.puller (`.#substrate-puller`): pnpm install --prod from the shared pnpmDeps, the tree with its production node_modules copied to $out/lib/substrate-apps, node wrapped over apps/puller/bin/substrate-puller.mjs (86M, closure 327 MB, MEASURED). checkPhase starts it with no config and requires exit 78 with a config-invalid line, so every import resolved and main() ran. modules/substrate.nix puller: package defaults to the real puller; its configuration is the [puller] table of a client config.toml (packages/api/src/config.ts): holder coordinator, link_token_file, state_dir, pidfile, max_runs 1, node_dispatch local, runtimes (the rendered TOML), seat cc2, cap 2, default_model, node_runs_on, optional ax_server. The unit renders config.toml and substrate.json into its RuntimeDirectory at start with the two credential paths (floor-token, link-token: a second agenix secret name, substrate-link-token-coordinator) and exports SUBSTRATE_CLIENT_CONFIG, SUBSTRATE_CONFIG, AX_CONWIP_RUNTIMES. The INFERRED command-line flags of the previous commit are gone. tests/substrate-modules: asserts both real packages on the declared host (evaluated, not built), the two LoadCredential entries, and parses the rendered config.toml (placeholders in place, seat cc2, holder coordinator, no /run/agenix or /run/user path in any rendered file). Co-Authored-By: Claude Fable 5.1 --- flake.nix | 6 +- modules/substrate.nix | 194 ++- pkgs/substrate-apps/SYNC.md | 33 +- pkgs/substrate-apps/default.nix | 59 +- .../src/apps/puller/bin/substrate-puller.mjs | 10 + .../src/apps/puller/package.json | 14 + .../src/apps/puller/src/connect.ts | 13 + .../src/apps/puller/src/execute.ts | 176 +++ .../src/apps/puller/src/main.ts | 64 + .../src/apps/puller/src/pidfile.ts | 21 + .../src/apps/puller/src/puller.ts | 256 ++++ .../src/apps/puller/src/state.ts | 35 + .../src/apps/puller/test/puller.test.ts | 183 +++ .../src/apps/puller/tsconfig.json | 5 + .../src/apps/puller/vitest.config.ts | 2 + .../src/deploy/client.config.example.toml | 24 + .../src/deploy/substrate.config.example.json | 21 + .../src/deploy/wrangler.local.example.jsonc | 43 + pkgs/substrate-apps/src/package.json | 2 +- .../src/packages/api/package.json | 18 + .../src/packages/api/src/api.ts | 82 + .../src/packages/api/src/client.ts | 138 ++ .../src/packages/api/src/config.ts | 76 + .../src/packages/api/src/index.ts | 4 + .../src/packages/api/src/schema.ts | 110 ++ .../src/packages/api/test/api.test.ts | 54 + .../src/packages/api/tsconfig.json | 5 + .../src/packages/api/vitest.config.ts | 2 + .../src/packages/factory/README.md | 42 + .../src/packages/factory/package.json | 30 + .../src/packages/factory/src/cardWriter.ts | 192 +++ .../src/packages/factory/src/http.ts | 454 ++++++ .../src/packages/factory/src/index.ts | 11 + .../src/packages/factory/src/inputs.ts | 130 ++ .../src/packages/factory/src/projection.ts | 410 +++++ .../packages/factory/src/projectionLoad.ts | 186 +++ .../packages/factory/src/projectionMirror.ts | 228 +++ .../src/packages/factory/src/receipt.ts | 48 + .../packages/factory/test/capacity.test.ts | 245 +++ .../factory/test/capacityReview1.test.ts | 283 ++++ .../src/packages/factory/test/factory.test.ts | 384 +++++ .../src/packages/factory/test/fixtures.ts | 109 ++ .../src/packages/factory/test/http.test.ts | 241 +++ .../src/packages/factory/test/inputs.test.ts | 58 + .../packages/factory/test/persistence.test.ts | 70 + .../packages/factory/test/projection.test.ts | 317 ++++ .../factory/test/projectionMirror.test.ts | 247 +++ .../src/packages/factory/tsconfig.json | 9 + .../src/packages/factory/vitest.config.ts | 8 + .../packages/interpreter/bin/naive-run.mjs | 12 + .../src/packages/interpreter/package.json | 32 + .../src/packages/interpreter/src/backend.ts | 207 +++ .../src/packages/interpreter/src/cli.ts | 304 ++++ .../src/packages/interpreter/src/index.ts | 8 + .../packages/interpreter/src/interpreter.ts | 619 ++++++++ .../src/packages/interpreter/src/journal.ts | 287 ++++ .../packages/interpreter/src/jsonschema.ts | 73 + .../src/packages/interpreter/src/key.ts | 89 ++ .../src/packages/interpreter/src/meta.ts | 259 ++++ .../src/packages/interpreter/src/replay.ts | 182 +++ .../src/packages/interpreter/src/sandbox.ts | 158 ++ .../interpreter/test/conformance.test.ts | 324 ++++ .../packages/interpreter/test/edges.test.ts | 189 +++ .../interpreter/test/fidelity.test.ts | 164 ++ .../src/packages/interpreter/test/helpers.ts | 44 + .../packages/interpreter/test/meta.test.ts | 59 + .../interpreter/test/semantics.test.ts | 367 +++++ .../interpreter/test/successor-r1.test.ts | 149 ++ .../interpreter/test/successor-r2.test.ts | 119 ++ .../interpreter/test/successor-r3.test.ts | 114 ++ .../interpreter/test/successor-r5.test.ts | 54 + .../src/packages/interpreter/tsconfig.json | 16 + .../src/packages/interpreter/vitest.config.ts | 9 + .../src/packages/planning/README.md | 15 + .../src/packages/planning/package.json | 26 + .../packages/planning/src/capacity/index.ts | 6 + .../packages/planning/src/capacity/ledger.ts | 295 ++++ .../planning/src/capacity/model-families.json | 4 + .../packages/planning/src/capacity/project.ts | 578 +++++++ .../planning/src/heuristics/bufferAndon.ts | 71 + .../planning/src/heuristics/conwip.ts | 158 ++ .../planning/src/heuristics/eddCertificate.ts | 153 ++ .../planning/src/heuristics/envelope.ts | 131 ++ .../planning/src/heuristics/fillerLane.ts | 250 +++ .../packages/planning/src/heuristics/index.ts | 35 + .../planning/src/heuristics/killRedispatch.ts | 35 + .../planning/src/heuristics/lengthTermFlip.ts | 126 ++ .../planning/src/heuristics/localFirst.ts | 174 +++ .../src/heuristics/namespaceRoundRobin.ts | 128 ++ .../planning/src/heuristics/paceLine.ts | 106 ++ .../planning/src/heuristics/redundancy.ts | 42 + .../planning/src/heuristics/runtimeCap.ts | 60 + .../src/heuristics/subassemblyBatching.ts | 164 ++ .../planning/src/heuristics/valueDensity.ts | 142 ++ .../src/packages/planning/src/index.ts | 19 + .../packages/planning/src/objects/factory.ts | 1354 +++++++++++++++++ .../packages/planning/src/objects/hasher.ts | 88 ++ .../packages/planning/src/objects/index.ts | 23 + .../packages/planning/src/objects/kernel.ts | 486 ++++++ .../packages/planning/src/objects/mirror.ts | 130 ++ .../packages/planning/src/objects/storage.ts | 150 ++ .../planning/src/release/evaluator.ts | 1002 ++++++++++++ .../packages/planning/src/release/index.ts | 11 + .../packages/planning/src/release/ordering.ts | 84 + .../planning/src/release/selectorRead.ts | 154 ++ .../src/packages/planning/src/schema/admit.ts | 231 +++ .../packages/planning/src/schema/backlog.ts | 219 +++ .../packages/planning/src/schema/capacity.ts | 181 +++ .../packages/planning/src/schema/catalog.ts | 70 + .../packages/planning/src/schema/errors.ts | 95 ++ .../packages/planning/src/schema/estimate.ts | 87 ++ .../src/packages/planning/src/schema/ids.ts | 177 +++ .../src/packages/planning/src/schema/index.ts | 29 + .../packages/planning/src/schema/levels.ts | 62 + .../planning/src/schema/levelsDocument.ts | 283 ++++ .../packages/planning/src/schema/namespace.ts | 88 ++ .../src/packages/planning/src/schema/plan.ts | 112 ++ .../packages/planning/src/schema/prices.ts | 72 + .../packages/planning/src/schema/records.ts | 124 ++ .../src/packages/planning/src/schema/rows.ts | 89 ++ .../planning/src/schema/seatCapacity.ts | 233 +++ .../packages/planning/src/schema/selection.ts | 198 +++ .../planning/src/schema/selectionDocument.ts | 452 ++++++ .../packages/planning/src/schema/uplink.ts | 100 ++ .../packages/planning/src/schema/worklist.ts | 57 + .../planning/test-selector/fixtures/bands.tsv | 33 + .../test-selector/fixtures/selection-1.tsv | 52 + .../test-selector/fixtures/selection-2.tsv | 52 + .../test-selector/selectorRead.test.ts | 399 +++++ .../planning/test/capacityAlarmProbe.test.ts | 60 + .../planning/test/capacityFactory.test.ts | 120 ++ .../src/packages/planning/test/conwip.test.ts | 87 ++ .../packages/planning/test/envelope.test.ts | 109 ++ .../packages/planning/test/factory.test.ts | 168 ++ .../packages/planning/test/fillerLane.test.ts | 356 +++++ .../src/packages/planning/test/fixtures.ts | 528 +++++++ .../planning/test/levelsDocument.test.ts | 113 ++ .../planning/test/multiExecutor.test.ts | 459 ++++++ .../packages/planning/test/paceLine.test.ts | 118 ++ .../planning/test/releaseEvaluator.test.ts | 305 ++++ .../planning/test/releasePolicy.test.ts | 193 +++ .../planning/test/seatCapacity.test.ts | 663 ++++++++ .../planning/test/seatCapacityFixtures.ts | 137 ++ .../planning/test/seatCapacityReview2.test.ts | 432 ++++++ .../planning/test/seatCapacityReview3.test.ts | 204 +++ .../planning/test/valueDensity.test.ts | 108 ++ .../src/packages/planning/tsconfig.json | 27 + .../src/packages/planning/vitest.config.ts | 8 + .../src/packages/runners/README.md | 41 + .../runners/bin/ax-conwip-runners.mjs | 12 + .../src/packages/runners/bin/herdr-job.mjs | 97 ++ .../src/packages/runners/package.json | 32 + .../packages/runners/runtimes.example.toml | 66 + .../packages/runners/scripts/live-checks.ts | 156 ++ .../runners/scripts/live-tool-call.ts | 28 + .../src/packages/runners/src/ax.ts | 154 ++ .../src/packages/runners/src/backend.ts | 609 ++++++++ .../src/packages/runners/src/cli.ts | 101 ++ .../src/packages/runners/src/config.ts | 381 +++++ .../src/packages/runners/src/gvisor.ts | 333 ++++ .../src/packages/runners/src/harness.ts | 200 +++ .../src/packages/runners/src/herdr.ts | 585 +++++++ .../src/packages/runners/src/host.ts | 100 ++ .../src/packages/runners/src/index.ts | 14 + .../src/packages/runners/src/job.ts | 131 ++ .../src/packages/runners/src/microvm.ts | 262 ++++ .../src/packages/runners/src/models.ts | 87 ++ .../src/packages/runners/src/proc.ts | 361 +++++ .../src/packages/runners/src/registry.ts | 58 + .../src/packages/runners/src/ssh.ts | 216 +++ .../src/packages/runners/src/workerd.ts | 105 ++ .../src/packages/runners/test/backend.test.ts | 134 ++ .../src/packages/runners/test/cli.test.ts | 27 + .../src/packages/runners/test/config.test.ts | 126 ++ .../src/packages/runners/test/fences.test.ts | 11 + .../src/packages/runners/test/gvisor.test.ts | 128 ++ .../src/packages/runners/test/helpers.ts | 13 + .../src/packages/runners/test/herdr.test.ts | 168 ++ .../packages/runners/test/host-ssh.test.ts | 79 + .../test/interpreter-integration.test.ts | 82 + .../runners/test/microvm-workerd-ax.test.ts | 170 +++ .../src/packages/runners/test/models.test.ts | 78 + .../runners/test/runtime-test-runner.test.ts | 33 + .../runners/test/successor-r1.test.ts | 242 +++ .../runners/test/successor-r2.test.ts | 315 ++++ .../runners/test/successor-r3.test.ts | 339 +++++ .../runners/test/successor-r4.test.ts | 358 +++++ .../runners/test/successor-r5.test.ts | 163 ++ .../packages/runners/tools/check-fences.mjs | 48 + .../src/packages/runners/tsconfig.json | 16 + .../src/packages/runners/vitest.config.ts | 9 + .../src/packages/schema/README.md | 60 + .../src/packages/schema/package.json | 28 + .../src/packages/schema/src/Card.ts | 264 ++++ .../src/packages/schema/src/Common.ts | 183 +++ .../src/packages/schema/src/Evaluation.ts | 144 ++ .../src/packages/schema/src/Frontmatter.ts | 412 +++++ .../src/packages/schema/src/Receipt.ts | 227 +++ .../src/packages/schema/src/ReceiptStrict.ts | 454 ++++++ .../src/packages/schema/src/RunRecord.ts | 162 ++ .../src/packages/schema/src/Rung.ts | 163 ++ .../src/packages/schema/src/index.ts | 176 +++ .../packages/schema/test/Evaluation.test.ts | 95 ++ .../packages/schema/test/Frontmatter.test.ts | 133 ++ .../schema/test/ReceiptStrict.test.ts | 475 ++++++ .../packages/schema/test/RunRecord.test.ts | 94 ++ .../src/packages/schema/test/Schema.test.ts | 305 ++++ .../test/fixtures/receipt-strict-full.json | 141 ++ .../src/packages/schema/tsconfig.json | 8 + .../src/packages/schema/vitest.config.ts | 11 + .../src/packages/serializer/README.md | 61 + .../src/packages/serializer/package.json | 26 + .../src/packages/serializer/src/Iso8601.ts | 174 +++ .../src/packages/serializer/src/KeyOrder.ts | 61 + .../src/packages/serializer/src/Serializer.ts | 250 +++ .../src/packages/serializer/src/index.ts | 41 + .../packages/serializer/test/Iso8601.test.ts | 240 +++ .../serializer/test/Serializer.test.ts | 323 ++++ .../src/packages/serializer/tsconfig.json | 8 + .../src/packages/serializer/vitest.config.ts | 12 + pkgs/substrate-apps/src/pnpm-lock.yaml | 115 ++ pkgs/substrate-apps/src/proto/ax.proto | 387 +++++ pkgs/substrate-apps/src/src/axclient.ts | 155 ++ .../substrate-apps/src/src/capacity/config.ts | 85 ++ pkgs/substrate-apps/src/src/capacity/floor.ts | 175 +++ pkgs/substrate-apps/src/src/capacity/gate.ts | 244 +++ .../substrate-apps/src/src/capacity/source.ts | 111 ++ .../src/src/capacity/tally-meters.ts | 227 +++ pkgs/substrate-apps/src/src/cli.ts | 91 ++ pkgs/substrate-apps/src/src/deploy-config.ts | 133 ++ pkgs/substrate-apps/src/src/derive-only.ts | 64 + pkgs/substrate-apps/src/src/dispatch.ts | 370 +++++ pkgs/substrate-apps/src/src/e2e-live.ts | 167 ++ pkgs/substrate-apps/src/src/extract.ts | 540 +++++++ pkgs/substrate-apps/src/src/index.ts | 11 + pkgs/substrate-apps/src/src/integrated.ts | 421 +++++ pkgs/substrate-apps/src/src/jsonl.ts | 86 ++ pkgs/substrate-apps/src/src/ledger.ts | 115 ++ pkgs/substrate-apps/src/src/live.ts | 149 ++ pkgs/substrate-apps/src/src/loop.ts | 386 +++++ pkgs/substrate-apps/src/src/meter.ts | 391 +++++ pkgs/substrate-apps/src/src/pathguard.ts | 49 + pkgs/substrate-apps/src/src/record.ts | 344 +++++ pkgs/substrate-apps/src/src/release.ts | 110 ++ pkgs/substrate-apps/src/src/run-cli.ts | 164 ++ pkgs/substrate-apps/src/src/run-outcome.ts | 98 ++ pkgs/substrate-apps/src/src/schema.ts | 88 ++ pkgs/substrate-apps/src/src/seats-cli.ts | 77 + pkgs/substrate-apps/src/src/seats.ts | 487 ++++++ pkgs/substrate-apps/src/src/serve.ts | 1140 ++++++++++++++ pkgs/substrate-apps/src/src/slots.ts | 69 + pkgs/substrate-apps/src/src/submit.ts | 417 +++++ pkgs/substrate-apps/src/src/taskspec.ts | 169 ++ pkgs/substrate-apps/sync.sh | 2 +- tests/substrate-modules/default.nix | 43 +- 255 files changed, 42072 insertions(+), 87 deletions(-) create mode 100755 pkgs/substrate-apps/src/apps/puller/bin/substrate-puller.mjs create mode 100644 pkgs/substrate-apps/src/apps/puller/package.json create mode 100644 pkgs/substrate-apps/src/apps/puller/src/connect.ts create mode 100644 pkgs/substrate-apps/src/apps/puller/src/execute.ts create mode 100644 pkgs/substrate-apps/src/apps/puller/src/main.ts create mode 100644 pkgs/substrate-apps/src/apps/puller/src/pidfile.ts create mode 100644 pkgs/substrate-apps/src/apps/puller/src/puller.ts create mode 100644 pkgs/substrate-apps/src/apps/puller/src/state.ts create mode 100644 pkgs/substrate-apps/src/apps/puller/test/puller.test.ts create mode 100644 pkgs/substrate-apps/src/apps/puller/tsconfig.json create mode 100644 pkgs/substrate-apps/src/apps/puller/vitest.config.ts create mode 100644 pkgs/substrate-apps/src/deploy/client.config.example.toml create mode 100644 pkgs/substrate-apps/src/deploy/substrate.config.example.json create mode 100644 pkgs/substrate-apps/src/deploy/wrangler.local.example.jsonc create mode 100644 pkgs/substrate-apps/src/packages/api/package.json create mode 100644 pkgs/substrate-apps/src/packages/api/src/api.ts create mode 100644 pkgs/substrate-apps/src/packages/api/src/client.ts create mode 100644 pkgs/substrate-apps/src/packages/api/src/config.ts create mode 100644 pkgs/substrate-apps/src/packages/api/src/index.ts create mode 100644 pkgs/substrate-apps/src/packages/api/src/schema.ts create mode 100644 pkgs/substrate-apps/src/packages/api/test/api.test.ts create mode 100644 pkgs/substrate-apps/src/packages/api/tsconfig.json create mode 100644 pkgs/substrate-apps/src/packages/api/vitest.config.ts create mode 100644 pkgs/substrate-apps/src/packages/factory/README.md create mode 100644 pkgs/substrate-apps/src/packages/factory/package.json create mode 100644 pkgs/substrate-apps/src/packages/factory/src/cardWriter.ts create mode 100644 pkgs/substrate-apps/src/packages/factory/src/http.ts create mode 100644 pkgs/substrate-apps/src/packages/factory/src/index.ts create mode 100644 pkgs/substrate-apps/src/packages/factory/src/inputs.ts create mode 100644 pkgs/substrate-apps/src/packages/factory/src/projection.ts create mode 100644 pkgs/substrate-apps/src/packages/factory/src/projectionLoad.ts create mode 100644 pkgs/substrate-apps/src/packages/factory/src/projectionMirror.ts create mode 100644 pkgs/substrate-apps/src/packages/factory/src/receipt.ts create mode 100644 pkgs/substrate-apps/src/packages/factory/test/capacity.test.ts create mode 100644 pkgs/substrate-apps/src/packages/factory/test/capacityReview1.test.ts create mode 100644 pkgs/substrate-apps/src/packages/factory/test/factory.test.ts create mode 100644 pkgs/substrate-apps/src/packages/factory/test/fixtures.ts create mode 100644 pkgs/substrate-apps/src/packages/factory/test/http.test.ts create mode 100644 pkgs/substrate-apps/src/packages/factory/test/inputs.test.ts create mode 100644 pkgs/substrate-apps/src/packages/factory/test/persistence.test.ts create mode 100644 pkgs/substrate-apps/src/packages/factory/test/projection.test.ts create mode 100644 pkgs/substrate-apps/src/packages/factory/test/projectionMirror.test.ts create mode 100644 pkgs/substrate-apps/src/packages/factory/tsconfig.json create mode 100644 pkgs/substrate-apps/src/packages/factory/vitest.config.ts create mode 100755 pkgs/substrate-apps/src/packages/interpreter/bin/naive-run.mjs create mode 100644 pkgs/substrate-apps/src/packages/interpreter/package.json create mode 100644 pkgs/substrate-apps/src/packages/interpreter/src/backend.ts create mode 100644 pkgs/substrate-apps/src/packages/interpreter/src/cli.ts create mode 100644 pkgs/substrate-apps/src/packages/interpreter/src/index.ts create mode 100644 pkgs/substrate-apps/src/packages/interpreter/src/interpreter.ts create mode 100644 pkgs/substrate-apps/src/packages/interpreter/src/journal.ts create mode 100644 pkgs/substrate-apps/src/packages/interpreter/src/jsonschema.ts create mode 100644 pkgs/substrate-apps/src/packages/interpreter/src/key.ts create mode 100644 pkgs/substrate-apps/src/packages/interpreter/src/meta.ts create mode 100644 pkgs/substrate-apps/src/packages/interpreter/src/replay.ts create mode 100644 pkgs/substrate-apps/src/packages/interpreter/src/sandbox.ts create mode 100644 pkgs/substrate-apps/src/packages/interpreter/test/conformance.test.ts create mode 100644 pkgs/substrate-apps/src/packages/interpreter/test/edges.test.ts create mode 100644 pkgs/substrate-apps/src/packages/interpreter/test/fidelity.test.ts create mode 100644 pkgs/substrate-apps/src/packages/interpreter/test/helpers.ts create mode 100644 pkgs/substrate-apps/src/packages/interpreter/test/meta.test.ts create mode 100644 pkgs/substrate-apps/src/packages/interpreter/test/semantics.test.ts create mode 100644 pkgs/substrate-apps/src/packages/interpreter/test/successor-r1.test.ts create mode 100644 pkgs/substrate-apps/src/packages/interpreter/test/successor-r2.test.ts create mode 100644 pkgs/substrate-apps/src/packages/interpreter/test/successor-r3.test.ts create mode 100644 pkgs/substrate-apps/src/packages/interpreter/test/successor-r5.test.ts create mode 100644 pkgs/substrate-apps/src/packages/interpreter/tsconfig.json create mode 100644 pkgs/substrate-apps/src/packages/interpreter/vitest.config.ts create mode 100644 pkgs/substrate-apps/src/packages/planning/README.md create mode 100644 pkgs/substrate-apps/src/packages/planning/package.json create mode 100644 pkgs/substrate-apps/src/packages/planning/src/capacity/index.ts create mode 100644 pkgs/substrate-apps/src/packages/planning/src/capacity/ledger.ts create mode 100644 pkgs/substrate-apps/src/packages/planning/src/capacity/model-families.json create mode 100644 pkgs/substrate-apps/src/packages/planning/src/capacity/project.ts create mode 100644 pkgs/substrate-apps/src/packages/planning/src/heuristics/bufferAndon.ts create mode 100644 pkgs/substrate-apps/src/packages/planning/src/heuristics/conwip.ts create mode 100644 pkgs/substrate-apps/src/packages/planning/src/heuristics/eddCertificate.ts create mode 100644 pkgs/substrate-apps/src/packages/planning/src/heuristics/envelope.ts create mode 100644 pkgs/substrate-apps/src/packages/planning/src/heuristics/fillerLane.ts create mode 100644 pkgs/substrate-apps/src/packages/planning/src/heuristics/index.ts create mode 100644 pkgs/substrate-apps/src/packages/planning/src/heuristics/killRedispatch.ts create mode 100644 pkgs/substrate-apps/src/packages/planning/src/heuristics/lengthTermFlip.ts create mode 100644 pkgs/substrate-apps/src/packages/planning/src/heuristics/localFirst.ts create mode 100644 pkgs/substrate-apps/src/packages/planning/src/heuristics/namespaceRoundRobin.ts create mode 100644 pkgs/substrate-apps/src/packages/planning/src/heuristics/paceLine.ts create mode 100644 pkgs/substrate-apps/src/packages/planning/src/heuristics/redundancy.ts create mode 100644 pkgs/substrate-apps/src/packages/planning/src/heuristics/runtimeCap.ts create mode 100644 pkgs/substrate-apps/src/packages/planning/src/heuristics/subassemblyBatching.ts create mode 100644 pkgs/substrate-apps/src/packages/planning/src/heuristics/valueDensity.ts create mode 100644 pkgs/substrate-apps/src/packages/planning/src/index.ts create mode 100644 pkgs/substrate-apps/src/packages/planning/src/objects/factory.ts create mode 100644 pkgs/substrate-apps/src/packages/planning/src/objects/hasher.ts create mode 100644 pkgs/substrate-apps/src/packages/planning/src/objects/index.ts create mode 100644 pkgs/substrate-apps/src/packages/planning/src/objects/kernel.ts create mode 100644 pkgs/substrate-apps/src/packages/planning/src/objects/mirror.ts create mode 100644 pkgs/substrate-apps/src/packages/planning/src/objects/storage.ts create mode 100644 pkgs/substrate-apps/src/packages/planning/src/release/evaluator.ts create mode 100644 pkgs/substrate-apps/src/packages/planning/src/release/index.ts create mode 100644 pkgs/substrate-apps/src/packages/planning/src/release/ordering.ts create mode 100644 pkgs/substrate-apps/src/packages/planning/src/release/selectorRead.ts create mode 100644 pkgs/substrate-apps/src/packages/planning/src/schema/admit.ts create mode 100644 pkgs/substrate-apps/src/packages/planning/src/schema/backlog.ts create mode 100644 pkgs/substrate-apps/src/packages/planning/src/schema/capacity.ts create mode 100644 pkgs/substrate-apps/src/packages/planning/src/schema/catalog.ts create mode 100644 pkgs/substrate-apps/src/packages/planning/src/schema/errors.ts create mode 100644 pkgs/substrate-apps/src/packages/planning/src/schema/estimate.ts create mode 100644 pkgs/substrate-apps/src/packages/planning/src/schema/ids.ts create mode 100644 pkgs/substrate-apps/src/packages/planning/src/schema/index.ts create mode 100644 pkgs/substrate-apps/src/packages/planning/src/schema/levels.ts create mode 100644 pkgs/substrate-apps/src/packages/planning/src/schema/levelsDocument.ts create mode 100644 pkgs/substrate-apps/src/packages/planning/src/schema/namespace.ts create mode 100644 pkgs/substrate-apps/src/packages/planning/src/schema/plan.ts create mode 100644 pkgs/substrate-apps/src/packages/planning/src/schema/prices.ts create mode 100644 pkgs/substrate-apps/src/packages/planning/src/schema/records.ts create mode 100644 pkgs/substrate-apps/src/packages/planning/src/schema/rows.ts create mode 100644 pkgs/substrate-apps/src/packages/planning/src/schema/seatCapacity.ts create mode 100644 pkgs/substrate-apps/src/packages/planning/src/schema/selection.ts create mode 100644 pkgs/substrate-apps/src/packages/planning/src/schema/selectionDocument.ts create mode 100644 pkgs/substrate-apps/src/packages/planning/src/schema/uplink.ts create mode 100644 pkgs/substrate-apps/src/packages/planning/src/schema/worklist.ts create mode 100644 pkgs/substrate-apps/src/packages/planning/test-selector/fixtures/bands.tsv create mode 100644 pkgs/substrate-apps/src/packages/planning/test-selector/fixtures/selection-1.tsv create mode 100644 pkgs/substrate-apps/src/packages/planning/test-selector/fixtures/selection-2.tsv create mode 100644 pkgs/substrate-apps/src/packages/planning/test-selector/selectorRead.test.ts create mode 100644 pkgs/substrate-apps/src/packages/planning/test/capacityAlarmProbe.test.ts create mode 100644 pkgs/substrate-apps/src/packages/planning/test/capacityFactory.test.ts create mode 100644 pkgs/substrate-apps/src/packages/planning/test/conwip.test.ts create mode 100644 pkgs/substrate-apps/src/packages/planning/test/envelope.test.ts create mode 100644 pkgs/substrate-apps/src/packages/planning/test/factory.test.ts create mode 100644 pkgs/substrate-apps/src/packages/planning/test/fillerLane.test.ts create mode 100644 pkgs/substrate-apps/src/packages/planning/test/fixtures.ts create mode 100644 pkgs/substrate-apps/src/packages/planning/test/levelsDocument.test.ts create mode 100644 pkgs/substrate-apps/src/packages/planning/test/multiExecutor.test.ts create mode 100644 pkgs/substrate-apps/src/packages/planning/test/paceLine.test.ts create mode 100644 pkgs/substrate-apps/src/packages/planning/test/releaseEvaluator.test.ts create mode 100644 pkgs/substrate-apps/src/packages/planning/test/releasePolicy.test.ts create mode 100644 pkgs/substrate-apps/src/packages/planning/test/seatCapacity.test.ts create mode 100644 pkgs/substrate-apps/src/packages/planning/test/seatCapacityFixtures.ts create mode 100644 pkgs/substrate-apps/src/packages/planning/test/seatCapacityReview2.test.ts create mode 100644 pkgs/substrate-apps/src/packages/planning/test/seatCapacityReview3.test.ts create mode 100644 pkgs/substrate-apps/src/packages/planning/test/valueDensity.test.ts create mode 100644 pkgs/substrate-apps/src/packages/planning/tsconfig.json create mode 100644 pkgs/substrate-apps/src/packages/planning/vitest.config.ts create mode 100644 pkgs/substrate-apps/src/packages/runners/README.md create mode 100755 pkgs/substrate-apps/src/packages/runners/bin/ax-conwip-runners.mjs create mode 100755 pkgs/substrate-apps/src/packages/runners/bin/herdr-job.mjs create mode 100644 pkgs/substrate-apps/src/packages/runners/package.json create mode 100644 pkgs/substrate-apps/src/packages/runners/runtimes.example.toml create mode 100644 pkgs/substrate-apps/src/packages/runners/scripts/live-checks.ts create mode 100644 pkgs/substrate-apps/src/packages/runners/scripts/live-tool-call.ts create mode 100644 pkgs/substrate-apps/src/packages/runners/src/ax.ts create mode 100644 pkgs/substrate-apps/src/packages/runners/src/backend.ts create mode 100644 pkgs/substrate-apps/src/packages/runners/src/cli.ts create mode 100644 pkgs/substrate-apps/src/packages/runners/src/config.ts create mode 100644 pkgs/substrate-apps/src/packages/runners/src/gvisor.ts create mode 100644 pkgs/substrate-apps/src/packages/runners/src/harness.ts create mode 100644 pkgs/substrate-apps/src/packages/runners/src/herdr.ts create mode 100644 pkgs/substrate-apps/src/packages/runners/src/host.ts create mode 100644 pkgs/substrate-apps/src/packages/runners/src/index.ts create mode 100644 pkgs/substrate-apps/src/packages/runners/src/job.ts create mode 100644 pkgs/substrate-apps/src/packages/runners/src/microvm.ts create mode 100644 pkgs/substrate-apps/src/packages/runners/src/models.ts create mode 100644 pkgs/substrate-apps/src/packages/runners/src/proc.ts create mode 100644 pkgs/substrate-apps/src/packages/runners/src/registry.ts create mode 100644 pkgs/substrate-apps/src/packages/runners/src/ssh.ts create mode 100644 pkgs/substrate-apps/src/packages/runners/src/workerd.ts create mode 100644 pkgs/substrate-apps/src/packages/runners/test/backend.test.ts create mode 100644 pkgs/substrate-apps/src/packages/runners/test/cli.test.ts create mode 100644 pkgs/substrate-apps/src/packages/runners/test/config.test.ts create mode 100644 pkgs/substrate-apps/src/packages/runners/test/fences.test.ts create mode 100644 pkgs/substrate-apps/src/packages/runners/test/gvisor.test.ts create mode 100644 pkgs/substrate-apps/src/packages/runners/test/helpers.ts create mode 100644 pkgs/substrate-apps/src/packages/runners/test/herdr.test.ts create mode 100644 pkgs/substrate-apps/src/packages/runners/test/host-ssh.test.ts create mode 100644 pkgs/substrate-apps/src/packages/runners/test/interpreter-integration.test.ts create mode 100644 pkgs/substrate-apps/src/packages/runners/test/microvm-workerd-ax.test.ts create mode 100644 pkgs/substrate-apps/src/packages/runners/test/models.test.ts create mode 100644 pkgs/substrate-apps/src/packages/runners/test/runtime-test-runner.test.ts create mode 100644 pkgs/substrate-apps/src/packages/runners/test/successor-r1.test.ts create mode 100644 pkgs/substrate-apps/src/packages/runners/test/successor-r2.test.ts create mode 100644 pkgs/substrate-apps/src/packages/runners/test/successor-r3.test.ts create mode 100644 pkgs/substrate-apps/src/packages/runners/test/successor-r4.test.ts create mode 100644 pkgs/substrate-apps/src/packages/runners/test/successor-r5.test.ts create mode 100644 pkgs/substrate-apps/src/packages/runners/tools/check-fences.mjs create mode 100644 pkgs/substrate-apps/src/packages/runners/tsconfig.json create mode 100644 pkgs/substrate-apps/src/packages/runners/vitest.config.ts create mode 100644 pkgs/substrate-apps/src/packages/schema/README.md create mode 100644 pkgs/substrate-apps/src/packages/schema/package.json create mode 100644 pkgs/substrate-apps/src/packages/schema/src/Card.ts create mode 100644 pkgs/substrate-apps/src/packages/schema/src/Common.ts create mode 100644 pkgs/substrate-apps/src/packages/schema/src/Evaluation.ts create mode 100644 pkgs/substrate-apps/src/packages/schema/src/Frontmatter.ts create mode 100644 pkgs/substrate-apps/src/packages/schema/src/Receipt.ts create mode 100644 pkgs/substrate-apps/src/packages/schema/src/ReceiptStrict.ts create mode 100644 pkgs/substrate-apps/src/packages/schema/src/RunRecord.ts create mode 100644 pkgs/substrate-apps/src/packages/schema/src/Rung.ts create mode 100644 pkgs/substrate-apps/src/packages/schema/src/index.ts create mode 100644 pkgs/substrate-apps/src/packages/schema/test/Evaluation.test.ts create mode 100644 pkgs/substrate-apps/src/packages/schema/test/Frontmatter.test.ts create mode 100644 pkgs/substrate-apps/src/packages/schema/test/ReceiptStrict.test.ts create mode 100644 pkgs/substrate-apps/src/packages/schema/test/RunRecord.test.ts create mode 100644 pkgs/substrate-apps/src/packages/schema/test/Schema.test.ts create mode 100644 pkgs/substrate-apps/src/packages/schema/test/fixtures/receipt-strict-full.json create mode 100644 pkgs/substrate-apps/src/packages/schema/tsconfig.json create mode 100644 pkgs/substrate-apps/src/packages/schema/vitest.config.ts create mode 100644 pkgs/substrate-apps/src/packages/serializer/README.md create mode 100644 pkgs/substrate-apps/src/packages/serializer/package.json create mode 100644 pkgs/substrate-apps/src/packages/serializer/src/Iso8601.ts create mode 100644 pkgs/substrate-apps/src/packages/serializer/src/KeyOrder.ts create mode 100644 pkgs/substrate-apps/src/packages/serializer/src/Serializer.ts create mode 100644 pkgs/substrate-apps/src/packages/serializer/src/index.ts create mode 100644 pkgs/substrate-apps/src/packages/serializer/test/Iso8601.test.ts create mode 100644 pkgs/substrate-apps/src/packages/serializer/test/Serializer.test.ts create mode 100644 pkgs/substrate-apps/src/packages/serializer/tsconfig.json create mode 100644 pkgs/substrate-apps/src/packages/serializer/vitest.config.ts create mode 100644 pkgs/substrate-apps/src/proto/ax.proto create mode 100644 pkgs/substrate-apps/src/src/axclient.ts create mode 100644 pkgs/substrate-apps/src/src/capacity/config.ts create mode 100644 pkgs/substrate-apps/src/src/capacity/floor.ts create mode 100644 pkgs/substrate-apps/src/src/capacity/gate.ts create mode 100644 pkgs/substrate-apps/src/src/capacity/source.ts create mode 100644 pkgs/substrate-apps/src/src/capacity/tally-meters.ts create mode 100644 pkgs/substrate-apps/src/src/cli.ts create mode 100644 pkgs/substrate-apps/src/src/deploy-config.ts create mode 100644 pkgs/substrate-apps/src/src/derive-only.ts create mode 100644 pkgs/substrate-apps/src/src/dispatch.ts create mode 100644 pkgs/substrate-apps/src/src/e2e-live.ts create mode 100644 pkgs/substrate-apps/src/src/extract.ts create mode 100644 pkgs/substrate-apps/src/src/index.ts create mode 100644 pkgs/substrate-apps/src/src/integrated.ts create mode 100644 pkgs/substrate-apps/src/src/jsonl.ts create mode 100644 pkgs/substrate-apps/src/src/ledger.ts create mode 100644 pkgs/substrate-apps/src/src/live.ts create mode 100644 pkgs/substrate-apps/src/src/loop.ts create mode 100644 pkgs/substrate-apps/src/src/meter.ts create mode 100644 pkgs/substrate-apps/src/src/pathguard.ts create mode 100644 pkgs/substrate-apps/src/src/record.ts create mode 100644 pkgs/substrate-apps/src/src/release.ts create mode 100644 pkgs/substrate-apps/src/src/run-cli.ts create mode 100644 pkgs/substrate-apps/src/src/run-outcome.ts create mode 100644 pkgs/substrate-apps/src/src/schema.ts create mode 100644 pkgs/substrate-apps/src/src/seats-cli.ts create mode 100644 pkgs/substrate-apps/src/src/seats.ts create mode 100644 pkgs/substrate-apps/src/src/serve.ts create mode 100644 pkgs/substrate-apps/src/src/slots.ts create mode 100644 pkgs/substrate-apps/src/src/submit.ts create mode 100644 pkgs/substrate-apps/src/src/taskspec.ts diff --git a/flake.nix b/flake.nix index afcef388..b2b38a78 100644 --- a/flake.nix +++ b/flake.nix @@ -728,11 +728,13 @@ go_1_27 = inputs.nixpkgs-go.legacyPackages.${system}.go_1_27; }; }; - # `nix build .#substrate-pusher`, `.#substrate-apps-link`: the coordinator's - # gentle capacity pusher and the floor-to-ax link from the same pinned + # `nix build .#substrate-pusher`, `.#substrate-puller`, `.#substrate-apps-link`: + # the coordinator's gentle capacity pusher, its interpreter-host puller and + # the floor-to-ax link from the same pinned # substrate sha (pkgs/substrate-apps). Installed by modules/substrate.nix # (gates OFF) and hosts/nas/substrate-link.nix (still on pkgs/substrate-link). substrate-pusher = substrateApps.pusher; + substrate-puller = substrateApps.puller; substrate-apps-link = substrateApps.link; }; diff --git a/modules/substrate.nix b/modules/substrate.nix index 0ec5bc2e..630737c5 100644 --- a/modules/substrate.nix +++ b/modules/substrate.nix @@ -15,25 +15,27 @@ # policy (120 s floor, 900 s idle, 300 s active, 180 s above 85 percent, one read at reset + 90 s) and # posts seat-capacity/2 snapshots to the floor's POST /capacity/snapshots. A dead pusher fails closed # at the floor: readings grade STALE after 1200 s and admission stops. -# puller the interpreter host: leases runtime:interpreter work from the floor and runs each workflow's agent() -# calls on the runtimes this module renders into runtimes.toml (host, herdr, gvisor, ssh:worker) with -# the cc2 seat's config dir by path. PENDING upstream: apps/puller is not in the pinned source yet -# (pkgs/substrate-apps/SYNC.md), so `package` defaults to null and arming asserts. +# puller the interpreter host: leases runs (runtime:interpreter) from the floor with its per-link token, runs +# each workflow's agent() calls on the runtimes this module renders into runtimes.toml (host, herdr, +# gvisor, ssh:worker) with the cc2 seat's config dir by path, heartbeats, completes, resumes from the +# run's journal. Its config is the [puller] table of a client config.toml (packages/api/src/config.ts), +# rendered here with placeholders for the two credential paths and filled at unit start. # # WHY USER UNITS. Both programs speak for Tom's logins: `seats` reads each Claude seat's OAuth usage from its # config dir, and the puller runs claude, herdr and ssh as Tom. A system unit with User=tom would carry none of # the session (herdr socket, ssh agent, XDG dirs). ConditionUser pins each unit to that one user's manager. # -# THE TOKEN IS A PATH, NEVER A VALUE. The floor's operator bearer (FLOOR_TOKEN) is sealed as an agenix secret, -# decrypted at /run/agenix/ for the user (owner = user, 0400), and handed to each unit as a private -# copy by LoadCredential; the programs get `%d/floor-token` (the pusher's --token-file; the puller's -# capacityFloorTokenFile through a runtime-rendered substrate.json). Nothing here reads, prints or interpolates -# the value; the rendered pusher.json and substrate.json carry only paths. Minting the token (wrangler secret put -# FLOOR_TOKEN and this age file) is Tom's hand. +# THE TOKENS ARE PATHS, NEVER VALUES. The floor's operator bearer (FLOOR_TOKEN) and the puller's per-link token +# (one entry of the floor's LINK_TOKENS, bound to `holder`) are agenix secrets, decrypted at /run/agenix/ +# for the user (owner = user, 0400), and handed to each unit as private copies by LoadCredential; the pusher gets +# `--token-file %d/floor-token`, the puller a config.toml and a substrate.json written into its RuntimeDirectory +# at start with `$CREDENTIALS_DIRECTORY/floor-token` and `.../link-token` as token_file, link_token_file and +# capacityFloorTokenFile. Nothing here reads, prints or interpolates a value; every rendered file carries paths. +# Minting (wrangler secret put FLOOR_TOKEN and LINK_TOKENS, the two age files) is Tom's hand. # # THE GATE. Both `enable`s default to false and hosts/coordinator sets them false explicitly. Arming needs the -# floor deployed at floorUrl, the sealed token, and for the puller its package. tests/substrate-modules renders -# both units armed with fixtures so the shape is proven at evaluation time without a token on disk. +# floor deployed at floorUrl and the sealed tokens. tests/substrate-modules renders both units armed with fixtures +# so the shape is proven at evaluation time without a token on disk. let cfg = config.services.substrate; inherit (lib) @@ -81,24 +83,33 @@ let ) ); - pullerLabels = lib.concatStringsSep "," cfg.puller.servedLabels; + # The client config (packages/api/src/config.ts, deploy/client.config.example.toml): the floor, WHERE each + # credential is, and the [puller] table. Both token paths are placeholders until unit start. + clientConfigTemplate = tomlFormat.generate "substrate-client.toml.in" ({ + floor_url = cfg.floorUrl; + token_file = "@FLOOR_TOKEN_FILE@"; + puller = { + holder = cfg.puller.holder; + link_token_file = "@LINK_TOKEN_FILE@"; + state_dir = cfg.puller.stateDir; + pidfile = cfg.puller.pidfile; + max_runs = cfg.puller.maxRuns; + node_dispatch = cfg.puller.nodeDispatch; + runtimes = "${runtimesToml}"; + seat = cfg.puller.claudeSeat; + cap = cfg.puller.cap; + default_model = cfg.puller.defaultModel; + node_runs_on = cfg.puller.nodeRunsOn; + } + // lib.optionalAttrs (cfg.puller.axServer != null) { ax_server = cfg.puller.axServer; }; + }); + pullerExec = lib.escapeShellArgs ( lib.optionals (cfg.puller.runtimeTestWrapper != null) [ cfg.puller.runtimeTestWrapper "--" ] - ++ [ - (lib.getExe cfg.puller.package) - "--floor" - cfg.floorUrl - "--holder" - cfg.puller.holder - "--labels" - pullerLabels - "--max-in-flight" - (toString cfg.puller.maxInFlight) - ] - ++ cfg.puller.extraArgs + ++ [ (lib.getExe cfg.puller.package) ] ); pullerStart = pkgs.writeShellScript "substrate-puller-start" '' @@ -106,12 +117,16 @@ let : "''${CREDENTIALS_DIRECTORY:?substrate-puller needs LoadCredential (no CREDENTIALS_DIRECTORY)}" : "''${RUNTIME_DIRECTORY:?substrate-puller needs RuntimeDirectory}" token="$CREDENTIALS_DIRECTORY/floor-token" + link="$CREDENTIALS_DIRECTORY/link-token" test -r "$token" || { echo "substrate-puller: no floor-token credential" >&2; exit 78; } + test -r "$link" || { echo "substrate-puller: no link-token credential" >&2; exit 78; } umask 077 ${pkgs.gnused}/bin/sed "s|@FLOOR_TOKEN_FILE@|$token|" ${deployConfigTemplate} > "$RUNTIME_DIRECTORY/substrate.json" + ${pkgs.gnused}/bin/sed -e "s|@FLOOR_TOKEN_FILE@|$token|" -e "s|@LINK_TOKEN_FILE@|$link|" ${clientConfigTemplate} > "$RUNTIME_DIRECTORY/config.toml" export SUBSTRATE_CONFIG="$RUNTIME_DIRECTORY/substrate.json" + export SUBSTRATE_CLIENT_CONFIG="$RUNTIME_DIRECTORY/config.toml" export AX_CONWIP_RUNTIMES=${runtimesToml} - exec ${pullerExec} --token-file "$token" + exec ${pullerExec} ''; commonService = { @@ -247,31 +262,96 @@ in }; puller = { - enable = mkEnableOption "the coordinator puller (the interpreter host). OFF; PENDING its package, see pkgs/substrate-apps/SYNC.md"; + enable = mkEnableOption "the coordinator puller (the interpreter host). OFF; see this file's header"; package = mkOption { - type = types.nullOr types.package; - default = substrateApps.puller or null; - defaultText = lib.literalExpression "(pkgs.callPackage ../pkgs/substrate-apps { }).puller or null"; - description = "The puller. null until apps/puller is vendored; arming asserts on it."; + type = types.package; + default = substrateApps.puller; + defaultText = lib.literalExpression "(pkgs.callPackage ../pkgs/substrate-apps { }).puller"; + description = "The puller, built from pkgs/substrate-apps/src/apps/puller with its workspace."; }; holder = mkOption { type = types.str; - default = "coordinator-puller-1"; - description = "holderIdentity at the floor; one session per identity."; + default = "coordinator"; + description = "The holder its link token is bound to in the floor's LINK_TOKENS (which also binds the runs-on labels it may lease, runtime:interpreter and seat:coordinator in docs/DEPLOY.md). One session per holder (L5, exit 75)."; }; - servedLabels = mkOption { - type = types.listOf types.str; - default = [ "runtime:interpreter" ]; - description = "runs-on labels this puller leases (ARC W6: the coordinator serves the interpreter and the Claude seats; the NAS link serves seat:halogen runtime:gvisor)."; + linkTokenSecret = mkOption { + type = types.str; + default = "substrate-link-token-coordinator"; + description = "The agenix secret NAME holding this puller's per-link token (the LINK_TOKENS entry for `holder`). Handed to the unit as the credential `link-token`. A name, never a value."; }; - maxInFlight = mkOption { + linkTokenAgeFile = mkOption { + type = types.path; + default = ../secrets/substrate-link-token-coordinator.age; + defaultText = lib.literalExpression "../secrets/substrate-link-token-coordinator.age"; + description = "The sealed per-link token. Evaluated only when the puller is enabled."; + }; + + maxRuns = mkOption { type = types.ints.positive; default = 1; - description = "Runs interpreted at once on this host."; + description = "[puller].max_runs: runs held at once (the Lease capacity)."; + }; + + cap = mkOption { + type = types.ints.positive; + default = 2; + description = "[puller].cap: the WIP cap across this puller's agent() calls (node_dispatch = local)."; + }; + + nodeDispatch = mkOption { + type = types.enum [ + "local" + "floor" + ]; + default = "local"; + description = "[puller].node_dispatch: local runs agent() nodes on this box's runtimes; floor sends every node back to the floor as an AgentJob for whichever link serves its labels."; + }; + + nodeRunsOn = mkOption { + type = types.listOf types.str; + default = [ + "seat:halogen" + "runtime:gvisor" + ]; + description = "[puller].node_runs_on: runs-on for a node that names none (floor dispatch)."; + }; + + defaultModel = mkOption { + type = types.str; + default = "claude-opus-5-5"; + description = "[puller].default_model."; + }; + + axServer = mkOption { + type = types.nullOr types.str; + default = null; + example = "nas:7070"; + description = "[puller].ax_server: probed only to say why an ax node is refused here (the NAS link dispatches ax)."; + }; + + stateDir = mkOption { + type = types.str; + default = "${home}/.local/state/substrate/puller"; + defaultText = lib.literalExpression ''"''${home}/.local/state/substrate/puller"''; + description = "[puller].state_dir: held leases, verdict outbox, runs// journals."; + }; + + pidfile = mkOption { + type = types.str; + default = "${home}/.local/state/substrate/puller.pid"; + defaultText = lib.literalExpression ''"''${home}/.local/state/substrate/puller.pid"''; + description = "[puller].pidfile: one puller per box (a second exits 3)."; + }; + + clientConfigFile = mkOption { + type = types.path; + readOnly = true; + default = clientConfigTemplate; + description = "The rendered config.toml template, credential paths as placeholders (read-only; for tests)."; }; claudeSeat = mkOption { @@ -372,13 +452,7 @@ in ]; codexWorkdir = "~/mecattaf/substrate"; }; - description = "src/deploy-config.ts keys (SUBSTRATE_CONFIG). capacityFloorUrl and capacityFloorTokenFile are set by the unit from floorUrl and the credential."; - }; - - extraArgs = mkOption { - type = types.listOf types.str; - default = [ ]; - description = "Appended to the puller's command line."; + description = "src/deploy-config.ts keys (SUBSTRATE_CONFIG, read by the capacity gate). capacityFloorUrl and capacityFloorTokenFile are set by the unit from floorUrl and the credential."; }; extraPath = mkOption { @@ -396,8 +470,8 @@ in message = "services.substrate.floorUrl must be an https:// URL (the bearer never travels in clear)."; } { - assertion = !cfg.puller.enable || cfg.puller.package != null; - message = "services.substrate.puller has no package: apps/puller is not vendored yet (pkgs/substrate-apps/SYNC.md, Pending)."; + assertion = !cfg.puller.enable || cfg.puller.linkTokenSecret != cfg.tokenSecret; + message = "services.substrate.puller.linkTokenSecret must differ from tokenSecret: the per-link token is not the operator bearer."; } { assertion = !cfg.pusher.enable || !(cfg.pusher.extraConfig ? tokenFile); @@ -405,11 +479,22 @@ in } ]; - age.secrets.${cfg.tokenSecret} = { - file = cfg.tokenAgeFile; - owner = cfg.user; - mode = "0400"; - }; + age.secrets = lib.mkMerge [ + { + ${cfg.tokenSecret} = { + file = cfg.tokenAgeFile; + owner = cfg.user; + mode = "0400"; + }; + } + (mkIf cfg.puller.enable { + ${cfg.puller.linkTokenSecret} = { + file = cfg.puller.linkTokenAgeFile; + owner = cfg.user; + mode = "0400"; + }; + }) + ]; systemd.user.services.substrate-pusher = mkIf cfg.pusher.enable { description = "substrate-pusher: seats --json to the Substrate floor, gently"; @@ -445,6 +530,9 @@ in }; serviceConfig = commonService // { ExecStart = pullerStart; + LoadCredential = commonService.LoadCredential ++ [ + "link-token:${config.age.secrets.${cfg.puller.linkTokenSecret}.path}" + ]; RuntimeDirectory = "substrate-puller"; RuntimeDirectoryMode = "0700"; RestartPreventExitStatus = [ 78 ]; diff --git a/pkgs/substrate-apps/SYNC.md b/pkgs/substrate-apps/SYNC.md index 158993e4..4959a6b3 100644 --- a/pkgs/substrate-apps/SYNC.md +++ b/pkgs/substrate-apps/SYNC.md @@ -2,13 +2,16 @@ - Upstream: agency-agency/substrate (private; the repo formerly named ax-conwip, Tom 2026-09-23 E10), local checkout `/home/tom/mecattaf/substrate`, branch `main`. -- Source sha: `dc7cd1d05b1e3938dace9d3d3cddc1a22d98d6cc` (synced 2026-09-23T21:40Z, apps/puller absent) - "capacity: floor capacity service, gentle seats pusher, floor as the gate's default (G4)". +- Source sha: `b1051790f376c103ba4e901619ba11efeb78cef6` (synced 2026-09-23T21:27:53Z, apps/puller present) + "docs: DEPLOY.md, deploying your own floor from a fresh clone to a verified live Worker"; the puller landed one + commit earlier, 017893d "interfaces: typed HTTP API with /openapi.json, substrate CLI, MCP server in code mode, + coordinator puller (E11)". First vendored at dc7cd1d (link and pusher only), resynced the same night. - Taken with `git archive package.json pnpm-lock.yaml pnpm-workspace.yaml tsconfig.base.json tsconfig.json - apps/link apps/pusher`, unchanged (`./sync.sh /home/tom/mecattaf/substrate ` does exactly this). The root - `src/`, `test/`, `proto/`, `packages/*` and `apps/floor` are not vendored: the link bundle imports none of them, - the pusher's bin and src import only node: builtins and each other (MEASURED grep), and the floor is the - Cloudflare Worker, deployed from the substrate repo itself. + src proto packages deploy apps/link apps/pusher apps/puller`, unchanged (`./sync.sh /home/tom/mecattaf/substrate + ` does exactly this; 303 files, 3.2 MB). The puller imports the workspace root (`substrate/src/...`), + `@substrate/api`, `@substrate/link`, `@substrate/interpreter` and `@substrate/runners`, hence `src/`, `proto/` and + `packages/*`. Not vendored: `test/`, `tools/`, `docs/`, `apps/floor` (the Cloudflare Worker, deployed from the + substrate repo), `apps/cli` and `apps/mcp` (operator interfaces, not services of a box). - Not vendored on purpose: the pusher's own vitest suite depends on `@substrate/planning` and `@substrate/factory` (workspace packages); it runs upstream (1530 passed at this sha, REPORTED substrate/RECEIPT.md). The nix checkPhase runs one dry tick against `apps/pusher/test/fixtures/seats-v1.json` instead. @@ -23,17 +26,17 @@ |---|---|---|---| | `link` | `apps/link` | NAS (system unit) | `hosts/nas/substrate-link.nix` (still on pkgs/substrate-link a021003) | | `pusher` | `apps/pusher/bin/substrate-pusher.mjs` | coordinator (user unit) | `modules/substrate.nix` `services.substrate.pusher` | -| `puller` | `apps/puller` | coordinator (user unit) | `modules/substrate.nix` `services.substrate.puller` | +| `puller` | `apps/puller/bin/substrate-puller.mjs` | coordinator (user unit) | `modules/substrate.nix` `services.substrate.puller` | -## Pending: apps/puller +## How the puller is installed -At the pinned sha no branch of agency-agency/substrate has `apps/puller` (MEASURED `git ls-tree` over every ref, -2026-09-23 about 21:35Z; Lane B's Interfaces round was still open). The floor handoff names it: "the coordinator -puller (interpreter host). It leases runtime:interpreter, GETs /runs/:id/script, runs the interpreter with a floor -backend that POSTs /runs/:id/jobs and polls /runs/:id/events or /jobs/:id/output, then Completes the run." When it -lands: `./sync.sh ` picks it up, add a `puller` derivation here (bundle `apps/puller/src/main.ts` -the way `link` is bundled, or install its files the way `pusher` is), export it in the set, and drop the -`puller.package = null` default in `modules/substrate.nix`. +Upstream starts it as `node apps/puller/bin/substrate-puller.mjs`, which registers tsx's ESM loader and imports +`src/main.ts`; nothing is bundled upstream. The derivation keeps that: `pnpm install --prod --offline +--frozen-lockfile` from the shared `pnpmDeps`, then the whole tree with its production `node_modules` is copied to +`$out/lib/substrate-apps` and `bin/substrate-puller` wraps node over it. Its checkPhase starts it with no config +and requires exit 78 with a `config-invalid` line: every import resolved through tsx and the workspace links, and +`main()` ran to its first config check. Its configuration is the `[puller]` table of a client `config.toml` +(`packages/api/src/config.ts`, `deploy/client.config.example.toml`), which the module renders. ## To resync diff --git a/pkgs/substrate-apps/default.nix b/pkgs/substrate-apps/default.nix index fcd394bb..85ded330 100644 --- a/pkgs/substrate-apps/default.nix +++ b/pkgs/substrate-apps/default.nix @@ -14,8 +14,11 @@ # copy, a021003, stays the NAS module's default until hosts/nas is re-pointed). # pusher apps/pusher the coordinator's gentle capacity pusher: `seats --json --no-spend` under the # refresh policy, converted to seat-capacity/2 and posted to the floor. -# puller apps/puller PENDING: not in the source tree at the pinned sha (SYNC.md). The module -# services.substrate.puller takes it as an option and refuses to arm without it. +# puller apps/puller the coordinator's interpreter host: leases runs (runtime:interpreter) from the +# floor, runs them with the interpreter and the runners, heartbeats, completes, +# resumes from the run's journal after a kill. Upstream runs it through tsx's ESM +# loader over the workspace packages (substrate, @substrate/{api,link,interpreter, +# runners}), so it is installed as the whole tree with its production node_modules. # # One pnpm workspace, one fixed-output dependency tree (pnpmDeps) shared by every program that needs # node_modules. The pusher needs none: its bin and src import only node: builtins and each other @@ -26,8 +29,8 @@ let version = "0.1.0-unstable-2026-09-23"; src = ./src; - # The upstream commit ./src was taken from. Keep in step with SYNC.md. - sourceSha = "dc7cd1d05b1e3938dace9d3d3cddc1a22d98d6cc"; + # The upstream commit ./src was taken from. Keep in step with SYNC.md (sync.sh rewrites both). + sourceSha = "b1051790f376c103ba4e901619ba11efeb78cef6"; pnpmDeps = pnpm_10.fetchDeps { pname = "substrate-apps"; @@ -110,13 +113,59 @@ let platforms = lib.platforms.linux; }; }; + + puller = stdenvNoCC.mkDerivation { + pname = "substrate-puller"; + inherit version src pnpmDeps; + nativeBuildInputs = [ + nodejs_24 + pnpm_10.configHook + makeWrapper + ]; + # Production dependencies only: tsx, effect, smol-toml, grpc and the workspace links. vitest, + # typescript and fast-check stay out of the closure. + pnpmInstallFlags = [ "--prod" ]; + dontBuild = true; + doCheck = true; + # Start it with no config: every import resolves through tsx and the workspace links, main() runs, + # and it exits 78 with a `config-invalid` line ("[puller] holder is required"). Anything else, a + # missing module above all, is a different exit. + checkPhase = '' + runHook preCheck + export HOME=$PWD/home + mkdir -p $HOME + set +e + log=$(node apps/puller/bin/substrate-puller.mjs 2>&1) + rc=$? + set -e + printf '%s\n' "$log" | tail -n 3 + if [ "$rc" != 78 ]; then echo "expected exit 78 (config-invalid) with no config, got $rc"; exit 1; fi + printf '%s\n' "$log" | grep -q '"config-invalid"' + runHook postCheck + ''; + installPhase = '' + runHook preInstall + rm -rf home + mkdir -p $out/lib + cp -a . $out/lib/substrate-apps + makeWrapper ${lib.getExe nodejs_24} $out/bin/substrate-puller \ + --add-flags $out/lib/substrate-apps/apps/puller/bin/substrate-puller.mjs + runHook postInstall + ''; + passthru = { inherit sourceSha; }; + meta = { + description = "The Substrate interpreter host: leases runs from the floor and runs them on this box's runtimes"; + mainProgram = "substrate-puller"; + platforms = lib.platforms.linux; + }; + }; in { inherit link pusher + puller pnpmDeps sourceSha ; - # puller: absent at ${sourceSha}; added by the next sync once apps/puller lands upstream. } diff --git a/pkgs/substrate-apps/src/apps/puller/bin/substrate-puller.mjs b/pkgs/substrate-apps/src/apps/puller/bin/substrate-puller.mjs new file mode 100755 index 00000000..6160ba08 --- /dev/null +++ b/pkgs/substrate-apps/src/apps/puller/bin/substrate-puller.mjs @@ -0,0 +1,10 @@ +#!/usr/bin/env node +// Entry point: registers tsx's ESM loader, then runs src/main.ts. +import { register } from "tsx/esm/api"; +import { fileURLToPath } from "node:url"; +import { dirname, join } from "node:path"; + +register(); +const here = dirname(fileURLToPath(import.meta.url)); +const { main } = await import(join(here, "..", "src", "main.ts")); +process.exitCode = await main(process.env); diff --git a/pkgs/substrate-apps/src/apps/puller/package.json b/pkgs/substrate-apps/src/apps/puller/package.json new file mode 100644 index 00000000..f75a8c72 --- /dev/null +++ b/pkgs/substrate-apps/src/apps/puller/package.json @@ -0,0 +1,14 @@ +{ + "name": "@substrate/puller", + "version": "0.1.0", + "private": true, + "type": "module", + "description": "The coordinator-side interpreter host: leases runs (runtime:interpreter) from the floor over /rpc, runs each with the interpreter and the repo's runners (or dispatches its nodes back to the floor), heartbeats, completes, and resumes from the run's journal after a kill. A plain long-lived process with a pidfile.", + "license": "UNLICENSED", + "bin": { "substrate-puller": "./bin/substrate-puller.mjs" }, + "exports": { ".": "./src/puller.ts", "./*": "./src/*" }, + "scripts": { "check": "tsc --noEmit -p .", "test": "vitest run" }, + "dependencies": { "substrate": "workspace:*", "@substrate/api": "workspace:*", "@substrate/link": "workspace:*", "@substrate/interpreter": "workspace:*", "@substrate/runners": "workspace:*", "tsx": "^4.19.2" }, + "peerDependencies": { "effect": "4.0.0-rc.112" }, + "devDependencies": { "@substrate/floor": "workspace:*", "@types/node": "^22.10.2", "effect": "4.0.0-rc.112", "typescript": "^5.9.3", "vitest": "^3.2.4" } +} diff --git a/pkgs/substrate-apps/src/apps/puller/src/connect.ts b/pkgs/substrate-apps/src/apps/puller/src/connect.ts new file mode 100644 index 00000000..393d4d16 --- /dev/null +++ b/pkgs/substrate-apps/src/apps/puller/src/connect.ts @@ -0,0 +1,13 @@ +// The puller's floor connection: the link's own rpcFloor client (Effect RPC over HTTP, its retry classifier and +// per-call ceilings), held open in a scope for the life of the process. +import { Effect, Exit, Scope } from "effect" +import { rpcFloor } from "@substrate/link/floor.ts" +import type { FloorOptions } from "@substrate/link/floor.ts" +import { floorPort } from "./puller.ts" +import type { FloorPort } from "./puller.ts" + +export const connectFloor = async (o: FloorOptions): Promise<{ port: FloorPort; close: () => Promise }> => { + const scope = await Effect.runPromise(Scope.make()) + const api = await Effect.runPromise(Scope.provide(scope)(rpcFloor(o))) + return { port: floorPort(api), close: () => Effect.runPromise(Scope.close(scope, Exit.void)) } +} diff --git a/pkgs/substrate-apps/src/apps/puller/src/execute.ts b/pkgs/substrate-apps/src/apps/puller/src/execute.ts new file mode 100644 index 00000000..fdffcbba --- /dev/null +++ b/pkgs/substrate-apps/src/apps/puller/src/execute.ts @@ -0,0 +1,176 @@ +// What a leased run becomes. Two executors, one contract (puller.ts Executor): +// +// local the interpreter with the repo's runners on this host (src/integrated.ts runIntegrated): host, +// runtime-test, herdr, gvisor, ssh:worker for pi on Halogen, microvm, workerd, as runtimes.toml names them. +// The run directory is the puller's runs//, so a second attempt resumes from the first one's journal. +// An ax runtime is refused with the reason: ax Tasks are rendered and dispatched by the link on the NAS, +// which this host is not; use node_dispatch = "floor" to send nodes there. +// floor the interpreter here, every agent() node back to the floor as an AgentJob (POST /runs/:id/jobs), leased +// by whichever link serves its runs-on labels (the NAS link renders and dispatches it to ax) and read back +// from GET /jobs/:name/output. The journal stays here, so resume works the same way. +import { createHash } from "node:crypto" +import { connect } from "node:net" +import { existsSync, readFileSync } from "node:fs" +import { join } from "node:path" +import type { SubstrateClient, AgentJob } from "@substrate/api" +import { FileJournal, parseJournal, runWorkflow } from "@substrate/interpreter" +import type { AgentCall, AgentOutcome, Backend } from "@substrate/interpreter" +import { refusal } from "@substrate/runners" +import type { Runner, RuntimesConfig } from "@substrate/runners" +import type { CapacityGate } from "substrate/src/capacity/gate.ts" +import { runIntegrated } from "substrate/src/integrated.ts" +import type { Executor, RunVerdict } from "./puller.ts" + +const A = "ultracode.mecattaf.dev/" + +/** Can a TCP connection to host:port open within the timeout? */ +export const reachable = (hostPort: string, timeoutMs = 1500): Promise => new Promise((ok) => { + const i = hostPort.lastIndexOf(":") + const s = connect({ host: hostPort.slice(0, i), port: Number(hostPort.slice(i + 1)) }) + const t = setTimeout(() => { s.destroy(); ok(false) }, timeoutMs) + s.once("connect", () => { clearTimeout(t); s.destroy(); ok(true) }) + s.once("error", () => { clearTimeout(t); ok(false) }) +}) + +export const axRefusal = (axServer: string | undefined, isReachable: boolean | undefined): string => + `ax runtime: this puller does not dispatch ax Tasks; the NAS link renders and dispatches them (create-only, durable verdict outbox). ` + + (axServer === undefined ? "No ax_server is configured here. " : isReachable ? `ax-server ${axServer} is reachable, but dispatch from the coordinator is not its path. ` : `ax-server ${axServer} is not reachable. `) + + `Run with node_dispatch = "floor" so the node is enqueued for the link that serves its runs-on labels.` + +export interface LocalOptions { + readonly runtimes: RuntimesConfig + readonly seat: string + readonly defaultModel: string + readonly cap: number + readonly capacity?: CapacityGate + readonly concurrency?: number + readonly maxAttempts?: number + readonly axServer?: string + /** Runner overrides by runtime name (tests). */ + readonly runners?: Readonly> +} + +export const localExecutor = (o: LocalOptions): Executor => async (t) => { + const axNames = Object.entries(o.runtimes.runtimes).filter(([, r]) => r.type === "ax").map(([n]) => n) + const up = axNames.length > 0 && o.axServer !== undefined ? await reachable(o.axServer) : undefined + const why = axRefusal(o.axServer, up) + const axRunners = Object.fromEntries(axNames.map((n): [string, Runner] => [n, { name: n, type: "ax", refuses: () => why, run: async (job) => refusal(n, job, why) }])) + const run = await runIntegrated({ + scriptPath: t.scriptPath, dir: t.dir, runtimes: o.runtimes, seat: o.seat, defaultModel: o.defaultModel, cap: o.cap, + ...(t.args !== undefined ? { args: t.args } : {}), + ...(o.capacity ? { capacity: o.capacity } : {}), + ...(o.concurrency !== undefined ? { concurrency: o.concurrency } : {}), + ...(o.maxAttempts !== undefined ? { maxAttempts: o.maxAttempts } : {}), + runners: { ...axRunners, ...(o.runners ?? {}) }, + runIdPrefix: `${t.runId}-`, + signal: t.signal + }) + return { + result: run.result.status === "completed" ? "success" : "failure", + output: { + runId: t.runId, interpreterRunId: run.runId, resumed: run.resumed, outcome: run.outcome, callCounts: run.callCounts, dispatched: run.dispatched, + status: run.result.status, result: run.result.result ?? null, error: run.result.error ?? null, + ...(run.result.status === "completed" ? {} : { reason: "agent/script-failed", message: run.result.error ?? "the workflow failed" }) + } + } +} + +// ---------------------------------------------------------------- floor dispatch + +export interface FloorBackendOptions { + readonly client: Pick + readonly runId: string + readonly workflow: string + /** runs-on for a node that names none (opts.runsOn, or opts.seat and opts.runtime). */ + readonly defaultRunsOn: ReadonlyArray + readonly defaultModel: string + readonly pollMs?: number + readonly signal: AbortSignal +} + +const sha = (s: string) => createHash("sha256").update(s).digest("hex") +const str = (v: unknown) => (typeof v === "string" && v !== "" ? v : undefined) + +/** One agent() call as an AgentJob. The name is fixed by (index, attempt, run), so a resumed call re-enqueues the + * same job (idempotent by name) and reads the verdict the floor may already hold. */ +export const nodeJob = (call: AgentCall, o: Pick): AgentJob => { + const opts = call.opts as Record + const runsOn = Array.isArray(opts.runsOn) ? opts.runsOn.map(String) + : str(opts.seat) || str(opts.runtime) ? [`seat:${str(opts.seat) ?? o.defaultRunsOn.find((l) => l.startsWith("seat:"))?.slice(5) ?? "unknown"}`, ...(str(opts.runtime) ? [`runtime:${str(opts.runtime)}`] : o.defaultRunsOn.filter((l) => !l.startsWith("seat:")))] + : [...o.defaultRunsOn] + const digest = sha(call.prompt) + const w: Record = { prompt: call.prompt, prompt_ref: { sha256: digest, bytes: Buffer.byteLength(call.prompt), uri: `journal://${o.runId}/${call.index}/prompt.md` }, model: str(opts.model) ?? o.defaultModel } + if (str(opts.effort)) w.effort = opts.effort + if (opts.schema !== undefined) w.schema = opts.schema + if (str(opts.isolation)) w.isolation = opts.isolation + if (str(opts.agentType)) w["agent-type"] = opts.agentType + return { + apiVersion: "ultracode.mecattaf.dev/v1alpha1", kind: "AgentJob", + metadata: { + name: `n${call.index}-a${call.attempt}-${o.runId}`.toLowerCase(), + labels: { [A + "run-id"]: o.runId, [A + "workflow"]: o.workflow.toLowerCase().replace(/[^a-z0-9.-]+/g, "-").slice(0, 63), [A + "kind"]: "agent" }, + annotations: { [A + "run-id-raw"]: o.runId, [A + "journal-key"]: `${call.key}:${call.index}`, ...(call.opts.label ? { [A + "label"]: String(call.opts.label) } : {}), ...(call.phase ? { [A + "phase"]: call.phase } : {}) } + }, + spec: { "runs-on": runsOn, with: w as AgentJob["spec"]["with"] } + } +} + +/** The verdict as the interpreter's AgentOutcome. A failure is the call's error; the interpreter owns retries. */ +export const outcomeOf = (o: { result: string; output: unknown; usage: unknown }, wantObject: boolean): AgentOutcome => { + const u = o.usage as { prompt_tokens?: number; completion_tokens?: number } | null + const usage = u && typeof u.prompt_tokens === "number" ? { usage: { inputTokens: u.prompt_tokens, outputTokens: u.completion_tokens ?? 0 } } : {} + if (o.result !== "success") { + const f = (o.output ?? {}) as { reason?: string; message?: string } + return { error: `${o.result}: ${f.reason ?? "no reason"}${f.message ? `: ${f.message}` : ""}`, ...usage } + } + const v = o.output as { object?: unknown; text?: unknown } | string | null + if (wantObject) return { object: v !== null && typeof v === "object" && "object" in v ? v.object : v, ...usage } + return { text: typeof v === "string" ? v : v !== null && typeof v === "object" && typeof v.text === "string" ? v.text : JSON.stringify(v), ...usage } +} + +export class FloorBackend implements Backend { + readonly name = "floor" + readonly enqueued: Array = [] + constructor(readonly o: FloorBackendOptions) {} + async run(call: AgentCall): Promise { + const job = nodeJob(call, this.o) + const name = job.metadata.name + await this.o.client.enqueue(this.o.runId, [job]) + this.enqueued.push(name) + for (;;) { + if (this.o.signal.aborted) return new Promise(() => undefined) // in flight at the kill: journaled started-only, as a real kill leaves it + const v = await this.o.client.job(name) + if (v.state === "done") return outcomeOf(await this.o.client.output(name), call.opts.schema !== undefined) + await new Promise((ok) => { const t = setTimeout(ok, this.o.pollMs ?? 2000); this.o.signal.addEventListener("abort", () => { clearTimeout(t); ok(undefined) }, { once: true }) }) + } + } +} + +export interface FloorExecOptions extends Omit { + readonly concurrency?: number + readonly maxAttempts?: number + /** Observe each run's backend (tests). */ + readonly onBackend?: (b: FloorBackend) => void +} + +export const floorExecutor = (o: FloorExecOptions): Executor => async (t) => { + const backend = new FloorBackend({ ...o, runId: t.runId, workflow: t.workflowName, signal: t.signal }) + o.onBackend?.(backend) + const journalPath = join(t.dir, "journal.jsonl") + const resumed = existsSync(journalPath) && readFileSync(journalPath, "utf8").trim() !== "" + const aborted = new Promise<"aborted">((ok) => { if (t.signal.aborted) ok("aborted"); else t.signal.addEventListener("abort", () => ok("aborted"), { once: true }) }) + const r = await Promise.race([runWorkflow(t.script, { + backend, runId: `wf_${t.runId}`, scriptPath: t.scriptPath, journal: new FileJournal(journalPath), cacheIdentity: "content", defaultModel: o.defaultModel, + ...(resumed ? { resumeFrom: parseJournal(readFileSync(journalPath, "utf8")) } : {}), + ...(t.args !== undefined ? { args: t.args } : {}), + ...(o.concurrency !== undefined ? { concurrency: o.concurrency } : {}), + ...(o.maxAttempts !== undefined ? { maxAttempts: o.maxAttempts } : {}) + }), aborted]) + if (r === "aborted") return { result: "cancelled", output: { runId: t.runId, reason: "aborted" } } satisfies RunVerdict + const calls = r.calls.map((c) => c.state) + return { + result: r.status === "completed" ? "success" : "failure", + output: { runId: t.runId, resumed, status: r.status, result: r.result ?? null, error: r.error ?? null, calls: { total: calls.length, done: calls.filter((s) => s === "done").length, cached: calls.filter((s) => s === "cached").length, null: calls.filter((s) => s === "null").length }, enqueued: backend.enqueued.length, + ...(r.status === "completed" ? {} : { reason: "agent/script-failed", message: r.error ?? "the workflow failed" }) } + } +} diff --git a/pkgs/substrate-apps/src/apps/puller/src/main.ts b/pkgs/substrate-apps/src/apps/puller/src/main.ts new file mode 100644 index 00000000..2899a142 --- /dev/null +++ b/pkgs/substrate-apps/src/apps/puller/src/main.ts @@ -0,0 +1,64 @@ +// substrate-puller: a plain long-lived process. Configuration is the [puller] table of +// ~/.config/substrate/config.toml (see deploy/client.config.example.toml); the operator bearer and the per-link +// token are files (mode 600), read once and never logged. Logs are JSON lines on stdout. +// +// Exit codes: 0 stopped on SIGTERM or SIGINT; 3 another puller holds the pidfile; 75 another session holds this +// holder identity (L5); 78 configuration invalid, or the floor refused the token or answered a redirect. +import { homedir } from "node:os" +import { join } from "node:path" +import { ConfigError, clientFromConfig, expandHome, loadClientConfig, readSecretFile } from "@substrate/api" +import { loadRuntimes } from "@substrate/runners" +import { CapacityGate } from "substrate/src/capacity/gate.ts" +import { defaultCapacitySource } from "substrate/src/capacity/config.ts" +import { killLiveGroups } from "@substrate/runners" +import { connectFloor } from "./connect.ts" +import { floorExecutor, localExecutor } from "./execute.ts" +import { acquirePidfile, PidfileHeld } from "./pidfile.ts" +import { Puller } from "./puller.ts" +import type { Executor } from "./puller.ts" +import { PullerState } from "./state.ts" + +const log = (ev: string, f: Record = {}) => process.stdout.write(JSON.stringify({ t: new Date().toISOString(), ev, ...f }) + "\n") + +export const main = async (env: NodeJS.ProcessEnv = process.env): Promise => { + let release: (() => void) | undefined + try { + const cfg = loadClientConfig(env) + const p = cfg.puller ?? {} + if (!p.holder) throw new ConfigError("[puller] holder is required (the holder name its link token is bound to)") + if (!p.link_token_file) throw new ConfigError("[puller] link_token_file is required") + if (!cfg.floor_url) throw new ConfigError("floor_url is required") + const stateDir = expandHome(p.state_dir ?? "~/.local/state/substrate/puller") + const client = clientFromConfig(cfg) + const linkToken = readSecretFile(expandHome(p.link_token_file), "[puller] link_token_file") + release = acquirePidfile(expandHome(p.pidfile ?? join(homedir(), ".local/state/substrate/puller.pid"))) + const state = new PullerState(stateDir) + let execute: Executor + if ((p.node_dispatch ?? "local") === "floor") { + execute = floorExecutor({ client, defaultRunsOn: p.node_runs_on ?? ["seat:halogen", "runtime:gvisor"], defaultModel: p.default_model ?? "claude-opus-5-5", pollMs: 5000 }) + } else { + if (!p.seat) throw new ConfigError("[puller] seat is required for node_dispatch = \"local\" (the seat the capacity gate reads)") + const runtimes = loadRuntimes(p.runtimes ? expandHome(p.runtimes) : undefined) + let capacity: CapacityGate + try { capacity = new CapacityGate(defaultCapacitySource(), p.seat) } catch (e) { throw new ConfigError(String((e as Error).message)) } + execute = localExecutor({ runtimes, seat: p.seat, defaultModel: p.default_model ?? "claude-opus-5-5", cap: p.cap ?? 2, capacity, ...(p.ax_server ? { axServer: p.ax_server } : {}) }) + } + const floor = await connectFloor({ url: cfg.floor_url, token: linkToken, sessionId: state.sessionId() }) + const puller = new Puller({ holder: p.holder, maxRuns: p.max_runs ?? 1, stateDir, pollMs: 5000, heartbeatMs: 10_000 }, { floor: floor.port, client, execute, log }) + let signalled = false + for (const sig of ["SIGTERM", "SIGINT", "SIGHUP"] as const) process.on(sig, () => { + if (signalled) { killLiveGroups("SIGKILL"); process.exit(130) } + signalled = true; log("stop-requested", { signal: sig }); puller.stop() + }) + log("start", { holder: p.holder, stateDir, nodeDispatch: p.node_dispatch ?? "local", maxRuns: p.max_runs ?? 1, held: Object.keys(puller.held).length }) + try { await puller.run() } finally { await floor.close() } + log("stopped") + return 0 + } catch (e) { + if (e instanceof PidfileHeld) { log("pidfile-held", { pid: e.pid }); return 3 } + if (e instanceof ConfigError) { log("config-invalid", { why: e.message }); return 78 } + const kind = (e as { kind?: string })?.kind + log("fatal", { kind: kind ?? "defect", message: String((e as Error)?.message ?? e) }) + return kind === "session-conflict" ? 75 : kind === "auth" || kind === "redirect" || kind === "misrouted" ? 78 : 1 + } finally { release?.() } +} diff --git a/pkgs/substrate-apps/src/apps/puller/src/pidfile.ts b/pkgs/substrate-apps/src/apps/puller/src/pidfile.ts new file mode 100644 index 00000000..607cbb59 --- /dev/null +++ b/pkgs/substrate-apps/src/apps/puller/src/pidfile.ts @@ -0,0 +1,21 @@ +// One puller per state directory and holder: a pidfile, taken at start and removed at exit. A pidfile whose pid is +// gone, or is no longer a substrate-puller, is stale and replaced. +import { existsSync, readFileSync, rmSync, writeFileSync, mkdirSync } from "node:fs" +import { dirname } from "node:path" + +export class PidfileHeld extends Error { constructor(readonly pid: number, path: string) { super(`another puller (pid ${pid}) holds ${path}`) } } + +const alive = (pid: number, marker: string): boolean => { + try { process.kill(pid, 0) } catch (e) { return (e as NodeJS.ErrnoException).code === "EPERM" } + try { return readFileSync(`/proc/${pid}/cmdline`, "utf8").includes(marker) } catch { return true } +} + +export const acquirePidfile = (path: string, marker = "substrate-puller", self = process.pid): (() => void) => { + mkdirSync(dirname(path), { recursive: true }) + if (existsSync(path)) { + const pid = Number(readFileSync(path, "utf8").trim()) + if (Number.isInteger(pid) && pid > 0 && pid !== self && alive(pid, marker)) throw new PidfileHeld(pid, path) + } + writeFileSync(path, `${self}\n`, { mode: 0o644 }) + return () => { try { if (readFileSync(path, "utf8").trim() === String(self)) rmSync(path) } catch { /* gone */ } } +} diff --git a/pkgs/substrate-apps/src/apps/puller/src/puller.ts b/pkgs/substrate-apps/src/apps/puller/src/puller.ts new file mode 100644 index 00000000..fba0bad3 --- /dev/null +++ b/pkgs/substrate-apps/src/apps/puller/src/puller.ts @@ -0,0 +1,256 @@ +// The puller: the coordinator-side interpreter host (LINK-DESIGN: a holder advertising runtime:interpreter). +// +// It speaks the link's wire, unchanged: Lease with a journaled requestKey (Forgejo request key; a restart re-sends +// it and the floor answers the same grants, rule 2b), Heartbeat with the complete set it holds (level-triggered, +// ARC; an omitted lease is released, L3), Complete from a durable verdict outbox (Temporal RespondActivityTask*; a +// verdict survives a kill and is resent until the floor accepts or refuses it for good). +// +// A grant is one run. The puller fetches the run's script from the floor, checks it against the grant's content +// address, and hands it to an Executor with the run's own directory. The directory is keyed by run id, not by +// lease, so attempt n+1 of a run (after a kill, a lost lease or a restart) finds attempt n's journal and resumes +// from it: finished agent() calls are cache hits and are never dispatched again. +import { randomUUID } from "node:crypto" +import { writeFileSync } from "node:fs" +import { join } from "node:path" +import { Effect } from "effect" +import type { SubstrateClient } from "@substrate/api" +import type { AgentJob, Grant } from "@substrate/link/contract.ts" +import type { CompletePayload, FloorApi, LeaseReply } from "@substrate/link/floor.ts" +import { CompleteRefused, FloorError } from "@substrate/link/floor.ts" +import { PullerState } from "./state.ts" + +export const A = "ultracode.mecattaf.dev/" +export const INTERPRETER_LABEL = "runtime:interpreter" + +/** The floor's three calls as promises; a failure rejects with the link's own FloorError or CompleteRefused. */ +export interface FloorPort { + lease(p: { holderIdentity: string; capacity: number; requestKey: string }): Promise + heartbeat(p: { holderIdentity: string; leaseIds: ReadonlyArray; pendingRequestKey?: string }): Promise<{ renewed: ReadonlyArray; lost: ReadonlyArray; cancelRequested: ReadonlyArray }> + complete(p: CompletePayload): Promise<{ duplicate: boolean }> +} +const settle = (eff: Effect.Effect): Promise => + Effect.runPromise(Effect.match(eff, { onFailure: (e) => ({ e }), onSuccess: (a) => ({ a }) })).then((r) => { if ("e" in r) throw r.e; return r.a }) +export const floorPort = (api: FloorApi): FloorPort => ({ + lease: (p) => settle(api.lease(p)), + heartbeat: (p) => settle(api.heartbeat(p)), + complete: (p) => settle(api.complete(p)) +}) + +export interface RunTask { + readonly runId: string + readonly leaseId: string + readonly attempt: number + readonly job: AgentJob + readonly workflowName: string + readonly args: unknown + /** The run's directory; `script.js` is written there before the executor starts. */ + readonly dir: string + readonly scriptPath: string + readonly script: string + readonly signal: AbortSignal +} +export interface RunVerdict { readonly result: "success" | "failure" | "cancelled"; readonly output: unknown } +export type Executor = (t: RunTask) => Promise + +export interface PullerConfig { + readonly holder: string + readonly maxRuns: number + readonly stateDir: string + readonly pollMs?: number + readonly heartbeatMs?: number + /** Verdict outputs above this are cut to a preview (the floor's row limit is 2 MB). */ + readonly maxOutputBytes?: number +} +export interface PullerDeps { + readonly floor: FloorPort + readonly client: Pick + readonly execute: Executor + readonly log?: (ev: string, f?: Record) => void +} + +type Why = "lost" | "cancel" | "stop" | "superseded" +interface Active { readonly grant: Grant; readonly ac: AbortController; why?: Why; readonly done: Promise } + +const sha256 = async (s: string) => [...new Uint8Array(await crypto.subtle.digest("SHA-256", new TextEncoder().encode(s)))].map((b) => b.toString(16).padStart(2, "0")).join("") +const sleep = (ms: number, signal?: AbortSignal) => new Promise((ok) => { + const t = setTimeout(ok, ms) + signal?.addEventListener("abort", () => { clearTimeout(t); ok() }, { once: true }) +}) + +export class Puller { + readonly state: PullerState + readonly active = new Map() + #held: Record + #stop = new AbortController() + #hbMs: number + #fatal: unknown + readonly #log: NonNullable + + constructor(readonly cfg: PullerConfig, readonly deps: PullerDeps) { + this.state = new PullerState(cfg.stateDir) + this.#held = this.state.held() + this.#hbMs = cfg.heartbeatMs ?? 10_000 + this.#log = deps.log ?? (() => undefined) + } + + get held(): Readonly> { return this.#held } + + /** Runs until stop() or a fatal floor answer (bad token, second session, redirect, misroute), which it rethrows. */ + async run(): Promise { + const stop = this.#stop.signal + await this.flushOutbox() + await this.readopt() + const hb = this.heartbeatLoop() + while (!stop.aborted && this.#fatal === undefined) { + await this.flushOutbox() + const free = this.cfg.maxRuns - this.active.size + if (free > 0 && !stop.aborted) await this.lease(free) + await sleep(this.cfg.pollMs ?? 5_000, stop) + } + this.#stop.abort() + await hb + for (const a of this.active.values()) { a.why ??= "stop"; a.ac.abort() } + await Promise.all([...this.active.values()].map((a) => a.done)) + if (this.#fatal !== undefined) throw this.#fatal + } + + /** Stop leasing and abort every run; nothing is completed, so each lease is re-adopted by the next start. */ + stop() { this.#stop.abort() } + + private fatal(e: unknown) { if (e instanceof FloorError && e.fatal) { this.#fatal = e; this.#stop.abort() } } + private persistHeld() { this.state.putHeld(this.#held) } + + /** After a restart: vouch for every held lease; relaunch the renewed ones (they resume from their journal). */ + private async readopt() { + const ids = Object.keys(this.#held) + const pendingRequestKey = this.state.pendingKey() + if (ids.length === 0 && pendingRequestKey === undefined) return + try { + const r = await this.deps.floor.heartbeat({ holderIdentity: this.cfg.holder, leaseIds: ids, ...(pendingRequestKey ? { pendingRequestKey } : {}) }) + for (const id of r.lost) delete this.#held[id] + this.persistHeld() + for (const id of r.renewed) { const g = this.#held[id]; if (g && !this.active.has(id)) { this.#log("readopt", { leaseId: id }); this.launch(g) } } + for (const id of r.cancelRequested) this.abort(id, "cancel") + } catch (e) { this.#log("readopt-error", { message: String((e as Error)?.message ?? e) }); this.fatal(e) } + } + + private async lease(capacity: number) { + let key = this.state.pendingKey() + if (key === undefined) { key = randomUUID(); this.state.setPendingKey(key) } + let r: LeaseReply + try { r = await this.deps.floor.lease({ holderIdentity: this.cfg.holder, capacity, requestKey: key }) } catch (e) { + this.#log("lease-error", { kind: (e as FloorError)?.kind, message: String((e as Error)?.message ?? e) }); this.fatal(e); return + } + if (typeof r.heartbeatSeconds === "number" && r.heartbeatSeconds > 0) this.#hbMs = Math.min(this.cfg.heartbeatMs ?? Infinity, r.heartbeatSeconds * 1000) + for (const g of r.grants) this.#held[g.leaseId] = g + this.persistHeld() + this.state.setPendingKey(undefined) + for (const bad of r.invalid ?? []) { + if (bad.leaseId !== undefined && bad.attempt !== undefined) this.queueVerdict({ leaseId: bad.leaseId, attempt: bad.attempt, result: "failure", output: { reason: "pre-start/invalid-spec", message: bad.message } }) + } + for (const g of r.grants) { + for (const old of g.supersedes ?? []) { // L3: the earlier attempt stops before this one starts + const a = this.active.get(old) + if (a) { this.abort(old, "superseded"); await a.done } + delete this.#held[old] + } + this.persistHeld() + if (!this.active.has(g.leaseId)) this.launch(g) + } + } + + private abort(leaseId: string, why: Why) { + const a = this.active.get(leaseId) + if (a) { a.why ??= why; a.ac.abort() } + else if (why === "cancel" && this.#held[leaseId]) { + const g = this.#held[leaseId]! + this.queueVerdict({ leaseId, attempt: g.attempt, result: "cancelled", output: { reason: "cancelled" } }) + } + } + + private launch(g: Grant) { + const ac = new AbortController() + const entry: Active = { grant: g, ac, done: undefined as never } + const work = async (): Promise => { + const ann = g.job.metadata.annotations ?? {} + const runId = ann[A + "run-id-raw"] + if (!g.job.spec["runs-on"].includes(INTERPRETER_LABEL) || runId === undefined) + return { result: "failure", output: { reason: "pre-start/invalid-spec", message: "not a run: the puller takes runtime:interpreter jobs with a run-id-raw annotation" } } + const script = await this.deps.client.runScript(runId) + const want = g.job.spec.with.prompt_ref.sha256 + if (await sha256(script) !== want) return { result: "failure", output: { reason: "pre-start/script-mismatch", message: `the floor's script does not hash to ${want}` } } + const dir = this.state.runDir(runId) + const scriptPath = join(dir, "script.js") + writeFileSync(scriptPath, script) + let args: unknown + try { args = ann[A + "args"] === undefined ? undefined : JSON.parse(ann[A + "args"]!) } catch { return { result: "failure", output: { reason: "pre-start/invalid-spec", message: "the args annotation is not JSON" } } } + this.#log("run-start", { runId, leaseId: g.leaseId, attempt: g.attempt }) + return this.deps.execute({ runId, leaseId: g.leaseId, attempt: g.attempt, job: g.job, workflowName: ann[A + "workflow-name"] ?? runId, args, dir, scriptPath, script, signal: ac.signal }) + } + const done = work().then((v) => v, (e: unknown): RunVerdict => { + if (e instanceof FloorError) return { result: "failure", output: { reason: "infra/script-unreadable", message: e.message } } + return { result: "failure", output: { reason: "infra/puller-defect", message: String((e as Error)?.message ?? e).slice(0, 2000) } } + }).then((v) => { + const why = entry.why + this.#log("run-end", { leaseId: g.leaseId, result: v.result, ...(why ? { aborted: why } : {}) }) + if (why === "lost" || why === "stop" || why === "superseded") return // no verdict: the lease is the floor's again, or re-adopted on restart + this.queueVerdict({ leaseId: g.leaseId, attempt: g.attempt, result: why === "cancel" ? "cancelled" : v.result, output: this.cap(v.output) }) + }).finally(() => this.active.delete(g.leaseId)) + Object.assign(entry, { done }) + this.active.set(g.leaseId, entry) + } + + private cap(output: unknown): unknown { + const text = JSON.stringify(output ?? null) + const max = this.cfg.maxOutputBytes ?? 1_500_000 + return Buffer.byteLength(text) <= max ? output : { truncated: true, bytes: Buffer.byteLength(text), preview: text.slice(0, 4000) } + } + + private queueVerdict(v: CompletePayload) { this.state.putVerdict(v); void this.flushOutbox() } + + #flushing: Promise | undefined + #flushGen = 0 + /** Sends every verdict in the outbox; a transient failure leaves it for the next pass. */ + flushOutbox(): Promise { + if (this.#flushing) return this.#flushing + const gen = ++this.#flushGen + const p = (async () => { + await Promise.resolve() // the pass starts after #flushing is set, so its finally clears this pass and no other + try { + for (const v of this.state.outbox()) { + try { await this.deps.floor.complete(v) } catch (e) { + if (e instanceof CompleteRefused) this.#log("verdict-refused", { leaseId: v.leaseId, code: e.code }) + else if (e instanceof FloorError && e.kind === "rejected") this.#log("verdict-rejected", { leaseId: v.leaseId, message: e.message }) + else { this.#log("verdict-retry", { leaseId: v.leaseId, message: String((e as Error)?.message ?? e) }); this.fatal(e); continue } + } + this.state.dropVerdict(v.leaseId) + delete this.#held[v.leaseId] + this.persistHeld() + } + } finally { if (this.#flushGen === gen) this.#flushing = undefined } + })() + this.#flushing = p + return p + } + + private async heartbeatLoop() { + const stop = this.#stop.signal + while (!stop.aborted) { + await sleep(this.#hbMs, stop) + if (stop.aborted) break + const ids = Object.keys(this.#held) + const pendingRequestKey = this.state.pendingKey() + if (ids.length === 0 && pendingRequestKey === undefined) continue + try { + const r = await this.deps.floor.heartbeat({ holderIdentity: this.cfg.holder, leaseIds: ids, ...(pendingRequestKey ? { pendingRequestKey } : {}) }) + for (const id of r.lost) { + this.#log("lease-lost", { leaseId: id }) + this.abort(id, "lost") + if (!this.state.outbox().some((v) => v.leaseId === id)) delete this.#held[id] + } + if (r.lost.length > 0) this.persistHeld() + for (const id of r.cancelRequested) { this.#log("cancel-requested", { leaseId: id }); this.abort(id, "cancel") } + } catch (e) { this.#log("heartbeat-error", { message: String((e as Error)?.message ?? e) }); this.fatal(e) } + } + } +} diff --git a/pkgs/substrate-apps/src/apps/puller/src/state.ts b/pkgs/substrate-apps/src/apps/puller/src/state.ts new file mode 100644 index 00000000..9ad5f084 --- /dev/null +++ b/pkgs/substrate-apps/src/apps/puller/src/state.ts @@ -0,0 +1,35 @@ +// The puller's durable state, all under one directory, written before the act it records: +// session-id stable per directory: a restart is the same session (L5), a second copy is not +// held.json leaseId -> the grant, written when a Lease answer arrives, removed when its verdict lands +// pending-request-key the Lease requestKey in flight; a restart re-sends it and gets the same grants (rule 2b) +// outbox/.json a verdict not yet accepted by the floor; resent until it is +// runs// the run's own directory: script.js, journal.jsonl, events, jobs, result.json +import { randomUUID } from "node:crypto" +import { existsSync, mkdirSync, readdirSync, readFileSync, renameSync, rmSync, writeFileSync } from "node:fs" +import { join } from "node:path" +import type { Grant } from "@substrate/link/contract.ts" +import type { CompletePayload } from "@substrate/link/floor.ts" + +const atomic = (path: string, text: string) => { const tmp = `${path}.tmp-${process.pid}`; writeFileSync(tmp, text, { mode: 0o600 }); renameSync(tmp, path) } + +export class PullerState { + constructor(readonly dir: string) { mkdirSync(join(dir, "outbox"), { recursive: true }); mkdirSync(join(dir, "runs"), { recursive: true }) } + sessionId(): string { + const p = join(this.dir, "session-id") + if (!existsSync(p)) writeFileSync(p, randomUUID() + "\n", { mode: 0o600 }) + return readFileSync(p, "utf8").trim() + } + held(): Record { + const p = join(this.dir, "held.json") + return existsSync(p) ? JSON.parse(readFileSync(p, "utf8")) as Record : {} + } + putHeld(h: Record) { atomic(join(this.dir, "held.json"), JSON.stringify(h)) } + pendingKey(): string | undefined { const p = join(this.dir, "pending-request-key"); return existsSync(p) ? readFileSync(p, "utf8").trim() || undefined : undefined } + setPendingKey(k: string | undefined) { const p = join(this.dir, "pending-request-key"); if (k === undefined) rmSync(p, { force: true }); else atomic(p, k) } + outbox(): Array { + return readdirSync(join(this.dir, "outbox")).filter((f) => f.endsWith(".json")).sort().map((f) => JSON.parse(readFileSync(join(this.dir, "outbox", f), "utf8")) as CompletePayload) + } + putVerdict(v: CompletePayload) { atomic(join(this.dir, "outbox", `${v.leaseId}.json`), JSON.stringify(v)) } + dropVerdict(leaseId: string) { rmSync(join(this.dir, "outbox", `${leaseId}.json`), { force: true }) } + runDir(runId: string) { const d = join(this.dir, "runs", runId); mkdirSync(d, { recursive: true }); return d } +} diff --git a/pkgs/substrate-apps/src/apps/puller/test/puller.test.ts b/pkgs/substrate-apps/src/apps/puller/test/puller.test.ts new file mode 100644 index 00000000..c4c4e818 --- /dev/null +++ b/pkgs/substrate-apps/src/apps/puller/test/puller.test.ts @@ -0,0 +1,183 @@ +// The puller's lease loop against the real floor (the Worker entry and Floor object over node:sqlite, +// apps/floor/src/local.ts), through the link's own rpcFloor client: lease, run, heartbeat, complete; a kill and a +// restart that re-adopts the lease and resumes from the run's journal; a cancel; the local executor with the repo's +// runners (a fake claude binary) and the ax refusal; the pidfile. +import { chmodSync, existsSync, mkdtempSync, readFileSync, writeFileSync } from "node:fs" +import { tmpdir } from "node:os" +import { join } from "node:path" +import { afterAll, afterEach, beforeAll, describe, expect, it } from "vitest" +import { SubstrateClient } from "@substrate/api" +import { localFloor } from "@substrate/floor/local.ts" +import { parseRuntimesToml } from "@substrate/runners" +import { connectFloor } from "../src/connect.ts" +import { FloorBackend, floorExecutor, localExecutor } from "../src/execute.ts" +import { acquirePidfile, PidfileHeld } from "../src/pidfile.ts" +import { Puller } from "../src/puller.ts" +import type { Executor } from "../src/puller.ts" +import { main } from "../src/main.ts" + +const T = { floor: "puller-test-floor", coord: "puller-test-coord", nas: "puller-test-nas" } +const URL_ = "http://floor.test" +const cleanups: Array<() => unknown> = [] +afterEach(async () => { for (const c of cleanups.splice(0).reverse()) await c() }) + +const until = async (what: string, f: () => Promise | boolean, ms = 15_000) => { + const end = Date.now() + ms + while (!(await f())) { if (Date.now() > end) throw new Error(`timed out waiting for ${what}`); await new Promise((r) => setTimeout(r, 20)) } +} + +const world = async () => { + const f = localFloor({ floorToken: T.floor, links: [ + { token: T.coord, holder: "coord", labels: ["runtime:interpreter", "seat:interp"] }, + { token: T.nas, holder: "nas", labels: ["seat:t", "runtime:*"] } + ], vars: { FLOOR_UNGATED_SEATS: "*", FLOOR_INTERPRETER_SEAT: "interp", FLOOR_HEARTBEAT_SECONDS: "1", FLOOR_LEASE_SECONDS: "30", FLOOR_POLL_SECONDS: "1" } }) + cleanups.push(() => f.close()) + const client = new SubstrateClient({ url: URL_, token: T.floor, fetch: f.fetch }) + const stateDir = mkdtempSync(join(tmpdir(), "substrate-puller-")) + const nas = await connectFloor({ url: URL_, token: T.nas, sessionId: "nas-session", fetch: f.fetch }) + cleanups.push(() => nas.close()) + /** The NAS link's part, by hand: lease what is queued for it and answer each node by its prompt. */ + const serveNodes = async (answer: (prompt: string) => unknown | undefined) => { + const r = await nas.port.lease({ holderIdentity: "nas", capacity: 10, requestKey: crypto.randomUUID() }) + for (const g of r.grants) { + const out = answer(g.job.spec.with.prompt ?? "") + if (out !== undefined) await nas.port.complete({ leaseId: g.leaseId, attempt: g.attempt, result: "success", output: out, usage: { prompt_tokens: 3, completion_tokens: 1, tool_calls: 0 } }) + } + return r.grants.length + } + const start = async (execute: Executor, o: { hbMs?: number } = {}) => { + const conn = await connectFloor({ url: URL_, token: T.coord, sessionId: readFileSync(join(stateDir, "session-id"), "utf8").trim(), fetch: f.fetch }) + const events: Array<{ ev: string; f?: Record }> = [] + const p = new Puller({ holder: "coord", maxRuns: 1, stateDir, pollMs: 25, heartbeatMs: o.hbMs ?? 150 }, { floor: conn.port, client, execute, log: (ev, fields) => { if (process.env.PULLER_DEBUG) console.error(ev, JSON.stringify(fields)); events.push({ ev, ...(fields ? { f: fields } : {}) }) } }) + const done = p.run().finally(() => conn.close()) + cleanups.push(async () => { p.stop(); await done.catch(() => undefined) }) + return { p, done, events } + } + // The session id file is created by the first PullerState; make it exist before start() reads it. + const { PullerState } = await import("../src/state.ts") + new PullerState(stateDir).sessionId() + return { f, client, stateDir, serveNodes, start } +} + +const TWO_STEP = `export const meta = { name: "two-step", description: "d" } +const a = await agent("first", { label: "a" }) +const b = await agent("second after " + a, { label: "b" }) +return { a, b } +` + +describe("lease loop, floor dispatch", () => { + it("leases the run, dispatches its nodes to the floor, completes; after a kill it re-adopts the lease and resumes from the journal", async () => { + const w = await world() + await w.client.submitScript(TWO_STEP, { id: "pullrun0001" }) + const backends: Array = [] + const exec = floorExecutor({ client: w.client, defaultRunsOn: ["seat:t", "runtime:gvisor"], defaultModel: "test-model", pollMs: 25, onBackend: (b) => backends.push(b) }) + + const first = await w.start(exec) + await until("node 1 enqueued", async () => (await w.client.runJobs("pullrun0001")).some((j) => j.name.startsWith("n1-"))) + expect((await w.client.job("run-pullrun0001")).holder).toBe("coord") + expect(await w.serveNodes((p) => (p === "first" ? "A" : undefined))).toBe(1) + await until("node 2 enqueued", async () => (await w.client.runJobs("pullrun0001")).some((j) => j.name.startsWith("n2-"))) + // The kill: the process stops with node 2 in flight. No verdict is sent; the lease is kept for re-adoption. + first.p.stop(); await first.done + expect(Object.keys(JSON.parse(readFileSync(join(w.stateDir, "held.json"), "utf8")))).toEqual(["run-pullrun0001-a1"]) + const journal = readFileSync(join(w.stateDir, "runs", "pullrun0001", "journal.jsonl"), "utf8") + expect(journal.split("\n").filter((l) => l.includes("\"result\"")).length).toBe(1) + expect((await w.client.run("pullrun0001")).state).toBe("running") + + const second = await w.start(exec) + await until("re-adopted", () => second.events.some((e) => e.ev === "readopt")) + await until("node 2 served", async () => (await w.serveNodes((p) => (p.startsWith("second") ? "B" : undefined))) > 0) + const run = await w.client.waitRun("pullrun0001", { timeoutMs: 15_000, pollMs: 25 }) + expect([run.state, run.result]).toEqual(["done", "success"]) + const out = await w.client.output("run-pullrun0001") + expect(out).toMatchObject({ result: "success", attempt: 1, output: { resumed: true, status: "completed", result: { a: "A", b: "B" } } }) + // Node 1 was a journal hit on the resume: never enqueued again. Node 2 was re-enqueued under the same name. + expect(backends[1]!.enqueued).toEqual([backends[0]!.enqueued[1]]) + expect((await w.client.runJobs("pullrun0001")).filter((j) => j.kind === "agent").map((j) => j.name)).toHaveLength(2) + await until("verdict flushed", () => !existsSync(join(w.stateDir, "outbox", "run-pullrun0001-a1.json")) && Object.keys(second.p.held).length === 0) + const pullers = await w.client.pullers() + expect(pullers.map((p) => p.holder).sort()).toEqual(["coord", "nas"]) + }) + + it("a cancel reaches the running run through the heartbeat and is completed as cancelled", async () => { + const w = await world() + await w.client.submitScript(TWO_STEP, { id: "pullrun0002" }) + const exec = floorExecutor({ client: w.client, defaultRunsOn: ["seat:t"], defaultModel: "test-model", pollMs: 25 }) + const s = await w.start(exec) + await until("node 1 enqueued", async () => (await w.client.runJobs("pullrun0002")).some((j) => j.name.startsWith("n1-"))) + await w.client.cancelRun("pullrun0002") + await until("cancel requested", () => s.events.some((e) => e.ev === "cancel-requested")) + await until("interpreter job done", async () => (await w.client.job("run-pullrun0002")).state === "done") + expect((await w.client.output("run-pullrun0002")).result).toBe("cancelled") + expect(s.events.find((e) => e.ev === "run-end")?.f).toMatchObject({ aborted: "cancel" }) + }) + + it("refuses a script that does not hash to the grant's content address", async () => { + const w = await world() + await w.client.submitScript(TWO_STEP, { id: "pullrun0003" }) + const s = await w.start(async () => { throw new Error("must not run") }) + // Tamper with what the floor serves: the grant still carries the original sha256. + const orig = w.client.runScript.bind(w.client) + ;(s.p.deps.client as { runScript: (id: string) => Promise }).runScript = async (id) => (await orig(id)) + "\n// tampered" + await until("done", async () => (await w.client.job("run-pullrun0003")).state === "done") + expect((await w.client.output("run-pullrun0003")).output).toMatchObject({ reason: "pre-start/script-mismatch" }) + }) +}) + +describe("local executor (the repo's runners)", () => { + const bin = mkdtempSync(join(tmpdir(), "substrate-puller-bin-")) + const saved = process.env.PATH + beforeAll(() => { + writeFileSync(join(bin, "claude"), `#!/bin/sh\np=$(cat)\ncase "$p" in\n *gold*) echo '{"type":"result","subtype":"success","is_error":false,"session_id":"s-gold","result":"Au","usage":{"input_tokens":3,"output_tokens":1}}' ;;\nesac\n`) + chmodSync(join(bin, "claude"), 0o755) + process.env.PATH = `${bin}:${saved}` + }) + afterAll(() => { process.env.PATH = saved }) + + it("runs a leased run with runIntegrated on a host runtime and completes it with the return value", async () => { + const w = await world() + await w.client.submitScript(`export const meta = { name: "gold", description: "d" }\nreturn await agent("say gold")\n`, { id: "pullrun0004" }) + const runtimes = parseRuntimesToml(`default = "h"\n[runtime.h]\ntype = "host"\nharness = "claude"\n`, "test") + await w.start(localExecutor({ runtimes, seat: "fake", defaultModel: "fake-model", cap: 1, maxAttempts: 1 })) + const run = await w.client.waitRun("pullrun0004", { timeoutMs: 30_000, pollMs: 25 }) + expect([run.state, run.result]).toEqual(["done", "success"]) + expect((await w.client.output("run-pullrun0004")).output).toMatchObject({ status: "completed", result: "Au", outcome: "all-done" }) + expect(existsSync(join(w.stateDir, "runs", "pullrun0004", "journal.jsonl"))).toBe(true) + }) + + it("an ax runtime is refused with the reason, and the run fails with it", async () => { + const w = await world() + await w.client.submitScript(`export const meta = { name: "axed", description: "d" }\nconst r = await agent("say gold")\nif (r === null) throw new Error("the ax node was refused")\nreturn r\n`, { id: "pullrun0005" }) + const runtimes = parseRuntimesToml(`default = "axr"\n[runtime.axr]\ntype = "ax"\n`, "test") + await w.start(localExecutor({ runtimes, seat: "fake", defaultModel: "fake-model", cap: 1, maxAttempts: 1, axServer: "127.0.0.1:1" })) + const run = await w.client.waitRun("pullrun0005", { timeoutMs: 30_000, pollMs: 25 }) + expect([run.state, run.result]).toEqual(["done", "failure"]) + expect((await w.client.output("run-pullrun0005")).output).toMatchObject({ reason: "agent/script-failed" }) + const ledger = readFileSync(join(w.stateDir, "runs", "pullrun0005", "ledger.jsonl"), "utf8") + expect(ledger).toMatch(/ax-server 127\.0\.0\.1:1 is not reachable/) + }) +}) + +describe("process", () => { + it("holds a pidfile: a live holder refuses a second copy, a dead one is replaced", () => { + const dir = mkdtempSync(join(tmpdir(), "substrate-pid-")) + const path = join(dir, "puller.pid") + writeFileSync(path, `${process.pid}\n`) + expect(() => acquirePidfile(path, "node", 2_000_000_000)).toThrow(PidfileHeld) + writeFileSync(path, "2000000001\n") + const release = acquirePidfile(path, "node", 2_000_000_000) + expect(readFileSync(path, "utf8").trim()).toBe("2000000000") + release() + expect(existsSync(path)).toBe(false) + }) + + it("main refuses to start without a [puller] config (exit 78)", async () => { + const dir = mkdtempSync(join(tmpdir(), "substrate-pcfg-")) + writeFileSync(join(dir, "config.toml"), `floor_url = "http://floor.test"\n`) + const writes: Array = [] + const orig = process.stdout.write.bind(process.stdout) + process.stdout.write = ((s: string) => { writes.push(s); return true }) as typeof process.stdout.write + try { expect(await main({ SUBSTRATE_CLIENT_CONFIG: join(dir, "config.toml") })).toBe(78) } finally { process.stdout.write = orig } + expect(writes.join("")).toMatch(/config-invalid.*holder is required/) + }) +}) diff --git a/pkgs/substrate-apps/src/apps/puller/tsconfig.json b/pkgs/substrate-apps/src/apps/puller/tsconfig.json new file mode 100644 index 00000000..4edbf3bf --- /dev/null +++ b/pkgs/substrate-apps/src/apps/puller/tsconfig.json @@ -0,0 +1,5 @@ +{ + "extends": "../../tsconfig.base.json", + "compilerOptions": { "exactOptionalPropertyTypes": false, "types": ["node"], "allowImportingTsExtensions": true, "lib": ["ES2022", "DOM"] }, + "include": ["src/**/*.ts", "test/**/*.ts", "../floor/src/cf-types.d.ts"] +} diff --git a/pkgs/substrate-apps/src/apps/puller/vitest.config.ts b/pkgs/substrate-apps/src/apps/puller/vitest.config.ts new file mode 100644 index 00000000..49b4806f --- /dev/null +++ b/pkgs/substrate-apps/src/apps/puller/vitest.config.ts @@ -0,0 +1,2 @@ +import { defineConfig } from "vitest/config"; +export default defineConfig({ test: { name: "puller", include: ["test/**/*.test.ts"], environment: "node", testTimeout: 60000 } }); diff --git a/pkgs/substrate-apps/src/deploy/client.config.example.toml b/pkgs/substrate-apps/src/deploy/client.config.example.toml new file mode 100644 index 00000000..6b1ea6b0 --- /dev/null +++ b/pkgs/substrate-apps/src/deploy/client.config.example.toml @@ -0,0 +1,24 @@ +# ~/.config/substrate/config.toml: what the substrate CLI, the MCP server and the puller read. +# It names the floor and WHERE each credential is, never a credential. Every file named here is mode 600. +# SUBSTRATE_CLIENT_CONFIG moves this file; SUBSTRATE_URL and SUBSTRATE_TOKEN_FILE override the first two keys. + +floor_url = "https://substrate.example.dev" +token_file = "~/.local/state/substrate/floor-token" # the operator bearer (FLOOR_TOKEN) +# access_client_id_file = "~/.local/state/substrate/access-client-id" # a Cloudflare Access service +# access_client_secret_file = "~/.local/state/substrate/access-client-secret" # token pair, instead of or beside it + +[puller] +holder = "coordinator" # the holder its link token is bound to (LINK_TOKENS) +link_token_file = "~/.local/state/substrate/link-token-coordinator" +state_dir = "~/.local/state/substrate/puller" # held leases, verdict outbox, runs// journals +pidfile = "~/.local/state/substrate/puller.pid" +max_runs = 1 # runs held at once (the Lease capacity) +node_dispatch = "local" # "local": the runners here; "floor": nodes go back to the floor +# local +runtimes = "~/.config/substrate/runtimes.toml" +seat = "cc" # the seat the capacity gate reads for this host's calls +cap = 2 # the WIP cap across this puller's calls +default_model = "claude-opus-5-5" +# ax_server = "nas:7070" # probed only to say why an ax node is refused here +# floor +node_runs_on = ["seat:halogen", "runtime:gvisor"] # runs-on for a node that names none diff --git a/pkgs/substrate-apps/src/deploy/substrate.config.example.json b/pkgs/substrate-apps/src/deploy/substrate.config.example.json new file mode 100644 index 00000000..d06a235d --- /dev/null +++ b/pkgs/substrate-apps/src/deploy/substrate.config.example.json @@ -0,0 +1,21 @@ +{ + "comment": "Copy to ~/.config/substrate/substrate.json or point SUBSTRATE_CONFIG at it. Every key is optional; see src/deploy-config.ts.", + "forbiddenReadPrefixes": [ + "~/private", + "~/.local/state/secrets" + ], + "forbiddenLedgerPrefixes": [ + "/run/user", + "~/.local/state/substrate/meters" + ], + "metersDir": "~/.local/state/substrate/meters", + "axProtoFallbackPath": null, + "realSeats": [ + "claude-main", + "codex", + "gpu-worker" + ], + "codexWorkdir": "~/src/my-project", + "capacityFloorUrl": "https://substrate.example.dev", + "capacityFloorTokenFile": "~/.local/state/substrate/floor-token" +} diff --git a/pkgs/substrate-apps/src/deploy/wrangler.local.example.jsonc b/pkgs/substrate-apps/src/deploy/wrangler.local.example.jsonc new file mode 100644 index 00000000..dccbaab9 --- /dev/null +++ b/pkgs/substrate-apps/src/deploy/wrangler.local.example.jsonc @@ -0,0 +1,43 @@ +{ + // deploy/wrangler.local.example.jsonc: the deploy-local Worker config. + // + // Copy to apps/floor/wrangler.local.jsonc (gitignored by *.local.jsonc) and + // fill in YOUR account, route and seat names. Deploy with: + // cd apps/floor && wrangler deploy --config wrangler.local.jsonc + // The committed apps/floor/wrangler.jsonc names no account and no route, so + // no command run from a fresh clone can publish anything by accident. + "name": "substrate-floor", + "main": "src/index.ts", + "compatibility_date": "2026-05-18", + "account_id": "", + "routes": [{ "pattern": "substrate.example.com", "custom_domain": true }], + "durable_objects": { + "bindings": [ + { "name": "FACTORY", "class_name": "Factory" }, + { "name": "FLOOR", "class_name": "Floor" } + ] + }, + "migrations": [ + { "tag": "v1", "new_sqlite_classes": ["Factory"] }, + { "tag": "v2", "new_sqlite_classes": ["Floor"] } + ], + // Secrets are never vars. Set each with `wrangler secret put --config wrangler.local.jsonc`: + // FLOOR_TOKEN operator bearer (intake API, CLI, capacity pusher) + // LINK_TOKENS JSON {"": {"holder": "...", "labels": ["seat:...", "runtime:..."], "guest": false}} + // LEASE_TOKEN_KEY HMAC key for guest lease tokens (only needed when a holder has "guest": true) + // ACCESS_CLIENT_ID optional: a Cloudflare Access service token accepted in place of FLOOR_TOKEN + // ACCESS_CLIENT_SECRET + "vars": { + // The CONWIP cap over agent jobs, and the runs held by interpreter hosts at once. + "FLOOR_CAP": "2", + "FLOOR_RUN_CAP": "4", + // The seat an interpreter host (runtime:interpreter) serves; runs are handed out on it. + "FLOOR_INTERPRETER_SEAT": "coordinator", + // Seats admitted without a capacity reading (comma list). Every other seat needs a fresh seat-capacity/2 row. + "FLOOR_UNGATED_SEATS": "coordinator", + // runs-on seat name -> capacity seat id, when they differ (e.g. a GPU slot row). + "FLOOR_SEAT_CAPACITY_MAP": "{\"gpu\": \"gpu-worker\"}", + // Per-label caps, JSON. + "FLOOR_LABEL_CAPS": "{}" + } +} diff --git a/pkgs/substrate-apps/src/package.json b/pkgs/substrate-apps/src/package.json index 34a14c37..5541c17d 100644 --- a/pkgs/substrate-apps/src/package.json +++ b/pkgs/substrate-apps/src/package.json @@ -6,7 +6,7 @@ "description": "CONWIP factory: turns Claude ultracode workflow run records into ax Tasks under a WIP cap, plus the Effect workspace (schema, serializer, planning, factory) and the Cloudflare floor.", "scripts": { "test": "vitest run", - "typecheck": "tsc --noEmit -p . && tsc --noEmit -p packages/schema && tsc --noEmit -p packages/serializer && tsc --noEmit -p packages/planning && tsc --noEmit -p packages/factory && tsc --noEmit -p apps/floor && tsc --noEmit -p packages/interpreter && tsc --noEmit -p packages/runners && tsc --noEmit -p apps/link", + "typecheck": "tsc --noEmit -p . && tsc --noEmit -p packages/schema && tsc --noEmit -p packages/serializer && tsc --noEmit -p packages/planning && tsc --noEmit -p packages/factory && tsc --noEmit -p apps/floor && tsc --noEmit -p packages/interpreter && tsc --noEmit -p packages/runners && tsc --noEmit -p apps/link && tsc --noEmit -p packages/api && tsc --noEmit -p apps/floor/tsconfig.node.json && tsc --noEmit -p apps/cli && tsc --noEmit -p apps/mcp && tsc --noEmit -p apps/puller", "check:fences": "node tools/check-fences.mjs", "check": "pnpm typecheck && pnpm check:fences && pnpm test", "conwip": "tsx src/cli.ts", diff --git a/pkgs/substrate-apps/src/packages/api/package.json b/pkgs/substrate-apps/src/packages/api/package.json new file mode 100644 index 00000000..69673b8b --- /dev/null +++ b/pkgs/substrate-apps/src/packages/api/package.json @@ -0,0 +1,18 @@ +{ + "name": "@substrate/api", + "version": "0.1.0", + "private": true, + "type": "module", + "description": "The floor's HTTP API described once as Effect Schema (HttpApi): the generated OpenAPI 3.1 document, a typed Promise client, and the operator config shared by the CLI, the MCP server and the puller.", + "license": "UNLICENSED", + "exports": { ".": "./src/index.ts", "./*": "./src/*" }, + "scripts": { "check": "tsc --noEmit -p .", "test": "vitest run" }, + "dependencies": { "smol-toml": "^1.4.2", "@substrate/link": "workspace:*" }, + "peerDependencies": { "effect": "4.0.0-rc.112" }, + "devDependencies": { + "@types/node": "^22.10.2", + "effect": "4.0.0-rc.112", + "typescript": "^5.9.3", + "vitest": "^3.2.4" + } +} diff --git a/pkgs/substrate-apps/src/packages/api/src/api.ts b/pkgs/substrate-apps/src/packages/api/src/api.ts new file mode 100644 index 00000000..433b530e --- /dev/null +++ b/pkgs/substrate-apps/src/packages/api/src/api.ts @@ -0,0 +1,82 @@ +// The floor's routes as one Effect HttpApi. Nothing serves this definition directly: the floor's own router +// (apps/floor/src/link/floor-object.ts) is the implementation and apps/floor/test/openapi.test.ts holds the two to +// the same route table. What this buys is one description with two outputs: GET /openapi.json (OpenApi.fromApi) +// and the client's decoders (client.ts). /rpc is Effect RPC (Lease, Heartbeat, Complete) and is described in the +// document's info, not as REST paths. +import { Schema } from "effect" +import { HttpApi, HttpApiEndpoint, HttpApiGroup, HttpApiSchema, OpenApi } from "effect/unstable/httpapi" +import * as S from "./schema.ts" + +const Id = { id: Schema.String } +const Name = { name: Schema.String } +const errors = [S.ErrorBody] +const doc = unknown }>(e: E, summary: string, description?: string): E => + // eslint-disable-next-line @typescript-eslint/no-explicit-any + { let r: any = (e as any).annotate(OpenApi.Summary, summary); if (description) r = r.annotate(OpenApi.Description, description); return r } + +export const RunsGroup = HttpApiGroup.make("runs") + .add(doc(HttpApiEndpoint.post("submitRun", "/runs", { payload: S.SubmitRequest, success: S.SubmitReply.pipe(HttpApiSchema.status(201)), error: errors }), + "Submit a run", "A workflow script (JSON { script } or a text/javascript body) or a list of AgentJob. Answers 201 when created, 200 when the id already exists.")) + .add(doc(HttpApiEndpoint.get("listRuns", "/runs", { query: { limit: Schema.optionalKey(Schema.String) }, success: Schema.Struct({ runs: Schema.Array(S.RunView) }), error: errors }), "List runs, newest first")) + .add(doc(HttpApiEndpoint.get("getRun", "/runs/:id", { params: Id, success: Schema.Struct({ run: S.RunView }), error: errors }), "One run")) + .add(doc(HttpApiEndpoint.get("runJobs", "/runs/:id/jobs", { params: Id, success: Schema.Struct({ jobs: Schema.Array(S.JobSummary) }), error: errors }), "The jobs of a run")) + .add(doc(HttpApiEndpoint.post("enqueueRunJobs", "/runs/:id/jobs", { params: Id, payload: Schema.Union([S.AgentJob, Schema.Array(S.AgentJob)]), success: S.Enqueued.pipe(HttpApiSchema.status(201)), error: errors }), + "Enqueue nodes of a run", "The interpreter host's call: each agent() node as an AgentJob, idempotent by metadata.name.")) + .add(doc(HttpApiEndpoint.get("runScript", "/runs/:id/script", { params: Id, success: Schema.String.pipe(HttpApiSchema.asText({ contentType: "text/javascript" })), error: errors }), "The script of a script run")) + .add(doc(HttpApiEndpoint.get("runEvents", "/runs/:id/events", { params: Id, query: { after: Schema.optionalKey(Schema.String) }, success: Schema.Struct({ events: Schema.Array(S.FloorEvent) }), error: errors }), "Verdict events of a run")) + .add(doc(HttpApiEndpoint.post("cancelRun", "/runs/:id/cancel", { params: Id, success: Schema.Struct({ run: S.RunView }), error: errors }), "Cancel a run and every unfinished job in it")) + +export const JobsGroup = HttpApiGroup.make("jobs") + .add(doc(HttpApiEndpoint.get("getJob", "/jobs/:name", { params: Name, success: Schema.Struct({ job: S.JobView }), error: errors }), "One job")) + .add(doc(HttpApiEndpoint.get("jobOutput", "/jobs/:name/output", { params: Name, success: S.JobOutput, error: errors }), "A job's verdict and output", "404 no-output while the job has no verdict.")) + .add(doc(HttpApiEndpoint.post("cancelJob", "/jobs/:name/cancel", { params: Name, success: Schema.Struct({ job: S.JobView }), error: errors }), "Cancel one job")) + .add(doc(HttpApiEndpoint.get("events", "/events", { query: { after: Schema.optionalKey(Schema.String) }, success: Schema.Struct({ events: Schema.Array(S.FloorEvent) }), error: errors }), "Every verdict event after a sequence number")) + +export const HoldersGroup = HttpApiGroup.make("holders") + .add(doc(HttpApiEndpoint.get("listHolders", "/holders", { success: Schema.Struct({ holders: Schema.Array(S.Holder) }), error: errors }), "Pullers and links known to the floor")) + .add(doc(HttpApiEndpoint.post("pauseHolder", "/holders/:holder/pause", { params: { holder: Schema.String }, success: Schema.Struct({ holder: Schema.String, paused: Schema.Boolean }), error: errors }), "Stop granting to a holder")) + .add(doc(HttpApiEndpoint.post("resumeHolder", "/holders/:holder/resume", { params: { holder: Schema.String }, success: Schema.Struct({ holder: Schema.String, paused: Schema.Boolean }), error: errors }), "Grant to a holder again")) + +export const CapacityGroup = HttpApiGroup.make("capacity") + .add(doc(HttpApiEndpoint.get("capacity", "/capacity", { query: { model: Schema.optionalKey(Schema.String), min_headroom_pct: Schema.optionalKey(Schema.String), asOf: Schema.optionalKey(Schema.String) }, success: S.CapacityView, error: errors }), + "Every seat's projected capacity", "Grade, staleness, headroom, resets, the floor's WIP per seat and admit for ?model.")) + .add(doc(HttpApiEndpoint.get("capacityAdmit", "/capacity/admit", { query: { seat: Schema.String, model: Schema.optionalKey(Schema.String), min_headroom_pct: Schema.optionalKey(Schema.String) }, success: S.AdmitAnswer, error: errors }), "Would one seat admit a job now")) + .add(doc(HttpApiEndpoint.post("capacitySnapshot", "/capacity/snapshots", { payload: Schema.Unknown.annotate({ description: "A seat-capacity/2 snapshot." }), success: Schema.Record(Schema.String, Schema.Unknown), error: errors }), "Push a seat-capacity/2 snapshot (the pusher)")) + +export const FloorGroup = HttpApiGroup.make("floor") + .add(doc(HttpApiEndpoint.get("floorState", "/floor/state", { success: S.FloorState, error: errors }), "Floor statistics, run WIP, alarm and config")) + .add(doc(HttpApiEndpoint.get("openapi", "/openapi.json", { success: Schema.Record(Schema.String, Schema.Unknown) }), "This document", "Served without a credential: it carries no data.")) + +export const SubstrateApi = HttpApi.make("substrate") + .add(RunsGroup).add(JobsGroup).add(HoldersGroup).add(CapacityGroup).add(FloorGroup) + .annotate(OpenApi.Title, "substrate floor") + .annotate(OpenApi.Description, "The job-of-record floor. Operator routes take `Authorization: Bearer ` or a Cloudflare Access service token pair. " + + "Pullers and links use POST /rpc (Effect RPC over HTTP, JSON serialization) with a per-link token: Lease, Heartbeat, Complete (see apps/link/src/contract.ts).") + +/** Operator routes as (method, path template) pairs; the floor test holds the router to this list. */ +export const ROUTES: ReadonlyArray<{ readonly method: string; readonly path: string; readonly id: string; readonly public: boolean }> = + Object.values(SubstrateApi.groups).flatMap((g) => Object.values(g.endpoints).map((e) => ({ method: e.method, path: e.path, id: e.name, public: e.path === "/openapi.json" }))) + +let cached: Record | undefined +/** The OpenAPI 3.1 document, with the two credential schemes the door accepts. */ +export const openApiDocument = (): Record => { + if (cached) return cached + const spec = OpenApi.fromApi(SubstrateApi) as unknown as Record & { components?: Record; paths: Record>> } + const components = (spec.components ?? {}) as Record + components.securitySchemes = { + bearer: { type: "http", scheme: "bearer", description: "FLOOR_TOKEN (operator)." }, + accessClientId: { type: "apiKey", in: "header", name: "CF-Access-Client-Id" }, + accessClientSecret: { type: "apiKey", in: "header", name: "CF-Access-Client-Secret" } + } + spec.components = components + for (const [path, ops] of Object.entries(spec.paths)) for (const op of Object.values(ops)) + op.security = path === "/openapi.json" ? [] : [{ bearer: [] }, { accessClientId: [], accessClientSecret: [] }] + spec.servers = [{ url: "/" }] + // Refusals are the floor's 4xx (400, 401, 403, 404, 409, 413, 422), not the generator's default 500. + for (const ops of Object.values(spec.paths)) for (const op of Object.values(ops)) { + const r = op.responses as Record | undefined + if (r && r["500"] !== undefined) { r["4XX"] = r["500"]; delete r["500"] } + } + cached = spec + return spec +} diff --git a/pkgs/substrate-apps/src/packages/api/src/client.ts b/pkgs/substrate-apps/src/packages/api/src/client.ts new file mode 100644 index 00000000..037bc587 --- /dev/null +++ b/pkgs/substrate-apps/src/packages/api/src/client.ts @@ -0,0 +1,138 @@ +// The typed client over the floor's operator routes. Promise-based so a CLI verb, an MCP tool and a code-mode +// snippet all call it the same way; every answer is decoded with the schema api.ts publishes, so a drifted floor +// fails here, loudly, rather than in a table three layers up. +import { Schema } from "effect" +import * as S from "./schema.ts" + +export class SubstrateError extends Error { + override readonly name = "SubstrateError" + constructor(readonly status: number, readonly code: string, message: string) { super(`${status} ${code}: ${message}`) } +} + +export interface ClientOptions { + /** The floor's base URL, e.g. https://substrate.example.dev (no trailing path). */ + readonly url: string + /** The operator bearer (FLOOR_TOKEN). Never logged. */ + readonly token?: string + /** A Cloudflare Access service token pair, sent instead of or beside the bearer. */ + readonly access?: { readonly clientId: string; readonly clientSecret: string } + readonly fetch?: typeof globalThis.fetch + readonly timeoutMs?: number +} + +export interface SeatRow { + readonly seat: string; readonly grade: string; readonly staleness: string; readonly headroom_pct: number | null + readonly next_reset_at: string | null; readonly wip: number; readonly admit: boolean | null; readonly reason: string | null +} + +const decode = (schema: Schema.Codec, v: unknown, what: string): A => { + try { return Schema.decodeUnknownSync(schema as never)(v) as A } catch (e) { + throw new SubstrateError(0, "decode", `${what}: the floor's answer does not match the published schema: ${String((e as Error).message).slice(0, 600)}`) + } +} + +const RunsReply = Schema.Struct({ runs: Schema.Array(S.RunView) }) +const RunReply = Schema.Struct({ run: S.RunView }) +const JobsReply = Schema.Struct({ jobs: Schema.Array(S.JobSummary) }) +const JobReply = Schema.Struct({ job: S.JobView }) +const EventsReply = Schema.Struct({ events: Schema.Array(S.FloorEvent) }) +const HoldersReply = Schema.Struct({ holders: Schema.Array(S.Holder) }) +const PauseReply = Schema.Struct({ holder: Schema.String, paused: Schema.Boolean }) + +export class SubstrateClient { + readonly url: string + readonly #o: ClientOptions + constructor(o: ClientOptions) { this.url = o.url.replace(/\/+$/, ""); this.#o = o } + + private headers(extra: Record = {}): Headers { + const h = new Headers(extra) + if (this.#o.token) h.set("authorization", `Bearer ${this.#o.token}`) + if (this.#o.access) { h.set("cf-access-client-id", this.#o.access.clientId); h.set("cf-access-client-secret", this.#o.access.clientSecret) } + return h + } + + /** One request; a non-2xx answer becomes a SubstrateError carrying the floor's own code. */ + async request(method: string, path: string, body?: { json?: unknown; text?: string; contentType?: string }): Promise<{ status: number; text: string }> { + const f = this.#o.fetch ?? globalThis.fetch + const init: RequestInit = { method, headers: this.headers(body?.json !== undefined ? { "content-type": "application/json" } : body?.text !== undefined ? { "content-type": body.contentType ?? "text/plain" } : {}) } + if (body?.json !== undefined) init.body = JSON.stringify(body.json) + else if (body?.text !== undefined) init.body = body.text + if (this.#o.timeoutMs) init.signal = AbortSignal.timeout(this.#o.timeoutMs) + const res = await f(`${this.url}${path}`, init) + const text = await res.text() + if (!res.ok) { + let code = `http-${res.status}`, message = text.slice(0, 500) + try { + const e = (JSON.parse(text) as { error?: unknown }).error + if (typeof e === "string") { code = e; message = e } else if (e && typeof e === "object") { code = String((e as { code?: unknown }).code ?? code); message = String((e as { message?: unknown }).message ?? message) } + } catch { /* not JSON */ } + throw new SubstrateError(res.status, code, message) + } + return { status: res.status, text } + } + private async json(method: string, path: string, json?: unknown): Promise { + const r = await this.request(method, path, json === undefined ? undefined : { json }) + try { return JSON.parse(r.text) } catch { throw new SubstrateError(r.status, "not-json", `${method} ${path} answered a non-JSON body`) } + } + private p = (s: string) => encodeURIComponent(s) + + // ---- runs + /** Submit a workflow script (source text) or AgentJobs. Idempotent when `id` is given. */ + async submit(req: S.SubmitRequest): Promise { return decode(S.SubmitReply, await this.json("POST", "/runs", req), "submit") } + submitScript(script: string, o: { args?: unknown; id?: string } = {}): Promise { return this.submit({ script, ...o }) } + submitJobs(jobs: ReadonlyArray, o: { name?: string; id?: string } = {}): Promise { return this.submit({ jobs, ...o }) } + async runs(o: { limit?: number } = {}): Promise> { return decode(RunsReply, await this.json("GET", `/runs?limit=${o.limit ?? 50}`), "runs").runs } + async run(id: string): Promise { return decode(RunReply, await this.json("GET", `/runs/${this.p(id)}`), "run").run } + async runJobs(id: string): Promise> { return decode(JobsReply, await this.json("GET", `/runs/${this.p(id)}/jobs`), "runJobs").jobs } + async runScript(id: string): Promise { return (await this.request("GET", `/runs/${this.p(id)}/script`)).text } + async runEvents(id: string, after = 0): Promise> { return decode(EventsReply, await this.json("GET", `/runs/${this.p(id)}/events?after=${after}`), "runEvents").events } + async cancelRun(id: string): Promise { return decode(RunReply, await this.json("POST", `/runs/${this.p(id)}/cancel`), "cancelRun").run } + /** The interpreter host's call: enqueue agent() nodes of a script run as AgentJobs (idempotent by name). */ + async enqueue(id: string, jobs: ReadonlyArray): Promise { return decode(S.Enqueued, await this.json("POST", `/runs/${this.p(id)}/jobs`, jobs), "enqueue") } + /** Poll a run until it is done or the timeout passes; answers the last view either way. */ + async waitRun(id: string, o: { timeoutMs?: number; pollMs?: number } = {}): Promise { + const until = Date.now() + (o.timeoutMs ?? 60_000) + for (;;) { + const r = await this.run(id) + if (r.state === "done" || Date.now() >= until) return r + await new Promise((ok) => setTimeout(ok, o.pollMs ?? 2000)) + } + } + + // ---- jobs + async job(name: string): Promise { return decode(JobReply, await this.json("GET", `/jobs/${this.p(name)}`), "job").job } + async output(name: string): Promise { return decode(S.JobOutput, await this.json("GET", `/jobs/${this.p(name)}/output`), "output") } + async cancelJob(name: string): Promise { return decode(JobReply, await this.json("POST", `/jobs/${this.p(name)}/cancel`), "cancelJob").job } + async events(after = 0): Promise> { return decode(EventsReply, await this.json("GET", `/events?after=${after}`), "events").events } + + // ---- capacity and seats + async capacity(o: { model?: string; minHeadroomPct?: number } = {}): Promise { + const q = new URLSearchParams() + if (o.model) q.set("model", o.model) + if (o.minHeadroomPct !== undefined) q.set("min_headroom_pct", String(o.minHeadroomPct)) + const qs = q.toString() + return decode(S.CapacityView, await this.json("GET", `/capacity${qs ? `?${qs}` : ""}`), "capacity") + } + async admit(seat: string, o: { model?: string } = {}): Promise> { + const q = new URLSearchParams({ seat, ...(o.model ? { model: o.model } : {}) }) + return decode(S.AdmitAnswer, await this.json("GET", `/capacity/admit?${q}`), "admit") as Record + } + /** One row per seat: the `seats` view of GET /capacity. */ + async seats(o: { model?: string } = {}): Promise> { + const v = await this.capacity(o) + return v.seats.map((s) => { + const a = (s.admit ?? {}) as { admit?: unknown; reason?: unknown } + return { seat: s.seat, grade: s.grade, staleness: s.staleness, headroom_pct: s.headroom_pct, next_reset_at: s.next_reset_at, wip: s.wip, + admit: typeof a.admit === "boolean" ? a.admit : null, reason: typeof a.reason === "string" ? a.reason : null } + }) + } + + // ---- pullers (holders) + async pullers(): Promise> { return decode(HoldersReply, await this.json("GET", "/holders"), "pullers").holders } + async pausePuller(holder: string): Promise<{ holder: string; paused: boolean }> { return decode(PauseReply, await this.json("POST", `/holders/${this.p(holder)}/pause`), "pause") } + async resumePuller(holder: string): Promise<{ holder: string; paused: boolean }> { return decode(PauseReply, await this.json("POST", `/holders/${this.p(holder)}/resume`), "resume") } + + // ---- floor + async floorState(): Promise { return decode(S.FloorState, await this.json("GET", "/floor/state"), "floorState") } + async openapi(): Promise> { return await this.json("GET", "/openapi.json") as Record } +} diff --git a/pkgs/substrate-apps/src/packages/api/src/config.ts b/pkgs/substrate-apps/src/packages/api/src/config.ts new file mode 100644 index 00000000..9f0508ff --- /dev/null +++ b/pkgs/substrate-apps/src/packages/api/src/config.ts @@ -0,0 +1,76 @@ +// The operator config every interface reads: ~/.config/substrate/config.toml (SUBSTRATE_CLIENT_CONFIG overrides the +// path). It names the floor and WHERE the credential is, never the credential itself. Environment overrides: +// SUBSTRATE_URL, SUBSTRATE_TOKEN_FILE. A deployment's own values (domain, paths) live only in that file; the repo +// carries deploy/client.config.example.toml. +import { existsSync, readFileSync, statSync } from "node:fs" +import { homedir } from "node:os" +import { parse } from "smol-toml" +import { SubstrateClient } from "./client.ts" + +export interface PullerSection { + readonly holder?: string + readonly link_token_file?: string + readonly state_dir?: string + readonly runtimes?: string + readonly max_runs?: number + readonly seat?: string + readonly default_model?: string + readonly cap?: number + readonly node_dispatch?: "local" | "floor" + readonly ax_server?: string + readonly pidfile?: string + readonly node_runs_on?: ReadonlyArray +} +export interface ClientConfig { + readonly floor_url?: string + readonly token_file?: string + readonly access_client_id_file?: string + readonly access_client_secret_file?: string + readonly puller?: PullerSection + /** Where it was read from, or undefined when no file exists. */ + readonly source?: string +} + +export class ConfigError extends Error { override readonly name = "ConfigError" } + +export const expandHome = (p: string, home = homedir()) => (p === "~" ? home : p.startsWith("~/") ? home + p.slice(1) : p) +export const DEFAULT_CONFIG_PATH = "~/.config/substrate/config.toml" + +export const loadClientConfig = (env: NodeJS.ProcessEnv = process.env, home = homedir()): ClientConfig => { + const path = expandHome(env.SUBSTRATE_CLIENT_CONFIG ?? DEFAULT_CONFIG_PATH, home) + let file: Record = {} + if (existsSync(path)) { + try { file = parse(readFileSync(path, "utf8")) as Record } catch (e) { throw new ConfigError(`${path}: ${(e as Error).message}`) } + } + const str = (k: string) => (typeof file[k] === "string" ? (file[k] as string) : undefined) + const url = env.SUBSTRATE_URL ?? str("floor_url") + const tokenFile = env.SUBSTRATE_TOKEN_FILE ?? str("token_file") + return { + ...(url !== undefined ? { floor_url: url } : {}), + ...(tokenFile !== undefined ? { token_file: expandHome(tokenFile, home) } : {}), + ...(str("access_client_id_file") !== undefined ? { access_client_id_file: expandHome(str("access_client_id_file")!, home) } : {}), + ...(str("access_client_secret_file") !== undefined ? { access_client_secret_file: expandHome(str("access_client_secret_file")!, home) } : {}), + ...(typeof file.puller === "object" && file.puller !== null ? { puller: file.puller as PullerSection } : {}), + ...(existsSync(path) ? { source: path } : {}) + } +} + +/** Reads a secret file. Refuses a group- or world-readable one; the value never enters an error message. */ +export const readSecretFile = (path: string, what: string): string => { + if (!existsSync(path)) throw new ConfigError(`${what}: ${path} does not exist`) + const mode = statSync(path).mode & 0o077 + if (mode !== 0) throw new ConfigError(`${what}: ${path} is readable by group or others (chmod 600 it)`) + const v = readFileSync(path, "utf8").trim() + if (v === "") throw new ConfigError(`${what}: ${path} is empty`) + return v +} + +export const clientFromConfig = (c: ClientConfig, o: { fetch?: typeof globalThis.fetch; home?: string } = {}): SubstrateClient => { + if (!c.floor_url) throw new ConfigError(`no floor_url: set it in ${DEFAULT_CONFIG_PATH} or SUBSTRATE_URL`) + const token = c.token_file ? readSecretFile(c.token_file, "token_file") : undefined + const access = c.access_client_id_file && c.access_client_secret_file + ? { clientId: readSecretFile(c.access_client_id_file, "access_client_id_file"), clientSecret: readSecretFile(c.access_client_secret_file, "access_client_secret_file") } + : undefined + if (token === undefined && access === undefined) throw new ConfigError("no credential: set token_file (or the Access pair files)") + return new SubstrateClient({ url: c.floor_url, ...(token ? { token } : {}), ...(access ? { access } : {}), ...(o.fetch ? { fetch: o.fetch } : {}) }) +} diff --git a/pkgs/substrate-apps/src/packages/api/src/index.ts b/pkgs/substrate-apps/src/packages/api/src/index.ts new file mode 100644 index 00000000..05a4b991 --- /dev/null +++ b/pkgs/substrate-apps/src/packages/api/src/index.ts @@ -0,0 +1,4 @@ +export * from "./schema.ts" +export * from "./api.ts" +export * from "./client.ts" +export * from "./config.ts" diff --git a/pkgs/substrate-apps/src/packages/api/src/schema.ts b/pkgs/substrate-apps/src/packages/api/src/schema.ts new file mode 100644 index 00000000..a4b4b985 --- /dev/null +++ b/pkgs/substrate-apps/src/packages/api/src/schema.ts @@ -0,0 +1,110 @@ +// The floor's HTTP bodies, described once. The floor (apps/floor) serves them, the OpenAPI document (api.ts) is +// generated from them, and the client (client.ts) decodes every answer with them. Field names are the floor's own +// (engine.ts run(), jobView(), output(), events(); capacity.ts capacityView()). +import { AgentJob } from "@substrate/link/contract.ts" +import { Schema } from "effect" + +export { AgentJob } + +export const ErrorBody = Schema.Struct({ + error: Schema.Union([Schema.String, Schema.Struct({ code: Schema.String, message: Schema.String })]) +}).annotate({ identifier: "ErrorBody", description: "Every refusal: { error: { code, message } }, or { error: \"Unauthorized\" } at the door." }) +export type ErrorBody = typeof ErrorBody.Type + +export const JobSummary = Schema.Struct({ + name: Schema.String, + runId: Schema.NullOr(Schema.String), + kind: Schema.String, // agent | run + state: Schema.String, // queued | leased | orphaned | done + attempt: Schema.Number, + leaseId: Schema.NullOr(Schema.String), + holder: Schema.NullOr(Schema.String), + runsOn: Schema.Array(Schema.String), + cancel: Schema.NullOr(Schema.String), + result: Schema.NullOr(Schema.String), + reason: Schema.NullOr(Schema.String), + supersedes: Schema.Array(Schema.String), + createdAt: Schema.Number, + doneAt: Schema.NullOr(Schema.Number) +}).annotate({ identifier: "JobSummary" }) +export type JobSummary = typeof JobSummary.Type + +export const JobView = Schema.Struct({ ...JobSummary.fields, job: Schema.Unknown }).annotate({ identifier: "JobView", description: "A job with its AgentJob body." }) +export type JobView = typeof JobView.Type + +export const RunView = Schema.Struct({ + id: Schema.String, + kind: Schema.String, // script | jobs + name: Schema.NullOr(Schema.String), + meta: Schema.Unknown, + args: Schema.Unknown, + scriptSha256: Schema.NullOr(Schema.String), + state: Schema.String, // running | done + result: Schema.NullOr(Schema.String), + createdAt: Schema.Number, + doneAt: Schema.NullOr(Schema.Number), + jobs: Schema.Record(Schema.String, Schema.Number), + interpreter: Schema.NullOr(JobView) +}).annotate({ identifier: "RunView" }) +export type RunView = typeof RunView.Type + +export const JobOutput = Schema.Struct({ + result: Schema.String, attempt: Schema.Number, leaseId: Schema.String, + output: Schema.Unknown, usage: Schema.Unknown, bytes: Schema.Number +}).annotate({ identifier: "JobOutput", description: "A job's recorded verdict and output." }) +export type JobOutput = typeof JobOutput.Type + +export const FloorEvent = Schema.Struct({ + seq: Schema.Number, run_id: Schema.NullOr(Schema.String), name: Schema.String, lease_id: Schema.NullOr(Schema.String), + attempt: Schema.Number, result: Schema.String, reason: Schema.NullOr(Schema.String), at_ms: Schema.Number +}).annotate({ identifier: "FloorEvent", description: "One row of the verdict outbox." }) +export type FloorEvent = typeof FloorEvent.Type + +export const ScriptSubmit = Schema.Struct({ + script: Schema.String.annotate({ description: "An ultracode workflow script whose `export const meta` is a pure literal." }), + args: Schema.optionalKey(Schema.Unknown), + id: Schema.optionalKey(Schema.String.annotate({ description: "6 to 40 lowercase letters or digits; makes the submit idempotent." })) +}).annotate({ identifier: "ScriptSubmit" }) +export const JobsSubmit = Schema.Struct({ + jobs: Schema.Array(AgentJob), + name: Schema.optionalKey(Schema.String), + id: Schema.optionalKey(Schema.String) +}).annotate({ identifier: "JobsSubmit" }) +export const SubmitRequest = Schema.Union([ScriptSubmit, JobsSubmit]).annotate({ identifier: "SubmitRequest" }) +export type SubmitRequest = typeof SubmitRequest.Type +export const SubmitReply = Schema.Struct({ run: RunView, created: Schema.Boolean }).annotate({ identifier: "SubmitReply" }) +export type SubmitReply = typeof SubmitReply.Type + +export const Enqueued = Schema.Struct({ enqueued: Schema.Array(Schema.Struct({ enqueued: Schema.Boolean, name: Schema.String })) }) +export type Enqueued = typeof Enqueued.Type + +export const SeatView = Schema.Struct({ + seat: Schema.String, host: Schema.String, published_at: Schema.String, received_at: Schema.String, + provider: Schema.Unknown, dispatchable: Schema.Unknown, dispatchable_reason: Schema.Unknown, + grade: Schema.String, staleness: Schema.String, age_seconds: Schema.NullOr(Schema.Number), + headroom_pct: Schema.NullOr(Schema.Number), next_reset_at: Schema.NullOr(Schema.String), wip: Schema.Number, + reading: Schema.Unknown, lapses: Schema.Unknown, plan_lapses_at: Schema.Unknown, admit: Schema.Unknown +}).annotate({ identifier: "SeatView" }) +export type SeatView = typeof SeatView.Type +export const CapacityView = Schema.Struct({ + as_of: Schema.String, + job: Schema.Struct({ model: Schema.NullOr(Schema.String), min_headroom_pct: Schema.Number }), + seats: Schema.Array(SeatView), + next_transition_at: Schema.NullOr(Schema.String) +}).annotate({ identifier: "CapacityView" }) +export type CapacityView = typeof CapacityView.Type +export const AdmitAnswer = Schema.Record(Schema.String, Schema.Unknown).annotate({ identifier: "AdmitAnswer" }) + +export const Holder = Schema.Struct({ + holder: Schema.String, + paused: Schema.Boolean, + live: Schema.Boolean.annotate({ description: "A session renewed within three heartbeat intervals." }), + sessionUntil: Schema.NullOr(Schema.Number), + leased: Schema.Number, + labels: Schema.Array(Schema.String).annotate({ description: "The runs-on labels of the jobs it holds now." }), + lastLeaseAt: Schema.NullOr(Schema.Number) +}).annotate({ identifier: "Holder", description: "A puller or link: anything that leases over /rpc." }) +export type Holder = typeof Holder.Type + +export const FloorState = Schema.Struct({ stats: Schema.Unknown, runWip: Schema.Unknown, alarm: Schema.Unknown, config: Schema.Unknown }) +export type FloorState = typeof FloorState.Type diff --git a/pkgs/substrate-apps/src/packages/api/test/api.test.ts b/pkgs/substrate-apps/src/packages/api/test/api.test.ts new file mode 100644 index 00000000..07640060 --- /dev/null +++ b/pkgs/substrate-apps/src/packages/api/test/api.test.ts @@ -0,0 +1,54 @@ +// The client's error mapping and decode guard, and the operator config loader. +import { chmodSync, mkdtempSync, writeFileSync } from "node:fs" +import { tmpdir } from "node:os" +import { join } from "node:path" +import { describe, expect, it } from "vitest" +import { clientFromConfig, ConfigError, loadClientConfig, openApiDocument, SubstrateClient, SubstrateError } from "../src/index.ts" + +const answer = (status: number, body: unknown): typeof fetch => (async () => new Response(typeof body === "string" ? body : JSON.stringify(body), { status })) as typeof fetch + +describe("SubstrateClient", () => { + it("carries the floor's own error code, in both refusal shapes", async () => { + await expect(new SubstrateClient({ url: "http://f", token: "t", fetch: answer(404, { error: { code: "not-found", message: "no run x" } }) }).run("x")) + .rejects.toMatchObject({ status: 404, code: "not-found" }) + await expect(new SubstrateClient({ url: "http://f", fetch: answer(401, { error: "Unauthorized" }) }).runs()).rejects.toMatchObject({ status: 401, code: "Unauthorized" }) + await expect(new SubstrateClient({ url: "http://f", fetch: answer(502, "bad gateway") }).runs()).rejects.toMatchObject({ status: 502, code: "http-502" }) + }) + it("refuses an answer that does not match the published schema", async () => { + const e = await new SubstrateClient({ url: "http://f", fetch: answer(200, { runs: [{ id: 1 }] }) }).runs().catch((x: unknown) => x) + expect(e).toBeInstanceOf(SubstrateError) + expect((e as SubstrateError).code).toBe("decode") + }) + it("sends the bearer and the Access pair", async () => { + let seen: Headers | undefined + const f = (async (_: unknown, init?: RequestInit) => { seen = new Headers(init?.headers); return new Response(JSON.stringify({ runs: [] })) }) as typeof fetch + await new SubstrateClient({ url: "http://f/", token: "tok", access: { clientId: "id", clientSecret: "sec" }, fetch: f }).runs() + expect([seen!.get("authorization"), seen!.get("cf-access-client-id"), seen!.get("cf-access-client-secret")]).toEqual(["Bearer tok", "id", "sec"]) + }) +}) + +describe("config", () => { + it("reads the file, expands ~, lets the environment override, and refuses a loose secret file", () => { + const home = mkdtempSync(join(tmpdir(), "substrate-home-")) + const cfg = join(home, "c.toml") + writeFileSync(cfg, `floor_url = "https://floor.example"\ntoken_file = "~/tok"\n[puller]\nholder = "coord"\nmax_runs = 2\n`) + const c = loadClientConfig({ SUBSTRATE_CLIENT_CONFIG: cfg }, home) + expect(c).toMatchObject({ floor_url: "https://floor.example", token_file: join(home, "tok"), puller: { holder: "coord", max_runs: 2 }, source: cfg }) + expect(loadClientConfig({ SUBSTRATE_CLIENT_CONFIG: cfg, SUBSTRATE_URL: "http://other" }, home).floor_url).toBe("http://other") + expect(() => clientFromConfig(c)).toThrow(/does not exist/) + writeFileSync(join(home, "tok"), "secret-value\n") + chmodSync(join(home, "tok"), 0o644) + let msg = "" + try { clientFromConfig(c) } catch (e) { expect(e).toBeInstanceOf(ConfigError); msg = (e as Error).message } + expect(msg).toMatch(/chmod 600/) + expect(msg).not.toContain("secret-value") + chmodSync(join(home, "tok"), 0o600) + expect(clientFromConfig(c).url).toBe("https://floor.example") + expect(() => clientFromConfig(loadClientConfig({ SUBSTRATE_CLIENT_CONFIG: join(home, "none.toml") }, home))).toThrow(/no floor_url/) + }) + it("the OpenAPI document declares both credential schemes and no 500 responses", () => { + const d = openApiDocument() as { components: { securitySchemes: Record }; paths: Record } + expect(Object.keys(d.components.securitySchemes).sort()).toEqual(["accessClientId", "accessClientSecret", "bearer"]) + expect(JSON.stringify(d.paths)).not.toContain("\"500\"") + }) +}) diff --git a/pkgs/substrate-apps/src/packages/api/tsconfig.json b/pkgs/substrate-apps/src/packages/api/tsconfig.json new file mode 100644 index 00000000..40fe4b28 --- /dev/null +++ b/pkgs/substrate-apps/src/packages/api/tsconfig.json @@ -0,0 +1,5 @@ +{ + "extends": "../../tsconfig.base.json", + "compilerOptions": { "types": ["node"], "allowImportingTsExtensions": true, "lib": ["ES2022", "DOM"] }, + "include": ["src/**/*.ts", "test/**/*.ts"] +} diff --git a/pkgs/substrate-apps/src/packages/api/vitest.config.ts b/pkgs/substrate-apps/src/packages/api/vitest.config.ts new file mode 100644 index 00000000..11387e4c --- /dev/null +++ b/pkgs/substrate-apps/src/packages/api/vitest.config.ts @@ -0,0 +1,2 @@ +import { defineConfig } from "vitest/config"; +export default defineConfig({ test: { name: "api", include: ["test/**/*.test.ts"], environment: "node" } }); diff --git a/pkgs/substrate-apps/src/packages/factory/README.md b/pkgs/substrate-apps/src/packages/factory/README.md new file mode 100644 index 00000000..d25d6cac --- /dev/null +++ b/pkgs/substrate-apps/src/packages/factory/README.md @@ -0,0 +1,42 @@ +# @substrate/factory + +U-A11 supplies the host-neutral Factory service over the `PlanningStore` +interface. Tests compose the in-memory store, one constant `PriceVector`, and an +authored fake capacity feed. The package has no Worker entry and no SQLite +binding; those are U-A14's composition-root work. + +The service owns the `unclaimed → released → inflight → closed` lifecycle. +`handOut` makes the first transition, `observeOutcome` applies `Accepted`, +`Rejected`, or `NotYet`, and `observeVerdict` mirrors a chained kernel record, +deduplicates `(executor, seq, hash)`, updates WIP once, and wakes the evaluator. +Only an `inflight` item may consume a verdict; a merely `released` item is +refused with its current state and keeps its WIP unchanged. A non-pass verdict +retries under the plan's `attemptCap` with a new armed-plan-and-attempt dedup +identity; a pass closes the item and can release an item whose `dependsOn` edge +has just become satisfied. + +The service schema-encodes its mutable state under `FACTORY_STATE_KEY` after +each accepted mutation and restores it when another Factory is constructed over +the same `PlanningStore`. Inflight work, WIP, readings, mirrored chain state, +alarms, outcome idempotence, deferrals, heartbeats, receipts, and facts therefore +survive an in-memory recreation today and the U-A14 SQLite Layer later. +The backlog schema includes optional `argv_ref` and `mutation_hint` cells, so +the command reference and evaluator control carried by an armed item survive +that same boundary and are unchanged when the item is proposed after restart. + +`makeFactoryHttpHandler` exposes `/plans`, `/capacity`, `/proposals`, `/outcomes`, +`/verdicts`, `/receipts`, `/heartbeats`, and `/state` without instantiating a +Worker. A proposal pull returns `{ proposals, next_wake_at }`, including the wake +instant when every row is stopped and the proposal list is empty. Every +state-changing request—including that GET proposal pull—requires the bearer +token supplied by the host. Receipt ingestion first uses the strict §2.3 decoder, +then requires `oracle_rc`, `mutation_rc`, and +`oracle_output_sha256` to match one mirrored verdict exactly. + +The `/plans` route accepts the acceptor's producer body +`{ planHash, script_bytes_base64, args_bytes_base64, plan }`, hashes the decoded +script-plus-args bytes, and expands the emitted items into the Factory backlog. +The same plan hash is also the stable internal plan id, so posting the identical +producer payload again is append-only and duplicate-free. + +The dominant checks are documented in [docs/factory.md](../../docs/factory.md). diff --git a/pkgs/substrate-apps/src/packages/factory/package.json b/pkgs/substrate-apps/src/packages/factory/package.json new file mode 100644 index 00000000..6c8c2a34 --- /dev/null +++ b/pkgs/substrate-apps/src/packages/factory/package.json @@ -0,0 +1,30 @@ +{ + "name": "@substrate/factory", + "private": true, + "type": "module", + "version": "0.1.0", + "description": "The Factory Durable Object as a service: backlog, WIP cap, release evaluator, capacity routes. Ported from tally-ts-sdk packages/factory.", + "license": "UNLICENSED", + "exports": { + ".": "./src/index.ts", + "./*": "./src/*" + }, + "scripts": { + "check": "tsc --noEmit -p .", + "test": "vitest run" + }, + "dependencies": { + "@substrate/planning": "workspace:*", + "@substrate/schema": "workspace:*", + "@substrate/serializer": "workspace:*" + }, + "peerDependencies": { + "effect": "4.0.0-rc.112" + }, + "devDependencies": { + "effect": "4.0.0-rc.112", + "fast-check": "^4.9.0", + "typescript": "^5.9.3", + "vitest": "^3.2.4" + } +} diff --git a/pkgs/substrate-apps/src/packages/factory/src/cardWriter.ts b/pkgs/substrate-apps/src/packages/factory/src/cardWriter.ts new file mode 100644 index 00000000..d4d03f42 --- /dev/null +++ b/pkgs/substrate-apps/src/packages/factory/src/cardWriter.ts @@ -0,0 +1,192 @@ +/** + * cardWriter.ts — U-A13 LAKE-PROJECTION, the writer. + * + * §2.2e: "`--write` fills only `actuals`, `result.grade`, `result.observed`, + * `result.receipt_sha256`, `prior_gap`, `outcome_for_calibration` on cards (the + * operator's fields); `status` and `outcome_ruled` never." + * + * LINE-ORIENTED, NOT A YAML EMITTER. A card is a file a human wrote: it carries + * trailing comments (`status: DRAFT # DRAFT -> ARMED -> …`), block + * scalars, and a field order someone chose. Re-emitting its front matter from a + * parsed value would rewrite every one of those and the diff would be a diff of + * this tool's formatting opinions. So the writer finds the ONE line a cell lives + * on and replaces the text after the colon, keeping the trailing comment byte + * for byte; a cell with no line yet is appended to its block. + * + * THE FENCE IS THE VOCABULARY. `applyCells` writes the keys it is handed and has + * no path that constructs a key of its own. `status` and `outcome_ruled` are + * never in `PROJECTION_CELLS`, so no value the tool computes can name them — + * and `fencedLines` lets the caller re-read the result and refuse anyway, + * because a fence that is only an argument about scope is not a fence. + * + * IDEMPOTENT BY CONSTRUCTION. Values render through one function, and a cell + * already carrying its rendered form is replaced by the same bytes. Applying the + * same card twice is a no-op, which is the issue's "Re-applying the same card + * reports zero changes" read at the level of the file. + */ +import { Frontmatter } from "@substrate/schema"; +import { NEVER_WRITTEN } from "./projection.ts"; + +/** One cell value, rendered as the front matter spells it. */ +const renderValue = (value: string | number | boolean | null): string => { + if (value === null) return "null"; + if (typeof value === "number") return String(value); + if (typeof value === "boolean") return value ? "true" : "false"; + return JSON.stringify(value); +}; + +/** + * The trailing `# …` comment of a front-matter line, with the whitespace that + * separated it from the value, or the empty string. + * + * The padding comes along because it is part of what the human wrote: a card + * that reads `status: DRAFT # DRAFT -> ARMED -> …` keeps its column + * when a value beside it changes, and a line with no comment gains no trailing + * space. + */ +const trailingComment = (line: string): string => { + const stripped = Frontmatter.stripComment(line); + const comment = line.slice(stripped.length); + if (comment === "") return ""; + return stripped.slice(stripped.trimEnd().length) + comment; +}; + +const indentOf = (line: string): number => line.length - line.replace(/^ +/, "").length; + +/** The half-open line range of the block opened at `head`, by indentation. */ +const blockRange = ( + lines: ReadonlyArray, + head: number +): { readonly start: number; readonly end: number; readonly indent: number } => { + const indent = indentOf(lines[head] ?? ""); + let end = head + 1; + while (end < lines.length) { + const line = lines[end] ?? ""; + if (line.trim() === "" || line.trimStart().startsWith("#")) { + end += 1; + continue; + } + if (indentOf(line) <= indent) break; + end += 1; + } + // A trailing run of blank or comment lines belongs to whatever comes next, + // not to this block: appending after them would move a comment's subject. + while (end > head + 1) { + const line = lines[end - 1] ?? ""; + if (line.trim() === "" || line.trimStart().startsWith("#")) end -= 1; + else break; + } + return { start: head + 1, end, indent: indent + 2 }; +}; + +/** The index of the line opening `key` at indent 0, or -1. */ +const headOf = (lines: ReadonlyArray, key: string): number => + lines.findIndex((line) => line === `${key}:` || line.startsWith(`${key}: `) || line.startsWith(`${key}:#`)); + +/** Set `key` inside `[start, end)`, or insert it at `end`. Returns the new end. */ +const setInBlock = ( + lines: Array, + start: number, + end: number, + indent: number, + key: string, + value: string | number | boolean | null +): number => { + const prefix = `${" ".repeat(indent)}${key}:`; + for (let i = start; i < end; i++) { + const line = lines[i] ?? ""; + if (!line.startsWith(prefix)) continue; + const rest = line.slice(prefix.length); + if (rest !== "" && !rest.startsWith(" ") && !rest.startsWith("\t") && !rest.startsWith("#")) continue; + lines[i] = `${prefix} ${renderValue(value)}${trailingComment(line)}`; + return end; + } + lines.splice(end, 0, `${" ".repeat(indent)}${key}: ${renderValue(value)}`); + return end + 1; +}; + +/** + * Put the operator's cells on one card's front matter. + * + * @param text - The card, whole. + * @param cells - The cells to set, keyed as `PROJECTION_CELLS` keys them + * (`actuals.tokens`, `result.grade`, `outcome_for_calibration`, …). + * @returns The card with those cells set and nothing else touched. + */ +export const applyCells = ( + text: string, + cells: ReadonlyMap +): string => { + const split = Frontmatter.split(text); + if (split === null) throw new Error("the card opens with no front-matter fence"); + const lines = split.frontmatter.split("\n"); + + const group = (name: string, after: number): number => { + let head = headOf(lines, name); + if (head === -1) { + lines.splice(after, 0, `${name}:`); + head = after; + } + let range = blockRange(lines, head); + for (const [cell, value] of cells) { + if (!cell.startsWith(`${name}.`)) continue; + range = { + ...range, + end: setInBlock(lines, range.start, range.end, range.indent, cell.slice(name.length + 1), value) + }; + } + return range.end; + }; + + const afterActuals = group("actuals", lines.length); + group("result", afterActuals); + + for (const [cell, value] of cells) { + if (cell.includes(".")) continue; + setInBlock(lines, 0, lines.length, 0, cell, value); + } + + return `---\n${lines.join("\n")}\n---\n${split.body}`; +}; + +/** The front-matter lines whose key is one this projection never writes. */ +export const fencedLines = (text: string): ReadonlyArray => { + const split = Frontmatter.split(text); + if (split === null) return []; + return split.frontmatter + .split("\n") + .filter((line) => NEVER_WRITTEN.some((key) => line.startsWith(`${key}:`))); +}; + +/** + * `git diff --stat`'s shape for one file. + * + * Computed here because git is not on the wrapped PATH: `scripts/node-env.sh` + * puts node on it and nothing else, and the evaluator runs the oracle on a bare + * PATH where the only executables are `/usr/bin/env` and `/bin/sh` (U-A2, + * MEASURED). The oracle's parenthetical asks for the reading, not for the + * program that usually prints it. + */ +export const statOf = ( + name: string, + before: string, + after: string +): { readonly name: string; readonly added: number; readonly removed: number } => { + const a = before.split("\n"); + const b = after.split("\n"); + // Longest common subsequence, so an inserted block counts as an insertion and + // not as a rewrite of every line below it. A card is ~120 lines; the quadratic + // table is the honest reading and costs nothing at that size. + const table: Array> = Array.from({ length: a.length + 1 }, () => + new Array(b.length + 1).fill(0) + ); + for (let i = a.length - 1; i >= 0; i--) { + for (let j = b.length - 1; j >= 0; j--) { + table[i]![j] = a[i] === b[j] + ? (table[i + 1]![j + 1] ?? 0) + 1 + : Math.max(table[i + 1]![j] ?? 0, table[i]![j + 1] ?? 0); + } + } + const common = table[0]![0] ?? 0; + return { name, added: b.length - common, removed: a.length - common }; +}; diff --git a/pkgs/substrate-apps/src/packages/factory/src/http.ts b/pkgs/substrate-apps/src/packages/factory/src/http.ts new file mode 100644 index 00000000..40df649d --- /dev/null +++ b/pkgs/substrate-apps/src/packages/factory/src/http.ts @@ -0,0 +1,454 @@ +/** + * The Factory HTTP surface as a library handler. + * + * U-A11 built every route but one and left the Worker entry to U-A14, which + * added `GET /projection` — §2.2b's last unbuilt line — and the store the route + * reads. The handler still instantiates no Worker and names no binding. + */ +import { Effect, Schema } from "effect"; +import { + ArtifactHasher, + type IArtifactHasher +} from "@substrate/planning/objects/hasher.ts"; +import type { IFactory } from "@substrate/planning/objects/factory.ts"; +import { parseAdmitOutcomeReport } from "@substrate/planning/schema/admit.ts"; +import { parseBacklog } from "@substrate/planning/schema/backlog.ts"; +import { parseCapacityReading } from "@substrate/planning/schema/capacity.ts"; +import { + Instant, + parseSeatCapacitySnapshot, + SeatJob +} from "@substrate/planning/schema/seatCapacity.ts"; +import { FactoryError } from "@substrate/planning/schema/errors.ts"; +import { ExecutorId, PlanId } from "@substrate/planning/schema/ids.ts"; +import { parsePlanArm } from "@substrate/planning/schema/plan.ts"; +import { parseVerdict } from "@substrate/planning/schema/records.ts"; +import { Heartbeat } from "@substrate/planning/schema/uplink.ts"; +import type { IPlanningStore } from "@substrate/planning/objects/storage.ts"; +import { serializeValue } from "@substrate/serializer"; +import { NONE } from "./projection.ts"; +import { + mirrorProjectionRows, + projectionBody, + retainedReceiptOf, + retainedReceipts, + retainReceipt +} from "./projectionMirror.ts"; +import { postReceipt } from "./receipt.ts"; + +const parseExecutor = Schema.decodeUnknownEffect(ExecutorId); +const parsePlanId = Schema.decodeUnknownEffect(PlanId); +const parseHeartbeat = Schema.decodeUnknownEffect(Heartbeat); +const parseInstant = Schema.decodeUnknownEffect(Instant); +const parseSeatJob = Schema.decodeUnknownEffect(SeatJob); + +interface FactoryHttpOptions { + readonly factory: IFactory; + readonly token: string; + readonly artifactHasher: IArtifactHasher; + /** + * The same store the object was built over, for `GET /projection` (U-A14). + * + * Optional because the route is the only thing that needs it and a host that + * does not serve the projection should not have to name a store. Without it, + * an accepted receipt's measured cells are not retained and the projection is + * empty — which is an honest answer and not a silent one: an empty projection + * says "this object mirrors no receipt", which is exactly true of a handler + * that was given nowhere to keep one. + */ + readonly store?: IPlanningStore; + /** + * The host's clock, as an RFC 3339 instant. + * + * The object never reads a clock; the host hands one in. It is the default + * `asOf` for `GET /capacity`, the receipt time for pushed seats (a reading + * dated beyond the skew tolerance ahead of it is refused) and the floor under + * every alarm a `POST /capacity` computes. Without it, `POST /capacity/seats` + * and a `POST /capacity` that carries `seats` are refused (501 NoClock) + * rather than accepted against no clock (the publisher's own `published_at` + * is never the receiving clock), and `GET /capacity` requires `asOf`. + */ + readonly now?: () => string; +} + +type FactoryHttpHandler = (request: Request) => Promise; + +const response = (status: number, body: unknown): Response => + new Response(`${JSON.stringify(body)}\n`, { + status, + headers: { "content-type": "application/json; charset=utf-8" } + }); + +/** + * The projection's own response. + * + * Its body is JSON Lines produced by U-A12's serializer and is served verbatim: + * re-encoding it through `response` above would put `JSON.stringify`'s + * insertion order back on the wire and undo the one property the route exists + * to have. + */ +const ndjson = (body: string): Response => + new Response(body, { + status: 200, + headers: { "content-type": "application/x-ndjson; charset=utf-8" } + }); + +const asRecord = (value: unknown): Record => { + if (value === null || typeof value !== "object" || Array.isArray(value)) { + throw new TypeError("request body must be a JSON object"); + } + return value as Record; +}; + +const decodeBase64 = (value: unknown, field: string): Uint8Array => { + if (typeof value !== "string") throw new TypeError(`${field} must be base64 text`); + try { + const binary = atob(value); + return Uint8Array.from(binary, (character) => character.charCodeAt(0)); + } catch { + throw new TypeError(`${field} must be valid base64`); + } +}; + +const concatenate = (...parts: ReadonlyArray): Uint8Array => { + const joined = new Uint8Array(parts.reduce((length, part) => length + part.length, 0)); + let offset = 0; + for (const part of parts) { + joined.set(part, offset); + offset += part.length; + } + return joined; +}; + +const statusFor = (error: unknown): number => { + if (error instanceof FactoryError) { + if (error.reason === "ReceiptMismatch" || error.reason === "ContinuityGap") return 409; + if (error.reason === "InvalidTransition" || error.reason === "DedupMismatch") return 409; + if (error.reason === "ReceiptInvalid") return 400; + if (error.reason === "UnknownTask" || error.reason === "PlanNotArmed") return 404; + } + return 400; +}; + +const errorBody = (error: unknown): Record => { + if (error instanceof FactoryError) { + return { error: error.reason, operation: error.operation, subject: error.subject }; + } + if ( + error !== null && + typeof error === "object" && + "_tag" in error && + typeof (error as { readonly _tag?: unknown })._tag === "string" + ) { + return { error: String((error as { readonly _tag: string })._tag) }; + } + return { error: error instanceof Error ? error.message : "invalid request" }; +}; + +/** + * Builds the route handler without creating a Worker entry or selecting a + * persistence binding. Every state-changing request checks the supplied bearer + * token, including the proposal pull that releases backlog items. + */ +export const makeFactoryHttpHandler = (options: FactoryHttpOptions): FactoryHttpHandler => { + const run = (effect: Effect.Effect): Promise => + Effect.runPromise( + effect.pipe( + Effect.provideService( + ArtifactHasher, + ArtifactHasher.of(options.artifactHasher) + ) + ) + ); + + /** `?asOf=` when given (validated), else the host's clock, else nothing. */ + const asOfFrom = async (url: URL): Promise => { + const raw = url.searchParams.get("asOf"); + if (raw !== null) return Effect.runPromise(parseInstant(raw)); + return options.now?.(); + }; + + return async (request: Request): Promise => { + const url = new URL(request.url); + const changesState = + (request.method !== "GET" && request.method !== "HEAD") || + (request.method === "GET" && url.pathname === "/proposals"); + if ( + changesState && + request.headers.get("authorization") !== `Bearer ${options.token}` + ) { + // Drain before refusing. The refusal itself needs nothing from the body, + // but a Worker that answers while the request stream is still open makes + // workerd log `Can't read from request stream after response has been + // sent` — an uncaught TypeError in the dev log of an otherwise green + // smoke, which is exactly the kind of noise a reader learns to ignore. + await request.arrayBuffer().catch(() => undefined); + return response(401, { error: "Unauthorized" }); + } + + try { + if (request.method === "GET" && url.pathname === "/projection") { + if (options.store === undefined) return ndjson(""); + const retained = await Effect.runPromise(retainedReceipts(options.store)); + const state = await run(options.factory.stateView); + return ndjson(projectionBody(mirrorProjectionRows(retained, state.receipts))); + } + + if (request.method === "GET" && url.pathname === "/state") { + return response(200, await run(options.factory.stateView)); + } + + if (request.method === "GET" && url.pathname === "/proposals") { + const raw = url.searchParams.get("executor"); + if (raw === null) return response(400, { error: "executor is required" }); + const executor = await Effect.runPromise(parseExecutor(raw)); + const proposals = await run(options.factory.handOut(executor)); + const state = await run(options.factory.stateView); + return response(200, { + proposals, + next_wake_at: state.alarm_at + }); + } + + if (request.method === "POST" && url.pathname === "/capacity") { + const body = await request.json(); + const record = asRecord(body); + const { seats, ...reading } = await Effect.runPromise( + parseCapacityReading(record["reading"] ?? body) + ); + const asOf = options.now?.(); + // Seats need a receiving clock, the same as on POST /capacity/seats. + // Refused whole, before anything is folded in, so the two doors agree. + if (seats !== undefined && asOf === undefined) { + return response(501, { error: "NoClock", subject: "/capacity" }); + } + // The seats are the ledger's: folded in on their own monotonic rule, + // whatever becomes of the reading's seq. + const seatReport = + seats === undefined || asOf === undefined + ? undefined + : await run(options.factory.observeSeats(seats, asOf)); + const accepted = await run(options.factory.observeCapacity(reading, asOf)); + return response(200, seatReport === undefined ? { accepted } : { accepted, seats: seatReport }); + } + + if (request.method === "POST" && url.pathname === "/capacity/seats") { + const receivedAt = options.now?.(); + if (receivedAt === undefined) { + await request.arrayBuffer().catch(() => undefined); + return response(501, { error: "NoClock", subject: "/capacity/seats" }); + } + const body = asRecord(await request.json()); + const snapshot = await Effect.runPromise( + parseSeatCapacitySnapshot(body["snapshot"] ?? body) + ); + return response(200, await run(options.factory.observeSeats(snapshot, receivedAt))); + } + + if (request.method === "GET" && url.pathname === "/capacity") { + const asOf = await asOfFrom(url); + if (asOf === undefined) return response(400, { error: "asOf is required" }); + return response(200, await run(options.factory.capacityAt(asOf))); + } + + if (request.method === "GET" && url.pathname === "/capacity/admit") { + const seat = url.searchParams.get("seat"); + if (seat === null) return response(400, { error: "seat is required" }); + const asOf = await asOfFrom(url); + if (asOf === undefined) return response(400, { error: "asOf is required" }); + const minimum = url.searchParams.get("min_headroom_pct"); + const job = await Effect.runPromise( + parseSeatJob({ + model: url.searchParams.get("model"), + min_headroom_pct: minimum === null ? 0 : Number(minimum) + }) + ); + return response(200, await run(options.factory.admitSeat(seat, job, asOf))); + } + + if (request.method === "POST" && url.pathname === "/outcomes") { + const report = await Effect.runPromise( + parseAdmitOutcomeReport(await request.json()) + ); + return response(200, await run(options.factory.observeOutcome(report))); + } + + if (request.method === "POST" && url.pathname === "/verdicts") { + const verdict = await Effect.runPromise(parseVerdict(await request.json())); + return response(200, await run(options.factory.observeVerdict(verdict))); + } + + if (request.method === "POST" && url.pathname === "/receipts") { + const body = asRecord(await request.json()); + const verdictHash = body["verdict_hash"]; + if (typeof verdictHash !== "string") { + return response(400, { error: "verdict_hash is required" }); + } + let receipt: unknown = body["receipt"]; + if (receipt === undefined) { + const inline = { ...body }; + delete inline["verdict_hash"]; + receipt = inline; + } + const observation = await run(postReceipt(options.factory, receipt, verdictHash)); + + // Accepted. The three evidence cells are now in the mirror; the + // measured cells (`seconds`, `tokens`, `disposition`) live only in + // these bytes, so they are retained beside the object's state for + // `GET /projection`. The digest is taken over the receipt's CANONICAL + // bytes — U-A12's serializer — so the same receipt yields the same + // `receipt_sha256` whichever order its keys arrived in. + if (options.store !== undefined) { + const path = body["receipt_path"]; + const canonical = serializeValue(receipt); + const digest = await Effect.runPromise( + options.artifactHasher.hash( + new TextEncoder().encode(canonical), + "receipt" + ) + ); + const retained = retainedReceiptOf( + receipt, + typeof path === "string" ? path : NONE, + digest + ); + if (retained.ok) { + await Effect.runPromise(retainReceipt(options.store, retained.retained)); + } + } + return response(200, observation); + } + + if (request.method === "POST" && url.pathname === "/heartbeats") { + const body = asRecord(await request.json()); + const heartbeat = await Effect.runPromise(parseHeartbeat(body["heartbeat"] ?? body)); + const lane = typeof body["lane"] === "string" ? body["lane"] : undefined; + const observedAt = + typeof body["observed_at"] === "string" ? body["observed_at"] : undefined; + await run(options.factory.observeHeartbeat(heartbeat, lane, observedAt)); + return response(200, { accepted: true }); + } + + if (request.method === "POST" && url.pathname === "/plans") { + const body = asRecord(await request.json()); + + // The acceptor owns this producer shape. Its planHash is over the + // concatenated script and args bytes, and its plan items are enriched + // here with the release-state fields that are Factory concerns. + if (body["plan"] !== undefined) { + const plan = asRecord(body["plan"]); + const meta = plan["meta"] === undefined ? {} : asRecord(plan["meta"]); + const arm = await Effect.runPromise( + parsePlanArm({ + label: + typeof meta["name"] === "string" && meta["name"] !== "none" + ? meta["name"] + : "conwip plan arm", + kind: "workflow", + level: plan["level"], + namespace: plan["namespace"], + planHash: body["planHash"], + author: + typeof plan["acceptor"] === "string" + ? plan["acceptor"] + : "conwip plan arm" + }) + ); + const planId = await Effect.runPromise(parsePlanId(arm.planHash)); + if (plan["planHash"] !== arm.planHash) { + throw new FactoryError({ + reason: "PlanHashMismatch", + operation: "FactoryHttp.POST /plans", + subject: planId + }); + } + + const rawItems = plan["items"]; + if (!Array.isArray(rawItems)) throw new TypeError("plan.items must be an array"); + const current = await run(options.factory.releaseState); + const itemRecords = rawItems.map((value) => asRecord(value)); + const decodedItems = await Effect.runPromise( + parseBacklog( + itemRecords.map((item) => { + if (item["planHash"] !== arm.planHash) { + throw new FactoryError({ + reason: "PlanHashMismatch", + operation: "FactoryHttp.POST /plans", + subject: planId + }); + } + const namespace = item["namespace"] ?? arm.namespace; + const namespaceValue = current.namespaces.namespaces.find( + (entry) => entry.name === namespace + )?.value; + const mutationHint = item["mutation_hint"]; + const argvRef = item["argv_ref"]; + return { + taskId: item["taskId"], + planId, + namespace, + family: item["family"] ?? "build", + level: item["level"] ?? arm.level, + rank: item["rank"], + needs: item["needs"], + dependsOn: item["dependsOn"] ?? [], + evidence: item["evidence"] ?? [], + briefHash: item["briefHash"], + planHash: item["planHash"], + dedupKey: item["dedupKey"], + state: item["state"], + deferrals: item["deferrals"] ?? 0, + attempt: item["attempt"] ?? 1, + outcome: item["outcome"] ?? null, + value: item["value"] ?? namespaceValue ?? 1, + estimate: item["estimate"] ?? current.estimates.mainEffect, + gate: item["gate"], + subassembly: item["subassembly"] ?? null, + dueBy: item["dueBy"] ?? null, + float: item["float"] ?? null, + envelopeParent: item["envelopeParent"] ?? null, + ...(typeof mutationHint === "string" && mutationHint !== "none" + ? { mutation_hint: mutationHint } + : {}), + ...(typeof argvRef === "string" ? { argv_ref: argvRef } : {}) + }; + }) + ) + ); + const bytes = concatenate( + decodeBase64(body["script_bytes_base64"], "script_bytes_base64"), + decodeBase64(body["args_bytes_base64"], "args_bytes_base64") + ); + const row = await run( + options.factory.armPlan(arm, { planId, bytes, items: decodedItems }) + ); + return response(200, row); + } + + const artifactBody = asRecord(body["artifact"]); + const arm = await Effect.runPromise(parsePlanArm(body["arm"])); + const planId = await Effect.runPromise(parsePlanId(artifactBody["planId"])); + const items = await Effect.runPromise(parseBacklog(artifactBody["items"])); + const bytes = artifactBody["bytes"]; + if ( + !Array.isArray(bytes) || + !bytes.every((entry) => Number.isInteger(entry) && entry >= 0 && entry <= 255) + ) { + return response(400, { error: "artifact.bytes must be byte values" }); + } + const row = await run( + options.factory.armPlan(arm, { + planId, + bytes: Uint8Array.from(bytes as ReadonlyArray), + items + }) + ); + return response(200, row); + } + + return response(404, { error: "NotFound" }); + } catch (error) { + return response(statusFor(error), errorBody(error)); + } + }; +}; diff --git a/pkgs/substrate-apps/src/packages/factory/src/index.ts b/pkgs/substrate-apps/src/packages/factory/src/index.ts new file mode 100644 index 00000000..2ae8d0a6 --- /dev/null +++ b/pkgs/substrate-apps/src/packages/factory/src/index.ts @@ -0,0 +1,11 @@ +/** Public surface of the host-neutral Factory package. */ +export * from "@substrate/planning/objects/factory.ts"; +export * from "@substrate/planning/objects/storage.ts"; +export * from "@substrate/planning/objects/hasher.ts"; +export * from "./inputs.ts"; +export * from "./receipt.ts"; +export * from "./http.ts"; +export * from "./projection.ts"; +export * from "./cardWriter.ts"; +export * from "./projectionLoad.ts"; +export * from "./projectionMirror.ts"; diff --git a/pkgs/substrate-apps/src/packages/factory/src/inputs.ts b/pkgs/substrate-apps/src/packages/factory/src/inputs.ts new file mode 100644 index 00000000..7015cdbd --- /dev/null +++ b/pkgs/substrate-apps/src/packages/factory/src/inputs.ts @@ -0,0 +1,130 @@ +/** + * Host-neutral input Layers used to compose the Factory. + * + * Capacity arrives two ways and both cross `FactoryCapacityFeed`: authored + * readings and seat snapshots (`fakeCapacityLayer`, the hermetic bench) and the + * seats the capacity ledger holds in the object's own store + * (`ledgerCapacityFeedLayer`, backed by Durable Object storage in the Worker and + * by the in-memory store in tests). + */ +import { Context, Effect, Layer } from "effect"; +import { + CapacityLedger, + capacityLedgerLayer +} from "@substrate/planning/capacity/ledger.ts"; +import { PlanningStore } from "@substrate/planning/objects/storage.ts"; +import type { CapacityReading } from "@substrate/planning/schema/capacity.ts"; +import { + SEAT_CAPACITY_SCHEMA_VERSION, + type SeatCapacitySnapshot +} from "@substrate/planning/schema/seatCapacity.ts"; +import type { PriceVector } from "@substrate/planning/schema/prices.ts"; +import { Factory } from "@substrate/planning/objects/factory.ts"; +import { FactoryError } from "@substrate/planning/schema/errors.ts"; + +interface IFactoryCapacityFeed { + readonly readings: Effect.Effect>; + /** Per-seat capacity snapshots (`seat-capacity/2`). */ + readonly seats: Effect.Effect>; +} + +/** A capacity source; production and test sources cross the same interface. */ +export class FactoryCapacityFeed extends Context.Service< + FactoryCapacityFeed, + IFactoryCapacityFeed +>()("@substrate/factory/FactoryCapacityFeed") {} + +interface IFactoryPriceSource { + readonly prices: PriceVector; +} + +/** The immutable price vector used for every evaluation in one composition. */ +export class FactoryPriceSource extends Context.Service< + FactoryPriceSource, + IFactoryPriceSource +>()("@substrate/factory/FactoryPriceSource") {} + +/** Provides authored fake readings and seat snapshots without a live capacity plane. */ +export const fakeCapacityLayer = ( + readings: ReadonlyArray, + seats: ReadonlyArray = [] +): Layer.Layer => + Layer.succeed(FactoryCapacityFeed)( + FactoryCapacityFeed.of({ readings: Effect.succeed(readings), seats: Effect.succeed(seats) }) + ); + +/** + * The real capacity feed: the seats the capacity ledger holds, one snapshot per + * publishing host, read from whichever `PlanningStore` is provided. + * + * Readings are not replayed from here: the kernel's readings are the Factory's + * own stored state and arrive by `POST /capacity`. A store that cannot be read + * yields no seats, which the admission predicate answers `unknown-seat`, so a + * storage failure can only ever refuse work, never admit it. + */ +const ledgerCapacityFeedLayer: Layer.Layer = + Layer.effect( + FactoryCapacityFeed, + Effect.gen(function* () { + const ledger = yield* CapacityLedger; + const seats = ledger.latest.pipe( + Effect.map((held) => { + const byHost = new Map>(); + for (const entry of held) byHost.set(entry.host, [...(byHost.get(entry.host) ?? []), entry]); + return [...byHost].map( + ([host, entries]): SeatCapacitySnapshot => ({ + schema_version: SEAT_CAPACITY_SCHEMA_VERSION, + host, + published_at: entries + .map((entry) => entry.published_at) + .reduce((latest, next) => (Date.parse(next) > Date.parse(latest) ? next : latest)), + seats: entries.map((entry) => entry.seat) + }) + ); + }), + Effect.orElseSucceed((): ReadonlyArray => []) + ); + return FactoryCapacityFeed.of({ readings: Effect.succeed([]), seats }); + }) + ); + +/** The real capacity feed over the provided store, ledger included. */ +export const storedCapacityLayer: Layer.Layer = + ledgerCapacityFeedLayer.pipe(Layer.provide(capacityLedgerLayer)); + +/** Provides one constant, authored price vector. */ +export const constantPriceVectorLayer = ( + prices: PriceVector +): Layer.Layer => + Layer.succeed(FactoryPriceSource)(FactoryPriceSource.of({ prices })); + +/** + * Wires the service to its constant price vector and fake/read-only capacity + * source. The production Worker supplies different Layers; the state machine is + * unchanged. + * + * @param receivedAt the receiving clock for seat snapshots. Required when the + * feed carries any: a snapshot's own `published_at` is never the receipt + * time, so without a clock seats are refused `NoClock`. Re-feeding what the + * ledger already holds is a no-op (idempotent). + */ +export const loadFactoryInputs = (receivedAt?: string) => + Effect.gen(function* () { + const factory = yield* Factory; + const capacity = yield* FactoryCapacityFeed; + const price = yield* FactoryPriceSource; + const current = yield* factory.releaseState; + yield* factory.restore({ ...current, prices: price.prices }); + const readings = yield* capacity.readings; + for (const reading of readings) yield* factory.observeCapacity(reading, receivedAt); + const seats = yield* capacity.seats; + if (seats.length > 0) { + if (receivedAt === undefined) { + return yield* Effect.fail( + new FactoryError({ reason: "NoClock", operation: "loadFactoryInputs", subject: "capacity/seats" }) + ); + } + for (const snapshot of seats) yield* factory.observeSeats(snapshot, receivedAt); + } + return yield* factory.stateView; + }); diff --git a/pkgs/substrate-apps/src/packages/factory/src/projection.ts b/pkgs/substrate-apps/src/packages/factory/src/projection.ts new file mode 100644 index 00000000..435b046e --- /dev/null +++ b/pkgs/substrate-apps/src/packages/factory/src/projection.ts @@ -0,0 +1,410 @@ +/** + * projection.ts — U-A13 LAKE-PROJECTION, the part that touches nothing. + * + * §2.2e names one tool, `tools/ladder-projection.mjs`, and one thing it fills: + * "`--write` fills only `actuals`, `result.grade`, `result.observed`, + * `result.receipt_sha256`, `prior_gap`, `outcome_for_calibration` on cards (the + * operator's fields); `status` and `outcome_ruled` never." + * + * This module is the operator's fields as a value. It reads no path, opens no + * socket and writes no byte: the tool reads the register and hands what it read + * in here as data, and the tool writes back what comes out. Everything that can + * be decided without the disk is decided here so the suite can decide it too. + * + * THE THREE THINGS IT KNOWS + * + * 1. `bankedRowOf` — what one banked receipt says, normalised. The banked + * corpus is heterogeneous: MEASURED 2026-09-06 over + * `receipts/FACTORY-2026-09-06/*​/receipt.json`, 52 files carry `unit`, + * `disposition`, `verdict`, `oracle_rc`, `tokens{6 cells}` and `seconds`; + * 27 also carry `outcome_for_calibration`, `prior_gap` and `mutation_rc`; + * 47 carry `oracle_output_sha256`; and 0 of 52 decode under the strict + * §2.3 `Receipt` (U-A16) — they are the factory run's own receipts, written + * before that type existed. A projection that refused them would project + * nothing, so this reader takes the cells that are there, names the ones + * that are not, and never invents one. + * + * 2. `projectionRowOf` — the operator's fields for one unit, built from the + * banked cells AND from what the Factory object handed back. The evidence + * cells (`result.observed`, `result.grade`) come from the mirror, not from + * the file: a row exists only for a unit the object accepted, which is what + * makes `--negative-control drop-one-receipt` visible at all. + * + * 3. `diffProjections` — one line per disagreement, in a fixed order. Empty + * output is the green; every line names the unit and the cell. + * + * WHAT IS NOT HERE. `status` and `outcome_ruled` are absent from + * `PROJECTION_CELLS`, from `cardCellsOf` and from every writer path — not + * guarded against, absent. A field the writer has no name for cannot be written + * by a mistake in the writer. + */ + +/** The one release cycle this projection covers. */ +export const RELEASE_CYCLE = "FACTORY-2026-09-06"; + +/** + * The card cells `--write` may fill, spelled as §2.2e spells them. + * + * `status` and `outcome_ruled` are not here and never are. The tool's writer + * takes its whole vocabulary from this list. + */ +export const PROJECTION_CELLS: ReadonlyArray = [ + "actuals.tokens", + "actuals.seconds", + "actuals.outcome", + "actuals.receipt_sha256", + "actuals.prior_gap", + "result.grade", + "result.observed", + "result.receipt_sha256", + "outcome_for_calibration" +]; + +/** The cells no projection may ever write, named so the suite can assert it. */ +export const NEVER_WRITTEN: ReadonlyArray = ["status", "outcome_ruled"]; + +/** The string D-B22 gives an inapplicable field: never a null, never a guess. */ +export const NONE = "none"; + +/** One banked receipt, normalised to the cells the projection uses. */ +export interface BankedRow { + /** The card id the receipt is banked under — the directory name. */ + readonly unit: string; + /** The receipt path, relative to the register root. */ + readonly receipt_path: string; + /** sha256 over the receipt's bytes, computed by the caller. */ + readonly receipt_sha256: string; + /** `oracle_rc`, the gate metric of every card in this cycle. */ + readonly oracle_rc: number; + /** `oracle_output_sha256`, or `"none"` where the receipt carries none. */ + readonly oracle_output_sha256: string; + /** `mutation_rc` where the receipt carries one. */ + readonly mutation_rc: number | null; + /** `verdict`, normalised to the mirror's vocabulary. */ + readonly outcome: "pass" | "fail"; + /** `disposition`, verbatim. */ + readonly disposition: string; + /** `seconds`, measured. */ + readonly seconds: number; + /** `tokens.total`, the cell the ladder's `actuals.tokens` carries. */ + readonly tokens: number; + /** `tokens.out`, the declared cell (D-B19). */ + readonly tokens_out: number; + /** `outcome_for_calibration` where the receipt carries one. */ + readonly outcome_for_calibration: string | null; + /** The declared-cell prior gap, where it can be read or derived. */ + readonly prior_gap: number | null; + /** Which cells the receipt did not carry. Reported, never filled. */ + readonly absent: ReadonlyArray; +} + +/** The operator's fields for one unit, as one canonical record. */ +export interface ProjectionRow { + readonly actuals: { + readonly outcome: string; + readonly prior_gap: number | null; + readonly receipt_sha256: string; + readonly seconds: number; + readonly tokens: number; + }; + readonly outcome_for_calibration: string | null; + readonly receipt_path: string; + readonly result: { + readonly grade: "MEASURED" | "CLAIMED"; + readonly observed: number; + readonly receipt_sha256: string; + }; + readonly unit: string; +} + +const isRecord = (value: unknown): value is Record => + value !== null && typeof value === "object" && !Array.isArray(value); + +const numberAt = (source: Record, key: string): number | null => { + const value = source[key]; + return typeof value === "number" && Number.isFinite(value) ? value : null; +}; + +const stringAt = (source: Record, key: string): string | null => { + const value = source[key]; + return typeof value === "string" && value !== "" ? value : null; +}; + +/** + * The declared-cell prior gap. + * + * The corpus MEASURED two spellings of `prior_gap`: the strict §2.3 block + * `{tokens_by_cell{out, in_uncached+out, total}, ...}` (1 receipt) and the older + * flat `{tokens_out, p_pass_vs_outcome}` (26). Both name the same cell, `out`, + * which is the cell every prior in this cycle declares (D-B19), so both read as + * one number. A receipt carrying neither yields `null` and is named in `absent` + * — the gap is not derived from a prior the receipt does not carry, because a + * number computed against an unknown prediction is not a measurement. + */ +export const priorGapOf = (value: unknown): number | null => { + if (!isRecord(value)) return null; + const flat = numberAt(value, "tokens_out"); + if (flat !== null) return flat; + const byCell = value["tokens_by_cell"]; + if (isRecord(byCell)) { + const out = numberAt(byCell, "out"); + if (out !== null) return out; + } + return null; +}; + +/** The verdict vocabulary of the banked corpus, mapped onto the mirror's. */ +const outcomeOf = (verdict: unknown): "pass" | "fail" => + typeof verdict === "string" && verdict.trim().toUpperCase() === "PASS" ? "pass" : "fail"; + +/** + * Normalise one banked receipt. + * + * @param unit - The card id the receipt is banked under. + * @param receiptPath - The path, relative to the register root. + * @param receiptSha256 - sha256 over the file's bytes. + * @param parsed - The receipt's JSON, already parsed. + * @returns The row, or the reason it is not one. A receipt missing a cell the + * projection cannot do without (`oracle_rc`, `seconds`, `tokens.total`, + * `disposition`, `verdict`) is refused by name rather than filled with a zero. + */ +export const bankedRowOf = ( + unit: string, + receiptPath: string, + receiptSha256: string, + parsed: unknown +): { readonly ok: true; readonly row: BankedRow } | { readonly ok: false; readonly why: string } => { + if (!isRecord(parsed)) return { ok: false, why: "the receipt is not a JSON object" }; + const missing: Array = []; + const absent: Array = []; + + const oracleRc = numberAt(parsed, "oracle_rc"); + if (oracleRc === null) missing.push("oracle_rc"); + const seconds = numberAt(parsed, "seconds"); + if (seconds === null) missing.push("seconds"); + const disposition = stringAt(parsed, "disposition"); + if (disposition === null) missing.push("disposition"); + const verdict = stringAt(parsed, "verdict"); + if (verdict === null) missing.push("verdict"); + + const tokensBlock = parsed["tokens"]; + const tokens = isRecord(tokensBlock) ? numberAt(tokensBlock, "total") : null; + const tokensOut = isRecord(tokensBlock) ? numberAt(tokensBlock, "out") : null; + if (tokens === null) missing.push("tokens.total"); + if (tokensOut === null) missing.push("tokens.out"); + + if (missing.length > 0) { + return { ok: false, why: `carries no ${missing.join(", ")}` }; + } + + const outputSha = stringAt(parsed, "oracle_output_sha256"); + if (outputSha === null) absent.push("oracle_output_sha256"); + const mutationRc = numberAt(parsed, "mutation_rc"); + if (mutationRc === null) absent.push("mutation_rc"); + const forCalibration = stringAt(parsed, "outcome_for_calibration"); + if (forCalibration === null) absent.push("outcome_for_calibration"); + const gap = priorGapOf(parsed["prior_gap"]); + if (gap === null) absent.push("prior_gap"); + + return { + ok: true, + row: { + unit, + receipt_path: receiptPath, + receipt_sha256: receiptSha256, + oracle_rc: oracleRc as number, + oracle_output_sha256: outputSha ?? NONE, + mutation_rc: mutationRc, + outcome: outcomeOf(verdict), + disposition: disposition as string, + seconds: seconds as number, + tokens: tokens as number, + tokens_out: tokensOut as number, + outcome_for_calibration: forCalibration, + prior_gap: gap, + absent + } + }; +}; + +/** + * The banked cells one projection row is built from. + * + * A `BankedRow` is one of these and more: the extra cells (`oracle_rc`, + * `oracle_output_sha256`, `mutation_rc`, `outcome`) are the evidence cells, and + * the row below never reads them from the file — it reads them from the mirror, + * which is the whole mechanism of `--negative-control drop-one-receipt`. Naming + * the smaller shape lets the Worker serve `GET /projection` from the receipts it + * has itself accepted (U-A14) through this one definition of the operator's + * fields, rather than a second copy of it. + */ +type ProjectionCells = Pick< + BankedRow, + | "unit" + | "receipt_path" + | "receipt_sha256" + | "disposition" + | "seconds" + | "tokens" + | "outcome_for_calibration" + | "prior_gap" +>; + +/** What the Factory object handed back for one unit. */ +export interface MirroredEvidence { + readonly id: string; + readonly oracle_rc: number; + readonly oracle_output_sha256: string; + readonly mutation_rc?: number; + readonly verdict_hash: string; +} + +/** + * The operator's fields for one unit. + * + * `result.observed` and `result.grade` are read off the MIRROR — the evidence + * the object accepted — and not off the file the tool read. That is the whole + * mechanism of the negative control: a receipt dropped from the load is a unit + * the object never accepted, so no row exists for it, so `--diff` names it. + * + * `result.grade` is `MEASURED` when the object holds the receipt's evidence and + * the receipt carries a measured token total; `CLAIMED` otherwise. Nothing here + * decides `KEEP` — that is Tom's, and `outcome_ruled` is never written. + */ +export const projectionRowOf = ( + banked: ProjectionCells, + mirrored: MirroredEvidence +): ProjectionRow => ({ + actuals: { + outcome: banked.disposition, + prior_gap: banked.prior_gap, + receipt_sha256: banked.receipt_sha256, + seconds: banked.seconds, + tokens: banked.tokens + }, + outcome_for_calibration: banked.outcome_for_calibration, + receipt_path: banked.receipt_path, + result: { + grade: banked.tokens > 0 ? "MEASURED" : "CLAIMED", + observed: mirrored.oracle_rc, + receipt_sha256: banked.receipt_sha256 + }, + unit: banked.unit +}); + +/** The cells `--write` sets on one card, keyed by their §2.2e spelling. */ +export const cardCellsOf = (row: ProjectionRow): ReadonlyMap => + new Map([ + ["actuals.tokens", row.actuals.tokens], + ["actuals.seconds", row.actuals.seconds], + ["actuals.outcome", row.actuals.outcome], + ["actuals.receipt_sha256", row.actuals.receipt_sha256], + ["actuals.prior_gap", row.actuals.prior_gap], + ["result.grade", row.result.grade], + ["result.observed", row.result.observed], + ["result.receipt_sha256", row.result.receipt_sha256], + ["outcome_for_calibration", row.outcome_for_calibration] + ]); + +/** One card as the diff needs to see it: its id, whether it maps, its cells. */ +export interface CardView { + readonly id: string; + /** `repo:` on the card, or null. §2.2e's `unmapped`. */ + readonly repo: string | null; + /** The card's current value for each cell of `PROJECTION_CELLS`. */ + readonly cells: ReadonlyMap; +} + +const cellText = (value: string | number | boolean | null): string => + value === null ? "null" : String(value); + +/** + * Whether a card cell is unwritten. + * + * A card minted by U-E8 carries `actuals: {tokens: null, ...}` and no `result:` + * block at all: the operator's fields exist and are empty, which is what "not + * yet run" looks like. An unwritten cell is not a disagreement — it is what + * `--write` is for — so the diff is silent about it and loud about a cell that + * carries a DIFFERENT value. + */ +const unwritten = (value: string | number | boolean | null | undefined): boolean => + value === undefined || value === null || value === "" || value === NONE; + +/** + * Every disagreement between the mirror and the register, one line each. + * + * The line vocabulary, in the order the lines are emitted: + * + * `missing ` the register banks a receipt the mirror has no + * row for — the object never accepted it. This is + * what `drop-one-receipt` produces. + * `extra ` the mirror holds a row the register does not + * bank. The lake is a mirror and never a ledger; + * a row without a banked receipt is the lake + * having authored something. + * `cell mirror= register=` + * two projections of the same receipt disagree. + * `contradiction card= projection=` + * the card already carries a DIFFERENT value in + * a cell `--write` would fill. `--write` refuses + * to overwrite it; the operator settles it. + * `unmapped ` a card carries no `repo:` (§2.2e: expected to + * name 89 against the main checkout until + * TL-12's merge; 0 against the named worktree). + * + * @param mirror - Rows read back out of the Factory object. + * @param register - Rows computed straight off the register's banked receipts. + * @param cards - The register's ladder cards. + * @returns The lines, sorted within each kind by unit id. Empty is the green. + */ +export const diffProjections = ( + mirror: ReadonlyArray, + register: ReadonlyArray, + cards: ReadonlyArray +): ReadonlyArray => { + const lines: Array = []; + const byUnit = new Map(mirror.map((row) => [row.unit, row])); + const registerByUnit = new Map(register.map((row) => [row.unit, row])); + + for (const row of [...register].sort((a, b) => a.unit.localeCompare(b.unit))) { + if (!byUnit.has(row.unit)) lines.push(`missing ${row.unit} ${row.receipt_path}`); + } + for (const row of [...mirror].sort((a, b) => a.unit.localeCompare(b.unit))) { + if (!registerByUnit.has(row.unit)) lines.push(`extra ${row.unit}`); + } + for (const row of [...mirror].sort((a, b) => a.unit.localeCompare(b.unit))) { + const other = registerByUnit.get(row.unit); + if (other === undefined) continue; + const here = cardCellsOf(row); + const there = cardCellsOf(other); + for (const cell of PROJECTION_CELLS) { + const a = here.get(cell) ?? null; + const b = there.get(cell) ?? null; + if (cellText(a) !== cellText(b)) { + lines.push(`cell ${row.unit} ${cell} mirror=${cellText(a)} register=${cellText(b)}`); + } + } + } + + const cardById = new Map(cards.map((card) => [card.id, card])); + for (const row of [...mirror].sort((a, b) => a.unit.localeCompare(b.unit))) { + const card = cardById.get(row.unit); + if (card === undefined) continue; + for (const [cell, projected] of cardCellsOf(row)) { + const current = card.cells.get(cell); + if (unwritten(current)) continue; + if (projected === null) continue; + if (cellText(current ?? null) !== cellText(projected)) { + lines.push( + `contradiction ${row.unit} ${cell} card=${cellText(current ?? null)} projection=${cellText(projected)}` + ); + } + } + } + + for (const card of [...cards].sort((a, b) => a.id.localeCompare(b.id))) { + if (card.repo === null || card.repo === "") lines.push(`unmapped ${card.id}`); + } + + return lines; +}; diff --git a/pkgs/substrate-apps/src/packages/factory/src/projectionLoad.ts b/pkgs/substrate-apps/src/packages/factory/src/projectionLoad.ts new file mode 100644 index 00000000..e5e75953 --- /dev/null +++ b/pkgs/substrate-apps/src/packages/factory/src/projectionLoad.ts @@ -0,0 +1,186 @@ +/** + * projectionLoad.ts — U-A13 LAKE-PROJECTION, the load. + * + * `ACTION-PLAN.md` U-A13, verbatim: "loads the banked corpus into the Factory + * object under the in-memory storage Layer and projects the mirror back". This + * is that sentence, and it is the reason the projection is not a `cat` of the + * register into a different file. + * + * THE CORPUS GOES THROUGH THE STATE MACHINE, NOT AROUND IT. Every banked + * receipt is walked through the same four states an item walks in production — + * `unclaimed → released → inflight → closed` — by the same calls the uplink + * makes: + * + * observeCapacity(reading) a reading is what makes anything admissible + * handOut(executor) the release evaluator picks; this module does not + * observeOutcome(Accepted) the kernel's answer, which is what makes it inflight + * observeVerdict(verdict) the mirrored witness record, chained + * observeReceipt(evidence) §2.2b's exact chain match + * + * The object refuses a receipt whose three evidence cells do not equal a + * mirrored verdict's (`Factory.observeReceipt`, reason `ReceiptMismatch`) and + * refuses a verdict that breaks its executor's chain (`ContinuityGap`). So a + * unit reaches the mirror only if its evidence survived both, and the mirror is + * the thing the projection reads. A receipt dropped from the load is a unit the + * object never accepted, no row is projected for it, and `--diff` names it: + * that is the whole of `mutation_hint`, "drop one banked receipt → --diff + * non-empty", implemented rather than asserted. + * + * THIS MODULE AUTHORS NOTHING. It constructs no verdict of its own: the outcome, + * the rc, the mutation rc and the output digest all come from the banked bytes, + * and the hash chain is supplied by the caller. The lake is a mirror and never a + * ledger; what it mirrors here is a corpus that was banked by someone else. + * + * IN-MEMORY ONLY. The caller provides the object; the tool provides it under + * `planningStoreInMemoryLayer`. Nothing is persisted anywhere. + */ +import { Effect, Schema } from "effect"; +import type { CapacityReading } from "@substrate/planning/schema/capacity.ts"; +import type { ExecutorId } from "@substrate/planning/schema/ids.ts"; +import { Verdict } from "@substrate/planning/schema/records.ts"; +import { AdmitOutcomeReport } from "@substrate/planning/schema/admit.ts"; +import type { FactoryError } from "@substrate/planning/schema/errors.ts"; +import type { IFactory } from "@substrate/planning/objects/factory.ts"; +import { + projectionRowOf, + type BankedRow, + type MirroredEvidence, + type ProjectionRow +} from "./projection.ts"; + +const decodeVerdict = Schema.decodeUnknownSync(Verdict); +const decodeOutcome = Schema.decodeUnknownSync(AdmitOutcomeReport); + +/** One link of the witness chain the caller supplies. */ +interface ChainLink { + readonly seq: number; + readonly hash: string; + readonly prevHash: string; +} + +/** What one load produced. */ +interface LoadResult { + /** One row per unit the object accepted, sorted by unit id. */ + readonly rows: ReadonlyArray; + /** Units armed but never proposed before the loop stopped making progress. */ + readonly unreleased: ReadonlyArray; + /** Units the object refused, with the reason it gave. */ + readonly refused: ReadonlyArray<{ readonly unit: string; readonly reason: string }>; + /** Passes of the release loop the corpus needed. */ + readonly passes: number; +} + +/** + * Walk the banked corpus through one Factory object and read the mirror back. + * + * @param factory - The object, already built over the in-memory store. + * @param rows - The banked receipts, normalised. Order is the arming order. + * @param link - The witness chain link for the nth verdict, supplied by the + * caller so this module computes no hash and therefore mints no proof. + * @param readingFor - A capacity reading for pass `n`. One per pass, because a + * reading is a complete fixture and re-observing the same `seq` is a no-op. + * @param maxPasses - A bound on the release loop, so a corpus the evaluator will + * not drain stops rather than spins. + * @returns The mirror's rows plus everything that did not reach it, named. + */ +export const loadBankedIntoFactory = ( + factory: IFactory, + rows: ReadonlyArray, + link: (index: number) => ChainLink, + readingFor: (pass: number) => CapacityReading, + maxPasses = 512 +): Effect.Effect => + Effect.gen(function* () { + const pending = new Map(rows.map((row) => [row.unit, row])); + const witnessed = new Set(); + const refused: Array<{ unit: string; reason: string }> = []; + let written = 0; + let pass = 0; + + while (witnessed.size + refused.length < rows.length && pass < maxPasses) { + const reading = readingFor(pass); + pass += 1; + yield* factory.observeCapacity(reading); + const proposals = yield* factory.handOut(reading.executor as ExecutorId); + const fresh = proposals.filter( + (proposal) => pending.has(proposal.taskId) && !witnessed.has(proposal.taskId) + ); + if (fresh.length === 0) break; + + for (const proposal of fresh) { + const banked = pending.get(proposal.taskId); + if (banked === undefined) continue; + yield* factory.observeOutcome( + decodeOutcome({ + taskId: proposal.taskId, + dedupKey: proposal.dedupKey, + outcome: "Accepted", + row: proposal.rows[0]?.row ?? "build" + }) + ); + const chain = link(written); + written += 1; + const record = decodeVerdict({ + _tag: "Verdict", + taskId: banked.unit, + executor: reading.executor, + seq: chain.seq, + hash: chain.hash, + prevHash: chain.prevHash, + outcome: banked.outcome, + serviceSeconds: banked.seconds, + unit_id: banked.unit, + oracle_rc: banked.oracle_rc, + oracle_output_sha256: banked.oracle_output_sha256, + ...(banked.mutation_rc === null ? {} : { mutation_rc: banked.mutation_rc }) + }); + const observed = yield* factory.observeVerdict(record).pipe( + Effect.map(() => null), + Effect.catch((error: FactoryError) => Effect.succeed(error.reason)) + ); + if (observed !== null) { + refused.push({ unit: banked.unit, reason: observed }); + witnessed.add(banked.unit); + continue; + } + const accepted = yield* factory + .observeReceipt({ + id: banked.unit, + oracle_rc: banked.oracle_rc, + oracle_output_sha256: banked.oracle_output_sha256, + verdict_hash: chain.hash, + ...(banked.mutation_rc === null ? {} : { mutation_rc: banked.mutation_rc }) + }) + .pipe( + Effect.map(() => null), + Effect.catch((error: FactoryError) => Effect.succeed(error.reason)) + ); + if (accepted !== null) refused.push({ unit: banked.unit, reason: accepted }); + witnessed.add(banked.unit); + } + } + + const view = yield* factory.stateView; + const evidence = new Map(); + for (const receipt of view.receipts) evidence.set(receipt.id, receipt); + + const projected: Array = []; + for (const row of rows) { + const mirrored = evidence.get(row.unit); + if (mirrored === undefined) continue; + projected.push(projectionRowOf(row, mirrored)); + } + projected.sort((a, b) => a.unit.localeCompare(b.unit)); + + const unreleased = rows + .filter((row) => !witnessed.has(row.unit)) + .map((row) => row.unit) + .sort((a, b) => a.localeCompare(b)); + + return { + rows: projected, + unreleased, + refused: refused.sort((a, b) => a.unit.localeCompare(b.unit)), + passes: pass + }; + }); diff --git a/pkgs/substrate-apps/src/packages/factory/src/projectionMirror.ts b/pkgs/substrate-apps/src/packages/factory/src/projectionMirror.ts new file mode 100644 index 00000000..5ac1d9dd --- /dev/null +++ b/pkgs/substrate-apps/src/packages/factory/src/projectionMirror.ts @@ -0,0 +1,228 @@ +/** + * projectionMirror.ts — U-A14. `GET /projection`, served from the object's own + * mirror. + * + * §2.2b's route table has one line for it — `GET /projection` → §2.2e — and + * §2.2e says what a projection is: the operator's fields over U-A12's + * deterministic serializer (stable key order, LF, trailing newline, ISO-8601 + * UTC). U-A13 built that projection for the TOOL, which reads a banked corpus + * off the disk. This module builds the same rows for the WORKER, which has no + * disk and no register: what it has is the receipts it accepted. + * + * THE ROW IS THE SAME ROW. `projectionRowOf` is U-A13's and is not copied here. + * The two halves it joins are the two halves it always joins: + * + * the banked cells `actuals.*`, `outcome_for_calibration`, `receipt_path` + * — retained below from the receipt's own bytes + * the evidence cells `result.observed`, `result.grade` + * — read off `stateView.receipts`, the mirror + * + * so a unit has a row only if the object accepted its evidence against a + * mirrored kernel verdict (§2.2b's exact three-cell match). A receipt the object + * refused is a unit with no row, on the Worker for the same reason as in the + * tool. + * + * WHY THE CELLS ARE RETAINED AT ALL. `Factory.observeReceipt` keeps + * `ReceiptEvidence` — five cells, the ones the chain match is about — and + * deliberately not the receipt. But `actuals.seconds`, `actuals.tokens` and + * `actuals.outcome` are measurements that live nowhere else, and a projection + * that filled them with zeros would be inventing them. So the HTTP layer writes + * the receipt's own measured cells into the `PlanningStore` beside the object's + * state, under this module's prefix, and reads them back here. Durable Object + * SQLite is what makes that survive an eviction; the in-memory Layer makes it + * survive a test. + * + * NOTHING IS AUTHORED. Every cell below comes from the receipt's bytes or from + * the mirror. `status` and `outcome_ruled` are not reachable from here — they + * are not in `PROJECTION_CELLS`, not in `ProjectionRow`, and not named in this + * file. + */ +import { Effect, Option, Schema } from "effect"; +import { serializeRecords } from "@substrate/serializer"; +import { + StorageKey, + type IPlanningStore +} from "@substrate/planning/objects/storage.ts"; +import type { ReceiptEvidence } from "@substrate/planning/objects/factory.ts"; +import { + NONE, + projectionRowOf, + type ProjectionRow +} from "./projection.ts"; + +/** Where a retained receipt's measured cells live in the store. */ +const MIRROR_RECEIPT_PREFIX = "factory/projection/"; + +/** The key one unit's retained cells are written under. */ +const mirrorReceiptKey = (unit: string): StorageKey => + StorageKey(`${MIRROR_RECEIPT_PREFIX}${unit}`); + +/** + * The measured cells kept from an accepted receipt. + * + * Exactly the banked half of a projection row and not one field more. It is a + * schema rather than an interface because storage is a boundary: what comes back + * out of SQLite is parsed before any row is built from it, by the same rule the + * `PlanningStore` header states. + */ +const RetainedReceipt = Schema.Struct({ + unit: Schema.String, + receipt_path: Schema.String, + receipt_sha256: Schema.String, + disposition: Schema.String, + seconds: Schema.Number, + tokens: Schema.Number, + outcome_for_calibration: Schema.NullOr(Schema.String), + prior_gap: Schema.NullOr(Schema.Number) +}); + +/** The measured cells kept from an accepted receipt. */ +type RetainedReceipt = typeof RetainedReceipt.Type; + +const parseRetained = Schema.decodeUnknownEffect(RetainedReceipt); +const encodeRetained = Schema.encodeUnknownSync(RetainedReceipt); + +const isRecord = (value: unknown): value is Record => + value !== null && typeof value === "object" && !Array.isArray(value); + +const numberAt = (source: Record, key: string): number | null => { + const value = source[key]; + return typeof value === "number" && Number.isFinite(value) ? value : null; +}; + +const stringAt = (source: Record, key: string): string | null => { + const value = source[key]; + return typeof value === "string" && value !== "" ? value : null; +}; + +/** + * The declared-cell prior gap of a §2.3 receipt. + * + * The strict type spells it `prior_gap{tokens_by_cell{out, …}}`; `out` is the + * cell every prior in this cycle declares (D-B19), which is the same reading + * `projection.ts` takes over the banked corpus. + */ +const priorGapOf = (value: unknown): number | null => { + if (!isRecord(value)) return null; + const byCell = value["tokens_by_cell"]; + if (isRecord(byCell)) return numberAt(byCell, "out"); + return null; +}; + +/** + * Reads the measured cells off one receipt. + * + * The receipt has already been decoded strictly by `postReceipt` when this is + * called, so the required cells are there by construction; the reader is written + * defensively anyway, because "already validated upstream" is a claim about a + * caller and not about a value. + * + * @param receipt - The receipt as it arrived, parsed. + * @param receiptPath - Where the poster banked it, or `"none"` (D-B22). + * @param receiptSha256 - sha256 over the receipt's canonical bytes. + * @returns The retained cells, or the reason they are not a row. + */ +export const retainedReceiptOf = ( + receipt: unknown, + receiptPath: string, + receiptSha256: string +): + | { readonly ok: true; readonly retained: RetainedReceipt } + | { readonly ok: false; readonly why: string } => { + if (!isRecord(receipt)) return { ok: false, why: "the receipt is not a JSON object" }; + const missing: Array = []; + const unit = stringAt(receipt, "id"); + if (unit === null) missing.push("id"); + const seconds = numberAt(receipt, "seconds"); + if (seconds === null) missing.push("seconds"); + const disposition = stringAt(receipt, "disposition"); + if (disposition === null) missing.push("disposition"); + const tokensBlock = receipt["tokens"]; + const tokens = isRecord(tokensBlock) ? numberAt(tokensBlock, "total") : null; + if (tokens === null) missing.push("tokens.total"); + if (missing.length > 0) return { ok: false, why: `carries no ${missing.join(", ")}` }; + + return { + ok: true, + retained: { + unit: unit as string, + receipt_path: receiptPath === "" ? NONE : receiptPath, + receipt_sha256: receiptSha256, + disposition: disposition as string, + seconds: seconds as number, + tokens: tokens as number, + outcome_for_calibration: stringAt(receipt, "outcome_for_calibration"), + prior_gap: priorGapOf(receipt["prior_gap"]) + } + }; +}; + +/** Writes one unit's retained cells into the store. */ +export const retainReceipt = ( + store: IPlanningStore, + retained: RetainedReceipt +): Effect.Effect => + store + .put(mirrorReceiptKey(retained.unit), encodeRetained(retained)) + .pipe(Effect.catch(() => Effect.void)); + +/** + * Joins the retained cells with the mirror, in unit order. + * + * A retained receipt with no matching evidence yields no row: the object holds + * the cells only because it accepted the evidence, so the two disagreeing means + * the store is ahead of the mirror and the row would be a claim the object + * cannot back. Evidence with no retained cells yields no row either, and for the + * mirror image of the same reason. + */ +export const mirrorProjectionRows = ( + retained: ReadonlyArray, + evidence: ReadonlyArray +): ReadonlyArray => { + const byUnit = new Map(evidence.map((entry) => [entry.id, entry] as const)); + const rows: Array = []; + for (const cells of retained) { + const mirrored = byUnit.get(cells.unit); + if (mirrored === undefined) continue; + rows.push(projectionRowOf(cells, mirrored)); + } + return rows.sort((left, right) => (left.unit < right.unit ? -1 : left.unit > right.unit ? 1 : 0)); +}; + +/** + * Reads every retained receipt back out of the store, in key order. + * + * A record that does not parse is skipped rather than thrown on: the projection + * is a mirror, and a mirror that refuses to show anything because one record is + * malformed shows less truth, not more. It cannot be written by this repository + * in the first place — the only writer is `retainReceipt` above. + */ +export const retainedReceipts = ( + store: IPlanningStore +): Effect.Effect, never> => + Effect.gen(function* () { + const keys = yield* store + .list(MIRROR_RECEIPT_PREFIX) + .pipe(Effect.catch(() => Effect.succeed>([]))); + const found: Array = []; + for (const key of keys) { + const stored = yield* store + .get(key) + .pipe(Effect.catch(() => Effect.succeed(Option.none()))); + if (Option.isNone(stored)) continue; + const parsed = yield* parseRetained(stored.value).pipe(Effect.option); + if (Option.isSome(parsed)) found.push(parsed.value); + } + return found; + }); + +/** + * The projection body, byte for byte. + * + * `serializeRecords` is U-A12's: sorted keys, LF, one record per line, a + * trailing newline. That is what makes `cmp` against a banked fixture a + * statement about the projection rather than about `JSON.stringify`'s insertion + * order, and it is why the smoke's `cmp` step is worth running at all. + */ +export const projectionBody = (rows: ReadonlyArray): string => + serializeRecords(rows as ReadonlyArray); diff --git a/pkgs/substrate-apps/src/packages/factory/src/receipt.ts b/pkgs/substrate-apps/src/packages/factory/src/receipt.ts new file mode 100644 index 00000000..ec4d7298 --- /dev/null +++ b/pkgs/substrate-apps/src/packages/factory/src/receipt.ts @@ -0,0 +1,48 @@ +/** Strict receipt decoding followed by the Factory's exact verdict match. */ +import { Effect } from "effect"; +import { decodeReceipt } from "@substrate/schema"; +import { + type IFactory, + type ReceiptEvidence, + type ReceiptObservation +} from "@substrate/planning/objects/factory.ts"; +import { FactoryError } from "@substrate/planning/schema/errors.ts"; + +const possibleId = (input: unknown): string => { + if (input === null || typeof input !== "object") return "receipt"; + const id = (input as Record)["id"]; + return typeof id === "string" ? id : "receipt"; +}; + +/** + * Decodes the complete §2.3 receipt. Only then are its three evidence cells + * compared with a mirrored kernel verdict carrying `verdictHash`. + */ +export const postReceipt = ( + factory: IFactory, + incoming: unknown, + verdictHash: string +): Effect.Effect => { + const decoded = decodeReceipt(incoming); + if (!decoded.ok) { + return Effect.fail( + new FactoryError({ + reason: "ReceiptInvalid", + operation: "Factory.observeReceipt", + subject: possibleId(incoming) + }) + ); + } + const receipt = decoded.value; + const base = { + id: receipt.id, + oracle_rc: receipt.oracle_rc, + oracle_output_sha256: receipt.oracle_output_sha256, + verdict_hash: verdictHash + }; + const evidence: ReceiptEvidence = + receipt.mutation.kind === "hint" + ? { ...base, mutation_rc: receipt.mutation.rc } + : base; + return factory.observeReceipt(evidence); +}; diff --git a/pkgs/substrate-apps/src/packages/factory/test/capacity.test.ts b/pkgs/substrate-apps/src/packages/factory/test/capacity.test.ts new file mode 100644 index 00000000..09a52e48 --- /dev/null +++ b/pkgs/substrate-apps/src/packages/factory/test/capacity.test.ts @@ -0,0 +1,245 @@ +/** + * Seat capacity through the host-neutral surface: the ingestion routes, the + * projection and admission reads, the capacity feed Layers, and the Worker's + * alarm clamp. + */ +import { Effect, Schema } from "effect"; +import { describe, expect, it } from "vitest"; +import { + Factory, + FactoryCapacityFeed, + fakeCapacityLayer, + loadFactoryInputs, + makeFactory, + makeFactoryHttpHandler, + makeInMemoryPlanningStore, + makeTableArtifactHasher, + PlanningStore, + storedCapacityLayer, + constantPriceVectorLayer +} from "../src/index.ts"; +import { alarmToSet, ALARM_FLOOR_MS } from "../../../apps/floor/src/alarm.ts"; +import { CapacityReading } from "@substrate/planning/schema/capacity.ts"; +import { PriceVector } from "@substrate/planning/schema/prices.ts"; +import { + CC_OBSERVED, + cc3Seat, + claudeSeat, + halogenSeat, + plus, + qwenSeat, + snapshot +} from "../../planning/test/seatCapacityFixtures.ts"; +import { artifactBytes, artifactDigest, emptyFactory, reading, releaseState, run } from "./fixtures.ts"; + +const encodeReading = Schema.encodeSync(CapacityReading); +const T0 = plus(CC_OBSERVED, 60); +/** Today's fleet, as the pusher would publish it. */ +const today = () => snapshot([claudeSeat(), cc3Seat(), qwenSeat(), halogenSeat(0)]); + +const request = (path: string, method: "GET" | "POST", body?: unknown, authenticated = true) => + new Request(`https://factory.invalid${path}`, { + method, + headers: { + ...(authenticated ? { authorization: "Bearer test-token" } : {}), + ...(body === undefined ? {} : { "content-type": "application/json" }) + }, + ...(body === undefined ? {} : { body: JSON.stringify(body) }) + }); + +const handlerAt = async (clock: { now: string } | undefined) => { + const factory = await emptyFactory(); + const handler = makeFactoryHttpHandler({ + factory, + token: "test-token", + artifactHasher: makeTableArtifactHasher([[artifactBytes, artifactDigest]]), + ...(clock === undefined ? {} : { now: () => clock.now }) + }); + return { factory, handler }; +}; + +const json = async (response: Response) => ({ status: response.status, body: await response.json() }); + +describe("POST /capacity/seats", () => { + it("requires the bearer", async () => { + const { handler } = await handlerAt({ now: T0 }); + expect((await handler(request("/capacity/seats", "POST", today(), false))).status).toBe(401); + }); + + it("refuses without a host clock rather than accepting against none", async () => { + const { handler } = await handlerAt(undefined); + expect(await json(await handler(request("/capacity/seats", "POST", today())))).toEqual({ + status: 501, + body: { error: "NoClock", subject: "/capacity/seats" } + }); + }); + + it("validates the snapshot against the schema", async () => { + const { handler } = await handlerAt({ now: T0 }); + const broken = { ...today(), seats: [{ ...claudeSeat(), observed_at: "2026-09-23 04:52" }] }; + expect((await handler(request("/capacity/seats", "POST", broken))).status).toBe(400); + const badVersion = { ...today(), schema_version: "seat-capacity/1" }; + expect((await handler(request("/capacity/seats", "POST", badVersion))).status).toBe(400); + }); + + it("is idempotent, monotonic, and refuses a reading dated in the future", async () => { + const clock = { now: T0 }; + const { handler } = await handlerAt(clock); + const first = await json(await handler(request("/capacity/seats", "POST", today()))); + expect(first.status).toBe(200); + expect(first.body.changed).toBe(true); + const again = await json(await handler(request("/capacity/seats", "POST", { snapshot: today() }))); + expect(again.body).toEqual({ + results: ["cc", "cc3", "pi-qwencloud", "gpu-worker"].map((seat) => ({ seat, outcome: "duplicate" })), + changed: false + }); + const older = snapshot([claudeSeat({ observed_at: plus(CC_OBSERVED, -300) })], plus(T0, 10)); + expect((await json(await handler(request("/capacity/seats", "POST", older)))).body.results).toEqual([ + { seat: "cc", outcome: "older" } + ]); + const future = snapshot([claudeSeat({ observed_at: plus(T0, 900) })], plus(T0, 900)); + expect((await json(await handler(request("/capacity/seats", "POST", future)))).body.results).toEqual([ + { seat: "cc", outcome: "future" } + ]); + }); +}); + +describe("POST /capacity with seats attached", () => { + it("keeps today's uplink body working and folds attached seats in", async () => { + const { handler, factory } = await handlerAt({ now: T0 }); + const plain = await json(await handler(request("/capacity", "POST", encodeReading(reading({}))))); + expect(plain.body).toEqual({ accepted: true }); + const withSeats = { ...encodeReading(reading({})), seats: today() }; + const attached = await json(await handler(request("/capacity", "POST", withSeats))); + expect(attached.body.accepted).toBe(false); + expect(attached.body.seats.changed).toBe(true); + const state = await run(factory.stateView); + expect(state.seat_readings).toHaveLength(4); + expect(state.latest_readings.coordinator).not.toHaveProperty("seats"); + }); + + it("never sets alarm_at at or before the host clock", async () => { + const { handler, factory } = await handlerAt({ now: T0 }); + const past = encodeReading(reading({ nextWindowAt: "2026-09-12T11:00:00Z" })); + await handler(request("/capacity", "POST", past)); + expect((await run(factory.stateView)).alarm_at).toBeNull(); + }); +}); + +describe("GET /capacity and GET /capacity/admit", () => { + const loaded = async () => { + const clock = { now: T0 }; + const context = await handlerAt(clock); + await context.handler(request("/capacity/seats", "POST", today())); + return { ...context, clock }; + }; + + it("projects every seat at the given asOf, defaulting to the host clock", async () => { + const { handler, clock } = await loaded(); + const now = await json(await handler(request("/capacity", "GET"))); + expect(now.body.as_of).toBe(T0); + expect(now.body.next_transition_at).toBe(plus(CC_OBSERVED, 1200)); + const headroom = Object.fromEntries( + now.body.seats.map((seat: { reading: { seat: string }; headroom_pct: number | null }) => [ + seat.reading.seat, + seat.headroom_pct + ]) + ); + expect(headroom).toEqual({ cc: 28, cc3: 100, "gpu-worker": null, "pi-qwencloud": null }); + clock.now = plus(T0, 3600); + const later = await json(await handler(request(`/capacity?asOf=${encodeURIComponent(T0)}`, "GET"))); + expect(later.body.as_of).toBe(T0); + }); + + it("answers the admission question per job class", async () => { + const { handler } = await loaded(); + const ask = async (query: string) => (await json(await handler(request(`/capacity/admit?${query}`, "GET")))).body; + expect(await ask("seat=cc&model=Opus")).toMatchObject({ admit: true, headroom_pct: 28 }); + expect(await ask("seat=cc&model=Fable")).toMatchObject({ admit: false, reason: "severity-critical" }); + expect(await ask("seat=cc&model=Opus&min_headroom_pct=30")).toMatchObject({ + admit: false, + reason: "window-exhausted" + }); + expect(await ask("seat=cc3&model=Opus")).toMatchObject({ admit: false, reason: "not-dispatchable" }); + expect(await ask("seat=nobody")).toMatchObject({ admit: false, reason: "unknown-seat" }); + expect(await ask(`seat=cc&model=Opus&asOf=${encodeURIComponent(plus(CC_OBSERVED, 2000))}`)).toMatchObject({ + admit: false, + reason: "stale" + }); + }); + + it("refuses a malformed asOf, a missing seat and a malformed headroom", async () => { + const { handler } = await loaded(); + expect((await handler(request("/capacity?asOf=tomorrow", "GET"))).status).toBe(400); + expect((await handler(request("/capacity/admit", "GET"))).status).toBe(400); + expect((await handler(request("/capacity/admit?seat=cc&min_headroom_pct=lots", "GET"))).status).toBe(400); + }); + + it("without a host clock, GET /capacity requires asOf", async () => { + const { handler } = await handlerAt(undefined); + expect((await handler(request("/capacity", "GET"))).status).toBe(400); + expect((await handler(request(`/capacity?asOf=${encodeURIComponent(T0)}`, "GET"))).status).toBe(200); + }); +}); + +describe("the capacity feed Layers", () => { + const prices = Schema.decodeUnknownSync(PriceVector)({ + hash: "7".repeat(64), + rows: [], + drum: 1, + stage: [] + }); + + it("the fake Layer carries seats built from today's shapes into the Factory", async () => { + const factory = await emptyFactory(); + const state = await Effect.runPromise( + loadFactoryInputs(T0).pipe( + Effect.provideService(Factory, Factory.of(factory)), + Effect.provide(fakeCapacityLayer([reading({})], [today()])), + Effect.provide(constantPriceVectorLayer(prices)) + ) + ); + expect(state.seat_readings.map((entry) => entry.seat.seat)).toEqual([ + "cc", + "cc3", + "gpu-worker", + "pi-qwencloud" + ]); + }); + + it("the stored Layer reads back what the object persisted, from the same store", async () => { + const store = await run(makeInMemoryPlanningStore); + const factory = await run( + makeFactory(releaseState([], { plans: [] })).pipe(Effect.provideService(PlanningStore, store)) + ); + await run(factory.observeSeats(today(), T0)); + await run(factory.observeSeats(snapshot([halogenSeat(1)], plus(T0, 1), "worker"), T0)); + const snapshots = await run( + Effect.gen(function* () { + const feed = yield* FactoryCapacityFeed; + return yield* feed.seats; + }).pipe(Effect.provide(storedCapacityLayer), Effect.provideService(PlanningStore, store)) + ); + expect(snapshots.map((entry) => [entry.host, entry.seats.map((seat) => seat.seat)])).toEqual([ + ["coordinator", ["cc", "cc3", "pi-qwencloud"]], + ["worker", ["gpu-worker"]] + ]); + // Feeding them back is a no-op: ingestion is idempotent. + const before = await run(factory.stateView); + for (const entry of snapshots) { + expect((await run(factory.observeSeats(entry, T0))).changed).toBe(false); + } + expect(await run(factory.stateView)).toEqual(before); + }); +}); + +describe("the Worker's alarm clamp", () => { + it("never hands setAlarm an instant at or before now", () => { + const now = Date.parse(T0); + expect(alarmToSet(null, now)).toBeNull(); + expect(alarmToSet(Number.NaN, now)).toBeNull(); + expect(alarmToSet(now + 60_000, now)).toBe(now + 60_000); + expect(alarmToSet(now, now)).toBe(now + ALARM_FLOOR_MS); + expect(alarmToSet(Date.parse("2026-09-12T11:00:00Z"), now)).toBe(now + ALARM_FLOOR_MS); + }); +}); diff --git a/pkgs/substrate-apps/src/packages/factory/test/capacityReview1.test.ts b/pkgs/substrate-apps/src/packages/factory/test/capacityReview1.test.ts new file mode 100644 index 00000000..9a48c637 --- /dev/null +++ b/pkgs/substrate-apps/src/packages/factory/test/capacityReview1.test.ts @@ -0,0 +1,283 @@ +/** + * Capacity review round 1 (evals-2026-09-23/capacity/IMPL.md, "Fix round 1"). + * + * Each case pins a defect the reviewers reproduced on the branch: + * + * - a request that lands after a due alarm and before its delivery must not + * cancel or replace that alarm (the Worker's post-request reconcile); + * - a clock-less host must not ingest seats against the publisher's own + * `published_at`, on any door; + * - an undecodable `capacity/seats` key must not take down routes that have + * nothing to do with seat capacity, nor the Worker's post-request step. + */ +import { Effect, Option, Schema } from "effect"; +import { describe, expect, it } from "vitest"; +import { + Factory, + fakeCapacityLayer, + loadFactoryInputs, + makeFactory, + makeFactoryHttpHandler, + makeInMemoryPlanningStore, + makeTableArtifactHasher, + PlanningStore, + StorageKey, + constantPriceVectorLayer, + type IFactory +} from "../src/index.ts"; +import { reconcileRuntimeAlarm, type AlarmStorage } from "../../../apps/floor/src/alarm.ts"; +import { CapacityReading } from "@substrate/planning/schema/capacity.ts"; +import { PriceVector } from "@substrate/planning/schema/prices.ts"; +import { FactoryError } from "@substrate/planning/schema/errors.ts"; +import { + CC_OBSERVED, + claudeSeat, + plus, + snapshot +} from "../../planning/test/seatCapacityFixtures.ts"; +import { artifactBytes, artifactDigest, emptyFactory, reading, releaseState, run } from "./fixtures.ts"; +import { cloudReading } from "../../planning/test/fixtures.ts"; + +const encodeReading = Schema.encodeSync(CapacityReading); +const T0 = plus(CC_OBSERVED, 60); + +const request = (path: string, method: "GET" | "POST", body?: unknown) => + new Request(`https://factory.invalid${path}`, { + method, + headers: { + authorization: "Bearer test-token", + ...(body === undefined ? {} : { "content-type": "application/json" }) + }, + ...(body === undefined ? {} : { body: JSON.stringify(body) }) + }); + +const handlerOver = (factory: IFactory, now?: () => string) => + makeFactoryHttpHandler({ + factory, + token: "test-token", + artifactHasher: makeTableArtifactHasher([[artifactBytes, artifactDigest]]), + ...(now === undefined ? {} : { now }) + }); + +const json = async (response: Response) => ({ status: response.status, body: await response.json() }); + +/** A Durable Object's alarm cell, as the Worker sees it through ctx.storage. */ +const fakeStorage = (alarm: number | null) => { + const log: Array = []; + const storage: AlarmStorage & { alarm: number | null; readonly log: Array } = { + alarm, + log, + getAlarm: async () => storage.alarm, + setAlarm: async (at: number) => { + storage.alarm = at; + log.push(`set ${new Date(at).toISOString()}`); + }, + deleteAlarm: async () => { + storage.alarm = null; + log.push("delete"); + } + }; + return storage; +}; + +describe("the Worker's reconcile never drops a due, undelivered alarm", () => { + const T = "2026-09-23T06:00:00.000Z"; + const armedAt = async (readings: ReadonlyArray>) => { + const factory = await emptyFactory(); + for (const entry of readings) { + await run(factory.observeCapacity(entry, "2026-09-23T05:00:00.000Z")); + } + expect((await run(factory.stateView)).alarm_at).toBe(T); + return factory; + }; + + it("a GET 20 ms after T, before delivery, leaves the alarm at T; its delivery wakes the object", async () => { + const factory = await armedAt([reading({ seq: 1, nextWindowAt: T })]); + const storage = fakeStorage(Date.parse(T)); + await reconcileRuntimeAlarm({ storage, factory, nowMs: Date.parse(T) + 20, path: "request" }); + expect(storage.alarm).toBe(Date.parse(T)); + expect(storage.log).toEqual([]); + // The runtime delivers the alarm (and clears it) 30 ms after T. + storage.alarm = null; + const firedAt = new Date(Date.parse(T) + 30).toISOString(); + await run(factory.alarm(firedAt)); + await reconcileRuntimeAlarm({ storage, factory, nowMs: Date.parse(firedAt), path: "alarm" }); + expect((await run(factory.stateView)).alarm_fired_at).toBe(firedAt); + expect(storage.alarm).toBeNull(); + }); + + it("with a later transition held, the due alarm is not replaced by the later one", async () => { + const later = "2026-09-23T07:00:00.000Z"; + const factory = await armedAt([reading({ seq: 1, nextWindowAt: T })]); + // A second executor's row carries the later transition. + await run(factory.observeCapacity(cloudReading({ seq: 1, nextWindowAt: later }), "2026-09-23T05:00:00.000Z")); + const storage = fakeStorage(Date.parse(T)); + await reconcileRuntimeAlarm({ storage, factory, nowMs: Date.parse(T) + 20, path: "request" }); + expect(storage.alarm).toBe(Date.parse(T)); + storage.alarm = null; + const firedAt = new Date(Date.parse(T) + 30).toISOString(); + await run(factory.alarm(firedAt)); + await reconcileRuntimeAlarm({ storage, factory, nowMs: Date.parse(firedAt), path: "alarm" }); + expect(storage.alarm).toBe(Date.parse(later)); + }); + + it("an alarm not yet due is still moved or cleared as before", async () => { + const factory = await armedAt([reading({ seq: 1, nextWindowAt: T })]); + const early = fakeStorage(Date.parse("2026-09-23T08:00:00.000Z")); + await reconcileRuntimeAlarm({ storage: early, factory, nowMs: Date.parse(T) - 60_000, path: "request" }); + expect(early.alarm).toBe(Date.parse(T)); + const empty = await emptyFactory(); + const stray = fakeStorage(Date.parse(T)); + await reconcileRuntimeAlarm({ storage: stray, factory: empty, nowMs: Date.parse(T) - 60_000, path: "request" }); + expect(stray.log).toEqual(["delete"]); + }); + + it("on the alarm path a stored instant at or before now is replaced, never kept", async () => { + const factory = await armedAt([reading({ seq: 1, nextWindowAt: T })]); + const firedAt = new Date(Date.parse(T) + 30).toISOString(); + await run(factory.alarm(firedAt)); + const storage = fakeStorage(Date.parse(T)); + await reconcileRuntimeAlarm({ storage, factory, nowMs: Date.parse(firedAt), path: "alarm" }); + expect(storage.log).toEqual(["delete"]); + }); + + it("a failing rearm on the request path is contained; on the alarm path it is raised", async () => { + const broken = { + rearm: () => + Effect.fail( + new FactoryError({ reason: "PersistenceFailed", operation: "Factory.rearm", subject: "capacity/seats" }) + ) + } as unknown as IFactory; + const storage = fakeStorage(null); + const outcome = await reconcileRuntimeAlarm({ storage, factory: broken, nowMs: Date.parse(T), path: "request" }); + expect(outcome).toMatchObject({ action: "failed" }); + await expect( + reconcileRuntimeAlarm({ storage, factory: broken, nowMs: Date.parse(T), path: "alarm" }) + ).rejects.toBeDefined(); + }); +}); + +describe("a clock-less host never ingests seats", () => { + const far = "2099-01-01T00:00:00Z"; + const poisoned = () => snapshot([claudeSeat({ observed_at: far })], far); + + it("POST /capacity with seats and no clock is refused 501 NoClock, and nothing is folded in", async () => { + const factory = await emptyFactory(); + const handler = handlerOver(factory); + const body = { ...encodeReading(reading({})), seats: poisoned() }; + expect(await json(await handler(request("/capacity", "POST", body)))).toEqual({ + status: 501, + body: { error: "NoClock", subject: "/capacity" } + }); + const state = await run(factory.stateView); + expect(state.seat_readings).toEqual([]); + expect(state.latest_readings).toEqual({}); + }); + + it("POST /capacity without seats still works without a clock", async () => { + const handler = handlerOver(await emptyFactory()); + expect(await json(await handler(request("/capacity", "POST", encodeReading(reading({})))))).toEqual({ + status: 200, + body: { accepted: true } + }); + }); + + it("with a clock, the future-dated seat is refused and an honest one is then accepted", async () => { + const handler = handlerOver(await emptyFactory(), () => T0); + const first = await json( + await handler(request("/capacity", "POST", { ...encodeReading(reading({ seq: 1 })), seats: poisoned() })) + ); + expect(first.body.seats.results).toEqual([{ seat: "cc", outcome: "future" }]); + const honest = await json( + await handler( + request("/capacity", "POST", { ...encodeReading(reading({ seq: 2 })), seats: snapshot([claudeSeat()], T0) }) + ) + ); + expect(honest.body.seats.results).toEqual([{ seat: "cc", outcome: "accepted" }]); + }); + + it("Factory.observeCapacity refuses seats without asOf, and takes them with one", async () => { + const factory = await emptyFactory(); + const withSeats = { ...reading({ seq: 1 }), seats: poisoned() }; + const failure = await Effect.runPromise(Effect.flip(factory.observeCapacity(withSeats))); + expect(failure).toMatchObject({ _tag: "FactoryError", reason: "NoClock", operation: "Factory.observeCapacity" }); + expect((await run(factory.stateView)).seat_readings).toEqual([]); + expect(await run(factory.observeCapacity({ ...reading({ seq: 1 }), seats: snapshot([claudeSeat()], T0) }, T0))).toBe( + true + ); + }); + + it("the bench loads authored seats only against an explicit clock", async () => { + const prices = Schema.decodeUnknownSync(PriceVector)({ hash: "7".repeat(64), rows: [], drum: 1, stage: [] }); + const load = async (clock: string | undefined) => { + const factory = await emptyFactory(); + return Effect.runPromise( + Effect.flip( + loadFactoryInputs(clock).pipe( + Effect.provideService(Factory, Factory.of(factory)), + Effect.provide(fakeCapacityLayer([reading({})], [poisoned()])), + Effect.provide(constantPriceVectorLayer(prices)) + ) + ) + ); + }; + expect(await load(undefined)).toMatchObject({ reason: "NoClock" }); + }); +}); + +describe("an undecodable capacity/seats key is contained to the capacity routes", () => { + const brokenFactory = async () => { + const store = await run(makeInMemoryPlanningStore); + // What a future seat-capacity/3 row, or any schema tightening, looks like here. + await run( + store.put(StorageKey("capacity/seats"), [ + { seat: { seat: "cc", provider: "gemini" }, host: "coordinator" } + ]) + ); + return run( + makeFactory(releaseState([], { plans: [] })).pipe(Effect.provideService(PlanningStore, store)) + ); + }; + + it("a plain kernel reading is accepted, GET /state answers with a marker, and rearm resolves", async () => { + const factory = await brokenFactory(); + const handler = handlerOver(factory, () => T0); + expect(await json(await handler(request("/capacity", "POST", encodeReading(reading({})))))).toEqual({ + status: 200, + body: { accepted: true } + }); + const state = await json(await handler(request("/state", "GET"))); + expect(state.status).toBe(200); + expect(state.body.seat_readings).toEqual([]); + expect(state.body.seat_readings_error).toMatch(/capacity\/seats/); + expect(Option.isOption(await run(factory.rearm(T0)))).toBe(true); + const storage = fakeStorage(null); + expect( + await reconcileRuntimeAlarm({ storage, factory, nowMs: Date.parse(T0), path: "request" }) + ).not.toMatchObject({ action: "failed" }); + }); + + it("the capacity reads say PersistenceFailed rather than answering from nothing", async () => { + const handler = handlerOver(await brokenFactory(), () => T0); + expect((await json(await handler(request("/capacity", "GET")))).body).toMatchObject({ + error: "PersistenceFailed" + }); + expect((await json(await handler(request("/capacity/admit?seat=cc", "GET")))).body).toMatchObject({ + error: "PersistenceFailed" + }); + }); + + it("the next accepted push replaces the unreadable value, and says so", async () => { + const factory = await brokenFactory(); + const handler = handlerOver(factory, () => T0); + const pushed = await json(await handler(request("/capacity/seats", "POST", snapshot([claudeSeat()], T0)))); + expect(pushed).toMatchObject({ + status: 200, + body: { results: [{ seat: "cc", outcome: "accepted" }], changed: true, discarded_unreadable: true } + }); + const state = await run(factory.stateView); + expect(state.seat_readings.map((entry) => entry.seat.seat)).toEqual(["cc"]); + expect(state.seat_readings_error).toBeNull(); + expect((await json(await handler(request("/capacity/admit?seat=cc", "GET")))).body).toMatchObject({ admit: true }); + }); +}); diff --git a/pkgs/substrate-apps/src/packages/factory/test/factory.test.ts b/pkgs/substrate-apps/src/packages/factory/test/factory.test.ts new file mode 100644 index 00000000..fcce2c7f --- /dev/null +++ b/pkgs/substrate-apps/src/packages/factory/test/factory.test.ts @@ -0,0 +1,384 @@ +import { Option } from "effect"; +import { describe, expect, it } from "vitest"; +import { cloudReading, armedPlan } from "../../planning/test/fixtures.ts"; +import { + armItems, + emptyFactory, + factoryFor, + heartbeat, + item, + reading, + report, + run, + verdict +} from "./fixtures.ts"; + +const capped = (cap = 1) => [ + { axis: "level" as const, subject: "approved", cap }, + { axis: "namespace" as const, subject: "mecattaf/conwip", cap }, + { axis: "family" as const, subject: "build", cap } +]; + +describe("the Factory state machine", () => { + it("an accepted, unclosed item is not proposed on the next evaluate", async () => { + const factory = await factoryFor( + [item({ taskId: "A", rank: 0 }), item({ taskId: "B", rank: 1 })], + { caps: capped(1) } + ); + const capacity = reading({}); + await run(factory.observeCapacity(capacity)); + expect((await run(factory.handOut(capacity.executor))).map((p) => p.taskId)).toEqual(["A"]); + await run(factory.observeOutcome(report("A", "Accepted"))); + + expect(await run(factory.handOut(capacity.executor))).toEqual([]); + expect((await run(factory.releaseState)).backlog[0]?.state).toBe("inflight"); + }); + + it("wip after verdict equals wip before admit", async () => { + const factory = await factoryFor([item({ taskId: "A" })], { caps: capped(1) }); + const before = (await run(factory.releaseState)).wip; + const capacity = reading({}); + await run(factory.observeCapacity(capacity)); + await run(factory.handOut(capacity.executor)); + await run(factory.observeOutcome(report("A", "Accepted"))); + const during = await run(factory.stateView); + expect(during.wip).toEqual({ + level: { approved: 1 }, + namespace: { "mecattaf/conwip": 1 }, + family: { build: 1 } + }); + + await run(factory.observeVerdict(verdict("A", "pass"))); + expect((await run(factory.releaseState)).wip).toEqual(before); + }); + + it("a NotYet returns the item with deferrals+1", async () => { + const factory = await factoryFor([item({ taskId: "A" })]); + const capacity = reading({}); + await run(factory.observeCapacity(capacity)); + await run(factory.handOut(capacity.executor)); + await run(factory.observeOutcome(report("A", "NotYet"))); + + const state = await run(factory.releaseState); + expect(state.backlog[0]?.state).toBe("unclaimed"); + expect(state.backlog[0]?.deferrals).toBe(1); + expect((await run(factory.stateView)).deferrals_by_row).toEqual({ + "gpu-coordinator": 1 + }); + }); + + it("applies the same NotYet report again after a new release cycle", async () => { + const factory = await factoryFor([item({ taskId: "A" })]); + const capacity = reading({}); + const notYet = report("A", "NotYet"); + await run(factory.observeCapacity(capacity)); + + await run(factory.handOut(capacity.executor)); + expect((await run(factory.observeOutcome(notYet))).idempotent).toBe(false); + expect((await run(factory.observeOutcome(notYet))).idempotent).toBe(true); + + await run(factory.handOut(capacity.executor)); + expect((await run(factory.observeOutcome(notYet))).idempotent).toBe(false); + const state = await run(factory.stateView); + expect(state.backlog[0]).toMatchObject({ state: "unclaimed", deferrals: 2 }); + expect(state.deferrals_by_row).toEqual({ "gpu-coordinator": 2 }); + }); + + it("the same verdict posted twice mirrors once and frees one slot", async () => { + const factory = await factoryFor([item({ taskId: "A" })], { caps: capped(1) }); + const capacity = reading({}); + await run(factory.observeCapacity(capacity)); + await run(factory.handOut(capacity.executor)); + await run(factory.observeOutcome(report("A", "Accepted"))); + const first = await run(factory.observeVerdict(verdict("A", "pass"))); + const second = await run(factory.observeVerdict(verdict("A", "pass"))); + + expect(first.idempotent).toBe(false); + expect(second.idempotent).toBe(true); + const state = await run(factory.stateView); + expect(state.mirrored_verdicts).toBe(1); + expect(state.wip.level.approved).toBe(0); + }); + + it("a re-arm with the same planHash adds only new ordinals", async () => { + const factory = await emptyFactory(); + const original = item({ taskId: "ordinal-3", rank: 3, value: 10 }); + await armItems(factory, [original]); + const capacity = reading({}); + await run(factory.observeCapacity(capacity)); + await run(factory.handOut(capacity.executor)); + await run(factory.observeOutcome(report("ordinal-3", "Accepted"))); + await run(factory.observeVerdict(verdict("ordinal-3", "pass"))); + + await armItems(factory, [ + item({ taskId: "ordinal-3-rewritten", rank: 3, value: 999 }), + item({ taskId: "ordinal-4", rank: 4 }) + ]); + const state = await run(factory.releaseState); + expect(state.backlog.map((entry) => entry.rank)).toEqual([3, 4]); + expect(state.backlog[0]).toMatchObject({ + taskId: "ordinal-3", + state: "closed", + value: 10 + }); + expect(state.backlog[1]).toMatchObject({ taskId: "ordinal-4", state: "unclaimed" }); + }); + + it("moves a handed-out item from unclaimed to released", async () => { + const factory = await factoryFor([item({ taskId: "A" })]); + const capacity = reading({}); + await run(factory.observeCapacity(capacity)); + const proposals = await run(factory.handOut(capacity.executor)); + expect(proposals[0]).toMatchObject({ + taskId: "A", + row: "gpu-coordinator", + argv_ref: "A" + }); + expect((await run(factory.releaseState)).backlog[0]?.state).toBe("released"); + }); + + it("reserves cap slack while a released proposal awaits its answer", async () => { + const factory = await factoryFor( + [item({ taskId: "A", rank: 0 }), item({ taskId: "B", rank: 1 })], + { caps: capped(1) } + ); + const capacity = reading({}); + await run(factory.observeCapacity(capacity)); + expect((await run(factory.handOut(capacity.executor))).map((p) => p.taskId)).toEqual(["A"]); + expect(await run(factory.handOut(capacity.executor))).toEqual([]); + expect((await run(factory.releaseState)).wip).toEqual([]); + }); + + it("refuses a verdict before Accepted and leaves released WIP unchanged", async () => { + const factory = await factoryFor([item({ taskId: "A" })]); + const capacity = reading({}); + await run(factory.observeCapacity(capacity)); + await run(factory.handOut(capacity.executor)); + const before = await run(factory.releaseState); + expect(before.backlog[0]).toMatchObject({ state: "released", outcome: Option.none() }); + + await expect(run(factory.observeVerdict(verdict("A", "pass")))).rejects.toMatchObject({ + reason: "InvalidTransition", + operation: "Factory.observeVerdict", + subject: "A (state released)" + }); + + const after = await run(factory.releaseState); + expect(after.backlog[0]).toMatchObject({ state: "released", outcome: Option.none() }); + expect(after.wip).toEqual(before.wip); + expect((await run(factory.stateView)).mirrored_verdicts).toBe(0); + }); + + it("closes a rejected release without minting a verdict outcome", async () => { + const factory = await factoryFor([item({ taskId: "A" })]); + const capacity = reading({}); + await run(factory.observeCapacity(capacity)); + await run(factory.handOut(capacity.executor)); + await run(factory.observeOutcome(report("A", "Rejected", { code: "not-armed" }))); + const state = await run(factory.stateView); + expect(state.backlog[0]).toMatchObject({ state: "closed", outcome: null }); + expect(state.facts.at(-1)).toMatchObject({ kind: "rejected", detail: "not-armed" }); + }); + + it("deduplicates an outcome report before changing WIP", async () => { + const factory = await factoryFor([item({ taskId: "A" })]); + const capacity = reading({}); + await run(factory.observeCapacity(capacity)); + await run(factory.handOut(capacity.executor)); + const first = await run(factory.observeOutcome(report("A", "Accepted", { lease_id: "L1" }))); + const second = await run(factory.observeOutcome(report("A", "Accepted", { lease_id: "L1" }))); + expect(first.idempotent).toBe(false); + expect(second.idempotent).toBe(true); + expect((await run(factory.releaseState)).wip[0]?.count).toBe(1); + }); + + it("does not confuse outcome identities containing embedded NULs", async () => { + const factory = await factoryFor([item({ taskId: "A" })]); + const capacity = reading({}); + const first = report("A", "NotYet", { + lease_id: "lease\u0000part", + row: "row" + }); + const boundaryShifted = report("A", "NotYet", { + lease_id: "lease", + row: "part\u0000row" + }); + await run(factory.observeCapacity(capacity)); + await run(factory.handOut(capacity.executor)); + await run(factory.observeOutcome(first)); + + await expect(run(factory.observeOutcome(boundaryShifted))).rejects.toMatchObject({ + reason: "InvalidTransition" + }); + await run(factory.handOut(capacity.executor)); + expect((await run(factory.observeOutcome(boundaryShifted))).idempotent).toBe(false); + expect((await run(factory.releaseState)).backlog[0]).toMatchObject({ + state: "unclaimed", + deferrals: 2 + }); + }); + + it("refuses an outcome carrying another proposal's dedup key", async () => { + const factory = await factoryFor([item({ taskId: "A" })]); + const capacity = reading({}); + await run(factory.observeCapacity(capacity)); + await run(factory.handOut(capacity.executor)); + await expect( + run(factory.observeOutcome(report("A", "Accepted", { dedupKey: "wrong" }))) + ).rejects.toMatchObject({ reason: "DedupMismatch" }); + }); +}); + +describe("dependency release and verdict replay", () => { + it("proposes B only after A closes pass", async () => { + const factory = await factoryFor( + [ + item({ taskId: "A", rank: 0 }), + item({ taskId: "B", rank: 1, dependsOn: ["A"] }) + ], + { caps: capped(2) } + ); + const capacity = reading({}); + await run(factory.observeCapacity(capacity)); + expect((await run(factory.handOut(capacity.executor))).map((p) => p.taskId)).toEqual(["A"]); + await run(factory.observeOutcome(report("A", "Accepted"))); + await run(factory.observeVerdict(verdict("A", "pass"))); + expect((await run(factory.handOut(capacity.executor))).map((p) => p.taskId)).toEqual(["B"]); + }); + + it("returns A to attempt 2 after fail and keeps B dependency-blocked", async () => { + const factory = await factoryFor( + [ + item({ taskId: "A", rank: 0 }), + item({ taskId: "B", rank: 1, dependsOn: ["A"] }) + ], + { caps: capped(2) } + ); + const capacity = reading({}); + await run(factory.observeCapacity(capacity)); + await run(factory.handOut(capacity.executor)); + await run(factory.observeOutcome(report("A", "Accepted"))); + await run(factory.observeVerdict(verdict("A", "fail"))); + const state = await run(factory.releaseState); + expect(state.backlog[0]).toMatchObject({ state: "unclaimed", attempt: 2 }); + expect((await run(factory.handOut(capacity.executor))).map((p) => p.taskId)).toEqual(["A"]); + }); + + it("gives a retried attempt a new admit identity", async () => { + const factory = await factoryFor([item({ taskId: "A" })]); + const capacity = reading({}); + await run(factory.observeCapacity(capacity)); + const first = await run(factory.handOut(capacity.executor)); + const firstKey = first[0]?.dedupKey; + await run( + factory.observeOutcome( + report("A", "Accepted", { dedupKey: firstKey, lease_id: "lease-1" }) + ) + ); + await run(factory.observeVerdict(verdict("A", "fail"))); + + const second = await run(factory.handOut(capacity.executor)); + const secondKey = second[0]?.dedupKey; + expect(secondKey).not.toBe(firstKey); + const accepted = await run( + factory.observeOutcome( + report("A", "Accepted", { dedupKey: secondKey, lease_id: "lease-2" }) + ) + ); + expect(accepted.idempotent).toBe(false); + expect((await run(factory.releaseState)).backlog[0]).toMatchObject({ + state: "inflight", + attempt: 2, + dedupKey: secondKey + }); + expect((await run(factory.stateView)).wip.level.approved).toBe(1); + }); + + it("closes a failed item at its authored attempt cap", async () => { + const factory = await factoryFor([item({ taskId: "A" })], { + plans: [{ ...armedPlan, attemptCap: Option.some(1) }] + }); + const capacity = reading({}); + await run(factory.observeCapacity(capacity)); + await run(factory.handOut(capacity.executor)); + await run(factory.observeOutcome(report("A", "Accepted"))); + await run(factory.observeVerdict(verdict("A", "fail"))); + expect((await run(factory.releaseState)).backlog[0]).toMatchObject({ + state: "closed", + attempt: 1 + }); + }); + + it("refuses a first mirrored verdict that skips the chain", async () => { + const factory = await factoryFor([item({ taskId: "A" })]); + const capacity = reading({}); + await run(factory.observeCapacity(capacity)); + await run(factory.handOut(capacity.executor)); + await run(factory.observeOutcome(report("A", "Accepted"))); + await expect(run(factory.observeVerdict(verdict("A", "pass", 2)))).rejects.toMatchObject({ + reason: "ContinuityGap" + }); + const state = await run(factory.stateView); + expect(state.continuity_gaps).toHaveLength(1); + expect(state.chains).toEqual({}); + }); +}); + +describe("capacity wake state", () => { + it("treats SLOW as STOP for proposals", async () => { + const factory = await factoryFor([item({ taskId: "A" })]); + const capacity = reading({ signal: "SLOW" }); + await run(factory.observeCapacity(capacity)); + expect(await run(factory.handOut(capacity.executor))).toEqual([]); + }); + + it("keeps only the newest reading from each executor", async () => { + const factory = await factoryFor([]); + expect(await run(factory.observeCapacity(reading({ seq: 2 })))).toBe(true); + expect(await run(factory.observeCapacity(reading({ seq: 1 })))).toBe(false); + expect((await run(factory.stateView)).latest_readings.coordinator?.seq).toBe(2); + }); + + it("sets alarm_at to the minimum next_window_at and records alarm firing", async () => { + const factory = await factoryFor([]); + await run( + factory.observeCapacity(reading({ nextWindowAt: "2026-09-06T18:00:00Z" })) + ); + await run( + factory.observeCapacity(cloudReading({ nextWindowAt: "2026-09-06T17:00:00Z" })) + ); + expect((await run(factory.stateView)).alarm_at).toBe("2026-09-06T17:00:00Z"); + await run(factory.alarm("2026-09-06T17:00:00Z")); + expect((await run(factory.stateView)).alarm_fired_at).toBe("2026-09-06T17:00:00Z"); + }); + + it("returns chain high-water marks and the constant price hash", async () => { + const factory = await factoryFor([]); + await run(factory.observeVerdict(verdict("external-task", "pass"))); + const state = await run(factory.stateView); + expect(state.chains.coordinator).toEqual({ + last_seq: 1, + last_hash: verdict("external-task", "pass").hash + }); + expect(state.price_hash).toBe("d".repeat(64)); + }); + + it("reports the latest heartbeat by lane", async () => { + const factory = await factoryFor([]); + await run( + factory.observeHeartbeat( + heartbeat("lane-A", 1), + "lane-A", + "2026-09-06T16:00:00Z" + ) + ); + await run(factory.observeHeartbeat(heartbeat("lane-A", 1), "lane-A")); + expect((await run(factory.stateView)).heartbeats_by_lane["lane-A"]).toEqual({ + executor: "coordinator", + task_id: "lane-A", + last_seq: 1, + status: "live", + observed_at: "2026-09-06T16:00:00Z" + }); + }); +}); diff --git a/pkgs/substrate-apps/src/packages/factory/test/fixtures.ts b/pkgs/substrate-apps/src/packages/factory/test/fixtures.ts new file mode 100644 index 00000000..e70e66ea --- /dev/null +++ b/pkgs/substrate-apps/src/packages/factory/test/fixtures.ts @@ -0,0 +1,109 @@ +import { Effect, Schema } from "effect"; +import { + artifactHasherTableLayer, + makeFactory, + planningStoreInMemoryLayer, + type ArmedArtifact, + type IFactory +} from "../src/index.ts"; +import { AdmitOutcomeReport } from "@substrate/planning/schema/admit.ts"; +import { Heartbeat } from "@substrate/planning/schema/uplink.ts"; +import { PlanArm } from "@substrate/planning/schema/plan.ts"; +import { Verdict } from "@substrate/planning/schema/records.ts"; +import { Sha256Hex } from "@substrate/planning/schema/ids.ts"; +import type { ReleaseState } from "@substrate/planning/release/evaluator.ts"; +import { + digest, + item, + reading, + releaseState +} from "../../planning/test/fixtures.ts"; + +const decodeOutcome = Schema.decodeUnknownSync(AdmitOutcomeReport); +const decodeHeartbeat = Schema.decodeUnknownSync(Heartbeat); +const decodePlanArm = Schema.decodeUnknownSync(PlanArm); +const decodeVerdict = Schema.decodeUnknownSync(Verdict); +const decodeSha = Schema.decodeUnknownSync(Sha256Hex); + +export { digest, item, reading, releaseState }; + +export const artifactBytes = Uint8Array.of(1, 2, 3, 4); +export const artifactDigest = decodeSha(digest("e")); + +export const armRequest = decodePlanArm({ + label: "night.js", + kind: "workflow", + level: "approved", + namespace: "mecattaf/conwip", + planHash: artifactDigest, + author: "tom" +}); + +export const factoryFor = ( + items: ReleaseState["backlog"], + overrides: Partial = {} +): Promise => + Effect.runPromise( + makeFactory(releaseState(items, overrides)).pipe( + Effect.provide(planningStoreInMemoryLayer) + ) + ); + +export const emptyFactory = (): Promise => + factoryFor([], { plans: [] }); + +export const armItems = ( + factory: IFactory, + items: ArmedArtifact["items"] +): Promise => + Effect.runPromise( + factory + .armPlan(armRequest, { + planId: items[0]?.planId ?? item({ taskId: "placeholder" }).planId, + bytes: artifactBytes, + items + }) + .pipe(Effect.provide(artifactHasherTableLayer([[artifactBytes, artifactDigest]]))) + ); + +export const report = ( + taskId: string, + outcome: "Accepted" | "Rejected" | "NotYet", + extra: Record = {} +) => + decodeOutcome({ + taskId, + dedupKey: `dedup-${taskId}`, + outcome, + row: "gpu-coordinator", + ...extra + }); + +export const verdict = ( + taskId: string, + outcome: "pass" | "fail" | "cancelled" | "preempted" | "expired", + sequence = 1, + extra: Record = {} +) => + decodeVerdict({ + _tag: "Verdict", + taskId, + executor: "coordinator", + seq: sequence, + hash: digest(String(sequence % 10)), + prevHash: digest(String(Math.max(0, sequence - 1) % 10)), + outcome, + serviceSeconds: 1, + ...extra + }); + +export const heartbeat = (taskId: string | null, sequence = 1) => + decodeHeartbeat({ + _tag: "Heartbeat", + executor: "coordinator", + taskId, + seq: sequence + }); + +export const run = (effect: Effect.Effect): Promise => + Effect.runPromise(effect); diff --git a/pkgs/substrate-apps/src/packages/factory/test/http.test.ts b/pkgs/substrate-apps/src/packages/factory/test/http.test.ts new file mode 100644 index 00000000..57bf5fa6 --- /dev/null +++ b/pkgs/substrate-apps/src/packages/factory/test/http.test.ts @@ -0,0 +1,241 @@ +import { readFileSync } from "node:fs"; +import { fileURLToPath } from "node:url"; +import { Schema } from "effect"; +import { describe, expect, it } from "vitest"; +import { + makeFactoryHttpHandler, + makeTableArtifactHasher +} from "../src/index.ts"; +import { BacklogItem } from "@substrate/planning/schema/backlog.ts"; +import { CapacityReading } from "@substrate/planning/schema/capacity.ts"; +import { + armRequest, + artifactBytes, + artifactDigest, + emptyFactory, + factoryFor, + item, + reading, + report, + run, + verdict +} from "./fixtures.ts"; + +const RECEIPT = fileURLToPath( + new URL("../../schema/test/fixtures/receipt-strict-full.json", import.meta.url) +); +const receipt = (): Record => JSON.parse(readFileSync(RECEIPT, "utf8")); +const encodeReading = Schema.encodeSync(CapacityReading); +const encodeItem = Schema.encodeSync(BacklogItem); + +const request = ( + path: string, + method: "GET" | "POST", + body?: unknown, + authenticated = true +) => + new Request(`https://factory.invalid${path}`, { + method, + headers: { + ...(authenticated ? { authorization: "Bearer test-token" } : {}), + ...(body === undefined ? {} : { "content-type": "application/json" }) + }, + ...(body === undefined ? {} : { body: JSON.stringify(body) }) + }); + +describe("the host-neutral HTTP surface", () => { + it("requires bearer authorization on every write", async () => { + const factory = await emptyFactory(); + const handler = makeFactoryHttpHandler({ + factory, + token: "test-token", + artifactHasher: makeTableArtifactHasher([[artifactBytes, artifactDigest]]) + }); + const result = await handler( + request("/capacity", "POST", encodeReading(reading({})), false) + ); + expect(result.status).toBe(401); + }); + + it("requires bearer authorization before a proposal pull can release work", async () => { + const factory = await factoryFor([item({ taskId: "A" })]); + await run(factory.observeCapacity(reading({}))); + const handler = makeFactoryHttpHandler({ + factory, + token: "test-token", + artifactHasher: makeTableArtifactHasher([]) + }); + + const refused = await handler( + request("/proposals?executor=coordinator", "GET", undefined, false) + ); + expect(refused.status).toBe(401); + expect((await run(factory.releaseState)).backlog[0]?.state).toBe("unclaimed"); + + const accepted = await handler( + request("/proposals?executor=coordinator", "GET") + ); + expect(accepted.status).toBe(200); + expect((await run(factory.releaseState)).backlog[0]?.state).toBe("released"); + }); + + it("serves plans, capacity, proposals, outcomes and state without a Worker entry", async () => { + const factory = await emptyFactory(); + const handler = makeFactoryHttpHandler({ + factory, + token: "test-token", + artifactHasher: makeTableArtifactHasher([[artifactBytes, artifactDigest]]) + }); + const a = item({ taskId: "A", rank: 0 }); + expect( + ( + await handler( + request("/plans", "POST", { + arm: armRequest, + artifact: { + planId: a.planId, + bytes: [...artifactBytes], + items: [encodeItem(a)] + } + }) + ) + ).status + ).toBe(200); + expect( + ( + await handler( + request("/capacity", "POST", encodeReading(reading({}))) + ) + ).status + ).toBe(200); + + const proposals = await handler(request("/proposals?executor=coordinator", "GET")); + expect(proposals.status).toBe(200); + expect( + (await proposals.json()) as { + proposals: Array<{ taskId: string }>; + next_wake_at: string | null; + } + ).toMatchObject({ proposals: [{ taskId: "A" }], next_wake_at: null }); + expect((await handler(request("/outcomes", "POST", report("A", "Accepted")))).status).toBe( + 200 + ); + const state = await handler(request("/state", "GET", undefined, false)); + expect(state.status).toBe(200); + expect((await state.json()) as { backlog: Array<{ state: string }> }).toMatchObject({ + backlog: [{ state: "inflight" }] + }); + }); + + it("returns next_wake_at when every row is STOP and no proposal is admissible", async () => { + const factory = await emptyFactory(); + const handler = makeFactoryHttpHandler({ + factory, + token: "test-token", + artifactHasher: makeTableArtifactHasher([[artifactBytes, artifactDigest]]) + }); + const a = item({ taskId: "A", rank: 0 }); + expect( + ( + await handler( + request("/plans", "POST", { + arm: armRequest, + artifact: { + planId: a.planId, + bytes: [...artifactBytes], + items: [encodeItem(a)] + } + }) + ) + ).status + ).toBe(200); + + const nextWakeAt = "2026-09-06T18:00:00Z"; + const encoded = encodeReading(reading({ signal: "STOP", nextWindowAt: nextWakeAt })); + const allStopped = { + ...encoded, + rows: encoded.rows.map((row) => ({ ...row, signal: "STOP" as const })) + }; + expect((await handler(request("/capacity", "POST", allStopped))).status).toBe(200); + + const response = await handler(request("/proposals?executor=coordinator", "GET")); + expect(response.status).toBe(200); + expect(await response.json()).toEqual({ proposals: [], next_wake_at: nextWakeAt }); + }); + + it("returns 409 when a strict receipt has no matching mirrored verdict", async () => { + const factory = await emptyFactory(); + const handler = makeFactoryHttpHandler({ + factory, + token: "test-token", + artifactHasher: makeTableArtifactHasher([]) + }); + const result = await handler( + request("/receipts", "POST", { + receipt: receipt(), + verdict_hash: "1".repeat(64) + }) + ); + expect(result.status).toBe(409); + }); + + it("accepts a receipt only when all three evidence cells match", async () => { + const factory = await emptyFactory(); + const handler = makeFactoryHttpHandler({ + factory, + token: "test-token", + artifactHasher: makeTableArtifactHasher([]) + }); + const line = receipt(); + const mirrored = verdict("receipt-evaluator", "pass", 1, { + unit_id: line["id"], + oracle_rc: line["oracle_rc"], + mutation_rc: (line["mutation"] as { rc: number }).rc, + oracle_output_sha256: line["oracle_output_sha256"] + }); + expect((await handler(request("/verdicts", "POST", mirrored))).status).toBe(200); + expect( + ( + await handler( + request("/receipts", "POST", { + receipt: line, + verdict_hash: mirrored.hash + }) + ) + ).status + ).toBe(200); + + const changed = { ...line, oracle_rc: 7 }; + expect( + ( + await handler( + request("/receipts", "POST", { + receipt: changed, + verdict_hash: mirrored.hash + }) + ) + ).status + ).toBe(409); + expect((await run(factory.stateView)).receipts).toHaveLength(1); + }); + + it("strictly rejects a receipt with a null measured cell", async () => { + const factory = await emptyFactory(); + const handler = makeFactoryHttpHandler({ + factory, + token: "test-token", + artifactHasher: makeTableArtifactHasher([]) + }); + const invalid = { ...receipt(), tokens: null }; + expect( + ( + await handler( + request("/receipts", "POST", { + receipt: invalid, + verdict_hash: "1".repeat(64) + }) + ) + ).status + ).toBe(400); + }); +}); diff --git a/pkgs/substrate-apps/src/packages/factory/test/inputs.test.ts b/pkgs/substrate-apps/src/packages/factory/test/inputs.test.ts new file mode 100644 index 00000000..5545f620 --- /dev/null +++ b/pkgs/substrate-apps/src/packages/factory/test/inputs.test.ts @@ -0,0 +1,58 @@ +import { Effect, Schema } from "effect"; +import { describe, expect, it } from "vitest"; +import { + Factory, + FactoryCapacityFeed, + FactoryPriceSource, + constantPriceVectorLayer, + fakeCapacityLayer, + loadFactoryInputs +} from "../src/index.ts"; +import { PriceVector } from "@substrate/planning/schema/prices.ts"; +import { factoryFor, item, reading } from "./fixtures.ts"; + +const decodePrices = Schema.decodeUnknownSync(PriceVector); + +describe("the in-memory composition Layers", () => { + it("wires a constant PriceVector and fake capacity readings into the service", async () => { + const factory = await factoryFor([item({ taskId: "A" })]); + const prices = decodePrices({ + hash: "9".repeat(64), + rows: [{ row: "gpu-coordinator", price: 0 }], + drum: 1, + stage: [{ family: "build", holdingCost: 0 }] + }); + const capacity = reading({ nextWindowAt: "2026-09-06T17:00:00Z" }); + const state = await Effect.runPromise( + loadFactoryInputs().pipe( + Effect.provideService(Factory, Factory.of(factory)), + Effect.provide(fakeCapacityLayer([capacity])), + Effect.provide(constantPriceVectorLayer(prices)) + ) + ); + expect(state.price_hash).toBe("9".repeat(64)); + expect(state.latest_readings.coordinator?.seq).toBe(1); + expect(state.alarm_at).toBe("2026-09-06T17:00:00Z"); + }); + + it("exposes the two test inputs as distinct Effect services", async () => { + const capacity = reading({}); + const prices = decodePrices({ + hash: "8".repeat(64), + rows: [], + drum: 1, + stage: [] + }); + const values = await Effect.runPromise( + Effect.gen(function* () { + const feed = yield* FactoryCapacityFeed; + const price = yield* FactoryPriceSource; + return [(yield* feed.readings).length, price.prices.hash] as const; + }).pipe( + Effect.provide(fakeCapacityLayer([capacity])), + Effect.provide(constantPriceVectorLayer(prices)) + ) + ); + expect(values).toEqual([1, "8".repeat(64)]); + }); +}); diff --git a/pkgs/substrate-apps/src/packages/factory/test/persistence.test.ts b/pkgs/substrate-apps/src/packages/factory/test/persistence.test.ts new file mode 100644 index 00000000..7456aee7 --- /dev/null +++ b/pkgs/substrate-apps/src/packages/factory/test/persistence.test.ts @@ -0,0 +1,70 @@ +import { Effect, Option } from "effect"; +import { describe, expect, it } from "vitest"; + +import { + FACTORY_STATE_KEY, + makeFactory, + makeInMemoryPlanningStore, + PlanningStore +} from "../src/index.ts"; +import { item, reading, releaseState, report, run } from "./fixtures.ts"; + +describe("Factory state in PlanningStore", () => { + it("restores inflight work, WIP, alarms, readings and outcome identity", async () => { + const store = await run(makeInMemoryPlanningStore); + const create = (initial: ReturnType) => + run(makeFactory(initial).pipe(Effect.provideService(PlanningStore, store))); + const alarmAt = "2026-09-07T03:04:05Z"; + + const first = await create(releaseState([item({ taskId: "A" })])); + const capacity = reading({ nextWindowAt: alarmAt }); + await run(first.observeCapacity(capacity)); + const proposals = await run(first.handOut(capacity.executor)); + const outcome = report("A", "Accepted", { + dedupKey: proposals[0]?.dedupKey, + lease_id: "lease-persisted" + }); + await run(first.observeOutcome(outcome)); + + expect(Option.isSome(await run(store.get(FACTORY_STATE_KEY)))).toBe(true); + const recreated = await create(releaseState([], { plans: [], wip: [] })); + const state = await run(recreated.stateView); + expect(state.backlog[0]?.state).toBe("inflight"); + expect(state.wip.level.approved).toBe(1); + expect(state.alarm_at).toBe(alarmAt); + expect(state.latest_readings.coordinator?.seq).toBe(1); + expect(await run(recreated.handOut(capacity.executor))).toEqual([]); + expect((await run(recreated.observeOutcome(outcome))).idempotent).toBe(true); + expect((await run(recreated.stateView)).wip.level.approved).toBe(1); + }); + + it("restores explicit proposal command and mutation references", async () => { + const store = await run(makeInMemoryPlanningStore); + const create = (initial: ReturnType) => + run(makeFactory(initial).pipe(Effect.provideService(PlanningStore, store))); + const enriched = item({ + taskId: "A", + argv_ref: "cards/U-A11.json#A", + mutation_hint: "delete dependencyUnmet" + }); + + const first = await create(releaseState([enriched])); + expect((await run(first.releaseState)).backlog[0]).toMatchObject({ + argv_ref: "cards/U-A11.json#A", + mutation_hint: "delete dependencyUnmet" + }); + + const recreated = await create(releaseState([], { plans: [], wip: [] })); + expect((await run(recreated.releaseState)).backlog[0]).toMatchObject({ + argv_ref: "cards/U-A11.json#A", + mutation_hint: "delete dependencyUnmet" + }); + const capacity = reading({}); + await run(recreated.observeCapacity(capacity)); + expect((await run(recreated.handOut(capacity.executor)))[0]).toMatchObject({ + taskId: "A", + argv_ref: "cards/U-A11.json#A", + mutation_hint: "delete dependencyUnmet" + }); + }); +}); diff --git a/pkgs/substrate-apps/src/packages/factory/test/projection.test.ts b/pkgs/substrate-apps/src/packages/factory/test/projection.test.ts new file mode 100644 index 00000000..553725fe --- /dev/null +++ b/pkgs/substrate-apps/src/packages/factory/test/projection.test.ts @@ -0,0 +1,317 @@ +/** + * U-A13 LAKE-PROJECTION — the suite. + * + * The unit's DOMINANT oracle runs against the register on this disk, which is a + * corpus that belongs to other lanes and moves while they bank into it. This + * suite is the part that does not: hermetic fixtures, no path opened, one + * assertion per sentence of §2.2e. + * + * What it asserts, in the order §2.2e says it: + * + * the banked corpus is read as it is, not as a schema wishes it were + * the mirror is what the Factory object accepted, and nothing else + * the diff sees a dropped receipt — the unit's own mutation_hint + * the diff sees a card that contradicts the projection, and is silent about + * one that is merely unwritten + * `--write` fills the nine cells and never `status` or `outcome_ruled` + * re-applying the same projection changes nothing + */ +import { Effect, Option } from "effect"; +import { describe, expect, it } from "vitest"; +import { + applyCells, + bankedRowOf, + cardCellsOf, + diffProjections, + fencedLines, + loadBankedIntoFactory, + makeFactory, + NEVER_WRITTEN, + planningStoreInMemoryLayer, + priorGapOf, + projectionRowOf, + PROJECTION_CELLS, + statOf, + type BankedRow, + type CardView, + type ProjectionRow +} from "../src/index.ts"; +import { armedPlan, item, reading, releaseState } from "../../planning/test/fixtures.ts"; + +/** + * One banked receipt in the shape the corpus actually has. + * + * MEASURED 2026-09-06 over `receipts/FACTORY-2026-09-06/*​/receipt.json`: every + * one of the 52 files carries `unit`, `disposition`, `verdict`, `oracle_rc`, + * `seconds` and a six-cell `tokens`; 27 also carry `outcome_for_calibration`, + * `prior_gap` and `mutation_rc`; 0 of 52 decode under the strict §2.3 `Receipt`. + */ +const receipt = (overrides: Record = {}) => ({ + id: "U-X1-2026-09-06-r0", + unit: "U-X1", + kind: "build", + disposition: "KEEP", + verdict: "PASS", + seconds: 1200, + oracle_rc: 0, + oracle_output_sha256: "a".repeat(64), + mutation_rc: 1, + outcome_for_calibration: "pass", + prior_gap: { tokens_out: 4200, p_pass_vs_outcome: "0.7 vs 1" }, + tokens: { + in_uncached: 1, + cache_read: 2, + cache_write: 0, + out: 3, + reasoning: 4, + total: 10 + }, + ...overrides +}); + +const rowOf = (unit: string, overrides: Record = {}): BankedRow => { + const read = bankedRowOf( + unit, + `receipts/FACTORY-2026-09-06/${unit}/receipt.json`, + unit.padEnd(64, "0"), + receipt({ unit, ...overrides }) + ); + if (!read.ok) throw new Error(`the fixture receipt did not read: ${read.why}`); + return read.row; +}; + +const chainLink = (index: number) => ({ + seq: index + 1, + hash: String(index + 1).padStart(64, "0"), + prevHash: String(index).padStart(64, "0") +}); + +const load = async (rows: ReadonlyArray) => { + const items = rows.map((row, index) => item({ taskId: row.unit, rank: index })); + const factory = await Effect.runPromise( + makeFactory( + releaseState(items, { plans: [{ ...armedPlan, attemptCap: Option.some(1) }] }) + ).pipe(Effect.provide(planningStoreInMemoryLayer)) + ); + return Effect.runPromise( + loadBankedIntoFactory(factory, rows, chainLink, (pass) => reading({ seq: pass + 1 })) + ); +}; + +const registerSide = (rows: ReadonlyArray): ReadonlyArray => + rows + .map((row) => + projectionRowOf(row, { + id: row.unit, + oracle_rc: row.oracle_rc, + oracle_output_sha256: row.oracle_output_sha256, + verdict_hash: "" + }) + ) + .sort((a, b) => a.unit.localeCompare(b.unit)); + +const card = (id: string, overrides: Partial = {}): CardView => ({ + id, + repo: "mecattaf/substrate", + cells: new Map(), + ...overrides +}); + +const CARD_TEXT = [ + "---", + 'id: "U-X1"', + "status: DRAFT # DRAFT -> ARMED -> RUNNING -> KEEP | DISCARD", + 'repo: "mecattaf/substrate"', + "prior:", + " p_pass: 0.55", + " predicted_tokens_cell: out # DECISIONS.md D-B19", + 'grade: "CLAIMED"', + "actuals: # written after the run", + " tokens: null", + " seconds: null", + " outcome: null", + " receipt_sha256: null", + " prior_gap: null", + "---", + "", + "# U-X1 — the body, which the writer never touches", + "" +].join("\n"); + +describe("reading the banked corpus", () => { + it("takes the cells the receipt carries and names the ones it does not", () => { + const read = bankedRowOf("U-X1", "p", "s", receipt({ mutation_rc: undefined })); + expect(read.ok).toBe(true); + if (!read.ok) return; + expect(read.row.absent).toContain("mutation_rc"); + expect(read.row.mutation_rc).toBeNull(); + expect(read.row.tokens).toBe(10); + expect(read.row.outcome).toBe("pass"); + }); + + it("refuses a receipt missing a cell it cannot do without, by name", () => { + const read = bankedRowOf("U-X1", "p", "s", receipt({ seconds: undefined, tokens: {} })); + expect(read.ok).toBe(false); + if (read.ok) return; + expect(read.why).toContain("seconds"); + expect(read.why).toContain("tokens.total"); + }); + + it("carries `none` rather than a null where §2.3 has no digest (D-B22)", () => { + const row = rowOf("U-X1", { oracle_output_sha256: undefined }); + expect(row.oracle_output_sha256).toBe("none"); + }); + + it("reads both spellings of prior_gap as the one declared cell", () => { + expect(priorGapOf({ tokens_out: 7, p_pass_vs_outcome: "0.5 vs 1" })).toBe(7); + expect(priorGapOf({ tokens_by_cell: { out: -9, "in_uncached+out": 1, total: 2 } })).toBe(-9); + expect(priorGapOf(undefined)).toBeNull(); + }); + + it("reads a FAIL verdict as the mirror's `fail`, not as an absence", () => { + expect(rowOf("U-X1", { verdict: "FAIL" }).outcome).toBe("fail"); + }); +}); + +describe("loading the corpus into the Factory object", () => { + it("walks every receipt through the state machine and mirrors it back", async () => { + const rows = [rowOf("U-X1"), rowOf("U-X2"), rowOf("U-X3")]; + const result = await load(rows); + expect(result.refused).toEqual([]); + expect(result.unreleased).toEqual([]); + expect(result.rows.map((row) => row.unit)).toEqual(["U-X1", "U-X2", "U-X3"]); + expect(result.rows[0]?.result.observed).toBe(0); + expect(result.rows[0]?.result.grade).toBe("MEASURED"); + }); + + it("agrees with the same rows read straight off the register", async () => { + const rows = [rowOf("U-X1"), rowOf("U-X2")]; + const result = await load(rows); + expect(diffProjections(result.rows, registerSide(rows), [])).toEqual([]); + }); + + it("names the unit when one banked receipt is dropped from the load", async () => { + // The unit's mutation_hint: "drop one banked receipt → --diff non-empty". + const rows = [rowOf("U-X1"), rowOf("U-X2"), rowOf("U-X3")]; + const result = await load(rows.filter((row) => row.unit !== "U-X2")); + const lines = diffProjections(result.rows, registerSide(rows), []); + expect(lines).toEqual(["missing U-X2 receipts/FACTORY-2026-09-06/U-X2/receipt.json"]); + }); + + it("names a mirrored row the register does not bank", async () => { + const rows = [rowOf("U-X1"), rowOf("U-X2")]; + const result = await load(rows); + const lines = diffProjections(result.rows, registerSide([rows[0]!]), []); + expect(lines).toEqual(["extra U-X2"]); + }); + + it("projects no row for a receipt whose evidence the object refuses", async () => { + // A verdict that skips the chain is a ContinuityGap; the object refuses it, + // the receipt then has no verdict to match, and the mirror holds neither. + const rows = [rowOf("U-X1"), rowOf("U-X2")]; + const items = rows.map((row, index) => item({ taskId: row.unit, rank: index })); + const factory = await Effect.runPromise( + makeFactory( + releaseState(items, { plans: [{ ...armedPlan, attemptCap: Option.some(1) }] }) + ).pipe(Effect.provide(planningStoreInMemoryLayer)) + ); + const skipping = (index: number) => ({ + seq: (index + 1) * 7, + hash: String(index + 1).padStart(64, "0"), + prevHash: String(index).padStart(64, "0") + }); + const result = await Effect.runPromise( + loadBankedIntoFactory(factory, rows, skipping, (pass) => reading({ seq: pass + 1 })) + ); + expect(result.rows).toEqual([]); + expect(result.refused.map((entry) => entry.reason)).toContain("ContinuityGap"); + }); +}); + +describe("the diff against the cards", () => { + it("is silent about a cell the card has not written yet", async () => { + const rows = [rowOf("U-X1")]; + const result = await load(rows); + const unwritten = card("U-X1", { + cells: new Map(PROJECTION_CELLS.map((cell) => [cell, null])) + }); + expect(diffProjections(result.rows, registerSide(rows), [unwritten])).toEqual([]); + }); + + it("names a card that carries a different value in a cell it would fill", async () => { + const rows = [rowOf("U-X1")]; + const result = await load(rows); + const stale = card("U-X1", { cells: new Map([["actuals.tokens", 999]]) }); + expect(diffProjections(result.rows, registerSide(rows), [stale])).toEqual([ + "contradiction U-X1 actuals.tokens card=999 projection=10" + ]); + }); + + it("names a card that carries no repo: (§2.2e's unmapped)", async () => { + const rows = [rowOf("U-X1")]; + const result = await load(rows); + const lines = diffProjections(result.rows, registerSide(rows), [ + card("U-X1"), + card("U-X9", { repo: null }) + ]); + expect(lines).toEqual(["unmapped U-X9"]); + }); +}); + +describe("writing the operator's fields", () => { + const cellsFor = (row: ProjectionRow) => { + const cells = new Map(); + for (const [cell, value] of cardCellsOf(row)) if (value !== null) cells.set(cell, value); + return cells; + }; + + it("fills exactly the cells §2.2e names and leaves the body alone", async () => { + const rows = [rowOf("U-X1")]; + const result = await load(rows); + const after = applyCells(CARD_TEXT, cellsFor(result.rows[0]!)); + expect(after).toContain(" tokens: 10"); + expect(after).toContain(" seconds: 1200"); + expect(after).toContain(' outcome: "KEEP"'); + expect(after).toContain(" prior_gap: 4200"); + expect(after).toContain(' grade: "MEASURED"'); + expect(after).toContain(" observed: 0"); + expect(after).toContain('outcome_for_calibration: "pass"'); + expect(after).toContain("# U-X1 — the body, which the writer never touches"); + }); + + it("never moves a status: or outcome_ruled: line", async () => { + const rows = [rowOf("U-X1")]; + const result = await load(rows); + const after = applyCells(CARD_TEXT, cellsFor(result.rows[0]!)); + expect(fencedLines(after)).toEqual(fencedLines(CARD_TEXT)); + expect(after).toContain("status: DRAFT # DRAFT -> ARMED -> RUNNING -> KEEP | DISCARD"); + for (const key of NEVER_WRITTEN) { + expect([...cardCellsOf(result.rows[0]!).keys()]).not.toContain(key); + expect(PROJECTION_CELLS).not.toContain(key); + } + }); + + it("keeps a trailing comment, and its column, when a value beside it changes", () => { + const after = applyCells(CARD_TEXT, new Map([["status", "KEEP"]])); + expect(after).toContain('status: "KEEP" # DRAFT -> ARMED -> RUNNING -> KEEP | DISCARD'); + }); + + it("is idempotent: re-applying the same cells changes no byte", async () => { + const rows = [rowOf("U-X1")]; + const result = await load(rows); + const once = applyCells(CARD_TEXT, cellsFor(result.rows[0]!)); + const twice = applyCells(once, cellsFor(result.rows[0]!)); + expect(twice).toBe(once); + }); + + it("reads the change as an insertion and not as a rewrite of the file", async () => { + const rows = [rowOf("U-X1")]; + const result = await load(rows); + const after = applyCells(CARD_TEXT, cellsFor(result.rows[0]!)); + const stat = statOf("U-X1.md", CARD_TEXT, after); + // five actuals lines replaced, plus `result:` with its three cells and + // `outcome_for_calibration:` inserted. + expect(stat.removed).toBe(5); + expect(stat.added).toBe(10); + }); +}); diff --git a/pkgs/substrate-apps/src/packages/factory/test/projectionMirror.test.ts b/pkgs/substrate-apps/src/packages/factory/test/projectionMirror.test.ts new file mode 100644 index 00000000..33ff3179 --- /dev/null +++ b/pkgs/substrate-apps/src/packages/factory/test/projectionMirror.test.ts @@ -0,0 +1,247 @@ +import { readFileSync } from "node:fs"; +import { fileURLToPath } from "node:url"; +import { Effect } from "effect"; +import { describe, expect, it } from "vitest"; +import { + makeFactoryHttpHandler, + makeInMemoryPlanningStore, + makeFactory, + mirrorProjectionRows, + NEVER_WRITTEN, + PlanningStore, + retainedReceiptOf, + type IFactory, + type IPlanningStore +} from "../src/index.ts"; +import { checkLineDiscipline } from "@substrate/serializer"; +import { releaseState, run, verdict } from "./fixtures.ts"; + +const RECEIPT = fileURLToPath( + new URL("../../schema/test/fixtures/receipt-strict-full.json", import.meta.url) +); +const receipt = (): Record => JSON.parse(readFileSync(RECEIPT, "utf8")); + +/** + * A hasher that answers for any bytes. + * + * The table hasher refuses bytes it has not been told about, which is right for + * an ARMING — an unknown artifact has no authority. A receipt digest is not an + * authority, it is a label on a row, so the fake here answers by length. It is + * still not sha-256 and still does not pretend to be: the digest's value is + * never asserted below, only its presence and its stability. + */ +const lengthHasher = { + hash: (bytes: Uint8Array) => + Effect.succeed(String(bytes.length).padStart(64, "0") as never) +}; + +const openHandler = async (): Promise<{ + readonly factory: IFactory; + readonly store: IPlanningStore; + readonly handler: (request: Request) => Promise; +}> => { + const store = await run(makeInMemoryPlanningStore); + const factory = await run( + makeFactory(releaseState([], { plans: [] })).pipe( + Effect.provideService(PlanningStore, store) + ) + ); + return { + factory, + store, + handler: makeFactoryHttpHandler({ + factory, + token: "test-token", + artifactHasher: lengthHasher, + store + }) + }; +}; + +const request = (path: string, method: "GET" | "POST", body?: unknown) => + new Request(`https://factory.invalid${path}`, { + method, + headers: { + authorization: "Bearer test-token", + ...(body === undefined ? {} : { "content-type": "application/json" }) + }, + ...(body === undefined ? {} : { body: JSON.stringify(body) }) + }); + +/** Posts one mirrored verdict and the receipt whose evidence matches it. */ +const ingest = async ( + handler: (request: Request) => Promise, + line: Record, + sequence = 1 +): Promise => { + const mirrored = verdict("receipt-evaluator", "pass", sequence, { + unit_id: line["id"], + oracle_rc: line["oracle_rc"], + mutation_rc: (line["mutation"] as { readonly rc: number }).rc, + oracle_output_sha256: line["oracle_output_sha256"] + }); + expect((await handler(request("/verdicts", "POST", mirrored))).status).toBe(200); + const posted = await handler( + request("/receipts", "POST", { + receipt: line, + verdict_hash: mirrored.hash, + receipt_path: `receipts/${String(line["id"])}/receipt.json` + }) + ); + expect(posted.status).toBe(200); + return mirrored.hash; +}; + +describe("the measured cells retained from an accepted receipt", () => { + it("reads exactly the banked half of a projection row", () => { + const read = retainedReceiptOf(receipt(), "receipts/U-A16/receipt.json", "a".repeat(64)); + expect(read.ok).toBe(true); + if (!read.ok) return; + expect(read.retained).toEqual({ + unit: "U-A16-FIXTURE-0", + receipt_path: "receipts/U-A16/receipt.json", + receipt_sha256: "a".repeat(64), + disposition: "CRASH", + seconds: 265, + tokens: 34206, + outcome_for_calibration: "fail", + prior_gap: -134694 + }); + }); + + it("names the cell it cannot do without rather than filling it with a zero", () => { + const withoutSeconds = { ...receipt() }; + delete withoutSeconds["seconds"]; + const read = retainedReceiptOf(withoutSeconds, "none", "a".repeat(64)); + expect(read.ok).toBe(false); + if (read.ok) return; + expect(read.why).toContain("seconds"); + }); + + it("carries an absent optional cell as null and never as a guess", () => { + const without = { ...receipt() }; + delete without["outcome_for_calibration"]; + delete without["prior_gap"]; + const read = retainedReceiptOf(without, "none", "a".repeat(64)); + expect(read.ok).toBe(true); + if (!read.ok) return; + expect(read.retained.outcome_for_calibration).toBeNull(); + expect(read.retained.prior_gap).toBeNull(); + }); +}); + +describe("the projection the object serves from its own mirror", () => { + it("yields a row only where the retained cells and the mirror agree on a unit", () => { + const cells = { + unit: "U-1", + receipt_path: "none", + receipt_sha256: "a".repeat(64), + disposition: "CRASH", + seconds: 1, + tokens: 2, + outcome_for_calibration: null, + prior_gap: null + }; + const evidence = { + id: "U-1", + oracle_rc: 0, + oracle_output_sha256: "sha256:x", + verdict_hash: "h" + }; + expect(mirrorProjectionRows([cells], [evidence])).toHaveLength(1); + expect(mirrorProjectionRows([cells], [])).toHaveLength(0); + expect(mirrorProjectionRows([], [evidence])).toHaveLength(0); + }); + + it("sorts by unit, so two objects holding the same receipts serve the same bytes", () => { + const cellsFor = (unit: string) => ({ + unit, + receipt_path: "none", + receipt_sha256: "a".repeat(64), + disposition: "CRASH", + seconds: 1, + tokens: 2, + outcome_for_calibration: null, + prior_gap: null + }); + const evidenceFor = (id: string) => ({ + id, + oracle_rc: 0, + oracle_output_sha256: "sha256:x", + verdict_hash: "h" + }); + const rows = mirrorProjectionRows( + [cellsFor("U-B"), cellsFor("U-A")], + [evidenceFor("U-A"), evidenceFor("U-B")] + ); + expect(rows.map((row) => row.unit)).toEqual(["U-A", "U-B"]); + }); + + it("serves an accepted receipt as one canonical line, twice identically", async () => { + const { handler } = await openHandler(); + await ingest(handler, receipt()); + + const first = await handler(request("/projection", "GET")); + expect(first.status).toBe(200); + const body = await first.text(); + expect(checkLineDiscipline(body)).toEqual([]); + expect(body.split("\n").filter((line) => line !== "")).toHaveLength(1); + expect(JSON.parse(body)).toMatchObject({ + unit: "U-A16-FIXTURE-0", + actuals: { seconds: 265, tokens: 34206, outcome: "CRASH" }, + result: { grade: "MEASURED", observed: 0 } + }); + + const second = await handler(request("/projection", "GET")); + expect(await second.text()).toBe(body); + }); + + it("never serves a cell the operator does not own", async () => { + const { handler } = await openHandler(); + // The fixture receipt carries `outcome_ruled: DISCARD`. It is Tom's field, + // and the projection has no name for it: §2.2e's `--write` fills only the + // operator's cells, and `status` and `outcome_ruled` never. + expect(receipt()["outcome_ruled"]).toBe("DISCARD"); + await ingest(handler, receipt()); + const body = await (await handler(request("/projection", "GET"))).text(); + for (const forbidden of NEVER_WRITTEN) { + expect(body).not.toContain(`"${forbidden}"`); + } + }); + + it("keeps the retained cells across a recreation over the same store", async () => { + const { handler, store } = await openHandler(); + await ingest(handler, receipt()); + const before = await (await handler(request("/projection", "GET"))).text(); + + const recreated = await run( + makeFactory(releaseState([], { plans: [], wip: [] })).pipe( + Effect.provideService(PlanningStore, store) + ) + ); + const second = makeFactoryHttpHandler({ + factory: recreated, + token: "test-token", + artifactHasher: lengthHasher, + store + }); + expect(await (await second(request("/projection", "GET"))).text()).toBe(before); + }); + + it("serves an empty body, not an error, when the host named no store", async () => { + const store = await run(makeInMemoryPlanningStore); + const factory = await run( + makeFactory(releaseState([], { plans: [] })).pipe( + Effect.provideService(PlanningStore, store) + ) + ); + const handler = makeFactoryHttpHandler({ + factory, + token: "test-token", + artifactHasher: lengthHasher + }); + const served = await handler(request("/projection", "GET")); + expect(served.status).toBe(200); + expect(await served.text()).toBe(""); + }); +}); diff --git a/pkgs/substrate-apps/src/packages/factory/tsconfig.json b/pkgs/substrate-apps/src/packages/factory/tsconfig.json new file mode 100644 index 00000000..553fbec5 --- /dev/null +++ b/pkgs/substrate-apps/src/packages/factory/tsconfig.json @@ -0,0 +1,9 @@ +{ + "extends": "../../tsconfig.base.json", + "compilerOptions": { + "lib": ["ES2022", "DOM"], + "types": [], + "allowImportingTsExtensions": true + }, + "include": ["src/**/*.ts"] +} diff --git a/pkgs/substrate-apps/src/packages/factory/vitest.config.ts b/pkgs/substrate-apps/src/packages/factory/vitest.config.ts new file mode 100644 index 00000000..21f4ddd0 --- /dev/null +++ b/pkgs/substrate-apps/src/packages/factory/vitest.config.ts @@ -0,0 +1,8 @@ +import { defineConfig } from "vitest/config" + +export default defineConfig({ + test: { + include: ["test/**/*.test.ts", "test-*/**/*.test.ts"], + environment: "node" + } +}) diff --git a/pkgs/substrate-apps/src/packages/interpreter/bin/naive-run.mjs b/pkgs/substrate-apps/src/packages/interpreter/bin/naive-run.mjs new file mode 100755 index 00000000..8666ab45 --- /dev/null +++ b/pkgs/substrate-apps/src/packages/interpreter/bin/naive-run.mjs @@ -0,0 +1,12 @@ +#!/usr/bin/env node +// Entry point: registers tsx's ESM loader, then runs src/cli.ts. +import { register } from "tsx/esm/api"; +import { fileURLToPath } from "node:url"; +import { dirname, join } from "node:path"; + +register(); +const here = dirname(fileURLToPath(import.meta.url)); +const { main } = await import(join(here, "..", "src", "cli.ts")); +const { code, out } = await main(process.argv.slice(2)); +(code === 2 ? process.stderr : process.stdout).write(out + "\n"); +process.exitCode = code; diff --git a/pkgs/substrate-apps/src/packages/interpreter/package.json b/pkgs/substrate-apps/src/packages/interpreter/package.json new file mode 100644 index 00000000..4b2d13be --- /dev/null +++ b/pkgs/substrate-apps/src/packages/interpreter/package.json @@ -0,0 +1,32 @@ +{ + "name": "@substrate/interpreter", + "version": "0.1.0", + "private": true, + "type": "module", + "exports": { + ".": "./src/index.ts", + "./meta": "./src/meta.ts" + }, + "description": "Naive clean-room interpreter for Claude Code workflow scripts: meta literal, agent/parallel/pipeline/phase/log/budget/workflow, chained resume journal, mock and replay backends. No live model calls.", + "bin": { + "naive-run": "./bin/naive-run.mjs" + }, + "scripts": { + "test": "vitest run", + "typecheck": "tsc -p tsconfig.json", + "naive-run": "node bin/naive-run.mjs" + }, + "dependencies": { + "ajv": "^8.17.1" + }, + "devDependencies": { + "@types/node": "^22.10.2", + "tsx": "^4.19.2", + "typescript": "^5.9.3", + "vitest": "^3.2.4", + "effect": "4.0.0-rc.112" + }, + "peerDependencies": { + "effect": "4.0.0-rc.112" + } +} diff --git a/pkgs/substrate-apps/src/packages/interpreter/src/backend.ts b/pkgs/substrate-apps/src/packages/interpreter/src/backend.ts new file mode 100644 index 00000000..1ceeb9a8 --- /dev/null +++ b/pkgs/substrate-apps/src/packages/interpreter/src/backend.ts @@ -0,0 +1,207 @@ +/** + * The Backend seam: what one agent() call becomes. + * + * The interpreter owns the script, the key chain, the journal, the caps and the + * retry loop. A backend owns exactly one attempt at one call and reports what + * happened. It never throws for an ordinary failure: it returns `error`. + */ +import { createHash } from "node:crypto"; + +export interface AgentOpts { + readonly label?: string; + readonly phase?: string; + readonly schema?: Record; + readonly model?: string; + readonly effort?: string; + readonly isolation?: string; + readonly agentType?: string; + readonly [k: string]: unknown; +} + +export interface AgentCall { + /** 1-based invocation index within the run (nested workflows share the counter). */ + readonly index: number; + /** The harness-compatible chained resume key. */ + readonly key: string; + readonly prompt: string; + readonly opts: AgentOpts; + /** The phase in force: opts.phase, else the last phase() title. */ + readonly phase: string | undefined; + /** 1-based attempt number; attempt > 1 is a retry after a schema mismatch or an error. */ + readonly attempt: number; + /** Validation errors from the previous attempt, for a backend that can feed them back. */ + readonly previousErrors?: readonly string[]; + readonly signal?: AbortSignal; + /** + * Which occurrence of the same content (prompt and keyed opts) this call is + * in the run, fixed at invocation: the same on every attempt and on every + * resume (successor review r3: a backend that counted its own occurrences + * gave a retry, or a resumed call, another call's identity). + */ + readonly occurrence?: number; + /** The call's content identity (key.ts contentId), fixed at invocation. */ + readonly cid?: string; + /** + * The run's budget ceiling at the moment a backend admits the call: a + * reason when the budget is spent, else undefined. A backend that queues + * calls (a WIP cap) asks it again when the call leaves the queue, and + * answers `budgetExhausted` instead of running (successor review r3: calls + * that passed the interpreter's check waited for a WIP slot and then ran + * after the budget was spent). + */ + readonly budgetExhausted?: () => string | undefined; +} + +export interface Usage { + readonly inputTokens: number; + readonly outputTokens: number; +} + +export interface AgentOutcome { + /** Unstructured result (no schema). */ + readonly text?: string; + /** Structured result (schema given); validated by the interpreter, not the backend. */ + readonly object?: unknown; + readonly usage?: Usage; + /** A terminal failure of this attempt. */ + readonly error?: string; + /** The backend's own id for the agent (session id, subagent id). */ + readonly agentId?: string; + /** A backend may declare the call skipped: the interpreter returns null without retrying. */ + readonly skipped?: boolean; + /** The backend refused to run the call because the run's budget was spent at admission. */ + readonly budgetExhausted?: boolean; +} + +export interface Backend { + readonly name: string; + run(call: AgentCall): Promise; +} + +// ---------------------------------------------------------------- MockBackend + +export interface MockOptions { + /** Length of every generated array (clamped to minItems/maxItems). Default 1. */ + readonly arrayLength?: number; + /** + * Simulated latency in wall-clock ms (setTimeout). When given, completion order + * follows the timers and is only as deterministic as they are. Default: none; + * see `turns`. + */ + readonly latency?: (call: AgentCall) => number; + /** + * Simulated latency in event-loop turns (setImmediate), used when `latency` is + * not given. Default: a function of the call ORDINAL, (index * 7) % 5, so calls + * finish out of invocation order (exercising pipelines) yet every run of the + * same script completes its calls in the same order: same keys, same results. + */ + readonly turns?: (call: AgentCall) => number; + /** Override the value for a call; return undefined to use the stub. */ + readonly respond?: (call: AgentCall) => AgentOutcome | undefined | Promise; + /** Boolean fields' stub value. Default true (so `filter(c => c.flag)` keeps items). */ + readonly booleanValue?: boolean; +} + +const digest = (s: string) => createHash("sha256").update(s).digest("hex"); + +/** + * A schema-satisfying stub. Deterministic: the same (schema, tag) gives the + * same value. Covers type (and type arrays), enum, const, properties, required, + * items/prefixItems, min/maxItems, anyOf/oneOf/allOf (first branch), minimum, + * minLength, and `$ref` into local `$defs`/`definitions`. + */ +export function stubFromSchema( + schema: unknown, + tag: string, + o: { arrayLength?: number; booleanValue?: boolean } = {}, + root: unknown = schema, + path = "$", + depth = 0, +): unknown { + if (depth > 32) return null; + if (schema === true || schema === undefined) return `${tag} ${path}`; + if (!schema || typeof schema !== "object") return null; + const s = schema as Record; + const rec = (sub: unknown, p: string) => stubFromSchema(sub, tag, o, root, p, depth + 1); + if (typeof s.$ref === "string") { + const m = /^#\/(\$defs|definitions)\/(.+)$/.exec(s.$ref); + const defs = m ? ((root as Record>)[m[1]!] ?? {}) : {}; + return rec(m ? defs[m[2]!] : undefined, path); + } + if ("const" in s) return s.const; + if (Array.isArray(s.enum) && s.enum.length) return s.enum[0]; + for (const k of ["anyOf", "oneOf"] as const) { + if (Array.isArray(s[k]) && (s[k] as unknown[]).length) { + const branches = s[k] as Record[]; + const nonNull = branches.find((b) => b && b.type !== "null") ?? branches[0]; + return rec(nonNull, path); + } + } + if (Array.isArray(s.allOf) && s.allOf.length) { + return rec(Object.assign({}, ...(s.allOf as object[])), path); + } + let t = s.type; + if (Array.isArray(t)) t = t.find((x) => x !== "null") ?? t[0]; + if (t === undefined && s.properties) t = "object"; + if (t === undefined && s.items) t = "array"; + switch (t) { + case "object": { + const out: Record = {}; + for (const [k, sub] of Object.entries((s.properties ?? {}) as Record)) out[k] = rec(sub, `${path}.${k}`); + return out; + } + case "array": { + const min = typeof s.minItems === "number" ? s.minItems : 0; + const max = typeof s.maxItems === "number" ? s.maxItems : Infinity; + const n = Math.min(max, Math.max(min, o.arrayLength ?? 1)); + const prefix = Array.isArray(s.prefixItems) ? (s.prefixItems as unknown[]) : []; + return Array.from({ length: n }, (_, i) => rec(prefix[i] ?? s.items, `${path}[${i}]`)); + } + case "boolean": + return o.booleanValue ?? true; + case "integer": + case "number": { + const min = typeof s.minimum === "number" ? s.minimum : typeof s.exclusiveMinimum === "number" ? s.exclusiveMinimum + 1 : 1; + const max = typeof s.maximum === "number" ? s.maximum : Infinity; + return Math.min(max, t === "integer" ? Math.ceil(min) : min); + } + case "null": + return null; + default: { + let v = `${tag} ${path}`; + if (typeof s.minLength === "number") while (v.length < s.minLength) v += "_"; + if (typeof s.maxLength === "number") v = v.slice(0, s.maxLength); + return v; + } + } +} + +/** + * Deterministic, offline. Every value carries a provenance tag `[[m: