diff --git a/.impeccable/config.json b/.impeccable/config.json new file mode 100644 index 0000000..c037945 --- /dev/null +++ b/.impeccable/config.json @@ -0,0 +1,14 @@ +{ + "detector": { + "ignoreRules": [], + "ignoreFiles": [], + "ignoreValues": [ + { + "rule": "overused-font", + "value": "inter", + "createdAt": "2026-08-31T12:14:04.965Z", + "reason": "Project AGENTS/TASKS UI freeze: accepted typography must be preserved; issue #96 permits functional and accessibility fixes, not redesign" + } + ] + } +} diff --git a/Dockerfile.live-test b/Dockerfile.live-test new file mode 100644 index 0000000..142e060 --- /dev/null +++ b/Dockerfile.live-test @@ -0,0 +1,25 @@ +ARG NODE_IMAGE=node:22-bookworm-slim +ARG DOCKER_CLI_IMAGE=docker:cli + +FROM ${DOCKER_CLI_IMAGE} AS engine-client + +FROM ${NODE_IMAGE} +WORKDIR /app + +RUN apt-get update \ + && apt-get install -y --no-install-recommends ca-certificates git ripgrep \ + && rm -rf /var/lib/apt/lists/* + +COPY --from=engine-client /usr/local/bin/docker /usr/local/bin/docker +COPY package.json package-lock.json tsconfig.base.json ./ +COPY apps/server/package.json apps/server/package.json +COPY apps/web/package.json apps/web/package.json +RUN npm ci + +COPY --chown=node:node apps ./apps +COPY --chown=node:node scripts ./scripts +RUN mkdir -p /app/.tmp /app/state \ + && chown -R node:node /app/.tmp /app/state + +USER node +CMD ["node", "scripts/run-live-vitest.mjs", "runtime"] diff --git a/README.md b/README.md index af95dd8..d511486 100644 --- a/README.md +++ b/README.md @@ -285,20 +285,26 @@ cp deploy/volcengine/terraform.tfvars.example \ deploy/volcengine/terraform.tfvars export VOLCENGINE_ACCESS_KEY=your-access-key export VOLCENGINE_SECRET_KEY=your-secret-key +export VOLCENGINE_SSH_PRIVATE_KEY=/absolute/path/to/the-matching-private-key.pem +# Optional when the image does not use Ubuntu's default account: +export VOLCENGINE_SSH_USER=ubuntu ./scripts/deploy-volcengine.sh ``` Set Ark values only in `.env.production` and region/zone/image/instance/key/CIDR/ repository values in `terraform.tfvars`. Provide Volcengine account credentials only through the current shell; never give account AK/SK values to an Agent Runtime. -After Terraform prints `app_url`, allow cloud-init several minutes and inspect it -with `cloud-init status --wait` and `/var/log/cloud-init-output.log` over SSH. +Terraform provisions only infrastructure and public repository metadata. The deploy +script waits for `cloud-init status --wait`, copies the ignored `.env.production` +over SSH, installs it as root-owned mode `0600`, and then starts the application. +Runtime credentials therefore do not enter Terraform variables, plans, state, +cloud-init, or instance user data. Use a dedicated test instance. Never commit `.env.production`, Terraform variables -or state, Ark keys, or Volcengine account credentials. The PoC places the Ark key in -Terraform user data/state; production requires managed secrets and encrypted remote -state. `terraform -chdir=deploy/volcengine destroy` removes the ECS instance, system -disk, and Agent workspaces—back up required code and data first. +or state, Ark keys, SSH keys, or Volcengine account credentials. Protect Terraform +state because it still contains infrastructure identifiers and network metadata. +`terraform -chdir=deploy/volcengine destroy` removes the ECS instance, system disk, +and Agent workspaces—back up required code and data first. ## Configuration @@ -338,16 +344,22 @@ boundaries. ```bash npm run check -terraform fmt -check -recursive deploy/volcengine +npm run test:coverage +npm run test:terraform docker compose config ``` +`npm run test:terraform` uses local Terraform when available and otherwise the +pinned `hashicorp/terraform:1.9.8` image. It formats, initializes, and validates a +disposable copy so generated provider files never dirty the checkout. + ## Documentation - [Current task ledger and completion cut](docs/TASKS.md) - [Shepherd product requirements](docs/PRD.md) - [Shepherd design and trust boundaries](docs/SHEPHERD.md) - [Architecture](docs/ARCHITECTURE.md) +- [Release test report](docs/SHEPHERD_TEST_REPORT.md) - [Local POC](docs/LOCAL_POC.md) - [Track 1 TechJam brief](docs/TECHJAM.md) - [Hackathon extension guide](docs/HACKATHON_EXTENSION_GUIDE.md) diff --git a/apps/server/src/agent-service.test.ts b/apps/server/src/agent-service.test.ts index ed13873..fba19d5 100644 --- a/apps/server/src/agent-service.test.ts +++ b/apps/server/src/agent-service.test.ts @@ -197,6 +197,59 @@ describe("Agent lifecycle", () => { expect(service.listAgents()).toHaveLength(0); }); + it("refuses Agent admission at durable capacity without creating an orphan workspace", async () => { + const service = await makeService(); + const first = await service.createAgent({ name: "Capacity sentinel" }); + const internals = service as unknown as { store: JsonStore }; + await internals.store.mutate((database) => { + database.agents.push( + ...Array.from({ length: 9_999 }, (_, index) => ({ + ...structuredClone(first), + id: `capacity-agent-${index}`, + name: `Capacity Agent ${index}`, + workspacePath: path.join(path.dirname(first.workspacePath), `capacity-agent-${index}`), + })), + ); + }); + const workspaceNamesBefore = await readdir(path.dirname(first.workspacePath)); + + await expect(service.createAgent({ name: "One Agent too many" })).rejects.toMatchObject({ + name: "DurableCapacityError", + statusCode: 507, + }); + expect(service.listAgents()).toHaveLength(10_000); + expect(await readdir(path.dirname(first.workspacePath))).toEqual(workspaceNamesBefore); + }); + + it("reserves the final durable Agent slot across concurrent admissions", async () => { + const service = await makeService(); + const first = await service.createAgent({ name: "Capacity sentinel" }); + const internals = service as unknown as { store: JsonStore }; + await internals.store.mutate((database) => { + database.agents.push( + ...Array.from({ length: 9_998 }, (_, index) => ({ + ...structuredClone(first), + id: `concurrent-capacity-agent-${index}`, + name: `Concurrent Capacity Agent ${index}`, + workspacePath: path.join( + path.dirname(first.workspacePath), + `concurrent-capacity-agent-${index}`, + ), + })), + ); + }); + + const results = await Promise.allSettled([ + service.createAgent({ name: "Final slot contender A" }), + service.createAgent({ name: "Final slot contender B" }), + ]); + + expect(results.filter(({ status }) => status === "fulfilled")).toHaveLength(1); + expect(results.filter(({ status }) => status === "rejected")).toHaveLength(1); + expect(service.listAgents()).toHaveLength(10_000); + expect(await readdir(path.dirname(first.workspacePath))).toHaveLength(3); + }); + it("preserves an Agent and workspace while a private Contract prompt references it", async () => { const service = await makeService(); const agent = await service.createAgent({ diff --git a/apps/server/src/agent-service.ts b/apps/server/src/agent-service.ts index e3f9f8e..eaa4e79 100644 --- a/apps/server/src/agent-service.ts +++ b/apps/server/src/agent-service.ts @@ -1,4 +1,8 @@ import { randomUUID } from "node:crypto"; +import { + appendWithinDurableCapacity, + assertWithinDurableCapacity, +} from "./collection-capacity.js"; import type { AppConfig } from "./config.js"; import { isArkConfigured, isShepherdModelReviewConfigured } from "./config.js"; import { HttpError, RunCancelledError } from "./errors.js"; @@ -194,6 +198,7 @@ export class AgentService { private readonly activeExecutions = new Map>(); private readonly cancellationRequests = new Set(); private readonly deletionReservations = new Set(); + private pendingAgentCreations = 0; constructor( private readonly config: AppConfig, @@ -260,9 +265,21 @@ export class AgentService { createdAt: timestamp, updatedAt: timestamp, }; - await this.workspaces.create(agent); - await this.store.mutate((database) => database.agents.push(agent)); - return agent; + assertWithinDurableCapacity( + this.store.snapshot().agents, + this.pendingAgentCreations + 1, + "Agent", + ); + this.pendingAgentCreations += 1; + try { + await this.workspaces.create(agent); + await this.store.mutate((database) => + appendWithinDurableCapacity(database.agents, [agent], "Agent"), + ); + return agent; + } finally { + this.pendingAgentCreations -= 1; + } } async updateAgent(id: string, input: UpdateAgentInput): Promise { @@ -511,8 +528,8 @@ export class AgentService { if (storedAgent.currentContractId) { throw new HttpError(409, "Shepherd is currently using this Agent"); } - database.runs.push(run); - database.messages.push(message); + appendWithinDurableCapacity(database.runs, [run], "Agent run"); + appendWithinDurableCapacity(database.messages, [message], "Agent message"); const snapshot = structuredClone(storedAgent); storedAgent.status = "busy"; storedAgent.lastError = null; @@ -592,14 +609,14 @@ export class AgentService { storedRun.output = result.output; storedRun.usage = result.usage; storedRun.completedAt = completedAt; - database.messages.push({ + appendWithinDurableCapacity(database.messages, [{ id: randomUUID(), agentId: agent.id, runId: run.id, role: "assistant", content: result.output, createdAt: completedAt, - }); + }], "Agent message"); agent.status = "ready"; agent.codexThreadId = result.threadId; agent.lastError = null; diff --git a/apps/server/src/app.test.ts b/apps/server/src/app.test.ts index 29a229a..edd5667 100644 --- a/apps/server/src/app.test.ts +++ b/apps/server/src/app.test.ts @@ -785,6 +785,39 @@ describe("HTTP boundary", () => { await app.close(); }); + it("rejects hostile Host and Origin values before tokenless mutation handlers", async () => { + const app = await createApp( + loadConfig({ NODE_ENV: "test" }), + createAgentService(), + createShepherdService(), + ); + const route = `/api/agents/${agentId}/start`; + for (const headers of [ + { host: "attacker.example" }, + { host: "localhost", origin: "https://attacker.example" }, + { host: "localhost", origin: "null" }, + { host: "localhost", origin: "http://127.0.0.1" }, + ]) { + const denied = await app.inject({ method: "POST", url: route, headers }); + expect(denied.statusCode, JSON.stringify(headers)).toBe(403); + expect(denied.json()).toEqual({ error: "Local API origin required" }); + } + + const sameOrigin = await app.inject({ + method: "POST", + url: route, + headers: { host: "localhost", origin: "http://localhost" }, + }); + expect(sameOrigin.statusCode).toBe(200); + const commandLine = await app.inject({ + method: "POST", + url: route, + headers: { host: "127.0.0.1:3000" }, + }); + expect(commandLine.statusCode).toBe(200); + await app.close(); + }); + it("protects API routes spelled with a percent-encoded prefix", async () => { // The hook decides whether to authenticate from the RAW request target while // the router percent-decodes before matching, so any encoded spelling of the diff --git a/apps/server/src/app.ts b/apps/server/src/app.ts index 4638158..d9bd06a 100644 --- a/apps/server/src/app.ts +++ b/apps/server/src/app.ts @@ -4,7 +4,7 @@ import Fastify, { type FastifyInstance } from "fastify"; import { timingSafeEqual } from "node:crypto"; import { fileURLToPath } from "node:url"; import { z } from "zod"; -import type { AppConfig } from "./config.js"; +import { isLoopbackHost, type AppConfig } from "./config.js"; import { HttpError } from "./errors.js"; import { agentAuthorityPreset, @@ -906,6 +906,31 @@ export async function createApp( routedPath = "/api/"; routedUrl = "/api/"; } + if (!config.authToken && routedPath.startsWith("/api/")) { + const hostHeader = request.headers.host?.trim().toLowerCase() ?? ""; + let requestHostname = ""; + try { + requestHostname = new URL(`http://${hostHeader}`).hostname; + } catch { + requestHostname = ""; + } + const originHeader = request.headers.origin; + let originAllowed = originHeader === undefined; + if (typeof originHeader === "string") { + try { + const origin = new URL(originHeader); + originAllowed = + origin.protocol === `${request.protocol}:` && + origin.host.toLowerCase() === hostHeader && + isLoopbackHost(origin.hostname); + } catch { + originAllowed = false; + } + } + if (!isLoopbackHost(requestHostname) || !originAllowed) { + return reply.code(403).send({ error: "Local API origin required" }); + } + } if ( !config.authToken || !routedPath.startsWith("/api/") || diff --git a/apps/server/src/codex-runner.test.ts b/apps/server/src/codex-runner.test.ts index ab8db6e..0801a7b 100644 --- a/apps/server/src/codex-runner.test.ts +++ b/apps/server/src/codex-runner.test.ts @@ -1,10 +1,12 @@ import { describe, expect, it } from "vitest"; import { buildCodexArgs, + CodexRunner, parseCodexEventLine, requireEphemeralThreadId, resolveRunnerTimeoutMs, } from "./codex-runner.js"; +import { loadConfig } from "./config.js"; describe("Codex runner protocol", () => { it("builds a new-session invocation", () => { @@ -119,4 +121,97 @@ describe("Codex runner protocol", () => { parsed.threadIds.splice(0, 2, "thread-1\n"); expect(() => requireEphemeralThreadId(parsed)).toThrow("invalid thread ID"); }); + + it("ignores malformed events and bounds every optional event field", () => { + const parsed = { + messages: [] as string[], + threadId: null as string | null, + usage: null as Record | null, + errors: [] as string[], + threadIds: [] as string[], + }; + for (const event of [ + "not-json", + JSON.stringify({ type: "thread.started", thread_id: 42 }), + JSON.stringify({ type: "item.completed", item: null }), + JSON.stringify({ type: "item.completed", item: { type: "tool", text: "private" } }), + JSON.stringify({ type: "item.completed", item: { type: "agent_message", text: 42 } }), + JSON.stringify({ + type: "turn.completed", + usage: { input_tokens: -1, cached_input_tokens: 2, output_tokens: -1 }, + }), + JSON.stringify({ type: "error", message: "primary" }), + JSON.stringify({ type: "error", error: "fallback" }), + JSON.stringify({ type: "error" }), + ]) { + parseCodexEventLine(event, parsed); + } + expect(parsed.messages).toEqual([]); + expect(parsed.usage).toEqual({ cachedInputTokens: 2 }); + expect(parsed.errors).toEqual([ + "primary", + "fallback", + "Codex reported an unknown error", + ]); + }); + + it("rejects invalid fresh requests, timeouts, and thread identifiers", () => { + const invalidFresh = { + mode: "fresh-ephemeral" as const, + agentId: "plane", + workspacePath: "/tmp/workspace", + prompt: "bounded", + threadId: "must-not-resume", + codexHome: "/tmp/codex-home", + timeoutMs: 999, + }; + const malformedFresh = invalidFresh as unknown as Parameters[0]; + expect(() => buildCodexArgs(malformedFresh, "workspace-write")).toThrow( + "cannot resume", + ); + expect(() => resolveRunnerTimeoutMs(malformedFresh, 2_000)).toThrow( + "at least 1000", + ); + expect(() => resolveRunnerTimeoutMs({ + agentId: "agent", + workspacePath: "/tmp/workspace", + prompt: "bounded", + threadId: null, + }, Number.NaN)).toThrow("at least 1000"); + + const parsed = { + messages: [], + threadId: null, + usage: null, + errors: [], + threadIds: ["x".repeat(513)], + }; + expect(() => requireEphemeralThreadId(parsed)).toThrow("invalid thread ID"); + parsed.threadIds = [" thread-id"]; + expect(() => requireEphemeralThreadId(parsed)).toThrow("invalid thread ID"); + parsed.threadIds = ["thread-id"]; + expect(requireEphemeralThreadId(parsed)).toBe("thread-id"); + expect(() => requireEphemeralThreadId({ + messages: [], + threadId: null, + usage: null, + errors: [], + })).toThrow("did not emit"); + }); + + it("rejects duplicate local runs and treats cancellation of an idle Agent as a no-op", async () => { + const runner = new CodexRunner(loadConfig({ + NODE_ENV: "test", + CODEX_BIN: "/definitely/missing/codex", + })); + await expect(runner.cancel("idle-agent")).resolves.toBe(false); + const internals = runner as unknown as { active: Map }; + internals.active.set("busy-agent", {}); + await expect(runner.run({ + agentId: "busy-agent", + workspacePath: "/tmp/workspace", + prompt: "bounded", + threadId: null, + })).rejects.toThrow("already has an active Codex process"); + }); }); diff --git a/apps/server/src/collection-capacity.ts b/apps/server/src/collection-capacity.ts new file mode 100644 index 0000000..cfff03d --- /dev/null +++ b/apps/server/src/collection-capacity.ts @@ -0,0 +1,33 @@ +export const MAX_DURABLE_COLLECTION_ITEMS = 10_000; + +export class DurableCapacityError extends Error { + readonly statusCode = 507; + + constructor(readonly collection: string) { + super(`Durable ${collection} capacity is exhausted`); + this.name = "DurableCapacityError"; + } +} + +export function assertWithinDurableCapacity( + collection: readonly unknown[], + additionalItems: number, + label: string, +): void { + if ( + !Number.isSafeInteger(additionalItems) || + additionalItems < 0 || + collection.length + additionalItems > MAX_DURABLE_COLLECTION_ITEMS + ) { + throw new DurableCapacityError(label); + } +} + +export function appendWithinDurableCapacity( + collection: T[], + values: readonly T[], + label: string, +): void { + assertWithinDurableCapacity(collection, values.length, label); + collection.push(...values); +} diff --git a/apps/server/src/config.test.ts b/apps/server/src/config.test.ts index 9a350d4..41d24ba 100644 --- a/apps/server/src/config.test.ts +++ b/apps/server/src/config.test.ts @@ -106,19 +106,36 @@ describe("Shepherd configuration", () => { expect(config.shepherdRoot).toBe("/app/state/data/shepherd"); }); - it("starts on any bind without a token and still carries one when configured", () => { - // The token is optional everywhere: an empty token disables the bearer check, - // so no bind requires configuration before the server will start. + it("allows tokenless loopback exposure and rejects tokenless public binds", () => { for (const environment of ["development", "test", "production"] as const) { - for (const host of ["0.0.0.0", "192.0.2.10", "127.0.0.1", "::"]) { + for (const host of ["127.0.0.1", "127.2.3.4", "localhost", "::1"]) { const config = loadConfig({ HOST: host, NODE_ENV: environment }); expect(config.host).toBe(host); expect(config.authToken).toBe(""); } } - // A genuine token is carried verbatim on any bind. Weakness is asserted - // separately, so this case keeps proving only what it is named for. + for (const host of ["0.0.0.0", "192.0.2.10", "::"]) { + expect(() => loadConfig({ HOST: host, NODE_ENV: "production" })).toThrow( + /APP_AUTH_TOKEN.*non-loopback/u, + ); + } + + expect( + loadConfig({ + HOST: "0.0.0.0", + PUBLIC_BIND_ADDR: "127.0.0.1", + NODE_ENV: "production", + }).authToken, + ).toBe(""); + expect(() => + loadConfig({ + HOST: "0.0.0.0", + PUBLIC_BIND_ADDR: "0.0.0.0", + NODE_ENV: "production", + }), + ).toThrow(/APP_AUTH_TOKEN.*non-loopback/u); + expect( loadConfig({ HOST: "0.0.0.0", diff --git a/apps/server/src/config.ts b/apps/server/src/config.ts index 16b5837..d864efe 100644 --- a/apps/server/src/config.ts +++ b/apps/server/src/config.ts @@ -1,5 +1,6 @@ import { randomBytes } from "node:crypto"; import { constants } from "node:fs"; +import { isIP } from "node:net"; import { chmod, link, @@ -21,12 +22,24 @@ const verifierInstallationNoncePattern = /^[a-f0-9]{32}$/u; const runtimeInstanceIdPattern = /^[a-zA-Z0-9_.-]{1,48}$/u; const containerVolumeNamePattern = /^[a-zA-Z0-9][a-zA-Z0-9_.-]{0,63}$/u; +export function isLoopbackHost(input: string): boolean { + const host = input.trim().toLowerCase().replace(/^\[|\]$/gu, ""); + if (host === "localhost") return true; + const family = isIP(host); + if (family === 4) return host.split(".")[0] === "127"; + if (family === 6) { + return host === "::1" || /^::ffff:127(?:\.|$)/u.test(host); + } + return false; +} + const booleanEnvironmentValue = z .enum(["true", "false"]) .transform((value) => value === "true"); const envSchema = z.object({ HOST: z.string().trim().min(1).default("127.0.0.1"), + PUBLIC_BIND_ADDR: z.string().trim().min(1).optional(), PORT: z.coerce.number().int().min(1).max(65535).default(3000), LOG_LEVEL: z.string().default("info"), APP_DATA_DIR: z.string().default(path.resolve(".data")), @@ -281,11 +294,11 @@ export function resolveContainerStateMount(input: { export function loadConfig(environment: NodeJS.ProcessEnv = process.env) { const env = envSchema.parse(environment); - // `APP_AUTH_TOKEN` is optional everywhere. An empty token disables the bearer - // check outright, so the server starts with no configuration on any bind. Set a - // token to require one; the request hook then enforces it on every `/api/` - // route. Setting one is still strongly advised for any non-loopback deployment. const authToken = env.APP_AUTH_TOKEN?.trim() ?? ""; + const publicBindAddress = env.PUBLIC_BIND_ADDR ?? env.HOST; + if (!authToken && !isLoopbackHost(publicBindAddress)) { + throw new Error("APP_AUTH_TOKEN is required for a non-loopback public bind"); + } const arkApiKey = env.ARK_API_KEY?.trim() ?? ""; const arkModel = env.ARK_MODEL?.trim() ?? ""; const defaultContainerUser = @@ -319,6 +332,7 @@ export function loadConfig(environment: NodeJS.ProcessEnv = process.env) { }); return { host: env.HOST, + publicBindAddress, port: env.PORT, logLevel: env.LOG_LEVEL, dataDirectory, diff --git a/apps/server/src/database-schema.ts b/apps/server/src/database-schema.ts index f9febc6..8c1af23 100644 --- a/apps/server/src/database-schema.ts +++ b/apps/server/src/database-schema.ts @@ -1,6 +1,7 @@ import { z } from "zod"; import path from "node:path"; import type { DatabaseV1, DatabaseV2 } from "./types.js"; +import { MAX_DURABLE_COLLECTION_ITEMS } from "./collection-capacity.js"; import { normalizeRepoPath, normalizeRepoPattern } from "./shepherd/authority.js"; import { AUTH_BACKEND_CHECK_ID, @@ -15,7 +16,7 @@ import { assertSafeGitBranch, } from "./shepherd/git-client.js"; -const MAX_COLLECTION_ITEMS = 10_000; +const MAX_COLLECTION_ITEMS = MAX_DURABLE_COLLECTION_ITEMS; const MAX_ID_LENGTH = 200; const MAX_PATH_LENGTH = 4_096; const MAX_TEXT_LENGTH = 1_048_576; diff --git a/apps/server/src/database.test.ts b/apps/server/src/database.test.ts index f51b2cd..b63c875 100644 --- a/apps/server/src/database.test.ts +++ b/apps/server/src/database.test.ts @@ -631,6 +631,28 @@ describe("Database V2", () => { expect(() => shepherdEventsAfter(database, -1)).toThrow("non-negative"); }); + it("refuses an event beyond durable capacity without consuming its cursor", () => { + const database = emptyDatabase(timestamp); + for (let index = 0; index < 10_000; index += 1) { + appendShepherdEvent(database, { + ...eventInput(`event-${index}`), + id: `event-${index}`, + }); + } + + expect(() => + appendShepherdEvent(database, { + ...eventInput("overflow"), + id: "event-overflow", + }), + ).toThrow(expect.objectContaining({ + name: "DurableCapacityError", + statusCode: 507, + })); + expect(database.shepherd.events).toHaveLength(10_000); + expect(database.shepherd.nextEventSequence).toBe(10_001); + }); + it("accepts reset-created event gaps without reusing a polling cursor", () => { const database = emptyDatabase(timestamp); appendShepherdEvent(database, { ...eventInput("one"), missionId: null }); diff --git a/apps/server/src/database.ts b/apps/server/src/database.ts index c8f30be..3beeb49 100644 --- a/apps/server/src/database.ts +++ b/apps/server/src/database.ts @@ -1,4 +1,5 @@ import { randomUUID } from "node:crypto"; +import { appendWithinDurableCapacity } from "./collection-capacity.js"; import type { ProjectGroupMessage, ShepherdDatabase, @@ -162,7 +163,7 @@ export const appendShepherdEvent = ( candidateId: input.candidateId, details: boundedDetails(input.details), }; - database.shepherd.events.push(event); + appendWithinDurableCapacity(database.shepherd.events, [event], "Shepherd event"); database.shepherd.nextEventSequence = sequence + 1; return structuredClone(event); }; @@ -193,6 +194,10 @@ export const appendProjectGroupMessage = ( message: ProjectGroupMessage, ): ProjectGroupMessage => { const persisted = structuredClone(message); - database.shepherd.groupMessages.push(persisted); + appendWithinDurableCapacity( + database.shepherd.groupMessages, + [persisted], + "Project Group message", + ); return structuredClone(persisted); }; diff --git a/apps/server/src/runner-factory.test.ts b/apps/server/src/runner-factory.test.ts new file mode 100644 index 0000000..b846048 --- /dev/null +++ b/apps/server/src/runner-factory.test.ts @@ -0,0 +1,15 @@ +import { describe, expect, it } from "vitest"; +import { CodexRunner } from "./codex-runner.js"; +import { ContainerCodexRunner } from "./container-codex-runner.js"; +import { loadConfig } from "./config.js"; +import { createRunner } from "./runner-factory.js"; + +describe("runner factory", () => { + it("selects only the configured Runtime boundary", () => { + expect(createRunner(loadConfig({ NODE_ENV: "test" }))).toBeInstanceOf(CodexRunner); + expect(createRunner(loadConfig({ + NODE_ENV: "test", + RUNTIME_PROVIDER: "container", + }))).toBeInstanceOf(ContainerCodexRunner); + }); +}); diff --git a/apps/server/src/shepherd/live-model-review.integration.test.ts b/apps/server/src/shepherd/live-model-review.integration.test.ts index 1e91a95..058fe81 100644 --- a/apps/server/src/shepherd/live-model-review.integration.test.ts +++ b/apps/server/src/shepherd/live-model-review.integration.test.ts @@ -154,7 +154,11 @@ describe.skipIf(!liveEnabled)("Shepherd live SHEPHERD_MODEL advisory review", () // This reviewer-only gate never starts the HTTP server. Preserve every real // provider/reviewer value while isolating it from an unrelated hosting bind // in the caller's environment. Production loadConfig validation is unchanged. - const config = loadConfig({ ...process.env, HOST: "127.0.0.1" }); + const config = loadConfig({ + ...process.env, + HOST: "127.0.0.1", + PUBLIC_BIND_ADDR: "127.0.0.1", + }); // Fail loudly rather than silently passing an unconfigured no-op. expect(isShepherdModelReviewConfigured(config)).toBe(true); diff --git a/apps/server/src/shepherd/live-runtime-gate.test.ts b/apps/server/src/shepherd/live-runtime-gate.test.ts new file mode 100644 index 0000000..dd98226 --- /dev/null +++ b/apps/server/src/shepherd/live-runtime-gate.test.ts @@ -0,0 +1,41 @@ +import path from "node:path"; +import { describe, expect, it } from "vitest"; +import { resolveLiveRuntimeLayout } from "./live-runtime-gate.js"; + +describe("live Runtime gate layout", () => { + it("keeps every mutable root inside the wrapper-provided named volume", () => { + const layout = resolveLiveRuntimeLayout( + { + SHEPHERD_LIVE_GATE_ROOT: "/app/state/live-gate", + CONTAINER_STATE_ROOT: "/app/state", + CONTAINER_STATE_VOLUME: "shepherd-live-state-unit", + }, + "/app", + ); + + expect(layout.containerStateRoot).toBe("/app/state"); + expect(layout.containerStateVolume).toBe("shepherd-live-state-unit"); + for (const root of [ + layout.liveRoot, + layout.dataDirectory, + layout.workspaceRoot, + layout.codexHome, + layout.shepherdRoot, + layout.shepherdCodexHomeRoot, + ]) { + expect(path.relative("/app/state", root)).toMatch(/^(?!\.\.)[^/]/u); + } + }); + + it("rejects a partial or escaping wrapper volume contract", () => { + expect(() => resolveLiveRuntimeLayout({ + SHEPHERD_LIVE_GATE_ROOT: "/app/state/live-gate", + CONTAINER_STATE_ROOT: "/app/state", + }, "/app")).toThrow(/must be provided together/u); + expect(() => resolveLiveRuntimeLayout({ + SHEPHERD_LIVE_GATE_ROOT: "/app/escape", + CONTAINER_STATE_ROOT: "/app/state", + CONTAINER_STATE_VOLUME: "shepherd-live-state-unit", + }, "/app")).toThrow(/inside CONTAINER_STATE_ROOT/u); + }); +}); diff --git a/apps/server/src/shepherd/live-runtime-gate.ts b/apps/server/src/shepherd/live-runtime-gate.ts new file mode 100644 index 0000000..d6682d2 --- /dev/null +++ b/apps/server/src/shepherd/live-runtime-gate.ts @@ -0,0 +1,60 @@ +import path from "node:path"; + +export interface LiveRuntimeLayout { + liveRoot: string; + dataDirectory: string; + workspaceRoot: string; + codexHome: string; + shepherdRoot: string; + shepherdCodexHomeRoot: string; + containerStateRoot: string | undefined; + containerStateVolume: string | undefined; +} + +function isStrictChild(root: string, candidate: string): boolean { + const relative = path.relative(root, candidate); + return relative.length > 0 + && relative !== ".." + && !relative.startsWith(`..${path.sep}`) + && !path.isAbsolute(relative); +} + +export function resolveLiveRuntimeLayout( + environment: NodeJS.ProcessEnv, + repositoryRoot: string, +): LiveRuntimeLayout { + const requestedRoot = environment.SHEPHERD_LIVE_GATE_ROOT; + const stateRoot = environment.CONTAINER_STATE_ROOT; + const stateVolume = environment.CONTAINER_STATE_VOLUME; + const volumeMode = [requestedRoot, stateRoot, stateVolume].some( + (value) => value !== undefined, + ); + if (volumeMode && (!requestedRoot || !stateRoot || !stateVolume)) { + throw new Error( + "SHEPHERD_LIVE_GATE_ROOT, CONTAINER_STATE_ROOT, and CONTAINER_STATE_VOLUME must be provided together", + ); + } + + const liveRoot = volumeMode + ? path.resolve(requestedRoot!) + : path.join(repositoryRoot, ".tmp", "shepherd-live-gate"); + if (volumeMode) { + if (path.resolve(stateRoot!) !== stateRoot || path.resolve(requestedRoot!) !== requestedRoot) { + throw new Error("Live Runtime volume paths must be absolute canonical paths"); + } + if (!isStrictChild(stateRoot!, liveRoot)) { + throw new Error("SHEPHERD_LIVE_GATE_ROOT must be inside CONTAINER_STATE_ROOT"); + } + } + + return { + liveRoot, + dataDirectory: path.join(liveRoot, "data"), + workspaceRoot: path.join(liveRoot, "agent-workspaces"), + codexHome: path.join(liveRoot, "shared-codex-home"), + shepherdRoot: path.join(liveRoot, "managed"), + shepherdCodexHomeRoot: path.join(liveRoot, "data", "shepherd-codex-homes"), + containerStateRoot: volumeMode ? stateRoot : undefined, + containerStateVolume: volumeMode ? stateVolume : undefined, + }; +} diff --git a/apps/server/src/shepherd/live-runtime.integration.test.ts b/apps/server/src/shepherd/live-runtime.integration.test.ts index 91a5609..c122094 100644 --- a/apps/server/src/shepherd/live-runtime.integration.test.ts +++ b/apps/server/src/shepherd/live-runtime.integration.test.ts @@ -31,6 +31,7 @@ import type { } from "../types.js"; import { BEARER_TRANSPORT, COOKIE_TRANSPORT } from "./auth-fixture.js"; import { CodexShepherdExecutor } from "./codex-executor.js"; +import { resolveLiveRuntimeLayout } from "./live-runtime-gate.js"; import type { ShepherdMissionDetail } from "./service.js"; import { AUTH_BACKEND_PROFILE_ID, @@ -42,7 +43,8 @@ import { ContainerVerifier, TrustedCheckRegistry } from "./verifier.js"; const execFileAsync = promisify(execFile); const repositoryRoot = fileURLToPath(new URL("../../../../", import.meta.url)); -const liveRoot = path.join(repositoryRoot, ".tmp", "shepherd-live-gate"); +const liveLayout = resolveLiveRuntimeLayout(process.env, repositoryRoot); +const liveRoot = liveLayout.liveRoot; const liveRootSentinel = path.join(liveRoot, ".live-gate-root"); const liveRootSentinelValue = "shepherd live Runtime gate only\n"; const liveEnabled = process.env.SHEPHERD_LIVE_TEST === "true"; @@ -122,9 +124,15 @@ async function entryExists(target: string): Promise { } async function prepareLiveRoot(): Promise { - const expectedRelative = path.join(".tmp", "shepherd-live-gate"); - if (path.relative(repositoryRoot, liveRoot) !== expectedRelative) { - throw new Error("Live gate root is not the exact repository-local target"); + if (liveLayout.containerStateRoot) { + if (!isStrictChild(liveLayout.containerStateRoot, liveRoot)) { + throw new Error("Live gate root escaped the wrapper-provided state volume"); + } + } else { + const expectedRelative = path.join(".tmp", "shepherd-live-gate"); + if (path.relative(repositoryRoot, liveRoot) !== expectedRelative) { + throw new Error("Live gate root is not the exact repository-local target"); + } } if (await entryExists(liveRoot)) { const metadata = await lstat(liveRoot); @@ -135,7 +143,8 @@ async function prepareLiveRoot(): Promise { throw new Error("Live gate root sentinel mismatch"); } const canonicalRoot = await realpath(liveRoot); - if (!isStrictChild(repositoryRoot, canonicalRoot)) { + const cleanupRoot = liveLayout.containerStateRoot ?? repositoryRoot; + if (!isStrictChild(cleanupRoot, canonicalRoot)) { throw new Error("Live gate cleanup target escaped the repository"); } await rm(canonicalRoot, { recursive: true, force: true }); @@ -157,15 +166,12 @@ function liveConfig(): AppConfig { return loadConfig({ ...process.env, HOST: "127.0.0.1", - APP_DATA_DIR: path.join(liveRoot, "data"), - AGENT_WORKSPACE_ROOT: path.join(liveRoot, "agent-workspaces"), - CODEX_HOME: path.join(liveRoot, "shared-codex-home"), - SHEPHERD_ROOT: path.join(liveRoot, "managed"), - SHEPHERD_CODEX_HOME_ROOT: path.join( - liveRoot, - "data", - "shepherd-codex-homes", - ), + PUBLIC_BIND_ADDR: "127.0.0.1", + APP_DATA_DIR: liveLayout.dataDirectory, + AGENT_WORKSPACE_ROOT: liveLayout.workspaceRoot, + CODEX_HOME: liveLayout.codexHome, + SHEPHERD_ROOT: liveLayout.shepherdRoot, + SHEPHERD_CODEX_HOME_ROOT: liveLayout.shepherdCodexHomeRoot, SHEPHERD_EXECUTION_MODE: "live", SHEPHERD_DEMO_MODE: "true", RUNTIME_PROVIDER: "container", @@ -179,10 +185,8 @@ function liveConfig(): AppConfig { CONTAINER_MEMORY_LIMIT: "1g", CONTAINER_PIDS_LIMIT: "128", CONTAINER_USER: `${uid}:${gid}`, - // This gate pins its own roots outside any state volume, so the state-volume - // settings a `.env` may carry must not reach it through the spread above. - CONTAINER_STATE_ROOT: undefined, - CONTAINER_STATE_VOLUME: undefined, + CONTAINER_STATE_ROOT: liveLayout.containerStateRoot, + CONTAINER_STATE_VOLUME: liveLayout.containerStateVolume, NODE_ENV: "test", }); } diff --git a/apps/server/src/shepherd/service.ts b/apps/server/src/shepherd/service.ts index a4b4573..166af6e 100644 --- a/apps/server/src/shepherd/service.ts +++ b/apps/server/src/shepherd/service.ts @@ -1,6 +1,7 @@ import { createHash, randomUUID } from "node:crypto"; import { lstat, mkdir, readFile, realpath, rm } from "node:fs/promises"; import path from "node:path"; +import { appendWithinDurableCapacity } from "../collection-capacity.js"; import { appendProjectGroupMessage, appendShepherdEvent, @@ -716,7 +717,9 @@ interface CandidateWork { function replaceById(collection: T[], value: T): void { const index = collection.findIndex((item) => item.id === value.id); - if (index === -1) collection.push(structuredClone(value)); + if (index === -1) { + appendWithinDurableCapacity(collection, [structuredClone(value)], "Shepherd record"); + } else collection[index] = structuredClone(value); } @@ -1918,7 +1921,7 @@ export class ShepherdService { } return; } - database.shepherd.projects.push({ + appendWithinDurableCapacity(database.shepherd.projects, [{ id: project.projectId, displayName: "Authentication collision demo", repositoryPath: project.repositoryPath, @@ -1927,7 +1930,7 @@ export class ShepherdService { activeMissionId: null, createdAt: timestamp, updatedAt: timestamp, - }); + }], "Shepherd project"); } async submitPrivateContractPrompt( @@ -3395,8 +3398,8 @@ export class ShepherdService { verifiedAt: null, completedAt: null, }; - database.shepherd.missions.push(mission); - database.shepherd.contracts.push(contract); + appendWithinDurableCapacity(database.shepherd.missions, [mission], "Shepherd Mission"); + appendWithinDurableCapacity(database.shepherd.contracts, [contract], "Execution Contract"); for (const message of draftMessages) { message.missionId = missionId; message.contractId = contractId; @@ -4328,7 +4331,7 @@ export class ShepherdService { startedAt: null, completedAt: null, }; - database.shepherd.missions.push(mission); + appendWithinDurableCapacity(database.shepherd.missions, [mission], "Shepherd Mission"); const contracts = [ this.makeContract({ id: frontendContractId, @@ -4353,7 +4356,7 @@ export class ShepherdService { createdAt, }), ]; - database.shepherd.contracts.push(...contracts); + appendWithinDurableCapacity(database.shepherd.contracts, contracts, "Execution Contract"); if (contractPromptRecords) { for (const record of [ contractPromptRecords.frontend, @@ -4693,7 +4696,7 @@ export class ShepherdService { plane.generalPromotionState = "promoting"; plane.generalPromotionEvidence = structuredClone(evidence); if (!plane.verificationEvidenceIds.includes(evidence.id)) { - plane.verificationEvidenceIds.push(evidence.id); + appendWithinDurableCapacity(plane.verificationEvidenceIds, [evidence.id], "Plane evidence reference"); } plane.updatedAt = promotingAt; }); @@ -5275,7 +5278,7 @@ export class ShepherdService { try { await this.store.mutate((database) => { this.assertMissionRunnable(database, prepared.missionId); - database.shepherd.planes.push(plane); + appendWithinDurableCapacity(database.shepherd.planes, [plane], "Plane"); const persisted = database.shepherd.contracts.find( (item) => item.id === contractId, ); @@ -6013,7 +6016,7 @@ export class ShepherdService { ); if (persistedPlane) { persistedPlane.state = "verified"; - persistedPlane.verificationEvidenceIds.push(evidence.id); + appendWithinDurableCapacity(persistedPlane.verificationEvidenceIds, [evidence.id], "Plane evidence reference"); persistedPlane.updatedAt = verifiedAt; } const persistedContract = database.shepherd.contracts.find( @@ -6223,7 +6226,7 @@ export class ShepherdService { try { await this.store.mutate((database) => { this.assertMissionRunnable(database, prepared.missionId); - database.shepherd.planes.push(integration); + appendWithinDurableCapacity(database.shepherd.planes, [integration], "Plane"); }); } catch (error) { await prepared.planeManager.destroyPlane(integration).catch(() => undefined); @@ -6354,13 +6357,13 @@ export class ShepherdService { const collisionAt = this.timestamp(); await this.store.mutate((database) => { this.assertMissionRunnable(database, missionId); - database.shepherd.collisions.push(collision); + appendWithinDurableCapacity(database.shepherd.collisions, [collision], "semantic collision"); const mission = database.shepherd.missions.find( (item) => item.id === missionId, ); if (!mission) throw new Error("Mission disappeared during collision detection"); - mission.collisionIds.push(collision.id); - mission.resolutionIds.push(collision.id); + appendWithinDurableCapacity(mission.collisionIds, [collision.id], "Mission collision reference"); + appendWithinDurableCapacity(mission.resolutionIds, [collision.id], "Mission resolution reference"); const persistedIntegration = database.shepherd.planes.find( (item) => item.id === integrationPlane.id, ); @@ -6621,13 +6624,13 @@ export class ShepherdService { try { await this.store.mutate((database) => { this.assertMissionRunnable(database, prepared.missionId); - database.shepherd.planes.push(plane); - database.shepherd.candidates.push(candidate); + appendWithinDurableCapacity(database.shepherd.planes, [plane], "Plane"); + appendWithinDurableCapacity(database.shepherd.candidates, [candidate], "resolution candidate"); const persistedCollision = database.shepherd.collisions.find( (item) => item.id === collision.id, ); if (!persistedCollision) throw new Error("Collision disappeared"); - persistedCollision.candidateIds.push(candidate.id); + appendWithinDurableCapacity(persistedCollision.candidateIds, [candidate.id], "collision candidate reference"); persistedCollision.updatedAt = createdAt; this.recordEvent(database, { type: "candidate_created", @@ -6886,7 +6889,7 @@ export class ShepherdService { candidate.failure = failure; candidate.updatedAt = completedAt; persistedPlane.state = candidatePassed ? "verified" : "failed"; - persistedPlane.verificationEvidenceIds.push(evidence.id); + appendWithinDurableCapacity(persistedPlane.verificationEvidenceIds, [evidence.id], "Plane evidence reference"); persistedPlane.updatedAt = completedAt; this.recordEvent(database, { type: candidatePassed ? "candidate_passed" : "candidate_failed", @@ -7054,7 +7057,7 @@ export class ShepherdService { persisted.retryCount = 1; persisted.failure = null; persisted.updatedAt = retryAt; - database.shepherd.planes.push(retryPlane); + appendWithinDurableCapacity(database.shepherd.planes, [retryPlane], "Plane"); this.recordEvent(database, { type: "candidate_retried", summary: "Started the candidate's single transient retry", @@ -7215,8 +7218,10 @@ export class ShepherdService { promotion.verificationEvidence.id, ) ) { - persistedPlane.verificationEvidenceIds.push( - promotion.verificationEvidence.id, + appendWithinDurableCapacity( + persistedPlane.verificationEvidenceIds, + [promotion.verificationEvidence.id], + "Plane evidence reference", ); persistedPlane.updatedAt = failedAt; } @@ -7286,8 +7291,10 @@ export class ShepherdService { promotion.verificationEvidence.id, ) ) { - persistedPlane.verificationEvidenceIds.push( - promotion.verificationEvidence.id, + appendWithinDurableCapacity( + persistedPlane.verificationEvidenceIds, + [promotion.verificationEvidence.id], + "Plane evidence reference", ); } persistedPlane.state = "verified"; @@ -7430,7 +7437,7 @@ export class ShepherdService { persistedCandidate.promotionEvidence = structuredClone(evidence); persistedCandidate.updatedAt = promotingAt; if (!persistedPlane.verificationEvidenceIds.includes(evidence.id)) { - persistedPlane.verificationEvidenceIds.push(evidence.id); + appendWithinDurableCapacity(persistedPlane.verificationEvidenceIds, [evidence.id], "Plane evidence reference"); } persistedPlane.updatedAt = promotingAt; }); diff --git a/apps/server/vitest.config.ts b/apps/server/vitest.config.ts index a9666a7..92cfe5e 100644 --- a/apps/server/vitest.config.ts +++ b/apps/server/vitest.config.ts @@ -14,5 +14,25 @@ export default defineConfig({ test: { clearMocks: true, restoreMocks: true, + coverage: { + provider: "v8", + include: ["src/**/*.ts"], + exclude: [ + "src/**/*.test.ts", + "src/**/*.integration.test.ts", + "src/**/test-fixtures/**", + "src/shepherd/auth-fixture.ts", + ], + reporter: ["text-summary", "json"], + reportsDirectory: fileURLToPath( + new URL("../../.tmp/coverage/server/", import.meta.url), + ), + thresholds: { + statements: 80, + branches: 80, + functions: 80, + lines: 80, + }, + }, }, }); diff --git a/apps/web/src/pages/AgentFormPage.tsx b/apps/web/src/pages/AgentFormPage.tsx index 06fed8b..799903b 100644 --- a/apps/web/src/pages/AgentFormPage.tsx +++ b/apps/web/src/pages/AgentFormPage.tsx @@ -1,4 +1,5 @@ -import { useEffect, useMemo, useState, type FormEvent } from "react"; +import { useEffect, useMemo, useState, type Dispatch, type FormEvent, type SetStateAction } from "react"; +import { flushSync } from "react-dom"; import { api } from "../api"; import { Link, navigate } from "../router"; import type { @@ -57,22 +58,22 @@ function matchingPreset(agent: Agent, presets: AuthorityPresetDefinition[]): Aut ) ?? null; } -function AuthorityFields({ form, setForm }: { form: AgentFormState; setForm: (form: AgentFormState) => void }) { +function AuthorityFields({ form, setForm }: { form: AgentFormState; setForm: Dispatch> }) { return (