diff --git a/e2e/fixtures/fixture-tasks/.gitignore b/e2e/fixtures/fixture-tasks/.gitignore new file mode 100644 index 000000000..9a7c2abb6 --- /dev/null +++ b/e2e/fixtures/fixture-tasks/.gitignore @@ -0,0 +1,11 @@ +node_modules +.env* +.eve +.vercel +.next +.output +.nitro +dist +.DS_Store +*.tsbuildinfo +.env*.local diff --git a/e2e/fixtures/fixture-tasks/.vercelignore b/e2e/fixtures/fixture-tasks/.vercelignore new file mode 100644 index 000000000..357664320 --- /dev/null +++ b/e2e/fixtures/fixture-tasks/.vercelignore @@ -0,0 +1,6 @@ +node_modules +.eve +.next +.output +.nitro +dist diff --git a/e2e/fixtures/fixture-tasks/agent/agent.ts b/e2e/fixtures/fixture-tasks/agent/agent.ts new file mode 100644 index 000000000..7cf10a84d --- /dev/null +++ b/e2e/fixtures/fixture-tasks/agent/agent.ts @@ -0,0 +1,323 @@ +import { e2eAgentConfig } from "@eve-e2e/config"; +import { defineAgent } from "eve"; +import { + mockModel, + type MockModelRequest, + type MockModelResponse, + type MockModelToolResult, +} from "eve/evals"; + +const TASK_ID_PATTERN = /task_[a-z0-9]+/iu; + +function respond(request: MockModelRequest): MockModelResponse | string { + const message = request.lastUserMessage ?? ""; + if (message.includes("TASK-FANOUT-INTERACTIVE-CHECK")) return "TASK-FANOUT-INTERACTIVE-OK"; + if (message.includes("TASK-CANCEL-NOW")) return cancelWorkerTask(request); + if (message.includes("CHILD-TASK-EXCLUSIVITY-RACE")) return raceBusyWorker(request); + if (message.startsWith("CHILD-TASK-EXCLUSIVITY-LATER ")) { + return laterBusyWorker(request, message); + } + if (message.startsWith("Background task ")) { + // Scenarios that act on wake notifications route to their script; + // every other scenario acknowledges them without running tools. + // The exclusivity race delegates so a completion notification that + // coalesces into the RACE turn still runs the same-batch sends. + if (request.userMessages.includes("TASK-FAN-IN")) return fanInNotification(request); + if (request.userMessages.includes("CHILD-TASK-EXCLUSIVITY-RACE")) { + return raceBusyWorker(request); + } + return "TASK-NOTIFICATION-ACK"; + } + + if (message === "TASK-FANOUT-PARENT-UPDATES") return fanoutTasks(request); + if (message === "TASK-FAN-IN") return fanInTasks(request); + if (message === "TASK-CANCEL-SETUP") return setupCancelWorker(request); + if (message.startsWith("TASK-CANCEL-VERIFY ")) { + return peekTask(request, "task-cancel-verify", "TASK-CANCEL-STATUS", message); + } + + if (message.startsWith("TASK-HITL-VERIFY ")) { + return peekTask(request, "task-hitl-verify", "TASK-HITL-STATUS", message); + } + if (message.startsWith("TASK-INPUT-BATCH-VERIFY ")) { + return peekTask(request, "task-input-batch-verify", "TASK-INPUT-BATCH-STATUS", message); + } + if (message.startsWith("CHILD-TASK-EXCLUSIVITY-VERIFY ")) { + return peekTask( + request, + "child-task-exclusivity-verify", + "CHILD-TASK-EXCLUSIVITY-STATUS", + message, + ); + } + if (message === "TASK-HITL-ROUTING") { + return startApprovalWorker(request, "task-hitl-worker", "TASK-HITL-STARTED"); + } + if (message === "TASK-INPUT-BATCH-ORDERING") { + return startApprovalWorker(request, "task-input-batch-worker", "TASK-INPUT-BATCH-STARTED"); + } + if (message === "TASK-CONTINUATION-HITL-SETUP") { + return startApprovalWorker( + request, + "task-continuation-hitl-worker", + "TASK-CONTINUATION-HITL-READY", + ); + } + if (message.startsWith("TASK-CONTINUATION-HITL-SEND ")) { + return continueApprovalWorker(request, message); + } + if (message === "CHILD-TASK-EXCLUSIVITY-SETUP") return setupBusyWorker(request); + + return `Mock reply: ${message}`; +} + +function fanoutTasks(request: MockModelRequest): MockModelResponse | string { + const pending = Array.from({ length: 10 }, (_, index) => index + 1).filter( + (index) => resultById(request, `task-fanout-${index}`) === undefined, + ); + if (pending.length > 0) { + return { + toolCalls: pending.map((index) => ({ + id: `task-fanout-${index}`, + input: { message: `FANOUT-WORKER-${index}` }, + name: "fanout-worker", + })), + }; + } + return "TASK-FANOUT-STARTED"; +} + +const FAN_IN_CALL_IDS = ["task-fan-in-1", "task-fan-in-2"] as const; + +function fanInTasks(request: MockModelRequest): MockModelResponse | string { + const pending = FAN_IN_CALL_IDS.filter((id) => resultById(request, id) === undefined); + if (pending.length > 0) { + return { + toolCalls: pending.map((id) => ({ + id, + input: { message: `Run the release gate, then return ${id.toUpperCase()}.` }, + name: "fanout-worker", + })), + }; + } + return "TASK-FAN-IN-STARTED"; +} + +/** + * The deterministic join predicate: on every wake, peek every fan-in task + * and answer only when all of them are completed. This is the model-side + * join contract — the framework delivers one wake + * per ready transition and the model decides whether the state suffices. + */ +function fanInNotification(request: MockModelRequest): MockModelResponse | string { + const callId = `task-fan-in-check-${request.userMessageCount}`; + const checked = resultById(request, callId); + const taskIds = FAN_IN_CALL_IDS.map((id) => findTaskId(resultById(request, id)?.output)).filter( + (taskId): taskId is string => taskId !== undefined, + ); + if (taskIds.length !== FAN_IN_CALL_IDS.length) { + throw new Error("Fan-in notification arrived before both task receipts."); + } + if (checked === undefined) { + return { toolCalls: [{ id: callId, input: { taskIds }, name: "task_peek" }] }; + } + return allTasksCompleted(checked.output, taskIds) + ? "TASK-FAN-IN-COMPLETE" + : "TASK-FAN-IN-WAITING"; +} + +function allTasksCompleted(output: unknown, expectedTaskIds: readonly string[]): boolean { + if (output === null || typeof output !== "object") return false; + const tasks = Reflect.get(output, "tasks"); + if (!Array.isArray(tasks) || tasks.length !== expectedTaskIds.length) return false; + const byId = new Map( + tasks + .filter((task) => task !== null && typeof task === "object") + .map((task) => [Reflect.get(task, "taskId"), task] as const), + ); + return expectedTaskIds.every( + (taskId) => Reflect.get(byId.get(taskId) ?? {}, "status") === "completed", + ); +} + +function setupCancelWorker(request: MockModelRequest): MockModelResponse | string { + if (resultById(request, "task-cancel-worker") === undefined) { + return { + toolCalls: [ + { + id: "task-cancel-worker", + input: { message: "Run the release gate, then return CANCEL-WORKER-DONE." }, + name: "fanout-worker", + }, + ], + }; + } + return "TASK-CANCEL-READY"; +} + +function cancelWorkerTask(request: MockModelRequest): MockModelResponse | string { + const callId = `task-cancel-call-${request.userMessageCount}`; + if (resultById(request, callId) === undefined) { + const taskId = findTaskId(resultById(request, "task-cancel-worker")?.output); + if (taskId === undefined) throw new Error("Cancel scenario has no initial task id."); + return { toolCalls: [{ id: callId, input: { taskIds: [taskId] }, name: "task_cancel" }] }; + } + return "TASK-CANCEL-DONE"; +} + +function startApprovalWorker( + request: MockModelRequest, + callId: string, + completedText: string, +): MockModelResponse | string { + if (resultById(request, callId) === undefined) { + return { + toolCalls: [ + { + id: callId, + input: { + message: + callId === "task-hitl-worker" + ? "Run three approval gates in order, then return CHILD-GATES-COMPLETE." + : "Run both approval gates in order, then return CHILD-GATES-COMPLETE.", + }, + name: "approval-worker", + }, + ], + }; + } + return completedText; +} + +function peekTask( + request: MockModelRequest, + callIdPrefix: string, + completedText: string, + message: string, +): MockModelResponse | string { + const callId = `${callIdPrefix}-${request.userMessageCount}`; + if (resultById(request, callId) === undefined) { + const taskId = TASK_ID_PATTERN.exec(message)?.[0]; + if (taskId === undefined) throw new Error(`Verification message has no task id: ${message}`); + return { toolCalls: [{ id: callId, input: { taskIds: [taskId] }, name: "task_peek" }] }; + } + return completedText; +} + +function setupBusyWorker(request: MockModelRequest): MockModelResponse | string { + const delegated = resultById(request, "child-task-exclusivity-initial-worker"); + if (delegated === undefined) { + return { + toolCalls: [ + { + id: "child-task-exclusivity-initial-worker", + input: { message: "Return BUSY-WORKER-INITIAL." }, + name: "busy-worker", + }, + ], + }; + } + + return "CHILD-TASK-EXCLUSIVITY-READY"; +} + +function raceBusyWorker(request: MockModelRequest): MockModelResponse | string { + const first = resultById(request, "child-task-exclusivity-send-a"); + const second = resultById(request, "child-task-exclusivity-send-b"); + if (first === undefined && second === undefined) { + const initial = resultById(request, "child-task-exclusivity-initial-worker"); + const taskId = findTaskId(initial?.output); + if (taskId === undefined) throw new Error("Busy-worker race has no initial task id."); + return { + toolCalls: [ + { + id: "child-task-exclusivity-send-a", + input: { message: "Return BUSY-WORKER-A.", taskId }, + name: "task_send", + }, + { + id: "child-task-exclusivity-send-b", + input: { message: "Return BUSY-WORKER-B.", taskId }, + name: "task_send", + }, + ], + }; + } + return "CHILD-TASK-EXCLUSIVITY-RACE-DONE"; +} + +function laterBusyWorker(request: MockModelRequest, message: string): MockModelResponse | string { + const result = resultById(request, "child-task-exclusivity-later"); + if (result === undefined) { + const taskId = findTaskId(message); + if (taskId === undefined) throw new Error("Later exclusivity send has no task id."); + return { + toolCalls: [ + { + id: "child-task-exclusivity-later", + input: { message: "Return BUSY-WORKER-LATER.", taskId }, + name: "task_send", + }, + ], + }; + } + return "CHILD-TASK-EXCLUSIVITY-LATER-DONE"; +} + +function continueApprovalWorker(request: MockModelRequest, message: string): MockModelResponse | string { + const callId = "task-continuation-hitl-send"; + const result = resultById(request, callId); + if (result === undefined) { + const taskId = findTaskId(message); + if (taskId === undefined) throw new Error("Continuation HITL send has no task id."); + return { + toolCalls: [ + { + id: callId, + input: { + message: + "Run four continuation approval gates in order, then return CHILD-GATES-COMPLETE.", + taskId, + }, + name: "task_send", + }, + ], + }; + } + return "TASK-CONTINUATION-HITL-STARTED"; +} + +function resultById(request: MockModelRequest, id: string): MockModelToolResult | undefined { + return request.toolResults.find((result) => result.id === id); +} + +function findTaskId(value: unknown): string | undefined { + if (typeof value === "string") return TASK_ID_PATTERN.exec(value)?.[0]; + if (Array.isArray(value)) { + for (const entry of value) { + const taskId = findTaskId(entry); + if (taskId !== undefined) return taskId; + } + return undefined; + } + if (value !== null && typeof value === "object") { + const taskId = Reflect.get(value, "taskId"); + if (typeof taskId === "string") return taskId; + for (const entry of Object.values(value)) { + const nested = findTaskId(entry); + if (nested !== undefined) return nested; + } + } + return undefined; +} + +const base = e2eAgentConfig(); + +export default defineAgent({ + ...base, + experimental: { ...base.experimental, tasks: true }, + // These evals target orchestration, not model planning. Keep every suite + // deterministic while retaining the workflow-world override from `base`. + model: mockModel(respond), + modelContextWindowTokens: 1_000_000, +}); diff --git a/e2e/fixtures/fixture-tasks/agent/channels/eve.ts b/e2e/fixtures/fixture-tasks/agent/channels/eve.ts new file mode 100644 index 000000000..6db5dd635 --- /dev/null +++ b/e2e/fixtures/fixture-tasks/agent/channels/eve.ts @@ -0,0 +1,27 @@ +import type { AuthFn } from "eve/channels/auth"; +import { eveChannel } from "eve/channels/eve"; +import type { SessionAuthContext } from "eve/context"; + +const PRINCIPAL_A = "Bearer e2e-task-operation-a"; +const PRINCIPAL_B = "Bearer e2e-task-operation-b"; + +function principal(principalId: string): SessionAuthContext { + return { + attributes: {}, + authenticator: "e2e-task-fixture", + issuer: "e2e", + principalId, + principalType: "user", + subject: principalId, + }; +} + +const authenticateA: AuthFn = (request) => + request.headers.get("authorization") === PRINCIPAL_A ? principal("operation-user-a") : null; + +const authenticateB: AuthFn = (request) => + request.headers.get("authorization") === PRINCIPAL_B ? principal("operation-user-b") : null; + +const authenticateEvalDriver: AuthFn = () => principal("eval-driver"); + +export default eveChannel({ auth: [authenticateA, authenticateB, authenticateEvalDriver] }); diff --git a/e2e/fixtures/fixture-tasks/agent/instructions.md b/e2e/fixtures/fixture-tasks/agent/instructions.md new file mode 100644 index 000000000..f3275e7f0 --- /dev/null +++ b/e2e/fixtures/fixture-tasks/agent/instructions.md @@ -0,0 +1 @@ +Follow the scripted model's task-control calls exactly. This fixture validates durable background-task orchestration, not model planning. diff --git a/e2e/fixtures/fixture-tasks/agent/subagents/approval-worker/agent.ts b/e2e/fixtures/fixture-tasks/agent/subagents/approval-worker/agent.ts new file mode 100644 index 000000000..43f908acb --- /dev/null +++ b/e2e/fixtures/fixture-tasks/agent/subagents/approval-worker/agent.ts @@ -0,0 +1,49 @@ +import { defineAgent } from "eve"; +import { mockModel, type MockModelRequest, type MockModelResponse } from "eve/evals"; + +function respond(request: MockModelRequest): MockModelResponse | string { + const continuation = request.userMessages.some((message) => + message.includes("continuation approval gates"), + ); + const threeGates = + continuation || request.userMessages.some((message) => message.includes("three approval gates")); + const id = (gate: string) => (continuation ? `continuation-${gate}` : `approval-${gate}`); + const first = request.toolResults.find((result) => result.id === id("first")); + if (first === undefined) { + return { + toolCalls: [{ id: id("first"), input: { marker: "FIRST" }, name: "first_gate" }], + }; + } + + const second = request.toolResults.find((result) => result.id === id("second")); + if (second === undefined) { + return { + toolCalls: [{ id: id("second"), input: { marker: "SECOND" }, name: "second_gate" }], + }; + } + + if (threeGates) { + const third = request.toolResults.find((result) => result.id === id("third")); + if (third === undefined) { + return { + toolCalls: [{ id: id("third"), input: { marker: "THIRD" }, name: "third_gate" }], + }; + } + if (continuation) { + const fourth = request.toolResults.find((result) => result.id === id("fourth")); + if (fourth === undefined) { + return { + toolCalls: [{ id: id("fourth"), input: { marker: "FOURTH" }, name: "fourth_gate" }], + }; + } + } + } + + return "CHILD-GATES-COMPLETE"; +} + +export default defineAgent({ + description: "Run deterministic approval gates in order.", + model: mockModel(respond), + modelContextWindowTokens: 1_000_000, +}); diff --git a/e2e/fixtures/fixture-tasks/agent/subagents/approval-worker/tools/first_gate.ts b/e2e/fixtures/fixture-tasks/agent/subagents/approval-worker/tools/first_gate.ts new file mode 100644 index 000000000..8c2afeb0f --- /dev/null +++ b/e2e/fixtures/fixture-tasks/agent/subagents/approval-worker/tools/first_gate.ts @@ -0,0 +1,10 @@ +import { defineTool } from "eve/tools"; +import { once } from "eve/tools/approval"; +import { z } from "zod"; + +export default defineTool({ + description: "First deterministic approval gate.", + inputSchema: z.object({ marker: z.literal("FIRST") }), + approval: once(), + execute: async ({ marker }) => ({ marker, passed: true }), +}); diff --git a/e2e/fixtures/fixture-tasks/agent/subagents/approval-worker/tools/fourth_gate.ts b/e2e/fixtures/fixture-tasks/agent/subagents/approval-worker/tools/fourth_gate.ts new file mode 100644 index 000000000..ec285ecf6 --- /dev/null +++ b/e2e/fixtures/fixture-tasks/agent/subagents/approval-worker/tools/fourth_gate.ts @@ -0,0 +1,10 @@ +import { defineTool } from "eve/tools"; +import { once } from "eve/tools/approval"; +import { z } from "zod"; + +export default defineTool({ + description: "Fourth deterministic approval gate for continued-task HITL routing.", + inputSchema: z.object({ marker: z.literal("FOURTH") }), + approval: once(), + execute: async ({ marker }) => ({ marker, passed: true }), +}); diff --git a/e2e/fixtures/fixture-tasks/agent/subagents/approval-worker/tools/second_gate.ts b/e2e/fixtures/fixture-tasks/agent/subagents/approval-worker/tools/second_gate.ts new file mode 100644 index 000000000..a3ec99c56 --- /dev/null +++ b/e2e/fixtures/fixture-tasks/agent/subagents/approval-worker/tools/second_gate.ts @@ -0,0 +1,10 @@ +import { defineTool } from "eve/tools"; +import { once } from "eve/tools/approval"; +import { z } from "zod"; + +export default defineTool({ + description: "Second deterministic approval gate.", + inputSchema: z.object({ marker: z.literal("SECOND") }), + approval: once(), + execute: async ({ marker }) => ({ marker, passed: true }), +}); diff --git a/e2e/fixtures/fixture-tasks/agent/subagents/approval-worker/tools/third_gate.ts b/e2e/fixtures/fixture-tasks/agent/subagents/approval-worker/tools/third_gate.ts new file mode 100644 index 000000000..6977b743a --- /dev/null +++ b/e2e/fixtures/fixture-tasks/agent/subagents/approval-worker/tools/third_gate.ts @@ -0,0 +1,10 @@ +import { defineTool } from "eve/tools"; +import { once } from "eve/tools/approval"; +import { z } from "zod"; + +export default defineTool({ + description: "Third deterministic approval gate for repeated HITL routing.", + inputSchema: z.object({ marker: z.literal("THIRD") }), + approval: once(), + execute: async ({ marker }) => ({ marker, passed: true }), +}); diff --git a/e2e/fixtures/fixture-tasks/agent/subagents/busy-worker/agent.ts b/e2e/fixtures/fixture-tasks/agent/subagents/busy-worker/agent.ts new file mode 100644 index 000000000..28d2dd9c0 --- /dev/null +++ b/e2e/fixtures/fixture-tasks/agent/subagents/busy-worker/agent.ts @@ -0,0 +1,18 @@ +import { defineAgent } from "eve"; +import { mockModel } from "eve/evals"; + +export default defineAgent({ + description: "Return one deterministic marker for each message.", + model: mockModel((request) => { + const message = request.lastUserMessage ?? ""; + if (message.includes("BUSY-WORKER-A") || message.includes("BUSY-WORKER-B")) { + if (!request.toolResults.some((result) => result.id === "exclusivity-hold")) { + return { + toolCalls: [{ id: "exclusivity-hold", input: { marker: "HOLD" }, name: "hold" }], + }; + } + } + return `BUSY-WORKER:${message}`; + }), + modelContextWindowTokens: 1_000_000, +}); diff --git a/e2e/fixtures/fixture-tasks/agent/subagents/busy-worker/tools/hold.ts b/e2e/fixtures/fixture-tasks/agent/subagents/busy-worker/tools/hold.ts new file mode 100644 index 000000000..d843ccc01 --- /dev/null +++ b/e2e/fixtures/fixture-tasks/agent/subagents/busy-worker/tools/hold.ts @@ -0,0 +1,11 @@ +import { defineTool } from "eve/tools"; +import { z } from "zod"; + +export default defineTool({ + description: "Keep an admitted continuation nonterminal long enough for a later-turn check.", + inputSchema: z.object({ marker: z.literal("HOLD") }), + execute: async ({ marker }) => { + await new Promise((resolve) => setTimeout(resolve, 5_000)); + return { marker, released: true }; + }, +}); diff --git a/e2e/fixtures/fixture-tasks/agent/subagents/fanout-worker/agent.ts b/e2e/fixtures/fixture-tasks/agent/subagents/fanout-worker/agent.ts new file mode 100644 index 000000000..ea7c4d1fc --- /dev/null +++ b/e2e/fixtures/fixture-tasks/agent/subagents/fanout-worker/agent.ts @@ -0,0 +1,16 @@ +import { defineAgent } from "eve"; +import { mockModel, type MockModelRequest, type MockModelResponse } from "eve/evals"; + +function respond(request: MockModelRequest): MockModelResponse | string { + const released = request.toolResults.find((result) => result.name === "release"); + if (released === undefined) { + return { toolCalls: [{ input: { marker: "RELEASE" }, name: "release" }] }; + } + return `FANOUT-COMPLETE:${request.lastUserMessage ?? ""}`; +} + +export default defineAgent({ + description: "Complete one fanout task with its deterministic marker.", + model: mockModel(respond), + modelContextWindowTokens: 1_000_000, +}); diff --git a/e2e/fixtures/fixture-tasks/agent/subagents/fanout-worker/tools/release.ts b/e2e/fixtures/fixture-tasks/agent/subagents/fanout-worker/tools/release.ts new file mode 100644 index 000000000..90d3c4070 --- /dev/null +++ b/e2e/fixtures/fixture-tasks/agent/subagents/fanout-worker/tools/release.ts @@ -0,0 +1,10 @@ +import { defineTool } from "eve/tools"; +import { once } from "eve/tools/approval"; +import { z } from "zod"; + +export default defineTool({ + description: "Release one fanout task after its parent remains interactive.", + inputSchema: z.object({ marker: z.literal("RELEASE") }), + approval: once(), + execute: async ({ marker }) => ({ marker, released: true }), +}); diff --git a/e2e/fixtures/fixture-tasks/evals/child-task-exclusivity.eval.ts b/e2e/fixtures/fixture-tasks/evals/child-task-exclusivity.eval.ts new file mode 100644 index 000000000..b66fcd821 --- /dev/null +++ b/e2e/fixtures/fixture-tasks/evals/child-task-exclusivity.eval.ts @@ -0,0 +1,91 @@ +import { defineEval, type EveEvalToolCall } from "eve/evals"; +import { satisfies } from "eve/evals/expect"; + +import { + requireBackgroundTaskId, + sendAndFollowQueuedTurn, + waitForCompletedTask, +} from "./shared.js"; + +/** One persistent child session admits at most one nonterminal task. */ +export default defineEval({ + description: + "Two same-batch task_send calls to one child admit one task and reject the other as AGENT_BUSY.", + async test(t) { + t.log("starting initial busy-worker task"); + const setup = await t.send("CHILD-TASK-EXCLUSIVITY-SETUP"); + t.log("initial busy-worker task settled"); + setup.expectOk(); + setup.messageIncludes("CHILD-TASK-EXCLUSIVITY-READY"); + const initialTaskId = requireBackgroundTaskId(setup); + await waitForCompletedTask(t, t, "CHILD-TASK-EXCLUSIVITY-VERIFY", initialTaskId); + + t.log("sending two same-batch continuations"); + const race = await sendAndFollowQueuedTurn(t, "CHILD-TASK-EXCLUSIVITY-RACE"); + const raced = race.turn; + t.log("same-batch continuation turn settled"); + raced.expectOk(); + raced.messageIncludes("CHILD-TASK-EXCLUSIVITY-RACE-DONE"); + + const sends = raced.toolCalls.filter((call) => call.name === "task_send"); + await t.require( + sends, + satisfies((calls: readonly EveEvalToolCall[]) => { + const admitted = calls.find( + (call) => + call.output !== null && + typeof call.output === "object" && + typeof Reflect.get(call.output, "agentId") === "string" && + typeof Reflect.get(call.output, "taskId") === "string" && + Reflect.get(call.output, "status") === "working", + ); + const admittedTaskId = + admitted?.output !== null && typeof admitted?.output === "object" + ? Reflect.get(admitted.output, "taskId") + : undefined; + const rejected = calls.find((call) => { + if (call.output === null || typeof call.output !== "object") return false; + const keys = Object.keys(call.output); + const message = Reflect.get(call.output, "message"); + return ( + keys.length === 1 && + keys[0] === "message" && + typeof message === "string" && + message.startsWith("AGENT_BUSY") && + (typeof admittedTaskId !== "string" || message.includes(admittedTaskId)) + ); + }); + return calls.length === 2 && admitted !== undefined && rejected !== undefined; + }, "exactly one same-batch task_send is admitted"), + ); + + const admitted = sends.find( + (call) => + call.output !== null && + typeof call.output === "object" && + Reflect.get(call.output, "status") === "working", + ); + const admittedTaskId = + admitted?.output !== null && typeof admitted?.output === "object" + ? Reflect.get(admitted.output, "taskId") + : undefined; + if (typeof admittedTaskId !== "string") throw new Error("No admitted continuation task id."); + + const later = await sendAndFollowQueuedTurn( + t, + `CHILD-TASK-EXCLUSIVITY-LATER ${initialTaskId}`, + race.session, + ); + const laterSend = later.turn.toolCalls.find((call) => call.name === "task_send"); + await t.require( + laterSend?.output, + satisfies( + (output) => JSON.stringify(output).includes("AGENT_BUSY"), + "a later parent turn remains excluded while the admitted task is nonterminal", + ), + ); + + await t.sleep(5_000); + await waitForCompletedTask(t, later.session, "CHILD-TASK-EXCLUSIVITY-VERIFY", admittedTaskId); + }, +}); diff --git a/e2e/fixtures/fixture-tasks/evals/evals.config.ts b/e2e/fixtures/fixture-tasks/evals/evals.config.ts new file mode 100644 index 000000000..3a273e03d --- /dev/null +++ b/e2e/fixtures/fixture-tasks/evals/evals.config.ts @@ -0,0 +1,3 @@ +import { defineEvalConfig } from "eve/evals"; + +export default defineEvalConfig({ timeoutMs: 120_000 }); diff --git a/e2e/fixtures/fixture-tasks/evals/shared.ts b/e2e/fixtures/fixture-tasks/evals/shared.ts new file mode 100644 index 000000000..c22d7618d --- /dev/null +++ b/e2e/fixtures/fixture-tasks/evals/shared.ts @@ -0,0 +1,159 @@ +import type { EveEvalContext, EveEvalSession, EveEvalTurn, InputRequest } from "eve/evals"; +import { satisfies } from "eve/evals/expect"; + +export type TaskEvalSessionDriver = Pick< + EveEvalSession, + "pendingInputRequests" | "respond" | "send" | "sessionId" | "state" +>; + +export interface PendingTaskInput { + readonly request: InputRequest; + readonly session: TaskEvalSessionDriver; +} + +export interface FollowedQueuedTurn { + readonly observedTurns: readonly EveEvalTurn[]; + readonly session: TaskEvalSessionDriver; + readonly turn: EveEvalTurn; +} + +/** Waits across server-initiated parent turns for one task-owned input request. */ +export async function waitForTaskInput( + t: EveEvalContext, + initialSession: TaskEvalSessionDriver, + toolName: string, +): Promise { + let session = initialSession; + for (let attempt = 0; attempt < 5; attempt += 1) { + const pending = session.pendingInputRequests.find( + (request) => request.action.toolName === toolName, + ); + if (pending !== undefined) return { request: pending, session }; + + const sessionId = session.sessionId; + if (sessionId === undefined) throw new Error("Task input wait has no parent session id."); + const live = t.target.watchTurn(sessionId, { startIndex: session.state.streamIndex }); + await live.result(); + session = live.session; + } + throw new Error(`Task did not surface input for tool "${toolName}" after five turns.`); +} + +/** Reads the task receipt attached to a background `subagent.completed` event. */ +export function requireBackgroundTaskId(turn: EveEvalTurn): string { + for (const event of turn.events) { + if (event.type === "subagent.completed" && event.data.backgroundTask !== undefined) { + return event.data.backgroundTask.taskId; + } + } + throw new Error("Turn completed without a background task receipt."); +} + +/** Returns the exact model-visible task view for one owned task id. */ +export function requireTaskView(output: unknown, taskId: string): Record { + if (output === null || typeof output !== "object") { + throw new Error("task_peek did not return an object."); + } + const tasks = Reflect.get(output, "tasks"); + if (!Array.isArray(tasks)) throw new Error("task_peek did not return a tasks array."); + const matches = tasks.filter( + (candidate) => + candidate !== null && + typeof candidate === "object" && + Reflect.get(candidate, "taskId") === taskId, + ); + if (matches.length !== 1) { + throw new Error(`task_peek returned ${matches.length} views for ${taskId}.`); + } + return matches[0] as Record; +} + +/** Follows queued server turns until the requested user message owns a turn. */ +export async function sendAndFollowQueuedTurn( + t: EveEvalContext, + message: string, + initialSession: TaskEvalSessionDriver = t, +): Promise { + let session = initialSession; + let turn = await session.send(message); + const observedTurns = [turn]; + for (let attempt = 0; attempt < 20; attempt += 1) { + const received = turn.events.some( + (event) => + event.type === "message.received" && messageText(event.data.message).includes(message), + ); + if (received) return { observedTurns, session, turn }; + + const sessionId = session.sessionId; + if (sessionId === undefined) throw new Error("Queued turn follow-up has no session id."); + const live = t.target.watchTurn(sessionId, { startIndex: session.state.streamIndex }); + turn = await live.result(); + observedTurns.push(turn); + session = live.session; + } + throw new Error(`Queued message "${message}" was not received after 20 turns.`); +} + +/** Polls the non-blocking task view until the expected task is completed. */ +export async function waitForCompletedTask( + t: EveEvalContext, + session: TaskEvalSessionDriver, + verificationMessage: string, + taskId: string, +): Promise { + return await waitForTaskStatus(t, session, verificationMessage, taskId, "completed"); +} + +/** Polls the non-blocking task view until the expected task reaches `status`. */ +export async function waitForTaskStatus( + t: EveEvalContext, + session: TaskEvalSessionDriver, + verificationMessage: string, + taskId: string, + status: string, +): Promise { + for (let attempt = 0; attempt < 20; attempt += 1) { + const turn = await session.send(`${verificationMessage} ${taskId}`); + const peeked = turn.toolCalls.find((call) => call.name === "task_peek"); + if (taskStatus(peeked?.output, taskId) === status) { + await t.require( + peeked?.output, + satisfies( + (output) => taskStatus(output, taskId) === status, + `task_peek returns ${status} task ${taskId}`, + ), + ); + return turn; + } + await t.sleep(100); + } + throw new Error(`Task ${taskId} did not reach "${status}" after 20 task_peek attempts.`); +} + +function messageText(message: unknown): string { + if (typeof message === "string") return message; + if (!Array.isArray(message)) return ""; + return message + .flatMap((part) => + part !== null && + typeof part === "object" && + Reflect.get(part, "type") === "text" && + typeof Reflect.get(part, "text") === "string" + ? [Reflect.get(part, "text") as string] + : [], + ) + .join("\n"); +} + +function taskStatus(output: unknown, taskId: string): unknown { + if (output === null || typeof output !== "object") return undefined; + const tasks = Reflect.get(output, "tasks"); + if (!Array.isArray(tasks)) return undefined; + const task = tasks.find( + (candidate) => + candidate !== null && + typeof candidate === "object" && + Reflect.get(candidate, "taskId") === taskId, + ); + return task === undefined ? undefined : Reflect.get(task, "status"); +} diff --git a/e2e/fixtures/fixture-tasks/evals/task-cancellation.eval.ts b/e2e/fixtures/fixture-tasks/evals/task-cancellation.eval.ts new file mode 100644 index 000000000..6ae721524 --- /dev/null +++ b/e2e/fixtures/fixture-tasks/evals/task-cancellation.eval.ts @@ -0,0 +1,99 @@ +import { defineEval, type EveEvalContext, type EveEvalTurn } from "eve/evals"; +import { satisfies } from "eve/evals/expect"; + +import { + requireBackgroundTaskId, + requireTaskView, + sendAndFollowQueuedTurn, + waitForTaskInput, + waitForTaskStatus, +} from "./shared.js"; + +/** + * Cancellation finality: `task_cancel` commits `cancelled` on a + * nonterminal task, the snapshot stays readable, and repeating the + * cancel is an idempotent no-op — a terminal state can never change. + */ +export default defineEval({ + description: "task_cancel commits a final cancelled state and repeated cancels are no-ops.", + async test(t) { + const started = await t.send("TASK-CANCEL-SETUP"); + started.expectOk(); + started.messageIncludes("TASK-CANCEL-READY"); + started.event("subagent.completed", { + count: 1, + data: { backgroundTask: { status: "working" }, subagentName: "fanout-worker" }, + }); + const taskId = requireBackgroundTaskId(started); + + // The worker blocks on its release gate, so the task is durably + // nonterminal when the cancel lands. + const blocked = await waitForTaskInput(t, t, "release"); + + const cancelled = await sendAndFollowQueuedTurn(t, "TASK-CANCEL-NOW", blocked.session); + cancelled.turn.expectOk(); + cancelled.turn.messageIncludes("TASK-CANCEL-DONE"); + cancelled.turn.calledTool("task_cancel", { input: { taskIds: [taskId] } }); + const cancelledCall = cancelled.turn.toolCalls.find((call) => call.name === "task_cancel"); + const cancelledView = requireTaskView(cancelledCall?.output, taskId); + + const verified = await waitForTaskStatus( + t, + cancelled.session, + "TASK-CANCEL-VERIFY", + taskId, + "cancelled", + ); + verified.expectOk(); + verified.messageIncludes("TASK-CANCEL-STATUS"); + await requireSameView(t, verified, taskId, cancelledView, "peek after cancellation"); + + // Cancelling an already-cancelled task changes nothing and is not an + // error: the tool returns the same terminal view. + const repeated = await sendAndFollowQueuedTurn(t, "TASK-CANCEL-NOW", cancelled.session); + repeated.turn.expectOk(); + repeated.turn.messageIncludes("TASK-CANCEL-DONE"); + repeated.turn.calledTool("task_cancel", { input: { taskIds: [taskId] } }); + const repeatedCall = repeated.turn.toolCalls.find((call) => call.name === "task_cancel"); + await t.require( + requireTaskView(repeatedCall?.output, taskId), + satisfies( + (view) => JSON.stringify(view) === JSON.stringify(cancelledView), + "repeated cancellation returns the identical model-visible view", + ), + ); + + const still = await waitForTaskStatus( + t, + repeated.session, + "TASK-CANCEL-VERIFY", + taskId, + "cancelled", + ); + still.expectOk(); + await requireSameView( + t, + still, + taskId, + cancelledView, + "later peek after repeated cancellation", + ); + }, +}); + +async function requireSameView( + t: EveEvalContext, + turn: EveEvalTurn, + taskId: string, + expected: Record, + description: string, +): Promise { + const peeked = turn.toolCalls.find((call) => call.name === "task_peek"); + await t.require( + requireTaskView(peeked?.output, taskId), + satisfies( + (view) => JSON.stringify(view) === JSON.stringify(expected), + `${description} is structurally identical`, + ), + ); +} diff --git a/e2e/fixtures/fixture-tasks/evals/task-continuation-hitl.eval.ts b/e2e/fixtures/fixture-tasks/evals/task-continuation-hitl.eval.ts new file mode 100644 index 000000000..44d1ce441 --- /dev/null +++ b/e2e/fixtures/fixture-tasks/evals/task-continuation-hitl.eval.ts @@ -0,0 +1,64 @@ +import { defineEval } from "eve/evals"; + +import { + requireBackgroundTaskId, + sendAndFollowQueuedTurn, + waitForCompletedTask, + waitForTaskInput, +} from "./shared.js"; + +/** A reused child must route every new-turn event through the new owning task. */ +export default defineEval({ + description: "task_send rebinds a reused child's HITL events to the new task lifecycle.", + async test(t) { + t.log("starting first task"); + const setup = await t.send("TASK-CONTINUATION-HITL-SETUP"); + setup.expectOk(); + const firstTaskId = requireBackgroundTaskId(setup); + + const first = await waitForTaskInput(t, t, "first_gate"); + t.log("answering first task gate one"); + const firstAnswered = await first.session.respond({ + optionId: "approve", + requestId: first.request.requestId, + }); + firstAnswered.notEvent("step.started"); + const second = await waitForTaskInput(t, first.session, "second_gate"); + t.log("answering first task gate two"); + await second.session.respond({ optionId: "approve", requestId: second.request.requestId }); + await waitForCompletedTask(t, second.session, "TASK-HITL-VERIFY", firstTaskId); + t.log("first task completed; sending continuation"); + + const continued = await sendAndFollowQueuedTurn( + t, + `TASK-CONTINUATION-HITL-SEND ${firstTaskId}`, + second.session, + ); + continued.turn.expectOk(); + const send = continued.turn.toolCalls.find((call) => call.name === "task_send"); + const secondTaskId = + send?.output !== null && typeof send?.output === "object" + ? Reflect.get(send.output, "taskId") + : undefined; + if (typeof secondTaskId !== "string") throw new Error("task_send returned no new task id."); + t.log("continuation admitted; waiting for rebound HITL"); + const continuedThird = await waitForTaskInput(t, continued.session, "third_gate"); + const continuedThirdAnswer = await continuedThird.session.respond({ + optionId: "approve", + requestId: continuedThird.request.requestId, + }); + continuedThirdAnswer.notEvent("step.started"); + const continuedFourth = await waitForTaskInput(t, continuedThird.session, "fourth_gate"); + await continuedFourth.session.respond({ + optionId: "approve", + requestId: continuedFourth.request.requestId, + }); + await waitForCompletedTask( + t, + continuedFourth.session, + "TASK-HITL-VERIFY", + secondTaskId, + ); + t.noFailedActions(); + }, +}); diff --git a/e2e/fixtures/fixture-tasks/evals/task-fan-in-join.eval.ts b/e2e/fixtures/fixture-tasks/evals/task-fan-in-join.eval.ts new file mode 100644 index 000000000..1f0fe3700 --- /dev/null +++ b/e2e/fixtures/fixture-tasks/evals/task-fan-in-join.eval.ts @@ -0,0 +1,201 @@ +import { defineEval, type EveEvalContext, type EveEvalTurn, type InputRequest } from "eve/evals"; +import { satisfies } from "eve/evals/expect"; + +import type { TaskEvalSessionDriver } from "./shared.js"; + +const FAN_IN_SIZE = 2; +const MAX_WAKE_TURNS = 8; + +/** + * The join contract: the framework delivers one + * wake per ready transition and the model decides sufficiency. On every + * wake the scripted model peeks both tasks; with one task released it must + * answer WAITING, and only after the second completes may it answer + * COMPLETE from a peek showing both tasks completed. + */ +export default defineEval({ + description: + "Two background tasks joined across completion wakes: peek on every wake, final answer only when all are completed.", + async test(t) { + const started = await t.send("TASK-FAN-IN"); + started.expectOk(); + started.messageIncludes("TASK-FAN-IN-STARTED"); + started.calledSubagent("fanout-worker", { count: FAN_IN_SIZE }); + + const taskIds = backgroundTaskIds(started); + await t.require( + taskIds, + satisfies( + (ids: readonly string[]) => ids.length === FAN_IN_SIZE && new Set(ids).size === FAN_IN_SIZE, + `${FAN_IN_SIZE} distinct background task receipts`, + ), + ); + + const blocked = await waitForReleaseRequests(t, t, started, FAN_IN_SIZE); + + // Release one task only: its completion wake must produce a peek that + // still sees the other task blocked, so the model holds the answer. + // The wake may coalesce into the respond turn or arrive on its own. + const [firstRequest, secondRequest] = blocked.requests; + if (firstRequest === undefined || secondRequest === undefined) { + throw new Error("Fan-in did not surface two release requests."); + } + const firstReleased = await blocked.session.respond({ + optionId: "approve", + requestId: firstRequest.requestId, + }); + firstReleased.expectOk(); + + const waiting = await waitForTurnMessage( + t, + blocked.session, + "TASK-FAN-IN-WAITING", + firstReleased, + ); + const waitingPeek = waiting.turn.toolCalls.find((call) => call.name === "task_peek"); + await requireExactPeek( + t, + waitingPeek, + taskIds, + ["completed", "input_required"], + "first completion keeps the other task blocked", + ); + for (const turn of waiting.observedTurns) { + await t.require( + turn.message ?? "", + satisfies( + (message) => !String(message).includes("TASK-FAN-IN-COMPLETE"), + "no final answer before every task completed", + ), + ); + } + + // Release the second task: the next peek sees both completed and the + // model may finally answer. + const secondReleased = await waiting.session.respond({ + optionId: "approve", + requestId: secondRequest.requestId, + }); + secondReleased.expectOk(); + + const complete = await waitForTurnMessage( + t, + waiting.session, + "TASK-FAN-IN-COMPLETE", + secondReleased, + ); + const completePeek = complete.turn.toolCalls.find((call) => call.name === "task_peek"); + await requireExactPeek( + t, + completePeek, + taskIds, + ["completed", "completed"], + "final join observes both exact tasks completed", + ); + t.noFailedActions(); + }, +}); + +async function requireExactPeek( + t: EveEvalContext, + call: EveEvalTurn["toolCalls"][number] | undefined, + taskIds: readonly string[], + statuses: readonly string[], + description: string, +): Promise { + await t.require( + { input: call?.input, output: call?.output }, + satisfies((subject: { readonly input: unknown; readonly output: unknown }) => { + const inputIds = Reflect.get(subject.input ?? {}, "taskIds"); + const tasks = Reflect.get(subject.output ?? {}, "tasks"); + if (!Array.isArray(inputIds) || !Array.isArray(tasks)) return false; + const expected = [...taskIds].sort(); + const outputIds = tasks.map((task) => Reflect.get(task, "taskId")).sort(); + const outputStatuses = tasks.map((task) => Reflect.get(task, "status")).sort(); + return ( + JSON.stringify([...inputIds].sort()) === JSON.stringify(expected) && + JSON.stringify(outputIds) === JSON.stringify(expected) && + JSON.stringify(outputStatuses) === JSON.stringify([...statuses].sort()) + ); + }, description), + ); +} + +interface BlockedFanIn { + readonly requests: readonly InputRequest[]; + readonly session: TaskEvalSessionDriver; +} + +async function waitForReleaseRequests( + t: EveEvalContext, + initialSession: TaskEvalSessionDriver, + initialTurn: EveEvalTurn, + expected: number, +): Promise { + const requests = new Map(); + let session = initialSession; + collectReleaseRequests(initialTurn.inputRequests, requests); + for (let attempt = 0; attempt < MAX_WAKE_TURNS && requests.size < expected; attempt += 1) { + const sessionId = session.sessionId; + if (sessionId === undefined) throw new Error("Fan-in has no parent session id."); + const live = t.target.watchTurn(sessionId, { startIndex: session.state.streamIndex }); + const turn = await live.result(); + collectReleaseRequests(turn.inputRequests, requests); + session = live.session; + } + if (requests.size !== expected) { + throw new Error(`Expected ${expected} release requests; received ${requests.size}.`); + } + return { requests: [...requests.values()], session }; +} + +function collectReleaseRequests( + inputRequests: readonly InputRequest[], + requests: Map, +): void { + for (const request of inputRequests) { + if (request.action.toolName === "release") requests.set(request.requestId, request); + } +} + +interface ObservedTurnMessage { + readonly observedTurns: readonly EveEvalTurn[]; + readonly session: TaskEvalSessionDriver; + readonly turn: EveEvalTurn; +} + +/** + * Finds the first turn whose reply carries `token`, starting with an + * already-consumed turn (a wake can coalesce into a respond turn) before + * watching for later server-initiated turns. + */ +async function waitForTurnMessage( + t: EveEvalContext, + initialSession: TaskEvalSessionDriver, + token: string, + initialTurn: EveEvalTurn, +): Promise { + const observedTurns: EveEvalTurn[] = [initialTurn]; + let session = initialSession; + if ((initialTurn.message ?? "").includes(token)) { + return { observedTurns, session, turn: initialTurn }; + } + for (let attempt = 0; attempt < MAX_WAKE_TURNS; attempt += 1) { + const sessionId = session.sessionId; + if (sessionId === undefined) throw new Error("Fan-in has no parent session id."); + const live = t.target.watchTurn(sessionId, { startIndex: session.state.streamIndex }); + const turn = await live.result(); + observedTurns.push(turn); + session = live.session; + if ((turn.message ?? "").includes(token)) return { observedTurns, session, turn }; + } + throw new Error(`No turn carried "${token}" after ${MAX_WAKE_TURNS} turns.`); +} + +function backgroundTaskIds(turn: EveEvalTurn): readonly string[] { + return turn.events.flatMap((event) => + event.type === "subagent.completed" && event.data.backgroundTask !== undefined + ? [event.data.backgroundTask.taskId] + : [], + ); +} diff --git a/e2e/fixtures/fixture-tasks/evals/task-fanout-parent-updates.eval.ts b/e2e/fixtures/fixture-tasks/evals/task-fanout-parent-updates.eval.ts new file mode 100644 index 000000000..c10c0810e --- /dev/null +++ b/e2e/fixtures/fixture-tasks/evals/task-fanout-parent-updates.eval.ts @@ -0,0 +1,140 @@ +import { defineEval, type EveEvalTurn, type InputRequest } from "eve/evals"; +import { satisfies } from "eve/evals/expect"; + +import { sendAndFollowQueuedTurn, type TaskEvalSessionDriver } from "./shared.js"; + +const FANOUT_SIZE = 10; + +/** Many background tasks complete independently and publish lifecycle updates to their parent. */ +export default defineEval({ + description: + "A ten-task fanout returns distinct receipts and sends every completion update to the parent session.", + async test(t) { + const started = await t.send("TASK-FANOUT-PARENT-UPDATES"); + started.expectOk(); + started.messageIncludes("TASK-FANOUT-STARTED"); + started.calledSubagent("fanout-worker", { count: FANOUT_SIZE }); + + const taskIds = backgroundTaskIds(started); + await t.require( + taskIds, + satisfies( + (ids: readonly string[]) => ids.length === FANOUT_SIZE && new Set(ids).size === FANOUT_SIZE, + `${FANOUT_SIZE} distinct background task receipts`, + ), + ); + + const blocked = await waitForReleaseRequests(t, t, started); + const interactive = await sendAndFollowQueuedTurn( + t, + "TASK-FANOUT-INTERACTIVE-CHECK", + blocked.session, + ); + interactive.turn.expectOk(); + interactive.turn.messageIncludes("TASK-FANOUT-INTERACTIVE-OK"); + interactive.turn.usedNoTools(); + + const released = await interactive.session.respond( + ...blocked.requests.map((request) => ({ + optionId: "approve", + requestId: request.requestId, + })), + ); + released.expectOk(); + + const updated = new Set([ + ...completedTaskUpdates(started, taskIds), + ...blocked.observedTurns.flatMap((turn) => completedTaskUpdates(turn, taskIds)), + ...interactive.observedTurns.flatMap((turn) => completedTaskUpdates(turn, taskIds)), + ...completedTaskUpdates(released, taskIds), + ]); + let startIndex = interactive.session.state.streamIndex; + for (let attempt = 0; attempt < FANOUT_SIZE && updated.size < FANOUT_SIZE; attempt += 1) { + const sessionId = t.sessionId; + if (sessionId === undefined) throw new Error("Task fanout has no parent session id."); + const live = t.target.watchTurn(sessionId, { startIndex }); + const turn = await live.result(); + for (const taskId of completedTaskUpdates(turn, taskIds)) updated.add(taskId); + startIndex = live.session.state.streamIndex; + } + + await t.require( + [...updated], + satisfies( + (ids: readonly string[]) => ids.length === FANOUT_SIZE, + "every fanout task sends a completed update to the parent session", + ), + ); + t.noFailedActions(); + }, +}); + +interface BlockedFanout { + readonly observedTurns: readonly EveEvalTurn[]; + readonly requests: readonly InputRequest[]; + readonly session: TaskEvalSessionDriver; +} + +async function waitForReleaseRequests( + t: Parameters[0], + initialSession: TaskEvalSessionDriver, + initialTurn: EveEvalTurn, +): Promise { + const requests = new Map(); + const observedTurns = [initialTurn]; + let session = initialSession; + collectReleaseRequests(initialTurn, requests); + for (let attempt = 0; attempt < FANOUT_SIZE && requests.size < FANOUT_SIZE; attempt += 1) { + const sessionId = session.sessionId; + if (sessionId === undefined) throw new Error("Task fanout has no parent session id."); + const live = t.target.watchTurn(sessionId, { startIndex: session.state.streamIndex }); + const turn = await live.result(); + observedTurns.push(turn); + collectReleaseRequests(turn, requests); + session = live.session; + } + if (requests.size !== FANOUT_SIZE) { + throw new Error(`Expected ${FANOUT_SIZE} release requests; received ${requests.size}.`); + } + return { observedTurns, requests: [...requests.values()], session }; +} + +function collectReleaseRequests(turn: EveEvalTurn, requests: Map): void { + for (const request of turn.inputRequests) { + if (request.action.toolName === "release") requests.set(request.requestId, request); + } +} + +function backgroundTaskIds(turn: EveEvalTurn): readonly string[] { + return turn.events.flatMap((event) => + event.type === "subagent.completed" && event.data.backgroundTask !== undefined + ? [event.data.backgroundTask.taskId] + : [], + ); +} + +function completedTaskUpdates(turn: EveEvalTurn, taskIds: readonly string[]): readonly string[] { + const messages = turn.events.flatMap((event) => + event.type === "message.received" ? [messageText(event.data.message)] : [], + ); + return taskIds.filter((taskId) => + messages.some( + (message) => message.includes(`Background task ${taskId} `) && message.includes("completed"), + ), + ); +} + +function messageText(message: unknown): string { + if (typeof message === "string") return message; + if (!Array.isArray(message)) return ""; + return message + .flatMap((part) => + part !== null && + typeof part === "object" && + Reflect.get(part, "type") === "text" && + typeof Reflect.get(part, "text") === "string" + ? [Reflect.get(part, "text") as string] + : [], + ) + .join("\n"); +} diff --git a/e2e/fixtures/fixture-tasks/evals/task-hitl-routing.eval.ts b/e2e/fixtures/fixture-tasks/evals/task-hitl-routing.eval.ts new file mode 100644 index 000000000..8e949c6e7 --- /dev/null +++ b/e2e/fixtures/fixture-tasks/evals/task-hitl-routing.eval.ts @@ -0,0 +1,53 @@ +import { defineEval } from "eve/evals"; + +import { requireBackgroundTaskId, waitForCompletedTask, waitForTaskInput } from "./shared.js"; + +/** + * Task-owned child HITL must surface on the parent session, and answering it + * must route to the child without running the parent model. + */ +export default defineEval({ + description: + "A background child's approval surfaces on the parent and routes back without a parent model step.", + async test(t) { + const started = await t.send("TASK-HITL-ROUTING"); + started.expectOk(); + started.event("subagent.completed", { + count: 1, + data: { backgroundTask: { status: "working" }, subagentName: "approval-worker" }, + }); + const taskId = requireBackgroundTaskId(started); + + const first = await waitForTaskInput(t, t, "first_gate"); + const answered = await first.session.respond({ + optionId: "approve", + requestId: first.request.requestId, + }); + answered.expectOk(); + answered.notEvent("step.started"); + + // A second child request proves the first answer reached the child even + // though the parent model did not run. + const second = await waitForTaskInput(t, first.session, "second_gate"); + const answeredSecond = await second.session.respond({ + optionId: "approve", + requestId: second.request.requestId, + }); + answeredSecond.expectOk(); + answeredSecond.notEvent("step.started"); + + const third = await waitForTaskInput(t, second.session, "third_gate"); + const finished = await third.session.respond({ + optionId: "approve", + requestId: third.request.requestId, + }); + finished.expectOk(); + + const verified = await waitForCompletedTask(t, third.session, "TASK-HITL-VERIFY", taskId); + verified.expectOk(); + verified.messageIncludes("TASK-HITL-STATUS"); + + t.event("input.requested", { data: { requests: [{ action: { toolName: "first_gate" } }] } }); + t.noFailedActions(); + }, +}); diff --git a/e2e/fixtures/fixture-tasks/evals/task-input-batch-ordering.eval.ts b/e2e/fixtures/fixture-tasks/evals/task-input-batch-ordering.eval.ts new file mode 100644 index 000000000..5b4f1f4cf --- /dev/null +++ b/e2e/fixtures/fixture-tasks/evals/task-input-batch-ordering.eval.ts @@ -0,0 +1,75 @@ +import { defineEval } from "eve/evals"; + +import { + requireBackgroundTaskId, + requireTaskView, + sendAndFollowQueuedTurn, + waitForCompletedTask, + waitForTaskInput, +} from "./shared.js"; +import { satisfies } from "eve/evals/expect"; + +/** + * Replaying Q1 after the child has raised Q2 must neither deliver Q1 again nor + * clear Q2 from the task snapshot. + */ +export default defineEval({ + description: "A stale task answer cannot unblock or erase the child's newer approval request.", + async test(t) { + const started = await t.send("TASK-INPUT-BATCH-ORDERING"); + started.expectOk(); + const taskId = requireBackgroundTaskId(started); + + const first = await waitForTaskInput(t, t, "first_gate"); + const firstAnswer = await first.session.respond({ + optionId: "approve", + requestId: first.request.requestId, + }); + firstAnswer.expectOk(); + + const second = await waitForTaskInput(t, first.session, "second_gate"); + const stale = await second.session.respond({ + optionId: "approve", + requestId: first.request.requestId, + }); + stale.expectOk(); + + const afterStale = await sendAndFollowQueuedTurn( + t, + `TASK-INPUT-BATCH-VERIFY ${taskId}`, + second.session, + ); + const peeked = afterStale.turn.toolCalls.find((call) => call.name === "task_peek"); + await t.require( + peeked?.output, + satisfies((output) => { + const view = requireTaskView(output, taskId); + const requests = Reflect.get(view, "inputRequests"); + return ( + Reflect.get(view, "status") === "input_required" && + Array.isArray(requests) && + requests.length === 1 && + Reflect.get(requests[0], "requestId") === second.request.requestId + ); + }, "the stale Q1 answer leaves exactly Q2 outstanding"), + ); + + // If the stale Q1 answer cleared Q2, this exact Q2 response cannot resume + // the child and the task never reaches `completed`. + const secondAnswer = await afterStale.session.respond({ + optionId: "approve", + requestId: second.request.requestId, + }); + secondAnswer.expectOk(); + + const verified = await waitForCompletedTask( + t, + afterStale.session, + "TASK-INPUT-BATCH-VERIFY", + taskId, + ); + verified.expectOk(); + verified.messageIncludes("TASK-INPUT-BATCH-STATUS"); + t.noFailedActions(); + }, +}); diff --git a/e2e/fixtures/fixture-tasks/package.json b/e2e/fixtures/fixture-tasks/package.json new file mode 100644 index 000000000..ceb406f2f --- /dev/null +++ b/e2e/fixtures/fixture-tasks/package.json @@ -0,0 +1,23 @@ +{ + "name": "fixture-tasks", + "version": "0.0.0", + "private": true, + "type": "module", + "scripts": { + "build": "eve build", + "dev": "eve dev", + "start": "eve start", + "typecheck": "eve build && tsc", + "test:e2e": "eve eval --strict" + }, + "dependencies": { + "@eve-e2e/config": "workspace:*", + "@workflow/world-postgres": "catalog:", + "eve": "workspace:*", + "zod": "catalog:" + }, + "devDependencies": { + "@types/node": "catalog:", + "typescript": "catalog:" + } +} diff --git a/e2e/fixtures/fixture-tasks/tsconfig.json b/e2e/fixtures/fixture-tasks/tsconfig.json new file mode 100644 index 000000000..85e3eceb8 --- /dev/null +++ b/e2e/fixtures/fixture-tasks/tsconfig.json @@ -0,0 +1,17 @@ +{ + "compilerOptions": { + "target": "ES2022", + "module": "esnext", + "moduleResolution": "bundler", + "outDir": "dist", + "rootDir": ".", + "strict": true, + "esModuleInterop": true, + "skipLibCheck": true, + "forceConsistentCasingInFileNames": true, + "declaration": true, + "noEmit": true, + "types": ["node"] + }, + "include": ["agent/**/*.ts", "evals/**/*.ts"] +} diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 4e95497d1..6a854ec36 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -919,6 +919,28 @@ importers: specifier: 'catalog:' version: 7.0.2 + e2e/fixtures/fixture-tasks: + dependencies: + '@eve-e2e/config': + specifier: workspace:* + version: link:../e2e-config + '@workflow/world-postgres': + specifier: 'catalog:' + version: 5.0.0-beta.31(@opentelemetry/api@1.9.1)(@types/pg@8.20.0)(@upstash/redis@1.38.0)(sql.js@1.14.1)(supports-color@10.2.2)(typescript@7.0.2) + eve: + specifier: workspace:* + version: link:../../../packages/eve + zod: + specifier: 'catalog:' + version: 4.4.3 + devDependencies: + '@types/node': + specifier: 'catalog:' + version: 25.9.1 + typescript: + specifier: 'catalog:' + version: 7.0.2 + e2e/fixtures/agent-tools: dependencies: '@eve-e2e/config':