feat(eve): publish durable runtime action events

.changeset/instrumentation-runtime-actions.md

@@ -0,0 +1,5 @@
+---
+"eve": patch
+---
+
+Publish durable action lifecycle events for tools, skills, subagents, and remote agents while preserving the AI SDK tool-call boundary separately.

packages/eve/src/evals/runner/derive-run-facts.test.ts

@@ -162,6 +162,39 @@ describe("deriveRunFacts", () => {
     expect(facts.toolCallCount).toBe(2);
   });
 
+  it("preserves framework skill input in the eval tool-call view", () => {
+    const facts = derive([
+      turnStarted("t1", 0),
+      {
+        type: "actions.requested",
+        data: {
+          actions: [
+            {
+              callId: "skill-1",
+              input: { skill: "research" },
+              kind: "load-skill",
+            },
+          ],
+          sequence: 1,
+          stepIndex: 0,
+          turnId: "t1",
+        },
+      },
+      actionResult({ callId: "skill-1", toolName: "load_skill", output: "Skill body" }),
+    ]);
+
+    expect(facts.toolCalls).toEqual([
+      {
+        input: { skill: "research" },
+        name: "load_skill",
+        output: "Skill body",
+        sessionId: undefined,
+        status: "completed",
+        turnIndex: 0,
+      },
+    ]);
+  });
+
   it("uses the normalized failed lifecycle status for error results", () => {
     const events: UnstampedMessageStreamEvent[] = [
       actionsRequested([{ callId: "c1", toolName: "bash" }]),

packages/eve/src/evals/runner/derive-run-facts.ts

@@ -1,4 +1,5 @@
 import type { MessageStreamEvent } from "#protocol/message.js";
+import { LOAD_SKILL_TOOL_NAME } from "#runtime/skills/fragment-context.js";
 import type { InputRequest } from "#runtime/input/types.js";
 import type { JsonObject, JsonValue } from "#shared/json.js";
 import type { EveEvalDerivedFacts, EveEvalSubagentCall, EveEvalToolCall } from "#evals/types.js";
@@ -107,8 +108,11 @@ export function deriveRunFacts(
 
       case "actions.requested": {
         for (const action of event.data.actions) {
-          if (action.kind !== "tool-call") continue;
-          ensureToolCall(action.callId, action.toolName, action.input);
+          if (action.kind === "tool-call") {
+            ensureToolCall(action.callId, action.toolName, action.input);
+          } else if (action.kind === "load-skill") {
+            ensureToolCall(action.callId, LOAD_SKILL_TOOL_NAME, action.input);
+          }
         }
         break;
       }

packages/eve/src/execution/node-step.ts

@@ -4,6 +4,7 @@ import type { Runtime, SessionCapabilities } from "#channel/types.js";
 import { dispatchDynamicModelEvent } from "#context/dynamic-model-lifecycle.js";
 import { createHarnessDelegationToolDefinition } from "#execution/delegation-tool.js";
 import type { HarnessToolDefinition } from "#harness/execute-tool.js";
+import { LOAD_SKILL_TOOL_NAME } from "#runtime/skills/fragment-context.js";
 import { createToolLoopHarness } from "#harness/tool-loop.js";
 import type { HandleEventFn, HarnessToolMap, StepFn } from "#harness/types.js";
 import { resolveInstalledPackageInfo } from "#internal/application/package.js";
@@ -260,6 +261,8 @@ function resolveHarnessToolDefinition(input: {
       rawExecute,
       scope: def.name,
     }),
+    frameworkAction:
+      isFrameworkTool && def.name === LOAD_SKILL_TOOL_NAME ? "load-skill" : undefined,
     inputSchema: def.inputSchema ?? UNSPECIFIED_INPUT_SCHEMA,
     name: def.name,
     approval: def.approval,

packages/eve/src/harness/ai-sdk-hook-bridge.test.ts

@@ -379,7 +379,6 @@ describe("createAiSdkHookBridge", () => {
           callId: "tool-1",
           idempotencyKey: `tool:${scope.attemptId}:tool-1:0`,
           input: { q: "eve" },
-          kind: "tool-call",
           scope,
           toolName: "search",
           type: "tool.call.started",
@@ -398,25 +397,6 @@ describe("createAiSdkHookBridge", () => {
     },
   );
 
-  it("labels a tool call with the kind the harness resolves", async () => {
-    const started = vi.fn();
-    const hooks = createInstrumentationHooks([{ events: { "tool.call.started": started } }]);
-    const bridge = createAiSdkHookBridge(scope, hooks, undefined, (toolName) =>
-      toolName === "research" ? "subagent-call" : "tool-call",
-    );
-
-    for (const toolName of ["research", "search"]) {
-      await Reflect.apply(bridge.onToolExecutionStart!, bridge, [
-        { callId: `call-${toolName}`, toolCall: { input: {}, toolCallId: toolName, toolName } },
-      ]);
-    }
-
-    expect(started.mock.calls.map(([event]) => [event.toolName, event.kind])).toEqual([
-      ["research", "subagent-call"],
-      ["search", "tool-call"],
-    ]);
-  });
-
   it("keeps each provider's state to itself", async () => {
     const observed = new Map<string, unknown>();
     const provider = (name: string): InstrumentationProviderDefinition => {

packages/eve/src/harness/ai-sdk-hook-bridge.ts

@@ -1,7 +1,6 @@
 import type { Telemetry } from "ai";
 
 import type {
-  InstrumentationActionKind,
   InstrumentationAttemptScope,
   InstrumentationStepAttemptStartedEvent,
   InstrumentationContentPart,
@@ -23,15 +22,8 @@ import {
 
 type TelemetryEvent<TKey extends keyof Telemetry> = Parameters<NonNullable<Telemetry[TKey]>>[0];
 
-/**
- * Reports what eve dispatches one tool name as. The AI SDK only knows the
- * name, so the kind has to come back from the harness.
- */
-export type ActionKindResolver = (toolName: string) => InstrumentationActionKind;
-
 interface AttemptState {
   readonly modelKeys: Map<string, string>;
-  readonly resolveActionKind: ActionKindResolver;
   readonly scope: InstrumentationAttemptScope;
   readonly toolKeys: Map<string, string>;
   operation?: InstrumentationOperationRef;
@@ -44,11 +36,9 @@ export function createAiSdkHookBridge(
   scope: InstrumentationAttemptScope,
   hooks: InstrumentationHooks,
   runInContext: InstrumentationContextRunner = directRunInContext,
-  resolveActionKind: ActionKindResolver = defaultResolveActionKind,
 ): Telemetry {
   const state: AttemptState = {
     modelKeys: new Map(),
-    resolveActionKind,
     scope,
     toolKeys: new Map(),
   };
@@ -151,8 +141,6 @@ export function createAiSdkHookBridge(
 
 const directRunInContext: InstrumentationContextRunner = (_operation, execute) => execute();
 
-const defaultResolveActionKind: ActionKindResolver = () => "tool-call";
-
 function toStepAttemptStarted(
   state: AttemptState,
 ): InstrumentationStepAttemptStartedEvent | undefined {
@@ -260,7 +248,6 @@ function toToolCallStarted(
     callId: source.toolCall.toolCallId,
     idempotencyKey,
     input: source.toolCall.input,
-    kind: state.resolveActionKind(source.toolCall.toolName),
     scope: state.scope,
     toolName: source.toolCall.toolName,
     type: "tool.call.started",

packages/eve/src/harness/execute-tool.ts

@@ -23,6 +23,7 @@ export interface HarnessToolDefinition {
   readonly approvalKey?: (toolInput: Readonly<Record<string, unknown>>) => string;
   readonly description: string;
   readonly execute?: (input: any, options: ToolExecuteOptions) => any;
+  readonly frameworkAction?: "load-skill";
   readonly inputSchema: FlexibleSchema;
   readonly name: string;
   readonly approval?: Approval;

packages/eve/src/harness/instrumentation-lifecycle.test.ts

@@ -3,6 +3,7 @@ import { describe, expect, it, vi } from "vitest";
 import { ContextContainer, contextStorage } from "#context/container.js";
 import { deserializeContext, serializeContext } from "#context/serialize.js";
 import {
+  actionIdempotencyKey,
   attemptIdempotencyKey,
   createInstrumentationHooks,
   modelCallIdempotencyKey,
@@ -11,7 +12,11 @@ import {
   turnIdempotencyKey,
   type InstrumentationAttemptScope,
 } from "#harness/instrumentation-lifecycle.js";
-import { instrumentationStateSlot } from "#harness/instrumentation-state.js";
+import {
+  findInstrumentationActionScopeForCall,
+  instrumentationStateSlot,
+  rememberInstrumentationActionScope,
+} from "#harness/instrumentation-state.js";
 
 const scope: InstrumentationAttemptScope = {
   attemptId: "session-1:turn-1:0:0",
@@ -116,6 +121,68 @@ describe("provider state lifecycle", () => {
       expect(instrumentationStateSlot("sink", modelKey).get()).toBeUndefined();
     });
   });
+
+  it("keeps action state past the originating attempt", async () => {
+    const actionKey = actionIdempotencyKey(scope.sessionId, scope.turnId, "call-1");
+    const hooks = createInstrumentationHooks([
+      {
+        events: { "action.started": (_event, ctx) => ctx.state.set("open") },
+        name: "sink",
+      },
+    ]);
+    await contextStorage.run(new ContextContainer(), async () => {
+      await hooks.publish({
+        callId: "call-1",
+        idempotencyKey: actionKey,
+        input: {},
+        kind: "tool-call",
+        name: "tool",
+        scope,
+        type: "action.started",
+      });
+      await hooks.publish({
+        idempotencyKey: attemptIdempotencyKey(scope),
+        scope,
+        type: "step.attempt.completed",
+      });
+      expect(instrumentationStateSlot("sink", actionKey).get()).toBe("open");
+    });
+  });
+
+  it("terminalizes and releases pending actions when a turn is cancelled", async () => {
+    const actionKey = actionIdempotencyKey(scope.sessionId, scope.turnId, "call-1");
+    const failed = vi.fn();
+    const hooks = createInstrumentationHooks([
+      {
+        events: {
+          "action.failed": failed,
+          "action.started": (_event, ctx) => ctx.state.set("open"),
+        },
+        name: "sink",
+      },
+    ]);
+    await contextStorage.run(new ContextContainer(), async () => {
+      rememberInstrumentationActionScope(actionKey, scope);
+      await hooks.publish({
+        callId: "call-1",
+        idempotencyKey: actionKey,
+        input: {},
+        kind: "tool-call",
+        name: "tool",
+        scope,
+        type: "action.started",
+      });
+      await hooks.publish({
+        idempotencyKey: turnIdempotencyKey(scope.sessionId, scope.turnId),
+        sessionId: scope.sessionId,
+        turnId: scope.turnId,
+        type: "turn.cancelled",
+      });
+      expect(instrumentationStateSlot("sink", actionKey).get()).toBeUndefined();
+      expect(findInstrumentationActionScopeForCall(scope.sessionId, "call-1")).toBeUndefined();
+    });
+    expect(failed).toHaveBeenCalledOnce();
+  });
 });
 
 describe("provider handler deadlines", () => {