feat(eve): let a provider say how much of an event it needs

.changeset/capture-below-the-otel-layer.md

@@ -0,0 +1,10 @@
+---
+"eve": patch
+---
+
+Let an instrumentation provider declare how much of each event it wants. Under
+the experimental provider layout, `capture: "content"` opts a provider into the
+prompt, the response, and tool payloads; the default `"metadata"` leaves it
+structure, usage, and timing. Content is now built only when something asked for
+it, so an agent whose providers and destinations all decline never serializes a
+prompt at all.

docs/guides/instrumentation-providers.md

@@ -82,6 +82,39 @@ Pass `spanProcessors` instead when the destination needs its own batching, sampl
 
 `otel()` and `otelIntegration()` come from `eve/instrumentation/otel`, a separate entrypoint from `eve/instrumentation`.
 
+## Content is per provider
+
+A provider declares how much of each event it wants. The default is
+`"metadata"`: structure, identity, usage, and timing, but not what the
+conversation said. `"content"` adds the prompt, the response, tool arguments,
+and tool results.
+
+```ts title="agent/instrumentation/audit.ts"
+export default defineInstrumentation({
+  capture: "content",
+  events: {
+    "model.call.completed": (event) => {
+      console.log(event.content);
+    },
+  },
+});
+```
+
+Asking is what makes eve build the projection at all. A directory in which no
+provider asked — and whose destinations all declined — never serializes a
+prompt, so declining is cheaper than filtering as well as safer. Providers that
+did not ask receive the same events with the content fields absent; a
+`capture: "content"` provider beside them changes nothing about what they see.
+
+Structure survives declining. `action.completed` still says whether the tool
+returned or threw, and `model.call.completed` still carries its finish reason
+and token usage — a provider counting failures or cost never has to ask for
+content to get them.
+
+Content fields are therefore optional on the event types that carry them:
+`input` on `model.call.started` and `action.started`, `content` on
+`model.call.completed`, and the payloads inside `action.completed`'s `output`.
+
 ## Content is per destination
 
 `recordInputs` and `recordOutputs` belong to a destination, not to the process. Content is written onto a span if any destination wants it, and each destination that declined never exports it. A local spool and a hosted backend no longer have to agree:
@@ -94,7 +127,7 @@ export default otelIntegration({
 });
 ```
 
-An agent whose every destination declines still never materializes a prompt — the union of nothing is nothing. Declining wraps every processor in that file, an author's included: they are this destination, and the point of declining is that nothing under it sees what was said. The wrapper copies the span rather than editing it, because the span it is handed is shared with every other destination in the pipeline.
+An agent whose every destination declines still never materializes a prompt — the OpenTelemetry pipeline is itself one provider, and a pipeline whose destinations all declined asks for `"metadata"` like any other. Declining wraps every processor in that file, an author's included: they are this destination, and the point of declining is that nothing under it sees what was said. The wrapper copies the span rather than editing it, because the span it is handed is shared with every other destination in the pipeline.
 
 For sensitive, regulated, or production data, decline content on any destination whose retention path you have not reviewed. You are responsible for ensuring an observability or eval provider is approved for what is exported to it.
 

packages/eve/src/harness/ai-sdk-hook-bridge.test.ts

@@ -250,21 +250,27 @@ describe("createAiSdkHookBridge", () => {
 
   it("projects the model call callbacks onto eve fields only", async () => {
     const before = vi.fn((event: InstrumentationModelCallStartedEvent) => {
+      if (event.input === undefined) throw new Error("expected model input");
       expect(Object.isFrozen(event)).toBe(true);
       expect(Object.isFrozen(event.input)).toBe(true);
       expect(Object.isFrozen(event.input.messages)).toBe(true);
       expect(Object.isFrozen(event.model)).toBe(true);
     });
     const after = vi.fn((event: InstrumentationModelCallTerminalEvent) => {
       if (event.type !== "model.call.completed") throw new Error("expected completed model call");
+      if (event.content === undefined) throw new Error("expected model content");
       expect(Object.isFrozen(event)).toBe(true);
       expect(Object.isFrozen(event.content)).toBe(true);
       expect(event.content.every((part) => Object.isFrozen(part))).toBe(true);
       expect(Object.isFrozen(event.usage)).toBe(true);
       expect(Object.isFrozen(event.usage.inputTokenDetails)).toBe(true);
     });
     const hooks = createInstrumentationHooks([
-      { events: { "model.call.completed": after, "model.call.started": before }, name: "spy" },
+      {
+        capture: "content",
+        events: { "model.call.completed": after, "model.call.started": before },
+        name: "spy",
+      },
     ]);
     const bridge = createAiSdkHookBridge(scope, hooks);
 
@@ -356,7 +362,11 @@ describe("createAiSdkHookBridge", () => {
         expect(Object.isFrozen(event.output)).toBe(true);
       });
       const hooks = createInstrumentationHooks([
-        { events: { "action.completed": after, "action.started": before }, name: "spy" },
+        {
+          capture: "content",
+          events: { "action.completed": after, "action.started": before },
+          name: "spy",
+        },
       ]);
       const bridge = createAiSdkHookBridge(scope, hooks);
       const toolCall = { input: { q: "eve" }, toolCallId: "tool-1", toolName: "search" };
@@ -390,6 +400,83 @@ describe("createAiSdkHookBridge", () => {
     },
   );
 
+  it("omits content from the projection when no provider asked for it", async () => {
+    const modelStarted = vi.fn();
+    const modelCompleted = vi.fn();
+    const actionStarted = vi.fn();
+    const actionCompleted = vi.fn();
+    const hooks = createInstrumentationHooks([
+      {
+        events: {
+          "action.completed": actionCompleted,
+          "action.started": actionStarted,
+          "model.call.completed": modelCompleted,
+          "model.call.started": modelStarted,
+        },
+        name: "metadata-only",
+      },
+    ]);
+    const bridge = createAiSdkHookBridge(scope, hooks);
+    const toolCall = { input: { q: "eve" }, toolCallId: "tool-1", toolName: "search" };
+
+    await Reflect.apply(bridge.onLanguageModelCallStart!, bridge, [
+      {
+        callId: "call-1",
+        instructions: "be brief",
+        messages: [{ content: "hi", role: "user" }],
+        modelId: "model",
+        provider: "test",
+        tools: undefined,
+      },
+    ]);
+    await Reflect.apply(bridge.onLanguageModelCallEnd!, bridge, [
+      {
+        callId: "call-1",
+        content: [{ text: "hello", type: "text" }],
+        finishReason: "stop",
+        performance: { responseTimeMs: 1 },
+        responseId: "response-1",
+        usage: { inputTokens: 1, outputTokens: 2 },
+      },
+    ]);
+    await Reflect.apply(bridge.onToolExecutionStart!, bridge, [{ callId: "call-1", toolCall }]);
+    await Reflect.apply(bridge.onToolExecutionEnd!, bridge, [
+      {
+        callId: "call-1",
+        toolCall,
+        toolExecutionMs: 1,
+        toolOutput: { output: "ok", type: "tool-result" },
+      },
+    ]);
+
+    expect(modelStarted.mock.calls[0]?.[0].input).toBeUndefined();
+    expect(modelCompleted.mock.calls[0]?.[0].content).toBeUndefined();
+    // Structure survives: usage, the finish reason, and the tool's identity are
+    // not what was said.
+    expect(modelCompleted.mock.calls[0]?.[0].finishReason).toBe("stop");
+    expect(actionStarted.mock.calls[0]?.[0].input).toBeUndefined();
+    expect(actionStarted.mock.calls[0]?.[0].name).toBe("search");
+    expect(actionCompleted.mock.calls[0]?.[0].output).toEqual({ type: "result" });
+  });
+
+  it("withholds content from a metadata provider sharing a bus with a content one", async () => {
+    const metadataOnly = vi.fn();
+    const wantsContent = vi.fn();
+    const hooks = createInstrumentationHooks([
+      { events: { "action.started": metadataOnly }, name: "metadata-only" },
+      { capture: "content", events: { "action.started": wantsContent }, name: "wants-content" },
+    ]);
+    const bridge = createAiSdkHookBridge(scope, hooks);
+
+    await Reflect.apply(bridge.onToolExecutionStart!, bridge, [
+      { callId: "call-1", toolCall: { input: { q: "eve" }, toolCallId: "t", toolName: "search" } },
+    ]);
+
+    expect(wantsContent.mock.calls[0]?.[0].input).toEqual({ q: "eve" });
+    expect(metadataOnly.mock.calls[0]?.[0].input).toBeUndefined();
+    expect(metadataOnly.mock.calls[0]?.[0].name).toBe("search");
+  });
+
   it("labels a tool call with the kind the harness resolves", async () => {
     const started = vi.fn();
     const hooks = createInstrumentationHooks([

packages/eve/src/harness/ai-sdk-hook-bridge.ts

@@ -30,6 +30,8 @@ export type ActionKindResolver = (toolName: string) => InstrumentationActionKind
 
 interface AttemptState {
   readonly actionKeys: Map<string, string>;
+  /** False when no provider asked for content, so none is projected at all. */
+  readonly capturesContent: boolean;
   readonly modelKeys: Map<string, string>;
   readonly resolveActionKind: ActionKindResolver;
   readonly scope: InstrumentationAttemptScope;
@@ -47,6 +49,7 @@ export function createAiSdkHookBridge(
 ): Telemetry {
   const state: AttemptState = {
     actionKeys: new Map(),
+    capturesContent: hooks.capturesContent,
     modelKeys: new Map(),
     resolveActionKind,
     scope,
@@ -167,10 +170,12 @@ function toModelCallStarted(
 ): InstrumentationModelCallStartedEvent {
   return Object.freeze({
     idempotencyKey,
-    input: Object.freeze({
-      instructions: source.instructions,
-      messages: Object.freeze([...source.messages]),
-    }),
+    input: state.capturesContent
+      ? Object.freeze({
+          instructions: source.instructions,
+          messages: Object.freeze([...source.messages]),
+        })
+      : undefined,
     model: Object.freeze({ modelId: source.modelId, provider: source.provider }),
     scope: state.scope,
     type: "model.call.started",
@@ -183,7 +188,7 @@ function toModelCallCompleted(
   source: TelemetryEvent<"onLanguageModelCallEnd">,
 ): InstrumentationModelCallCompletedEvent {
   return Object.freeze({
-    content: toContentParts(source.content),
+    content: state.capturesContent ? toContentParts(source.content) : undefined,
     finishReason: source.finishReason,
     idempotencyKey,
     scope: state.scope,
@@ -267,7 +272,7 @@ function toActionStarted(
   return Object.freeze({
     callId: source.toolCall.toolCallId,
     idempotencyKey,
-    input: source.toolCall.input,
+    input: state.capturesContent ? source.toolCall.input : undefined,
     kind: state.resolveActionKind(source.toolCall.toolName),
     name: source.toolCall.toolName,
     scope: state.scope,
@@ -282,16 +287,23 @@ function toActionCompleted(
 ): InstrumentationActionCompletedEvent {
   return Object.freeze({
     idempotencyKey,
-    output: toActionOutput(source.toolOutput),
+    output: toActionOutput(source.toolOutput, state.capturesContent),
     scope: state.scope,
     type: "action.completed",
   });
 }
 
+/** Whether the action errored is structure, so it survives declining content. */
 function toActionOutput(
   toolOutput: TelemetryEvent<"onToolExecutionEnd">["toolOutput"],
+  capturesContent: boolean,
 ): InstrumentationActionOutput {
-  return toolOutput.type === "tool-result"
-    ? Object.freeze({ output: toolOutput.output, type: "result" })
-    : Object.freeze({ error: toolOutput.error, type: "error" });
+  if (toolOutput.type === "tool-result") {
+    return Object.freeze(
+      capturesContent ? { output: toolOutput.output, type: "result" } : { type: "result" },
+    );
+  }
+  return Object.freeze(
+    capturesContent ? { error: toolOutput.error, type: "error" } : { type: "error" },
+  );
 }

packages/eve/src/harness/instrumentation-lifecycle.test.ts

@@ -242,3 +242,17 @@ describe("handler state", () => {
     expect(read).toHaveBeenCalledWith(undefined);
   });
 });
+
+describe("capture", () => {
+  it("reports content capture only when some provider asked for it", () => {
+    expect(createInstrumentationHooks([{ name: "quiet" }]).capturesContent).toBe(false);
+    expect(
+      createInstrumentationHooks([{ capture: "metadata", name: "quiet" }, { name: "also-quiet" }])
+        .capturesContent,
+    ).toBe(false);
+    expect(
+      createInstrumentationHooks([{ name: "quiet" }, { capture: "content", name: "loud" }])
+        .capturesContent,
+    ).toBe(true);
+  });
+});