diff --git a/src/config.ts b/src/config.ts index a0425a9..9c68329 100644 --- a/src/config.ts +++ b/src/config.ts @@ -4,7 +4,7 @@ import { LEVELS, type Level } from "./types.ts" export type MetricsTemporality = "cumulative" | "delta" | "lowmemory" /** Valid trace types emitted by the plugin. */ -export const TRACE_TYPES = ["session", "llm", "tool"] as const +export const TRACE_TYPES = ["session", "llm", "tool", "permission"] as const const VALID_TEMPORALITIES: ReadonlySet = new Set(["cumulative", "delta", "lowmemory"]) const TRACE_DISABLE_ALL_VALUES = new Set(["all", "*", "true", "1"]) diff --git a/src/handlers/permission.ts b/src/handlers/permission.ts index ba57fbe..a4d445d 100644 --- a/src/handlers/permission.ts +++ b/src/handlers/permission.ts @@ -1,8 +1,10 @@ +import { SpanStatusCode, trace } from "@opentelemetry/api" import { SeverityNumber } from "@opentelemetry/api-logs" +import { AGENT_NAME, SESSION_ID } from "@arizeai/openinference-semantic-conventions" import type { EventOf, HandlerContext } from "../types.ts" -import { agentAttrs, getSessionAgentMeta, setBoundedMap } from "../util.ts" +import { agentAttrs, getSessionAgentMeta, isTraceEnabled, resolveRunContext, setBoundedMap } from "../util.ts" -/** Stores an answered permission prompt for correlation when the reply arrives. */ +/** Starts a permission span for a permission prompt so approval waits are visible in traces. */ export function handlePermissionAsked(e: EventOf<"permission.asked">, ctx: HandlerContext) { const d = e.data setBoundedMap(ctx.tracing.pendingPermissions, d.id, { @@ -15,9 +17,29 @@ export function handlePermissionAsked(e: EventOf<"permission.asked">, ctx: Handl sessionID: d.sessionID, action: d.action, }) + if (!isTraceEnabled("permission", ctx)) return + // The approval happens between a step's model response and its tool execution, + // so parent the span to the active step (the LLM span) and fall back to the run span. + const activeStep = ctx.tracing.activeStepSpans.get(d.sessionID) + const parent = activeStep + ? trace.setSpan(resolveRunContext(d.sessionID, ctx), activeStep) + : resolveRunContext(d.sessionID, ctx) + const span = ctx.tracer.startSpan( + `${ctx.tracePrefix}permission`, + { + startTime: e.created, + attributes: { + [SESSION_ID]: d.sessionID, + "permission.action": d.action, + ...ctx.commonAttrs, + }, + }, + parent, + ) + setBoundedMap(ctx.tracing.permissionSpans, d.id, span) } -/** Emits a `tool_decision` log event recording whether the permission was accepted or rejected. */ +/** Ends the permission span and emits a `tool_decision` log event recording whether the permission was accepted or rejected. */ export function handlePermissionReplied(e: EventOf<"permission.replied">, ctx: HandlerContext) { const d = e.data const pending = ctx.tracing.pendingPermissions.get(d.requestID) @@ -25,6 +47,20 @@ export function handlePermissionReplied(e: EventOf<"permission.replied">, ctx: H const decision = d.reply === "reject" ? "reject" : "accept" const { agentName, agentType } = getSessionAgentMeta(d.sessionID, ctx) + const span = ctx.tracing.permissionSpans.get(d.requestID) + if (span) { + span.setAttributes({ + "permission.decision": decision, + "permission.source": d.reply, + ...(pending ? { "permission.action": pending.action } : {}), + [AGENT_NAME]: agentName, + "agent.type": agentType, + }) + span.setStatus({ code: decision === "accept" ? SpanStatusCode.OK : SpanStatusCode.ERROR }) + span.end(e.created) + ctx.tracing.permissionSpans.delete(d.requestID) + } + ctx.emitLog({ severityNumber: SeverityNumber.INFO, severityText: "INFO", @@ -43,3 +79,15 @@ export function handlePermissionReplied(e: EventOf<"permission.replied">, ctx: H }, }) } + +/** Ends permission spans still open when a session execution ends. */ +export function sweepPermissionSpans(sessionID: string, ctx: HandlerContext) { + for (const [id, span] of ctx.tracing.permissionSpans) { + const pending = ctx.tracing.pendingPermissions.get(id) + if (pending?.sessionID !== sessionID) continue + span.setAttributes({ "permission.decision": "cancelled" }) + span.setStatus({ code: SpanStatusCode.ERROR, message: "session execution ended before the permission was answered" }) + span.end() + ctx.tracing.permissionSpans.delete(id) + } +} diff --git a/src/handlers/session.ts b/src/handlers/session.ts index 3d9c65a..1a98250 100644 --- a/src/handlers/session.ts +++ b/src/handlers/session.ts @@ -22,6 +22,7 @@ import { setBoundedMap, totalTokens, } from "../util.ts" +import { sweepPermissionSpans } from "./permission.ts" const OPENINFERENCE_SPAN_KIND = SemanticConventions.OPENINFERENCE_SPAN_KIND @@ -423,6 +424,9 @@ function sweepExecution(sessionID: string, ctx: HandlerContext) { ctx.tracing.stepMeta.delete(messageID) ctx.tracing.stepOutputs.delete(messageID) } + // Sweep permission spans before dropping their pending entries: the sweep + // matches spans to sessions through pendingPermissions. + sweepPermissionSpans(sessionID, ctx) for (const [id, perm] of ctx.tracing.pendingPermissions) { if (perm.sessionID === sessionID) ctx.tracing.pendingPermissions.delete(id) } diff --git a/src/state.ts b/src/state.ts index e3395c0..fcf2002 100644 --- a/src/state.ts +++ b/src/state.ts @@ -116,6 +116,7 @@ export function acquireTracingState(): TracingState { stepSpanContexts: new Map(), toolSpans: new Map(), toolSpanContexts: new Map(), + permissionSpans: new Map(), subagentParents: new Map(), consumedSubagentDispatches: new Set(), toolMeta: new Map(), diff --git a/src/types.ts b/src/types.ts index 5c4ec11..560de3e 100644 --- a/src/types.ts +++ b/src/types.ts @@ -155,6 +155,7 @@ export type TracingState = { stepSpanContexts: Map toolSpans: Map toolSpanContexts: Map + permissionSpans: Map subagentParents: Map consumedSubagentDispatches: Set toolMeta: Map diff --git a/tests/handlers/permission.test.ts b/tests/handlers/permission.test.ts index 5ddf496..22d9c1a 100644 --- a/tests/handlers/permission.test.ts +++ b/tests/handlers/permission.test.ts @@ -1,31 +1,68 @@ import { describe, test, expect } from "bun:test" -import { handlePermissionAsked, handlePermissionReplied } from "../../src/handlers/permission.ts" +import { SpanStatusCode } from "@opentelemetry/api" +import { handleExecutionEnded } from "../../src/handlers/session.ts" +import { handlePermissionAsked, handlePermissionReplied, sweepPermissionSpans } from "../../src/handlers/permission.ts" +import { handleStepStarted } from "../../src/handlers/step.ts" import { makeCtx, evt } from "../helpers.ts" -describe("handlePermissionAsked", () => { - test("stores the pending permission", () => { - const { ctx } = makeCtx() - handlePermissionAsked(evt("permission.asked", { id: "perm_1", sessionID: "ses_1", action: "bash", resources: ["git push"] }), ctx) - expect(ctx.tracing.pendingPermissions.get("perm_1")!.action).toBe("bash") +function asked(id = "perm_1", at = 1000) { + return evt("permission.asked", { id, sessionID: "ses_1", action: "bash", resources: ["ls -la"] }, at) +} + +function replied(id = "perm_1", reply: "accept" | "reject" | "always" = "accept", at = 5000) { + return evt("permission.replied", { requestID: id, sessionID: "ses_1", reply }, at) +} + +describe("permission spans", () => { + test("records the approval wait under the active step span", () => { + const { ctx, tracer } = makeCtx() + handleStepStarted(evt("session.step.started", { sessionID: "ses_1", assistantMessageID: "msg_1", agent: "build", model: { id: "m", providerID: "p" }, started: 1 }), ctx) + handlePermissionAsked(asked(), ctx) + handlePermissionReplied(replied(), ctx) + const step = tracer.spans.find((s) => s.name === "opencode.llm")! + const span = tracer.spans.find((s) => s.name === "opencode.permission")! + expect(span.parentSpanContext).toBe(step.spanContext()) + expect(span.startTime).toBe(1000) + expect(span.endTime).toBe(5000) + expect(span.attributes["permission.decision"]).toBe("accept") + expect(span.status.code).toBe(SpanStatusCode.OK) + expect(ctx.tracing.permissionSpans.size).toBe(0) + }) + + test("falls back to the run span when no step is active", () => { + const { ctx, tracer } = makeCtx() + handlePermissionAsked(asked(), ctx) + handlePermissionReplied(replied("perm_1", "reject"), ctx) + const span = tracer.spans.find((s) => s.name === "opencode.permission")! + expect(span.attributes["permission.decision"]).toBe("reject") + expect(span.status.code).toBe(SpanStatusCode.ERROR) + }) + + test("does not create spans when permission tracing is disabled", () => { + const { ctx, tracer } = makeCtx("proj_test", [], ["permission"]) + handlePermissionAsked(asked(), ctx) + handlePermissionReplied(replied(), ctx) + expect(tracer.spans.filter((s) => s.name === "opencode.permission")).toHaveLength(0) }) -}) -describe("handlePermissionReplied", () => { - test("emits an accept tool_decision", () => { - const { ctx, logger } = makeCtx() - handlePermissionAsked(evt("permission.asked", { id: "perm_1", sessionID: "ses_1", action: "bash", resources: ["git push"] }), ctx) - handlePermissionReplied(evt("permission.replied", { sessionID: "ses_1", requestID: "perm_1", reply: "always" }), ctx) - const record = logger.records.at(-1)! - expect(record.body).toBe("tool_decision") - expect(record.attributes?.["decision"]).toBe("accept") - expect(record.attributes?.["tool_name"]).toBe("bash") - expect(record.attributes?.["source"]).toBe("always") + test("sweep ends a permission span that was never answered", () => { + const { ctx, tracer } = makeCtx() + handlePermissionAsked(asked(), ctx) + sweepPermissionSpans("ses_1", ctx) + const span = tracer.spans.find((s) => s.name === "opencode.permission")! + expect(span.ended).toBe(true) + expect(span.attributes["permission.decision"]).toBe("cancelled") + expect(span.status.code).toBe(SpanStatusCode.ERROR) }) - test("emits a reject tool_decision", () => { - const { ctx, logger } = makeCtx() - handlePermissionAsked(evt("permission.asked", { id: "perm_2", sessionID: "ses_1", action: "edit", resources: ["a.ts"] }), ctx) - handlePermissionReplied(evt("permission.replied", { sessionID: "ses_1", requestID: "perm_2", reply: "reject" }), ctx) - expect(logger.records.at(-1)!.attributes?.["decision"]).toBe("reject") + test("execution-end sweep ends unanswered permission spans before pending entries drop", () => { + const { ctx, tracer } = makeCtx() + handlePermissionAsked(asked(), ctx) + // The real path: handleExecutionEnded -> sweepExecution deletes pendingPermissions, + // so the span sweep must run before that or it can no longer match the session. + handleExecutionEnded(evt("session.execution.succeeded", { sessionID: "ses_1" }), ctx, { type: "succeeded" }) + const span = tracer.spans.find((s) => s.name === "opencode.permission")! + expect(span.ended).toBe(true) + expect(span.attributes["permission.decision"]).toBe("cancelled") }) }) diff --git a/tests/helpers.ts b/tests/helpers.ts index 2af8128..2cb79df 100644 --- a/tests/helpers.ts +++ b/tests/helpers.ts @@ -145,6 +145,7 @@ export function makeTracingState(): TracingState { stepSpanContexts: new Map(), toolSpans: new Map(), toolSpanContexts: new Map(), + permissionSpans: new Map(), subagentParents: new Map(), consumedSubagentDispatches: new Set(), toolMeta: new Map(),