Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion src/config.ts
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@ import { LEVELS, type Level } from "./types.ts"
export type MetricsTemporality = "cumulative" | "delta" | "lowmemory"

/** Valid trace types emitted by the plugin. */
export const TRACE_TYPES = ["session", "llm", "tool"] as const
export const TRACE_TYPES = ["session", "llm", "tool", "permission"] as const

const VALID_TEMPORALITIES: ReadonlySet<MetricsTemporality> = new Set<MetricsTemporality>(["cumulative", "delta", "lowmemory"])
const TRACE_DISABLE_ALL_VALUES = new Set(["all", "*", "true", "1"])
Expand Down
54 changes: 51 additions & 3 deletions src/handlers/permission.ts
Original file line number Diff line number Diff line change
@@ -1,8 +1,10 @@
import { SpanStatusCode, trace } from "@opentelemetry/api"
import { SeverityNumber } from "@opentelemetry/api-logs"
import { AGENT_NAME, SESSION_ID } from "@arizeai/openinference-semantic-conventions"
import type { EventOf, HandlerContext } from "../types.ts"
import { agentAttrs, getSessionAgentMeta, setBoundedMap } from "../util.ts"
import { agentAttrs, getSessionAgentMeta, isTraceEnabled, resolveRunContext, setBoundedMap } from "../util.ts"

/** Stores an answered permission prompt for correlation when the reply arrives. */
/** Starts a permission span for a permission prompt so approval waits are visible in traces. */
export function handlePermissionAsked(e: EventOf<"permission.asked">, ctx: HandlerContext) {
const d = e.data
setBoundedMap(ctx.tracing.pendingPermissions, d.id, {
Expand All @@ -15,16 +17,50 @@ export function handlePermissionAsked(e: EventOf<"permission.asked">, ctx: Handl
sessionID: d.sessionID,
action: d.action,
})
if (!isTraceEnabled("permission", ctx)) return
// The approval happens between a step's model response and its tool execution,
// so parent the span to the active step (the LLM span) and fall back to the run span.
const activeStep = ctx.tracing.activeStepSpans.get(d.sessionID)
const parent = activeStep
? trace.setSpan(resolveRunContext(d.sessionID, ctx), activeStep)
: resolveRunContext(d.sessionID, ctx)
const span = ctx.tracer.startSpan(
`${ctx.tracePrefix}permission`,
{
startTime: e.created,
attributes: {
[SESSION_ID]: d.sessionID,
"permission.action": d.action,
...ctx.commonAttrs,
},
},
parent,
)
setBoundedMap(ctx.tracing.permissionSpans, d.id, span)
}

/** Emits a `tool_decision` log event recording whether the permission was accepted or rejected. */
/** Ends the permission span and emits a `tool_decision` log event recording whether the permission was accepted or rejected. */
export function handlePermissionReplied(e: EventOf<"permission.replied">, ctx: HandlerContext) {
const d = e.data
const pending = ctx.tracing.pendingPermissions.get(d.requestID)
ctx.tracing.pendingPermissions.delete(d.requestID)
const decision = d.reply === "reject" ? "reject" : "accept"
const { agentName, agentType } = getSessionAgentMeta(d.sessionID, ctx)

const span = ctx.tracing.permissionSpans.get(d.requestID)
if (span) {
span.setAttributes({
"permission.decision": decision,
"permission.source": d.reply,
...(pending ? { "permission.action": pending.action } : {}),
[AGENT_NAME]: agentName,
"agent.type": agentType,
})
span.setStatus({ code: decision === "accept" ? SpanStatusCode.OK : SpanStatusCode.ERROR })
span.end(e.created)
ctx.tracing.permissionSpans.delete(d.requestID)
}

ctx.emitLog({
severityNumber: SeverityNumber.INFO,
severityText: "INFO",
Expand All @@ -43,3 +79,15 @@ export function handlePermissionReplied(e: EventOf<"permission.replied">, ctx: H
},
})
}

/** Ends permission spans still open when a session execution ends. */
export function sweepPermissionSpans(sessionID: string, ctx: HandlerContext) {
for (const [id, span] of ctx.tracing.permissionSpans) {
const pending = ctx.tracing.pendingPermissions.get(id)
if (pending?.sessionID !== sessionID) continue
span.setAttributes({ "permission.decision": "cancelled" })
span.setStatus({ code: SpanStatusCode.ERROR, message: "session execution ended before the permission was answered" })
span.end()
ctx.tracing.permissionSpans.delete(id)
}
}
4 changes: 4 additions & 0 deletions src/handlers/session.ts
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@ import {
setBoundedMap,
totalTokens,
} from "../util.ts"
import { sweepPermissionSpans } from "./permission.ts"

const OPENINFERENCE_SPAN_KIND = SemanticConventions.OPENINFERENCE_SPAN_KIND

Expand Down Expand Up @@ -423,6 +424,9 @@ function sweepExecution(sessionID: string, ctx: HandlerContext) {
ctx.tracing.stepMeta.delete(messageID)
ctx.tracing.stepOutputs.delete(messageID)
}
// Sweep permission spans before dropping their pending entries: the sweep
// matches spans to sessions through pendingPermissions.
sweepPermissionSpans(sessionID, ctx)
for (const [id, perm] of ctx.tracing.pendingPermissions) {
if (perm.sessionID === sessionID) ctx.tracing.pendingPermissions.delete(id)
}
Expand Down
1 change: 1 addition & 0 deletions src/state.ts
Original file line number Diff line number Diff line change
Expand Up @@ -116,6 +116,7 @@ export function acquireTracingState(): TracingState {
stepSpanContexts: new Map(),
toolSpans: new Map(),
toolSpanContexts: new Map(),
permissionSpans: new Map(),
subagentParents: new Map(),
consumedSubagentDispatches: new Set(),
toolMeta: new Map(),
Expand Down
1 change: 1 addition & 0 deletions src/types.ts
Original file line number Diff line number Diff line change
Expand Up @@ -155,6 +155,7 @@ export type TracingState = {
stepSpanContexts: Map<string, SpanContext>
toolSpans: Map<string, Span>
toolSpanContexts: Map<string, SpanContext>
permissionSpans: Map<string, Span>
subagentParents: Map<string, { spanContext: SpanContext; callID: string }>
consumedSubagentDispatches: Set<string>
toolMeta: Map<string, ToolMeta>
Expand Down
81 changes: 59 additions & 22 deletions tests/handlers/permission.test.ts
Original file line number Diff line number Diff line change
@@ -1,31 +1,68 @@
import { describe, test, expect } from "bun:test"
import { handlePermissionAsked, handlePermissionReplied } from "../../src/handlers/permission.ts"
import { SpanStatusCode } from "@opentelemetry/api"
import { handleExecutionEnded } from "../../src/handlers/session.ts"
import { handlePermissionAsked, handlePermissionReplied, sweepPermissionSpans } from "../../src/handlers/permission.ts"
import { handleStepStarted } from "../../src/handlers/step.ts"
import { makeCtx, evt } from "../helpers.ts"

describe("handlePermissionAsked", () => {
test("stores the pending permission", () => {
const { ctx } = makeCtx()
handlePermissionAsked(evt("permission.asked", { id: "perm_1", sessionID: "ses_1", action: "bash", resources: ["git push"] }), ctx)
expect(ctx.tracing.pendingPermissions.get("perm_1")!.action).toBe("bash")
function asked(id = "perm_1", at = 1000) {
return evt("permission.asked", { id, sessionID: "ses_1", action: "bash", resources: ["ls -la"] }, at)
}

function replied(id = "perm_1", reply: "accept" | "reject" | "always" = "accept", at = 5000) {
return evt("permission.replied", { requestID: id, sessionID: "ses_1", reply }, at)
}

describe("permission spans", () => {
test("records the approval wait under the active step span", () => {
const { ctx, tracer } = makeCtx()
handleStepStarted(evt("session.step.started", { sessionID: "ses_1", assistantMessageID: "msg_1", agent: "build", model: { id: "m", providerID: "p" }, started: 1 }), ctx)
handlePermissionAsked(asked(), ctx)
handlePermissionReplied(replied(), ctx)
const step = tracer.spans.find((s) => s.name === "opencode.llm")!
const span = tracer.spans.find((s) => s.name === "opencode.permission")!
expect(span.parentSpanContext).toBe(step.spanContext())
expect(span.startTime).toBe(1000)
expect(span.endTime).toBe(5000)
expect(span.attributes["permission.decision"]).toBe("accept")
expect(span.status.code).toBe(SpanStatusCode.OK)
expect(ctx.tracing.permissionSpans.size).toBe(0)
})

test("falls back to the run span when no step is active", () => {
const { ctx, tracer } = makeCtx()
handlePermissionAsked(asked(), ctx)
handlePermissionReplied(replied("perm_1", "reject"), ctx)
const span = tracer.spans.find((s) => s.name === "opencode.permission")!
expect(span.attributes["permission.decision"]).toBe("reject")
expect(span.status.code).toBe(SpanStatusCode.ERROR)
})

test("does not create spans when permission tracing is disabled", () => {
const { ctx, tracer } = makeCtx("proj_test", [], ["permission"])
handlePermissionAsked(asked(), ctx)
handlePermissionReplied(replied(), ctx)
expect(tracer.spans.filter((s) => s.name === "opencode.permission")).toHaveLength(0)
})
})

describe("handlePermissionReplied", () => {
test("emits an accept tool_decision", () => {
const { ctx, logger } = makeCtx()
handlePermissionAsked(evt("permission.asked", { id: "perm_1", sessionID: "ses_1", action: "bash", resources: ["git push"] }), ctx)
handlePermissionReplied(evt("permission.replied", { sessionID: "ses_1", requestID: "perm_1", reply: "always" }), ctx)
const record = logger.records.at(-1)!
expect(record.body).toBe("tool_decision")
expect(record.attributes?.["decision"]).toBe("accept")
expect(record.attributes?.["tool_name"]).toBe("bash")
expect(record.attributes?.["source"]).toBe("always")
test("sweep ends a permission span that was never answered", () => {
const { ctx, tracer } = makeCtx()
handlePermissionAsked(asked(), ctx)
sweepPermissionSpans("ses_1", ctx)
const span = tracer.spans.find((s) => s.name === "opencode.permission")!
expect(span.ended).toBe(true)
expect(span.attributes["permission.decision"]).toBe("cancelled")
expect(span.status.code).toBe(SpanStatusCode.ERROR)
})

test("emits a reject tool_decision", () => {
const { ctx, logger } = makeCtx()
handlePermissionAsked(evt("permission.asked", { id: "perm_2", sessionID: "ses_1", action: "edit", resources: ["a.ts"] }), ctx)
handlePermissionReplied(evt("permission.replied", { sessionID: "ses_1", requestID: "perm_2", reply: "reject" }), ctx)
expect(logger.records.at(-1)!.attributes?.["decision"]).toBe("reject")
test("execution-end sweep ends unanswered permission spans before pending entries drop", () => {
const { ctx, tracer } = makeCtx()
handlePermissionAsked(asked(), ctx)
// The real path: handleExecutionEnded -> sweepExecution deletes pendingPermissions,
// so the span sweep must run before that or it can no longer match the session.
handleExecutionEnded(evt("session.execution.succeeded", { sessionID: "ses_1" }), ctx, { type: "succeeded" })
const span = tracer.spans.find((s) => s.name === "opencode.permission")!
expect(span.ended).toBe(true)
expect(span.attributes["permission.decision"]).toBe("cancelled")
})
})
1 change: 1 addition & 0 deletions tests/helpers.ts
Original file line number Diff line number Diff line change
Expand Up @@ -145,6 +145,7 @@ export function makeTracingState(): TracingState {
stepSpanContexts: new Map(),
toolSpans: new Map(),
toolSpanContexts: new Map(),
permissionSpans: new Map(),
subagentParents: new Map(),
consumedSubagentDispatches: new Set(),
toolMeta: new Map(),
Expand Down
Loading