From fa32ba8c0d051a9157db0391df0ab11158fce877 Mon Sep 17 00:00:00 2001 From: Elliott de Launay Date: Tue, 13 Oct 2026 20:38:58 -0400 Subject: [PATCH 1/3] feat(task): add run-state divergence comparison and metrics --- packages/types/src/telemetry.ts | 16 + src/__mocks__/vscode.js | 1 + src/core/task/Task.ts | 30 +- .../task/__tests__/Task.run-shadow.spec.ts | 77 ++++- .../__tests__/runStateDivergence.spec.ts | 298 ++++++++++++++++++ .../__tests__/runStateShadow.spec.ts | 6 +- src/core/task/run-state/runStateDivergence.ts | 86 +++++ src/core/task/run-state/runStateShadow.ts | 10 +- 8 files changed, 516 insertions(+), 8 deletions(-) create mode 100644 src/core/task/run-state/__tests__/runStateDivergence.spec.ts create mode 100644 src/core/task/run-state/runStateDivergence.ts diff --git a/packages/types/src/telemetry.ts b/packages/types/src/telemetry.ts index d69476836e..949e28bedf 100644 --- a/packages/types/src/telemetry.ts +++ b/packages/types/src/telemetry.ts @@ -86,6 +86,8 @@ export enum TelemetryEventName { TELEMETRY_SETTINGS_CHANGED = "Telemetry Settings Changed", MODEL_CACHE_EMPTY_RESPONSE = "Model Cache Empty Response", READ_FILE_LEGACY_FORMAT_USED = "Read File Legacy Format Used", + RUN_STATE_DIVERGENCE = "Run State Divergence", + RUN_STATE_KNOWN_RACE = "Run State Known Race", } /** @@ -221,6 +223,20 @@ export const rooCodeTelemetryEventSchema = z.discriminatedUnion("type", [ ]), properties: telemetryPropertiesSchema, }), + z.object({ + type: z.enum([TelemetryEventName.RUN_STATE_DIVERGENCE, TelemetryEventName.RUN_STATE_KNOWN_RACE]), + properties: z.object({ + ...telemetryPropertiesSchema.shape, + taskId: z.string(), + kind: z.enum(["mismatch", "rejection"]), + raceId: z.string().optional(), + eventTag: z.string(), + mismatched: z.array(z.string()), + phase: z.string(), + streamTag: z.string(), + abort: z.boolean(), + }), + }), z.object({ type: z.literal(TelemetryEventName.TELEMETRY_SETTINGS_CHANGED), properties: z.object({ diff --git a/src/__mocks__/vscode.js b/src/__mocks__/vscode.js index 8ace25aeb0..d96b5d9ca4 100644 --- a/src/__mocks__/vscode.js +++ b/src/__mocks__/vscode.js @@ -107,6 +107,7 @@ export const env = { openExternal: () => Promise.resolve(), } +export const ExtensionMode = { Production: 1, Development: 2, Test: 3 } export const Uri = mockUri export const Range = mockRange export const Position = mockPosition diff --git a/src/core/task/Task.ts b/src/core/task/Task.ts index 500622ee17..2f04024396 100644 --- a/src/core/task/Task.ts +++ b/src/core/task/Task.ts @@ -151,6 +151,12 @@ import { shouldAddUserMessageToHistory } from "./messageCounting" import { type TaskExecutionContext } from "./providerHandoff" import { initialRunState, type Ask, type RunEvent, type RunState } from "./run-state/runState" import { applyRunEvent, type RunRejection } from "./run-state/runStateShadow" +import { + divergenceKey, + findDivergences, + isDivergenceCheckEnabled, + reportDivergence, +} from "./run-state/runStateDivergence" const MAX_EXPONENTIAL_BACKOFF_SECONDS = 600 // 10 minutes const DEFAULT_USAGE_COLLECTION_TIMEOUT_MS = 5000 // 5 seconds @@ -409,6 +415,8 @@ export class Task extends EventEmitter implements TaskLike { runState: RunState = initialRunState readonly runRejections: RunRejection[] = [] + private readonly divergenceCheckEnabled: boolean + private readonly reportedDivergences = new Set() // API apiConfiguration: ProviderSettings @@ -734,6 +742,7 @@ export class Task extends EventEmitter implements TaskLike { ) this.providerRef = new WeakRef(provider) this.globalStoragePath = provider.context.globalStorageUri.fsPath + this.divergenceCheckEnabled = isDivergenceCheckEnabled(Package.releaseChannel, provider.context.extensionMode) this.diffViewProvider = new DiffViewProvider(this.cwd, this) this.enableCheckpoints = enableCheckpoints this.checkpointTimeout = checkpointTimeout @@ -3458,7 +3467,26 @@ export class Task extends EventEmitter implements TaskLike { } private driveRunEvent(event: RunEvent): void { - this.runState = applyRunEvent(this.runState, event, this.runRejections) + const { state, rejected } = applyRunEvent(this.runState, event, this.runRejections) + this.runState = state + + if (this.divergenceCheckEnabled) { + const fields = { + abort: this.abort, + abandoned: this.abandoned, + abortReason: this.abortReason, + isInitialized: this.isInitialized, + isStreaming: this.isStreaming, + didFinishAbortingStream: this.didFinishAbortingStream, + } + for (const divergence of findDivergences({ event, state, fields, rejected })) { + if (this.reportedDivergences.has(divergenceKey(divergence))) { + continue + } + this.reportedDivergences.add(divergenceKey(divergence)) + reportDivergence(this.taskId, divergence) + } + } } public abortTask(isAbandoned = false): Promise { diff --git a/src/core/task/__tests__/Task.run-shadow.spec.ts b/src/core/task/__tests__/Task.run-shadow.spec.ts index 4e0c2b169b..ced7c3298f 100644 --- a/src/core/task/__tests__/Task.run-shadow.spec.ts +++ b/src/core/task/__tests__/Task.run-shadow.spec.ts @@ -1,6 +1,8 @@ // npx vitest run core/task/__tests__/Task.run-shadow.spec.ts +import * as vscode from "vscode" import { providerIdentifiers } from "@roo-code/types/provider-identifiers" +import { TelemetryEventName } from "@roo-code/types" import { TelemetryService } from "@roo-code/telemetry" import { makeExtensionContext, makeUri } from "../../../test-utils/vscode" @@ -11,10 +13,13 @@ import { initialRunState } from "../run-state/runState" vi.mock("../../../i18n", () => ({ t: (key: string) => key })) vi.mock("../../ignore/RooIgnoreController") -function makeTask(): Task { +function makeTask(extensionMode = vscode.ExtensionMode.Production): Task { const provider = { log: vi.fn(), - context: makeExtensionContext({ globalStorageUri: makeUri("/mock/global-storage") }), + context: makeExtensionContext({ + globalStorageUri: makeUri("/mock/global-storage"), + extensionMode, + }), } // Double cast: the constructor reads only these provider members. return new Task({ @@ -129,3 +134,71 @@ describe("Task run-state write helpers", () => { expect(task.runRejections).toEqual([]) }) }) + +describe("Task run-state divergence report", () => { + let warn: ReturnType + let captureEvent: ReturnType + + const divergenceMetrics = () => + captureEvent.mock.calls.filter( + ([name]: unknown[]) => + name === TelemetryEventName.RUN_STATE_DIVERGENCE || name === TelemetryEventName.RUN_STATE_KNOWN_RACE, + ) + + const divergenceWarnings = () => + warn.mock.calls.filter(([message]: unknown[]) => message === "[RunState] divergence") + + beforeEach(() => { + if (!TelemetryService.hasInstance()) { + TelemetryService.createInstance([]) + } + warn = vi.spyOn(console, "warn").mockImplementation(() => {}) + captureEvent = vi.spyOn(TelemetryService.instance, "captureEvent").mockImplementation(() => {}) + }) + + afterEach(() => { + vi.restoreAllMocks() + }) + + it("logs and counts a rejected event once when the check is enabled", () => { + const task = makeTask(vscode.ExtensionMode.Test) + + task.requestAbandon() + + expect(divergenceWarnings()).toHaveLength(1) + expect(divergenceWarnings()[0][1]).toMatchObject({ + taskId: task.taskId, + kind: "rejection", + event: { tag: "abandonRequested" }, + state: initialRunState, + mismatched: [], + }) + expect(divergenceMetrics()).toEqual([ + [ + TelemetryEventName.RUN_STATE_DIVERGENCE, + expect.objectContaining({ taskId: task.taskId, kind: "rejection", eventTag: "abandonRequested" }), + ], + ]) + }) + + it("reports a repeated rejection of the same event once per task", () => { + const task = makeTask(vscode.ExtensionMode.Test) + + task.requestAbandon() + task.requestAbandon() + + expect(task.runRejections).toHaveLength(2) + expect(divergenceWarnings()).toHaveLength(1) + expect(divergenceMetrics()).toHaveLength(1) + }) + + it("stays silent for a production build on the stable channel", () => { + const task = makeTask(vscode.ExtensionMode.Production) + + task.requestAbandon() + + expect(task.runRejections).toHaveLength(1) + expect(divergenceWarnings()).toHaveLength(0) + expect(divergenceMetrics()).toHaveLength(0) + }) +}) diff --git a/src/core/task/run-state/__tests__/runStateDivergence.spec.ts b/src/core/task/run-state/__tests__/runStateDivergence.spec.ts new file mode 100644 index 0000000000..afdcae6d48 --- /dev/null +++ b/src/core/task/run-state/__tests__/runStateDivergence.spec.ts @@ -0,0 +1,298 @@ +import { TelemetryEventName } from "@roo-code/types" +import { TelemetryService } from "@roo-code/telemetry" + +import { initialRunState, nextRunState, type RunEvent, type RunState } from "../runState" +import { + divergenceKey, + findDivergences, + isDivergenceCheckEnabled, + KNOWN_RACES, + reportDivergence, + type KnownRace, +} from "../runStateDivergence" +import type { RunStateFields } from "../runStateShadow" + +const idle: RunStateFields = { + abort: false, + abandoned: false, + isInitialized: false, + isStreaming: false, + didFinishAbortingStream: false, +} + +function drive(events: RunEvent[]): RunState { + return events.reduce((state, event) => { + const result = nextRunState(state, event) + if (!("ok" in result)) { + throw new Error(`expected ${event.tag} to be accepted`) + } + return result.ok + }, initialRunState as RunState) +} + +const fixtureRaces: KnownRace[] = [ + { + id: "R-1", + matches: (d) => d.mismatched.length === 1 && d.mismatched[0] === "didFinishAbortingStream", + }, + { + id: "R-3", + matches: (d) => + d.kind === "rejection" && + d.event.tag === "askStarted" && + d.state.phase === "running" && + d.state.latches.abort, + }, + { + id: "R-4", + matches: (d) => d.mismatched.length === 1 && d.mismatched[0] === "abortReason", + }, +] + +const allFields: Array = [ + "abort", + "abandoned", + "abortReason", + "isInitialized", + "isStreaming", + "didFinishAbortingStream", +] + +describe("findDivergences", () => { + it("reports nothing when the fields and the kernel agree", () => { + const state = drive([{ tag: "initialized" }]) + const divergences = findDivergences({ + event: { tag: "initialized" }, + state, + fields: { ...idle, isInitialized: true }, + rejected: false, + routed: allFields, + }) + expect(divergences).toEqual([]) + }) + + it("reports an untagged mismatch for a routed field", () => { + const state = drive([{ tag: "initialized" }]) + const [divergence, ...rest] = findDivergences({ + event: { tag: "initialized" }, + state, + fields: { ...idle, isInitialized: true, abort: true }, + rejected: false, + routed: allFields, + }) + expect(rest).toEqual([]) + expect(divergence).toMatchObject({ kind: "mismatch", mismatched: ["abort"] }) + expect(divergence.raceId).toBeUndefined() + }) + + it("ignores a field that is not routed", () => { + const divergences = findDivergences({ + event: { tag: "initialized" }, + state: initialRunState, + fields: { ...idle, abort: true }, + rejected: false, + routed: ["isInitialized"], + }) + expect(divergences).toEqual([]) + }) + + it("reports an untagged rejection", () => { + const [divergence] = findDivergences({ + event: { tag: "streamEnded" }, + state: initialRunState, + fields: idle, + rejected: true, + routed: [], + }) + expect(divergence).toMatchObject({ kind: "rejection", event: { tag: "streamEnded" } }) + expect(divergence.raceId).toBeUndefined() + }) + + it("tags a stale didFinishAbortingStream on a new stream as R-1", () => { + const state = drive([{ tag: "initialized" }, { tag: "streamStarted" }]) + const [divergence] = findDivergences({ + event: { tag: "streamStarted" }, + state, + fields: { ...idle, isInitialized: true, isStreaming: true, didFinishAbortingStream: true }, + rejected: false, + routed: allFields, + races: fixtureRaces, + }) + expect(divergence).toMatchObject({ kind: "mismatch", raceId: "R-1" }) + }) + + it("tags an ask rejected after abort as R-3", () => { + const state = drive([{ tag: "initialized" }, { tag: "abortRequested", abandoned: false }]) + const [divergence] = findDivergences({ + event: { tag: "askStarted", ask: { tag: "approval", askTs: 1 } }, + state, + fields: { ...idle, isInitialized: true, abort: true }, + rejected: true, + routed: allFields, + races: fixtureRaces, + }) + expect(divergence).toMatchObject({ kind: "rejection", raceId: "R-3" }) + }) + + it("does not tag an ask rejected before initialization as R-3", () => { + const [divergence] = findDivergences({ + event: { tag: "askStarted", ask: { tag: "approval", askTs: 1 } }, + state: initialRunState, + fields: idle, + rejected: true, + routed: allFields, + races: fixtureRaces, + }) + expect(divergence.raceId).toBeUndefined() + }) + + it("tags a reason overwritten after a cancel as R-4", () => { + const state = drive([{ tag: "reasonSet", reason: "user_cancelled" }]) + const [divergence] = findDivergences({ + event: { tag: "reasonSet", reason: "streaming_failed" }, + state, + fields: { ...idle, abortReason: "streaming_failed" }, + rejected: false, + routed: allFields, + races: fixtureRaces, + }) + expect(divergence).toMatchObject({ kind: "mismatch", mismatched: ["abortReason"], raceId: "R-4" }) + }) + + it("leaves a mismatch untagged when an unknown field mismatches beside a known race", () => { + const state = drive([{ tag: "reasonSet", reason: "user_cancelled" }]) + const [divergence] = findDivergences({ + event: { tag: "reasonSet", reason: "streaming_failed" }, + state, + fields: { ...idle, abortReason: "streaming_failed", abandoned: true }, + rejected: false, + routed: allFields, + races: fixtureRaces, + }) + expect(divergence.mismatched).toEqual(["abandoned", "abortReason"]) + expect(divergence.raceId).toBeUndefined() + }) +}) + +describe("KNOWN_RACES", () => { + it("is empty because the fixes for R-1, R-3, and R-4 have merged", () => { + expect(KNOWN_RACES).toEqual([]) + }) + + it("leaves a stale didFinishAbortingStream untagged by default", () => { + const state = drive([{ tag: "initialized" }, { tag: "streamStarted" }]) + const [divergence] = findDivergences({ + event: { tag: "streamStarted" }, + state, + fields: { ...idle, isInitialized: true, isStreaming: true, didFinishAbortingStream: true }, + rejected: false, + routed: allFields, + }) + expect(divergence).toMatchObject({ kind: "mismatch", mismatched: ["didFinishAbortingStream"] }) + expect(divergence.raceId).toBeUndefined() + }) +}) + +describe("divergenceKey", () => { + it("is the same for the same kind, event tag, and fields", () => { + const base = { kind: "mismatch" as const, event: { tag: "initialized" } as const, state: initialRunState } + expect(divergenceKey({ ...base, fields: idle, mismatched: ["abort"] })).toBe( + divergenceKey({ ...base, fields: { ...idle, abort: true }, mismatched: ["abort"] }), + ) + expect(divergenceKey({ ...base, fields: idle, mismatched: ["abort"] })).not.toBe( + divergenceKey({ ...base, fields: idle, mismatched: ["abandoned"] }), + ) + }) +}) + +describe("divergenceKey rejections", () => { + it("differs when the kernel rejects the same event for a different reason", () => { + const event: RunEvent = { tag: "askStarted", ask: { tag: "approval", askTs: 1 } } + const rejection = (state: RunState) => ({ + kind: "rejection" as const, + event, + state, + fields: idle, + mismatched: [], + }) + + const beforeInit = rejection(initialRunState) + const afterAbort = rejection(drive([{ tag: "initialized" }, { tag: "abortRequested", abandoned: false }])) + + expect(divergenceKey(beforeInit)).not.toBe(divergenceKey(afterAbort)) + }) +}) + +describe("isDivergenceCheckEnabled", () => { + it.each([ + ["stable", 1, false], + ["stable", undefined, false], + ["prerelease", 1, true], + ["stable", 2, true], + ["stable", 3, true], + ])("channel %s, mode %s -> %s", (channel, mode, expected) => { + expect(isDivergenceCheckEnabled(channel, mode)).toBe(expected) + }) +}) + +describe("reportDivergence", () => { + let captureEvent: ReturnType + let warn: ReturnType + + beforeEach(() => { + if (!TelemetryService.hasInstance()) { + TelemetryService.createInstance([]) + } + captureEvent = vi.spyOn(TelemetryService.instance, "captureEvent").mockImplementation(() => {}) + warn = vi.spyOn(console, "warn").mockImplementation(() => {}) + }) + + afterEach(() => { + vi.restoreAllMocks() + }) + + const state = drive([{ tag: "initialized" }]) + const mismatch = { + kind: "mismatch" as const, + event: { tag: "initialized" } as const, + state, + fields: { ...idle, abort: true }, + mismatched: ["abort" as const], + } + + it("logs once and counts an untagged divergence on the divergence metric", () => { + reportDivergence("task-1", mismatch) + + expect(warn).toHaveBeenCalledTimes(1) + expect(warn).toHaveBeenCalledWith("[RunState] divergence", { + taskId: "task-1", + kind: "mismatch", + raceId: undefined, + event: mismatch.event, + state, + fields: mismatch.fields, + mismatched: ["abort"], + }) + expect(captureEvent).toHaveBeenCalledTimes(1) + expect(captureEvent).toHaveBeenCalledWith(TelemetryEventName.RUN_STATE_DIVERGENCE, { + taskId: "task-1", + kind: "mismatch", + raceId: undefined, + eventTag: "initialized", + mismatched: ["abort"], + phase: "running", + streamTag: "none", + abort: false, + }) + }) + + it("counts a tagged divergence on the known-race metric with its race ID", () => { + reportDivergence("task-1", { ...mismatch, raceId: "R-3" }) + + expect(captureEvent).toHaveBeenCalledTimes(1) + expect(captureEvent).toHaveBeenCalledWith( + TelemetryEventName.RUN_STATE_KNOWN_RACE, + expect.objectContaining({ raceId: "R-3" }), + ) + }) +}) diff --git a/src/core/task/run-state/__tests__/runStateShadow.spec.ts b/src/core/task/run-state/__tests__/runStateShadow.spec.ts index fa7712ff77..8e6e67966c 100644 --- a/src/core/task/run-state/__tests__/runStateShadow.spec.ts +++ b/src/core/task/run-state/__tests__/runStateShadow.spec.ts @@ -81,14 +81,16 @@ describe("applyRunEvent", () => { it("returns the next state and records nothing when the kernel accepts the event", () => { const rejections: RunRejection[] = [] const next = applyRunEvent(initialRunState, { tag: "initialized" }, rejections) - expect(next.phase).toBe("running") + expect(next.state.phase).toBe("running") + expect(next.rejected).toBe(false) expect(rejections).toEqual([]) }) it("keeps the state and records the event and state when the kernel rejects the event", () => { const rejections: RunRejection[] = [] const next = applyRunEvent(initialRunState, { tag: "streamEnded" }, rejections) - expect(next).toBe(initialRunState) + expect(next.state).toBe(initialRunState) + expect(next.rejected).toBe(true) expect(rejections).toEqual([{ event: { tag: "streamEnded" }, state: initialRunState }]) }) diff --git a/src/core/task/run-state/runStateDivergence.ts b/src/core/task/run-state/runStateDivergence.ts new file mode 100644 index 0000000000..83bd075c27 --- /dev/null +++ b/src/core/task/run-state/runStateDivergence.ts @@ -0,0 +1,86 @@ +import { TelemetryEventName } from "@roo-code/types" +import { TelemetryService } from "@roo-code/telemetry" + +import type { RunEvent, RunState } from "./runState" +import { projectRunState, readRunStateFields, type RunStateFields } from "./runStateShadow" + +export type RaceId = `R-${number}` + +export type Divergence = { + kind: "mismatch" | "rejection" + raceId?: RaceId + event: RunEvent + state: RunState + fields: RunStateFields + mismatched: Array +} + +export const ROUTED_FIELDS: ReadonlyArray = [] + +export type KnownRace = { id: RaceId; matches: (divergence: Divergence) => boolean } + +// Source: "Known code races" in task-lifecycle-target-architecture.md. +// Add an entry only for a race that still has an open fix issue. Delete it when that issue merges. +export const KNOWN_RACES: ReadonlyArray = [] + +// Literals, not the vscode enum: specs that mock "vscode" without ExtensionMode would throw on the read. +const EXTENSION_MODE_DEVELOPMENT = 2 +const EXTENSION_MODE_TEST = 3 + +export function isDivergenceCheckEnabled(releaseChannel: string, extensionMode: number | undefined): boolean { + return ( + releaseChannel === "prerelease" || + extensionMode === EXTENSION_MODE_DEVELOPMENT || + extensionMode === EXTENSION_MODE_TEST + ) +} + +export const divergenceKey = (d: Divergence): string => + `${d.kind}:${d.event.tag}:${d.mismatched.join(",")}:${d.state.phase}:${d.state.latches.abort}` + +export function findDivergences(input: { + event: RunEvent + state: RunState + fields: RunStateFields + rejected: boolean + routed?: ReadonlyArray + races?: ReadonlyArray +}): Divergence[] { + const { event, state, fields, rejected, routed = ROUTED_FIELDS, races = KNOWN_RACES } = input + const expected = projectRunState(fields) + const actual = readRunStateFields(state) + const mismatched = routed.filter((field) => expected[field] !== actual[field]) + + const candidates: Divergence[] = [] + if (rejected) { + candidates.push({ kind: "rejection", event, state, fields, mismatched: [] }) + } + if (mismatched.length > 0) { + candidates.push({ kind: "mismatch", event, state, fields, mismatched }) + } + return candidates.map((divergence) => { + const race = races.find((known) => known.matches(divergence)) + return race ? { ...divergence, raceId: race.id } : divergence + }) +} + +export function reportDivergence(taskId: string, divergence: Divergence): void { + const { kind, raceId, event, state, fields, mismatched } = divergence + console.warn("[RunState] divergence", { taskId, kind, raceId, event, state, fields, mismatched }) + + if (TelemetryService.hasInstance()) { + TelemetryService.instance.captureEvent( + raceId ? TelemetryEventName.RUN_STATE_KNOWN_RACE : TelemetryEventName.RUN_STATE_DIVERGENCE, + { + taskId, + kind, + raceId, + eventTag: event.tag, + mismatched, + phase: state.phase, + streamTag: state.stream.tag, + abort: state.latches.abort, + }, + ) + } +} diff --git a/src/core/task/run-state/runStateShadow.ts b/src/core/task/run-state/runStateShadow.ts index 1010108714..9d39138573 100644 --- a/src/core/task/run-state/runStateShadow.ts +++ b/src/core/task/run-state/runStateShadow.ts @@ -38,14 +38,18 @@ export function projectRunState(fields: RunStateFields): RunStateFields { return { ...fields, didFinishAbortingStream: fields.isStreaming && fields.didFinishAbortingStream } } -export function applyRunEvent(state: RunState, event: RunEvent, rejections: RunRejection[]): RunState { +export function applyRunEvent( + state: RunState, + event: RunEvent, + rejections: RunRejection[], +): { state: RunState; rejected: boolean } { const result = nextRunState(state, event) if ("ok" in result) { - return result.ok + return { state: result.ok, rejected: false } } rejections.push({ event, state }) if (rejections.length > MAX_RUN_REJECTIONS) { rejections.shift() } - return state + return { state, rejected: true } } From d57dd483e3d577b390e70d7f2ac1dc24d4cd5fd4 Mon Sep 17 00:00:00 2001 From: Elliott de Launay Date: Tue, 13 Oct 2026 21:14:52 -0400 Subject: [PATCH 2/3] fix(task): key divergence dedupe on the event payload and test the schema --- .../types/src/__tests__/telemetry.test.ts | 78 +++++++++++++++++++ .../__tests__/runStateDivergence.spec.ts | 19 +++++ src/core/task/run-state/runStateDivergence.ts | 2 +- 3 files changed, 98 insertions(+), 1 deletion(-) diff --git a/packages/types/src/__tests__/telemetry.test.ts b/packages/types/src/__tests__/telemetry.test.ts index 6b8a9fc30c..926ab4dcfe 100644 --- a/packages/types/src/__tests__/telemetry.test.ts +++ b/packages/types/src/__tests__/telemetry.test.ts @@ -13,6 +13,8 @@ import { ConsecutiveMistakeError, isConsecutiveMistakeError, extractConsecutiveMistakeErrorProperties, + rooCodeTelemetryEventSchema, + TelemetryEventName, } from "../telemetry.js" describe("telemetry error utilities", () => { @@ -575,3 +577,79 @@ describe("telemetry error utilities", () => { }) }) }) + +describe("rooCodeTelemetryEventSchema run-state events", () => { + const appProperties = { + appName: "zoo-code", + appVersion: "1.0.0", + vscodeVersion: "1.100.0", + platform: "linux", + editorName: "vscode", + language: "en", + mode: "code", + } + const runStateProperties = { + ...appProperties, + taskId: "task-1", + kind: "rejection", + eventTag: "askStarted", + mismatched: [], + phase: "running", + streamTag: "none", + abort: true, + } + + it.each([TelemetryEventName.RUN_STATE_DIVERGENCE, TelemetryEventName.RUN_STATE_KNOWN_RACE])( + "keeps the run-state properties of %s", + (type) => { + const result = rooCodeTelemetryEventSchema.safeParse({ + type, + properties: { ...runStateProperties, raceId: "R-9", mismatched: ["abortReason"] }, + }) + + expect(result.success).toBe(true) + expect(result.data?.properties).toMatchObject({ + taskId: "task-1", + kind: "rejection", + raceId: "R-9", + eventTag: "askStarted", + mismatched: ["abortReason"], + phase: "running", + streamTag: "none", + abort: true, + }) + }, + ) + + it("accepts an event without a race ID", () => { + const result = rooCodeTelemetryEventSchema.safeParse({ + type: TelemetryEventName.RUN_STATE_DIVERGENCE, + properties: runStateProperties, + }) + + expect(result.success).toBe(true) + }) + + it("rejects an unknown kind", () => { + const result = rooCodeTelemetryEventSchema.safeParse({ + type: TelemetryEventName.RUN_STATE_DIVERGENCE, + properties: { ...runStateProperties, kind: "other" }, + }) + + expect(result.success).toBe(false) + }) + + it.each(["taskId", "eventTag", "mismatched", "phase", "streamTag", "abort"])( + "rejects an event without %s", + (property) => { + const { [property]: _omitted, ...properties } = runStateProperties as Record + + const result = rooCodeTelemetryEventSchema.safeParse({ + type: TelemetryEventName.RUN_STATE_KNOWN_RACE, + properties, + }) + + expect(result.success).toBe(false) + }, + ) +}) diff --git a/src/core/task/run-state/__tests__/runStateDivergence.spec.ts b/src/core/task/run-state/__tests__/runStateDivergence.spec.ts index afdcae6d48..c69d1fe4b5 100644 --- a/src/core/task/run-state/__tests__/runStateDivergence.spec.ts +++ b/src/core/task/run-state/__tests__/runStateDivergence.spec.ts @@ -223,6 +223,25 @@ describe("divergenceKey rejections", () => { }) }) +describe("divergenceKey event payload", () => { + const state = drive([{ tag: "initialized" }, { tag: "abortRequested", abandoned: false }]) + const rejectedAsk = (askTs: number) => ({ + kind: "rejection" as const, + event: { tag: "askStarted", ask: { tag: "approval", askTs } } as const, + state, + fields: idle, + mismatched: [], + }) + + it("differs for rejected events with different payloads", () => { + expect(divergenceKey(rejectedAsk(1))).not.toBe(divergenceKey(rejectedAsk(2))) + }) + + it("is the same for a repeated identical event", () => { + expect(divergenceKey(rejectedAsk(1))).toBe(divergenceKey(rejectedAsk(1))) + }) +}) + describe("isDivergenceCheckEnabled", () => { it.each([ ["stable", 1, false], diff --git a/src/core/task/run-state/runStateDivergence.ts b/src/core/task/run-state/runStateDivergence.ts index 83bd075c27..6eda0e97bb 100644 --- a/src/core/task/run-state/runStateDivergence.ts +++ b/src/core/task/run-state/runStateDivergence.ts @@ -36,7 +36,7 @@ export function isDivergenceCheckEnabled(releaseChannel: string, extensionMode: } export const divergenceKey = (d: Divergence): string => - `${d.kind}:${d.event.tag}:${d.mismatched.join(",")}:${d.state.phase}:${d.state.latches.abort}` + `${d.kind}:${JSON.stringify(d.event)}:${d.mismatched.join(",")}:${d.state.phase}:${d.state.latches.abort}` export function findDivergences(input: { event: RunEvent From 9d1e716d91c2d2ada7572b4fa788f570e463644e Mon Sep 17 00:00:00 2001 From: Elliott de Launay Date: Tue, 13 Oct 2026 23:21:38 -0400 Subject: [PATCH 3/3] fix(task): sort event keys in the divergence key --- .../__tests__/runStateDivergence.spec.ts | 18 ++++++++++++++++++ src/core/task/run-state/runStateDivergence.ts | 7 ++++++- 2 files changed, 24 insertions(+), 1 deletion(-) diff --git a/src/core/task/run-state/__tests__/runStateDivergence.spec.ts b/src/core/task/run-state/__tests__/runStateDivergence.spec.ts index c69d1fe4b5..9e866b4262 100644 --- a/src/core/task/run-state/__tests__/runStateDivergence.spec.ts +++ b/src/core/task/run-state/__tests__/runStateDivergence.spec.ts @@ -240,6 +240,15 @@ describe("divergenceKey event payload", () => { it("is the same for a repeated identical event", () => { expect(divergenceKey(rejectedAsk(1))).toBe(divergenceKey(rejectedAsk(1))) }) + + it("is the same when the event properties are built in a different order", () => { + const reordered = { + ...rejectedAsk(1), + event: { ask: { askTs: 1, tag: "approval" }, tag: "askStarted" } as const, + } + + expect(divergenceKey(reordered)).toBe(divergenceKey(rejectedAsk(1))) + }) }) describe("isDivergenceCheckEnabled", () => { @@ -305,6 +314,15 @@ describe("reportDivergence", () => { }) }) + it("still logs when telemetry is not created", () => { + vi.spyOn(TelemetryService, "hasInstance").mockReturnValue(false) + + reportDivergence("task-1", mismatch) + + expect(warn).toHaveBeenCalledTimes(1) + expect(captureEvent).not.toHaveBeenCalled() + }) + it("counts a tagged divergence on the known-race metric with its race ID", () => { reportDivergence("task-1", { ...mismatch, raceId: "R-3" }) diff --git a/src/core/task/run-state/runStateDivergence.ts b/src/core/task/run-state/runStateDivergence.ts index 6eda0e97bb..eb1190e0e2 100644 --- a/src/core/task/run-state/runStateDivergence.ts +++ b/src/core/task/run-state/runStateDivergence.ts @@ -35,8 +35,13 @@ export function isDivergenceCheckEnabled(releaseChannel: string, extensionMode: ) } +const sortKeys = (_key: string, value: unknown): unknown => + value && typeof value === "object" && !Array.isArray(value) + ? Object.fromEntries(Object.entries(value).sort(([a], [b]) => (a < b ? -1 : 1))) + : value + export const divergenceKey = (d: Divergence): string => - `${d.kind}:${JSON.stringify(d.event)}:${d.mismatched.join(",")}:${d.state.phase}:${d.state.latches.abort}` + `${d.kind}:${JSON.stringify(d.event, sortKeys)}:${d.mismatched.join(",")}:${d.state.phase}:${d.state.latches.abort}` export function findDivergences(input: { event: RunEvent