From 304c2ef28fba80ce691aa7ed081601289f0dde81 Mon Sep 17 00:00:00 2001 From: Nathaniel Pogue Date: Mon, 25 May 2026 14:02:55 -0400 Subject: [PATCH 1/5] Fix WebRTC alignment and RN setVolume on LiveKit voice sessions --- packages/client/src/VoiceConversation.test.ts | 213 ++++++++++++++++++ packages/client/src/VoiceConversation.ts | 8 +- .../client/src/utils/WebRTCConnection.test.ts | 177 ++++++++++++--- packages/client/src/utils/WebRTCConnection.ts | 37 ++- .../src/utils/WebSocketConnection.test.ts | 32 +++ 5 files changed, 430 insertions(+), 37 deletions(-) create mode 100644 packages/client/src/VoiceConversation.test.ts diff --git a/packages/client/src/VoiceConversation.test.ts b/packages/client/src/VoiceConversation.test.ts new file mode 100644 index 00000000..e4a02ce9 --- /dev/null +++ b/packages/client/src/VoiceConversation.test.ts @@ -0,0 +1,213 @@ +import { describe, it, expect, vi, beforeEach, afterEach } from "vitest"; + +import { VoiceConversation } from "./VoiceConversation.js"; +import { + BaseConversation, + type Options, + type PartialOptions, +} from "./BaseConversation.js"; +import type { BaseConnection } from "./utils/BaseConnection.js"; +import type { InputController } from "./InputController.js"; +import type { OutputController } from "./OutputController.js"; +import type { AgentAudioEvent } from "./utils/events.js"; + +const noopInput = { + close: vi.fn(), + setDevice: vi.fn(), + setMuted: vi.fn(), + isMuted: () => false, + getAnalyser: () => undefined, + getVolume: () => 0, + getByteFrequencyData: () => {}, +} satisfies InputController; + +const noopOutput = { + close: vi.fn(), + setDevice: vi.fn(), + setVolume: vi.fn(), + interrupt: vi.fn(), + getAnalyser: () => undefined, + getVolume: () => 0, + getByteFrequencyData: () => {}, +} satisfies OutputController; + +export class TestVoiceConversation extends VoiceConversation { + public static create( + options: Partial = {}, + connection: BaseConnection = { + conversationId: "test-conversation-id", + inputFormat: { format: "pcm", sampleRate: 16000 }, + outputFormat: { format: "pcm", sampleRate: 16000 }, + onMessage: () => {}, + onDisconnect: () => {}, + onModeChange: () => {}, + close: () => {}, + sendMessage: () => {}, + } as unknown as BaseConnection + ) { + const fullOptions = BaseConversation.getFullOptions({ + agentId: "test-agent-id", + connectionType: "webrtc", + ...options, + } as PartialOptions); + + return new TestVoiceConversation( + fullOptions, + connection, + noopInput, + noopOutput, + null, + async () => {} + ); + } + + public handleAudioEvent(event: AgentAudioEvent) { + this.handleAudio(event); + } + + public simulateInterruption(eventId: number) { + this.handleInterruption({ + type: "interruption", + interruption_event: { event_id: eventId }, + }); + } +} + +describe("VoiceConversation", () => { + const alignment = { + chars: ["H", "e", "l", "l", "o"], + char_start_times_ms: [0, 80, 160, 240, 320], + char_durations_ms: [80, 80, 80, 80, 120], + }; + + it("fires onAudioAlignment when an audio event includes alignment data", () => { + const onAudioAlignment = vi.fn(); + const conversation = TestVoiceConversation.create({ onAudioAlignment }); + + conversation.handleAudioEvent({ + type: "audio", + audio_event: { + audio_base_64: "dGVzdA==", + event_id: 10, + alignment, + }, + }); + + expect(onAudioAlignment).toHaveBeenCalledWith(alignment); + }); + + it("does not fire onAudioAlignment for stale events after an interruption", () => { + const onAudioAlignment = vi.fn(); + const conversation = TestVoiceConversation.create({ onAudioAlignment }); + + conversation.simulateInterruption(20); + conversation.handleAudioEvent({ + type: "audio", + audio_event: { + audio_base_64: "dGVzdA==", + event_id: 5, + alignment, + }, + }); + + expect(onAudioAlignment).not.toHaveBeenCalled(); + }); + + it("fires onAudioAlignment without calling onAudio when audio_base_64 is omitted", () => { + const onAudioAlignment = vi.fn(); + const onAudio = vi.fn(); + const conversation = TestVoiceConversation.create({ + onAudioAlignment, + onAudio, + }); + + conversation.handleAudioEvent({ + type: "audio", + audio_event: { + audio_base_64: "", + event_id: 11, + alignment, + }, + }); + + expect(onAudioAlignment).toHaveBeenCalledWith(alignment); + expect(onAudio).not.toHaveBeenCalled(); + }); +}); + +describe("VoiceConversation transport integration", () => { + const alignment = { + chars: ["A", "B"], + char_start_times_ms: [0, 100], + char_durations_ms: [100, 100], + }; + + let listeners: Map void)[]>; + let mockSocket: Record; + + beforeEach(() => { + listeners = new Map(); + mockSocket = { + addEventListener: vi.fn((type: string, handler: (event: { data: string }) => void) => { + if (!listeners.has(type)) listeners.set(type, []); + listeners.get(type)!.push(handler); + }), + removeEventListener: vi.fn(), + send: vi.fn(), + close: vi.fn(), + }; + vi.stubGlobal( + "WebSocket", + vi.fn(function WebSocket() { + return mockSocket; + }) + ); + }); + + afterEach(() => { + vi.unstubAllGlobals(); + }); + + function emitMessage(data: unknown) { + for (const handler of listeners.get("message") ?? []) { + handler({ data: JSON.stringify(data) }); + } + } + + it("delivers onAudioAlignment over WebSocket", async () => { + const { WebSocketConnection } = await import("./utils/WebSocketConnection.js"); + const onAudioAlignment = vi.fn(); + + const promise = WebSocketConnection.create({ + agentId: "test-agent", + connectionType: "websocket", + }); + + for (const handler of listeners.get("open") ?? []) { + handler({ data: "" }); + } + emitMessage({ + type: "conversation_initiation_metadata", + conversation_initiation_metadata_event: { + conversation_id: "test-conv-id", + agent_output_audio_format: "pcm_16000", + user_input_audio_format: "pcm_16000", + }, + }); + + const connection = await promise; + TestVoiceConversation.create({ onAudioAlignment }, connection); + + emitMessage({ + type: "audio", + audio_event: { + audio_base_64: "dGVzdA==", + event_id: 12, + alignment, + }, + }); + + expect(onAudioAlignment).toHaveBeenCalledWith(alignment); + connection.close(); + }); +}); diff --git a/packages/client/src/VoiceConversation.ts b/packages/client/src/VoiceConversation.ts index caeadd4e..9c432775 100644 --- a/packages/client/src/VoiceConversation.ts +++ b/packages/client/src/VoiceConversation.ts @@ -113,11 +113,11 @@ export class VoiceConversation extends BaseConversation { protected override handleAudio(event: AgentAudioEvent) { super.handleAudio(event); - if (event.audio_event.alignment && this.options.onAudioAlignment) { - this.options.onAudioAlignment(event.audio_event.alignment); - } - if (this.lastInterruptTimestamp <= event.audio_event.event_id) { + if (event.audio_event.alignment && this.options.onAudioAlignment) { + this.options.onAudioAlignment(event.audio_event.alignment); + } + if (event.audio_event.audio_base_64) { this.options.onAudio?.(event.audio_event.audio_base_64); // Audio routing is handled by attachConnectionToOutput for WebSocket diff --git a/packages/client/src/utils/WebRTCConnection.test.ts b/packages/client/src/utils/WebRTCConnection.test.ts index fc59ae1d..23a6dcab 100644 --- a/packages/client/src/utils/WebRTCConnection.test.ts +++ b/packages/client/src/utils/WebRTCConnection.test.ts @@ -45,6 +45,7 @@ vi.mock("livekit-client", () => { ConnectionStateChanged: "connectionStateChanged", DataReceived: "dataReceived", TrackSubscribed: "trackSubscribed", + TrackUnsubscribed: "trackUnsubscribed", ActiveSpeakersChanged: "activeSpeakersChanged", ParticipantDisconnected: "participantDisconnected", }, @@ -64,11 +65,41 @@ import { WebRTCConnection } from "./WebRTCConnection.js"; import { Room, createLocalAudioTrack } from "livekit-client"; import { setWebRTCAudioAdapterFactory } from "../WebRTCAudioAdapter.js"; import { WebAudioAdapter } from "../platform/web/webAudioAdapter.js"; +import { TestVoiceConversation } from "../VoiceConversation.test.js"; + +async function createConnectionWithEventHandlers() { + const mockRoom = new Room() as any; + const eventHandlers = new Map void>(); + + (mockRoom.on as ReturnType).mockImplementation( + (event: string, callback: (...args: unknown[]) => void) => { + eventHandlers.set(event, callback); + if (event === "connected") { + queueMicrotask(() => callback()); + } + } + ); + (mockRoom.once as ReturnType).mockImplementation( + (event: string, callback: (...args: unknown[]) => void) => { + if (event === "signalConnected") { + queueMicrotask(() => callback()); + } + } + ); + + const connection = await WebRTCConnection.create({ + conversationToken: "test-token", + connectionType: "webrtc", + }); + + return { connection, eventHandlers, mockRoom }; +} describe("WebRTCConnection", () => { beforeEach(() => { vi.clearAllMocks(); vi.unstubAllGlobals(); + setWebRTCAudioAdapterFactory(undefined as never); (globalThis as Record).__mockCalls__ = { setMicrophoneEnabled: [], }; @@ -260,36 +291,9 @@ describe("WebRTCConnection", () => { }); describe("disconnection context", () => { - async function createWithHandlers() { - const mockRoom = new Room() as any; - const eventHandlers = new Map void>(); - - (mockRoom.on as ReturnType).mockImplementation( - (event: string, callback: (...args: unknown[]) => void) => { - eventHandlers.set(event, callback); - if (event === "connected") { - queueMicrotask(() => callback()); - } - } - ); - (mockRoom.once as ReturnType).mockImplementation( - (event: string, callback: (...args: unknown[]) => void) => { - if (event === "signalConnected") { - queueMicrotask(() => callback()); - } - } - ); - - const connection = await WebRTCConnection.create({ - conversationToken: "test-token", - connectionType: "webrtc", - }); - - return { connection, eventHandlers, mockRoom }; - } - it("emits agent disconnect with context on RoomEvent.Disconnected", async () => { - const { connection, eventHandlers } = await createWithHandlers(); + const { connection, eventHandlers } = + await createConnectionWithEventHandlers(); const onDisconnect = vi.fn(); connection.onDisconnect(onDisconnect); @@ -302,7 +306,8 @@ describe("WebRTCConnection", () => { }); it("emits error disconnect with context on ConnectionStateChanged to Disconnected", async () => { - const { connection, eventHandlers } = await createWithHandlers(); + const { connection, eventHandlers } = + await createConnectionWithEventHandlers(); const onDisconnect = vi.fn(); connection.onDisconnect(onDisconnect); @@ -316,7 +321,8 @@ describe("WebRTCConnection", () => { }); it("emits agent disconnect with context on agent ParticipantDisconnected", async () => { - const { connection, eventHandlers } = await createWithHandlers(); + const { connection, eventHandlers } = + await createConnectionWithEventHandlers(); const onDisconnect = vi.fn(); connection.onDisconnect(onDisconnect); @@ -329,6 +335,115 @@ describe("WebRTCConnection", () => { context: { type: "close", reason: "agent disconnected" }, }); }); + + }); + + it("forwards alignment on LiveKit RoomEvent.DataReceived without playback audio", async () => { + const alignment = { + chars: ["H", "i"], + char_start_times_ms: [0, 100], + char_durations_ms: [100, 150], + }; + const { connection, eventHandlers } = + await createConnectionWithEventHandlers(); + const onMessage = vi.fn(); + connection.onMessage(onMessage); + + eventHandlers.get("dataReceived")?.( + new TextEncoder().encode( + JSON.stringify({ + type: "audio", + audio_event: { + audio_base_64: "dGVzdA==", + event_id: 42, + alignment, + }, + }) + ) + ); + + expect(onMessage).toHaveBeenCalledWith({ + type: "audio", + audio_event: { + audio_base_64: "", + event_id: 42, + alignment, + }, + }); + connection.close(); + }); + + it("drops audio data channel messages without alignment", async () => { + const { connection, eventHandlers } = + await createConnectionWithEventHandlers(); + const onMessage = vi.fn(); + connection.onMessage(onMessage); + + eventHandlers.get("dataReceived")?.( + new TextEncoder().encode( + JSON.stringify({ + type: "audio", + audio_event: { + audio_base_64: "dGVzdA==", + event_id: 44, + }, + }) + ) + ); + + expect(onMessage).not.toHaveBeenCalled(); + connection.close(); + }); + + it("forwards onAudioAlignment through VoiceConversation over LiveKit WebRTC", async () => { + const alignment = { + chars: ["W", "o", "r", "d"], + char_start_times_ms: [0, 50, 100, 150], + char_durations_ms: [50, 50, 50, 80], + }; + const { connection, eventHandlers } = + await createConnectionWithEventHandlers(); + const onAudioAlignment = vi.fn(); + const onAudio = vi.fn(); + + TestVoiceConversation.create({ onAudioAlignment, onAudio }, connection); + + eventHandlers.get("dataReceived")?.( + new TextEncoder().encode( + JSON.stringify({ + type: "audio", + audio_event: { + audio_base_64: "dGVzdA==", + event_id: 100, + alignment, + }, + }) + ) + ); + + expect(onAudioAlignment).toHaveBeenCalledWith(alignment); + expect(onAudio).not.toHaveBeenCalled(); + connection.close(); + }); + + it("setVolume uses LiveKit track.setVolume when no audio adapter is registered", async () => { + const { connection, eventHandlers } = + await createConnectionWithEventHandlers(); + const mockSetVolume = vi.fn(); + + await eventHandlers.get("trackSubscribed")?.( + { + kind: "audio", + mediaStreamTrack: { id: "remote-track" }, + setVolume: mockSetVolume, + }, + {}, + { identity: "agent_123" } + ); + + connection.output.setVolume(0.75); + expect(mockSetVolume).toHaveBeenCalledWith(0.75); + connection.close(); }); it.each([ diff --git a/packages/client/src/utils/WebRTCConnection.ts b/packages/client/src/utils/WebRTCConnection.ts index d48a6979..ea699e23 100644 --- a/packages/client/src/utils/WebRTCConnection.ts +++ b/packages/client/src/utils/WebRTCConnection.ts @@ -62,6 +62,8 @@ export class WebRTCConnection extends BaseConnection { private audioAdapter: WebRTCAudioAdapter | null; + private remoteAudioTracks: RemoteAudioTrack[] = []; + private inputAnalyser: unknown = undefined; private inputVolumeProvider: VolumeProvider = NO_VOLUME; @@ -397,7 +399,15 @@ export class WebRTCConnection extends BaseConnection { const message = JSON.parse(new TextDecoder().decode(payload)); // Filter out audio messages for WebRTC - they're handled via audio tracks - if (message.type === "audio") { + if (isValidSocketEvent(message) && message.type === "audio") { + if (message.audio_event.alignment) { + const { audio_base_64: _audioBase64, ...audioEvent } = + message.audio_event; + this.handleMessage({ + type: "audio", + audio_event: { ...audioEvent, audio_base_64: "" }, + }); + } return; } @@ -425,6 +435,7 @@ export class WebRTCConnection extends BaseConnection { participant.identity.includes("agent") ) { const remoteAudioTrack = track as RemoteAudioTrack; + this.remoteAudioTracks.push(remoteAudioTrack); if (this.audioAdapter) { // Delegate playback to the platform-specific adapter @@ -442,6 +453,20 @@ export class WebRTCConnection extends BaseConnection { } ); + this.room.on( + RoomEvent.TrackUnsubscribed, + (track: Track, _publication: TrackPublication, participant: Participant) => { + if ( + track.kind === Track.Kind.Audio && + participant.identity.includes("agent") + ) { + this.remoteAudioTracks = this.remoteAudioTracks.filter( + remoteTrack => remoteTrack !== track + ); + } + } + ); + this.room.on( RoomEvent.ActiveSpeakersChanged, async (speakers: Participant[]) => { @@ -485,6 +510,7 @@ export class WebRTCConnection extends BaseConnection { // Delegate all audio cleanup to the adapter this.audioAdapter?.cleanup(); + this.remoteAudioTracks = []; this.inputAnalyser = undefined; this.outputAnalyser = undefined; this.inputVolumeProvider = NO_VOLUME; @@ -598,7 +624,14 @@ export class WebRTCConnection extends BaseConnection { } public setAudioVolume(volume: number) { - this.audioAdapter?.setVolume(volume); + if (this.audioAdapter) { + this.audioAdapter.setVolume(volume); + return; + } + + for (const track of this.remoteAudioTracks) { + track.setVolume?.(volume); + } } public async setAudioOutputDevice(deviceId: string): Promise { diff --git a/packages/client/src/utils/WebSocketConnection.test.ts b/packages/client/src/utils/WebSocketConnection.test.ts index d0c9953d..2ca24801 100644 --- a/packages/client/src/utils/WebSocketConnection.test.ts +++ b/packages/client/src/utils/WebSocketConnection.test.ts @@ -168,4 +168,36 @@ describe("WebSocketConnection", () => { expect(listener).not.toHaveBeenCalled(); }); + + it("forwards alignment-bearing audio on the WebSocket without filtering", async () => { + const connection = await createConnection(); + const onMessage = vi.fn(); + const alignment = { + chars: ["H", "i"], + char_start_times_ms: [0, 100], + char_durations_ms: [100, 150], + }; + + connection.onMessage(onMessage); + + emit("message", { + data: JSON.stringify({ + type: "audio", + audio_event: { + audio_base_64: "dGVzdA==", + event_id: 2, + alignment, + }, + }), + }); + + expect(onMessage).toHaveBeenCalledWith({ + type: "audio", + audio_event: { + audio_base_64: "dGVzdA==", + event_id: 2, + alignment, + }, + }); + }); }); From 468e2b72df60d511f51518280316b45afae9c554 Mon Sep 17 00:00:00 2001 From: Nathaniel Pogue Date: Mon, 25 May 2026 14:25:13 -0400 Subject: [PATCH 2/5] fixed cursorbot issue --- packages/client/src/VoiceConversation.test.ts | 13 ++++++++----- packages/client/src/utils/WebRTCConnection.test.ts | 1 - packages/client/src/utils/WebRTCConnection.ts | 6 +++++- 3 files changed, 13 insertions(+), 7 deletions(-) diff --git a/packages/client/src/VoiceConversation.test.ts b/packages/client/src/VoiceConversation.test.ts index e4a02ce9..119bab27 100644 --- a/packages/client/src/VoiceConversation.test.ts +++ b/packages/client/src/VoiceConversation.test.ts @@ -148,10 +148,12 @@ describe("VoiceConversation transport integration", () => { beforeEach(() => { listeners = new Map(); mockSocket = { - addEventListener: vi.fn((type: string, handler: (event: { data: string }) => void) => { - if (!listeners.has(type)) listeners.set(type, []); - listeners.get(type)!.push(handler); - }), + addEventListener: vi.fn( + (type: string, handler: (event: { data: string }) => void) => { + if (!listeners.has(type)) listeners.set(type, []); + listeners.get(type)!.push(handler); + } + ), removeEventListener: vi.fn(), send: vi.fn(), close: vi.fn(), @@ -175,7 +177,8 @@ describe("VoiceConversation transport integration", () => { } it("delivers onAudioAlignment over WebSocket", async () => { - const { WebSocketConnection } = await import("./utils/WebSocketConnection.js"); + const { WebSocketConnection } = + await import("./utils/WebSocketConnection.js"); const onAudioAlignment = vi.fn(); const promise = WebSocketConnection.create({ diff --git a/packages/client/src/utils/WebRTCConnection.test.ts b/packages/client/src/utils/WebRTCConnection.test.ts index 23a6dcab..d3b981fc 100644 --- a/packages/client/src/utils/WebRTCConnection.test.ts +++ b/packages/client/src/utils/WebRTCConnection.test.ts @@ -335,7 +335,6 @@ describe("WebRTCConnection", () => { context: { type: "close", reason: "agent disconnected" }, }); }); - }); it("forwards alignment on LiveKit RoomEvent.DataReceived without playback audio", async () => { diff --git a/packages/client/src/utils/WebRTCConnection.ts b/packages/client/src/utils/WebRTCConnection.ts index ea699e23..4a9e7aa0 100644 --- a/packages/client/src/utils/WebRTCConnection.ts +++ b/packages/client/src/utils/WebRTCConnection.ts @@ -455,7 +455,11 @@ export class WebRTCConnection extends BaseConnection { this.room.on( RoomEvent.TrackUnsubscribed, - (track: Track, _publication: TrackPublication, participant: Participant) => { + ( + track: Track, + _publication: TrackPublication, + participant: Participant + ) => { if ( track.kind === Track.Kind.Audio && participant.identity.includes("agent") From 4f6999bdad9d88aefa2a02f1d652e38de35a4aab Mon Sep 17 00:00:00 2001 From: Nathaniel Pogue Date: Mon, 25 May 2026 14:38:31 -0400 Subject: [PATCH 3/5] Fix WebRTC alignment and RN setVolume on LiveKit voice sessions --- packages/client/src/VoiceConversation.test.ts | 48 +-- .../client/src/utils/WebRTCConnection.test.ts | 289 ++++++++++-------- packages/client/src/utils/WebRTCConnection.ts | 4 +- .../src/utils/WebSocketConnection.test.ts | 2 +- 4 files changed, 184 insertions(+), 159 deletions(-) diff --git a/packages/client/src/VoiceConversation.test.ts b/packages/client/src/VoiceConversation.test.ts index 119bab27..00e767fe 100644 --- a/packages/client/src/VoiceConversation.test.ts +++ b/packages/client/src/VoiceConversation.test.ts @@ -135,7 +135,7 @@ describe("VoiceConversation", () => { }); }); -describe("VoiceConversation transport integration", () => { +describe("VoiceConversation WebSocket integration", () => { const alignment = { chars: ["A", "B"], char_start_times_ms: [0, 100], @@ -170,12 +170,6 @@ describe("VoiceConversation transport integration", () => { vi.unstubAllGlobals(); }); - function emitMessage(data: unknown) { - for (const handler of listeners.get("message") ?? []) { - handler({ data: JSON.stringify(data) }); - } - } - it("delivers onAudioAlignment over WebSocket", async () => { const { WebSocketConnection } = await import("./utils/WebSocketConnection.js"); @@ -189,26 +183,34 @@ describe("VoiceConversation transport integration", () => { for (const handler of listeners.get("open") ?? []) { handler({ data: "" }); } - emitMessage({ - type: "conversation_initiation_metadata", - conversation_initiation_metadata_event: { - conversation_id: "test-conv-id", - agent_output_audio_format: "pcm_16000", - user_input_audio_format: "pcm_16000", - }, - }); + for (const handler of listeners.get("message") ?? []) { + handler({ + data: JSON.stringify({ + type: "conversation_initiation_metadata", + conversation_initiation_metadata_event: { + conversation_id: "test-conv-id", + agent_output_audio_format: "pcm_16000", + user_input_audio_format: "pcm_16000", + }, + }), + }); + } const connection = await promise; TestVoiceConversation.create({ onAudioAlignment }, connection); - emitMessage({ - type: "audio", - audio_event: { - audio_base_64: "dGVzdA==", - event_id: 12, - alignment, - }, - }); + for (const handler of listeners.get("message") ?? []) { + handler({ + data: JSON.stringify({ + type: "audio", + audio_event: { + audio_base_64: "dGVzdA==", + event_id: 12, + alignment, + }, + }), + }); + } expect(onAudioAlignment).toHaveBeenCalledWith(alignment); connection.close(); diff --git a/packages/client/src/utils/WebRTCConnection.test.ts b/packages/client/src/utils/WebRTCConnection.test.ts index d3b981fc..f3ff90ad 100644 --- a/packages/client/src/utils/WebRTCConnection.test.ts +++ b/packages/client/src/utils/WebRTCConnection.test.ts @@ -67,39 +67,10 @@ import { setWebRTCAudioAdapterFactory } from "../WebRTCAudioAdapter.js"; import { WebAudioAdapter } from "../platform/web/webAudioAdapter.js"; import { TestVoiceConversation } from "../VoiceConversation.test.js"; -async function createConnectionWithEventHandlers() { - const mockRoom = new Room() as any; - const eventHandlers = new Map void>(); - - (mockRoom.on as ReturnType).mockImplementation( - (event: string, callback: (...args: unknown[]) => void) => { - eventHandlers.set(event, callback); - if (event === "connected") { - queueMicrotask(() => callback()); - } - } - ); - (mockRoom.once as ReturnType).mockImplementation( - (event: string, callback: (...args: unknown[]) => void) => { - if (event === "signalConnected") { - queueMicrotask(() => callback()); - } - } - ); - - const connection = await WebRTCConnection.create({ - conversationToken: "test-token", - connectionType: "webrtc", - }); - - return { connection, eventHandlers, mockRoom }; -} - describe("WebRTCConnection", () => { beforeEach(() => { vi.clearAllMocks(); vi.unstubAllGlobals(); - setWebRTCAudioAdapterFactory(undefined as never); (globalThis as Record).__mockCalls__ = { setMicrophoneEnabled: [], }; @@ -291,9 +262,36 @@ describe("WebRTCConnection", () => { }); describe("disconnection context", () => { + async function createWithHandlers() { + const mockRoom = new Room() as any; + const eventHandlers = new Map void>(); + + (mockRoom.on as ReturnType).mockImplementation( + (event: string, callback: (...args: unknown[]) => void) => { + eventHandlers.set(event, callback); + if (event === "connected") { + queueMicrotask(() => callback()); + } + } + ); + (mockRoom.once as ReturnType).mockImplementation( + (event: string, callback: (...args: unknown[]) => void) => { + if (event === "signalConnected") { + queueMicrotask(() => callback()); + } + } + ); + + const connection = await WebRTCConnection.create({ + conversationToken: "test-token", + connectionType: "webrtc", + }); + + return { connection, eventHandlers, mockRoom }; + } + it("emits agent disconnect with context on RoomEvent.Disconnected", async () => { - const { connection, eventHandlers } = - await createConnectionWithEventHandlers(); + const { connection, eventHandlers } = await createWithHandlers(); const onDisconnect = vi.fn(); connection.onDisconnect(onDisconnect); @@ -306,8 +304,7 @@ describe("WebRTCConnection", () => { }); it("emits error disconnect with context on ConnectionStateChanged to Disconnected", async () => { - const { connection, eventHandlers } = - await createConnectionWithEventHandlers(); + const { connection, eventHandlers } = await createWithHandlers(); const onDisconnect = vi.fn(); connection.onDisconnect(onDisconnect); @@ -321,8 +318,7 @@ describe("WebRTCConnection", () => { }); it("emits agent disconnect with context on agent ParticipantDisconnected", async () => { - const { connection, eventHandlers } = - await createConnectionWithEventHandlers(); + const { connection, eventHandlers } = await createWithHandlers(); const onDisconnect = vi.fn(); connection.onDisconnect(onDisconnect); @@ -337,112 +333,139 @@ describe("WebRTCConnection", () => { }); }); - it("forwards alignment on LiveKit RoomEvent.DataReceived without playback audio", async () => { - const alignment = { - chars: ["H", "i"], - char_start_times_ms: [0, 100], - char_durations_ms: [100, 150], - }; - const { connection, eventHandlers } = - await createConnectionWithEventHandlers(); - const onMessage = vi.fn(); - connection.onMessage(onMessage); - - eventHandlers.get("dataReceived")?.( - new TextEncoder().encode( - JSON.stringify({ - type: "audio", - audio_event: { - audio_base_64: "dGVzdA==", - event_id: 42, - alignment, - }, - }) - ) - ); + describe("WebRTC audio data channel", () => { + async function createWithHandlers() { + const mockRoom = new Room() as any; + const eventHandlers = new Map void>(); - expect(onMessage).toHaveBeenCalledWith({ - type: "audio", - audio_event: { - audio_base_64: "", - event_id: 42, - alignment, - }, + (mockRoom.on as ReturnType).mockImplementation( + (event: string, callback: (...args: unknown[]) => void) => { + eventHandlers.set(event, callback); + if (event === "connected") { + queueMicrotask(() => callback()); + } + } + ); + (mockRoom.once as ReturnType).mockImplementation( + (event: string, callback: (...args: unknown[]) => void) => { + if (event === "signalConnected") { + queueMicrotask(() => callback()); + } + } + ); + + const connection = await WebRTCConnection.create({ + conversationToken: "test-token", + connectionType: "webrtc", + }); + + return { connection, eventHandlers, mockRoom }; + } + + it("forwards alignment on data channel without playback audio", async () => { + const alignment = { + chars: ["H", "i"], + char_start_times_ms: [0, 100], + char_durations_ms: [100, 150], + }; + const { connection, eventHandlers } = await createWithHandlers(); + const onMessage = vi.fn(); + connection.onMessage(onMessage); + + eventHandlers.get("dataReceived")?.( + new TextEncoder().encode( + JSON.stringify({ + type: "audio", + audio_event: { + audio_base_64: "dGVzdA==", + event_id: 42, + alignment, + }, + }) + ) + ); + + expect(onMessage).toHaveBeenCalledWith({ + type: "audio", + audio_event: { + audio_base_64: "", + event_id: 42, + alignment, + }, + }); + connection.close(); }); - connection.close(); - }); - it("drops audio data channel messages without alignment", async () => { - const { connection, eventHandlers } = - await createConnectionWithEventHandlers(); - const onMessage = vi.fn(); - connection.onMessage(onMessage); - - eventHandlers.get("dataReceived")?.( - new TextEncoder().encode( - JSON.stringify({ - type: "audio", - audio_event: { - audio_base_64: "dGVzdA==", - event_id: 44, - }, - }) - ) - ); + it("drops audio data channel messages without alignment", async () => { + const { connection, eventHandlers } = await createWithHandlers(); + const onMessage = vi.fn(); + connection.onMessage(onMessage); + + eventHandlers.get("dataReceived")?.( + new TextEncoder().encode( + JSON.stringify({ + type: "audio", + audio_event: { + audio_base_64: "dGVzdA==", + event_id: 44, + }, + }) + ) + ); - expect(onMessage).not.toHaveBeenCalled(); - connection.close(); - }); + expect(onMessage).not.toHaveBeenCalled(); + connection.close(); + }); - it("forwards onAudioAlignment through VoiceConversation over LiveKit WebRTC", async () => { - const alignment = { - chars: ["W", "o", "r", "d"], - char_start_times_ms: [0, 50, 100, 150], - char_durations_ms: [50, 50, 50, 80], - }; - const { connection, eventHandlers } = - await createConnectionWithEventHandlers(); - const onAudioAlignment = vi.fn(); - const onAudio = vi.fn(); - - TestVoiceConversation.create({ onAudioAlignment, onAudio }, connection); - - eventHandlers.get("dataReceived")?.( - new TextEncoder().encode( - JSON.stringify({ - type: "audio", - audio_event: { - audio_base_64: "dGVzdA==", - event_id: 100, - alignment, - }, - }) - ) - ); + it("forwards onAudioAlignment through VoiceConversation", async () => { + const alignment = { + chars: ["W", "o", "r", "d"], + char_start_times_ms: [0, 50, 100, 150], + char_durations_ms: [50, 50, 50, 80], + }; + const { connection, eventHandlers } = await createWithHandlers(); + const onAudioAlignment = vi.fn(); + const onAudio = vi.fn(); + + TestVoiceConversation.create({ onAudioAlignment, onAudio }, connection); + + eventHandlers.get("dataReceived")?.( + new TextEncoder().encode( + JSON.stringify({ + type: "audio", + audio_event: { + audio_base_64: "dGVzdA==", + event_id: 100, + alignment, + }, + }) + ) + ); - expect(onAudioAlignment).toHaveBeenCalledWith(alignment); - expect(onAudio).not.toHaveBeenCalled(); - connection.close(); - }); + expect(onAudioAlignment).toHaveBeenCalledWith(alignment); + expect(onAudio).not.toHaveBeenCalled(); + connection.close(); + }); - it("setVolume uses LiveKit track.setVolume when no audio adapter is registered", async () => { - const { connection, eventHandlers } = - await createConnectionWithEventHandlers(); - const mockSetVolume = vi.fn(); - - await eventHandlers.get("trackSubscribed")?.( - { - kind: "audio", - mediaStreamTrack: { id: "remote-track" }, - setVolume: mockSetVolume, - }, - {}, - { identity: "agent_123" } - ); + it("setVolume uses LiveKit track.setVolume when no audio adapter is registered", async () => { + setWebRTCAudioAdapterFactory(undefined as never); + const { connection, eventHandlers } = await createWithHandlers(); + const mockSetVolume = vi.fn(); + + await eventHandlers.get("trackSubscribed")?.( + { + kind: "audio", + mediaStreamTrack: { id: "remote-track" }, + setVolume: mockSetVolume, + }, + {}, + { identity: "agent_123" } + ); - connection.output.setVolume(0.75); - expect(mockSetVolume).toHaveBeenCalledWith(0.75); - connection.close(); + connection.output.setVolume(0.75); + expect(mockSetVolume).toHaveBeenCalledWith(0.75); + connection.close(); + }); }); it.each([ diff --git a/packages/client/src/utils/WebRTCConnection.ts b/packages/client/src/utils/WebRTCConnection.ts index 4a9e7aa0..90f58d8e 100644 --- a/packages/client/src/utils/WebRTCConnection.ts +++ b/packages/client/src/utils/WebRTCConnection.ts @@ -399,8 +399,8 @@ export class WebRTCConnection extends BaseConnection { const message = JSON.parse(new TextDecoder().decode(payload)); // Filter out audio messages for WebRTC - they're handled via audio tracks - if (isValidSocketEvent(message) && message.type === "audio") { - if (message.audio_event.alignment) { + if (message.type === "audio") { + if (message.audio_event?.alignment && isValidSocketEvent(message)) { const { audio_base_64: _audioBase64, ...audioEvent } = message.audio_event; this.handleMessage({ diff --git a/packages/client/src/utils/WebSocketConnection.test.ts b/packages/client/src/utils/WebSocketConnection.test.ts index 2ca24801..059732e7 100644 --- a/packages/client/src/utils/WebSocketConnection.test.ts +++ b/packages/client/src/utils/WebSocketConnection.test.ts @@ -169,7 +169,7 @@ describe("WebSocketConnection", () => { expect(listener).not.toHaveBeenCalled(); }); - it("forwards alignment-bearing audio on the WebSocket without filtering", async () => { + it("forwards alignment-bearing audio on the WebSocket", async () => { const connection = await createConnection(); const onMessage = vi.fn(); const alignment = { From 27caabfaeb33e307854ecfa134125dd1ce8f6438 Mon Sep 17 00:00:00 2001 From: Nathaniel Pogue Date: Mon, 25 May 2026 14:56:54 -0400 Subject: [PATCH 4/5] Fix TypeScript narrowing for WebRTC alignment audio events. Reorder guards so isValidSocketEvent narrows before accessing audio_event, fixing client tsc build in CI. Co-authored-by: Cursor --- packages/client/src/utils/WebRTCConnection.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/client/src/utils/WebRTCConnection.ts b/packages/client/src/utils/WebRTCConnection.ts index 90f58d8e..4a9e7aa0 100644 --- a/packages/client/src/utils/WebRTCConnection.ts +++ b/packages/client/src/utils/WebRTCConnection.ts @@ -399,8 +399,8 @@ export class WebRTCConnection extends BaseConnection { const message = JSON.parse(new TextDecoder().decode(payload)); // Filter out audio messages for WebRTC - they're handled via audio tracks - if (message.type === "audio") { - if (message.audio_event?.alignment && isValidSocketEvent(message)) { + if (isValidSocketEvent(message) && message.type === "audio") { + if (message.audio_event.alignment) { const { audio_base_64: _audioBase64, ...audioEvent } = message.audio_event; this.handleMessage({ From 158e536c1dd793b3b474a176dd4e0a9c41e59737 Mon Sep 17 00:00:00 2001 From: Nathaniel Pogue Date: Mon, 25 May 2026 15:02:08 -0400 Subject: [PATCH 5/5] fixed cursorbot issue --- packages/client/src/VoiceConversation.test.ts | 21 +++++++++++++++++++ packages/client/src/VoiceConversation.ts | 7 +++---- 2 files changed, 24 insertions(+), 4 deletions(-) diff --git a/packages/client/src/VoiceConversation.test.ts b/packages/client/src/VoiceConversation.test.ts index 00e767fe..34499b04 100644 --- a/packages/client/src/VoiceConversation.test.ts +++ b/packages/client/src/VoiceConversation.test.ts @@ -133,6 +133,27 @@ describe("VoiceConversation", () => { expect(onAudioAlignment).toHaveBeenCalledWith(alignment); expect(onAudio).not.toHaveBeenCalled(); }); + + it("does not update mode or feedback for alignment-only events", () => { + const onModeChange = vi.fn(); + const onCanSendFeedbackChange = vi.fn(); + const conversation = TestVoiceConversation.create({ + onModeChange, + onCanSendFeedbackChange, + }); + + conversation.handleAudioEvent({ + type: "audio", + audio_event: { + audio_base_64: "", + event_id: 11, + alignment, + }, + }); + + expect(onModeChange).not.toHaveBeenCalled(); + expect(onCanSendFeedbackChange).not.toHaveBeenCalled(); + }); }); describe("VoiceConversation WebSocket integration", () => { diff --git a/packages/client/src/VoiceConversation.ts b/packages/client/src/VoiceConversation.ts index 9c432775..e9b7c81d 100644 --- a/packages/client/src/VoiceConversation.ts +++ b/packages/client/src/VoiceConversation.ts @@ -122,11 +122,10 @@ export class VoiceConversation extends BaseConversation { this.options.onAudio?.(event.audio_event.audio_base_64); // Audio routing is handled by attachConnectionToOutput for WebSocket // WebRTC handles audio playback directly through LiveKit tracks + this.currentEventId = event.audio_event.event_id; + this.updateCanSendFeedback(); + this.updateMode("speaking"); } - - this.currentEventId = event.audio_event.event_id; - this.updateCanSendFeedback(); - this.updateMode("speaking"); } }