fix: stabilize browser media sessions

app/api/connection-details/route.ts

@@ -1,7 +1,6 @@
 import { NextResponse } from 'next/server';
 import { AccessToken, type AccessTokenOptions, type VideoGrant } from 'livekit-server-sdk';
 import { randomUUID } from 'node:crypto';
-import { RoomConfiguration } from '@livekit/protocol';
 import { deriveLiveKitRoomName, resolveConnectionSessionId } from '@/lib/connection-room-id';
 
 type ConnectionDetails = {
@@ -32,23 +31,19 @@ export async function POST(req: Request) {
       throw new Error('LIVEKIT_API_SECRET is not defined');
     }
 
-    // Parse room configuration from request body
     const body = await req.json();
-    const roomConfig = body?.room_config
-      ? RoomConfiguration.fromJson(body.room_config, { ignoreUnknownFields: true })
-      : new RoomConfiguration();
-    const tokenRoomConfig = buildTokenRoomConfig(roomConfig);
 
     // Generate participant token
     const participantName = 'user';
     const sessionId = resolveConnectionSessionId(body, randomUUID);
     const participantIdentity = `voice_assistant_user_${sessionId}`;
     const roomName = deriveLiveKitRoomName(sessionId);
 
+    // Explicit dispatch is handled by /api/session/dispatch. Omitting roomConfig
+    // also keeps participant tokens compatible with older LiveKit servers.
     const participantToken = await createParticipantToken(
       { identity: participantIdentity, name: participantName },
-      roomName,
-      tokenRoomConfig
+      roomName
     );
 
     // Return connection details
@@ -76,11 +71,7 @@ export async function POST(req: Request) {
   }
 }
 
-function createParticipantToken(
-  userInfo: AccessTokenOptions,
-  roomName: string,
-  roomConfig: RoomConfiguration | undefined
-): Promise<string> {
+function createParticipantToken(userInfo: AccessTokenOptions, roomName: string): Promise<string> {
   const at = new AccessToken(API_KEY, API_SECRET, {
     ...userInfo,
     ttl: '15m',
@@ -94,21 +85,5 @@ function createParticipantToken(
   };
   at.addGrant(grant);
 
-  if (roomConfig) {
-    at.roomConfig = roomConfig;
-  }
-
   return at.toJwt();
 }
-
-function buildTokenRoomConfig(roomConfig: RoomConfiguration) {
-  if (roomConfig.agents.length === 0) {
-    return roomConfig;
-  }
-
-  // Explicit dispatch is handled by /api/session/dispatch; token agents would create duplicate jobs.
-  return new RoomConfiguration({
-    ...roomConfig,
-    agents: [],
-  });
-}

app/api/session/stop/route.ts

@@ -10,6 +10,7 @@ import {
 } from '@/lib/connection-room-id';
 import {
   executeRoomInputStopsSequentially,
+  isLiveKitRoomNotFoundError,
   resolveRoomInputStopUrls as resolveConfiguredRoomInputStopUrls,
   resolveLiveKitHttpUrl,
 } from '@/lib/session-stop';
@@ -252,6 +253,10 @@ async function deleteLiveKitRoom(roomName: string): Promise<StopResult> {
     await roomService.deleteRoom(roomName);
     return { target: 'livekit_room', ok: true };
   } catch (error) {
+    if (isLiveKitRoomNotFoundError(error)) {
+      return { target: 'livekit_room', ok: true, skipped: true, status: 404 };
+    }
+
     return {
       target: 'livekit_room',
       ok: false,

components/livekit/filtered-audio-renderer.tsx

@@ -10,6 +10,7 @@ import {
   Track,
 } from 'livekit-client';
 import { useRoomContext } from '@livekit/components-react';
+import { buildBrowserAudioPlaybackDiagnostics } from '@/lib/browser-audio-capture';
 import { startMediaTrackAudioObserver } from '@/lib/frontend-audio-observer';
 import {
   FRONTEND_EVENTS,
@@ -398,6 +399,13 @@ export function FilteredAudioRenderer({
           if (!playbackSource || playbackObserverStops.has(elementKey)) {
             return;
           }
+          const playbackDiagnostics = buildBrowserAudioPlaybackDiagnostics(
+            participantIdentity,
+            trackName,
+            audioElements.values(),
+            createdAudioElement
+          );
+          debugAudioLog(debugAudio, '[browser-audio] playback diagnostics', playbackDiagnostics);
           pendingPlayback.delete(elementKey);
           startPlaybackObserver(
             elementKey,

hooks/useBrowserSourceClient.ts

@@ -11,7 +11,14 @@ import {
   createLocalVideoTrack,
 } from 'livekit-client';
 import type { AppConfig } from '@/app-config';
+import {
+  BROWSER_AUDIO_CONSTRAINTS,
+  assertBrowserEchoCancellationActive,
+  inspectBrowserAudioCapture,
+  runWithBrowserAudioTrackCleanup,
+} from '@/lib/browser-audio-capture';
 import { BrowserAudioGateDevice } from '@/lib/browser-audio-gate-device';
+import { awaitBrowserMediaCapture } from '@/lib/browser-media-capture-timeout';
 import {
   detachCurrentRuntime,
   isCurrentRuntime,
@@ -33,12 +40,8 @@ const DEFAULT_BROWSER_MEDIA_STREAM_NAME = 'browser_input';
 const BROWSER_VIDEO_DEFAULT_ENABLED = true;
 const BROWSER_VIDEO_STATS_INTERVAL_MS = 5000;
 const BROWSER_MEDIA_GATE_MAX_OPEN_LEASE_MS = 3000;
-const BROWSER_AUDIO_CONSTRAINTS: MediaTrackConstraints = {
-  echoCancellation: true,
-  noiseSuppression: true,
-  autoGainControl: true,
-};
-
+const BROWSER_VIDEO_CAPTURE_TIMEOUT_MS = 8000;
+const BROWSER_VIDEO_PUBLISH_TIMEOUT_MS = 5000;
 interface BrowserSourceRuntime {
   audioTrack: LocalAudioTrack | null;
   videoTrack: LocalVideoTrack | null;
@@ -162,10 +165,10 @@ export function useBrowserSourceClient(
           buildAudioCaptureOptions(audioDeviceIdRef.current)
         );
         recordFrontendObservability(FRONTEND_EVENTS.BROWSER_AUDIO_CAPTURE_FINISHED);
-        const captureTrack = audioTrack.mediaStreamTrack;
-        audioTrack.mediaStreamTrack.enabled = false;
-
-        try {
+        await runWithBrowserAudioTrackCleanup(audioTrack, async () => {
+          const captureTrack = audioTrack.mediaStreamTrack;
+          logBrowserAudioCaptureDiagnostics(captureTrack, appConfig.debugAudio);
+          audioTrack.mediaStreamTrack.enabled = false;
           await audioTrack.mute();
           if (runtimeRef.current !== runtime || !runtime.audioEnabled) {
             audioTrack.stop();
@@ -243,12 +246,7 @@ export function useBrowserSourceClient(
                 });
               });
           }
-        } catch (error) {
-          audioTrack.mediaStreamTrack.enabled = false;
-          void audioTrack.mute().catch(() => undefined);
-          audioTrack.stop();
-          throw error;
-        }
+        });
       })();
       runtime.audioPublishPromise = publishPromise;
       try {
@@ -262,6 +260,7 @@ export function useBrowserSourceClient(
     [
       appConfig.observabilityEnabled,
       appConfig.sandboxId,
+      appConfig.debugAudio,
       audioConfigured,
       browserMediaStreamName,
       recordFrontendObservability,
@@ -281,15 +280,22 @@ export function useBrowserSourceClient(
       }
 
       recordFrontendObservability(FRONTEND_EVENTS.BROWSER_VIDEO_CAPTURE_STARTED);
-      const videoTrack = await createLocalVideoTrack({
-        facingMode: 'user',
-        frameRate: { ideal: browserVideoFrameRate, max: browserVideoFrameRate },
-        resolution: {
-          width: browserVideoWidth,
-          height: browserVideoHeight,
-          frameRate: browserVideoFrameRate,
-        },
-      });
+      const videoTrack = await awaitBrowserMediaCapture(
+        createLocalVideoTrack({
+          facingMode: 'user',
+          frameRate: { ideal: browserVideoFrameRate, max: browserVideoFrameRate },
+          resolution: {
+            width: browserVideoWidth,
+            height: browserVideoHeight,
+            frameRate: browserVideoFrameRate,
+          },
+        }),
+        {
+          timeoutMs: BROWSER_VIDEO_CAPTURE_TIMEOUT_MS,
+          label: 'camera',
+          disposeLateResult: (track) => track.stop(),
+        }
+      );
       recordFrontendObservability(FRONTEND_EVENTS.BROWSER_VIDEO_CAPTURE_FINISHED);
       videoTrack.mediaStreamTrack.enabled = runtime.videoEnabled;
       if (!isCurrentRuntime(runtimeRef, runtime)) {
@@ -299,17 +305,26 @@ export function useBrowserSourceClient(
 
       try {
         recordFrontendObservability(FRONTEND_EVENTS.BROWSER_VIDEO_PUBLISH_STARTED);
-        const publication = await room.localParticipant.publishTrack(videoTrack, {
-          name: BROWSER_VIDEO_TRACK_NAME,
-          source: Track.Source.Camera,
-          stream: browserMediaStreamName,
-          simulcast: false,
-          degradationPreference: 'maintain-resolution',
-          videoEncoding: {
-            maxBitrate: browserVideoMaxBitrate,
-            maxFramerate: browserVideoFrameRate,
-          },
-        });
+        const publication = await awaitBrowserMediaCapture(
+          room.localParticipant.publishTrack(videoTrack, {
+            name: BROWSER_VIDEO_TRACK_NAME,
+            source: Track.Source.Camera,
+            stream: browserMediaStreamName,
+            simulcast: false,
+            degradationPreference: 'maintain-resolution',
+            videoEncoding: {
+              maxBitrate: browserVideoMaxBitrate,
+              maxFramerate: browserVideoFrameRate,
+            },
+          }),
+          {
+            timeoutMs: BROWSER_VIDEO_PUBLISH_TIMEOUT_MS,
+            label: 'camera publish',
+            disposeLateResult: () => {
+              void room.localParticipant.unpublishTrack(videoTrack, true).catch(() => undefined);
+            },
+          }
+        );
         recordFrontendObservability(FRONTEND_EVENTS.BROWSER_VIDEO_PUBLISH_FINISHED);
         if (!isCurrentRuntime(runtimeRef, runtime)) {
           await room.localParticipant.unpublishTrack(videoTrack, true).catch(() => undefined);
@@ -712,6 +727,17 @@ function buildAudioCaptureOptions(deviceId: string | null) {
   };
 }
 
+function logBrowserAudioCaptureDiagnostics(track: MediaStreamTrack, debugAudio?: boolean) {
+  const diagnostics = inspectBrowserAudioCapture(
+    track,
+    navigator.mediaDevices.getSupportedConstraints()
+  );
+  if (debugAudio) {
+    console.info('[browser-audio] capture diagnostics', diagnostics);
+  }
+  assertBrowserEchoCancellationActive(diagnostics);
+}
+
 function syncTrackEnabled(track: LocalAudioTrack | LocalVideoTrack | null, enabled: boolean) {
   if (!track) return;
 

hooks/useRoom.ts

@@ -259,6 +259,11 @@ export function useRoom(appConfig: AppConfig) {
 
     try {
       await waitForAgentSessionStop();
+      // A Room disconnect can make the welcome view visible without running the
+      // explicit End Call path. Clear the previous capture/gate runtime before
+      // reusing this Room, otherwise start() is a no-op and the new agent is
+      // dispatched into a room that never receives the browser microphone.
+      await browserSourceClient.stop();
       await waitForRoomDisconnected(room);
 
       if (usesManagedRoomInput) {

lib/browser-audio-capture.ts

@@ -0,0 +1,86 @@
+export const BROWSER_AUDIO_CONSTRAINTS: MediaTrackConstraints = {
+  echoCancellation: true,
+  noiseSuppression: true,
+  autoGainControl: true,
+};
+
+type InspectableAudioTrack = Pick<MediaStreamTrack, 'id' | 'getConstraints' | 'getSettings'>;
+
+export interface BrowserAudioCaptureDiagnostics {
+  trackId: string;
+  supported: MediaTrackSupportedConstraints;
+  constraints: MediaTrackConstraints;
+  settings: MediaTrackSettings;
+}
+
+export interface BrowserAudioPlaybackDiagnostics {
+  participantIdentity: string;
+  trackName: string;
+  activeAudioElementCount: number;
+  paused: boolean;
+  readyState: number;
+}
+
+type BrowserAudioElementState = Pick<HTMLAudioElement, 'ended' | 'paused' | 'readyState'>;
+
+export interface CleanableBrowserAudioTrack {
+  mediaStreamTrack: { enabled: boolean };
+  mute: () => Promise<unknown>;
+  stop: () => void;
+}
+
+export async function runWithBrowserAudioTrackCleanup<T>(
+  audioTrack: CleanableBrowserAudioTrack,
+  operation: () => Promise<T>
+): Promise<T> {
+  try {
+    return await operation();
+  } catch (error) {
+    audioTrack.mediaStreamTrack.enabled = false;
+    void audioTrack.mute().catch(() => undefined);
+    audioTrack.stop();
+    throw error;
+  }
+}
+
+export function buildBrowserAudioPlaybackDiagnostics(
+  participantIdentity: string,
+  trackName: string,
+  audioElements: Iterable<BrowserAudioElementState>,
+  currentElement: BrowserAudioElementState
+): BrowserAudioPlaybackDiagnostics {
+  return {
+    participantIdentity,
+    trackName,
+    activeAudioElementCount: Array.from(audioElements).filter(
+      (element) => !element.paused && !element.ended && element.readyState >= 2
+    ).length,
+    paused: currentElement.paused,
+    readyState: currentElement.readyState,
+  };
+}
+
+export function inspectBrowserAudioCapture(
+  track: InspectableAudioTrack,
+  supported: MediaTrackSupportedConstraints
+): BrowserAudioCaptureDiagnostics {
+  const constraints = track.getConstraints();
+  const settings = track.getSettings();
+
+  return {
+    trackId: track.id,
+    supported,
+    constraints,
+    settings,
+  };
+}
+
+export function assertBrowserEchoCancellationActive(
+  diagnostics: BrowserAudioCaptureDiagnostics
+): void {
+  if (diagnostics.supported.echoCancellation && diagnostics.settings.echoCancellation !== true) {
+    throw new Error(
+      'Browser echo cancellation was requested but is not active on the microphone track.'
+    );
+  }
+}