From 0b151261af28b8a914fbe86377db96dd6de63c69 Mon Sep 17 00:00:00 2001 From: supfeer Date: Thu, 23 Jul 2026 20:05:25 +0300 Subject: [PATCH] Delay dictation overlay until audio is ready --- Sources/Fluid/ContentView.swift | 13 ---- Sources/Fluid/Services/ASRService.swift | 33 +++++++-- Sources/Fluid/Services/MenuBarManager.swift | 82 ++++++++------------- 3 files changed, 57 insertions(+), 71 deletions(-) diff --git a/Sources/Fluid/ContentView.swift b/Sources/Fluid/ContentView.swift index 2410fd45..6e9565cb 100644 --- a/Sources/Fluid/ContentView.swift +++ b/Sources/Fluid/ContentView.swift @@ -3045,9 +3045,6 @@ struct ContentView: View { self.advanceOverlayLifecycle() self.setActiveRecordingMode(.dictate) let shouldShowDictationOverlay = !self.isRecordingForCommand && !self.isRecordingForRewrite - let shouldPlayStartSound = !self.isRecordingForCommand - && !self.isRecordingForRewrite - && self.asr.micStatus == .authorized // Ensure normal dictation mode is set (command/rewrite modes set their own) if shouldShowDictationOverlay { @@ -3055,15 +3052,9 @@ struct ContentView: View { } Task { - if shouldPlayStartSound, !self.asr.isRunning { - TranscriptionSoundPlayer.shared.playStartSound() - } await self.asr.start(onCaptureStarted: { self.captureRecordingContext() self.prewarmPrivateAIDictationIfNeeded(for: .primary) - if shouldShowDictationOverlay { - self.menuBarManager.showRecordingOverlayImmediately() - } }) if !self.asr.isRunning { self.menuBarManager.hideRecordingOverlayImmediately(reason: "asr_start_failed") @@ -3680,14 +3671,10 @@ extension ContentView { Task { let asrStartStartedAt = ProcessInfo.processInfo.systemUptime DebugLogger.shared.benchmark("APP_BENCH", message: "asr_start_call", source: "AppBenchmark") - if SettingsStore.shared.enableTranscriptionSounds, !self.asr.isRunning { - TranscriptionSoundPlayer.shared.playStartSound() - } await self.asr.start(onCaptureStarted: { self.captureRecordingContext() self.appBench("overlay_mode_request mode=Dictation") self.menuBarManager.setOverlayMode(.dictation) - self.menuBarManager.showRecordingOverlayImmediately() self.appBench("overlay_mode_requested mode=Dictation") self.prewarmPrivateAIDictationIfNeeded(for: slot) }) diff --git a/Sources/Fluid/Services/ASRService.swift b/Sources/Fluid/Services/ASRService.swift index 8c3855c1..0bccf16b 100644 --- a/Sources/Fluid/Services/ASRService.swift +++ b/Sources/Fluid/Services/ASRService.swift @@ -86,6 +86,7 @@ final class ASRService: ObservableObject { } @Published var isRunning: Bool = false + @Published private(set) var isAudioCaptureReady: Bool = false @Published var finalText: String = "" @Published var partialTranscription: String = "" @Published var wordBoostStatusText: String = "Word boost: off" @@ -105,6 +106,7 @@ final class ASRService: ObservableObject { private(set) var isStarting: Bool = false // Guard against re-entrant start() calls private var audioCaptureStartWaiters: [CheckedContinuation] = [] + private var pendingCaptureStartedHandler: (@MainActor () -> Void)? var isRunningOrStarting: Bool { self.isRunning || self.isStarting } private var hasCompletedFirstTranscription: Bool = false // Track if model has warmed up with first transcription private var lastBoostHitTerm: String? @@ -1068,14 +1070,15 @@ final class ASRService: ObservableObject { /// from CoreAudio's realtime callback thread. private lazy var audioCapturePipeline: AudioCapturePipeline = .init( audioBuffer: self.audioBuffer, - onFirstAudio: { sessionID, sampleCount, frameLength, sampleRate, acquisitionMs, elapsedMs in - DispatchQueue.main.async { + onFirstAudio: { [weak self] sessionID, sampleCount, frameLength, sampleRate, acquisitionMs, elapsedMs in + Task { @MainActor [weak self] in let bufferMs = Int((Double(frameLength) / sampleRate * 1000).rounded()) DebugLogger.shared.benchmark( "ASR_BENCH", message: "session=\(sessionID) first_audio sampleCount=\(sampleCount) frameLength=\(frameLength) sampleRate=\(Int(sampleRate.rounded())) bufferMs=\(bufferMs) acquisitionMs=\(acquisitionMs) elapsedMs=\(elapsedMs)", source: "ASRBenchmark" ) + self?.markAudioCaptureReady(sessionID: sessionID) } }, onDurationMismatch: { [weak self] sessionID, capturedMilliseconds, elapsedMilliseconds in @@ -1334,6 +1337,9 @@ final class ASRService: ObservableObject { return } self.isStarting = true + self.isAudioCaptureReady = false + self.pendingCaptureStartedHandler = onCaptureStarted + self.isDictionaryTrainingCaptureActive = forDictionaryTraining defer { self.finishAudioCaptureStart() } // Reset media pause state for this session @@ -1369,18 +1375,14 @@ final class ASRService: ObservableObject { self.benchmarkLog("recording_start model=\(dims.model) provider=\(dims.provider) supportsStreaming=\(SettingsStore.shared.selectedSpeechModel.supportsStreaming)") DebugLogger.shared.debug("✅ Buffers cleared", source: "ASRService") - self.isDictionaryTrainingCaptureActive = false - do { if SettingsStore.shared.microphoneSelectionMode == .manual { AppServices.shared.microphonePreferenceCoordinator.enforcePreferredInput(reason: "recording start") } try await self.startPreferredAudioCapture() - self.isDictionaryTrainingCaptureActive = forDictionaryTraining self.isRunning = true DebugLogger.shared.info("✅ Audio capture running", source: "ASRService") - onCaptureStarted?() // Pause only after capture is live so media control cannot delay the // first PCM packet. A quick stop while this await is in flight is @@ -1421,6 +1423,8 @@ final class ASRService: ObservableObject { DebugLogger.shared.info("✅ START() completed successfully", source: "ASRService") } catch { self.isDictionaryTrainingCaptureActive = false + self.isAudioCaptureReady = false + self.pendingCaptureStartedHandler = nil self.audioCapturePipeline.setRecordingEnabled(false) self.isRunning = false self.stopActiveAudioCapture() @@ -1460,6 +1464,19 @@ final class ASRService: ObservableObject { } } + private func markAudioCaptureReady(sessionID: Int) { + guard sessionID == self.benchmarkSessionID, + self.isRunning || self.isStarting, + self.isAudioCaptureReady == false + else { return } + + self.isAudioCaptureReady = true + let handler = self.pendingCaptureStartedHandler + self.pendingCaptureStartedHandler = nil + DebugLogger.shared.info("✅ Audio capture accepted its first buffer", source: "ASRService") + handler?() + } + func waitForPendingStart() async { guard self.isStarting else { return } await withCheckedContinuation { continuation in @@ -1549,6 +1566,8 @@ final class ASRService: ObservableObject { // Set isRunning to false before teardown so in-flight ASR chunks stop safely. DebugLogger.shared.debug("🚫 Setting isRunning = false...", source: "ASRService") + self.isAudioCaptureReady = false + self.pendingCaptureStartedHandler = nil self.isRunning = false DebugLogger.shared.debug("✅ isRunning disabled", source: "ASRService") @@ -1882,6 +1901,8 @@ final class ASRService: ObservableObject { DebugLogger.shared.info("🛑 Stopping recording - releasing audio devices", source: "ASRService") // CRITICAL: Set isRunning to false FIRST to signal any in-flight chunks to abort early + self.isAudioCaptureReady = false + self.pendingCaptureStartedHandler = nil self.isRunning = false self.audioCapturePipeline.setRecordingEnabled(false) diff --git a/Sources/Fluid/Services/MenuBarManager.swift b/Sources/Fluid/Services/MenuBarManager.swift index d331201a..84445242 100644 --- a/Sources/Fluid/Services/MenuBarManager.swift +++ b/Sources/Fluid/Services/MenuBarManager.swift @@ -47,6 +47,8 @@ final class MenuBarManager: NSObject, ObservableObject, NSMenuDelegate { private var pendingShowOperation: DispatchWorkItem? private var pendingHideOperation: DispatchWorkItem? private var pendingProcessingShowOperation: DispatchWorkItem? + /// Keep capture live briefly before presenting the recording UI. + private let recordingVisualDelay: DispatchTimeInterval = .milliseconds(500) /// Show immediately so users see the processing state right away. private let processingVisualDelay: DispatchTimeInterval = .milliseconds(0) /// Legacy debounce used by generic processing callers. Successful dictation @@ -72,16 +74,22 @@ final class MenuBarManager: NSObject, ObservableObject, NSMenuDelegate { func configure(asrService: ASRService) { self.asrService = asrService - // Subscribe to recording state changes + // Keep menu bar state tied to the capture lifecycle. asrService.$isRunning .receive(on: DispatchQueue.main) .sink { [weak self] isRunning in self?.isRecording = isRunning self?.updateMenuBarIcon() self?.updateMenu() + } + .store(in: &self.cancellables) - // Handle overlay lifecycle (independent of window state) - self?.handleOverlayState(isRunning: isRunning, asrService: asrService) + // The recording overlay represents usable audio capture, not merely a + // backend that has returned successfully from start(). + asrService.$isAudioCaptureReady + .receive(on: DispatchQueue.main) + .sink { [weak self] isCaptureReady in + self?.handleOverlayState(isCaptureReady: isCaptureReady, asrService: asrService) } .store(in: &self.cancellables) @@ -97,8 +105,8 @@ final class MenuBarManager: NSObject, ObservableObject, NSMenuDelegate { .store(in: &self.cancellables) } - private func handleOverlayState(isRunning: Bool, asrService: ASRService) { - self.overlayBench("handle_state isRunning=\(isRunning) overlayVisible=\(self.overlayVisible) processing=\(self.isProcessingActive) mode=\(self.currentOverlayMode.rawValue)") + private func handleOverlayState(isCaptureReady: Bool, asrService: ASRService) { + self.overlayBench("handle_state isCaptureReady=\(isCaptureReady) overlayVisible=\(self.overlayVisible) processing=\(self.isProcessingActive) mode=\(self.currentOverlayMode.rawValue)") // Dictionary training owns its recording controls, so showing the // regular dictation notch here would create two competing overlays. @@ -115,24 +123,28 @@ final class MenuBarManager: NSObject, ObservableObject, NSMenuDelegate { // Don't hide the overlay while AI processing is active. // Without this, the notch can disappear during the short "Refining..." phase because // `isRunning` becomes false before post-processing completes. - if !isRunning, self.isProcessingActive { + if !isCaptureReady, self.isProcessingActive { self.overlayBench("handle_state_return reason=processing_active") return } // Prevent rapid state changes that could cause cycles - guard self.overlayVisible != isRunning else { + guard self.overlayVisible != isCaptureReady else { self.overlayBench("handle_state_return reason=visibility_unchanged") return } - if isRunning { + if isCaptureReady { + if self.currentOverlayMode == .dictation { + AutomaticDictionaryCorrectionTracker.shared.cancel() + } + // Cancel any pending hide operation self.pendingHideOperation?.cancel() self.pendingHideOperation = nil self.overlayVisible = true - self.overlayBench("show_request mode=\(self.currentOverlayMode.rawValue)") + self.overlayBench("show_request mode=\(self.currentOverlayMode.rawValue) delayMs=500") // If expanded command output is showing, check if we should keep it or close it if NotchOverlayManager.shared.isCommandOutputExpanded { @@ -172,6 +184,11 @@ final class MenuBarManager: NSObject, ObservableObject, NSMenuDelegate { // Show notch overlay self.overlayBench("show_workitem_execute mode=\(self.currentOverlayMode.rawValue)") + if self.currentOverlayMode == .dictation, + SettingsStore.shared.enableTranscriptionSounds + { + TranscriptionSoundPlayer.shared.playStartSound() + } NotchOverlayManager.shared.show( audioLevelPublisher: asrService.audioLevelPublisher, mode: self.currentOverlayMode @@ -181,7 +198,10 @@ final class MenuBarManager: NSObject, ObservableObject, NSMenuDelegate { self.pendingShowOperation = nil } self.pendingShowOperation = showItem - DispatchQueue.main.async(execute: showItem) + DispatchQueue.main.asyncAfter( + deadline: .now() + self.recordingVisualDelay, + execute: showItem + ) } else { // Cancel any pending show operation self.pendingShowOperation?.cancel() @@ -222,48 +242,6 @@ final class MenuBarManager: NSObject, ObservableObject, NSMenuDelegate { } } - func showRecordingOverlayImmediately() { - AutomaticDictionaryCorrectionTracker.shared.cancel() - - guard let asrService else { - self.overlayBench("instant_show_return reason=no_asr_service") - return - } - - self.pendingHideOperation?.cancel() - self.pendingHideOperation = nil - self.pendingShowOperation?.cancel() - self.pendingShowOperation = nil - - guard !self.overlayVisible else { - self.overlayBench("instant_show_return reason=already_visible") - return - } - - self.overlayVisible = true - self.overlayBench("instant_show_request mode=\(self.currentOverlayMode.rawValue)") - - if NotchOverlayManager.shared.isCommandOutputExpanded { - if self.currentOverlayMode == .command, NotchOverlayManager.shared.supportsCommandNotchUI { - NotchContentState.shared.setRecordingInExpandedMode(true) - self.expandedModeAudioSubscription = asrService.audioLevelPublisher - .receive(on: DispatchQueue.main) - .sink { level in - NotchContentState.shared.updateExpandedModeAudioLevel(level) - } - return - } - NotchOverlayManager.shared.hideExpandedCommandOutput() - } - - self.overlayBench("show_workitem_execute mode=\(self.currentOverlayMode.rawValue)") - NotchOverlayManager.shared.show( - audioLevelPublisher: asrService.audioLevelPublisher, - mode: self.currentOverlayMode - ) - self.overlayBench("show_workitem_return mode=\(self.currentOverlayMode.rawValue)") - } - func hideRecordingOverlayImmediately(reason: String) { self.pendingShowOperation?.cancel() self.pendingShowOperation = nil