Skip to content

feat(audio): per-app input device selection via the direct Core Audio path #642

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
aguynamedryan wants to merge 7 commits into
base: main
Choose a base branch
from
+701 −40
Open

feat(audio): per-app input device selection via the direct Core Audio path #642

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
ba52b9d
feat(audio): restore independent input-device mode via direct Core Au…
aguynamedryan Jul 16, 2026
dc53e92
refactor(audio): remove unreachable AVAudioEngine device-binding code
aguynamedryan Jul 19, 2026
c55f060
Merge origin/main: rebuild FluidVoice-only input mode on MicrophonePr…
aguynamedryan Jul 20, 2026
78691e5
Merge origin/main: adopt AVAudioEngine retirement drain from #655
aguynamedryan Jul 21, 2026
e50f29c
fix(audio): fall back to the default mic in FluidVoice-only mode inst…
aguynamedryan Jul 22, 2026
e6251e1
fix(audio): post the mic-fallback notification only after capture starts
aguynamedryan Jul 24, 2026
18c10d7
fix(audio): don't interrupt a FluidVoice-only recording on unrelated …
aguynamedryan Jul 24, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions Fluid.xcodeproj/project.pbxproj
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@
86CAA2D4EF18433096185602 /* LLMClientRequestBodyTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = 343B29013F4441D6A797D12D /* LLMClientRequestBodyTests.swift */; };
272BFB5CB271489892CAE50C /* TemperatureSupportTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = 980330F3CE464336ADCE3E23 /* TemperatureSupportTests.swift */; };
A62300000000000000000002 /* AudioBufferConverterTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = A62300000000000000000001 /* AudioBufferConverterTests.swift */; };
A62300000000000000000004 /* FluidVoiceOnlyMicrophoneModeTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = A62300000000000000000003 /* FluidVoiceOnlyMicrophoneModeTests.swift */; };
C0DE63600000000000000002 /* AudioEngineRetirementDrainTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = C0DE63600000000000000001 /* AudioEngineRetirementDrainTests.swift */; };
7CDB0A2F2F3C4D5600FB7CAD /* dictation_fixture.wav in Resources */ = {isa = PBXBuildFile; fileRef = 7CDB0A2B2F3C4D5600FB7CAD /* dictation_fixture.wav */; };
7CDB0A302F3C4D5600FB7CAD /* XCTest.framework in Frameworks */ = {isa = PBXBuildFile; fileRef = 7CDB0A2C2F3C4D5600FB7CAD /* XCTest.framework */; };
Expand Down Expand Up @@ -53,6 +54,7 @@
343B29013F4441D6A797D12D /* LLMClientRequestBodyTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = LLMClientRequestBodyTests.swift; sourceTree = "<group>"; };
980330F3CE464336ADCE3E23 /* TemperatureSupportTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = TemperatureSupportTests.swift; sourceTree = "<group>"; };
A62300000000000000000001 /* AudioBufferConverterTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = AudioBufferConverterTests.swift; sourceTree = "<group>"; };
A62300000000000000000003 /* FluidVoiceOnlyMicrophoneModeTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = FluidVoiceOnlyMicrophoneModeTests.swift; sourceTree = "<group>"; };
C0DE63600000000000000001 /* AudioEngineRetirementDrainTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = AudioEngineRetirementDrainTests.swift; sourceTree = "<group>"; };
7C078D8F2E3B339200FB7CAC /* FluidVoice Debug.app */ = {isa = PBXFileReference; explicitFileType = wrapper.application; includeInIndex = 0; path = "FluidVoice Debug.app"; sourceTree = BUILT_PRODUCTS_DIR; };
7C91B0022F42AA0100C0DEF0 /* HotkeyShortcutTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = HotkeyShortcutTests.swift; sourceTree = "<group>"; };
Expand Down Expand Up @@ -131,6 +133,7 @@
343B29013F4441D6A797D12D /* LLMClientRequestBodyTests.swift */,
980330F3CE464336ADCE3E23 /* TemperatureSupportTests.swift */,
A62300000000000000000001 /* AudioBufferConverterTests.swift */,
A62300000000000000000003 /* FluidVoiceOnlyMicrophoneModeTests.swift */,
C0DE63600000000000000001 /* AudioEngineRetirementDrainTests.swift */,
);
path = FluidDictationIntegrationTests;
Expand Down Expand Up @@ -290,6 +293,7 @@
86CAA2D4EF18433096185602 /* LLMClientRequestBodyTests.swift in Sources */,
272BFB5CB271489892CAE50C /* TemperatureSupportTests.swift in Sources */,
A62300000000000000000002 /* AudioBufferConverterTests.swift in Sources */,
A62300000000000000000004 /* FluidVoiceOnlyMicrophoneModeTests.swift in Sources */,
C0DE63600000000000000002 /* AudioEngineRetirementDrainTests.swift in Sources */,
);
runOnlyForDeploymentPostprocessing = 0;
Expand Down
6 changes: 3 additions & 3 deletions Sources/Fluid/ContentView.swift
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -388,7 +388,7 @@ struct ContentView: View {
if let sysIn = AudioDevice.getDefaultInputDevice()?.uid {
self.selectedInputUID = sysIn
}
case .manual:
case .manual, .fluidVoiceOnly:
// The refreshed device list drives the displayed selection.
break
}
Expand Down Expand Up @@ -608,7 +608,7 @@ struct ContentView: View {
if let defaultUID = AudioDevice.getDefaultInputDevice()?.uid {
self.selectedInputUID = defaultUID
}
case .manual:
case .manual, .fluidVoiceOnly:
if let preferredUID = SettingsStore.shared.preferredInputDeviceUID, !preferredUID.isEmpty {
self.selectedInputUID = preferredUID
} else if let defaultUID = AudioDevice.getDefaultInputDevice()?.uid {
Expand Down Expand Up @@ -4319,7 +4319,7 @@ private extension ContentView {
switch SettingsStore.shared.microphoneSelectionMode {
case .system:
self.selectedInputUID = AudioDevice.getDefaultInputDevice()?.uid ?? ""
case .manual:
case .manual, .fluidVoiceOnly:
self.selectedInputUID = SettingsStore.shared.preferredInputDeviceUID ?? AudioDevice.getDefaultInputDevice()?.uid ?? ""
}
self.selectedOutputUID = SettingsStore.shared.preferredOutputDeviceUID ?? ""
Expand Down
38 changes: 34 additions & 4 deletions Sources/Fluid/Persistence/SettingsStore.swift
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -183,6 +183,12 @@ final class SettingsStore: ObservableObject {
enum MicrophoneSelectionMode: String, Codable, CaseIterable, Identifiable {
case system
case manual
/// Like `.manual`, but the preferred microphone is captured for FluidVoice alone and the
/// macOS default input is never changed. Only available on the direct Core Audio capture
/// path (`DirectCoreAudioInput`), which can bind an arbitrary device per-app. When the
/// preferred device is unavailable, capture falls back to the macOS default (still without
/// moving it) and the substitution is surfaced to the user rather than recorded silently.
case fluidVoiceOnly

var id: String {
self.rawValue
Expand All @@ -194,8 +200,15 @@ final class SettingsStore: ObservableObject {
return "Use macOS Default"
case .manual:
return "Use Preferred Microphone"
case .fluidVoiceOnly:
return "Use Preferred Microphone (FluidVoice Only)"
}
}

/// Whether capture resolves `preferredInputDeviceUID` instead of the macOS default input.
var usesPreferredInputDevice: Bool {
self != .system
}
}

enum DictationPromptSelection: Equatable {
Expand Down Expand Up @@ -1814,6 +1827,14 @@ final class SettingsStore: ObservableObject {
if let raw = self.defaults.string(forKey: Keys.microphoneSelectionMode),
let mode = MicrophoneSelectionMode(rawValue: raw)
{
// FluidVoice-only capture exists solely on the direct Core Audio path. If that path is
// off, degrade to `.manual` — which still honours the preferred microphone, by
// moving the system default — rather than silently recording from whatever macOS
// happens to be pointed at.
if mode == .fluidVoiceOnly, self.experimentalDirectAudioCaptureEnabled == false {
return .manual
}

return mode
}

Expand All @@ -1827,7 +1848,7 @@ final class SettingsStore: ObservableObject {

func recordInputDeviceSelection(_ uid: String) {
guard uid.isEmpty == false else { return }
guard self.microphoneSelectionMode == .manual else { return }
guard self.microphoneSelectionMode.usesPreferredInputDevice else { return }

self.preferredInputDeviceUID = uid
}
Expand All @@ -1844,8 +1865,13 @@ final class SettingsStore: ObservableObject {
) -> String? {
let previousMode = self.microphoneSelectionMode

if previousMode == .system,
mode == .manual,
// `.manual` is the one mode that moves the macOS default, so capture the user's own default
// on the way *in* — from any other mode, since neither `.system` nor `.fluidVoiceOnly`
// disturbs it, which makes the current value genuinely theirs. Capturing only from
// `.system` would miss the `.fluidVoiceOnly` -> `.manual` -> `.fluidVoiceOnly` round trip
// and leave the system input stuck on FluidVoice's choice after the user opted back out.
if mode == .manual,
previousMode != .manual,
let currentSystemInputUID,
currentSystemInputUID.isEmpty == false
{
Expand All @@ -1854,7 +1880,11 @@ final class SettingsStore: ObservableObject {

self.microphoneSelectionMode = mode

guard mode == .system else { return nil }
// `.manual` is the only mode that holds the macOS default on FluidVoice's choice, so only
// *leaving* it hands the default back to whatever the user had before — whether that is for
// `.system` or for `.fluidVoiceOnly`. Restoring on any other transition would clobber a
// default the user changed themselves while FluidVoice was keeping its hands off it.
guard previousMode == .manual, mode != .manual else { return nil }

if let previousSystemInputUID = self.defaults.string(forKey: Keys.systemInputDeviceUIDBeforeManual),
previousSystemInputUID.isEmpty == false,
Expand Down
107 changes: 95 additions & 12 deletions Sources/Fluid/Services/ASRService.swift
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -621,6 +621,10 @@ final class ASRService: ObservableObject {
private var directAudioInput: DirectCoreAudioInput?
private var activeAudioCaptureBackend: AudioCaptureBackend = .none
private var isFallingBackFromDirectCapture = false
/// UID of the device the FluidVoice-only "preferred mic unavailable" notification last named, so
/// a single fallback announces once rather than on every recording. Reset (to nil) when the
/// preferred device is used again. See `announcePreferredMicrophoneFallbackIfNeeded(recording:)`.
private var lastAnnouncedFallbackDeviceUID: String?

private var hasPreparedAudioCapture: Bool {
self.directAudioInput != nil || self.hasWarmAudioEngine
Expand Down Expand Up @@ -809,6 +813,12 @@ final class ASRService: ObservableObject {
}

private func startPreferredAudioCapture() async throws {
// FluidVoice-only mode prefers one microphone but does not require it. If the preferred
// device is unavailable — unplugged, or unbindable on this hardware — capture records the
// macOS default instead, through this same private path and without ever moving the system
// input. That substitution is announced (never silent) at whichever backend actually starts,
// so this mode records rather than refuses.

// A non-route path may have scheduled a fire-and-forget retirement. Do
// not let capture startup overlap any final AVAudioEngine release that is
// already queued.
Expand All @@ -830,6 +840,9 @@ final class ASRService: ObservableObject {
"audio_backend kind=direct_core_audio device=\(directAudioInput.deviceID) " +
"frames=\(directAudioInput.hardwareBufferFrameSize) callbackMs=\(callbackMs)"
)
// The direct path bound whatever resolution returned — the preferred device, or the
// macOS default when it was unavailable. Announce the latter.
self.announcePreferredMicrophoneFallbackIfNeeded(recording: self.resolvedInputDeviceForCapture())
return
} catch {
DebugLogger.shared.warning(
Expand All @@ -846,9 +859,50 @@ final class ASRService: ObservableObject {
self.directAudioInput = nil
}

// The AVAudioEngine path can only record the current macOS default input — it cannot bind an
// arbitrary device for this app alone (`kAudioUnitErr_InvalidPropertyValue`, -10851, on
// aggregate and Bluetooth devices). In FluidVoice-only mode the default is the correct
// target here: either the preferred device was unavailable and resolved to the default, or
// the direct path could not bind it on this hardware and the default is the best capture we
// have. Record it rather than refuse.
try await self.startCompatibilityAudioCapture(
reason: directCaptureEnabled ? "direct_unavailable" : "experimental_disabled"
)
// Announce the substitution only after the capture has actually started, so a failed start
// never leaves a misleading "recording through …" notification behind. This backend always
// records the macOS default, so pass it explicitly rather than the resolved preference. (The
// direct-capture site above announces after its `start()` for the same reason.)
self.announcePreferredMicrophoneFallbackIfNeeded(recording: AudioDevice.getDefaultInputDevice())
Comment on lines +871 to +875

@chatgpt-codex-connector chatgpt-codex-connector Bot Jul 24, 2026

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

P2 Badge Announce runtime fallback to the default mic

Fresh evidence in the current diff is that the new preferred-mic notification is only invoked after startPreferredAudioCapture() startup fallbacks; handleDirectCaptureDurationMismatch() still switches an active direct session with startCompatibilityAudioCapture(reason: "duration_mismatch") and never calls this helper. When .fluidVoiceOnly is recording the preferred mic and the direct path detects a duration mismatch, it falls back to AVAudioEngine/default input silently aside from the generic audio-capture-path notification, so users are not told their selected mic is no longer being used.

Useful? React with 👍 / 👎.

}

/// FluidVoice-only mode prefers one microphone but does not require it. When capture lands on a
/// device other than the pinned one — the preferred mic is unplugged, or the direct path cannot
/// bind it on this hardware — it records the macOS default through the private path, without ever
/// moving the system input. That substitution is invisible to the audio pipeline, so announce it
/// once per distinct fallback device; the record resets whenever the preferred device is used
/// again (`device.uid == preferredUID`), so a later fallback announces afresh.
///
/// `device` is the input actually being recorded, passed by the caller because it differs by
/// backend: the direct path binds the resolved device, while AVAudioEngine always records the
/// macOS default regardless of the preference.
private func announcePreferredMicrophoneFallbackIfNeeded(recording device: AudioDevice.Device?) {
guard SettingsStore.shared.microphoneSelectionMode == .fluidVoiceOnly,
let preferredUID = SettingsStore.shared.preferredInputDeviceUID,
preferredUID.isEmpty == false,
let device,
device.uid != preferredUID
else {
self.lastAnnouncedFallbackDeviceUID = nil
return
}
guard self.lastAnnouncedFallbackDeviceUID != device.uid else { return }
self.lastAnnouncedFallbackDeviceUID = device.uid

DebugLogger.shared.info(
"Preferred microphone unavailable; recording through fallback '\(device.name)'",
source: "ASRService"
)
NotificationService.showPreferredMicrophoneFallback(fallbackDeviceName: device.name)
}

private func startCompatibilityAudioCapture(reason: String) async throws {
Expand Down Expand Up @@ -1950,8 +2004,11 @@ final class ASRService: ObservableObject {
DebugLogger.shared.debug("✅ configureSession() - COMPLETED", source: "ASRService")
}

/// In independent mode, attempt to bind AVAudioEngine's input to the user's preferred input device.
/// In sync-with-system mode, we intentionally do nothing so the engine follows macOS defaults.
/// In `.manual` mode, attempt to bind AVAudioEngine's input to the user's preferred input device.
/// In `.system` mode we intentionally do nothing so the engine follows macOS defaults.
/// `.fluidVoiceOnly` also does nothing here: if it has reached the AVAudioEngine path at all, the
/// preferred device could not be captured directly, so recording the macOS default is the
/// intended fallback (announced by `announcePreferredMicrophoneFallbackIfNeeded()`).
/// Returns true if binding succeeded or if no binding was needed, false if binding failed completely.
@discardableResult
private func bindPreferredInputDeviceIfNeeded() -> Bool {
Expand Down Expand Up @@ -1988,8 +2045,8 @@ final class ASRService: ObservableObject {
return true
Comment on lines 2014 to 2045

@chatgpt-codex-connector chatgpt-codex-connector Bot Jul 19, 2026

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

P2 Badge Honor preferred input on direct-capture fallback

When sync is disabled but DirectCoreAudioInput cannot be used for the selected mic (for example the helper rejects devices with multiple input streams in CoreAudioCaptureSupport.c:54-57, or AudioDeviceStart fails), startPreferredAudioCapture() falls back to the AVAudioEngine path. This method now always succeeds without binding preferredInputDeviceUID, so the recording silently uses the macOS default while Settings/menu still show the dedicated mic; either fail/notify or bind/re-sync before using the fallback.

Useful? React with 👍 / 👎.

@aguynamedryan aguynamedryan Jul 23, 2026

Copy link
Copy Markdown
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This is now the intended, announced behaviour. When the direct path can't bind the preferred device (e.g. a multi-stream aggregate, or a device that fails to start), FluidVoice-only records the macOS default via AVAudioEngine — "prefer, else default" — and announcePreferredMicrophoneFallbackIfNeeded(recording:) posts a notification naming the device it fell back to, so the swap is visible rather than silent.

Verified live by forcing the reject with a purpose-built 2-stream aggregate as the preferred device: the direct path returns kAudioHardwareUnsupportedOperationError, AVAudioEngine records the system default, the fallback notification fires, and the macOS system default input is left unchanged.

}

/// In independent mode, attempt to bind AVAudioEngine's output to the user's preferred output device.
/// In sync-with-system mode, we intentionally do nothing so the engine follows macOS defaults.
/// Output always follows the macOS default: per-app selection is input-only, so there is no
/// preferred output device to bind here.
/// Returns true if binding succeeded or if no binding was needed, false if binding failed completely.
@discardableResult
private func bindPreferredOutputDeviceIfNeeded() -> Bool {
Expand Down Expand Up @@ -2546,14 +2603,25 @@ final class ASRService: ObservableObject {
}

private func handleDefaultInputChanged() {
if SettingsStore.shared.microphoneSelectionMode == .manual {
switch SettingsStore.shared.microphoneSelectionMode {
case .manual:
if self.isRunning {
AppServices.shared.microphonePreferenceCoordinator.stabilizePreferredInputAfterHardwareChange(
reason: "default input changed"
)
self.scheduleAudioRouteRecovery(reason: "manual preferred input reasserted")
}
return
case .fluidVoiceOnly:
// The macOS default moved, but FluidVoice-only capture neither follows it nor pushes it
// back: the device we are bound to is unaffected, so there is nothing to recover.
DebugLogger.shared.info(
"Default input changed; FluidVoice-only capture keeps its own device",
source: "ASRService"
)
return
case .system:
break
}

self.scheduleAudioRouteRecovery(reason: "default input changed")
Expand Down Expand Up @@ -2765,13 +2833,28 @@ final class ASRService: ObservableObject {

DebugLogger.shared.debug("Current input devices: \(currentDevices.map { $0.name }.joined(separator: ", "))", source: "ASRService")

if self.isRunning,
SettingsStore.shared.microphoneSelectionMode == .manual,
Set(currentDevices.map(\.uid)) != cachedUIDs
{
AppServices.shared.microphonePreferenceCoordinator.stabilizePreferredInputAfterHardwareChange(
reason: "input device list changed"
)
if self.isRunning, Set(currentDevices.map(\.uid)) != cachedUIDs {
switch SettingsStore.shared.microphoneSelectionMode {
case .manual:
AppServices.shared.microphonePreferenceCoordinator
.stabilizePreferredInputAfterHardwareChange(reason: "input device list changed")
case .fluidVoiceOnly:
// A device-list change never interrupts an in-progress FluidVoice-only
// recording. Direct capture is bound to one device by ID and is unaffected by
// unrelated hardware coming or going, so reacting here would needlessly halt
// the pipeline (missed audio, gap in transcription) on any unrelated USB or
// Bluetooth event. If the *bound* device itself disconnects, the per-device
// availability monitor (handleDeviceAvailabilityChanged) recovers it
// separately; a changed device preference is applied on the next recording,
// which re-resolves. So leave the running capture alone.
DebugLogger.shared.debug(
"Input device list changed during FluidVoice-only recording; " +
"leaving direct capture undisturbed (re-resolves on next recording)",
source: "ASRService"
)
case .system:
break
}
}

self.cacheCurrentDeviceList(currentDevices)
Expand Down
Loading
Loading