Stream the PC audio to the tablet

README.md

@@ -61,16 +61,16 @@ Working today, over USB with ADB:
 | Locally drawn cursor | ✅ | ✅ | 🧪 | 🧪 | 🧪 |
 | Zero-copy hardware encode | ✅ | ✅ | 🧪 | ❌ shared-memory capture | ❌ shared-memory capture |
 | Touch, pen pressure, tilt and hover | ✅ | ✅ | 🧪 | 🧪 | 🧪 |
+| PC audio on the tablet, with volume kept in sync | 🧪 audio only, no volume sync | ✅ | 🧪 | 🧪 | 🧪 |
 | Tray app with settings and live statistics | ✅ | ✅ | 🧪 | 🧪 | 🧪 |
 | Installer / packages | ✅ `.exe` | ✅ AppImage, `.deb`, Arch | 🧪 AppImage, `.deb`, Arch | 🧪 AppImage, `.deb`, Arch | 🧪 AppImage, `.deb`, Arch |
 
 🧪 New and not yet tested on a real session with a tablet: GNOME has been tested against a
 headless Mutter (the tray needs the AppIndicator extension there), Sway against a headless sway,
 Hyprland only through unit tests, and X11 against Xorg with the dummy video driver (i3, xfwm4).
 X11 works with any window manager or desktop that honours RandR 1.5 monitors; the proprietary
-NVIDIA driver is untested.
-
-**Not yet:** audio.
+NVIDIA driver is untested. Audio on Windows has only been tested through unit tests, and on Linux
+only on KDE Plasma.
 
 ## How it works
 
@@ -96,6 +96,20 @@ All channels share one prioritized, multiplexed link (ADB over USB today), so in
 updates are never stuck behind a video frame. Details: [docs/architecture.md](docs/architecture.md),
 wire format: [docs/protocol.md](docs/protocol.md), input path: [docs/input.md](docs/input.md).
 
+## Audio
+
+The PC's sound plays on the tablet while a session runs.
+
+- **Linux:** the host adds a PipeWire output named `ezdisplay` and makes it the default for as long
+  as the session lasts, then puts the previous default back. Its volume and mute follow the tablet's
+  and the other way round.
+- **Windows:** the host captures what the default output plays (WASAPI loopback). The PC keeps
+  playing it too, and the volume is not synced.
+- Over a fast link the audio is uncompressed PCM, otherwise Opus. Choose with `--audio
+  auto|pcm|opus|off` or `audio = "..."` in `config.toml`, or in the tray settings.
+- The tablet buffers 20 to 80 ms of audio, growing the buffer when playback stutters and shrinking
+  it again once it is steady.
+
 ## USB without developer mode
 
 Besides adb, ezdisplay can talk to the tablet over USB with the Android Open Accessory protocol,

android/app/src/main/kotlin/com/librestatic/ezdisplay/Capabilities.kt

@@ -62,6 +62,13 @@ object Capabilities {
         return json.toString()
     }
 
+    /**
+     * [caps] (from [toJson]) with the `audio` switch set. The switch is a user setting that can change between
+     * sessions, so it is applied on every connect instead of being cached with the rest.
+     */
+    fun withAudio(caps: String, enabled: Boolean): String =
+        JSONObject(caps).put("audio", enabled).toString()
+
     /** HDR10 support and luminance of the panel; luminance members are 0 when the platform does not report them. */
     @Suppress("DEPRECATION")
     private fun hdr(display: Display): JSONObject {

android/app/src/main/kotlin/com/librestatic/ezdisplay/MainActivity.kt

@@ -14,6 +14,7 @@ import android.content.IntentFilter
 import android.content.res.Configuration
 import android.hardware.usb.UsbAccessory
 import android.hardware.usb.UsbManager
+import android.media.AudioManager
 import android.os.Build
 import android.os.Bundle
 import android.os.Handler
@@ -96,6 +97,16 @@ class MainActivity : Activity(), SurfaceHolder.Callback, Choreographer.FrameCall
     private lateinit var telemetryView: TextView
 
     private val handler = Handler(Looper.getMainLooper())
+    private val volumeWatcher by lazy {
+        VolumeWatcher(
+            this, handler,
+            VolumeSync(
+                AudioManagerVolume(getSystemService(AudioManager::class.java)),
+                NativeBridge::setLocalVolume,
+                NativeBridge::pollHostVolume,
+            ),
+        )
+    }
     private var chosenMode: android.view.Display.Mode? = null
     private var capsJson: String? = null
 
@@ -176,6 +187,7 @@ class MainActivity : Activity(), SurfaceHolder.Callback, Choreographer.FrameCall
     override fun onCreate(savedInstanceState: Bundle?) {
         super.onCreate(savedInstanceState)
         launchIntent = intent
+        volumeControlStream = AudioManager.STREAM_MUSIC // the volume keys always move the PC's audio
         window.addFlags(WindowManager.LayoutParams.FLAG_KEEP_SCREEN_ON)
         if (Build.VERSION.SDK_INT >= 28) {
             window.attributes = window.attributes.also {
@@ -474,8 +486,12 @@ class MainActivity : Activity(), SurfaceHolder.Callback, Choreographer.FrameCall
     private fun connect() {
         if (!started || sessionActive) return
         if (!surfaceReady) return // surfaceCreated reschedules
-        val caps = capsJson ?: Capabilities.toJson(this, chosenMode).also { capsJson = it }
+        val caps = Capabilities.withAudio(
+            capsJson ?: Capabilities.toJson(this, chosenMode).also { capsJson = it },
+            InputPrefs.prefs(this).getBoolean(SettingsStore.AUDIO_ENABLED, true),
+        )
         pushKeyboardState()
+        volumeWatcher.start()
         val acc = accessory?.takeIf { usbManager.hasPermission(it) }
         val wifiChoice = if (tryWifiNext) wifiTarget() else null
         val route = chooseRoute(acc != null, tryWifiNext, wifiChoice != null)
@@ -491,6 +507,7 @@ class MainActivity : Activity(), SurfaceHolder.Callback, Choreographer.FrameCall
         }
         if (err != null) {
             Log.w(TAG, "start failed: $err")
+            volumeWatcher.stop()
             setStatus(
                 if (route == Route.ACCESSORY) getString(R.string.status_usb_open_failed)
                 else getString(R.string.status_bad_caps, err),
@@ -539,6 +556,7 @@ class MainActivity : Activity(), SurfaceHolder.Callback, Choreographer.FrameCall
 
     private fun stopSession() {
         releaseAllKeys()
+        volumeWatcher.stop()
         if (sessionActive) {
             NativeBridge.stop()
             sessionActive = false
@@ -577,6 +595,8 @@ class MainActivity : Activity(), SurfaceHolder.Callback, Choreographer.FrameCall
         val now = SystemClock.uptimeMillis()
         if (now - lastStatsPoll >= STATS_INTERVAL_MS) {
             lastStatsPoll = now
+            volumeWatcher.pollHost()
+            volumeWatcher.checkLocal()
             updateStats(now)
         }
     }
@@ -862,6 +882,13 @@ class MainActivity : Activity(), SurfaceHolder.Callback, Choreographer.FrameCall
     }
 
     override fun dispatchKeyEvent(event: KeyEvent): Boolean {
+        if (inputActive()) {
+            volumeKeyDirection(event.keyCode)?.let { direction ->
+                // Handled here so the system volume panel stays hidden: its blur stalls the video.
+                if (event.action == KeyEvent.ACTION_DOWN) volumeWatcher.onKey(direction)
+                return true
+            }
+        }
         if (inputActive() && isHardwareKey(event)) {
             val code = event.keyCode
             val nanos = event.eventTime * 1_000_000L
@@ -884,6 +911,14 @@ class MainActivity : Activity(), SurfaceHolder.Callback, Choreographer.FrameCall
         return super.dispatchKeyEvent(event)
     }
 
+    /** The volume change a volume key asks for, or null for any other key. */
+    private fun volumeKeyDirection(code: Int): Int? = when (code) {
+        KeyEvent.KEYCODE_VOLUME_UP -> AudioManager.ADJUST_RAISE
+        KeyEvent.KEYCODE_VOLUME_DOWN -> AudioManager.ADJUST_LOWER
+        KeyEvent.KEYCODE_VOLUME_MUTE -> AudioManager.ADJUST_TOGGLE_MUTE
+        else -> null
+    }
+
     /** A key from a physical keyboard that Android does not need for itself. */
     private fun isHardwareKey(event: KeyEvent): Boolean {
         if (event.source and InputDevice.SOURCE_KEYBOARD != InputDevice.SOURCE_KEYBOARD) return false

android/app/src/main/kotlin/com/librestatic/ezdisplay/NativeBridge.kt

@@ -93,6 +93,19 @@ object NativeBridge {
      */
     @JvmStatic external fun setKeyboardAttached(attached: Boolean)
 
+    /**
+     * The tablet's media volume as a perceptual 0..1 value plus the mute flag. Kept across
+     * sessions: the host gets the last value once audio starts, and every later change at once.
+     * Do not report a value that was just applied from the host ([pollHostVolume]).
+     */
+    @JvmStatic external fun setLocalVolume(volume: Float, muted: Boolean)
+
+    /**
+     * `[seq, volume, muted (0/1)]`: the latest volume the host sent in this session. `seq` grows
+     * with every message and is 0 while there is none.
+     */
+    @JvmStatic external fun pollHostVolume(): FloatArray
+
     /** Fires the engine timers that are due; call once per frame. */
     @JvmStatic external fun inputTick()
 

android/app/src/main/kotlin/com/librestatic/ezdisplay/VolumeSync.kt

@@ -0,0 +1,155 @@
+package com.librestatic.ezdisplay
+
+import android.content.Context
+import android.database.ContentObserver
+import android.media.AudioManager
+import android.os.Handler
+import android.provider.Settings
+import android.util.Log
+import kotlin.math.roundToInt
+
+/** The tablet's media volume as the platform exposes it. */
+interface MusicVolume {
+    val max: Int
+    var index: Int
+    var muted: Boolean
+
+    /**
+     * Moves the volume one step ([AudioManager.ADJUST_RAISE] or `ADJUST_LOWER`) or toggles mute.
+     *
+     * Whole steps through [index], not `adjustStreamVolume`: some devices move a finer hidden
+     * scale on each press, which the step the PC follows only sees every few presses. No flag
+     * shows the system volume panel either: its blur over the video surface stalls the decoder
+     * for as long as the panel is up.
+     */
+    fun adjust(direction: Int) {
+        when (direction) {
+            AudioManager.ADJUST_TOGGLE_MUTE -> muted = !muted
+            else -> index = (index + direction).coerceIn(0, max)
+        }
+    }
+}
+
+/** [MusicVolume] on top of [AudioManager]'s `STREAM_MUSIC`. */
+class AudioManagerVolume(private val audio: AudioManager) : MusicVolume {
+    override val max: Int get() = audio.getStreamMaxVolume(AudioManager.STREAM_MUSIC).coerceAtLeast(1)
+
+    override var index: Int
+        get() = audio.getStreamVolume(AudioManager.STREAM_MUSIC)
+        set(value) = audio.setStreamVolume(AudioManager.STREAM_MUSIC, value, 0)
+
+    override var muted: Boolean
+        get() = audio.isStreamMute(AudioManager.STREAM_MUSIC)
+        set(value) = audio.adjustStreamVolume(
+            AudioManager.STREAM_MUSIC,
+            if (value) AudioManager.ADJUST_MUTE else AudioManager.ADJUST_UNMUTE,
+            0,
+        )
+}
+
+/** Perceptual volume (0..1, what a slider shows) of a volume step. */
+fun volumeOfIndex(index: Int, max: Int): Float = (index.toFloat() / max.coerceAtLeast(1)).coerceIn(0f, 1f)
+
+/** The volume step closest to the perceptual [volume]. */
+fun indexOfVolume(volume: Float, max: Int): Int =
+    (volume.coerceIn(0f, 1f) * max).roundToInt().coerceIn(0, max)
+
+/**
+ * Keeps the tablet's media volume and the PC's volume in step (protocol 0.7 `AudioVolume`).
+ *
+ * The native layer holds the tablet's value and sends it to the host once audio starts, then on
+ * every change; a volume received from the host is polled with [pollHost] and applied to the
+ * hardware volume. To avoid sending a host change back, this class remembers the state it last
+ * saw (applied or reported) as a volume step plus the mute flag and reports only a different
+ * one: this is the only place that suppresses echoes.
+ */
+class VolumeSync(
+    private val music: MusicVolume,
+    private val report: (volume: Float, muted: Boolean) -> Unit,
+    private val pollHostVolume: () -> FloatArray,
+) {
+    private var knownIndex = -1
+    private var knownMuted = false
+    private var hostSeq = 0f
+
+    /** Starts a session: reports the current volume (kept natively until audio starts). */
+    fun begin() {
+        hostSeq = 0f
+        knownIndex = -1
+        onLocalChange()
+    }
+
+    /** The tablet's volume may have changed (volume keys, the system panel, another app). */
+    fun onLocalChange() {
+        val index = music.index
+        val muted = music.muted
+        if (index == knownIndex && muted == knownMuted) return
+        knownIndex = index
+        knownMuted = muted
+        report(volumeOfIndex(index, music.max), muted)
+    }
+
+    /** A volume key pressed on the tablet: changes the volume without the system panel and reports it. */
+    fun onKey(direction: Int) {
+        try {
+            music.adjust(direction)
+        } catch (e: SecurityException) {
+            Log.w(TAG, "cannot set the media volume", e)
+        }
+        onLocalChange()
+    }
+
+    /** Applies a volume the host sent since the last call. */
+    fun pollHost() {
+        val v = pollHostVolume()
+        if (v.size < 3 || v[0] == 0f || v[0] == hostSeq) return
+        hostSeq = v[0]
+        val index = indexOfVolume(v[1], music.max)
+        val muted = v[2] != 0f
+        // Remember before applying so the change notification that follows is not reported.
+        knownIndex = index
+        knownMuted = muted
+        try {
+            if (music.index != index) music.index = index
+            // Setting a volume can unmute, so the mute state goes last.
+            if (music.muted != muted) music.muted = muted
+        } catch (e: SecurityException) {
+            Log.w(TAG, "cannot set the media volume", e)
+        }
+    }
+
+    private companion object {
+        const val TAG = "VolumeSync"
+    }
+}
+
+/** Runs [sync]'s local check whenever the system settings (where the volumes live) change. */
+class VolumeWatcher(private val context: Context, handler: Handler, private val sync: VolumeSync) {
+    private val observer = object : ContentObserver(handler) {
+        override fun onChange(selfChange: Boolean) = sync.onLocalChange()
+    }
+    private var registered = false
+
+    /** Registers the observer (once) and reports the current volume. */
+    fun start() {
+        if (!registered) {
+            context.contentResolver.registerContentObserver(Settings.System.CONTENT_URI, true, observer)
+            registered = true
+        }
+        sync.begin()
+    }
+
+    fun stop() {
+        if (registered) {
+            context.contentResolver.unregisterContentObserver(observer)
+            registered = false
+        }
+    }
+
+    fun pollHost() = sync.pollHost()
+
+    fun onKey(direction: Int) = sync.onKey(direction)
+
+    /** Safety net for devices that do not notify through the settings provider. */
+    fun checkLocal() = sync.onLocalChange()
+}

android/app/src/main/kotlin/com/librestatic/ezdisplay/settings/Sections.kt

@@ -1,6 +1,7 @@
 package com.librestatic.ezdisplay.settings
 
 import androidx.annotation.DrawableRes
+import androidx.annotation.StringRes
 import androidx.compose.animation.animateContentSize
 import androidx.compose.foundation.BorderStroke
 import androidx.compose.foundation.layout.Arrangement
@@ -521,28 +522,52 @@ private fun DoubleClickSlider(ms: Int, actions: SettingsActions, shape: Shape) {
 /** Palm rejection toggle. */
 @Composable
 fun PalmRow(on: Boolean, onChange: (Boolean) -> Unit, modifier: Modifier = Modifier) {
+    SwitchRow(
+        on, onChange, R.drawable.ic_back_hand, R.string.settings_palm_title, R.string.settings_palm_desc,
+        modifier.testTag("palm_row"),
+    )
+}
+
+/** Audio toggle: play the PC's audio on this device. */
+@Composable
+fun AudioRow(on: Boolean, onChange: (Boolean) -> Unit, modifier: Modifier = Modifier) {
+    SwitchRow(
+        on, onChange, R.drawable.ic_volume_up, R.string.settings_audio_title, R.string.settings_audio_desc,
+        modifier.testTag("audio_row"),
+    )
+}
+
+@Composable
+private fun SwitchRow(
+    on: Boolean,
+    onChange: (Boolean) -> Unit,
+    @DrawableRes icon: Int,
+    @StringRes title: Int,
+    @StringRes desc: Int,
+    modifier: Modifier,
+) {
     val colors = MaterialTheme.colorScheme
     // The whole row is the switch, so TalkBack reads one control.
     Surface(
         checked = on,
         onCheckedChange = onChange,
         shape = RoundedCornerShape(GroupEnd),
         color = colors.surfaceBright,
-        modifier = modifier.fillMaxWidth().semantics { role = Role.Switch }.testTag("palm_row"),
+        modifier = modifier.fillMaxWidth().semantics { role = Role.Switch },
     ) {
         Row(
             Modifier.heightIn(min = RowMinHeight).padding(RowPadding),
             verticalAlignment = Alignment.CenterVertically,
             horizontalArrangement = Arrangement.spacedBy(12.dp),
         ) {
-            Symbol(R.drawable.ic_back_hand, tint = colors.secondary)
+            Symbol(icon, tint = colors.secondary)
             Column(Modifier.weight(1f)) {
                 Text(
-                    stringResource(R.string.settings_palm_title),
+                    stringResource(title),
                     style = MaterialTheme.typography.titleSmall, fontWeight = FontWeight.Bold,
                 )
                 Text(
-                    stringResource(R.string.settings_palm_desc),
+                    stringResource(desc),
                     style = MaterialTheme.typography.bodySmall, color = colors.onSurfaceVariant,
                 )
             }