diff --git a/Cargo.lock b/Cargo.lock index fb85d87..cc96e00 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2348,7 +2348,7 @@ dependencies = [ [[package]] name = "localdesktop" -version = "1.0.4" +version = "1.1.0" dependencies = [ "android-sdkmanager-rs", "android_logger", diff --git a/Cargo.toml b/Cargo.toml index ea8104c..e088fa5 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "localdesktop" -version = "1.0.4" +version = "1.1.0" edition = "2021" build = "build.rs" default-run = "build_apk" diff --git a/README.md b/README.md index ecb6810..c64576b 100644 --- a/README.md +++ b/README.md @@ -12,6 +12,12 @@ Wayland compositor in the Android NDK plus proot) and replaces what runs on it. ideally with a keyboard and mouse. It brings a desktop to Android; it does not turn the desktop into a phone UI. +On a phone, connect a monitor (USB-C to HDMI, a dock, or a wireless display): the desktop moves +to it and the phone becomes a trackpad, with buttons to turn the screen and to type with the +Android keyboard. + +elementary OS on a 4K monitor The phone as its trackpad + ## How it works 1. The APK carries an elementary OS 8 root filesystem as a ~190 MB xz archive @@ -24,7 +30,13 @@ into a phone UI. Classic session inside it: Wingpanel, the Dock, notifications and the apps. Rendering is done on the CPU (Mesa llvmpipe): Android's GPU drivers cannot be used from a glibc -userland. +userland. llvmpipe compiles shaders for the CPU it reads from `/proc/cpuinfo`; on phones whose +firmware turns off SVE (Snapdragon 8 Gen 1 to 3, for example) the app hides the CPU model from the +guest, or Gala would crash on the first shader that uses it. + +The desktop's scale follows Android's display density, lowered when the desktop would otherwise be +smaller than 540×900 logical pixels (but never below half the size of Android's own UI): a 520 dpi +foldable gets 2x on both its screens instead of 3x. ### Updates @@ -61,7 +73,7 @@ Crash reporting is off. Set `SENTRY_DSN` when building to send crashes to your o Android 12 and later kill an app's child processes once it has more than 32 of them (the "phantom process killer"); everything in the desktop then disappears at once. The session is trimmed to about 23 processes at idle, which leaves room for a couple of apps (Terminal and Web together use -6). To open more at the same time, lift the limit once: +7). To open more at the same time, lift the limit once: - Android 14 and later: Developer options → **Disable child process restrictions**. - Otherwise, from a computer with adb: @@ -81,8 +93,8 @@ about 23 processes at idle, which leaves room for a couple of apps (Terminal and managed by Android. Screen locking is too. - Translations are stripped to keep the APK small, so the desktop is in English. - Shut down and restart in the session menu (there is no logind). Log out starts a fresh session. -- Known rough edges: the panel and dock sometimes appear only after the first tap, and sound is - not working reliably yet. +- Known rough edges: sound is not working reliably yet, and on narrow screens (a foldable's cover + screen) some windows open wider than the screen. ## Building diff --git a/docs/architecture.md b/docs/architecture.md index bbd2194..14d6814 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -57,7 +57,9 @@ The stages, in order: `/etc/localdesktop`, so they survive OS updates) and seed the user's home directory. 4. `setup_machine_id` — give D-Bus a machine ID. 5. `setup_network_and_time` — write Android's DNS servers and timezone into the guest. -6. `fix_xkb_symlink` — repoint an absolute xkb symlink to a relative one (it's resolved from NDK, not the chroot root). +6. `setup_cpuinfo` — give the guest a `/proc/cpuinfo` without the CPU model when the kernel has SVE + turned off, so llvmpipe's shaders do not use it (`core::cpuinfo`). +7. `fix_xkb_symlink` — repoint an absolute xkb symlink to a relative one (it's resolved from NDK, not the chroot root). The first stage is the heaviest: the archive is ~1 GB unpacked. Whether it needs to run is decided by a stamp file holding the archive's SHA-256 from its manifest. diff --git a/docs/img/external-display-monitor.jpg b/docs/img/external-display-monitor.jpg new file mode 100644 index 0000000..295386a Binary files /dev/null and b/docs/img/external-display-monitor.jpg differ diff --git a/docs/img/external-display-trackpad.png b/docs/img/external-display-trackpad.png new file mode 100644 index 0000000..defa620 Binary files /dev/null and b/docs/img/external-display-trackpad.png differ diff --git a/patches/winit/src/platform/android.rs b/patches/winit/src/platform/android.rs index 65d6717..f1b8102 100644 --- a/patches/winit/src/platform/android.rs +++ b/patches/winit/src/platform/android.rs @@ -186,3 +186,9 @@ pub mod activity { #[doc(hidden)] pub struct AndroidApp; } + +/// The physical key winit reports for an Android `KeyEvent` key code. +#[cfg(android_platform)] +pub fn physical_key(keycode: u32) -> crate::keyboard::PhysicalKey { + crate::platform_impl::keycodes::to_physical_key(keycode.into()) +} diff --git a/patches/winit/src/platform_impl/android/mod.rs b/patches/winit/src/platform_impl/android/mod.rs index be304f3..a8a33c6 100644 --- a/patches/winit/src/platform_impl/android/mod.rs +++ b/patches/winit/src/platform_impl/android/mod.rs @@ -27,7 +27,7 @@ use crate::window::{ WindowButtons, WindowLevel, }; -mod keycodes; +pub(crate) mod keycodes; pub(crate) use crate::cursor::{ NoCustomCursor as PlatformCustomCursor, NoCustomCursor as PlatformCustomCursorSource, diff --git a/patches/xbuild/xbuild/src/gradle/mod.rs b/patches/xbuild/xbuild/src/gradle/mod.rs index 5f8f76d..46179b5 100644 --- a/patches/xbuild/xbuild/src/gradle/mod.rs +++ b/patches/xbuild/xbuild/src/gradle/mod.rs @@ -252,7 +252,19 @@ pub fn build(env: &BuildEnv, libraries: Vec<(Target, PathBuf)>, out: &Path) -> R scaler.optimize(); let anydpi = res.join("mipmap-anydpi-v26"); std::fs::create_dir_all(&anydpi)?; - std::fs::write(anydpi.join("ic_launcher.xml"), IC_LAUNCHER)?; + // A monochrome layer is the icon's silhouette, which for an opaque icon is a square: + // themed icons then show the icon in its own colors instead. + let variants: &[&str] = if scaler.is_opaque() { + let xml = String::from_utf8_lossy(IC_LAUNCHER).replace( + " \n", + "", + ); + std::fs::write(anydpi.join("ic_launcher.xml"), xml)?; + &["foreground"] + } else { + std::fs::write(anydpi.join("ic_launcher.xml"), IC_LAUNCHER)?; + &["foreground", "monochrome"] + }; let dpis = [ ("m", 48), ("h", 72), @@ -270,10 +282,11 @@ pub fn build(env: &BuildEnv, libraries: Vec<(Target, PathBuf)>, out: &Path) -> R let mut icon = std::fs::File::create(dir.join("ic_launcher.png"))?; scaler.write(&mut icon, opts)?; } - for variant in ["foreground", "monochrome"] { + // Adaptive icon layers are 108 dp where the legacy icon is 48 dp. + for variant in variants { let mut icon = std::fs::File::create(dir.join(format!("ic_launcher_{}.png", variant)))?; - scaler.write(&mut icon, opts)?; + scaler.write_adaptive_layer(&mut icon, size * 108 / 48)?; } } manifest.application.icon = Some("@mipmap/ic_launcher".into()); diff --git a/patches/xbuild/xcommon/src/lib.rs b/patches/xbuild/xcommon/src/lib.rs index 61e7c0a..ad13b78 100644 --- a/patches/xbuild/xcommon/src/lib.rs +++ b/patches/xbuild/xcommon/src/lib.rs @@ -75,6 +75,27 @@ impl Scaler { Ok(()) } + /// Whether the icon has no transparent pixels (after `optimize`). + pub fn is_opaque(&self) -> bool { + !self.img.color().has_alpha() + } + + /// An adaptive icon layer, `size` px for its 108 dp, with the whole icon in the 72 dp that + /// launchers show. The icon's edge pixels continue to the layer's edges, so a full-bleed + /// icon shows no other color around it and a transparent one stays transparent. + pub fn write_adaptive_layer(&self, w: &mut W, size: u32) -> Result<()> { + let inner = size * 2 / 3; + let offset = (size - inner) / 2; + let resized = self + .img + .resize_exact(inner, inner, FilterType::Lanczos3) + .to_rgba8(); + let edge = |v: u32| v.saturating_sub(offset).min(inner - 1); + let layer = RgbaImage::from_fn(size, size, |x, y| *resized.get_pixel(edge(x), edge(y))); + layer.write_to(w, ImageOutputFormat::Png)?; + Ok(()) + } + pub fn to_vec(&self, opts: ScalerOpts) -> Vec { let mut buf = vec![]; let mut cursor = Cursor::new(&mut buf); diff --git a/rootfs/overlay/etc/elementary-mobile/xdg/autostart/pantheon-gnome-keyring-secrets.desktop b/rootfs/overlay/etc/elementary-mobile/xdg/autostart/pantheon-gnome-keyring-secrets.desktop new file mode 100644 index 0000000..955845d --- /dev/null +++ b/rootfs/overlay/etc/elementary-mobile/xdg/autostart/pantheon-gnome-keyring-secrets.desktop @@ -0,0 +1,4 @@ +[Desktop Entry] +Type=Application +Name=pantheon-gnome-keyring-secrets +Hidden=true diff --git a/rootfs/overlay/usr/local/bin/elementary-mobile-session b/rootfs/overlay/usr/local/bin/elementary-mobile-session index 76de7c7..fe13935 100755 --- a/rootfs/overlay/usr/local/bin/elementary-mobile-session +++ b/rootfs/overlay/usr/local/bin/elementary-mobile-session @@ -84,9 +84,31 @@ export LIBGL_ALWAYS_SOFTWARE=1 GSK_RENDERER=cairo export WEBKIT_DISABLE_SANDBOX_THIS_IS_DANGEROUS=1 ELECTRON_DISABLE_SANDBOX=1 export WEBKIT_DISABLE_DMABUF_RENDERER=1 WEBKIT_DISABLE_COMPOSITING_MODE=1 # Android kills apps with more than 32 child processes, so the session avoids helper processes: -# a forked bus instead of dbus-run-session, and Gala started directly rather than supervised. +# a forked bus instead of dbus-run-session, and one shell that keeps Gala running. bus_pid_file="$XDG_RUNTIME_DIR/session-bus.pid" +# What Gala starts and leaves behind when it dies. A new Gala starts them again, and Wingpanel and +# the Dock refuse to run twice, so it would keep restarting them every second. +gala_helpers='(/usr/libexec/)?(io\.elementary\.(wingpanel|dock|notifications)|gala-daemon-gtk3|mutter-x11-frames)' + +# Gala runs outside gnome-session: it reports readiness to systemd only, so as a session component +# it would hit gnome-session's registration timeout. Its screen locker also crashes without +# XDG_SESSION_ID when logind is missing. Without a window manager the screen goes black and stops +# responding, so it is started again if it dies. +run_gala() { + trap 'kill "$wm" 2>/dev/null; wait "$wm"; exit' TERM + while :; do + started=$(date +%s) + XDG_SESSION_ID=elementary-mobile gala & + wm=$! + wait "$wm" + log "Gala exited with status $?, restarting it" + pkill -x -f "$gala_helpers" + # Do not spin if it cannot start at all. + [ $(($(date +%s) - started)) -ge 10 ] || sleep 10 + done +} + # Start a fresh session after "Log Out", like a display manager with automatic login. while kill -0 "$xwayland" 2>/dev/null; do DBUS_SESSION_BUS_ADDRESS=$(dbus-daemon --session --fork --print-address=1 --print-pid=3 3>"$bus_pid_file") @@ -94,10 +116,7 @@ while kill -0 "$xwayland" 2>/dev/null; do gsettings set org.gnome.desktop.interface scaling-factor "$scale" - # Gala runs outside gnome-session: it reports readiness to systemd only, so as a session - # component it would hit gnome-session's registration timeout. Its screen locker also crashes - # without XDG_SESSION_ID when logind is missing. - XDG_SESSION_ID=elementary-mobile gala & + run_gala & gala=$! gnome-session --session=elementary-mobile diff --git a/rootfs/overlay/usr/local/share/dbus-1/services/org.gnome.zeitgeist.Engine.service b/rootfs/overlay/usr/local/share/dbus-1/services/org.gnome.zeitgeist.Engine.service new file mode 100644 index 0000000..0694f0a --- /dev/null +++ b/rootfs/overlay/usr/local/share/dbus-1/services/org.gnome.zeitgeist.Engine.service @@ -0,0 +1,5 @@ +# Shadows zeitgeist's own service file (the session bus prefers /usr/local/share), which starts the +# daemon from a shell that waits for it: one more process towards Android's limit of 32. +[D-BUS Service] +Name=org.gnome.zeitgeist.Engine +Exec=/usr/bin/zeitgeist-daemon diff --git a/src/android/accessibility.rs b/src/android/accessibility.rs index 25b02a8..38c5365 100644 --- a/src/android/accessibility.rs +++ b/src/android/accessibility.rs @@ -17,6 +17,10 @@ pub enum AppUserEvent { AccessibilityInputReady, /// The first-run install finished; switch from the progress page to the desktop. SetupFinished, + /// An external monitor's surface appeared, changed or went away. + ExternalDisplayChanged, + /// A mouse (or touch) on the external monitor moved, clicked or scrolled. + ExternalPointer, } #[derive(Clone, Copy, Debug)] @@ -147,21 +151,23 @@ fn enqueue_key_event(action: jint, key_code: jint, scan_code: jint, event_time_m _ => return false, }; - if scan_code <= 0 { + if scan_code <= 0 || !bridge().lock().unwrap().runtime_active { return false; } + inject_key(scan_code as u32, state, event_time_ms.max(0) as u64) +} + +/// Type a key into the desktop. +pub fn inject_key(scancode: u32, state: ElementState, event_time_ms: u64) -> bool { let mut bridge = bridge() .lock() .expect("Failed to lock accessibility bridge"); - if !bridge.runtime_active { - return false; - } bridge.pending_events.push_back(AccessibilityKeyEvent { - scancode: scan_code as u32, + scancode, state, - event_time_ms: event_time_ms.max(0) as u64, + event_time_ms, }); let proxy = bridge.proxy.clone(); diff --git a/src/android/app/run.rs b/src/android/app/run.rs index 213eec1..6f93da4 100644 --- a/src/android/app/run.rs +++ b/src/android/app/run.rs @@ -1,4 +1,6 @@ +use std::sync::Arc; use std::thread; +use std::time::{Duration, Instant}; use super::build::{PolarBearApp, PolarBearBackend}; use crate::android::{ @@ -6,12 +8,12 @@ use crate::android::{ backend::{ pipewire_standalone_aaudio, wayland::{ - bind, centralize, centralize_injected_keyboard, handle, - write_guest_output_state, - CentralizedEvent, State, + bind, centralize, centralize_injected_keyboard, handle, on_external_pointer, + paint_trackpad, write_guest_output_state, CentralizedEvent, State, WaylandBackend, }, webview::ErrorVariant, }, + external_display, proot::{launch::launch, setup::build_wayland_backend}, utils::{ ndk::{self, run_in_jvm}, @@ -28,16 +30,22 @@ use winit::event::WindowEvent; use winit::event_loop::{ActiveEventLoop, ControlFlow}; use winit::window::WindowId; -fn configure_output(backend: &mut crate::android::backend::wayland::WaylandBackend) { +fn configure_output(backend: &mut WaylandBackend) { let Some(winit) = backend.graphic_renderer.as_ref() else { return; }; - let window_size = winit.window_size(); + let window_size = winit.output_size(); let size = (window_size.w, window_size.h); - // Not `winit.scale_factor()`: that reads `AConfiguration`, which still reports the 160 dpi - // default on the first launch and only becomes accurate after a configuration change. - let guest_scale_factor = ndk::scale_factor(&backend.android_app); + let (guest_scale_factor, refresh_millihz) = match winit.external_surface() { + Some(surface) => ( + (surface.density_dpi as f64 / 160.0).max(1.0), + surface.refresh_millihz, + ), + // Not `winit.scale_factor()`: that reads `AConfiguration`, which still reports the 160 + // dpi default on the first launch and only becomes accurate after a configuration change. + None => (ndk::scale_factor(&backend.android_app), 60000), + }; backend.guest_scale_factor = guest_scale_factor; backend.compositor.state.size = size.into(); @@ -65,14 +73,15 @@ fn configure_output(backend: &mut crate::android::backend::wayland::WaylandBacke output.change_current_state( Some(Mode { size: size.into(), - refresh: 60000, + refresh: refresh_millihz, }), Some(Transform::Normal), Some(Scale::Integer(1)), Some((0, 0).into()), ); - let guest_scale = crate::core::config::guest_ui_scale(guest_scale_factor); + let guest_scale = + crate::core::config::guest_ui_scale(guest_scale_factor, window_size.w, window_size.h); write_guest_output_state(window_size.w, window_size.h, guest_scale); for surface in backend.compositor.state.xdg_shell_state.toplevel_surfaces() { @@ -80,6 +89,44 @@ fn configure_output(backend: &mut crate::android::backend::wayland::WaylandBacke } } +/// Draw to the external monitor if one is connected, otherwise back to the phone's screen. +fn apply_external_display(backend: &mut WaylandBackend) { + let Some(winit) = backend.graphic_renderer.as_mut() else { + return; + }; + let surface = external_display::surface(); + let unchanged = match (&surface, winit.external_surface()) { + (Some(new), Some(current)) => Arc::ptr_eq(new, current), + (None, None) => true, + _ => false, + }; + if unchanged { + return; + } + let attached = match surface { + Some(surface) => match winit.attach_external(surface) { + Ok(()) => true, + Err(error) => { + log::error!("{error}"); + false + } + }, + None => false, + }; + if !attached { + winit.detach_external(); + } + external_display::set_in_use(attached); + + backend.reset_touch_state(); + configure_output(backend); + if attached { + let size = backend.compositor.state.size; + backend.cursor_position = (size.w as f64 / 2.0, size.h as f64 / 2.0).into(); + paint_trackpad(backend); + } +} + fn configure_toplevel(surface: &ToplevelSurface, width: i32, height: i32) { surface.with_pending_state(|state| { state.size.replace((width, height).into()); @@ -130,12 +177,20 @@ impl ApplicationHandler for PolarBearApp { return; } } - } else { - log::info!("Ignoring redundant resume while renderer is already active"); + } else if let Some(winit) = backend.graphic_renderer.as_mut() { + // Back from the background, where only the external display was drawn to. + if let Err(error) = winit.resume_window() { + log::error!("Failed to draw to the window again on resume: {error}"); + } } + apply_external_display(backend); configure_output(backend); + if backend.is_trackpad() { + paint_trackpad(backend); + } accessibility::set_runtime_active(true); + run_in_jvm(external_display::start, self.frontend.android_app.clone()); if let Some(winit) = backend.graphic_renderer.as_ref() { winit.window().request_redraw(); @@ -156,8 +211,9 @@ impl ApplicationHandler for PolarBearApp { |env, _| close_webview_popups(env), self.frontend.android_app.clone(), ); - self.backend = - PolarBearBackend::Wayland(build_wayland_backend(self.frontend.android_app.clone())); + self.backend = PolarBearBackend::Wayland(build_wayland_backend( + self.frontend.android_app.clone(), + )); // Same as coming back to the app: bind the renderer and start the session. If the // app is in the background, this fails quietly and the next resume does it. self.resumed(event_loop); @@ -170,14 +226,40 @@ impl ApplicationHandler for PolarBearApp { return; }; - for event in accessibility::drain_pending_events() { - let event = centralize_injected_keyboard( - event.scancode, - event.state, - event.event_time_ms, - backend, - ); - handle(event, backend, event_loop); + match event { + AppUserEvent::ExternalDisplayChanged => { + apply_external_display(backend); + handle(CentralizedEvent::Redraw, backend, event_loop); + } + AppUserEvent::ExternalPointer => { + for event in external_display::drain_pointer_events() { + on_external_pointer(backend, event); + } + } + _ => { + for event in accessibility::drain_pending_events() { + let event = centralize_injected_keyboard( + event.scancode, + event.state, + event.event_time_ms, + backend, + ); + handle(event, backend, event_loop); + } + } + } + } + + fn about_to_wait(&mut self, event_loop: &ActiveEventLoop) { + // In the background the window no longer asks for redraws, but the desktop is still on + // the external display. + if let PolarBearBackend::Wayland(backend) = &mut self.backend { + if backend.is_trackpad() && !backend.window_visible() { + handle(CentralizedEvent::Redraw, backend, event_loop); + event_loop.set_control_flow(ControlFlow::WaitUntil( + Instant::now() + Duration::from_millis(16), + )); + } } } @@ -208,10 +290,25 @@ impl ApplicationHandler for PolarBearApp { event_loop.set_control_flow(ControlFlow::Wait); if let PolarBearBackend::Wayland(backend) = &mut self.backend { - backend.graphic_renderer = None; backend.key_counter = 0; backend.reset_touch_state(); backend.pointer_pressed = false; + + if backend.is_trackpad() { + // The monitor stays a desktop display, for a mouse and keyboard on it. + if let Some(winit) = backend.graphic_renderer.as_mut() { + winit.suspend_window(); + } + run_in_jvm( + external_display::background, + self.frontend.android_app.clone(), + ); + return; + } + + backend.graphic_renderer = None; + external_display::set_in_use(false); + run_in_jvm(external_display::stop, self.frontend.android_app.clone()); // Kill the standalone-client PipeWire/AAudio backend if it was started. pipewire_standalone_aaudio::shutdown(); } diff --git a/src/android/backend/wayland/compositor.rs b/src/android/backend/wayland/compositor.rs index e3d3f24..ec02796 100644 --- a/src/android/backend/wayland/compositor.rs +++ b/src/android/backend/wayland/compositor.rs @@ -3,7 +3,7 @@ use smithay::{ backend::renderer::utils::on_commit_buffer_handler, delegate_compositor, delegate_data_device, delegate_output, delegate_seat, delegate_shm, delegate_xdg_shell, - input::{self, keyboard::KeyboardHandle, touch::TouchHandle, Seat, SeatHandler, SeatState}, + input::{keyboard::KeyboardHandle, touch::TouchHandle, Seat, SeatHandler, SeatState}, output::Output, reexports::{ wayland_protocols::xdg::shell::server::xdg_toplevel, @@ -30,7 +30,7 @@ use smithay::{ }, }; use smithay::{ - input::pointer::PointerHandle, + input::pointer::{CursorImageStatus, PointerHandle}, reexports::wayland_server::{ backend::{ClientData, ClientId, DisconnectReason, GlobalId}, protocol::{wl_buffer, wl_surface::WlSurface}, @@ -60,6 +60,8 @@ pub struct State { pub data_device_state: DataDeviceState, pub seat_state: SeatState, pub size: Size, + /// What the client wants the cursor to look like. + pub cursor_image: CursorImageStatus, } impl BufferHandler for State { @@ -142,7 +144,9 @@ impl SeatHandler for State { } fn focus_changed(&mut self, _seat: &Seat, _focused: Option<&WlSurface>) {} - fn cursor_image(&mut self, _seat: &Seat, _image: input::pointer::CursorImageStatus) {} + fn cursor_image(&mut self, _seat: &Seat, image: CursorImageStatus) { + self.cursor_image = image; + } } pub fn send_frames_surface_tree(surface: &WlSurface, time: u32) { @@ -215,6 +219,7 @@ impl Compositor { data_device_state: DataDeviceState::new::(&dh), seat_state, size: (1920, 1080).into(), + cursor_image: CursorImageStatus::default_named(), }; Ok(Compositor { diff --git a/src/android/backend/wayland/event_centralizer.rs b/src/android/backend/wayland/event_centralizer.rs index 4441147..58e6a4f 100644 --- a/src/android/backend/wayland/event_centralizer.rs +++ b/src/android/backend/wayland/event_centralizer.rs @@ -35,6 +35,18 @@ pub enum CentralizedEvent { /// A redraw was requested Redraw, + /// The trackpad moved the cursor by this many phone pixels. + TrackpadMotion { dx: f64, dy: f64, time: u64 }, + + /// The last finger left the trackpad. + TrackpadUp { + /// What the gesture had been resolved to. + mode: TouchMode, + /// Most fingers down at once during the gesture. + fingers: usize, + time: u64, + }, + /// TODO: Support these events Unsupported, } @@ -136,6 +148,9 @@ pub fn centralize(event: WindowEvent, backend: &mut WaylandBackend) -> Centraliz }; CentralizedEvent::Input(event) } + WindowEvent::Touch(touch) if backend.is_trackpad() => { + centralize_trackpad(touch, time, backend) + } WindowEvent::Touch(Touch { phase: TouchPhase::Started, location, @@ -186,29 +201,10 @@ pub fn centralize(event: WindowEvent, backend: &mut WaylandBackend) -> Centraliz match backend.touch_mode { // Still deciding between a tap, a scroll and a long press: don't move the // cursor yet, or a scroll would drag it along. - TouchMode::Undecided | TouchMode::LongPress => CentralizedEvent::Unsupported, - TouchMode::Scroll => { - // Scroll by the centroid delta, which is the finger itself when only one - // is down. Positive axis values scroll the view down, so the raw delta - // (negated once more in `WinitMouseWheelEvent::amount`) makes the content - // follow the finger the way Android does. - let new_centroid = centroid(&backend.touch_points); - let last = backend.scroll_centroid.replace(new_centroid); - let Some(last) = last else { - return CentralizedEvent::Unsupported; - }; - let dx = new_centroid.x - last.x; - let dy = new_centroid.y - last.y; - if dx == 0.0 && dy == 0.0 { - return CentralizedEvent::Unsupported; - } - CentralizedEvent::Input(InputEvent::PointerAxis { - event: WinitMouseWheelEvent { - time, - delta: MouseScrollDelta::PixelDelta(PhysicalPosition { x: dx, y: dy }), - }, - }) + TouchMode::Undecided | TouchMode::LongPress | TouchMode::Move => { + CentralizedEvent::Unsupported } + TouchMode::Scroll => scroll(backend, time), TouchMode::Drag => CentralizedEvent::Input(InputEvent::TouchMotion { event: WinitTouchMovedEvent { time, @@ -270,6 +266,113 @@ pub fn centralize(event: WindowEvent, backend: &mut WaylandBackend) -> Centraliz }; } +/// Scroll by the centroid delta, which is the finger itself when only one is down. Positive +/// axis values scroll the view down, so the raw delta (negated once more in +/// `WinitMouseWheelEvent::amount`) makes the content follow the finger the way Android does. +fn scroll(backend: &mut WaylandBackend, time: u64) -> CentralizedEvent { + let new_centroid = centroid(&backend.touch_points); + let last = backend.scroll_centroid.replace(new_centroid); + let Some(last) = last else { + return CentralizedEvent::Unsupported; + }; + let dx = new_centroid.x - last.x; + let dy = new_centroid.y - last.y; + if dx == 0.0 && dy == 0.0 { + return CentralizedEvent::Unsupported; + } + CentralizedEvent::Input(InputEvent::PointerAxis { + event: WinitMouseWheelEvent { + time, + delta: MouseScrollDelta::PixelDelta(PhysicalPosition { x: dx, y: dy }), + }, + }) +} + +/// The phone as a trackpad for the external monitor: one finger moves the cursor, taps click +/// (two fingers for a right click), two fingers scroll, and a long press starts a drag. +fn centralize_trackpad(touch: Touch, time: u64, backend: &mut WaylandBackend) -> CentralizedEvent { + let Touch { + phase, + location, + id, + .. + } = touch; + match phase { + TouchPhase::Started => { + backend.touch_points.insert(id, location); + backend.touch_fingers = backend.touch_fingers.max(backend.touch_points.len()); + backend.scroll_centroid = Some(centroid(&backend.touch_points)); + if backend.touch_points.len() == 1 { + backend.touch_mode = TouchMode::Undecided; + backend.touch_down_position = Some(location); + backend.touch_down_time = Some(time); + } else if backend.touch_mode == TouchMode::LongPress { + // Could still be a two-finger tap, until the fingers move together. + backend.touch_mode = TouchMode::Undecided; + } + CentralizedEvent::Unsupported + } + TouchPhase::Moved => { + let last = backend.touch_points.insert(id, location); + + if backend.touch_points.len() >= 2 { + if backend.touch_mode != TouchMode::Scroll { + let Some(anchor) = backend.scroll_centroid else { + return CentralizedEvent::Unsupported; + }; + let now = centroid(&backend.touch_points); + let (dx, dy) = (now.x - anchor.x, now.y - anchor.y); + if dx * dx + dy * dy <= backend.touch_slop_px * backend.touch_slop_px { + return CentralizedEvent::Unsupported; + } + backend.touch_mode = TouchMode::Scroll; + backend.scroll_centroid = Some(centroid(&backend.touch_points)); + } + return scroll(backend, time); + } + + let past_slop = travelled_past_slop(backend, location); + match backend.touch_mode { + TouchMode::Undecided if past_slop => backend.touch_mode = TouchMode::Move, + TouchMode::LongPress if past_slop => backend.touch_mode = TouchMode::Drag, + TouchMode::Scroll => return scroll(backend, time), + _ => {} + } + match (backend.touch_mode, last) { + (TouchMode::Move | TouchMode::Drag, Some(last)) => { + CentralizedEvent::TrackpadMotion { + dx: location.x - last.x, + dy: location.y - last.y, + time, + } + } + _ => CentralizedEvent::Unsupported, + } + } + TouchPhase::Ended | TouchPhase::Cancelled => { + backend.touch_points.remove(&id); + if !backend.touch_points.is_empty() { + backend.scroll_centroid = Some(centroid(&backend.touch_points)); + return CentralizedEvent::Unsupported; + } + + let mode = backend.touch_mode; + let fingers = backend.touch_fingers; + backend.reset_touch_state(); + if phase == TouchPhase::Cancelled { + return CentralizedEvent::Input(InputEvent::TouchCancel { + event: WinitTouchCancelledEvent { time, id }, + }); + } + CentralizedEvent::TrackpadUp { + mode, + fingers, + time, + } + } + } +} + /// Normalize a window-relative pixel position into the 0..1 range the input backend expects. fn relative_position( backend: &WaylandBackend, diff --git a/src/android/backend/wayland/event_handler.rs b/src/android/backend/wayland/event_handler.rs index e6eabb8..e78769c 100644 --- a/src/android/backend/wayland/event_handler.rs +++ b/src/android/backend/wayland/event_handler.rs @@ -4,19 +4,22 @@ use crate::android::{ compositor::{send_frames_surface_tree, ClientState, State}, write_guest_output_state, CentralizedEvent, TouchMode, WaylandBackend, }, + external_display::PointerEvent, }; -use smithay::backend::input::ButtonState; +use smithay::backend::input::{AxisSource, ButtonState}; +use smithay::backend::renderer::element::solid::SolidColorRenderElement; use smithay::backend::renderer::element::surface::{ render_elements_from_surface_tree, WaylandSurfaceRenderElement, }; -use smithay::backend::renderer::element::Kind; +use smithay::backend::renderer::element::{render_elements, Id, Kind}; use smithay::backend::renderer::gles::GlesRenderer; -use smithay::backend::renderer::utils::draw_render_elements; +use smithay::backend::renderer::utils::{draw_render_elements, CommitCounter}; use smithay::backend::renderer::{Color32F, Frame, Renderer}; use smithay::input::keyboard::FilterResult; -use smithay::input::pointer; +use smithay::input::pointer::{self, CursorImageStatus, CursorImageSurfaceData}; use smithay::reexports::wayland_server::protocol::wl_pointer::ButtonState as WlButtonState; use smithay::utils::{Point, Rectangle, Transform, SERIAL_COUNTER}; +use smithay::wayland::compositor::with_states; use smithay::wayland::shell::xdg::ToplevelSurface; use smithay::{ backend::input::{ @@ -32,6 +35,14 @@ use winit::event_loop::{ActiveEventLoop, ControlFlow}; const BTN_LEFT: u32 = 0x110; /// Linux input event code for the right mouse button (`BTN_RIGHT`). const BTN_RIGHT: u32 = 0x111; +/// Linux input event code for the middle mouse button (`BTN_MIDDLE`). +const BTN_MIDDLE: u32 = 0x112; +/// External display pixels the cursor moves per phone pixel the finger moves. +const TRACKPAD_SPEED: f64 = 1.5; +/// What the phone shows while it is a trackpad. +const TRACKPAD_COLOR: Color32F = Color32F::new(0.0, 0.0, 0.0, 1.0); +/// Pixels a wheel notch scrolls, as libinput reports for a mouse wheel. +const WHEEL_STEP: f64 = 15.0; /** * As we currently use Xwayland, there is only 1 surface @@ -159,6 +170,9 @@ fn poll_long_press(backend: &mut WaylandBackend) { } backend.touch_mode = TouchMode::LongPress; + if backend.is_trackpad() { + return; + } // Anchor the pointer where the finger landed, so a drag selects from there. emit_pointer_motion( &mut backend.compositor, @@ -268,7 +282,7 @@ pub fn handle(event: CentralizedEvent, backend: &mut WaylandBackend, event_loop: time, ), // A scroll consumed the gesture; nothing to click. - TouchMode::Scroll | TouchMode::Drag => {} + TouchMode::Scroll | TouchMode::Drag | TouchMode::Move => {} } } } @@ -376,6 +390,45 @@ pub fn handle(event: CentralizedEvent, backend: &mut WaylandBackend, event_loop: } _ => {} }, + CentralizedEvent::TrackpadMotion { dx, dy, time } => { + if backend.touch_mode == TouchMode::Drag && !backend.pointer_pressed { + emit_pointer_press(&mut backend.compositor, BTN_LEFT, time as u32); + backend.pointer_pressed = true; + } + let size = backend.compositor.state.size; + let cursor = &mut backend.cursor_position; + cursor.x = (cursor.x + dx * TRACKPAD_SPEED).clamp(0.0, (size.w - 1).max(0) as f64); + cursor.y = (cursor.y + dy * TRACKPAD_SPEED).clamp(0.0, (size.h - 1).max(0) as f64); + let cursor = backend.cursor_position; + emit_pointer_motion(&mut backend.compositor, cursor.x, cursor.y, time as u32); + } + CentralizedEvent::TrackpadUp { + mode, + fingers, + time, + } => { + let cursor = backend.cursor_position; + if backend.pointer_pressed { + emit_pointer_release(&mut backend.compositor, BTN_LEFT, time as u32); + backend.pointer_pressed = false; + return; + } + let button = match (mode, fingers) { + (TouchMode::Undecided, 1) => BTN_LEFT, + (TouchMode::Undecided, 2) | (TouchMode::LongPress, 1) => BTN_RIGHT, + (TouchMode::Undecided, _) => BTN_MIDDLE, + _ => return, + }; + emit_pointer_click( + &mut backend.compositor, + button, + cursor.x, + cursor.y, + time as u32, + ); + } + // The desktop is on the external display, so the phone's own window doesn't size it. + CentralizedEvent::Resized { .. } if backend.is_trackpad() => paint_trackpad(backend), CentralizedEvent::Resized { size, guest_scale_factor, @@ -394,7 +447,8 @@ pub fn handle(event: CentralizedEvent, backend: &mut WaylandBackend, event_loop: ); } - let guest_scale = crate::core::config::guest_ui_scale(guest_scale_factor); + let guest_scale = + crate::core::config::guest_ui_scale(guest_scale_factor, size.w, size.h); write_guest_output_state(size.w, size.h, guest_scale); if let Some(surface) = get_surface(&backend.compositor.state) { @@ -405,13 +459,74 @@ pub fn handle(event: CentralizedEvent, backend: &mut WaylandBackend, event_loop: } } +/// Show the phone's screen as a blank trackpad; `TrackpadControls` draws its outline on top. +pub fn paint_trackpad(backend: &mut WaylandBackend) { + if let Some(winit) = backend.graphic_renderer.as_mut() { + if let Err(error) = winit.paint_window(TRACKPAD_COLOR) { + log::warn!("Failed to paint the trackpad: {error}"); + } + } +} + +/// A mouse on the external display: it points where it is, and its buttons and wheel go +/// straight to the desktop. +pub fn on_external_pointer(backend: &mut WaylandBackend, event: PointerEvent) { + let time = event.time_ms as u32; + let size = backend.compositor.state.size; + backend.cursor_position = ( + event.x.clamp(0.0, (size.w - 1).max(0) as f64), + event.y.clamp(0.0, (size.h - 1).max(0) as f64), + ) + .into(); + let cursor = backend.cursor_position; + emit_pointer_motion(&mut backend.compositor, cursor.x, cursor.y, time); + + // `MotionEvent.BUTTON_PRIMARY`, `BUTTON_SECONDARY` and `BUTTON_TERTIARY`. + for (mask, button) in [(1, BTN_LEFT), (2, BTN_RIGHT), (4, BTN_MIDDLE)] { + let was = backend.external_buttons & mask != 0; + let is = event.buttons & mask != 0; + if is && !was { + emit_pointer_press(&mut backend.compositor, button, time); + } else if was && !is { + emit_pointer_release(&mut backend.compositor, button, time); + } + } + backend.external_buttons = event.buttons; + + if event.scroll_x != 0.0 || event.scroll_y != 0.0 { + // Android scrolls up and left with positive values, Wayland down and right. + let mut frame = pointer::AxisFrame::new(time).source(AxisSource::Wheel); + if event.scroll_x != 0.0 { + frame = frame + .value(Axis::Horizontal, event.scroll_x * WHEEL_STEP) + .v120(Axis::Horizontal, (event.scroll_x * 120.0) as i32); + } + if event.scroll_y != 0.0 { + frame = frame + .value(Axis::Vertical, -event.scroll_y * WHEEL_STEP) + .v120(Axis::Vertical, (-event.scroll_y * 120.0) as i32); + } + let compositor = &mut backend.compositor; + let pointer = compositor.pointer.clone(); + pointer.axis(&mut compositor.state, frame); + pointer.frame(&mut compositor.state); + } +} + +render_elements! { + SurfaceElement<=GlesRenderer>; + Surface=WaylandSurfaceRenderElement, + Solid=SolidColorRenderElement, +} + fn redraw(backend: &mut WaylandBackend) -> Result<(), String> { let Some(winit) = backend.graphic_renderer.as_mut() else { return Ok(()); }; - let size = winit.window_size(); + let size = winit.output_size(); let damage = Rectangle::from_size(size); + let trackpad = winit.external_surface().is_some(); { let (renderer, mut framebuffer) = winit .bind() @@ -419,13 +534,48 @@ fn redraw(backend: &mut WaylandBackend) -> Result<(), String> { let compositor = &mut backend.compositor; - let elements = compositor - .state - .xdg_shell_state - .toplevel_surfaces() - .iter() - .flat_map(|surface| { - render_elements_from_surface_tree( + // Elements are drawn front to back, so the cursor goes first. + let mut elements = Vec::new(); + if trackpad { + let position = backend.cursor_position.to_physical(1.0).to_i32_round(); + match &compositor.state.cursor_image { + CursorImageStatus::Surface(surface) => { + let hotspot = with_states(surface, |states| { + states + .data_map + .get::() + .map(|data| data.lock().unwrap().hotspot) + }) + .unwrap_or_default(); + elements.extend( + render_elements_from_surface_tree::<_, WaylandSurfaceRenderElement<_>>( + renderer, + surface, + position - hotspot.to_physical(1), + 1.0, + 1.0, + Kind::Cursor, + ) + .into_iter() + .map(SurfaceElement::Surface), + ); + } + // The client didn't provide an image; a small square at least shows where it is. + CursorImageStatus::Named(_) => { + elements.push(SurfaceElement::Solid(SolidColorRenderElement::new( + Id::new(), + Rectangle::new(position, (8, 8).into()), + CommitCounter::default(), + Color32F::new(1.0, 1.0, 1.0, 1.0), + Kind::Cursor, + ))); + } + CursorImageStatus::Hidden => {} + } + } + for surface in compositor.state.xdg_shell_state.toplevel_surfaces() { + elements.extend( + render_elements_from_surface_tree::<_, WaylandSurfaceRenderElement<_>>( renderer, surface.wl_surface(), (0, 0), @@ -433,8 +583,10 @@ fn redraw(backend: &mut WaylandBackend) -> Result<(), String> { 1.0, Kind::Unspecified, ) - }) - .collect::>>(); + .into_iter() + .map(SurfaceElement::Surface), + ); + } let mut frame = renderer .render(&mut framebuffer, size, Transform::Flipped180) @@ -449,11 +601,12 @@ fn redraw(backend: &mut WaylandBackend) -> Result<(), String> { .finish() .map_err(|error| format!("Failed to finish frame: {error:?}"))?; + let time = compositor.start_time.elapsed().as_millis() as u32; for surface in compositor.state.xdg_shell_state.toplevel_surfaces() { - send_frames_surface_tree( - surface.wl_surface(), - compositor.start_time.elapsed().as_millis() as u32, - ); + send_frames_surface_tree(surface.wl_surface(), time); + } + if let CursorImageStatus::Surface(surface) = &compositor.state.cursor_image { + send_frames_surface_tree(surface, time); } match compositor.listener.accept() { diff --git a/src/android/backend/wayland/mod.rs b/src/android/backend/wayland/mod.rs index c356405..d222933 100644 --- a/src/android/backend/wayland/mod.rs +++ b/src/android/backend/wayland/mod.rs @@ -7,16 +7,17 @@ mod keymap; mod output_state; mod winit_backend; +pub use keymap::physicalkey_to_scancode; pub use output_state::write_guest_output_state; pub use compositor::{Compositor, State}; pub use event_centralizer::{centralize, centralize_injected_keyboard, CentralizedEvent}; -pub use event_handler::handle; +pub use event_handler::{handle, on_external_pointer, paint_trackpad}; pub use winit_backend::{bind, WinitGraphicsBackend}; use smithay::{ backend::renderer::gles::GlesRenderer, - utils::{Clock, Monotonic}, + utils::{Clock, Logical, Monotonic, Point}, }; use std::collections::HashMap; use winit::dpi::PhysicalPosition; @@ -29,6 +30,8 @@ pub enum TouchMode { Undecided, /// Moved past touch slop before the long press fired. Scroll, + /// As a trackpad: one finger moved past touch slop, so it moves the cursor. + Move, /// Long-press timeout elapsed without moving; no button sent yet. LongPress, /// Moved after a long press: left button held down. @@ -58,6 +61,12 @@ pub struct WaylandBackend { pub long_press_timeout_ms: u64, /// Whether a synthesized button press is currently held (an in-progress drag). pub pointer_pressed: bool, + /// Most fingers down at once during the current gesture. + pub touch_fingers: usize, + /// Where the trackpad has moved the cursor to on the external display. + pub cursor_position: Point, + /// Buttons held by a mouse on the external display (`MotionEvent.getButtonState()`). + pub external_buttons: i32, } impl WaylandBackend { @@ -68,5 +77,20 @@ impl WaylandBackend { self.touch_mode = TouchMode::Undecided; self.touch_down_position = None; self.touch_down_time = None; + self.touch_fingers = 0; + } + + /// Whether the phone's own window is there to draw to (not while the app is in the background). + pub fn window_visible(&self) -> bool { + self.graphic_renderer + .as_ref() + .is_some_and(|winit| winit.window_visible()) + } + + /// Whether the desktop is on an external monitor, with the phone as its trackpad. + pub fn is_trackpad(&self) -> bool { + self.graphic_renderer + .as_ref() + .is_some_and(|winit| winit.external_surface().is_some()) } } diff --git a/src/android/backend/wayland/winit_backend.rs b/src/android/backend/wayland/winit_backend.rs index 23e466f..40f4920 100644 --- a/src/android/backend/wayland/winit_backend.rs +++ b/src/android/backend/wayland/winit_backend.rs @@ -18,7 +18,9 @@ //! The other types in this module are the instances of the associated types of these //! two traits for the winit backend. +use crate::android::external_display::ExternalSurface; use khronos_egl::DynamicInstance; +use smithay::backend::egl::display::PixelFormat; use smithay::{ backend::{ egl::{ @@ -29,11 +31,11 @@ use smithay::{ }, renderer::{ gles::{GlesError, GlesRenderer}, - Bind, + Bind, Color32F, Frame, Renderer, }, SwapBuffersError, }, - utils::{Physical, Rectangle, Size}, + utils::{Physical, Rectangle, Size, Transform}, }; use std::ffi::c_void; use std::sync::Arc; @@ -208,7 +210,7 @@ pub fn bind(event_loop: &ActiveEventLoop) -> Result { let display = create_egl_display(handle) .map_err(|error| format!("Failed to create EGLDisplay: {error:?}"))?; @@ -218,18 +220,10 @@ pub fn bind(event_loop: &ActiveEventLoop) -> Result return Err(format!("Unsupported platform: {:?}", platform)), }; @@ -243,13 +237,32 @@ pub fn bind(event_loop: &ActiveEventLoop) -> Result Result { + unsafe { + EGLSurface::new( + display, + pixel_format, + context.config_id(), + AndroidNativeSurface { handle }, + ) + } + .map_err(|error| format!("Failed to create EGLSurface: {error}")) +} + /// Errors thrown by the `winit` backends #[derive(Debug)] pub enum Error { @@ -273,10 +286,114 @@ pub struct WinitGraphicsBackend { renderer: R, // The display isn't used past this point but must be kept alive. _display: EGLDisplay, - egl_surface: EGLSurface, + /// The phone's own window; gone while the app is in the background. + egl_surface: Option, window: Arc, damage_tracking: bool, bind_size: Option>, + pixel_format: PixelFormat, + /// Drawn to instead of `window` while an external monitor is connected. + external: Option, +} + +#[derive(Debug)] +struct ExternalTarget { + egl_surface: EGLSurface, + bind_size: Option>, + // Declared last: the EGL surface must go before the native window it was created on. + surface: Arc, +} + +impl WinitGraphicsBackend { + /// Draw to `surface` from now on, until `detach_external`. + pub fn attach_external(&mut self, surface: Arc) -> Result<(), String> { + // A native window takes one EGL surface at a time. + self.external = None; + let handle = AndroidNdkWindowHandle::new(surface.native_window()); + let egl_surface = unsafe { + EGLSurface::new( + &self._display, + self.pixel_format, + self.renderer.egl_context().config_id(), + AndroidNativeSurface { handle }, + ) + } + .map_err(|error| { + format!("Failed to create EGLSurface for the external display: {error}") + })?; + self.external = Some(ExternalTarget { + egl_surface, + bind_size: None, + surface, + }); + Ok(()) + } + + pub fn detach_external(&mut self) { + self.external = None; + } + + /// Let go of the phone's window as the app goes to the background, but keep drawing to the + /// external monitor. + pub fn suspend_window(&mut self) { + self.egl_surface = None; + self.bind_size = None; + } + + pub fn window_visible(&self) -> bool { + self.egl_surface.is_some() + } + + /// Draw to the phone's window again once the app is back, if it had been let go. + pub fn resume_window(&mut self) -> Result<(), String> { + if self.egl_surface.is_some() { + return Ok(()); + } + let handle = match self.window.window_handle().map(|handle| handle.as_raw()) { + Ok(RawWindowHandle::AndroidNdk(handle)) => handle, + Ok(platform) => return Err(format!("Unsupported platform: {platform:?}")), + Err(error) => return Err(format!("Failed to get window handle: {error}")), + }; + let surface = create_window_surface( + &self._display, + self.pixel_format, + self.renderer.egl_context(), + handle, + )?; + self.egl_surface = Some(surface); + Ok(()) + } + + /// Fill the phone's own window with one color, for when it serves as a trackpad. + pub fn paint_window(&mut self, color: Color32F) -> Result<(), String> { + let size = self.window_size(); + let Some(egl_surface) = self.egl_surface.as_mut() else { + return Ok(()); + }; + if Some(size) != self.bind_size { + egl_surface.resize(size.w, size.h, 0, 0); + self.bind_size = Some(size); + } + { + let mut framebuffer = self + .renderer + .bind(egl_surface) + .map_err(|error| format!("Failed to bind the window: {error}"))?; + let mut frame = self + .renderer + .render(&mut framebuffer, size, Transform::Normal) + .map_err(|error| format!("Failed to render the window: {error:?}"))?; + frame + .clear(color, &[Rectangle::from_size(size)]) + .map_err(|error| format!("Failed to clear the window: {error:?}"))?; + let _ = frame + .finish() + .map_err(|error| format!("Failed to finish the window: {error:?}"))?; + } + egl_surface + .swap_buffers(None) + .map_err(|error| format!("Failed to submit the window: {error}")) + } } impl WinitGraphicsBackend @@ -290,6 +407,19 @@ where (w, h).into() } + /// The external monitor being drawn to, if any. + pub fn external_surface(&self) -> Option<&Arc> { + self.external.as_ref().map(|target| &target.surface) + } + + /// Size of what is drawn to: the external monitor if there is one, otherwise the window. + pub fn output_size(&self) -> Size { + match &self.external { + Some(target) => (target.surface.width, target.surface.height).into(), + None => self.window_size(), + } + } + /// Scale factor of the underlying window. pub fn scale_factor(&self) -> f64 { self.window.scale_factor() @@ -311,25 +441,26 @@ where // buffer will be latched. Some nvidia drivers may not like it, but a lot of wayland // software does the order that way due to mesa latching back buffer on each // `make_current`. - let window_size = self.window_size(); - if Some(window_size) != self.bind_size { - self.egl_surface.resize(window_size.w, window_size.h, 0, 0); + let output_size = self.output_size(); + let (egl_surface, bind_size) = match (&mut self.external, &mut self.egl_surface) { + (Some(target), _) => (&mut target.egl_surface, &mut target.bind_size), + (None, Some(egl_surface)) => (egl_surface, &mut self.bind_size), + (None, None) => { + return Err(SwapBuffersError::TemporaryFailure( + "the app is in the background".into(), + )) + } + }; + if Some(output_size) != *bind_size { + egl_surface.resize(output_size.w, output_size.h, 0, 0); } - self.bind_size = Some(window_size); + *bind_size = Some(output_size); - let fb = self.renderer.bind(&mut self.egl_surface)?; + let fb = self.renderer.bind(egl_surface)?; Ok((&mut self.renderer, fb)) } - /// Retrieve the underlying `EGLSurface` for advanced operations - /// - /// **Note:** Don't carelessly use this to manually bind the renderer to the surface, - /// `WinitGraphicsBackend::bind` transparently handles window resizes for you. - pub fn egl_surface(&self) -> &EGLSurface { - &self.egl_surface - } - /// Retrieve the buffer age of the current backbuffer of the window. /// /// This will only return a meaningful value, if this `WinitGraphicsBackend` @@ -340,7 +471,11 @@ where /// likely interpret an error just as if "0" was returned. pub fn buffer_age(&self) -> Option { if self.damage_tracking { - self.egl_surface.buffer_age().map(|x| x as usize) + let egl_surface = match &self.external { + Some(target) => &target.egl_surface, + None => self.egl_surface.as_ref()?, + }; + egl_surface.buffer_age().map(|x| x as usize) } else { Some(0) } @@ -352,11 +487,18 @@ where &mut self, damage: Option<&[Rectangle]>, ) -> Result<(), SwapBuffersError> { + let (egl_surface, bind_size) = match (&self.external, &self.egl_surface) { + (Some(target), _) => (&target.egl_surface, target.bind_size), + (None, Some(egl_surface)) => (egl_surface, self.bind_size), + (None, None) => { + return Err(SwapBuffersError::TemporaryFailure( + "the app is in the background".into(), + )) + } + }; let mut damage = match damage { Some(damage) if self.damage_tracking && !damage.is_empty() => { - let bind_size = self - .bind_size - .expect("submitting without ever binding the renderer."); + let bind_size = bind_size.expect("submitting without ever binding the renderer."); let damage = damage .iter() .map(|rect| { @@ -373,7 +515,7 @@ where // Request frame callback. self.window.pre_present_notify(); - self.egl_surface.swap_buffers(damage.as_deref_mut())?; + egl_surface.swap_buffers(damage.as_deref_mut())?; Ok(()) } } diff --git a/src/android/dex/README.md b/src/android/dex/README.md index 23580ce..ed4a1e8 100644 --- a/src/android/dex/README.md +++ b/src/android/dex/README.md @@ -1,6 +1,6 @@ This directory stores JVM bytecode artifacts that the on-device `cargo run` APK builder can embed without requiring a Java or Kotlin toolchain in Termux. -`classes.dex` currently contains the Java classes in [`../java/app/polarbear/`](../java/app/polarbear/): the keyboard accessibility service and the prompt that asks before turning it on. +`classes.dex` currently contains the Java classes in [`../java/app/polarbear/`](../java/app/polarbear/): the keyboard accessibility service, the prompt that asks before turning it on, and the external display presentation with its trackpad controls. To regenerate it on a machine with `javac`, `d8`, and Android platform `android.jar` available: diff --git a/src/android/dex/classes.dex b/src/android/dex/classes.dex index 4f26184..69d7252 100644 Binary files a/src/android/dex/classes.dex and b/src/android/dex/classes.dex differ diff --git a/src/android/external_display.rs b/src/android/external_display.rs new file mode 100644 index 0000000..728641a --- /dev/null +++ b/src/android/external_display.rs @@ -0,0 +1,267 @@ +use std::{ + collections::VecDeque, + ffi::c_void, + ptr::NonNull, + sync::{Arc, Condvar, Mutex, OnceLock}, + time::Duration, +}; + +use jni::{ + objects::{JClass, JObject}, + sys::{jboolean, jfloat, jint, jlong, jobject, JNI_TRUE}, + JNIEnv, +}; +use winit::{ + event::ElementState, + event_loop::EventLoopProxy, + platform::android::{self, activity::AndroidApp}, +}; + +use crate::android::{ + accessibility::{self, AppUserEvent}, + backend::wayland::physicalkey_to_scancode, +}; + +#[repr(C)] +#[derive(Debug)] +struct ANativeWindow { + _private: [u8; 0], +} + +extern "C" { + fn ANativeWindow_fromSurface( + env: *mut jni::sys::JNIEnv, + surface: jobject, + ) -> *mut ANativeWindow; + fn ANativeWindow_release(window: *mut ANativeWindow); +} + +/// The surface of the `Presentation` shown on an external monitor. +#[derive(Debug)] +pub struct ExternalSurface { + window: NonNull, + pub width: i32, + pub height: i32, + pub density_dpi: i32, + pub refresh_millihz: i32, +} + +unsafe impl Send for ExternalSurface {} +unsafe impl Sync for ExternalSurface {} + +impl ExternalSurface { + pub fn native_window(&self) -> NonNull { + self.window.cast() + } +} + +impl Drop for ExternalSurface { + fn drop(&mut self) { + unsafe { ANativeWindow_release(self.window.as_ptr()) }; + } +} + +/// A mouse (or touch) on the external monitor, in its pixels. +#[derive(Debug, Clone, Copy)] +pub struct PointerEvent { + pub x: f64, + pub y: f64, + /// `MotionEvent.getButtonState()`. + pub buttons: i32, + /// `AXIS_HSCROLL` and `AXIS_VSCROLL`, in wheel notches. + pub scroll_x: f64, + pub scroll_y: f64, + pub time_ms: u64, +} + +#[derive(Default)] +struct Bridge { + proxy: Option>, + surface: Option>, + /// The event loop holds an EGL surface on `surface`. + in_use: bool, + pointer_events: VecDeque, +} + +static BRIDGE: OnceLock<(Mutex, Condvar)> = OnceLock::new(); + +fn bridge() -> &'static (Mutex, Condvar) { + BRIDGE.get_or_init(Default::default) +} + +pub fn register_event_loop_proxy(proxy: EventLoopProxy) { + bridge().0.lock().unwrap().proxy = Some(proxy); +} + +/// The external monitor's surface, if one is connected. +pub fn surface() -> Option> { + bridge().0.lock().unwrap().surface.clone() +} + +/// Called by the event loop when it starts or stops rendering to `surface()`. +pub fn set_in_use(in_use: bool) { + let (lock, released) = bridge(); + lock.lock().unwrap().in_use = in_use; + if !in_use { + released.notify_all(); + } +} + +pub fn drain_pointer_events() -> Vec { + bridge() + .0 + .lock() + .unwrap() + .pointer_events + .drain(..) + .collect() +} + +fn notify(bridge: &Bridge, event: AppUserEvent) { + if let Some(proxy) = &bridge.proxy { + if let Err(error) = proxy.send_event(event) { + log::warn!("Failed to wake event loop for the external display: {error}"); + } + } +} + +fn call_java(env: &mut JNIEnv, android_app: &AndroidApp, method: &str) { + let activity = unsafe { JObject::from_raw(android_app.activity_as_ptr() as jobject) }; + let result = (|| -> jni::errors::Result<()> { + let loader = env + .call_method( + &activity, + "getClassLoader", + "()Ljava/lang/ClassLoader;", + &[], + )? + .l()?; + let name = env.new_string("app.polarbear.ExternalDisplay")?; + let class = env + .call_method( + &loader, + "loadClass", + "(Ljava/lang/String;)Ljava/lang/Class;", + &[(&name).into()], + )? + .l()?; + env.call_static_method( + JClass::from(class), + method, + "(Landroid/app/Activity;)V", + &[(&activity).into()], + )?; + Ok(()) + })(); + if let Err(error) = result { + log::warn!("Failed to call ExternalDisplay.{method}: {error}"); + let _ = env.exception_clear(); + } +} + +/// Start showing the desktop on an external monitor whenever one is connected. +pub fn start(env: &mut JNIEnv, android_app: &AndroidApp) { + call_java(env, android_app, "start"); +} + +/// Keep the desktop on the monitor while the app is in the background. +pub fn background(env: &mut JNIEnv, android_app: &AndroidApp) { + call_java(env, android_app, "background"); +} + +pub fn stop(env: &mut JNIEnv, android_app: &AndroidApp) { + call_java(env, android_app, "stop"); +} + +#[no_mangle] +pub extern "system" fn Java_app_polarbear_ExternalDisplay_nativeSurfaceChanged( + env: JNIEnv, + _class: JClass, + surface: JObject, + width: jint, + height: jint, + density_dpi: jint, + refresh_rate: jfloat, +) { + let window = unsafe { ANativeWindow_fromSurface(env.get_raw(), surface.as_raw()) }; + let Some(window) = NonNull::new(window) else { + log::warn!("The external display's surface has no native window"); + return; + }; + log::info!("External display surface {width}x{height} at {density_dpi} dpi"); + + let mut bridge = bridge().0.lock().unwrap(); + bridge.surface = Some(Arc::new(ExternalSurface { + window, + width, + height, + density_dpi, + refresh_millihz: (refresh_rate.max(1.0) * 1000.0).round() as i32, + })); + notify(&bridge, AppUserEvent::ExternalDisplayChanged); +} + +#[no_mangle] +pub extern "system" fn Java_app_polarbear_ExternalDisplay_nativeSurfaceDestroyed( + _env: JNIEnv, + _class: JClass, +) { + log::info!("External display surface destroyed"); + let (lock, released) = bridge(); + let mut bridge = lock.lock().unwrap(); + bridge.surface = None; + notify(&bridge, AppUserEvent::ExternalDisplayChanged); + // The surface must not be drawn to once this returns. + let (_bridge, timeout) = released + .wait_timeout_while(bridge, Duration::from_secs(1), |bridge| bridge.in_use) + .unwrap(); + if timeout.timed_out() { + log::warn!("Timed out waiting for the renderer to release the external display"); + } +} + +#[no_mangle] +pub extern "system" fn Java_app_polarbear_ExternalDisplay_nativeOnPointer( + _env: JNIEnv, + _class: JClass, + x: jfloat, + y: jfloat, + buttons: jint, + scroll_x: jfloat, + scroll_y: jfloat, + event_time_ms: jlong, +) { + let mut bridge = bridge().0.lock().unwrap(); + let wake = bridge.pointer_events.is_empty(); + bridge.pointer_events.push_back(PointerEvent { + x: x as f64, + y: y as f64, + buttons, + scroll_x: scroll_x as f64, + scroll_y: scroll_y as f64, + time_ms: event_time_ms.max(0) as u64, + }); + if wake { + notify(&bridge, AppUserEvent::ExternalPointer); + } +} + +/// A key from the Android keyboard shown over the trackpad. +#[no_mangle] +pub extern "system" fn Java_app_polarbear_TrackpadControls_nativeOnKey( + _env: JNIEnv, + _class: JClass, + down: jboolean, + key_code: jint, + event_time_ms: jlong, +) { + let Some(scancode) = physicalkey_to_scancode(android::physical_key(key_code as u32)) else { + return; + }; + let state = if down == JNI_TRUE { + ElementState::Pressed + } else { + ElementState::Released + }; + accessibility::inject_key(scancode, state, event_time_ms.max(0) as u64); +} diff --git a/src/android/java/app/polarbear/ExternalDisplay.java b/src/android/java/app/polarbear/ExternalDisplay.java new file mode 100644 index 0000000..02733a6 --- /dev/null +++ b/src/android/java/app/polarbear/ExternalDisplay.java @@ -0,0 +1,214 @@ +package app.polarbear; + +import android.app.Activity; +import android.app.Presentation; +import android.content.Context; +import android.content.pm.ActivityInfo; +import android.hardware.display.DisplayManager; +import android.os.Bundle; +import android.os.Handler; +import android.os.Looper; +import android.util.DisplayMetrics; +import android.view.Display; +import android.view.InputDevice; +import android.view.KeyEvent; +import android.view.MotionEvent; +import android.view.PointerIcon; +import android.view.Surface; +import android.view.SurfaceHolder; +import android.view.SurfaceView; +import android.view.View; +import android.view.WindowManager; +import android.widget.Toast; + +/** Shows the desktop on an external monitor while the phone becomes a trackpad. */ +public final class ExternalDisplay { + static { + System.loadLibrary("localdesktop"); + } + + private static Activity activity; + private static DisplayManager displayManager; + private static DesktopPresentation presentation; + private static TrackpadControls controls; + + private static final DisplayManager.DisplayListener LISTENER = new DisplayManager.DisplayListener() { + @Override + public void onDisplayAdded(int displayId) { + update(); + } + + @Override + public void onDisplayRemoved(int displayId) { + update(); + } + + @Override + public void onDisplayChanged(int displayId) {} + }; + + private ExternalDisplay() {} + + public static void start(final Activity activity) { + activity.runOnUiThread(new Runnable() { + @Override + public void run() { + if (ExternalDisplay.activity == null) { + ExternalDisplay.activity = activity; + displayManager = (DisplayManager) activity.getSystemService(Context.DISPLAY_SERVICE); + displayManager.registerDisplayListener(LISTENER, new Handler(Looper.getMainLooper())); + } + update(); + if (presentation != null && controls == null) { + controls = new TrackpadControls(activity); + controls.show(); + } + } + }); + } + + /** The app went to the background: keep the desktop on the monitor, put the phone's controls away. */ + public static void background(Activity activity) { + activity.runOnUiThread(new Runnable() { + @Override + public void run() { + if (controls != null) { + controls.dismiss(); + controls = null; + } + } + }); + } + + public static void stop(Activity activity) { + activity.runOnUiThread(new Runnable() { + @Override + public void run() { + if (displayManager != null) { + displayManager.unregisterDisplayListener(LISTENER); + } + dismiss(); + ExternalDisplay.activity = null; + } + }); + } + + private static void update() { + if (activity == null) { + return; + } + Display[] displays = displayManager.getDisplays(DisplayManager.DISPLAY_CATEGORY_PRESENTATION); + Display display = displays.length > 0 ? displays[0] : null; + if (presentation != null && presentation.getDisplay().getDisplayId() == (display == null ? -1 : display.getDisplayId())) { + return; + } + dismiss(); + if (display == null) { + return; + } + presentation = new DesktopPresentation(activity, display); + try { + presentation.show(); + } catch (WindowManager.InvalidDisplayException e) { + presentation = null; + return; + } + controls = new TrackpadControls(activity); + controls.show(); + Toast.makeText(activity, "The desktop is on the external display. Use this screen as a trackpad.", + Toast.LENGTH_LONG).show(); + } + + private static void dismiss() { + if (controls != null) { + controls.dismiss(); + controls = null; + } + if (activity != null) { + // The rotate button may have turned the screen; follow the phone again. + activity.setRequestedOrientation(ActivityInfo.SCREEN_ORIENTATION_UNSPECIFIED); + } + if (presentation != null) { + presentation.dismiss(); + presentation = null; + } + } + + private static final class DesktopPresentation extends Presentation implements SurfaceHolder.Callback { + DesktopPresentation(Context context, Display display) { + super(context, display); + } + + @Override + protected void onCreate(Bundle savedInstanceState) { + super.onCreate(savedInstanceState); + setCancelable(false); + getWindow().addFlags(WindowManager.LayoutParams.FLAG_KEEP_SCREEN_ON); + SurfaceView view = new SurfaceView(getContext()); + view.getHolder().addCallback(this); + // The desktop draws its own cursor where a mouse on this display points. + view.setPointerIcon(PointerIcon.getSystemIcon(getContext(), PointerIcon.TYPE_NULL)); + view.setOnGenericMotionListener(new View.OnGenericMotionListener() { + @Override + public boolean onGenericMotion(View view, MotionEvent event) { + return onPointer(event); + } + }); + view.setOnTouchListener(new View.OnTouchListener() { + @Override + public boolean onTouch(View view, MotionEvent event) { + return onPointer(event); + } + }); + setContentView(view); + } + + @Override + public boolean dispatchKeyEvent(KeyEvent event) { + // A keyboard typing while the desktop on this display has focus. + TrackpadControls.forwardKey(event); + return true; + } + + private boolean onPointer(MotionEvent event) { + int buttons = event.getButtonState(); + // A touch screen or pen on the monitor clicks like a mouse. + if (!event.isFromSource(InputDevice.SOURCE_MOUSE)) { + int action = event.getActionMasked(); + buttons = action == MotionEvent.ACTION_UP || action == MotionEvent.ACTION_CANCEL + ? 0 : MotionEvent.BUTTON_PRIMARY; + } + float scrollX = 0; + float scrollY = 0; + if (event.getActionMasked() == MotionEvent.ACTION_SCROLL) { + scrollX = event.getAxisValue(MotionEvent.AXIS_HSCROLL); + scrollY = event.getAxisValue(MotionEvent.AXIS_VSCROLL); + } + nativeOnPointer(event.getX(), event.getY(), buttons, scrollX, scrollY, event.getEventTime()); + return true; + } + + @Override + public void surfaceCreated(SurfaceHolder holder) {} + + @Override + public void surfaceChanged(SurfaceHolder holder, int format, int width, int height) { + DisplayMetrics metrics = getContext().getResources().getDisplayMetrics(); + nativeSurfaceChanged(holder.getSurface(), width, height, metrics.densityDpi, + getDisplay().getRefreshRate()); + } + + @Override + public void surfaceDestroyed(SurfaceHolder holder) { + nativeSurfaceDestroyed(); + } + } + + private static native void nativeSurfaceChanged( + Surface surface, int width, int height, int densityDpi, float refreshRate); + + private static native void nativeSurfaceDestroyed(); + + private static native void nativeOnPointer( + float x, float y, int buttons, float scrollX, float scrollY, long eventTimeMs); +} diff --git a/src/android/java/app/polarbear/TrackpadControls.java b/src/android/java/app/polarbear/TrackpadControls.java new file mode 100644 index 0000000..de11b5b --- /dev/null +++ b/src/android/java/app/polarbear/TrackpadControls.java @@ -0,0 +1,382 @@ +package app.polarbear; + +import android.app.Activity; +import android.app.Dialog; +import android.content.Context; +import android.content.pm.ActivityInfo; +import android.graphics.Canvas; +import android.graphics.Color; +import android.graphics.Paint; +import android.graphics.PorterDuff; +import android.graphics.RectF; +import android.graphics.drawable.AdaptiveIconDrawable; +import android.graphics.drawable.ColorDrawable; +import android.graphics.drawable.Drawable; +import android.os.Build; +import android.os.Bundle; +import android.text.InputType; +import android.view.Gravity; +import android.view.KeyEvent; +import android.view.Surface; +import android.view.View; +import android.view.Window; +import android.view.WindowInsets; +import android.view.WindowManager; +import android.view.inputmethod.BaseInputConnection; +import android.view.inputmethod.EditorInfo; +import android.view.inputmethod.InputConnection; +import android.view.inputmethod.InputMethodManager; +import android.widget.LinearLayout; + +/** Buttons on the phone while it is a trackpad: turn the screen, and type with the Android keyboard. */ +final class TrackpadControls extends Dialog { + private final Activity activity; + private final Pad pad; + private KeyboardButton keyboard; + private boolean keyboardShown; + /** Waiting for the window to take focus before the keyboard can come up. */ + private boolean keyboardPending; + /** The keyboard has been on screen since it was asked for. */ + private boolean keyboardSeen; + + TrackpadControls(Activity activity) { + super(activity); + this.activity = activity; + pad = new Pad(activity); + } + + @Override + public void show() { + // Under the buttons. + pad.show(); + super.show(); + } + + @Override + protected void onCreate(Bundle savedInstanceState) { + requestWindowFeature(Window.FEATURE_NO_TITLE); + super.onCreate(savedInstanceState); + setCancelable(false); + + Window window = getWindow(); + window.setBackgroundDrawable(new ColorDrawable(Color.TRANSPARENT)); + window.clearFlags(WindowManager.LayoutParams.FLAG_DIM_BEHIND); + // Touches elsewhere reach the trackpad, and keys the activity, until the keyboard is up. + window.addFlags(WindowManager.LayoutParams.FLAG_NOT_TOUCH_MODAL + | WindowManager.LayoutParams.FLAG_NOT_FOCUSABLE); + window.setGravity(Gravity.TOP | Gravity.RIGHT); + WindowManager.LayoutParams attributes = window.getAttributes(); + attributes.x = dp(Pad.INSET + 8); + attributes.y = dp(Pad.INSET + 8); + window.setAttributes(attributes); + + LinearLayout bar = new LinearLayout(getContext()); + bar.setOrientation(LinearLayout.HORIZONTAL); + IconButton rotate = new IconButton(getContext(), IconButton.ROTATE); + style(rotate, "Rotate screen"); + rotate.setOnClickListener(new View.OnClickListener() { + @Override + public void onClick(View view) { + rotateScreen(); + } + }); + keyboard = new KeyboardButton(getContext()); + style(keyboard, "Keyboard"); + keyboard.setOnClickListener(new View.OnClickListener() { + @Override + public void onClick(View view) { + setKeyboardShown(!keyboardShown); + } + }); + bar.addView(rotate); + bar.addView(keyboard); + setContentView(bar); + + if (Build.VERSION.SDK_INT >= 30) { + // The keyboard can also go away with the back gesture. + bar.setOnApplyWindowInsetsListener(new View.OnApplyWindowInsetsListener() { + @Override + public WindowInsets onApplyWindowInsets(View view, WindowInsets insets) { + boolean visible = insets.isVisible(WindowInsets.Type.ime()); + if (keyboardShown && visible) { + keyboardSeen = true; + } else if (keyboardShown && keyboardSeen) { + setKeyboardShown(false); + } + return insets; + } + }); + } + } + + @Override + public void onWindowFocusChanged(boolean hasFocus) { + super.onWindowFocusChanged(hasFocus); + if (hasFocus && keyboardPending) { + keyboardPending = false; + // After the input method has connected to the newly focused button. + keyboard.post(new Runnable() { + @Override + public void run() { + if (Build.VERSION.SDK_INT >= 30) { + getWindow().getInsetsController().show(WindowInsets.Type.ime()); + } + InputMethodManager imm = (InputMethodManager) + getContext().getSystemService(Context.INPUT_METHOD_SERVICE); + imm.showSoftInput(keyboard, 0); + } + }); + } + } + + @Override + public void dismiss() { + setKeyboardShown(false); + pad.dismiss(); + super.dismiss(); + } + + /** + * Turns the whole screen a quarter clockwise, Android keyboard included, for a phone lying + * sideways while auto-rotate keeps it upright. + */ + private void rotateScreen() { + int rotation = activity.getWindowManager().getDefaultDisplay().getRotation(); + boolean portrait = getContext().getResources().getConfiguration().orientation + == android.content.res.Configuration.ORIENTATION_PORTRAIT; + boolean naturallyPortrait = portrait == (rotation == Surface.ROTATION_0 || rotation == Surface.ROTATION_180); + // Orientations for ROTATION_0, 90, 180 and 270 of the display. + int[] orientations = naturallyPortrait + ? new int[] { + ActivityInfo.SCREEN_ORIENTATION_PORTRAIT, + ActivityInfo.SCREEN_ORIENTATION_LANDSCAPE, + ActivityInfo.SCREEN_ORIENTATION_REVERSE_PORTRAIT, + ActivityInfo.SCREEN_ORIENTATION_REVERSE_LANDSCAPE} + : new int[] { + ActivityInfo.SCREEN_ORIENTATION_LANDSCAPE, + ActivityInfo.SCREEN_ORIENTATION_REVERSE_PORTRAIT, + ActivityInfo.SCREEN_ORIENTATION_REVERSE_LANDSCAPE, + ActivityInfo.SCREEN_ORIENTATION_PORTRAIT}; + // The display turns the other way from the content: a quarter clockwise is ROTATION_270. + activity.setRequestedOrientation(orientations[(rotation + 3) % 4]); + } + + private void setKeyboardShown(boolean shown) { + if (keyboard == null || shown == keyboardShown) { + return; + } + keyboardShown = shown; + Window window = getWindow(); + InputMethodManager imm = + (InputMethodManager) getContext().getSystemService(Context.INPUT_METHOD_SERVICE); + if (shown) { + keyboardSeen = false; + keyboardPending = true; + window.clearFlags(WindowManager.LayoutParams.FLAG_NOT_FOCUSABLE); + // Only now: a view focusable in touch mode takes the first tap as focus, not a click. + keyboard.setFocusableInTouchMode(true); + keyboard.requestFocus(); + } else { + keyboardPending = false; + imm.hideSoftInputFromWindow(keyboard.getWindowToken(), 0); + keyboard.setFocusableInTouchMode(false); + window.addFlags(WindowManager.LayoutParams.FLAG_NOT_FOCUSABLE); + } + keyboard.setSelected(shown); + keyboard.invalidate(); + } + + private void style(View view, String label) { + view.setContentDescription(label); + LinearLayout.LayoutParams params = new LinearLayout.LayoutParams(dp(44), dp(44)); + params.setMargins(dp(4), 0, dp(4), 0); + view.setLayoutParams(params); + } + + private int dp(int value) { + return Math.round(value * getContext().getResources().getDisplayMetrics().density); + } + + /** + * The trackpad itself, drawn like Samsung DeX: a border, a cross through the middle, and the + * app's logo on top so the right way up is clear. Touches go through it to the activity. + */ + private static final class Pad extends Dialog { + static final int INSET = 12; + + Pad(Activity activity) { + // Not a floating dialog, which would keep margins around it. + super(activity, android.R.style.Theme_Translucent_NoTitleBar_Fullscreen); + } + + @Override + protected void onCreate(Bundle savedInstanceState) { + requestWindowFeature(Window.FEATURE_NO_TITLE); + super.onCreate(savedInstanceState); + setCancelable(false); + Window window = getWindow(); + window.setBackgroundDrawable(new ColorDrawable(Color.TRANSPARENT)); + window.clearFlags(WindowManager.LayoutParams.FLAG_DIM_BEHIND); + window.addFlags(WindowManager.LayoutParams.FLAG_NOT_TOUCHABLE + | WindowManager.LayoutParams.FLAG_NOT_FOCUSABLE + | WindowManager.LayoutParams.FLAG_LAYOUT_IN_SCREEN); + window.setLayout(WindowManager.LayoutParams.MATCH_PARENT, WindowManager.LayoutParams.MATCH_PARENT); + if (Build.VERSION.SDK_INT >= 28) { + WindowManager.LayoutParams attributes = window.getAttributes(); + attributes.layoutInDisplayCutoutMode = + WindowManager.LayoutParams.LAYOUT_IN_DISPLAY_CUTOUT_MODE_SHORT_EDGES; + window.setAttributes(attributes); + } + setContentView(new PadView(getContext())); + } + } + + private static final class PadView extends View { + private final Paint paint = new Paint(Paint.ANTI_ALIAS_FLAG); + private final Drawable logo; + /** `logo` is an adaptive icon layer, with the art in its middle two thirds. */ + private final boolean logoIsLayer; + + PadView(Context context) { + super(context); + paint.setStyle(Paint.Style.STROKE); + Drawable icon = context.getApplicationInfo().loadIcon(context.getPackageManager()); + if (Build.VERSION.SDK_INT >= 33 && icon instanceof AdaptiveIconDrawable + && ((AdaptiveIconDrawable) icon).getMonochrome() != null) { + // The silhouette, in the pad's grey. + logo = ((AdaptiveIconDrawable) icon).getMonochrome().mutate(); + logo.setColorFilter(0xFF8A8A8A, PorterDuff.Mode.SRC_IN); + logoIsLayer = true; + } else { + // An icon without one (a full-bleed picture) as the launcher shows it. + logo = icon; + logoIsLayer = false; + } + } + + @Override + protected void onDraw(Canvas canvas) { + float density = getResources().getDisplayMetrics().density; + float inset = Pad.INSET * density; + RectF border = new RectF(inset, inset, getWidth() - inset, getHeight() - inset); + float centerX = getWidth() / 2f; + float centerY = getHeight() / 2f; + + paint.setStrokeWidth(density); + paint.setColor(0xFF2E2E2E); + canvas.drawLine(centerX, border.top, centerX, border.bottom, paint); + canvas.drawLine(border.left, centerY, border.right, centerY, paint); + paint.setStrokeWidth(1.5f * density); + paint.setColor(0xFF4A4A4A); + canvas.drawRoundRect(border, 16 * density, 16 * density, paint); + + // Over the top of the cross, on a patch of the pad's black. + int size = Math.round(48 * density); + int left = Math.round(centerX - size / 2f); + int top = Math.round(border.top + 12 * density); + paint.setStyle(Paint.Style.FILL); + paint.setColor(Color.BLACK); + canvas.drawRect(left, top, left + size, top + size, paint); + paint.setStyle(Paint.Style.STROKE); + int bleed = logoIsLayer ? size / 4 : 0; + logo.setBounds(left - bleed, top - bleed, left + size + bleed, top + size + bleed); + logo.draw(canvas); + } + } + + /** A round button with a line icon drawn on a 24-unit grid, like Material icons. */ + private static class IconButton extends View { + static final int ROTATE = 0; + static final int KEYBOARD = 1; + + private final int icon; + private final Paint paint = new Paint(Paint.ANTI_ALIAS_FLAG); + + IconButton(Context context, int icon) { + super(context); + this.icon = icon; + paint.setStrokeCap(Paint.Cap.ROUND); + paint.setStrokeJoin(Paint.Join.ROUND); + } + + @Override + protected void onDraw(Canvas canvas) { + float size = Math.min(getWidth(), getHeight()); + paint.setStyle(Paint.Style.FILL); + paint.setColor(isSelected() ? 0x77FFFFFF : 0x26FFFFFF); + canvas.drawCircle(getWidth() / 2f, getHeight() / 2f, size / 2f, paint); + + // The icon takes the middle half of the button. + float unit = size / 48f; + canvas.save(); + canvas.translate((getWidth() - size) / 2f + 12 * unit, (getHeight() - size) / 2f + 12 * unit); + canvas.scale(unit, unit); + paint.setColor(Color.WHITE); + paint.setStyle(Paint.Style.STROKE); + paint.setStrokeWidth(2); + if (icon == ROTATE) { + // A phone with arrows turning it clockwise. + canvas.drawRoundRect(new RectF(8.5f, 5, 15.5f, 19), 1.5f, 1.5f, paint); + RectF orbit = new RectF(2, 2, 22, 22); + canvas.drawArc(orbit, -80, 60, false, paint); + canvas.drawLine(21.4f, 8.6f, 22.3f, 5.1f, paint); + canvas.drawLine(21.4f, 8.6f, 18.5f, 6.5f, paint); + canvas.drawArc(orbit, 100, 60, false, paint); + canvas.drawLine(2.6f, 15.4f, 1.7f, 18.9f, paint); + canvas.drawLine(2.6f, 15.4f, 5.5f, 17.5f, paint); + } else { + canvas.drawRoundRect(new RectF(2, 6, 22, 18), 2, 2, paint); + paint.setStyle(Paint.Style.FILL); + for (int x = 6; x <= 18; x += 4) { + canvas.drawCircle(x, 10, 1, paint); + canvas.drawCircle(x, 13, 1, paint); + } + paint.setStyle(Paint.Style.STROKE); + canvas.drawLine(8, 15.5f, 16, 15.5f, paint); + } + canvas.restore(); + } + } + + /** Takes what the Android keyboard types and passes it on as key presses. */ + private static final class KeyboardButton extends IconButton { + KeyboardButton(Context context) { + super(context, KEYBOARD); + } + + @Override + public boolean onCheckIsTextEditor() { + return true; + } + + @Override + public InputConnection onCreateInputConnection(EditorInfo outAttrs) { + // No suggestions, so each character arrives as soon as it is typed. + outAttrs.inputType = InputType.TYPE_CLASS_TEXT + | InputType.TYPE_TEXT_VARIATION_VISIBLE_PASSWORD + | InputType.TYPE_TEXT_FLAG_NO_SUGGESTIONS; + outAttrs.imeOptions = EditorInfo.IME_FLAG_NO_FULLSCREEN + | EditorInfo.IME_FLAG_NO_EXTRACT_UI + | EditorInfo.IME_ACTION_NONE; + return new BaseInputConnection(this, false); + } + + @Override + public boolean dispatchKeyEvent(KeyEvent event) { + if (event.getKeyCode() == KeyEvent.KEYCODE_BACK) { + return super.dispatchKeyEvent(event); + } + forwardKey(event); + return true; + } + } + + /** Passes a key on to the desktop, from the Android keyboard or one typing on the monitor. */ + static void forwardKey(KeyEvent event) { + if (event.getAction() == KeyEvent.ACTION_DOWN || event.getAction() == KeyEvent.ACTION_UP) { + nativeOnKey(event.getAction() == KeyEvent.ACTION_DOWN, event.getKeyCode(), event.getEventTime()); + } + } + + private static native void nativeOnKey(boolean down, int keyCode, long eventTimeMs); +} diff --git a/src/android/main.rs b/src/android/main.rs index 5ccac9f..06ae941 100644 --- a/src/android/main.rs +++ b/src/android/main.rs @@ -2,6 +2,7 @@ use crate::{ android::{ accessibility::{register_event_loop_proxy, AppUserEvent}, app::build::PolarBearApp, + external_display, utils::{ application_context::{get_application_context, ApplicationContext}, fullscreen_immersive::{enable_fullscreen_immersive_mode, keep_screen_on}, @@ -74,6 +75,7 @@ fn android_main(android_app: AndroidApp) { .build() .expect("Failed to create event loop"); register_event_loop_proxy(event_loop.create_proxy()); + external_display::register_event_loop_proxy(event_loop.create_proxy()); // ControlFlow::Poll continuously runs the event loop, even if the OS hasn't // dispatched any events. This is ideal for games and similar applications. diff --git a/src/android/proguard-rules.pro b/src/android/proguard-rules.pro index 62dbcf3..62d90cd 100644 --- a/src/android/proguard-rules.pro +++ b/src/android/proguard-rules.pro @@ -1,3 +1,14 @@ -keep class app.polarbear.KeyboardShortcutsPrompt { public static void showIfNeeded(android.app.Activity); } + +-keep class app.polarbear.ExternalDisplay { + public static void start(android.app.Activity); + public static void background(android.app.Activity); + public static void stop(android.app.Activity); + native ; +} + +-keep class app.polarbear.TrackpadControls { + native ; +} diff --git a/src/android/proot/process.rs b/src/android/proot/process.rs index b553027..d0873e2 100644 --- a/src/android/proot/process.rs +++ b/src/android/proot/process.rs @@ -1,4 +1,5 @@ use crate::android::utils::application_context::get_application_context; +use crate::core::cpuinfo::{kernel_enables_sve, GUEST_CPUINFO}; use std::ffi::CString; use std::fs; use std::io::{BufRead, BufReader, Read}; @@ -159,6 +160,10 @@ impl GuestProcess { .arg(format!("--bind={rootfs}/proc/.sysctl_inotify_max_user_watches:/proc/sys/fs/inotify/max_user_watches")) .arg(format!("--bind={rootfs}/sys/.empty:/sys/fs/selinux")); + if context.rootfs_dir.join(GUEST_CPUINFO).is_file() { + process.arg(format!("--bind={rootfs}/{GUEST_CPUINFO}:/proc/cpuinfo")); + } + // env vars process.arg("/usr/bin/env").arg("-i"); process.arg(format!("HOME={home}")); @@ -172,6 +177,10 @@ impl GuestProcess { .arg("TMPDIR=/tmp") .arg(format!("USER={}", user)) .arg(format!("LOGNAME={}", user)); + // See `core::cpuinfo`: llvmpipe's optimised shaders still crash Gala on these devices. + if !kernel_enables_sve(&fs::read_to_string("/proc/cpuinfo").unwrap_or_default()) { + process.arg("GALLIVM_PERF=nopt"); + } // user shell if user == "root" { diff --git a/src/android/proot/setup.rs b/src/android/proot/setup.rs index 4f7f3c0..151d966 100644 --- a/src/android/proot/setup.rs +++ b/src/android/proot/setup.rs @@ -10,7 +10,10 @@ use crate::{ utils::application_context::get_application_context, utils::ndk::{long_press_timeout_ms, run_in_jvm, scale_factor, touch_slop_px}, }, - core::rootfs::{self, RootfsManifest, ROOTFS_ARCHIVE, ROOTFS_MANIFEST}, + core::{ + cpuinfo::{guest_cpuinfo, GUEST_CPUINFO}, + rootfs::{self, RootfsManifest, ROOTFS_ARCHIVE, ROOTFS_MANIFEST}, + }, }; use jni::objects::{JObject, JString}; use pathdiff::diff_paths; @@ -303,6 +306,17 @@ fn setup_network_and_time(options: &SetupOptions) -> StageOutput { None } +/// Give the guest a `/proc/cpuinfo` that keeps llvmpipe from using CPU features the kernel turned +/// off (see `core::cpuinfo`). Every launch, so it always describes this device. +fn setup_cpuinfo(_: &SetupOptions) -> StageOutput { + let path = get_application_context().rootfs_dir.join(GUEST_CPUINFO); + match fs::read_to_string("/proc/cpuinfo") { + Ok(host) => replace_file(&path, &guest_cpuinfo(&host)), + Err(error) => log::warn!("Failed to read /proc/cpuinfo: {error}"), + } + None +} + /// Write `contents` to `path`, replacing a symlink or read-only file left there by the image. fn replace_file(path: &Path, contents: &str) { let _ = fs::remove_file(path); @@ -434,6 +448,9 @@ pub fn build_wayland_backend(android_app: AndroidApp) -> WaylandBackend { touch_slop_px: touch_slop_px(&android_app), long_press_timeout_ms: long_press_timeout_ms(&android_app), pointer_pressed: false, + touch_fingers: 0, + cursor_position: (0.0, 0.0).into(), + external_buttons: 0, android_app, } } @@ -484,7 +501,8 @@ pub fn setup(android_app: AndroidApp) -> PolarBearBackend { Box::new(setup_home), // Step 3. Persistent /home and /etc/localdesktop Box::new(setup_machine_id), // Step 4. Seed /etc/machine-id for D-Bus clients Box::new(setup_network_and_time), // Step 5. DNS servers and timezone from Android - Box::new(fix_xkb_symlink), // Step 6. Fix xkb symlink + Box::new(setup_cpuinfo), // Step 6. /proc/cpuinfo that llvmpipe can trust + Box::new(fix_xkb_symlink), // Step 7. Fix xkb symlink ]; let handle_stage_error = |e: Box, sender: &Sender| { diff --git a/src/core/config.rs b/src/core/config.rs index 25ca294..26e1b70 100644 --- a/src/core/config.rs +++ b/src/core/config.rs @@ -33,13 +33,31 @@ pub const PIPEWIRE_GUEST_RUNTIME_DIR: &str = "/tmp"; /// PipeWire-Pulse socket as seen from inside the proot guest. pub const PULSE_GUEST_SERVER: &str = "unix:/tmp/pulse/native"; -/// Whole-number UI scale for the desktop from Android's density scale (dpi / 160). +/// The smallest desktop, in logical pixels, that the panel, the dock and elementary's windows fit. +const MIN_DESKTOP_SHORT_SIDE: i32 = 540; +const MIN_DESKTOP_LONG_SIDE: i32 = 900; +/// How much denser than Android's own UI the desktop may get to reach that size. +const MAX_DENSITY_RATIO: f64 = 2.0; + +/// Whole-number UI scale for the desktop from Android's density scale (dpi / 160) and the window +/// size in physical pixels. /// /// Rounds down unless the density is close to the next step: a 420 dpi phone (2.625) gets 2x, a /// 480 dpi one 3x. Rounding to nearest gives 3x at 420 dpi, which leaves a desktop barely 360 -/// logical pixels wide in portrait. -pub fn guest_ui_scale(scale_factor: f64) -> i32 { - ((scale_factor + 0.25).floor() as i32).max(1) +/// logical pixels wide in portrait. Then it steps down while the desktop would be smaller than +/// `MIN_DESKTOP_SHORT_SIDE` x `MIN_DESKTOP_LONG_SIDE`, as long as text stays at least half the +/// size of Android's: dense screens that are small in pixels, like a foldable's cover screen (1116 +/// pixels wide at 520 dpi, so 372 at 3x), get 2x. +pub fn guest_ui_scale(scale_factor: f64, width: i32, height: i32) -> i32 { + let (short_side, long_side) = (width.min(height), width.max(height)); + let mut scale = ((scale_factor + 0.25).floor() as i32).max(1); + while scale > 1 + && (short_side / scale < MIN_DESKTOP_SHORT_SIDE || long_side / scale < MIN_DESKTOP_LONG_SIDE) + && scale_factor / f64::from(scale - 1) <= MAX_DENSITY_RATIO + { + scale -= 1; + } + scale } @@ -262,12 +280,35 @@ mod tests { #[test] fn picks_whole_number_scales() { - assert_eq!(guest_ui_scale(1.0), 1); // mdpi - assert_eq!(guest_ui_scale(1.5), 1); // hdpi - assert_eq!(guest_ui_scale(2.0), 2); // xhdpi - assert_eq!(guest_ui_scale(2.625), 2); // 420 dpi - assert_eq!(guest_ui_scale(2.75), 3); // 440 dpi - assert_eq!(guest_ui_scale(3.5), 3); // 560 dpi - assert_eq!(guest_ui_scale(0.75), 1); + let (w, h) = (3840, 2400); + assert_eq!(guest_ui_scale(1.0, w, h), 1); // mdpi + assert_eq!(guest_ui_scale(1.5, w, h), 1); // hdpi + assert_eq!(guest_ui_scale(2.0, w, h), 2); // xhdpi + assert_eq!(guest_ui_scale(2.625, w, h), 2); // 420 dpi + assert_eq!(guest_ui_scale(2.75, w, h), 3); // 440 dpi + assert_eq!(guest_ui_scale(3.5, w, h), 3); // 560 dpi + assert_eq!(guest_ui_scale(0.75, w, h), 1); + } + + #[test] + fn keeps_the_desktop_big_enough_to_use() { + // OnePlus Open at 520 dpi: cover screen and inner screen. + assert_eq!(guest_ui_scale(3.25, 1116, 2484), 2); + assert_eq!(guest_ui_scale(3.25, 2268, 2440), 2); + // Phones keep 2x in either orientation. + assert_eq!(guest_ui_scale(2.625, 1080, 1920), 2); + assert_eq!(guest_ui_scale(2.625, 2400, 1080), 2); + assert_eq!(guest_ui_scale(3.5, 1440, 3120), 2); + // A tablet keeps its density's scale. + assert_eq!(guest_ui_scale(2.0, 1600, 2560), 2); + // Never below 1x. + assert_eq!(guest_ui_scale(2.0, 600, 800), 1); + } + + #[test] + fn keeps_text_readable_on_small_screens() { + // Galaxy Z Fold5 cover screen at 400 dpi: 1x would fit more, with text 2.5 times smaller + // than Android's. + assert_eq!(guest_ui_scale(2.5, 904, 2316), 2); } } diff --git a/src/core/cpuinfo.rs b/src/core/cpuinfo.rs new file mode 100644 index 0000000..1de76a0 --- /dev/null +++ b/src/core/cpuinfo.rs @@ -0,0 +1,99 @@ +//! The guest's view of `/proc/cpuinfo`. +//! +//! The desktop is drawn by Mesa's llvmpipe, which compiles shaders with LLVM for "the host CPU". +//! On arm64, LLVM picks that CPU from the `CPU part` lines alone and assumes everything the core +//! can do. Many Armv9 phones (Snapdragon 8 Gen 1 to 3, for example) have cores whose SVE is turned +//! off by the firmware, so shaders compiled for them hit an illegal instruction: Gala, the window +//! manager, dies with SIGILL seconds after the desktop starts, and the screen stays black. +//! +//! Without the `CPU part` lines LLVM targets a generic Armv8 CPU, which every arm64 device runs. +//! The `Features` lines, which say what the kernel really enables, are kept. +//! +//! That is not always enough: on a OnePlus Open driving a 4K monitor, Gala still died with SIGILL +//! until llvmpipe stopped running LLVM's optimisation passes (`GALLIVM_PERF=nopt`), so the guest +//! gets that too on these devices. It costs little for a 2D desktop. + +/// Where the app writes the guest's `/proc/cpuinfo`, relative to the root filesystem. +pub const GUEST_CPUINFO: &str = "proc/.cpuinfo"; + +/// `/proc/cpuinfo` as the guest should see it, given the host's. +pub fn guest_cpuinfo(host: &str) -> String { + if kernel_enables_sve(host) { + return host.to_string(); + } + host.lines() + .filter(|line| !line.starts_with("CPU part")) + .map(|line| format!("{line}\n")) + .collect() +} + +/// Whether every CPU lists SVE among the features the kernel enables. +pub fn kernel_enables_sve(cpuinfo: &str) -> bool { + let mut features = cpuinfo + .lines() + .filter_map(|line| line.strip_prefix("Features")) + .peekable(); + features.peek().is_some() + && features.all(|line| { + line.trim_start_matches([' ', '\t', ':']) + .split_whitespace() + .any(|feature| feature == "sve") + }) +} + +#[cfg(test)] +mod tests { + use super::*; + + /// Two of the eight CPUs of a Snapdragon 8 Gen 2 (Cortex-A510 and Cortex-X3), SVE disabled. + const SNAPDRAGON_8_GEN_2: &str = "\ +processor\t: 0 +BogoMIPS\t: 38.40 +Features\t: fp asimd evtstrm aes pmull sha1 sha2 crc32 atomics fphp asimdhp cpuid asimdrdm jscvt fcma lrcpc dcpop sha3 sm3 sm4 asimddp sha512 asimdfhm dit uscat ilrcpc flagm sb paca pacg dcpodp flagm2 frint i8mm bf16 bti +CPU implementer\t: 0x41 +CPU architecture: 8 +CPU variant\t: 0x1 +CPU part\t: 0xd46 +CPU revision\t: 1 + +processor\t: 7 +BogoMIPS\t: 38.40 +Features\t: fp asimd evtstrm aes pmull sha1 sha2 crc32 atomics fphp asimdhp cpuid asimdrdm jscvt fcma lrcpc dcpop sha3 sm3 sm4 asimddp sha512 asimdfhm dit uscat ilrcpc flagm sb paca pacg dcpodp flagm2 frint i8mm bf16 bti +CPU implementer\t: 0x41 +CPU architecture: 8 +CPU variant\t: 0x1 +CPU part\t: 0xd4e +CPU revision\t: 0 +"; + + #[test] + fn hides_the_cpu_model_when_the_kernel_disables_sve() { + let guest = guest_cpuinfo(SNAPDRAGON_8_GEN_2); + + assert!(!guest.contains("CPU part")); + assert_eq!(guest.matches("Features\t: fp asimd").count(), 2); + assert_eq!(guest.matches("CPU implementer\t: 0x41").count(), 2); + assert!(guest.ends_with("CPU revision\t: 0\n")); + } + + #[test] + fn keeps_the_cpu_model_when_the_kernel_enables_sve() { + let host = SNAPDRAGON_8_GEN_2.replace(" bti", " bti sve sve2"); + + assert_eq!(guest_cpuinfo(&host), host); + } + + #[test] + fn hides_the_cpu_model_unless_every_cpu_has_sve() { + let host = SNAPDRAGON_8_GEN_2.replacen(" bti", " bti sve", 1); + + assert!(!guest_cpuinfo(&host).contains("CPU part")); + } + + #[test] + fn hides_the_cpu_model_when_features_are_unknown() { + let guest = guest_cpuinfo("processor\t: 0\nCPU part\t: 0xd4e\n"); + + assert_eq!(guest, "processor\t: 0\n"); + } +} diff --git a/src/lib.rs b/src/lib.rs index c5b5c8c..0d2bdc4 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -1,11 +1,13 @@ pub mod core { pub mod config; + pub mod cpuinfo; pub mod rootfs; } #[cfg(target_os = "android")] pub mod android { pub mod accessibility; + pub mod external_display; pub mod main; pub mod app {