diff --git a/README.md b/README.md index c64576b..65ae739 100644 --- a/README.md +++ b/README.md @@ -12,6 +12,9 @@ Wayland compositor in the Android NDK plus proot) and replaces what runs on it. ideally with a keyboard and mouse. It brings a desktop to Android; it does not turn the desktop into a phone UI. +Tap a text field and your phone's own keyboard comes up, with the desktop fitting above it. For +apps where it doesn't, open **On-screen keyboard** from the Applications menu. + On a phone, connect a monitor (USB-C to HDMI, a dock, or a wireless display): the desktop moves to it and the phone becomes a trackpad, with buttons to turn the screen and to type with the Android keyboard. diff --git a/docs/img/android-keyboard.jpg b/docs/img/android-keyboard.jpg new file mode 100644 index 0000000..da7427f Binary files /dev/null and b/docs/img/android-keyboard.jpg differ diff --git a/rootfs/packages.list b/rootfs/packages.list index 0b964a4..c764082 100644 --- a/rootfs/packages.list +++ b/rootfs/packages.list @@ -73,6 +73,10 @@ file-roller # On-screen keyboard (toggled from Quick Settings for touch-only use) onboard +# Reports focused text fields, so the app can open the Android keyboard +at-spi2-core +gir1.2-atspi-2.0 +python3-pyatspi # Command-line basics ca-certificates diff --git a/src/android/accessibility.rs b/src/android/accessibility.rs index 38c5365..bb57dfb 100644 --- a/src/android/accessibility.rs +++ b/src/android/accessibility.rs @@ -21,6 +21,10 @@ pub enum AppUserEvent { ExternalDisplayChanged, /// A mouse (or touch) on the external monitor moved, clicked or scrolled. ExternalPointer, + /// A text field on the desktop took focus. + KeyboardRequested, + /// The Android keyboard on the phone's screen came up, changed height or went away. + KeyboardInset, } #[derive(Clone, Copy, Debug)] diff --git a/src/android/app/run.rs b/src/android/app/run.rs index 6f93da4..96a8c49 100644 --- a/src/android/app/run.rs +++ b/src/android/app/run.rs @@ -9,7 +9,8 @@ use crate::android::{ pipewire_standalone_aaudio, wayland::{ bind, centralize, centralize_injected_keyboard, handle, on_external_pointer, - paint_trackpad, write_guest_output_state, CentralizedEvent, State, WaylandBackend, + paint_trackpad, set_keyboard_inset, write_guest_output_state, CentralizedEvent, State, + WaylandBackend, }, webview::ErrorVariant, }, @@ -236,6 +237,17 @@ impl ApplicationHandler for PolarBearApp { on_external_pointer(backend, event); } } + AppUserEvent::KeyboardRequested => { + if backend.window_visible() { + run_in_jvm( + external_display::show_keyboard, + self.frontend.android_app.clone(), + ); + } + } + AppUserEvent::KeyboardInset => { + set_keyboard_inset(backend, external_display::keyboard_inset()); + } _ => { for event in accessibility::drain_pending_events() { let event = centralize_injected_keyboard( diff --git a/src/android/backend/wayland/event_handler.rs b/src/android/backend/wayland/event_handler.rs index 35677e3..c2d938c 100644 --- a/src/android/backend/wayland/event_handler.rs +++ b/src/android/backend/wayland/event_handler.rs @@ -18,7 +18,7 @@ use smithay::backend::renderer::{Color32F, Frame, Renderer}; use smithay::input::keyboard::FilterResult; use smithay::input::pointer::{self, CursorImageStatus, CursorImageSurfaceData}; use smithay::reexports::wayland_server::protocol::wl_pointer::ButtonState as WlButtonState; -use smithay::utils::{Point, Rectangle, Transform, SERIAL_COUNTER}; +use smithay::utils::{Physical, Point, Rectangle, Size, Transform, SERIAL_COUNTER}; use smithay::wayland::compositor::with_states; use smithay::wayland::shell::xdg::ToplevelSurface; use smithay::{ @@ -430,34 +430,53 @@ pub fn handle(event: CentralizedEvent, backend: &mut WaylandBackend, event_loop: } // The desktop is on the external display, so the phone's own window doesn't size it. CentralizedEvent::Resized { .. } if backend.is_trackpad() => paint_trackpad(backend), - CentralizedEvent::Resized { - size, - guest_scale_factor, - } => { - backend.compositor.state.size = (size.w, size.h).into(); - - if let Some(output) = &backend.compositor.output { - output.change_current_state( - Some(Mode { - size: size.into(), - refresh: 60000, - }), - Some(Transform::Normal), - Some(Scale::Integer(1)), - Some((0, 0).into()), - ); - } + CentralizedEvent::Resized { size, .. } => resize_desktop(backend, size), + _ => (), + } +} - let guest_scale = - crate::core::config::guest_ui_scale(guest_scale_factor, size.w, size.h); - write_guest_output_state(size.w, size.h, guest_scale); +/// Size the desktop to the phone's window, less what the Android keyboard covers of it. +fn resize_desktop(backend: &mut WaylandBackend, window: Size) { + let size: Size = (window.w, (window.h - backend.keyboard_inset).max(1)).into(); + backend.compositor.state.size = (size.w, size.h).into(); + + if let Some(output) = &backend.compositor.output { + output.change_current_state( + Some(Mode { + size: size.into(), + refresh: 60000, + }), + Some(Transform::Normal), + Some(Scale::Integer(1)), + Some((0, 0).into()), + ); + } - if let Some(surface) = get_surface(&backend.compositor.state) { - surface.xdg_toplevel().configure(size.w, size.h, vec![]); - } - } - _ => (), + // From the whole window, so the UI keeps its size while the keyboard is up. + let guest_scale = + crate::core::config::guest_ui_scale(backend.guest_scale_factor, window.w, window.h); + write_guest_output_state(size.w, size.h, guest_scale); + + if let Some(surface) = get_surface(&backend.compositor.state) { + surface.xdg_toplevel().configure(size.w, size.h, vec![]); + } +} + +/// The Android keyboard on the phone's screen now covers its bottom `inset` pixels: fit the +/// desktop above it, as Android apps do, so the text field being typed into stays in sight. +pub fn set_keyboard_inset(backend: &mut WaylandBackend, inset: i32) { + if inset == backend.keyboard_inset { + return; } + backend.keyboard_inset = inset; + if backend.is_trackpad() { + return; + } + let window = match &backend.graphic_renderer { + Some(winit) => winit.window_size(), + None => return, + }; + resize_desktop(backend, window); } /// Show the phone's screen as a blank trackpad; `TrackpadControls` draws its outline on top. diff --git a/src/android/backend/wayland/mod.rs b/src/android/backend/wayland/mod.rs index d222933..bdb0db4 100644 --- a/src/android/backend/wayland/mod.rs +++ b/src/android/backend/wayland/mod.rs @@ -12,7 +12,7 @@ pub use output_state::write_guest_output_state; pub use compositor::{Compositor, State}; pub use event_centralizer::{centralize, centralize_injected_keyboard, CentralizedEvent}; -pub use event_handler::{handle, on_external_pointer, paint_trackpad}; +pub use event_handler::{handle, on_external_pointer, paint_trackpad, set_keyboard_inset}; pub use winit_backend::{bind, WinitGraphicsBackend}; use smithay::{ @@ -67,6 +67,8 @@ pub struct WaylandBackend { pub cursor_position: Point, /// Buttons held by a mouse on the external display (`MotionEvent.getButtonState()`). pub external_buttons: i32, + /// How much of the bottom of the phone's screen the Android keyboard covers, in pixels. + pub keyboard_inset: i32, } impl WaylandBackend { diff --git a/src/android/dex/classes.dex b/src/android/dex/classes.dex index 4a37509..fcda5ff 100644 Binary files a/src/android/dex/classes.dex and b/src/android/dex/classes.dex differ diff --git a/src/android/external_display.rs b/src/android/external_display.rs index e4d9090..1f008b7 100644 --- a/src/android/external_display.rs +++ b/src/android/external_display.rs @@ -1,8 +1,10 @@ use std::{ collections::VecDeque, ffi::c_void, + fs, ptr::NonNull, sync::{Arc, Condvar, Mutex, OnceLock}, + thread, time::Duration, }; @@ -20,6 +22,7 @@ use winit::{ use crate::android::{ accessibility::{self, AppUserEvent}, backend::wayland::physicalkey_to_scancode, + utils::application_context::get_application_context, }; #[repr(C)] @@ -83,16 +86,40 @@ struct Bridge { /// The event loop holds an EGL surface on `surface`. in_use: bool, pointer_events: VecDeque, + /// How much of the phone's screen the Android keyboard covers, in pixels. + keyboard_inset: i32, } static BRIDGE: OnceLock<(Mutex, Condvar)> = OnceLock::new(); +/// The guest touches this when a text field takes focus (see `elementary-mobile-keyboard`). +const KEYBOARD_REQUEST: &str = "tmp/localdesktop-keyboard"; + fn bridge() -> &'static (Mutex, Condvar) { BRIDGE.get_or_init(Default::default) } pub fn register_event_loop_proxy(proxy: EventLoopProxy) { - bridge().0.lock().unwrap().proxy = Some(proxy); + bridge().0.lock().unwrap().proxy = Some(proxy.clone()); + watch_keyboard_requests(proxy); +} + +fn watch_keyboard_requests(proxy: EventLoopProxy) { + thread::spawn(move || { + let path = get_application_context().rootfs_dir.join(KEYBOARD_REQUEST); + let modified = || fs::metadata(&path).and_then(|it| it.modified()).ok(); + let mut last = modified(); + loop { + thread::sleep(Duration::from_millis(200)); + let now = modified(); + if now != last { + last = now; + if now.is_some() && proxy.send_event(AppUserEvent::KeyboardRequested).is_err() { + return; + } + } + } + }); } /// The external monitor's surface, if one is connected. @@ -109,6 +136,10 @@ pub fn set_in_use(in_use: bool) { } } +pub fn keyboard_inset() -> i32 { + bridge().0.lock().unwrap().keyboard_inset +} + pub fn drain_pointer_events() -> Vec { bridge() .0 @@ -166,6 +197,11 @@ pub fn start(env: &mut JNIEnv, android_app: &AndroidApp) { call_java(env, android_app, "start"); } +/// Open the Android keyboard, over the trackpad or on the phone's screen. +pub fn show_keyboard(env: &mut JNIEnv, android_app: &AndroidApp) { + call_java(env, android_app, "showKeyboard"); +} + /// Keep the desktop on the monitor while the app is in the background. pub fn background(env: &mut JNIEnv, android_app: &AndroidApp) { call_java(env, android_app, "background"); @@ -295,3 +331,16 @@ pub extern "system" fn Java_app_polarbear_TrackpadControls_nativeOnKey( }; accessibility::inject_key(scancode, state, event_time_ms.max(0) as u64); } + +#[no_mangle] +pub extern "system" fn Java_app_polarbear_PhoneKeyboard_nativeOnInset( + _env: JNIEnv, + _class: JClass, + bottom: jint, +) { + let mut bridge = bridge().0.lock().unwrap(); + if bridge.keyboard_inset != bottom { + bridge.keyboard_inset = bottom; + notify(&bridge, AppUserEvent::KeyboardInset); + } +} diff --git a/src/android/java/app/polarbear/ExternalDisplay.java b/src/android/java/app/polarbear/ExternalDisplay.java index 9990b7f..dcaa8d8 100644 --- a/src/android/java/app/polarbear/ExternalDisplay.java +++ b/src/android/java/app/polarbear/ExternalDisplay.java @@ -80,6 +80,23 @@ public void run() { }); } + /** + * A text field on the desktop took focus: type into it with the Android keyboard, over the + * trackpad or on the phone's own screen. + */ + public static void showKeyboard(final Activity activity) { + activity.runOnUiThread(new Runnable() { + @Override + public void run() { + if (controls != null) { + controls.showKeyboard(); + } else { + PhoneKeyboard.show(activity); + } + } + }); + } + public static void stop(Activity activity) { activity.runOnUiThread(new Runnable() { @Override @@ -113,6 +130,7 @@ private static void update() { presentation = null; return; } + PhoneKeyboard.hide(); controls = new TrackpadControls(activity); controls.show(); Toast.makeText(activity, "The desktop is on the external display. Use this screen as a trackpad.", diff --git a/src/android/java/app/polarbear/PhoneKeyboard.java b/src/android/java/app/polarbear/PhoneKeyboard.java new file mode 100644 index 0000000..ca5c549 --- /dev/null +++ b/src/android/java/app/polarbear/PhoneKeyboard.java @@ -0,0 +1,109 @@ +package app.polarbear; + +import android.app.Activity; +import android.content.Context; +import android.os.Build; +import android.text.InputType; +import android.view.KeyEvent; +import android.view.View; +import android.view.ViewGroup; +import android.view.WindowInsets; +import android.view.inputmethod.BaseInputConnection; +import android.view.inputmethod.EditorInfo; +import android.view.inputmethod.InputConnection; +import android.view.inputmethod.InputMethodManager; + +/** + * The Android keyboard on the phone's own screen, typing into the desktop shown there. While it + * is up, the desktop fits above it. + */ +final class PhoneKeyboard { + private static KeyInput input; + + private PhoneKeyboard() {} + + static void show(Activity activity) { + final KeyInput input = attach(activity); + input.setFocusableInTouchMode(true); + input.requestFocus(); + // After the input method has connected to the newly focused view. + input.post(new Runnable() { + @Override + public void run() { + InputMethodManager imm = (InputMethodManager) + input.getContext().getSystemService(Context.INPUT_METHOD_SERVICE); + imm.showSoftInput(input, 0); + } + }); + } + + static void hide() { + if (input == null) { + return; + } + InputMethodManager imm = (InputMethodManager) + input.getContext().getSystemService(Context.INPUT_METHOD_SERVICE); + imm.hideSoftInputFromWindow(input.getWindowToken(), 0); + input.setFocusableInTouchMode(false); + input.clearFocus(); + } + + private static KeyInput attach(Activity activity) { + if (input != null && input.getContext() == activity) { + return input; + } + input = new KeyInput(activity); + activity.addContentView(input, new ViewGroup.LayoutParams(1, 1)); + if (Build.VERSION.SDK_INT >= 30) { + input.setOnApplyWindowInsetsListener(new View.OnApplyWindowInsetsListener() { + @Override + public WindowInsets onApplyWindowInsets(View view, WindowInsets insets) { + int bottom = insets.isVisible(WindowInsets.Type.ime()) + ? insets.getInsets(WindowInsets.Type.ime()).bottom : 0; + nativeOnInset(bottom); + if (bottom == 0) { + // Put away with the back gesture: the next tap on a text field asks again. + view.setFocusableInTouchMode(false); + } + return insets; + } + }); + } + return input; + } + + /** Takes what the Android keyboard types and passes it on as key presses. */ + private static final class KeyInput extends View { + KeyInput(Context context) { + super(context); + } + + @Override + public boolean onCheckIsTextEditor() { + return true; + } + + @Override + public InputConnection onCreateInputConnection(EditorInfo outAttrs) { + // No suggestions, so each character arrives as soon as it is typed. + outAttrs.inputType = InputType.TYPE_CLASS_TEXT + | InputType.TYPE_TEXT_VARIATION_VISIBLE_PASSWORD + | InputType.TYPE_TEXT_FLAG_NO_SUGGESTIONS; + outAttrs.imeOptions = EditorInfo.IME_FLAG_NO_FULLSCREEN + | EditorInfo.IME_FLAG_NO_EXTRACT_UI + | EditorInfo.IME_ACTION_NONE; + return new BaseInputConnection(this, false) { + @Override + public boolean sendKeyEvent(KeyEvent event) { + // Straight to the desktop: the activity's own key events go to native code, + // which doesn't see keys the input method makes up. + TrackpadControls.forwardKey(event); + return true; + } + }; + } + } + + /** How much of the screen's bottom the keyboard covers, in pixels; 0 once it is gone. */ + private static native void nativeOnInset(int bottom); +} diff --git a/src/android/java/app/polarbear/TrackpadControls.java b/src/android/java/app/polarbear/TrackpadControls.java index 0d001d8..18caa5c 100644 --- a/src/android/java/app/polarbear/TrackpadControls.java +++ b/src/android/java/app/polarbear/TrackpadControls.java @@ -275,6 +275,11 @@ private boolean onMouse(MotionEvent event) { return true; } + /** A text field on the desktop took focus. */ + void showKeyboard() { + setKeyboardShown(true); + } + /** * Turns the whole screen a quarter clockwise, Android keyboard included, for a phone lying * sideways while auto-rotate keeps it upright. diff --git a/src/android/proguard-rules.pro b/src/android/proguard-rules.pro index 62d90cd..bc12428 100644 --- a/src/android/proguard-rules.pro +++ b/src/android/proguard-rules.pro @@ -5,6 +5,7 @@ -keep class app.polarbear.ExternalDisplay { public static void start(android.app.Activity); public static void background(android.app.Activity); + public static void showKeyboard(android.app.Activity); public static void stop(android.app.Activity); native ; } @@ -12,3 +13,7 @@ -keep class app.polarbear.TrackpadControls { native ; } + +-keep class app.polarbear.PhoneKeyboard { + native ; +} diff --git a/src/android/proot/setup.rs b/src/android/proot/setup.rs index a38257c..64f3ae3 100644 --- a/src/android/proot/setup.rs +++ b/src/android/proot/setup.rs @@ -364,6 +364,87 @@ fn replace_file(path: &Path, contents: &str) { } } +/// Apps report focused text fields over the accessibility bus (AT-SPI): the app opens the Android +/// keyboard when one takes focus. Every launch, so updates reach existing installs. +fn setup_keyboard(_: &SetupOptions) -> StageOutput { + let rootfs = get_application_context().rootfs_dir; + let script = rootfs.join("usr/local/bin/elementary-mobile-keyboard"); + replace_file( + &script, + r#"#!/usr/bin/python3 +# Ask the app for the Android keyboard (see external_display.rs) when a text field takes focus. +import os, pyatspi + +REQUEST = "/tmp/localdesktop-keyboard" + +def on_focus(event): + if not event.detail1: + return + try: + editable = event.source.getState().contains(pyatspi.STATE_EDITABLE) + # Its terminal widget doesn't report itself as one. + terminal = event.source.getApplication().name == "io.elementary.terminal" + except Exception: + return + if editable or terminal: + with open(REQUEST, "a"): + os.utime(REQUEST) + +pyatspi.Registry.registerEventListener(on_focus, "object:state-changed:focused") +pyatspi.Registry.start() +"#, + ); + let _ = fs::set_permissions(&script, fs::Permissions::from_mode(0o755)); + replace_file( + &rootfs.join("etc/elementary-mobile/xdg/autostart/elementary-mobile-keyboard.desktop"), + "[Desktop Entry]\nType=Application\nName=Android keyboard\nExec=/usr/local/bin/elementary-mobile-keyboard\nNoDisplay=true\n", + ); + // For text fields the listener misses. + let applications = rootfs.join("usr/local/share/applications"); + let _ = fs::create_dir_all(&applications); + replace_file( + &applications.join("elementary-mobile-keyboard.desktop"), + "[Desktop Entry]\nType=Application\nName=On-screen keyboard\nComment=Show the Android keyboard\nExec=touch /tmp/localdesktop-keyboard\nIcon=input-keyboard\nCategories=Utility;Accessibility;\n", + ); + None +} + +/// The accessibility bus for `setup_keyboard`, on an OS unpacked before it came with it. +fn setup_keyboard_packages(options: &SetupOptions) -> StageOutput { + let rootfs = get_application_context().rootfs_dir; + let installed = [ + "usr/libexec/at-spi-bus-launcher", + "usr/lib/python3/dist-packages/pyatspi", + ]; + if installed.iter().all(|path| rootfs.join(path).exists()) { + return None; + } + let mpsc_sender = options.mpsc_sender.clone(); + Some(thread::spawn(move || { + mpsc_sender + .send(SetupMessage::Progress( + "Installing the Android keyboard helper...".to_string(), + )) + .unwrap_or(()); + let packages = "at-spi2-core python3-pyatspi gir1.2-atspi-2.0"; + let output = GuestProcess { + command: format!( + "export DEBIAN_FRONTEND=noninteractive; apt-get install -y --no-install-recommends {packages} || {{ apt-get update && apt-get install -y --no-install-recommends {packages}; }}" + ), + user: None, + log: None, + } + .run(); + if !output.status.success() { + // Offline: the next launch tries again. + log::warn!( + "Failed to install {packages}: {}", + String::from_utf8_lossy(&output.stderr) + ); + } + })) +} + /// The DNS servers of Android's active network, as IP address strings. fn android_dns_servers(android_app: &AndroidApp) -> Vec { fn query(env: &mut jni::JNIEnv, app: &AndroidApp) -> Option> { @@ -490,6 +571,7 @@ pub fn build_wayland_backend(android_app: AndroidApp) -> WaylandBackend { touch_fingers: 0, cursor_position: (0.0, 0.0).into(), external_buttons: 0, + keyboard_inset: 0, android_app, } } @@ -543,6 +625,8 @@ pub fn setup(android_app: AndroidApp) -> PolarBearBackend { Box::new(setup_network_and_time), // Step 5. DNS servers and timezone from Android Box::new(setup_cpuinfo), // Step 6. /proc/cpuinfo that llvmpipe can trust Box::new(fix_xkb_symlink), // Step 7. Fix xkb symlink + Box::new(setup_keyboard), // Step 8. Android keyboard for focused text fields + Box::new(setup_keyboard_packages), // Step 8b. Its accessibility bus on older installs ]; let handle_stage_error = |e: Box, sender: &Sender| {