Android keyboard on the phone's screen

README.md

@@ -12,6 +12,9 @@ Wayland compositor in the Android NDK plus proot) and replaces what runs on it.
 ideally with a keyboard and mouse. It brings a desktop to Android; it does not turn the desktop
 into a phone UI.
 
+Tap a text field and your phone's own keyboard comes up, with the desktop fitting above it. For
+apps where it doesn't, open **On-screen keyboard** from the Applications menu.
+
 On a phone, connect a monitor (USB-C to HDMI, a dock, or a wireless display): the desktop moves
 to it and the phone becomes a trackpad, with buttons to turn the screen and to type with the
 Android keyboard.

rootfs/packages.list

@@ -73,6 +73,10 @@ file-roller
 
 # On-screen keyboard (toggled from Quick Settings for touch-only use)
 onboard
+# Reports focused text fields, so the app can open the Android keyboard
+at-spi2-core
+gir1.2-atspi-2.0
+python3-pyatspi
 
 # Command-line basics
 ca-certificates

src/android/accessibility.rs

@@ -21,6 +21,10 @@ pub enum AppUserEvent {
     ExternalDisplayChanged,
     /// A mouse (or touch) on the external monitor moved, clicked or scrolled.
     ExternalPointer,
+    /// A text field on the desktop took focus.
+    KeyboardRequested,
+    /// The Android keyboard on the phone's screen came up, changed height or went away.
+    KeyboardInset,
 }
 
 #[derive(Clone, Copy, Debug)]

src/android/app/run.rs

@@ -9,7 +9,8 @@ use crate::android::{
         pipewire_standalone_aaudio,
         wayland::{
             bind, centralize, centralize_injected_keyboard, handle, on_external_pointer,
-            paint_trackpad, write_guest_output_state, CentralizedEvent, State, WaylandBackend,
+            paint_trackpad, set_keyboard_inset, write_guest_output_state, CentralizedEvent, State,
+            WaylandBackend,
         },
         webview::ErrorVariant,
     },
@@ -236,6 +237,17 @@ impl ApplicationHandler<AppUserEvent> for PolarBearApp {
                     on_external_pointer(backend, event);
                 }
             }
+            AppUserEvent::KeyboardRequested => {
+                if backend.window_visible() {
+                    run_in_jvm(
+                        external_display::show_keyboard,
+                        self.frontend.android_app.clone(),
+                    );
+                }
+            }
+            AppUserEvent::KeyboardInset => {
+                set_keyboard_inset(backend, external_display::keyboard_inset());
+            }
             _ => {
                 for event in accessibility::drain_pending_events() {
                     let event = centralize_injected_keyboard(

src/android/backend/wayland/event_handler.rs

@@ -18,7 +18,7 @@ use smithay::backend::renderer::{Color32F, Frame, Renderer};
 use smithay::input::keyboard::FilterResult;
 use smithay::input::pointer::{self, CursorImageStatus, CursorImageSurfaceData};
 use smithay::reexports::wayland_server::protocol::wl_pointer::ButtonState as WlButtonState;
-use smithay::utils::{Point, Rectangle, Transform, SERIAL_COUNTER};
+use smithay::utils::{Physical, Point, Rectangle, Size, Transform, SERIAL_COUNTER};
 use smithay::wayland::compositor::with_states;
 use smithay::wayland::shell::xdg::ToplevelSurface;
 use smithay::{
@@ -430,34 +430,53 @@ pub fn handle(event: CentralizedEvent, backend: &mut WaylandBackend, event_loop:
         }
         // The desktop is on the external display, so the phone's own window doesn't size it.
         CentralizedEvent::Resized { .. } if backend.is_trackpad() => paint_trackpad(backend),
-        CentralizedEvent::Resized {
-            size,
-            guest_scale_factor,
-        } => {
-            backend.compositor.state.size = (size.w, size.h).into();
-
-            if let Some(output) = &backend.compositor.output {
-                output.change_current_state(
-                    Some(Mode {
-                        size: size.into(),
-                        refresh: 60000,
-                    }),
-                    Some(Transform::Normal),
-                    Some(Scale::Integer(1)),
-                    Some((0, 0).into()),
-                );
-            }
+        CentralizedEvent::Resized { size, .. } => resize_desktop(backend, size),
+        _ => (),
+    }
+}
 
-            let guest_scale =
-                crate::core::config::guest_ui_scale(guest_scale_factor, size.w, size.h);
-            write_guest_output_state(size.w, size.h, guest_scale);
+/// Size the desktop to the phone's window, less what the Android keyboard covers of it.
+fn resize_desktop(backend: &mut WaylandBackend, window: Size<i32, Physical>) {
+    let size: Size<i32, Physical> = (window.w, (window.h - backend.keyboard_inset).max(1)).into();
+    backend.compositor.state.size = (size.w, size.h).into();
+
+    if let Some(output) = &backend.compositor.output {
+        output.change_current_state(
+            Some(Mode {
+                size: size.into(),
+                refresh: 60000,
+            }),
+            Some(Transform::Normal),
+            Some(Scale::Integer(1)),
+            Some((0, 0).into()),
+        );
+    }
 
-            if let Some(surface) = get_surface(&backend.compositor.state) {
-                surface.xdg_toplevel().configure(size.w, size.h, vec![]);
-            }
-        }
-        _ => (),
+    // From the whole window, so the UI keeps its size while the keyboard is up.
+    let guest_scale =
+        crate::core::config::guest_ui_scale(backend.guest_scale_factor, window.w, window.h);
+    write_guest_output_state(size.w, size.h, guest_scale);
+
+    if let Some(surface) = get_surface(&backend.compositor.state) {
+        surface.xdg_toplevel().configure(size.w, size.h, vec![]);
+    }
+}
+
+/// The Android keyboard on the phone's screen now covers its bottom `inset` pixels: fit the
+/// desktop above it, as Android apps do, so the text field being typed into stays in sight.
+pub fn set_keyboard_inset(backend: &mut WaylandBackend, inset: i32) {
+    if inset == backend.keyboard_inset {
+        return;
     }
+    backend.keyboard_inset = inset;
+    if backend.is_trackpad() {
+        return;
+    }
+    let window = match &backend.graphic_renderer {
+        Some(winit) => winit.window_size(),
+        None => return,
+    };
+    resize_desktop(backend, window);
 }
 
 /// Show the phone's screen as a blank trackpad; `TrackpadControls` draws its outline on top.

src/android/backend/wayland/mod.rs

@@ -12,7 +12,7 @@ pub use output_state::write_guest_output_state;
 
 pub use compositor::{Compositor, State};
 pub use event_centralizer::{centralize, centralize_injected_keyboard, CentralizedEvent};
-pub use event_handler::{handle, on_external_pointer, paint_trackpad};
+pub use event_handler::{handle, on_external_pointer, paint_trackpad, set_keyboard_inset};
 pub use winit_backend::{bind, WinitGraphicsBackend};
 
 use smithay::{
@@ -67,6 +67,8 @@ pub struct WaylandBackend {
     pub cursor_position: Point<f64, Logical>,
     /// Buttons held by a mouse on the external display (`MotionEvent.getButtonState()`).
     pub external_buttons: i32,
+    /// How much of the bottom of the phone's screen the Android keyboard covers, in pixels.
+    pub keyboard_inset: i32,
 }
 
 impl WaylandBackend {

src/android/external_display.rs

@@ -1,8 +1,10 @@
 use std::{
     collections::VecDeque,
     ffi::c_void,
+    fs,
     ptr::NonNull,
     sync::{Arc, Condvar, Mutex, OnceLock},
+    thread,
     time::Duration,
 };
 
@@ -20,6 +22,7 @@ use winit::{
 use crate::android::{
     accessibility::{self, AppUserEvent},
     backend::wayland::physicalkey_to_scancode,
+    utils::application_context::get_application_context,
 };
 
 #[repr(C)]
@@ -83,16 +86,40 @@ struct Bridge {
     /// The event loop holds an EGL surface on `surface`.
     in_use: bool,
     pointer_events: VecDeque<PointerEvent>,
+    /// How much of the phone's screen the Android keyboard covers, in pixels.
+    keyboard_inset: i32,
 }
 
 static BRIDGE: OnceLock<(Mutex<Bridge>, Condvar)> = OnceLock::new();
 
+/// The guest touches this when a text field takes focus (see `elementary-mobile-keyboard`).
+const KEYBOARD_REQUEST: &str = "tmp/localdesktop-keyboard";
+
 fn bridge() -> &'static (Mutex<Bridge>, Condvar) {
     BRIDGE.get_or_init(Default::default)
 }
 
 pub fn register_event_loop_proxy(proxy: EventLoopProxy<AppUserEvent>) {
-    bridge().0.lock().unwrap().proxy = Some(proxy);
+    bridge().0.lock().unwrap().proxy = Some(proxy.clone());
+    watch_keyboard_requests(proxy);
+}
+
+fn watch_keyboard_requests(proxy: EventLoopProxy<AppUserEvent>) {
+    thread::spawn(move || {
+        let path = get_application_context().rootfs_dir.join(KEYBOARD_REQUEST);
+        let modified = || fs::metadata(&path).and_then(|it| it.modified()).ok();
+        let mut last = modified();
+        loop {
+            thread::sleep(Duration::from_millis(200));
+            let now = modified();
+            if now != last {
+                last = now;
+                if now.is_some() && proxy.send_event(AppUserEvent::KeyboardRequested).is_err() {
+                    return;
+                }
+            }
+        }
+    });
 }
 
 /// The external monitor's surface, if one is connected.
@@ -109,6 +136,10 @@ pub fn set_in_use(in_use: bool) {
     }
 }
 
+pub fn keyboard_inset() -> i32 {
+    bridge().0.lock().unwrap().keyboard_inset
+}
+
 pub fn drain_pointer_events() -> Vec<PointerEvent> {
     bridge()
         .0
@@ -166,6 +197,11 @@ pub fn start(env: &mut JNIEnv, android_app: &AndroidApp) {
     call_java(env, android_app, "start");
 }
 
+/// Open the Android keyboard, over the trackpad or on the phone's screen.
+pub fn show_keyboard(env: &mut JNIEnv, android_app: &AndroidApp) {
+    call_java(env, android_app, "showKeyboard");
+}
+
 /// Keep the desktop on the monitor while the app is in the background.
 pub fn background(env: &mut JNIEnv, android_app: &AndroidApp) {
     call_java(env, android_app, "background");
@@ -295,3 +331,16 @@ pub extern "system" fn Java_app_polarbear_TrackpadControls_nativeOnKey(
     };
     accessibility::inject_key(scancode, state, event_time_ms.max(0) as u64);
 }
+
+#[no_mangle]
+pub extern "system" fn Java_app_polarbear_PhoneKeyboard_nativeOnInset(
+    _env: JNIEnv,
+    _class: JClass,
+    bottom: jint,
+) {
+    let mut bridge = bridge().0.lock().unwrap();
+    if bridge.keyboard_inset != bottom {
+        bridge.keyboard_inset = bottom;
+        notify(&bridge, AppUserEvent::KeyboardInset);
+    }
+}

src/android/java/app/polarbear/ExternalDisplay.java

@@ -80,6 +80,23 @@ public void run() {
         });
     }
 
+    /**
+     * A text field on the desktop took focus: type into it with the Android keyboard, over the
+     * trackpad or on the phone's own screen.
+     */
+    public static void showKeyboard(final Activity activity) {
+        activity.runOnUiThread(new Runnable() {
+            @Override
+            public void run() {
+                if (controls != null) {
+                    controls.showKeyboard();
+                } else {
+                    PhoneKeyboard.show(activity);
+                }
+            }
+        });
+    }
+
     public static void stop(Activity activity) {
         activity.runOnUiThread(new Runnable() {
             @Override
@@ -113,6 +130,7 @@ private static void update() {
             presentation = null;
             return;
         }
+        PhoneKeyboard.hide();
         controls = new TrackpadControls(activity);
         controls.show();
         Toast.makeText(activity, "The desktop is on the external display. Use this screen as a trackpad.",