Working sudo, mouse on the external display and wheel scrolling

Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "localdesktop"
-version = "1.1.1"
+version = "1.2.0"
 edition = "2021"
 build = "build.rs"
 default-run = "build_apk"

src/android/backend/wayland/compositor.rs

@@ -1,8 +1,8 @@
 use super::bind::bind_socket;
 use smithay::{
     backend::renderer::utils::on_commit_buffer_handler,
-    delegate_compositor, delegate_data_device, delegate_output, delegate_seat, delegate_shm,
-    delegate_xdg_shell,
+    delegate_compositor, delegate_data_device, delegate_output, delegate_relative_pointer,
+    delegate_seat, delegate_shm, delegate_xdg_shell,
     input::{keyboard::KeyboardHandle, touch::TouchHandle, Seat, SeatHandler, SeatState},
     output::Output,
     reexports::{
@@ -17,6 +17,7 @@ use smithay::{
             SurfaceAttributes, TraversalAction,
         },
         output::OutputHandler,
+        relative_pointer::RelativePointerManagerState,
         selection::{
             data_device::{
                 ClientDndGrabHandler, DataDeviceHandler, DataDeviceState, ServerDndGrabHandler,
@@ -59,6 +60,10 @@ pub struct State {
     pub shm_state: ShmState,
     pub data_device_state: DataDeviceState,
     pub seat_state: SeatState<Self>,
+    /// Xwayland sends scrolling through its relative pointer device, and pointer motion through
+    /// it too only when the compositor reports relative motion. Without this, X apps see the
+    /// pointer switch devices around every scroll and GTK drops the scroll.
+    pub relative_pointer_state: RelativePointerManagerState,
     pub size: Size<i32, Logical>,
     /// What the client wants the cursor to look like.
     pub cursor_image: CursorImageStatus,
@@ -191,6 +196,7 @@ delegate_shm!(State);
 delegate_seat!(State);
 delegate_data_device!(State);
 delegate_output!(State);
+delegate_relative_pointer!(State);
 
 impl Compositor {
     pub fn build() -> Result<Compositor, Box<dyn Error>> {
@@ -218,6 +224,7 @@ impl Compositor {
             shm_state: ShmState::new::<State>(&dh, vec![]),
             data_device_state: DataDeviceState::new::<State>(&dh),
             seat_state,
+            relative_pointer_state: RelativePointerManagerState::new::<State>(&dh),
             size: (1920, 1080).into(),
             cursor_image: CursorImageStatus::default_named(),
         };

src/android/backend/wayland/event_handler.rs

@@ -75,15 +75,26 @@ fn emit_pointer_motion(
     let state = &mut compositor.state;
     if let Some(focus) = pointer_focus(state) {
         let serial = SERIAL_COUNTER.next_serial();
+        let delta = Point::from((x, y)) - pointer.current_location();
         pointer.motion(
             state,
-            Some(focus),
+            Some(focus.clone()),
             &pointer::MotionEvent {
                 location: (x, y).into(),
                 serial,
                 time,
             },
         );
+        // In the same frame, so Xwayland moves its relative pointer device (see `State`).
+        pointer.relative_motion(
+            state,
+            Some(focus),
+            &pointer::RelativeMotionEvent {
+                delta,
+                delta_unaccel: delta,
+                utime: time as u64 * 1000,
+            },
+        );
         pointer.frame(state);
     }
 }
@@ -293,22 +304,12 @@ pub fn handle(event: CentralizedEvent, backend: &mut WaylandBackend, event_loop:
                 }
             }
             InputEvent::PointerMotionAbsolute { event, .. } => {
-                let compositor = &mut backend.compositor;
-                let pointer = compositor.pointer.clone();
-                let serial = SERIAL_COUNTER.next_serial();
-
-                if let Some(surface) = get_surface(&compositor.state) {
-                    pointer.motion(
-                        &mut compositor.state,
-                        Some((surface.wl_surface().clone(), (0f64, 0f64).into())),
-                        &pointer::MotionEvent {
-                            location: (event.x(), event.y()).into(),
-                            serial,
-                            time: event.time_msec(),
-                        },
-                    );
-                }
-                pointer.frame(&mut compositor.state);
+                emit_pointer_motion(
+                    &mut backend.compositor,
+                    event.x(),
+                    event.y(),
+                    event.time_msec(),
+                );
             }
             InputEvent::PointerButton { event, .. } => {
                 let serial = SERIAL_COUNTER.next_serial();
@@ -468,14 +469,22 @@ pub fn paint_trackpad(backend: &mut WaylandBackend) {
     }
 }
 
-/// A mouse on the external display: it points where it is, and its buttons and wheel go
-/// straight to the desktop.
+/// A mouse on the external display, or one plugged into the phone: it moves the cursor, and its
+/// buttons and wheel go straight to the desktop.
 pub fn on_external_pointer(backend: &mut WaylandBackend, event: PointerEvent) {
     let time = event.time_ms as u32;
     let size = backend.compositor.state.size;
+    let (x, y) = if event.relative {
+        // Mouse counts, scaled up on monitors larger than 1080p so crossing them is no chore.
+        let speed = (size.w as f64 / 1920.0).max(1.0);
+        let cursor = backend.cursor_position;
+        (cursor.x + event.x * speed, cursor.y + event.y * speed)
+    } else {
+        (event.x, event.y)
+    };
     backend.cursor_position = (
-        event.x.clamp(0.0, (size.w - 1).max(0) as f64),
-        event.y.clamp(0.0, (size.h - 1).max(0) as f64),
+        x.clamp(0.0, (size.w - 1).max(0) as f64),
+        y.clamp(0.0, (size.h - 1).max(0) as f64),
     )
         .into();
     let cursor = backend.cursor_position;
@@ -592,7 +601,7 @@ fn redraw(backend: &mut WaylandBackend) -> Result<(), String> {
             .render(&mut framebuffer, size, Transform::Flipped180)
             .map_err(|error| format!("Failed to render frame: {error:?}"))?;
         frame
-            .clear(Color32F::new(0.1, 0.0, 0.0, 1.0), &[damage])
+            .clear(Color32F::new(0.0, 0.0, 0.0, 1.0), &[damage])
             .map_err(|error| format!("Failed to clear frame: {error:?}"))?;
         draw_render_elements(&mut frame, 1.0, &elements, &[damage])
             .map_err(|error| format!("Failed to draw render elements: {error:?}"))?;

src/android/external_display.rs

@@ -61,11 +61,13 @@ impl Drop for ExternalSurface {
     }
 }
 
-/// A mouse (or touch) on the external monitor, in its pixels.
+/// A mouse (or touch) on the external monitor, or a mouse plugged into the phone.
 #[derive(Debug, Clone, Copy)]
 pub struct PointerEvent {
+    /// Where on the monitor, in its pixels, or how far a phone's mouse moved when `relative`.
     pub x: f64,
     pub y: f64,
+    pub relative: bool,
     /// `MotionEvent.getButtonState()`.
     pub buttons: i32,
     /// `AXIS_HSCROLL` and `AXIS_VSCROLL`, in wheel notches.
@@ -231,16 +233,44 @@ pub extern "system" fn Java_app_polarbear_ExternalDisplay_nativeOnPointer(
     scroll_y: jfloat,
     event_time_ms: jlong,
 ) {
-    let mut bridge = bridge().0.lock().unwrap();
-    let wake = bridge.pointer_events.is_empty();
-    bridge.pointer_events.push_back(PointerEvent {
+    push_pointer_event(PointerEvent {
         x: x as f64,
         y: y as f64,
+        relative: false,
         buttons,
         scroll_x: scroll_x as f64,
         scroll_y: scroll_y as f64,
         time_ms: event_time_ms.max(0) as u64,
     });
+}
+
+/// A mouse plugged into the phone, captured while it is a trackpad.
+#[no_mangle]
+pub extern "system" fn Java_app_polarbear_TrackpadControls_nativeOnMouse(
+    _env: JNIEnv,
+    _class: JClass,
+    dx: jfloat,
+    dy: jfloat,
+    buttons: jint,
+    scroll_x: jfloat,
+    scroll_y: jfloat,
+    event_time_ms: jlong,
+) {
+    push_pointer_event(PointerEvent {
+        x: dx as f64,
+        y: dy as f64,
+        relative: true,
+        buttons,
+        scroll_x: scroll_x as f64,
+        scroll_y: scroll_y as f64,
+        time_ms: event_time_ms.max(0) as u64,
+    });
+}
+
+fn push_pointer_event(event: PointerEvent) {
+    let mut bridge = bridge().0.lock().unwrap();
+    let wake = bridge.pointer_events.is_empty();
+    bridge.pointer_events.push_back(event);
     if wake {
         notify(&bridge, AppUserEvent::ExternalPointer);
     }

src/android/java/app/polarbear/ExternalDisplay.java

@@ -151,13 +151,13 @@ protected void onCreate(Bundle savedInstanceState) {
             view.setOnGenericMotionListener(new View.OnGenericMotionListener() {
                 @Override
                 public boolean onGenericMotion(View view, MotionEvent event) {
-                    return onPointer(event);
+                    return onPointer(event, false);
                 }
             });
             view.setOnTouchListener(new View.OnTouchListener() {
                 @Override
                 public boolean onTouch(View view, MotionEvent event) {
-                    return onPointer(event);
+                    return onPointer(event, true);
                 }
             });
             setContentView(view);
@@ -170,10 +170,12 @@ public boolean dispatchKeyEvent(KeyEvent event) {
             return true;
         }
 
-        private boolean onPointer(MotionEvent event) {
+        /** `pressing`: a touch event, which Android only sends while something is pressed. */
+        private boolean onPointer(MotionEvent event, boolean pressing) {
             int buttons = event.getButtonState();
-            // A touch screen or pen on the monitor clicks like a mouse.
-            if (!event.isFromSource(InputDevice.SOURCE_MOUSE)) {
+            // A touch screen or pen on the monitor clicks like a mouse, and so does a mouse that
+            // reports no button (a touchpad tapping, an injected click).
+            if (!event.isFromSource(InputDevice.SOURCE_MOUSE) || (pressing && buttons == 0)) {
                 int action = event.getActionMasked();
                 buttons = action == MotionEvent.ACTION_UP || action == MotionEvent.ACTION_CANCEL
                         ? 0 : MotionEvent.BUTTON_PRIMARY;

src/android/java/app/polarbear/KeyboardShortcutsPrompt.java

@@ -6,12 +6,15 @@
 import android.content.DialogInterface;
 import android.content.Intent;
 import android.content.SharedPreferences;
-import android.content.res.Configuration;
+import android.os.Bundle;
 import android.provider.Settings;
+import android.view.InputDevice;
 
 public final class KeyboardShortcutsPrompt {
     private static final String PREFS = "keyboard_shortcuts";
     private static final String ANSWERED = "answered";
+    /** "Not now" for as long as the app runs. */
+    private static boolean notNow;
 
     private KeyboardShortcutsPrompt() {}
 
@@ -27,38 +30,64 @@ public void run() {
     private static void show(final Activity activity) {
         final SharedPreferences prefs = activity.getSharedPreferences(PREFS, Activity.MODE_PRIVATE);
         if (prefs.getBoolean(ANSWERED, false)
-                || activity.getResources().getConfiguration().keyboard != Configuration.KEYBOARD_QWERTY
+                || notNow
+                || !hasPhysicalKeyboard()
                 || isServiceEnabled(activity)) {
             return;
         }
 
         CharSequence app = activity.getApplicationInfo().loadLabel(activity.getPackageManager());
         new AlertDialog.Builder(activity, android.R.style.Theme_DeviceDefault_Dialog_Alert)
-            .setTitle("Forward keyboard shortcuts?")
+            .setTitle("Use Alt+Tab on the desktop?")
             .setMessage(
-                "Android keeps some shortcuts of a physical keyboard for itself, such as Alt+Tab and the Meta key. "
-                    + app + " can pass them to the Linux desktop with Android's accessibility service.\n\n"
-                    + "When it is on, " + app + " receives the keys you press on a physical keyboard and uses them "
-                    + "only while the desktop is on screen. It does not read the screen or other apps, and it does "
-                    + "not store or send your keystrokes anywhere.\n\n"
-                    + "You can turn it off at any time in Android's accessibility settings.")
+                "Android keeps Alt+Tab and the Meta key for itself. The " + app
+                    + " keyboard service passes them to the desktop.\n\n"
+                    + "In the settings that open:\n"
+                    + "1. Tap Downloaded apps (or Installed apps)\n"
+                    + "2. Tap " + app + " and switch it on\n"
+                    + "3. Tap Allow")
             .setCancelable(false)
             .setPositiveButton("Turn on", new DialogInterface.OnClickListener() {
                 @Override
                 public void onClick(DialogInterface dialog, int which) {
                     prefs.edit().putBoolean(ANSWERED, true).apply();
-                    activity.startActivity(new Intent(Settings.ACTION_ACCESSIBILITY_SETTINGS));
+                    activity.startActivity(accessibilitySettings(activity));
                 }
             })
-            .setNegativeButton("No thanks", new DialogInterface.OnClickListener() {
+            .setNegativeButton("Not now", new DialogInterface.OnClickListener() {
                 @Override
                 public void onClick(DialogInterface dialog, int which) {
-                    prefs.edit().putBoolean(ANSWERED, true).apply();
+                    notNow = true;
                 }
             })
             .show();
     }
 
+    /** Unlike the configuration, this already knows about a keyboard that just connected. */
+    private static boolean hasPhysicalKeyboard() {
+        for (int id : InputDevice.getDeviceIds()) {
+            InputDevice device = InputDevice.getDevice(id);
+            if (device != null && !device.isVirtual()
+                    && device.getKeyboardType() == InputDevice.KEYBOARD_TYPE_ALPHABETIC) {
+                return true;
+            }
+        }
+        return false;
+    }
+
+    /**
+     * Android's accessibility settings, asked to highlight our service. Apps cannot open the
+     * service's own page; Settings apps that don't know the highlight extras just ignore them.
+     */
+    private static Intent accessibilitySettings(Activity activity) {
+        String key = new ComponentName(activity, KeyboardAccessibilityService.class).flattenToString();
+        Bundle args = new Bundle();
+        args.putString(":settings:fragment_args_key", key);
+        return new Intent(Settings.ACTION_ACCESSIBILITY_SETTINGS)
+            .putExtra(":settings:fragment_args_key", key)
+            .putExtra(":settings:show_fragment_args", args);
+    }
+
     private static boolean isServiceEnabled(Activity activity) {
         String enabled = Settings.Secure.getString(
             activity.getContentResolver(), Settings.Secure.ENABLED_ACCESSIBILITY_SERVICES);