feat(ui-egui): a pen on the page picks up a pen, typing goes back (#9)

crates/ui-egui/src/app.rs

@@ -930,6 +930,10 @@ pub struct NoteCraftApp {
     canvas_pan: egui::Vec2,
     /// Last middle-button position while panning the canvas.
     canvas_drag: Option<egui::Pos2>,
+    /// Words typed while a pen was in hand, held for the caret's line until it has the keyboard
+    /// again (issue #9), and how many frames they have waited.
+    typed_while_inking: Vec<egui::Event>,
+    typed_wait: u8,
     /// Collect All was asked for: the page gathers its boxes and ink when it is next drawn.
     collect_pending: bool,
     /// The page's sideways scroll bar and its thumb, when one was drawn last frame (issue #188).
@@ -1102,6 +1106,8 @@ impl NoteCraftApp {
             page_hbar: None,
             space_tool: false,
             canvas_drag: None,
+            typed_while_inking: Vec::new(),
+            typed_wait: 0,
             pointer: ribbon::Pointer::Write,
             rack,
             nib,
@@ -1496,6 +1502,71 @@ impl NoteCraftApp {
         }
     }
 
+    /// OneNote's automatic switching (issue #9): the window follows what the hand is holding.
+    /// A pen that comes down on the page while the pointer is the caret picks up the pen last in
+    /// hand, colour and thickness and all, and draws with it from that touch on; typing while a
+    /// pen or another ink tool is in hand puts it down and the keys go to the page as words. A
+    /// mouse, and a finger on a touch screen, change nothing: only a contact that reports a
+    /// force is a pen, so this works where the platform passes the pen's pressure on (see
+    /// `stylus.rs`), and elsewhere the tools are chosen by hand as before.
+    fn switch_by_device(&mut self, ctx: &Context) {
+        if self.dialog.is_some() || self.backstage.is_some() || !self.editor.as_ref().is_some_and(|ed| ed.writable) {
+            return;
+        }
+        if self.pointer == ribbon::Pointer::Write {
+            if self.stylus.landed().is_some_and(|at| self.page_rect.is_some_and(|page| page.contains(at))) {
+                self.ribbon_act(ribbon::Act::InkPen(self.nib));
+                crate::inkpad::claim_press(ctx);
+            }
+            return;
+        }
+        let typed = ctx.input(|i| {
+            !i.modifiers.command
+                && !i.modifiers.ctrl
+                && !i.modifiers.alt
+                && i.events.iter().any(|e| matches!(e, egui::Event::Text(t) if !t.trim().is_empty()))
+        });
+        if !typed || ctx.egui_wants_keyboard_input() {
+            return;
+        }
+        self.ribbon_act(ribbon::Act::InkPointer(ribbon::Pointer::Write));
+        // The words go back to the line the caret was in, where it was: the line's text box
+        // was not drawn while the pen was in hand, so it has to take the keyboard again before
+        // it can read them. They are held until it has (`replay_typing`).
+        let Some(ed) = self.editor.as_mut() else { return };
+        // A selection across lines left from before the pen would keep every line as finished
+        // words, with no text box to type into.
+        ed.across = None;
+        let Some(caret) = ed.caret.as_mut() else { return };
+        caret.focus = true;
+        caret.place = Some(caret.selection.1);
+        self.typed_while_inking = ctx.input_mut(|i| {
+            let (text, rest): (Vec<_>, Vec<_>) = std::mem::take(&mut i.events).into_iter().partition(|e| matches!(e, egui::Event::Text(_)));
+            i.events = rest;
+            text
+        });
+        self.typed_wait = 0;
+        ctx.request_repaint();
+    }
+
+    /// Gives the page the words that were typed with a pen in hand (see `switch_by_device`)
+    /// once the caret's line has the keyboard again, a frame or two after the switch.
+    fn replay_typing(&mut self, ctx: &Context) {
+        if self.typed_while_inking.is_empty() {
+            return;
+        }
+        let ready = self.editor.as_ref().and_then(|ed| ed.caret.as_ref()).is_some_and(|c| !c.focus) && ctx.memory(|m| m.focused().is_some());
+        if ready {
+            let typed = std::mem::take(&mut self.typed_while_inking);
+            ctx.input_mut(|i| i.events.extend(typed));
+        } else if self.typed_wait > 4 {
+            // The caret never came back (the page was closed, say): the words have nowhere to go.
+            self.typed_while_inking.clear();
+        } else {
+            self.typed_wait += 1;
+            ctx.request_repaint();
+        }
+    }
     /// Keys for the pens, so a tablet's express keys or a keyboard can change tools without
     /// reaching for the ribbon (issue #44): Alt+1 to Alt+9 pick up that pen on the Draw tab's
     /// rack, and Alt+0 puts the pens down and goes back to typing. On a Mac, where Option and a
@@ -8869,6 +8940,8 @@ impl NoteCraftApp {
             self.focus_search = true;
         }
         self.nav_keys(ctx);
+        self.replay_typing(ctx);
+        self.switch_by_device(ctx);
         self.pen_keys(ctx);
         self.direction_keys(ctx);
         self.hierarchy_keys(ctx);

crates/ui-egui/src/inkpad.rs

@@ -420,6 +420,17 @@ pub fn held_area(ctx: &egui::Context) -> Option<egui::Rect> {
     ctx.data(|d| d.get_temp::<egui::Rect>(held_area_id()))
 }
 
+/// Where the window asks the ink layer to take the press that is down now as its own: for the
+/// one frame after a pen touching the page picked up a pen (issue #9).
+pub fn claim_press_id() -> egui::Id {
+    egui::Id::new("notecraft-ink-claim-press")
+}
+
+/// Asks the ink layer, the next time it is drawn, to take the press that is down as its own.
+pub fn claim_press(ctx: &egui::Context) {
+    ctx.data_mut(|d| d.insert_temp(claim_press_id(), true));
+}
+
 fn held_area_id() -> egui::Id {
     egui::Id::new("inkpad-held-area")
 }
@@ -575,7 +586,15 @@ pub fn show(ui: &mut Ui, body: egui::Rect, origin: egui::Pos2, pad: &mut Pad, pe
 
     // While a pen is in hand the whole page belongs to it, so this goes on last and over
     // everything: a press must not land in the words underneath and move the caret.
-    let resp = ui.interact(body, ui.id().with("ink-layer"), Sense::click_and_drag());
+    let layer = ui.id().with("ink-layer");
+    // A pen that came down on the page while the caret was the pointer has just picked up a
+    // pen (issue #9). Its press began before this layer was here, so egui gave it to whatever
+    // was under it last frame: hand it to the layer, so the touch that switched the tool also
+    // draws.
+    if ui.data_mut(|d| d.remove_temp::<bool>(claim_press_id())).unwrap_or(false) && ui.input(|i| i.pointer.primary_down()) {
+        ui.ctx().set_dragged_id(layer);
+    }
+    let resp = ui.interact(body, layer, Sense::click_and_drag());
     ui.ctx().set_cursor_icon(match pen.pointer {
         Pointer::Erase => egui::CursorIcon::NotAllowed,
         Pointer::Pick => egui::CursorIcon::Grab,

crates/ui-egui/src/stylus.rs

@@ -61,12 +61,16 @@ pub struct Stylus {
     samples: u64,
     diagnostics: Diagnostics,
     last_event: &'static str,
+    /// Where a pen came down on the glass this frame, if one did: a touch that reports a force
+    /// as it starts. What the window uses to pick the pen up by itself (issue #9).
+    landed: Option<egui::Pos2>,
 }
 
 impl Stylus {
     /// Takes this frame's input events. Call it once per frame, before anything asks.
     pub fn update(&mut self, events: &[egui::Event]) {
         self.diagnostics.frames += 1;
+        self.landed = None;
         for e in events {
             self.diagnostics.events += 1;
             match e {
@@ -76,7 +80,7 @@ impl Stylus {
                 egui::Event::Touch { .. } => self.diagnostics.touch_events += 1,
                 _ => {}
             }
-            let egui::Event::Touch { phase, force, .. } = e else { continue };
+            let egui::Event::Touch { phase, force, pos, .. } = e else { continue };
             self.last_event = if force.is_some() { "touch with force" } else { "touch without force" };
             if force.is_some() {
                 self.diagnostics.touch_with_force += 1;
@@ -91,6 +95,9 @@ impl Stylus {
             }
             match phase {
                 egui::TouchPhase::Start | egui::TouchPhase::Move => {
+                    if *phase == egui::TouchPhase::Start && reported.is_some() {
+                        self.landed = Some(*pos);
+                    }
                     self.down = true;
                     if self.pen {
                         self.pressure = reported.unwrap_or(0.0);
@@ -139,6 +146,12 @@ impl Stylus {
         (self.highest, self.samples)
     }
 
+    /// Where a pen touched down this frame, in points: only a contact that reports a force as it
+    /// starts, so a mouse or a finger on a touch screen never counts.
+    pub fn landed(&self) -> Option<egui::Pos2> {
+        self.landed
+    }
+
     /// Event counters for the pressure diagnostic canvas and troubleshooting logs.
     pub fn diagnostics(&self) -> Diagnostics {
         self.diagnostics
@@ -181,6 +194,22 @@ mod tests {
         assert_eq!(s.sample(), Sample { pressure: 0.0, pen: true, down: false });
     }
 
+    /// Issue #9: a pen coming down is noticed for the one frame it happens in; a finger with no
+    /// force, or a pen that is already down and moving, is not a pen landing.
+    #[test]
+    fn a_pen_landing_is_seen_for_its_frame_only() {
+        let mut s = Stylus::default();
+        s.update(&[touch(egui::TouchPhase::Start, None)]);
+        assert_eq!(s.landed(), None, "a finger is not a pen");
+        s.update(&[touch(egui::TouchPhase::End, None)]);
+        s.update(&[touch(egui::TouchPhase::Start, Some(0.4))]);
+        assert_eq!(s.landed(), Some(egui::Pos2::ZERO), "a pen came down");
+        s.update(&[touch(egui::TouchPhase::Move, Some(0.6))]);
+        assert_eq!(s.landed(), None, "only the frame it came down in");
+        s.update(&[]);
+        assert_eq!(s.landed(), None);
+    }
+
     #[test]
     fn junk_readings_do_not_get_through() {
         let mut s = Stylus::default();

crates/ui-egui/tests/ui.rs

@@ -9552,6 +9552,90 @@ fn content_wider_than_the_view_gets_a_horizontal_scroll_bar() {
     }
 }
 
+/// Issue #9, OneNote's automatic switching: a pen that comes down on the page while the caret
+/// is the pointer picks up the pen last in hand and draws with it; typing puts it down again and
+/// the words go to the page. A mouse or a finger with no force changes nothing.
+#[test]
+fn a_pen_on_the_page_picks_up_the_last_pen_and_typing_puts_it_down() {
+    let (_t, mut app, s) = sample_app("ui-auto-pen");
+    app.open_path(&s.md_page.to_string_lossy());
+    let mut h = sized_harness(app, LIGHT, egui::vec2(1400.0, 1000.0));
+    settle(&mut h);
+    let ink = |h: &mut Harness<'_, NoteCraftApp>| {
+        let v = ask(h, "ui.inspect", json!({})).expect("inspect");
+        (
+            v.pointer("/ink/pointer").and_then(Value::as_str).unwrap_or_default().to_string(),
+            v.pointer("/ink/nib").and_then(Value::as_u64),
+            v.pointer("/ink/strokes").and_then(Value::as_u64).unwrap_or(0),
+        )
+    };
+    // The red pen was the last one in hand; then back to typing.
+    h.get_by_label("Draw").click();
+    settle(&mut h);
+    h.get_by_label("Pen red").click();
+    settle(&mut h);
+    let red = ink(&mut h).1;
+    h.get_by_label("Type").click();
+    settle(&mut h);
+    assert_eq!(ink(&mut h).0, "write");
+
+    let at = page_origin(&mut h) + egui::vec2(500.0, 420.0);
+    let stroke = |h: &mut Harness<'_, NoteCraftApp>, force: Option<f32>| {
+        let contact = |phase, pos| egui::Event::Touch { device_id: egui::TouchDeviceId(0), id: egui::TouchId(0), phase, pos, force };
+        h.input_mut().events.push(contact(egui::TouchPhase::Start, at));
+        h.input_mut().events.push(egui::Event::PointerMoved(at));
+        h.input_mut().events.push(egui::Event::PointerButton {
+            pos: at,
+            button: egui::PointerButton::Primary,
+            pressed: true,
+            modifiers: egui::Modifiers::NONE,
+        });
+        h.step();
+        for n in 1..=3 {
+            let to = at + egui::vec2(25.0 * n as f32, 6.0 * n as f32);
+            h.input_mut().events.push(contact(egui::TouchPhase::Move, to));
+            h.input_mut().events.push(egui::Event::PointerMoved(to));
+            h.step();
+        }
+        let to = at + egui::vec2(75.0, 18.0);
+        h.input_mut().events.push(contact(egui::TouchPhase::End, to));
+        h.input_mut().events.push(egui::Event::PointerButton {
+            pos: to,
+            button: egui::PointerButton::Primary,
+            pressed: false,
+            modifiers: egui::Modifiers::NONE,
+        });
+        h.step();
+        settle(h);
+    };
+
+    // A finger on a touch screen reports no force: it is not a pen, and nothing is drawn.
+    stroke(&mut h, None);
+    assert_eq!(ink(&mut h), ("write".to_string(), red, 0), "a finger leaves the caret as it was");
+
+    // The caret is in the first paragraph, where the words were last being written.
+    ask(&mut h, "ui.caret", json!({"block": 1})).expect("caret");
+    settle(&mut h);
+
+    // A pen picks up the red pen and the same touch draws.
+    stroke(&mut h, Some(0.6));
+    assert_eq!(ink(&mut h), ("draw".to_string(), red, 1), "the pen drew with the pen last in hand");
+
+    // A shortcut is not typing: the pen stays in hand.
+    ask(&mut h, "ui.key", json!({"key": "b", "ctrl": true})).expect("Ctrl+B");
+    settle(&mut h);
+    assert_eq!(ink(&mut h).0, "draw", "Ctrl+B does not put the pen down");
+
+    // Typing puts it down, and the words reach the page.
+    let before = h.state().body().unwrap_or_default().to_string();
+    h.input_mut().events.push(egui::Event::Text("Z".to_string()));
+    h.step();
+    settle(&mut h);
+    assert_eq!(ink(&mut h).0, "write", "typing goes back to the caret");
+    let after = h.state().body().unwrap_or_default().to_string();
+    assert_eq!(after.matches('Z').count(), before.matches('Z').count() + 1, "the Z went into the page");
+    assert!(after.contains("ZNoteCraft pages"), "where the caret was before the pen: {after}");
+}
 /// Issue #23: Draw > Collect All brings boxes and ink that drifted apart on the canvas back
 /// together, keeps their order and does not lay one over another; one Undo puts them all back.
 #[test]

docs/gaps.md

@@ -393,6 +393,12 @@ Playground hosts with the Files API continue using their existing persistence pa
   sit between. A drawn shape is ink like any other stroke, so the eraser, Select and Undo
   Stroke all work on it; it is not kept as a shape that can be turned back into a perfect
   circle later, and the file does not say which shape it was.
+  The window also switches by itself, as OneNote does: a pen that touches the page while the
+  caret is the pointer picks up the pen last in hand and that touch draws, and typing with an
+  ink tool in hand goes back to the caret, the words landing where it was. Only a contact that
+  reports a force counts as a pen, so this works where the pen's pressure arrives (Windows,
+  the web build); a mouse or a finger changes nothing. **Not built:** fingers panning and
+  pinching the canvas while a pen draws.
   Draw > Collect All gathers boxes and strokes that drifted apart on the open canvas: every
   empty stretch of the page wider than a small gap, across or down, is closed to that gap, so
   things keep their order and nothing ends up on anything it was not already on. Ink over a