feat(controller): new tabs start on the address bar, an arrow-and-hand pointer, a smooth glide

app/src/assets/spatial.js

@@ -254,41 +254,91 @@
 })();
 
 // The controller's pointer: the left stick moves it and A clicks where it points (real mouse events at the same spot,
-// sent by the shell). It is a ring, not an arrow, so it never passes for the system mouse; the dot in the middle is the
-// spot that gets clicked.
+// sent by the shell). An arrow that turns into a pointing hand over anything clickable, like a mouse; white with a black
+// edge and a shadow so it reads on any page. The tip of the arrow and of the finger is the spot that gets clicked.
 (() => {
   if (window.__joyCursor) return;
-  const SIZE = 34;
-  const ring = '<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 34 34">' +
-    '<circle cx="17" cy="17" r="12" fill="rgba(43,149,255,0.22)" stroke="#fff" stroke-width="5"/>' +
-    '<circle cx="17" cy="17" r="12" fill="none" stroke="#2B95FF" stroke-width="3"/>' +
-    '<circle cx="17" cy="17" r="3" fill="#2B95FF" stroke="#fff" stroke-width="1.5"/></svg>';
+  const svg = (box, body) => 'url("data:image/svg+xml;utf8,' + encodeURIComponent(
+    `<svg xmlns="http://www.w3.org/2000/svg" viewBox="${box}">${body}</svg>`) + '")';
+  const SHAPES = {
+    arrow: {
+      width: 24, height: 34, tipX: 2.4, tipY: 2.4,
+      image: svg('0 0 20 28', '<path d="M2 2 L2 22.5 L7.3 17.6 L10.9 25.8 L14.3 24.3 L10.8 16.3 L17.8 16.3 Z" ' +
+        'fill="#fff" stroke="#000" stroke-width="1.6" stroke-linejoin="round"/>'),
+    },
+    hand: {
+      width: 30, height: 35, tipX: 11.5, tipY: 2,
+      image: svg('0 0 26 30', '<path d="M10 1.6c1.1 0 2 .9 2 2v8.6h.6v-1.8c0-1.1.9-2 2-2s2 .9 2 2v2.2h.4v-1.4c0-1.1.9-2 2-2s2 .9 2 2v2h.3' +
+        'c0-1.1.9-2 2-2s2 .9 2 2V19c0 5.2-3.4 9.4-8.8 9.4h-1.7c-3.1 0-5.2-1.5-6.9-3.8L2 17.6c-.7-1-.5-2.4.5-3.1 1-.7 2.4-.5 3.1.5' +
+        'L8 18.4V3.6c0-1.1.9-2 2-2z" fill="#fff" stroke="#000" stroke-width="1.5" stroke-linejoin="round"/>' +
+        '<path d="M12 12.2V16.5M16.6 12.6V16.5M21 13.2V16.5" stroke="#000" stroke-width="1.2" stroke-linecap="round"/>'),
+    },
+  };
+  // What a mouse would show a hand over: links, buttons and the like, and anything a page gives cursor: pointer.
+  const CLICKABLE = 'a[href], button, summary, label, select, [role="button"], [role="link"], [role="tab"], [role="menuitem"], ' +
+    '[role="option"], input[type="submit"], input[type="button"], input[type="checkbox"], input[type="radio"]';
+  const overClickable = (x, y) => {
+    const el = document.elementFromPoint(x, y);
+    if (!el || el.tagName === 'IFRAME') return false;
+    return Boolean(el.closest(CLICKABLE)) || getComputedStyle(el).cursor === 'pointer';
+  };
   let el = null;
+  let shape = '';
   let at = { x: 0, y: 0 };
   const ensure = () => {
     if (el && document.contains(el)) return el;
     el = document.createElement('div');
     el.id = '__joy-cursor';
     el.setAttribute('aria-hidden', 'true');
-    el.style.cssText = `position:fixed;left:0;top:0;width:${SIZE}px;height:${SIZE}px;z-index:2147483647;pointer-events:none;display:none;` +
-      'filter:drop-shadow(0 1px 3px rgba(0,0,0,0.55));transition:scale 90ms ease-out;' +
-      'background:url("data:image/svg+xml;utf8,' + encodeURIComponent(ring) + '") no-repeat center / contain;';
+    el.style.cssText = 'position:fixed;left:0;top:0;z-index:2147483647;pointer-events:none;display:none;will-change:transform;' +
+      'background-repeat:no-repeat;background-size:contain;filter:drop-shadow(0 2px 3px rgba(0,0,0,0.45));' +
+      'transform-origin:var(--tip-x) var(--tip-y);transition:scale 90ms ease-out;';
+    shape = '';
     (document.body || document.documentElement).appendChild(el);
     return el;
   };
+  const setShape = (c, name) => {
+    if (shape === name) return;
+    shape = name;
+    const s = SHAPES[name];
+    c.style.width = `${s.width}px`;
+    c.style.height = `${s.height}px`;
+    c.style.backgroundImage = s.image;
+    c.style.setProperty('--tip-x', `${s.tipX}px`);
+    c.style.setProperty('--tip-y', `${s.tipY}px`);
+  };
+  // Positions arrive from the shell a few dozen times a second, unevenly; the page draws the pointer on every frame of the
+  // screen instead, gliding the rest of the way to the newest one, so it moves smoothly whatever the arrival rate. A click
+  // goes to the newest position, never to the drawn one.
+  let drawn = null;
+  let frame = 0;
+  const paint = () => {
+    frame = 0;
+    if (!el || !drawn) return;
+    drawn.x += (at.x - drawn.x) * 0.5;
+    drawn.y += (at.y - drawn.y) * 0.5;
+    if (Math.abs(at.x - drawn.x) < 0.3 && Math.abs(at.y - drawn.y) < 0.3) drawn = { x: at.x, y: at.y };
+    const s = SHAPES[shape];
+    el.style.transform = `translate(${drawn.x - s.tipX}px, ${drawn.y - s.tipY}px)`;
+    if (drawn.x !== at.x || drawn.y !== at.y) frame = requestAnimationFrame(paint);
+  };
   window.__joyCursor = {
     show(visible, x, y) {
       const c = ensure();
+      const appearing = c.style.display !== 'block';
       at = { x, y };
       c.style.display = visible ? 'block' : 'none';
-      // The click lands on (x, y): the ring is centred on it.
-      c.style.transform = `translate(${x - SIZE / 2}px, ${y - SIZE / 2}px)`;
+      if (!visible) return;
+      setShape(c, overClickable(x, y) ? 'hand' : 'arrow');
+      // Appearing, it shows up where it is rather than sliding in from where it was last seen.
+      if (appearing || !drawn) drawn = { x, y };
+      if (!frame) frame = requestAnimationFrame(paint);
     },
-    // A short squeeze on every click, so a press from the sofa is seen to land.
+    // A short squeeze around the tip on every click, so a press from the sofa is seen to land.
     press() {
       const c = ensure();
       if (c.style.display === 'none') this.show(true, at.x, at.y);
-      c.style.scale = '0.75';
+      c.style.scale = '0.82';
       setTimeout(() => { c.style.scale = '1'; }, 110);
     },
   };

app/src/main/controllerActions.ts

@@ -64,6 +64,8 @@ export class VirtualCursor {
   x = 200;
   y = 200;
   placed = false;
+  private drawing = false;
+  private redraw = false;
 
   place(x: number, y: number, width: number, height: number): void {
     this.x = Math.min(Math.max(0, x), Math.max(0, width - 1));
@@ -78,7 +80,28 @@ export class VirtualCursor {
   async moveBy(wc: WebContents, dx: number, dy: number, width: number, height: number): Promise<void> {
     this.place(this.x + dx, this.y + dy, width, height);
     wc.sendInputEvent({ type: "mouseMove", x: Math.round(this.x), y: Math.round(this.y) });
-    await this.show(wc, true);
+    await this.draw(wc);
+  }
+
+  /**
+   * Draws the pointer where it is now. Moves arrive about 60 times a second; while one drawing is still on its way into
+   * the page, the next ones only ask for another, which then uses the latest position: the pointer never lags behind a
+   * queue of old ones.
+   */
+  private async draw(wc: WebContents): Promise<void> {
+    if (this.drawing) {
+      this.redraw = true;
+      return;
+    }
+    this.drawing = true;
+    try {
+      do {
+        this.redraw = false;
+        await this.show(wc, true);
+      } while (this.redraw);
+    } finally {
+      this.drawing = false;
+    }
   }
 
   click(wc: WebContents, button: "left" | "right" = "left"): void {

app/src/main/services/navigation.ts

@@ -8,7 +8,8 @@ export interface NavigationHost {
   readonly tabs: TabManager;
   readonly settings: () => AppSettings;
   readonly httpAllowedHosts: Set<string>;
-  openKeyboard(target: "address" | "page"): void;
+  /** The address field takes the focus with the controller keyboard open on it: where a new tab starts. */
+  typeInAddress(): void;
   closeKeyboard(): void;
 }
 
@@ -39,6 +40,7 @@ export class Navigation {
   newTab(isPrivate = false): void {
     const target = isPrivate ? Pages.newtab : newTabTarget(this.host.settings());
     this.host.tabs.open(target, { isPrivate });
-    if (target !== Pages.newtab) this.host.openKeyboard("address");
+    // A new tab is for going somewhere: it opens on the address field with the keyboard up, whatever page it shows.
+    this.host.typeInAddress();
   }
 }

app/src/main/shell.ts

@@ -115,7 +115,7 @@ export class Shell {
       tabs: this.tabs,
       settings: () => this.deps.store.current,
       httpAllowedHosts: this.httpAllowedHosts,
-      openKeyboard: (target) => this.openKeyboard(target),
+      typeInAddress: () => this.typeInAddress(),
       closeKeyboard: () => this.closeKeyboard(),
     });
     this.wireTabs();
@@ -313,6 +313,17 @@ export class Shell {
       if (!this.wired.has(tab.webContents)) {
         this.wired.add(tab.webContents);
         this.attachShortcuts(tab.webContents);
+        // A view takes the window's focus as it is attached (a new tab, about 10 ms after the shell gave its address field the
+        // focus). While the keyboard types into that field the focus stays with the shell, or the D-pad walks dead keys.
+        // The tab hears of its focus before the shell hears it lost it, so the focus goes back once the swap is over.
+        tab.webContents.on("focus", () => {
+          if (this.keyboard?.target !== "address") return;
+          setImmediate(() => {
+            if (this.keyboard?.target !== "address" || this.window.isDestroyed()) return;
+            this.window.webContents.focus();
+            this.chromeFocused = true;
+          });
+        });
       }
       if (!contentView.children.includes(tab.view)) contentView.addChildView(tab.view);
       tab.view.setVisible(tab === this.tabs.active && !this.pageCovered);
@@ -563,6 +574,12 @@ export class Shell {
     this.window.webContents.send(Channels.shellFocusChrome);
   }
 
+  /** The toolbar takes the D-pad (see enterChrome) and the keyboard opens on the address field. */
+  typeInAddress(): void {
+    this.enterChrome();
+    this.openKeyboard("address");
+  }
+
   leaveChrome(): void {
     this.chromeFocused = false;
     this.tabs.activeWebContents?.focus();

app/src/renderer/shell/app.tsx

@@ -236,13 +236,17 @@ function AddressBar({ url, focused, setFocused, language, keyboard, children }:
   const fresh = keyboard && isInternal(url);
   const shown = fresh ? "" : url;
   const [value, setValue] = useState(shown);
-  useEffect(() => { if (!focused) setValue(shown); }, [shown, focused]);
+  // The field follows the tab's address until the user types in it: a new tab focuses the field, and a page that loads
+  // meanwhile must show its address, but text being typed is never overwritten.
+  const edited = useRef(false);
+  useEffect(() => { if (!focused || !edited.current) setValue(shown); }, [shown, focused]);
 
   const submit = () => cmd({ type: "search-or-navigate", text: value });
   return (
     <form
       class="address"
-      onSubmit={(e) => { e.preventDefault(); submit(); setFocused(false); (input.current as HTMLInputElement | null)?.blur(); }}
+      // Going somewhere hands the focus to the page that loads, as in Edge; the D-pad then walks the page, not the bar.
+      onSubmit={(e) => { e.preventDefault(); submit(); setFocused(false); (input.current as HTMLInputElement | null)?.blur(); cmd({ type: "focus-page" }); }}
     >
       <SearchIcon />
       <input
@@ -256,7 +260,7 @@ function AddressBar({ url, focused, setFocused, language, keyboard, children }:
         // A click (or A on the field from the D-pad, see gamepad.ts) opens the controller keyboard; focus alone does not, or
         // walking the toolbar with the D-pad would pop the keyboard up on the way past.
         onMouseDown={() => cmd({ type: "keyboard-open", target: "address" })}
-        onFocus={() => { setFocused(true); (input.current as HTMLInputElement).select(); }}
+        onFocus={() => { setFocused(true); edited.current = false; (input.current as HTMLInputElement).select(); }}
         onBlur={() => setFocused(false)}
         onKeyDown={(e) => {
           // Down leaves the field for the suggestions the controller keyboard shows, so both inputs reach them.
@@ -268,6 +272,7 @@ function AddressBar({ url, focused, setFocused, language, keyboard, children }:
         }}
         onInput={(e) => {
           const text = (e.target as HTMLInputElement).value;
+          edited.current = true;
           setValue(text);
           cmd({ type: "address-input", text });
         }}

app/src/renderer/shell/gamepad.ts

@@ -34,6 +34,7 @@ export function useGamepad(state: ShellState, overlay: Overlay, setOverlay: (o:
   const family = useRef<ControllerFamily | undefined>(undefined);
   const lastDirection = useRef(0);
   const lastStick = useRef(0);
+  const lastPointerMove = useRef(0);
   /**
    * Who A answers to on a page: the stick's pointer once the stick moved it, the D-pad's focus once the D-pad moved.
    * The last one used wins, so both can be mixed on the same page.
@@ -168,14 +169,20 @@ export function useGamepad(state: ShellState, overlay: Overlay, setOverlay: (o:
       const page = !overlayOpen && !chrome;
       const pointer = page && s.inputMode !== "Arrows";
 
-      // Buttons follow the poll rate; the sticks keep their own, slower cadence so the pointer moves at the same speed.
-      if (page && now - lastStick.current >= STICK_INTERVAL_MS) {
-        lastStick.current = now;
-        const velocity = pointer ? stickToVelocity(lx, ly) : { dx: 0, dy: 0 };
+      // The pointer moves on every poll, so it glides at the screen's rate instead of jumping 20 times a second; the step
+      // is scaled by the time since the last one, so the speed (set per STICK_INTERVAL_MS) does not depend on the rate.
+      const sinceMove = Math.min(now - lastPointerMove.current, STICK_INTERVAL_MS) / STICK_INTERVAL_MS;
+      lastPointerMove.current = now;
+      if (pointer) {
+        const velocity = stickToVelocity(lx, ly);
         if (velocity.dx !== 0 || velocity.dy !== 0) {
           pointing.current = true;
-          bridge().controller({ type: "cursor-move", dx: velocity.dx, dy: velocity.dy });
+          bridge().controller({ type: "cursor-move", dx: velocity.dx * sinceMove, dy: velocity.dy * sinceMove });
         }
+      }
+      // The wheel keeps the slower cadence: one notch per STICK_INTERVAL_MS is a readable scroll speed.
+      if (page && now - lastStick.current >= STICK_INTERVAL_MS) {
+        lastStick.current = now;
         const notches = stickToScroll(ry);
         const sideways = stickToScroll(rx);
         if (notches !== 0 || sideways !== 0) bridge().controller({ type: "scroll", notches, sideways });

app/tests/e2e/controller.spec.ts

@@ -320,3 +320,34 @@ test("the keyboard's own shortcuts: Y space, LT shift, X delete, LS the layer, a
   await step(10);
   await expect.poll(layerKey).toBe("abc");
 });
+
+test("a new tab opens on the address field with the keyboard up, and the D-pad types there", async () => {
+  await installPads([XBOX]);
+  await app.evaluate(() => globalThis.joy.command({ type: "new-tab" } as never));
+  await expect(chrome.locator("#keyboard")).toBeVisible();
+  await expect.poll(shellFocus).toBe("address");
+  await expect.poll(shellHasWindowFocus).toBe(true);
+  // The first press goes into the keys, at "q"; A types it into the address.
+  await step(Pad.Right);
+  await expect.poll(shellFocus).toBe("q");
+  await step(Pad.A);
+  await expect.poll(() => chrome.locator("#address").inputValue()).toBe("q");
+});
+
+test("the pointer is an arrow, and a hand over what can be clicked", async () => {
+  await app.evaluate(({ protocol }, page) => {
+    protocol.handle("https", () => new Response(page, { headers: { "content-type": "text/html" } }));
+  }, POINTER_PAGE);
+  await app.evaluate(() => globalThis.joy.command({ type: "navigate", url: "https://pointer.example/" } as never));
+  await expect.poll(async () => {
+    const tab = (await tabs()).find((t) => t.isActive);
+    return tab?.url === "https://pointer.example/" && !tab.isLoading;
+  }, { timeout: 30_000 }).toBe(true);
+  const shapeAt = async (where: string) => {
+    await app.evaluate(({}, code) => globalThis.joy.spatialEval(code), `(() => { const r = ${where}.getBoundingClientRect(); window.__joyCursor.show(true, r.left + r.width / 2, r.top + r.height / 2); })()`);
+    // The hand is the shape with fingers: its path starts at the fingertip.
+    return app.evaluate(() => globalThis.joy.pageEval("document.getElementById('__joy-cursor').style.backgroundImage.includes('M10%201.6') ? 'hand' : 'arrow'"));
+  };
+  expect(await shapeAt("document.getElementById('area')")).toBe("arrow");
+  expect(await shapeAt("document.getElementById('b')")).toBe("hand");
+});

app/tests/e2e/journeys.spec.ts

@@ -444,6 +444,11 @@ async function quickMenu(h: Harness, label: RegExp): Promise<void> {
   await focusOn(h, "shell", `[...document.querySelectorAll('.menu .menu-item')].find((b) => ${label.toString()}.test((b.textContent || '').trim()))`, `menu item ${label}`);
   await press(h, Pad.A);
   await expect(h.chrome.locator(".overlay .menu")).toHaveCount(0);
+  // A new tab opens on the address field with the keyboard up; B puts it away to go on with the pad.
+  if (await h.chrome.locator("#keyboard").count()) {
+    await press(h, Pad.B);
+    await expect(h.chrome.locator("#keyboard")).toHaveCount(0);
+  }
 }
 
 test("three tabs, switching in the tab switcher and closing one, all from the pad", async () => {