feat(controller): a pointer on the left stick, clicks with A, the wheel on the right stick

app/src/assets/spatial.js

@@ -253,25 +253,43 @@
   });
 })();
 
-// Virtual cursor overlay (Cursor input mode). The shell moves it and sends real mouse events at the same point.
+// The controller's pointer: the left stick moves it and A clicks where it points (real mouse events at the same spot,
+// sent by the shell). It is a ring, not an arrow, so it never passes for the system mouse; the dot in the middle is the
+// spot that gets clicked.
 (() => {
   if (window.__joyCursor) return;
+  const SIZE = 34;
+  const ring = '<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 34 34">' +
+    '<circle cx="17" cy="17" r="12" fill="rgba(43,149,255,0.22)" stroke="#fff" stroke-width="5"/>' +
+    '<circle cx="17" cy="17" r="12" fill="none" stroke="#2B95FF" stroke-width="3"/>' +
+    '<circle cx="17" cy="17" r="3" fill="#2B95FF" stroke="#fff" stroke-width="1.5"/></svg>';
   let el = null;
+  let at = { x: 0, y: 0 };
   const ensure = () => {
     if (el && document.contains(el)) return el;
     el = document.createElement('div');
     el.id = '__joy-cursor';
     el.setAttribute('aria-hidden', 'true');
-    el.style.cssText = 'position:fixed;left:0;top:0;width:18px;height:26px;z-index:2147483647;pointer-events:none;display:none;' +
-      'background:url("data:image/svg+xml;utf8,' + encodeURIComponent('<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 18 28"><path d="M1 1 L1 23 L7 18 L11 27 L14 26 L10 17 L17 17 Z" fill="#2B95FF" stroke="#0A0F1A" stroke-width="1.5"/></svg>') + '") no-repeat;';
+    el.style.cssText = `position:fixed;left:0;top:0;width:${SIZE}px;height:${SIZE}px;z-index:2147483647;pointer-events:none;display:none;` +
+      'filter:drop-shadow(0 1px 3px rgba(0,0,0,0.55));transition:scale 90ms ease-out;' +
+      'background:url("data:image/svg+xml;utf8,' + encodeURIComponent(ring) + '") no-repeat center / contain;';
     (document.body || document.documentElement).appendChild(el);
     return el;
   };
   window.__joyCursor = {
     show(visible, x, y) {
       const c = ensure();
+      at = { x, y };
       c.style.display = visible ? 'block' : 'none';
-      c.style.transform = `translate(${x}px, ${y}px)`;
+      // The click lands on (x, y): the ring is centred on it.
+      c.style.transform = `translate(${x - SIZE / 2}px, ${y - SIZE / 2}px)`;
+    },
+    // A short squeeze on every click, so a press from the sofa is seen to land.
+    press() {
+      const c = ensure();
+      if (c.style.display === 'none') this.show(true, at.x, at.y);
+      c.style.scale = '0.75';
+      setTimeout(() => { c.style.scale = '1'; }, 110);
     },
   };
 })();

app/src/main/controllerActions.ts

@@ -81,16 +81,20 @@ export class VirtualCursor {
     await this.show(wc, true);
   }
 
-  click(wc: WebContents): void {
+  click(wc: WebContents, button: "left" | "right" = "left"): void {
     const x = Math.round(this.x);
     const y = Math.round(this.y);
     wc.sendInputEvent({ type: "mouseMove", x, y });
-    wc.sendInputEvent({ type: "mouseDown", x, y, button: "left", clickCount: 1 });
-    wc.sendInputEvent({ type: "mouseUp", x, y, button: "left", clickCount: 1 });
+    wc.sendInputEvent({ type: "mouseDown", x, y, button, clickCount: 1 });
+    wc.sendInputEvent({ type: "mouseUp", x, y, button, clickCount: 1 });
+    void runInPageWorld(wc, "window.__joyCursor && window.__joyCursor.press()");
   }
 
-  /** Positive notches scroll down (Gamepad API stick Y grows downward); wheel deltaY is negative for down. */
-  scroll(wc: WebContents, notches: number): void {
-    wc.sendInputEvent({ type: "mouseWheel", x: Math.round(this.x), y: Math.round(this.y), deltaX: 0, deltaY: -notches * 120 });
+  /**
+   * Positive notches scroll down (Gamepad API stick Y grows downward) and positive `sideways` to the right; wheel deltas
+   * are negative for those. The wheel turns under the pointer, so whatever box it is over scrolls, like a real mouse.
+   */
+  scroll(wc: WebContents, notches: number, sideways = 0): void {
+    wc.sendInputEvent({ type: "mouseWheel", x: Math.round(this.x), y: Math.round(this.y), deltaX: -sideways * 120, deltaY: -notches * 120 });
   }
 }

app/src/main/services/inputRouter.ts

@@ -52,14 +52,24 @@ export class InputRouter {
           if ((await spatialActivate(wc)) === "edit") this.host.openKeyboard("page");
         } else sendKey(wc, "Return");
         break;
+      // The stick's pointer works in every mode but Arrows, whose sites (YouTube TV, Twitch) are built for the arrow keys.
       case "cursor-move":
-        if (wc && this.host.inputMode === "Cursor") {
+        if (wc && this.host.inputMode !== "Arrows") {
           const size = this.host.pageSize();
           await this.cursor.moveBy(wc, input.dx, input.dy, size.width, size.height);
         }
         break;
-      case "cursor-click": if (wc && this.host.inputMode === "Cursor") this.cursor.click(wc); break;
-      case "scroll": if (wc) this.cursor.scroll(wc, input.notches); break;
+      case "cursor-click": if (wc && this.host.inputMode !== "Arrows") this.cursor.click(wc, input.button); break;
+      case "cursor-hide": if (wc) await this.cursor.show(wc, false); break;
+      case "scroll":
+        if (!wc) return;
+        // Never pointed anywhere yet: the wheel turns over the middle of the page, not over a corner.
+        if (!this.cursor.placed) {
+          const size = this.host.pageSize();
+          this.cursor.place(size.width / 2, size.height / 2, size.width, size.height);
+        }
+        this.cursor.scroll(wc, input.notches, input.sideways);
+        break;
     }
   }
 }

app/src/renderer/shell/gamepad.ts

@@ -1,6 +1,6 @@
 import { useEffect, useRef } from "preact/hooks";
 import { controllerFamily, type ControllerFamily } from "../../shared/controllerGlyphs";
-import { Button, directionOf, stickToScroll, stickToVelocity, type Direction } from "../../shared/controllerInput";
+import { Button, DOUBLE_PRESS_MS, directionOf, stickToScroll, stickToVelocity, type Direction } from "../../shared/controllerInput";
 import { activePad, padsInUse, startedUsing, type PadSample } from "../../shared/controllerSelection";
 import { GAMEPAD_POLL_MS, gamepadPoll, STICK_INTERVAL_MS } from "../../shared/gamepadLoop";
 import type { ShellState } from "../../shared/ipc";
@@ -29,6 +29,13 @@ export function useGamepad(state: ShellState, overlay: Overlay, setOverlay: (o:
   const family = useRef<ControllerFamily | undefined>(undefined);
   const lastDirection = useRef(0);
   const lastStick = useRef(0);
+  /**
+   * Who A answers to on a page: the stick's pointer once the stick moved it, the D-pad's focus once the D-pad moved.
+   * The last one used wins, so both can be mixed on the same page.
+   */
+  const pointing = useRef(false);
+  /** A single A at the pointer waits here for a possible second one, which would make it a right click. */
+  const pendingClick = useRef<number | undefined>(undefined);
   const lastTick = useRef(0);
   const stateRef = useRef(state);
   stateRef.current = state;
@@ -117,6 +124,7 @@ export function useGamepad(state: ShellState, overlay: Overlay, setOverlay: (o:
       if (justPressed(Button.A)) {
         if (openOverlay !== "none" || s.permission || s.keyboard) (document.activeElement as HTMLElement | null)?.click();
         else if (chrome) activateChrome();
+        else if (pointing.current || s.inputMode === "Cursor") clickAtPointer();
         else bridge().controller({ type: "activate" });
         vibrate("confirm");
         return;
@@ -134,28 +142,56 @@ export function useGamepad(state: ShellState, overlay: Overlay, setOverlay: (o:
 
       const lx = pad.axes[0] ?? 0;
       const ly = pad.axes[1] ?? 0;
+      const rx = pad.axes[2] ?? 0;
       const ry = pad.axes[3] ?? 0;
       const dpad = { up: down(Button.DpadUp), down: down(Button.DpadDown), left: down(Button.DpadLeft), right: down(Button.DpadRight) };
-      const cursorMode = s.inputMode === "Cursor" && !overlayOpen;
+      // On a page the left stick is the pointer and the right stick the wheel. Arrows mode keeps the stick on the arrow
+      // keys its sites are built for; a dialog, the keyboard or the toolbar take the stick as a D-pad.
+      const page = !overlayOpen && !chrome;
+      const pointer = page && s.inputMode !== "Arrows";
 
-      // Buttons follow the poll rate; the stick keeps its own, slower cadence so the cursor moves at the same speed.
-      if (cursorMode && now - lastStick.current >= STICK_INTERVAL_MS) {
+      // Buttons follow the poll rate; the sticks keep their own, slower cadence so the pointer moves at the same speed.
+      if (page && now - lastStick.current >= STICK_INTERVAL_MS) {
         lastStick.current = now;
-        const velocity = stickToVelocity(lx, ly);
-        if (velocity.dx !== 0 || velocity.dy !== 0) bridge().controller({ type: "cursor-move", dx: velocity.dx, dy: velocity.dy });
+        const velocity = pointer ? stickToVelocity(lx, ly) : { dx: 0, dy: 0 };
+        if (velocity.dx !== 0 || velocity.dy !== 0) {
+          pointing.current = true;
+          bridge().controller({ type: "cursor-move", dx: velocity.dx, dy: velocity.dy });
+        }
         const notches = stickToScroll(ry);
-        if (notches !== 0) bridge().controller({ type: "scroll", notches });
+        const sideways = stickToScroll(rx);
+        if (notches !== 0 || sideways !== 0) bridge().controller({ type: "scroll", notches, sideways });
       }
 
-      const direction: Direction | undefined = cursorMode ? directionOf(dpad, 0, 0) : directionOf(dpad, lx, ly);
+      const direction: Direction | undefined = directionOf(dpad, pointer ? 0 : lx, pointer ? 0 : ly);
       if (direction && now - lastDirection.current >= 180) {
         lastDirection.current = now;
         if (moveSuggestionFocus(direction)) { /* the D-pad walked the address suggestions */ }
         else if (openOverlay !== "none" || s.permission || s.keyboard) moveOverlayFocus(direction);
         // Down with nothing under the toolbar goes back into the page.
         else if (chrome) { if (!moveFocusAmong(chromeControls(), direction) && direction === "down") leaveChrome(); }
-        else bridge().controller({ type: "direction", direction });
+        else {
+          // The D-pad takes over from the pointer (in Cursor mode the D-pad steps the pointer itself, so it stays).
+          if (pointing.current && s.inputMode !== "Cursor") {
+            pointing.current = false;
+            bridge().controller({ type: "cursor-hide" });
+          }
+          bridge().controller({ type: "direction", direction });
+        }
+      }
+    }
+    /** A at the pointer: the second press within DOUBLE_PRESS_MS turns the pending left click into a right click. */
+    function clickAtPointer(): void {
+      if (pendingClick.current !== undefined) {
+        clearTimeout(pendingClick.current);
+        pendingClick.current = undefined;
+        bridge().controller({ type: "cursor-click", button: "right" });
+        return;
       }
+      pendingClick.current = window.setTimeout(() => {
+        pendingClick.current = undefined;
+        bridge().controller({ type: "cursor-click", button: "left" });
+      }, DOUBLE_PRESS_MS);
     }
     // Up from the top of the page: the main process hands the focus over and the address field takes it.
     bridge().onFocusChrome(() => document.getElementById("address")?.focus());

app/src/shared/controllerInput.ts

@@ -5,6 +5,8 @@ import type { InputMode } from "./settings";
 export const STICK_DEADZONE = 0.25;
 export const DIRECTION_THRESHOLD = 0.5;
 export const MAX_CURSOR_SPEED = 22; // px per 50 ms tick at full deflection
+/** Two A presses closer than this are a right click at the pointer; a single press waits this long to be a left click. */
+export const DOUBLE_PRESS_MS = 280;
 
 /** Hosts whose pages already implement arrow-key navigation (10-foot UIs). */
 const ARROW_HOSTS = ["youtube.com", "tv.youtube.com", "twitch.tv"];

app/src/shared/ipc.ts

@@ -140,6 +140,10 @@ export type ControllerEvent =
   | { type: "direction"; direction: "up" | "down" | "left" | "right" }
   | { type: "activate" }
   | { type: "cursor-move"; dx: number; dy: number }
-  | { type: "cursor-click" }
-  | { type: "scroll"; notches: number }
+  /** A at the pointer: a left click, or a right click when A was pressed twice in a row. */
+  | { type: "cursor-click"; button?: "left" | "right" }
+  /** The D-pad took over: the pointer goes away until the stick moves it again. */
+  | { type: "cursor-hide" }
+  /** The right stick: wheel notches down (positive) and to the right (`sideways`, positive). */
+  | { type: "scroll"; notches: number; sideways?: number }
   | { type: "connected"; connected: boolean; family?: "xbox" | "playstation" | "switch" | "generic" };

app/tests/e2e/controller.spec.ts

@@ -234,3 +234,58 @@ test("up from the top of the page reaches the toolbar and the tabs, and down com
   await expect.poll(shellHasWindowFocus).toBe(false);
   expect(await chrome.evaluate(() => document.activeElement === document.body)).toBe(true);
 });
+
+// ---- The stick's pointer, its clicks and the right stick's wheel ----
+
+/** A focusable button on top and, under it, a tall area that logs every mouse press and context menu it gets. */
+const POINTER_PAGE = `<title>Pointer</title><body style="margin:0;font:20px sans-serif">` +
+  `<button id="b" style="margin:20px" onclick="window.__log.push('button')">Focus me</button>` +
+  `<div id="area" style="height:4000px;background:#eee"></div>` +
+  `<script>window.__log = [];` +
+  `addEventListener('mousedown', (e) => { if (e.target.id === 'area') __log.push('down' + e.button); });` +
+  `addEventListener('contextmenu', (e) => { e.preventDefault(); __log.push('context'); });</script></body>`;
+
+const tilt = (axis: number, value: number) => chrome.evaluate(([a, v]) => {
+  (window as unknown as { __pads: { axes: number[] }[] }).__pads[0]!.axes[a] = v;
+}, [axis, value] as const);
+const pageLog = () => app.evaluate(() => globalThis.joy.pageEval("window.__log.join(',')")) as Promise<string>;
+const pointerShown = () => app.evaluate(() => globalThis.joy.pageEval("(document.getElementById('__joy-cursor') || {}).style?.display === 'block'"));
+
+test("the left stick points and A clicks there, twice for a right click, until the D-pad takes over", async () => {
+  await installPads([XBOX]);
+  await app.evaluate(({ protocol }, page) => {
+    protocol.handle("https", () => new Response(page, { headers: { "content-type": "text/html" } }));
+  }, POINTER_PAGE);
+  await app.evaluate(() => globalThis.joy.command({ type: "navigate", url: "https://pointer.example/" } as never));
+  await expect.poll(async () => {
+    const tab = (await tabs()).find((t) => t.isActive);
+    return tab?.url === "https://pointer.example/" && !tab.isLoading;
+  }, { timeout: 30_000 }).toBe(true);
+
+  // Spatial is the default mode: the stick still brings the pointer up, over the area.
+  await tilt(1, 1);
+  await expect.poll(pointerShown, { timeout: 10_000 }).toBe(true);
+  await chrome.waitForTimeout(300);
+  await tilt(1, 0);
+
+  // One A is a left click where the pointer is (after the short wait for a second press).
+  await press(0, Pad.A);
+  await expect.poll(pageLog, { timeout: 5_000 }).toBe("down0");
+  // Two quick ones are a right click, and no left click on the way.
+  await press(0, Pad.A);
+  await press(0, Pad.A);
+  await expect.poll(pageLog, { timeout: 5_000 }).toBe("down0,down2,context");
+
+  // The D-pad takes over: the pointer goes away and A presses what the D-pad focused, not the spot under the pointer.
+  await step(Pad.Down);
+  await expect.poll(pageFocus).toBe("b");
+  await expect.poll(pointerShown).toBe(false);
+  await step(Pad.A);
+  await expect.poll(pageLog).toBe("down0,down2,context,button");
+
+  // The right stick is the wheel.
+  expect(await app.evaluate(() => globalThis.joy.pageEval("scrollY"))).toBe(0);
+  await tilt(3, 1);
+  await expect.poll(() => app.evaluate(() => globalThis.joy.pageEval("scrollY")) as Promise<number>, { timeout: 10_000 }).toBeGreaterThan(0);
+  await tilt(3, 0);
+});

docs/CONTROLLER-TEST.md

@@ -19,9 +19,10 @@ Para cada cenário: **passou / falhou**, quantos segundos demorou a primeira res
 4. **Três abas.** Abra três abas, alterne pelo seletor em grade e feche uma, só com o controle. Veja se as miniaturas aparecem e se o foco começa na aba ativa.
 5. **Campo de texto com A.** Numa página com um campo (a caixa de busca da nova aba, ou a do YouTube), leve o D-pad até o campo: o teclado **não** deve abrir só de passar por ele. Aperte A: o teclado abre, o D-pad e o analógico andam pelas teclas a partir da primeira, e A escreve no campo. Feche com B e repita: a segunda vez tem de funcionar igual à primeira.
 6. **Subir até a barra.** No topo de qualquer página, aperte para cima: o foco vai para a barra de endereço (sem abrir o teclado). Ande para os lados até voltar, avançar e recarregar, suba até as abas e o +, e aperte A em cada um. Para baixo (ou B) volta para a página. Os botões de minimizar, maximizar e fechar ficam de fora de propósito.
-7. **Permissão de microfone.** Abra um site que peça o microfone (ou o teste do WebRTC), percorra o prompt com o D-pad e escolha "permitir sempre". Confirme em Configurações que a permissão ficou salva para o site.
-8. **Atualização.** Em Windows/Linux: com uma versão nova publicada, deixe o download acontecer, feche o app e reabra; no `.deb`/`.rpm` use **Reiniciar e instalar** em Configurações → Sobre e confirme o pedido de senha. No macOS, o app deve avisar e abrir a página da release.
-9. **Streaming com DRM.** Abra a Netflix, entre na conta e toque um título. Registre a resolução que o player mostra e o que a linha "Widevine (DRM)" diz em Configurações → Sobre.
+7. **Cursor no analógico.** Numa página qualquer, mexa o analógico esquerdo: aparece um anel azul (o cursor do navegador, diferente da seta do mouse) e passa por cima dos itens. A clica onde ele aponta; A duas vezes rápido é o clique direito. Aperte o D-pad: o anel some e o A volta a apertar o item com foco. O analógico direito rola a página (para os lados também). Anote se a velocidade do cursor está boa da distância do sofá.
+8. **Permissão de microfone.** Abra um site que peça o microfone (ou o teste do WebRTC), percorra o prompt com o D-pad e escolha "permitir sempre". Confirme em Configurações que a permissão ficou salva para o site.
+9. **Atualização.** Em Windows/Linux: com uma versão nova publicada, deixe o download acontecer, feche o app e reabra; no `.deb`/`.rpm` use **Reiniciar e instalar** em Configurações → Sobre e confirme o pedido de senha. No macOS, o app deve avisar e abrir a página da release.
+10. **Streaming com DRM.** Abra a Netflix, entre na conta e toque um título. Registre a resolução que o player mostra e o que a linha "Widevine (DRM)" diz em Configurações → Sobre.
 
 Leitor de tela é o outro lado da mesma sessão: o roteiro de Orca, Narrator e VoiceOver está em [SCREEN-READER.md](SCREEN-READER.md).