diff --git a/app/src/assets/spatial.js b/app/src/assets/spatial.js index 8ffb4e7..d5f08ed 100644 --- a/app/src/assets/spatial.js +++ b/app/src/assets/spatial.js @@ -253,25 +253,43 @@ }); })(); -// Virtual cursor overlay (Cursor input mode). The shell moves it and sends real mouse events at the same point. +// The controller's pointer: the left stick moves it and A clicks where it points (real mouse events at the same spot, +// sent by the shell). It is a ring, not an arrow, so it never passes for the system mouse; the dot in the middle is the +// spot that gets clicked. (() => { if (window.__joyCursor) return; + const SIZE = 34; + const ring = '' + + '' + + '' + + ''; let el = null; + let at = { x: 0, y: 0 }; const ensure = () => { if (el && document.contains(el)) return el; el = document.createElement('div'); el.id = '__joy-cursor'; el.setAttribute('aria-hidden', 'true'); - el.style.cssText = 'position:fixed;left:0;top:0;width:18px;height:26px;z-index:2147483647;pointer-events:none;display:none;' + - 'background:url("data:image/svg+xml;utf8,' + encodeURIComponent('') + '") no-repeat;'; + el.style.cssText = `position:fixed;left:0;top:0;width:${SIZE}px;height:${SIZE}px;z-index:2147483647;pointer-events:none;display:none;` + + 'filter:drop-shadow(0 1px 3px rgba(0,0,0,0.55));transition:scale 90ms ease-out;' + + 'background:url("data:image/svg+xml;utf8,' + encodeURIComponent(ring) + '") no-repeat center / contain;'; (document.body || document.documentElement).appendChild(el); return el; }; window.__joyCursor = { show(visible, x, y) { const c = ensure(); + at = { x, y }; c.style.display = visible ? 'block' : 'none'; - c.style.transform = `translate(${x}px, ${y}px)`; + // The click lands on (x, y): the ring is centred on it. + c.style.transform = `translate(${x - SIZE / 2}px, ${y - SIZE / 2}px)`; + }, + // A short squeeze on every click, so a press from the sofa is seen to land. + press() { + const c = ensure(); + if (c.style.display === 'none') this.show(true, at.x, at.y); + c.style.scale = '0.75'; + setTimeout(() => { c.style.scale = '1'; }, 110); }, }; })(); diff --git a/app/src/main/controllerActions.ts b/app/src/main/controllerActions.ts index a111091..de0ec86 100644 --- a/app/src/main/controllerActions.ts +++ b/app/src/main/controllerActions.ts @@ -81,16 +81,20 @@ export class VirtualCursor { await this.show(wc, true); } - click(wc: WebContents): void { + click(wc: WebContents, button: "left" | "right" = "left"): void { const x = Math.round(this.x); const y = Math.round(this.y); wc.sendInputEvent({ type: "mouseMove", x, y }); - wc.sendInputEvent({ type: "mouseDown", x, y, button: "left", clickCount: 1 }); - wc.sendInputEvent({ type: "mouseUp", x, y, button: "left", clickCount: 1 }); + wc.sendInputEvent({ type: "mouseDown", x, y, button, clickCount: 1 }); + wc.sendInputEvent({ type: "mouseUp", x, y, button, clickCount: 1 }); + void runInPageWorld(wc, "window.__joyCursor && window.__joyCursor.press()"); } - /** Positive notches scroll down (Gamepad API stick Y grows downward); wheel deltaY is negative for down. */ - scroll(wc: WebContents, notches: number): void { - wc.sendInputEvent({ type: "mouseWheel", x: Math.round(this.x), y: Math.round(this.y), deltaX: 0, deltaY: -notches * 120 }); + /** + * Positive notches scroll down (Gamepad API stick Y grows downward) and positive `sideways` to the right; wheel deltas + * are negative for those. The wheel turns under the pointer, so whatever box it is over scrolls, like a real mouse. + */ + scroll(wc: WebContents, notches: number, sideways = 0): void { + wc.sendInputEvent({ type: "mouseWheel", x: Math.round(this.x), y: Math.round(this.y), deltaX: -sideways * 120, deltaY: -notches * 120 }); } } diff --git a/app/src/main/services/inputRouter.ts b/app/src/main/services/inputRouter.ts index 6c320f6..df6e5eb 100644 --- a/app/src/main/services/inputRouter.ts +++ b/app/src/main/services/inputRouter.ts @@ -52,14 +52,24 @@ export class InputRouter { if ((await spatialActivate(wc)) === "edit") this.host.openKeyboard("page"); } else sendKey(wc, "Return"); break; + // The stick's pointer works in every mode but Arrows, whose sites (YouTube TV, Twitch) are built for the arrow keys. case "cursor-move": - if (wc && this.host.inputMode === "Cursor") { + if (wc && this.host.inputMode !== "Arrows") { const size = this.host.pageSize(); await this.cursor.moveBy(wc, input.dx, input.dy, size.width, size.height); } break; - case "cursor-click": if (wc && this.host.inputMode === "Cursor") this.cursor.click(wc); break; - case "scroll": if (wc) this.cursor.scroll(wc, input.notches); break; + case "cursor-click": if (wc && this.host.inputMode !== "Arrows") this.cursor.click(wc, input.button); break; + case "cursor-hide": if (wc) await this.cursor.show(wc, false); break; + case "scroll": + if (!wc) return; + // Never pointed anywhere yet: the wheel turns over the middle of the page, not over a corner. + if (!this.cursor.placed) { + const size = this.host.pageSize(); + this.cursor.place(size.width / 2, size.height / 2, size.width, size.height); + } + this.cursor.scroll(wc, input.notches, input.sideways); + break; } } } diff --git a/app/src/renderer/shell/gamepad.ts b/app/src/renderer/shell/gamepad.ts index 8b4bdfb..b4f512d 100644 --- a/app/src/renderer/shell/gamepad.ts +++ b/app/src/renderer/shell/gamepad.ts @@ -1,6 +1,6 @@ import { useEffect, useRef } from "preact/hooks"; import { controllerFamily, type ControllerFamily } from "../../shared/controllerGlyphs"; -import { Button, directionOf, stickToScroll, stickToVelocity, type Direction } from "../../shared/controllerInput"; +import { Button, DOUBLE_PRESS_MS, directionOf, stickToScroll, stickToVelocity, type Direction } from "../../shared/controllerInput"; import { activePad, padsInUse, startedUsing, type PadSample } from "../../shared/controllerSelection"; import { GAMEPAD_POLL_MS, gamepadPoll, STICK_INTERVAL_MS } from "../../shared/gamepadLoop"; import type { ShellState } from "../../shared/ipc"; @@ -29,6 +29,13 @@ export function useGamepad(state: ShellState, overlay: Overlay, setOverlay: (o: const family = useRef(undefined); const lastDirection = useRef(0); const lastStick = useRef(0); + /** + * Who A answers to on a page: the stick's pointer once the stick moved it, the D-pad's focus once the D-pad moved. + * The last one used wins, so both can be mixed on the same page. + */ + const pointing = useRef(false); + /** A single A at the pointer waits here for a possible second one, which would make it a right click. */ + const pendingClick = useRef(undefined); const lastTick = useRef(0); const stateRef = useRef(state); stateRef.current = state; @@ -117,6 +124,7 @@ export function useGamepad(state: ShellState, overlay: Overlay, setOverlay: (o: if (justPressed(Button.A)) { if (openOverlay !== "none" || s.permission || s.keyboard) (document.activeElement as HTMLElement | null)?.click(); else if (chrome) activateChrome(); + else if (pointing.current || s.inputMode === "Cursor") clickAtPointer(); else bridge().controller({ type: "activate" }); vibrate("confirm"); return; @@ -134,28 +142,56 @@ export function useGamepad(state: ShellState, overlay: Overlay, setOverlay: (o: const lx = pad.axes[0] ?? 0; const ly = pad.axes[1] ?? 0; + const rx = pad.axes[2] ?? 0; const ry = pad.axes[3] ?? 0; const dpad = { up: down(Button.DpadUp), down: down(Button.DpadDown), left: down(Button.DpadLeft), right: down(Button.DpadRight) }; - const cursorMode = s.inputMode === "Cursor" && !overlayOpen; + // On a page the left stick is the pointer and the right stick the wheel. Arrows mode keeps the stick on the arrow + // keys its sites are built for; a dialog, the keyboard or the toolbar take the stick as a D-pad. + const page = !overlayOpen && !chrome; + const pointer = page && s.inputMode !== "Arrows"; - // Buttons follow the poll rate; the stick keeps its own, slower cadence so the cursor moves at the same speed. - if (cursorMode && now - lastStick.current >= STICK_INTERVAL_MS) { + // Buttons follow the poll rate; the sticks keep their own, slower cadence so the pointer moves at the same speed. + if (page && now - lastStick.current >= STICK_INTERVAL_MS) { lastStick.current = now; - const velocity = stickToVelocity(lx, ly); - if (velocity.dx !== 0 || velocity.dy !== 0) bridge().controller({ type: "cursor-move", dx: velocity.dx, dy: velocity.dy }); + const velocity = pointer ? stickToVelocity(lx, ly) : { dx: 0, dy: 0 }; + if (velocity.dx !== 0 || velocity.dy !== 0) { + pointing.current = true; + bridge().controller({ type: "cursor-move", dx: velocity.dx, dy: velocity.dy }); + } const notches = stickToScroll(ry); - if (notches !== 0) bridge().controller({ type: "scroll", notches }); + const sideways = stickToScroll(rx); + if (notches !== 0 || sideways !== 0) bridge().controller({ type: "scroll", notches, sideways }); } - const direction: Direction | undefined = cursorMode ? directionOf(dpad, 0, 0) : directionOf(dpad, lx, ly); + const direction: Direction | undefined = directionOf(dpad, pointer ? 0 : lx, pointer ? 0 : ly); if (direction && now - lastDirection.current >= 180) { lastDirection.current = now; if (moveSuggestionFocus(direction)) { /* the D-pad walked the address suggestions */ } else if (openOverlay !== "none" || s.permission || s.keyboard) moveOverlayFocus(direction); // Down with nothing under the toolbar goes back into the page. else if (chrome) { if (!moveFocusAmong(chromeControls(), direction) && direction === "down") leaveChrome(); } - else bridge().controller({ type: "direction", direction }); + else { + // The D-pad takes over from the pointer (in Cursor mode the D-pad steps the pointer itself, so it stays). + if (pointing.current && s.inputMode !== "Cursor") { + pointing.current = false; + bridge().controller({ type: "cursor-hide" }); + } + bridge().controller({ type: "direction", direction }); + } + } + } + /** A at the pointer: the second press within DOUBLE_PRESS_MS turns the pending left click into a right click. */ + function clickAtPointer(): void { + if (pendingClick.current !== undefined) { + clearTimeout(pendingClick.current); + pendingClick.current = undefined; + bridge().controller({ type: "cursor-click", button: "right" }); + return; } + pendingClick.current = window.setTimeout(() => { + pendingClick.current = undefined; + bridge().controller({ type: "cursor-click", button: "left" }); + }, DOUBLE_PRESS_MS); } // Up from the top of the page: the main process hands the focus over and the address field takes it. bridge().onFocusChrome(() => document.getElementById("address")?.focus()); diff --git a/app/src/shared/controllerInput.ts b/app/src/shared/controllerInput.ts index cc3d367..a038ce9 100644 --- a/app/src/shared/controllerInput.ts +++ b/app/src/shared/controllerInput.ts @@ -5,6 +5,8 @@ import type { InputMode } from "./settings"; export const STICK_DEADZONE = 0.25; export const DIRECTION_THRESHOLD = 0.5; export const MAX_CURSOR_SPEED = 22; // px per 50 ms tick at full deflection +/** Two A presses closer than this are a right click at the pointer; a single press waits this long to be a left click. */ +export const DOUBLE_PRESS_MS = 280; /** Hosts whose pages already implement arrow-key navigation (10-foot UIs). */ const ARROW_HOSTS = ["youtube.com", "tv.youtube.com", "twitch.tv"]; diff --git a/app/src/shared/ipc.ts b/app/src/shared/ipc.ts index 8ad25f0..0b69cd9 100644 --- a/app/src/shared/ipc.ts +++ b/app/src/shared/ipc.ts @@ -140,6 +140,10 @@ export type ControllerEvent = | { type: "direction"; direction: "up" | "down" | "left" | "right" } | { type: "activate" } | { type: "cursor-move"; dx: number; dy: number } - | { type: "cursor-click" } - | { type: "scroll"; notches: number } + /** A at the pointer: a left click, or a right click when A was pressed twice in a row. */ + | { type: "cursor-click"; button?: "left" | "right" } + /** The D-pad took over: the pointer goes away until the stick moves it again. */ + | { type: "cursor-hide" } + /** The right stick: wheel notches down (positive) and to the right (`sideways`, positive). */ + | { type: "scroll"; notches: number; sideways?: number } | { type: "connected"; connected: boolean; family?: "xbox" | "playstation" | "switch" | "generic" }; diff --git a/app/tests/e2e/controller.spec.ts b/app/tests/e2e/controller.spec.ts index cd74a77..f310ded 100644 --- a/app/tests/e2e/controller.spec.ts +++ b/app/tests/e2e/controller.spec.ts @@ -234,3 +234,58 @@ test("up from the top of the page reaches the toolbar and the tabs, and down com await expect.poll(shellHasWindowFocus).toBe(false); expect(await chrome.evaluate(() => document.activeElement === document.body)).toBe(true); }); + +// ---- The stick's pointer, its clicks and the right stick's wheel ---- + +/** A focusable button on top and, under it, a tall area that logs every mouse press and context menu it gets. */ +const POINTER_PAGE = `Pointer` + + `` + + `
` + + ``; + +const tilt = (axis: number, value: number) => chrome.evaluate(([a, v]) => { + (window as unknown as { __pads: { axes: number[] }[] }).__pads[0]!.axes[a] = v; +}, [axis, value] as const); +const pageLog = () => app.evaluate(() => globalThis.joy.pageEval("window.__log.join(',')")) as Promise; +const pointerShown = () => app.evaluate(() => globalThis.joy.pageEval("(document.getElementById('__joy-cursor') || {}).style?.display === 'block'")); + +test("the left stick points and A clicks there, twice for a right click, until the D-pad takes over", async () => { + await installPads([XBOX]); + await app.evaluate(({ protocol }, page) => { + protocol.handle("https", () => new Response(page, { headers: { "content-type": "text/html" } })); + }, POINTER_PAGE); + await app.evaluate(() => globalThis.joy.command({ type: "navigate", url: "https://pointer.example/" } as never)); + await expect.poll(async () => { + const tab = (await tabs()).find((t) => t.isActive); + return tab?.url === "https://pointer.example/" && !tab.isLoading; + }, { timeout: 30_000 }).toBe(true); + + // Spatial is the default mode: the stick still brings the pointer up, over the area. + await tilt(1, 1); + await expect.poll(pointerShown, { timeout: 10_000 }).toBe(true); + await chrome.waitForTimeout(300); + await tilt(1, 0); + + // One A is a left click where the pointer is (after the short wait for a second press). + await press(0, Pad.A); + await expect.poll(pageLog, { timeout: 5_000 }).toBe("down0"); + // Two quick ones are a right click, and no left click on the way. + await press(0, Pad.A); + await press(0, Pad.A); + await expect.poll(pageLog, { timeout: 5_000 }).toBe("down0,down2,context"); + + // The D-pad takes over: the pointer goes away and A presses what the D-pad focused, not the spot under the pointer. + await step(Pad.Down); + await expect.poll(pageFocus).toBe("b"); + await expect.poll(pointerShown).toBe(false); + await step(Pad.A); + await expect.poll(pageLog).toBe("down0,down2,context,button"); + + // The right stick is the wheel. + expect(await app.evaluate(() => globalThis.joy.pageEval("scrollY"))).toBe(0); + await tilt(3, 1); + await expect.poll(() => app.evaluate(() => globalThis.joy.pageEval("scrollY")) as Promise, { timeout: 10_000 }).toBeGreaterThan(0); + await tilt(3, 0); +}); diff --git a/docs/CONTROLLER-TEST.md b/docs/CONTROLLER-TEST.md index c024b87..22de239 100644 --- a/docs/CONTROLLER-TEST.md +++ b/docs/CONTROLLER-TEST.md @@ -19,9 +19,10 @@ Para cada cenário: **passou / falhou**, quantos segundos demorou a primeira res 4. **Três abas.** Abra três abas, alterne pelo seletor em grade e feche uma, só com o controle. Veja se as miniaturas aparecem e se o foco começa na aba ativa. 5. **Campo de texto com A.** Numa página com um campo (a caixa de busca da nova aba, ou a do YouTube), leve o D-pad até o campo: o teclado **não** deve abrir só de passar por ele. Aperte A: o teclado abre, o D-pad e o analógico andam pelas teclas a partir da primeira, e A escreve no campo. Feche com B e repita: a segunda vez tem de funcionar igual à primeira. 6. **Subir até a barra.** No topo de qualquer página, aperte para cima: o foco vai para a barra de endereço (sem abrir o teclado). Ande para os lados até voltar, avançar e recarregar, suba até as abas e o +, e aperte A em cada um. Para baixo (ou B) volta para a página. Os botões de minimizar, maximizar e fechar ficam de fora de propósito. -7. **Permissão de microfone.** Abra um site que peça o microfone (ou o teste do WebRTC), percorra o prompt com o D-pad e escolha "permitir sempre". Confirme em Configurações que a permissão ficou salva para o site. -8. **Atualização.** Em Windows/Linux: com uma versão nova publicada, deixe o download acontecer, feche o app e reabra; no `.deb`/`.rpm` use **Reiniciar e instalar** em Configurações → Sobre e confirme o pedido de senha. No macOS, o app deve avisar e abrir a página da release. -9. **Streaming com DRM.** Abra a Netflix, entre na conta e toque um título. Registre a resolução que o player mostra e o que a linha "Widevine (DRM)" diz em Configurações → Sobre. +7. **Cursor no analógico.** Numa página qualquer, mexa o analógico esquerdo: aparece um anel azul (o cursor do navegador, diferente da seta do mouse) e passa por cima dos itens. A clica onde ele aponta; A duas vezes rápido é o clique direito. Aperte o D-pad: o anel some e o A volta a apertar o item com foco. O analógico direito rola a página (para os lados também). Anote se a velocidade do cursor está boa da distância do sofá. +8. **Permissão de microfone.** Abra um site que peça o microfone (ou o teste do WebRTC), percorra o prompt com o D-pad e escolha "permitir sempre". Confirme em Configurações que a permissão ficou salva para o site. +9. **Atualização.** Em Windows/Linux: com uma versão nova publicada, deixe o download acontecer, feche o app e reabra; no `.deb`/`.rpm` use **Reiniciar e instalar** em Configurações → Sobre e confirme o pedido de senha. No macOS, o app deve avisar e abrir a página da release. +10. **Streaming com DRM.** Abra a Netflix, entre na conta e toque um título. Registre a resolução que o player mostra e o que a linha "Widevine (DRM)" diz em Configurações → Sobre. Leitor de tela é o outro lado da mesma sessão: o roteiro de Orca, Narrator e VoiceOver está em [SCREEN-READER.md](SCREEN-READER.md).