From 1da0e72c6f606f1a37d9d8dd88dc1a2541173eeb Mon Sep 17 00:00:00 2001 From: Filipe Moreira Date: Wed, 21 Oct 2026 16:18:46 -0300 Subject: [PATCH] fix(controller): A types in a field, the keyboard keys move, and up reaches the toolbar - A on a text field (any page, the new tab's box included) opens the controller keyboard. Walking over a field or a page's autofocus no longer pops it up; a real click still does (mouse, or A in Cursor mode). - The keyboard always takes the window's focus. Closing it hands the focus back to the tab, so from the second time on the D-pad walked the keys while Chromium drew no focus on them, and the keyboard looked dead. The first press now lands on the first key, not on the close button. - Up from the top of a page moves the D-pad to the toolbar (address field, back, forward, reload, favorite, menu) and the tab strip; A presses, the address field opens the keyboard, and down past the toolbar or B goes back to the page. Minimize, maximize and close stay off the D-pad. In TV mode the bar stays on screen while it has the focus. - The address field opens the keyboard on a click instead of on focus, so reaching it with the D-pad does not open it on the way past. - Spatial focus: the first press takes the first item instead of looking past the middle of the screen; sharing a column or a row counts more than where the centres sit, which made the D-pad jump over wide text fields. The shell walks its own controls edge to edge as well, so left from the address field reaches reload instead of the tab strip, and the close button inside a tab card stays reachable. Co-Authored-By: Claude Opus 5.5 --- app/src/assets/spatial.js | 13 +++- app/src/main/controllerActions.ts | 10 +-- app/src/main/services/commandRouter.ts | 4 +- app/src/main/services/inputRouter.ts | 13 ++-- app/src/main/services/shellHost.ts | 4 ++ app/src/main/shell.ts | 24 ++++++- app/src/preload/chrome.ts | 3 + app/src/preload/web.ts | 40 ++++++----- app/src/renderer/shell/app.tsx | 5 +- app/src/renderer/shell/bridge.ts | 2 + app/src/renderer/shell/gamepad.ts | 85 ++++++++++++++++++---- app/src/shared/commands.ts | 1 + app/src/shared/ipc.ts | 4 ++ app/tests/e2e/controller.spec.ts | 99 ++++++++++++++++++++++++++ app/tests/e2e/journeys.spec.ts | 20 +++--- docs/CONTROLLER-TEST.md | 8 ++- 16 files changed, 276 insertions(+), 59 deletions(-) diff --git a/app/src/assets/spatial.js b/app/src/assets/spatial.js index c8b43e1..8ffb4e7 100644 --- a/app/src/assets/spatial.js +++ b/app/src/assets/spatial.js @@ -60,7 +60,9 @@ const overlap = dir === 'up' || dir === 'down' ? Math.max(0, Math.min(r.right, fr.right) - Math.max(r.left, fr.left)) : Math.max(0, Math.min(r.bottom, fr.bottom) - Math.max(r.top, fr.top)); - const score = Math.max(0, ahead) + off * (overlap > 0 ? 0.6 : 2.2); + // Sharing the column (or the row) is what counts: a wide field under a short link is the next stop down, however far + // its centre sits (weighing centres made the D-pad jump over text fields); the centres only break ties. + const score = Math.max(0, ahead) + off * (overlap > 0 ? 0.1 : 2.2); if (score < bestScore) { best = el; bestScore = score; } } return best; @@ -102,13 +104,17 @@ }; const moveFrom = (fromEl, dir) => { const from = fromEl || focused(); - const next = pick(from ? from.getBoundingClientRect() : viewportCenter(), dir, from); + // Nothing focused yet: start from the edge the press comes from, so the first press takes the first item instead of + // looking past the middle of the screen and only scrolling. + const next = pick(from ? from.getBoundingClientRect() : enterRect(dir), dir, from); if (next) { mark(next); if (isFrame(next)) post(next.contentWindow, { joy: 'enter', dir }); return 'moved'; } if (window !== window.top) { post(window.parent, { joy: 'exit', dir }); return 'exit'; } + // Up with nothing above and the page already at its top: the D-pad leaves the page for the browser's own bar. + if (dir === 'up' && scrollY <= 0) return 'edge'; scroll(dir); // After scrolling new elements may be in view; try once more on the next frame. requestAnimationFrame(() => { @@ -201,7 +207,8 @@ const tag = el.tagName; const textual = tag === 'TEXTAREA' || tag === 'SELECT' || el.isContentEditable || (tag === 'INPUT' && !['checkbox', 'radio', 'submit', 'button', 'reset', 'image', 'file', 'color', 'range'].includes((el.type || 'text').toLowerCase())); - if (textual) { el.focus(); return 'focus'; } + // A on a text field means "type here": the shell opens the controller keyboard for it. + if (textual) { el.focus(); return 'edit'; } el.click(); return 'click'; }, diff --git a/app/src/main/controllerActions.ts b/app/src/main/controllerActions.ts index cff28f1..a111091 100644 --- a/app/src/main/controllerActions.ts +++ b/app/src/main/controllerActions.ts @@ -25,12 +25,14 @@ export async function runInPageWorld(wc: WebContents, code: string, userGestu } } -export async function spatialMove(wc: WebContents, direction: Direction): Promise { - await runInPageWorld(wc, `window.__joy && window.__joy.move(${JSON.stringify(direction)})`); +/** "edge" when the D-pad went up from the top of the page: there is nothing left in it to walk to. */ +export async function spatialMove(wc: WebContents, direction: Direction): Promise { + return runInPageWorld(wc, `window.__joy && window.__joy.move(${JSON.stringify(direction)})`); } -export async function spatialActivate(wc: WebContents): Promise { - await runInPageWorld(wc, "window.__joy && window.__joy.activate()"); +/** "edit" when A landed on a text field, which then wants the controller keyboard. */ +export async function spatialActivate(wc: WebContents): Promise { + return runInPageWorld(wc, "window.__joy && window.__joy.activate()"); } const MEDIA_CALLS: Record = { diff --git a/app/src/main/services/commandRouter.ts b/app/src/main/services/commandRouter.ts index ddadc48..888c65f 100644 --- a/app/src/main/services/commandRouter.ts +++ b/app/src/main/services/commandRouter.ts @@ -67,7 +67,9 @@ export class CommandRouter { break; case "find-close": h.find = undefined; wc?.stopFindInPage("clearSelection"); h.broadcast(); break; case "keyboard-open": h.openKeyboard(command.target); break; - case "keyboard-close": h.closeKeyboard(); wc?.focus(); break; + // Back to where the typing came from: the page, or the toolbar when the D-pad had reached the address field. + case "keyboard-close": h.closeKeyboard(); if (!h.chromeFocused) wc?.focus(); break; + case "focus-page": h.leaveChrome(); break; case "keyboard-insert": if (h.keyboard?.target === "page") await insertIntoPage(wc, command.text); break; case "keyboard-backspace": if (h.keyboard?.target === "page") await insertIntoPage(wc, ""); break; case "keyboard-submit": diff --git a/app/src/main/services/inputRouter.ts b/app/src/main/services/inputRouter.ts index 6c593c2..6c320f6 100644 --- a/app/src/main/services/inputRouter.ts +++ b/app/src/main/services/inputRouter.ts @@ -10,6 +10,9 @@ export interface InputHost { /** Whether the active tab is playing media, from the shell's own tab state. */ isPlayingMedia(): boolean; setControllerConnected(connected: boolean, family?: "xbox" | "playstation" | "switch" | "generic"): void; + openKeyboard(target: "address" | "page"): void; + /** The D-pad left the top of the page: the tabs and the toolbar take it. */ + enterChrome(): void; } /** @@ -27,8 +30,9 @@ export class InputRouter { break; case "direction": if (!wc) return; - if (this.host.inputMode === "Spatial") await spatialMove(wc, input.direction); - else if (this.host.inputMode === "Cursor") { + if (this.host.inputMode === "Spatial") { + if ((await spatialMove(wc, input.direction)) === "edge") this.host.enterChrome(); + } else if (this.host.inputMode === "Cursor") { const size = this.host.pageSize(); const step = 24; const dx = input.direction === "left" ? -step : input.direction === "right" ? step : 0; @@ -44,8 +48,9 @@ export class InputRouter { } // The tab state is the cheap gate: a page that never played media keeps A on focus, with no round trip. if (this.host.isPlayingMedia() && (await spatialPlayPause(wc)) === "media") break; - if (this.host.inputMode === "Spatial") await spatialActivate(wc); - else sendKey(wc, "Return"); + if (this.host.inputMode === "Spatial") { + if ((await spatialActivate(wc)) === "edit") this.host.openKeyboard("page"); + } else sendKey(wc, "Return"); break; case "cursor-move": if (wc && this.host.inputMode === "Cursor") { diff --git a/app/src/main/services/shellHost.ts b/app/src/main/services/shellHost.ts index ce5d6c8..d92cc26 100644 --- a/app/src/main/services/shellHost.ts +++ b/app/src/main/services/shellHost.ts @@ -34,6 +34,10 @@ export interface ShellHost { broadcast(): void; syncViews(): void; setLayout(top: number, bottom: number): void; + /** The D-pad came back down from the toolbar: the page takes the window's focus again. */ + leaveChrome(): void; + /** The D-pad is walking the tabs and the toolbar (see Shell.enterChrome). */ + readonly chromeFocused: boolean; /** Hides the active tab's view under a modal overlay, which the shell draws over a snapshot of it. */ setPageCovered(covered: boolean): void; setChromeVisible(visible: boolean): void; diff --git a/app/src/main/shell.ts b/app/src/main/shell.ts index 82f6025..ec23bca 100644 --- a/app/src/main/shell.ts +++ b/app/src/main/shell.ts @@ -64,6 +64,8 @@ export class Shell { readonly tabs: TabManager; private readonly layout: WindowLayout; chromeVisible = true; + /** True while the D-pad walks the tabs and the toolbar instead of the page; the TV-mode bar then stays on screen. */ + chromeFocused = false; /** A mode asked for on the command line (`--tv`, `--window`): it holds for this run without touching the setting. */ private launchMode: WindowMode | undefined; private pageCovered = false; @@ -273,6 +275,9 @@ export class Shell { } private wireWindow(): void { + // Whatever takes the window's focus from the shell (a click into the page, the page after the keyboard) ends the + // D-pad's walk on the toolbar. + this.window.webContents.on("blur", () => { this.chromeFocused = false; }); this.window.on("resize", () => this.layout.apply()); this.window.on("maximize", () => { this.layout.apply(); this.broadcast(); }); this.window.on("unmaximize", () => { this.layout.apply(); this.broadcast(); }); @@ -544,9 +549,26 @@ export class Shell { openKeyboard(target: "address" | "page"): void { this.revealChrome(); this.keyboard = { target, title: target === "address" ? "ADDRESS / SEARCH" : "PAGE TEXT ENTRY" }; + // The keys are the shell's: with the window's focus left on the tab (closing the keyboard hands it back there), the + // D-pad walked them but Chromium drew no focus on any of them, so the keyboard looked dead. + this.focusChrome(); this.broadcast(); } + /** The D-pad went up out of the page: the bar shows, takes the window's focus and puts it on the address field. */ + enterChrome(): void { + this.chromeFocused = true; + this.revealChrome(); + this.focusChrome(); + this.window.webContents.send(Channels.shellFocusChrome); + } + + leaveChrome(): void { + this.chromeFocused = false; + this.tabs.activeWebContents?.focus(); + this.scheduleChromeHide(); + } + /** * Hands the window's keyboard focus to the shell. Chromium keeps the focused document per view, so a tab view * that was focused stays focused while the shell's own controls take DOM focus, and a screen reader never gets @@ -659,7 +681,7 @@ export class Shell { const tab = this.tabs.active; if (!tab || tab.sleeping || isInternal(tab.url)) return; this.chromeHideTimer = setTimeout(() => { - if (this.windowMode === "tv" && !this.permissions.prompt && !this.keyboard) this.setChromeVisible(false); + if (this.windowMode === "tv" && !this.permissions.prompt && !this.keyboard && !this.chromeFocused) this.setChromeVisible(false); }, 4_000); this.chromeHideTimer.unref?.(); } diff --git a/app/src/preload/chrome.ts b/app/src/preload/chrome.ts index c55d85f..dd181b9 100644 --- a/app/src/preload/chrome.ts +++ b/app/src/preload/chrome.ts @@ -28,6 +28,9 @@ contextBridge.exposeInMainWorld("shell", { snapshot(): Promise { return ipcRenderer.invoke(Channels.shellSnapshot); }, + onFocusChrome(callback: () => void): void { + ipcRenderer.on(Channels.shellFocusChrome, () => callback()); + }, onActivity(callback: (activity: ShellActivity) => void): () => void { const listener = (_event: unknown, activity: ShellActivity): void => callback(activity); ipcRenderer.on(Channels.shellActivity, listener); diff --git a/app/src/preload/web.ts b/app/src/preload/web.ts index 306ffc0..d042a88 100644 --- a/app/src/preload/web.ts +++ b/app/src/preload/web.ts @@ -32,22 +32,26 @@ if (window.location.protocol === "joychromium:") { ipcRenderer.on(Channels.pageReply, (_event, message) => callback(message)); }, }); -} else { - // Web pages (isolated world): tell the shell when a text field takes focus so the controller keyboard can open. - // Nothing is exposed to the page. - document.addEventListener( - "focusin", - (event) => { - const field = event.target as HTMLElement | null; - if (!field || (field as HTMLInputElement).disabled || (field as HTMLInputElement).readOnly) return; - const tag = field.tagName; - const type = ((field as HTMLInputElement).type || "text").toLowerCase(); - const editable = - field.isContentEditable || - tag === "TEXTAREA" || - (tag === "INPUT" && ["text", "search", "url", "email", "password", "tel", "number"].includes(type)); - if (editable) ipcRenderer.send(Channels.pageFocusIn); - }, - true, - ); } + +// Every page, ours included: a text field clicked (with a mouse, or with A in Cursor mode, which is a real click) opens the +// controller keyboard. Only a click counts: a field the D-pad walks over or the page focuses by itself (autofocus) must not +// pop the keyboard up; with the D-pad, A on the field opens it (InputRouter). Nothing is exposed to the page. +let pressedAt = 0; +document.addEventListener("pointerdown", () => { pressedAt = Date.now(); }, true); +document.addEventListener( + "focusin", + (event) => { + if (Date.now() - pressedAt > 800) return; + const field = event.target as HTMLElement | null; + if (!field || (field as HTMLInputElement).disabled || (field as HTMLInputElement).readOnly) return; + const tag = field.tagName; + const type = ((field as HTMLInputElement).type || "text").toLowerCase(); + const editable = + field.isContentEditable || + tag === "TEXTAREA" || + (tag === "INPUT" && ["text", "search", "url", "email", "password", "tel", "number"].includes(type)); + if (editable) ipcRenderer.send(Channels.pageFocusIn); + }, + true, +); diff --git a/app/src/renderer/shell/app.tsx b/app/src/renderer/shell/app.tsx index 9f8d160..2b267b8 100644 --- a/app/src/renderer/shell/app.tsx +++ b/app/src/renderer/shell/app.tsx @@ -253,7 +253,10 @@ function AddressBar({ url, focused, setFocused, language, keyboard, children }: autocomplete="off" spellcheck={false} value={value} - onFocus={() => { setFocused(true); cmd({ type: "keyboard-open", target: "address" }); (input.current as HTMLInputElement).select(); }} + // A click (or A on the field from the D-pad, see gamepad.ts) opens the controller keyboard; focus alone does not, or + // walking the toolbar with the D-pad would pop the keyboard up on the way past. + onMouseDown={() => cmd({ type: "keyboard-open", target: "address" })} + onFocus={() => { setFocused(true); (input.current as HTMLInputElement).select(); }} onBlur={() => setFocused(false)} onKeyDown={(e) => { // Down leaves the field for the suggestions the controller keyboard shows, so both inputs reach them. diff --git a/app/src/renderer/shell/bridge.ts b/app/src/renderer/shell/bridge.ts index 1150fa1..51a6a0c 100644 --- a/app/src/renderer/shell/bridge.ts +++ b/app/src/renderer/shell/bridge.ts @@ -13,6 +13,8 @@ export interface ShellBridge { /** Whether the window is on screen, so idle loops park while it is minimized. Returns its own unsubscribe. */ onActivity(callback: (activity: ShellActivity) => void): () => void; thumbnails(): Promise<{ id: number; dataUrl: string }[]>; + /** The D-pad went up out of the page: focus the toolbar. */ + onFocusChrome(callback: () => void): void; /** The active page as an image, painted under a modal overlay while the live view is hidden. */ snapshot(): Promise; preferences(): Promise; diff --git a/app/src/renderer/shell/gamepad.ts b/app/src/renderer/shell/gamepad.ts index 9cf2f40..8b4bdfb 100644 --- a/app/src/renderer/shell/gamepad.ts +++ b/app/src/renderer/shell/gamepad.ts @@ -105,15 +105,18 @@ export function useGamepad(state: ShellState, overlay: Overlay, setOverlay: (o: cmd({ type: "close-tab" }); return; } + const chrome = !overlayOpen && inChrome(); if (justPressed(Button.B)) { if (openOverlay !== "none") setOverlay("none"); else if (s.permission) cmd({ type: "permission", allow: false, remember: false }); else if (s.keyboard) cmd({ type: "keyboard-close" }); + else if (chrome) leaveChrome(); else cmd({ type: "back" }); return; } if (justPressed(Button.A)) { if (openOverlay !== "none" || s.permission || s.keyboard) (document.activeElement as HTMLElement | null)?.click(); + else if (chrome) activateChrome(); else bridge().controller({ type: "activate" }); vibrate("confirm"); return; @@ -149,9 +152,13 @@ export function useGamepad(state: ShellState, overlay: Overlay, setOverlay: (o: lastDirection.current = now; if (moveSuggestionFocus(direction)) { /* the D-pad walked the address suggestions */ } else if (openOverlay !== "none" || s.permission || s.keyboard) moveOverlayFocus(direction); + // Down with nothing under the toolbar goes back into the page. + else if (chrome) { if (!moveFocusAmong(chromeControls(), direction) && direction === "down") leaveChrome(); } else bridge().controller({ type: "direction", direction }); } } + // Up from the top of the page: the main process hands the focus over and the address field takes it. + bridge().onFocusChrome(() => document.getElementById("address")?.focus()); resume(); const visibility = (): void => { if (!document.hidden) resume(); @@ -198,28 +205,76 @@ function moveSuggestionFocus(direction: Direction): boolean { return true; } +/** + * The D-pad is on the tabs or the toolbar: one of their controls has the focus and the shell's document has the + * window's. A click into the page leaves the shell's focused element in place, which is why the window focus counts too. + */ +function inChrome(): boolean { + return document.hasFocus() && Boolean((document.activeElement as HTMLElement | null)?.closest("#chrome")); +} + +/** What the D-pad walks on the tabs and the toolbar. Minimize, maximize and close stay off it: one A would close the app. */ +function chromeControls(): HTMLElement[] { + return [...document.querySelectorAll("#chrome button:not([disabled]), #chrome input")] + .filter((el) => !el.closest(".window-controls") && el.getClientRects().length > 0); +} + +/** A on the toolbar: the address field opens the controller keyboard, anything else is pressed. */ +function activateChrome(): void { + const el = document.activeElement as HTMLElement | null; + if (el?.id === "address") cmd({ type: "keyboard-open", target: "address" }); + else el?.click(); +} + +function leaveChrome(): void { + (document.activeElement as HTMLElement | null)?.blur(); + cmd({ type: "focus-page" }); +} + /** Spatial focus among the shell's own buttons, used while a dialog or the keyboard is open. */ function moveOverlayFocus(direction: Direction): void { const group = (document.querySelector('.overlay[role="dialog"]') as HTMLElement | null) ?? document.body; const buttons = [...group.querySelectorAll("button:not([disabled])")]; const current = document.activeElement as HTMLElement | null; - const index = current ? buttons.indexOf(current) : -1; - if (index < 0) { buttons[0]?.focus(); return; } - const rect = current!.getBoundingClientRect(); - const cx = rect.left + rect.width / 2; - const cy = rect.top + rect.height / 2; + // Nothing focused inside yet (a dialog opens with the focus on itself): the first press lands on the first key of the + // keyboard, or the first button of any other dialog, not on the close button in the corner. + if (!current || !buttons.includes(current)) { (group.querySelector(".key") ?? buttons[0])?.focus(); return; } + moveFocusAmong(buttons, direction); +} + +/** Moves the focus to the nearest control in `direction`; false when there is none, so the caller decides what that means. */ +function moveFocusAmong(controls: HTMLElement[], direction: Direction): boolean { + const current = document.activeElement as HTMLElement | null; + if (!current || !controls.includes(current)) { + if (!controls[0]) return false; + controls[0].focus(); + return true; + } + // Measured edge to edge, like the page's spatial.js: a wide control (the address field) has its centre far from its + // neighbours, and centre distances sent the D-pad to the tab strip instead of the reload button beside it. + const from = current.getBoundingClientRect(); + const fc = { x: from.left + from.width / 2, y: from.top + from.height / 2 }; let best: HTMLElement | undefined; let bestScore = Infinity; - for (const button of buttons) { - if (button === current) continue; - const r = button.getBoundingClientRect(); - const dx = r.left + r.width / 2 - cx; - const dy = r.top + r.height / 2 - cy; - const ahead = direction === "up" ? -dy : direction === "down" ? dy : direction === "left" ? -dx : dx; - if (ahead <= 4) continue; - const off = direction === "up" || direction === "down" ? Math.abs(dx) : Math.abs(dy); - const score = ahead + off * 2; - if (score < bestScore) { best = button; bestScore = score; } + for (const control of controls) { + if (control === current) continue; + const r = control.getBoundingClientRect(); + const c = { x: r.left + r.width / 2, y: r.top + r.height / 2 }; + const vertical = direction === "up" || direction === "down"; + // Ahead means its centre is ahead, so a control inside the current one (the close button on a tab card) is reachable; + // the distance is edge to edge, and overlapping the current control along the way counts as distance too. + const centreAhead = direction === "up" ? fc.y - c.y : direction === "down" ? c.y - fc.y : direction === "left" ? fc.x - c.x : c.x - fc.x; + if (centreAhead <= 4) continue; + const ahead = Math.abs(direction === "up" ? from.top - r.bottom : direction === "down" ? r.top - from.bottom + : direction === "left" ? from.left - r.right : r.left - from.right); + const off = vertical ? Math.abs(c.x - fc.x) : Math.abs(c.y - fc.y); + const overlap = vertical + ? Math.min(r.right, from.right) - Math.max(r.left, from.left) + : Math.min(r.bottom, from.bottom) - Math.max(r.top, from.top); + // Sharing the row or the column is what counts; the centres only break a tie between neighbours that both do. + const score = ahead + off * (overlap > 0 ? 0.1 : 2.2); + if (score < bestScore) { best = control; bestScore = score; } } best?.focus(); + return best !== undefined; } diff --git a/app/src/shared/commands.ts b/app/src/shared/commands.ts index 2da7e52..d8e79d6 100644 --- a/app/src/shared/commands.ts +++ b/app/src/shared/commands.ts @@ -67,6 +67,7 @@ export function isShellCommand(raw: unknown): raw is ShellCommand { case "window-mode": case "find-close": case "keyboard-close": + case "focus-page": case "keyboard-backspace": case "keyboard-submit": case "cycle-input-mode": diff --git a/app/src/shared/ipc.ts b/app/src/shared/ipc.ts index 7568062..8ad25f0 100644 --- a/app/src/shared/ipc.ts +++ b/app/src/shared/ipc.ts @@ -32,6 +32,8 @@ export const Channels = { shellActivity: "shell:activity", /** chrome renderer -> main: focus landed on a control of the shell, so the window's focus belongs to it and not to a tab */ shellFocus: "shell:focus", + /** main -> chrome renderer: the D-pad went up out of the page, so the toolbar takes the focus */ + shellFocusChrome: "shell:focus-chrome", } as const; /** Whether the window is on screen: sent when it is minimized, hidden, restored or shown. */ @@ -119,6 +121,8 @@ export type ShellCommand = | { type: "find-close" } | { type: "keyboard-open"; target: "address" | "page" } | { type: "keyboard-close" } + /** The D-pad left the tabs and the toolbar: the page takes the focus back. */ + | { type: "focus-page" } | { type: "keyboard-insert"; text: string } | { type: "keyboard-backspace" } | { type: "keyboard-submit" } diff --git a/app/tests/e2e/controller.spec.ts b/app/tests/e2e/controller.spec.ts index f384dac..cd74a77 100644 --- a/app/tests/e2e/controller.spec.ts +++ b/app/tests/e2e/controller.spec.ts @@ -135,3 +135,102 @@ test("a pad that connects but is not used does not take the input", async () => await press(1, Button.X); await expect.poll(hints).toContain("○ voltar"); }); + +// ---- Text fields, the controller keyboard and the toolbar, with the D-pad only ---- + +const Pad = { A: 0, B: 1, Up: 12, Down: 13, Left: 14, Right: 15 } as const; +const FORM = `FormLink one

` + + `

Link two`; + +/** A page with a link, a wide text field under it and another link, served locally (the CI has no network). */ +async function openForm(): Promise { + await app.evaluate(({ protocol }, page) => { + protocol.handle("https", () => new Response(page, { headers: { "content-type": "text/html" } })); + }, FORM); + await app.evaluate(() => globalThis.joy.command({ type: "navigate", url: "https://form.example/" } as never)); + await expect.poll(async () => { + const tab = (await tabs()).find((t) => t.isActive); + return tab?.url === "https://form.example/" && !tab.isLoading; + }, { timeout: 30_000 }).toBe(true); +} + +// The D-pad ignores a second direction inside its repeat gate (gamepad.ts), so each step waits it out. +const step = async (button: number) => { await press(0, button); await chrome.waitForTimeout(200); }; +const pageFocus = () => app.evaluate(() => globalThis.joy.pageEval("document.activeElement && (document.activeElement.id || document.activeElement.textContent.trim())")); +const shellFocus = () => chrome.evaluate(() => { + const el = document.activeElement as HTMLElement | null; + return el ? el.id || el.getAttribute("aria-label") || (el.textContent ?? "").trim() : ""; +}); +const shellHasWindowFocus = () => app.evaluate(({ BrowserWindow }) => BrowserWindow.getAllWindows()[0]!.webContents.isFocused()); +const keyboards = () => chrome.locator("#keyboard").count(); + +test("A on a text field opens the controller keyboard, which the D-pad walks, every time", async () => { + await installPads([XBOX]); + await openForm(); + // As after the keyboard was used once: the tab holds the window's focus. That is when the keys used to look dead. + await app.evaluate(({ BrowserWindow, webContents }) => { + const shell = BrowserWindow.getAllWindows()[0]!.webContents; + webContents.getAllWebContents().find((wc) => wc !== shell)!.focus(); + }); + + // The first press takes the first item; the wide field under a short link is the next stop down. + await step(Pad.Down); + await expect.poll(pageFocus).toBe("Link one"); + await step(Pad.Down); + await expect.poll(pageFocus).toBe("q"); + // Walking onto a field does not pop the keyboard up: A does. + expect(await keyboards()).toBe(0); + + await step(Pad.A); + await expect(chrome.locator("#keyboard")).toBeVisible(); + await expect.poll(shellHasWindowFocus).toBe(true); + // The first press lands on the first key, not on the close button; A types into the page's field. + await step(Pad.Right); + await expect.poll(shellFocus).toBe("q"); + await step(Pad.Right); + await expect.poll(shellFocus).toBe("w"); + await step(Pad.A); + await expect.poll(() => app.evaluate(() => globalThis.joy.pageEval("document.getElementById('q').value"))).toBe("w"); + + // B closes it and hands the focus back to the page; the second time works like the first. + await step(Pad.B); + await expect.poll(keyboards).toBe(0); + await step(Pad.A); + await expect(chrome.locator("#keyboard")).toBeVisible(); + await expect.poll(shellHasWindowFocus).toBe(true); + await step(Pad.Down); + await expect.poll(shellFocus).toBe("q"); +}); + +test("up from the top of the page reaches the toolbar and the tabs, and down comes back", async () => { + await installPads([XBOX]); + await openForm(); + await step(Pad.Down); + await expect.poll(pageFocus).toBe("Link one"); + + // Nothing above and the page at its top: the address field takes the focus, without opening the keyboard. + await step(Pad.Up); + await expect.poll(shellFocus).toBe("address"); + await expect.poll(shellHasWindowFocus).toBe(true); + expect(await keyboards()).toBe(0); + + // A on the field opens the keyboard; B closes it and the D-pad is still on the toolbar. + await step(Pad.A); + await expect(chrome.locator("#keyboard")).toBeVisible(); + await step(Pad.B); + await expect.poll(keyboards).toBe(0); + await expect.poll(shellFocus).toBe("address"); + + // Left reaches the reload button, up the tab strip. + await step(Pad.Left); + await expect.poll(shellFocus).toMatch(/Recarregar|Reload/); + await step(Pad.Up); + await expect.poll(() => chrome.evaluate(() => Boolean(document.activeElement?.closest(".tabstrip")))).toBe(true); + + // Down twice: the toolbar, then the page again. + await step(Pad.Down); + await expect.poll(() => chrome.evaluate(() => Boolean(document.activeElement?.closest(".bar")))).toBe(true); + await step(Pad.Down); + await expect.poll(shellHasWindowFocus).toBe(false); + expect(await chrome.evaluate(() => document.activeElement === document.body)).toBe(true); +}); diff --git a/app/tests/e2e/journeys.spec.ts b/app/tests/e2e/journeys.spec.ts index 6e8cd24..73f3b80 100644 --- a/app/tests/e2e/journeys.spec.ts +++ b/app/tests/e2e/journeys.spec.ts @@ -119,20 +119,22 @@ const OVERLAY_PLAN = (expr: string) => `(() => { if (targetIndex < 0) return { same: false, missing: true, dirs: [] }; const best = (current, direction) => { if (!current) return buttons[0] || null; - const rect = current.getBoundingClientRect(); - const cx = rect.left + rect.width / 2; - const cy = rect.top + rect.height / 2; + const from = current.getBoundingClientRect(); + const fc = { x: from.left + from.width / 2, y: from.top + from.height / 2 }; let found = null; let bestScore = Infinity; for (const button of buttons) { if (button === current) continue; const r = button.getBoundingClientRect(); - const dx = r.left + r.width / 2 - cx; - const dy = r.top + r.height / 2 - cy; - const ahead = direction === 'up' ? -dy : direction === 'down' ? dy : direction === 'left' ? -dx : dx; - if (ahead <= 4) continue; - const off = direction === 'up' || direction === 'down' ? Math.abs(dx) : Math.abs(dy); - const score = ahead + off * 2; + const c = { x: r.left + r.width / 2, y: r.top + r.height / 2 }; + const vertical = direction === 'up' || direction === 'down'; + const centreAhead = direction === 'up' ? fc.y - c.y : direction === 'down' ? c.y - fc.y : direction === 'left' ? fc.x - c.x : c.x - fc.x; + if (centreAhead <= 4) continue; + const ahead = Math.abs(direction === 'up' ? from.top - r.bottom : direction === 'down' ? r.top - from.bottom + : direction === 'left' ? from.left - r.right : r.left - from.right); + const off = vertical ? Math.abs(c.x - fc.x) : Math.abs(c.y - fc.y); + const overlap = vertical ? Math.min(r.right, from.right) - Math.max(r.left, from.left) : Math.min(r.bottom, from.bottom) - Math.max(r.top, from.top); + const score = ahead + off * (overlap > 0 ? 0.1 : 2.2); if (score < bestScore) { found = button; bestScore = score; } } return found; diff --git a/docs/CONTROLLER-TEST.md b/docs/CONTROLLER-TEST.md index 2028143..c024b87 100644 --- a/docs/CONTROLLER-TEST.md +++ b/docs/CONTROLLER-TEST.md @@ -17,9 +17,11 @@ Para cada cenário: **passou / falhou**, quantos segundos demorou a primeira res 2. **Tela inicial → YouTube TV.** Abra o app de TV pelo D-pad, entre num vídeo, veja se a barra se esconde sozinha sobre o conteúdo e se o B volta ao estado anterior. Anote se o foco "pula" algum card (pontuação do `spatial`). 3. **Digitar sem teclado físico.** Pesquise "previsão do tempo" com o teclado virtual (incluindo uma palavra com acento, como "ação"), navegue pelas sugestões e abra o resultado. Marque se o cursor virtual (modo Spatial) chegou em tudo. 4. **Três abas.** Abra três abas, alterne pelo seletor em grade e feche uma, só com o controle. Veja se as miniaturas aparecem e se o foco começa na aba ativa. -5. **Permissão de microfone.** Abra um site que peça o microfone (ou o teste do WebRTC), percorra o prompt com o D-pad e escolha "permitir sempre". Confirme em Configurações que a permissão ficou salva para o site. -6. **Atualização.** Em Windows/Linux: com uma versão nova publicada, deixe o download acontecer, feche o app e reabra; no `.deb`/`.rpm` use **Reiniciar e instalar** em Configurações → Sobre e confirme o pedido de senha. No macOS, o app deve avisar e abrir a página da release. -7. **Streaming com DRM.** Abra a Netflix, entre na conta e toque um título. Registre a resolução que o player mostra e o que a linha "Widevine (DRM)" diz em Configurações → Sobre. +5. **Campo de texto com A.** Numa página com um campo (a caixa de busca da nova aba, ou a do YouTube), leve o D-pad até o campo: o teclado **não** deve abrir só de passar por ele. Aperte A: o teclado abre, o D-pad e o analógico andam pelas teclas a partir da primeira, e A escreve no campo. Feche com B e repita: a segunda vez tem de funcionar igual à primeira. +6. **Subir até a barra.** No topo de qualquer página, aperte para cima: o foco vai para a barra de endereço (sem abrir o teclado). Ande para os lados até voltar, avançar e recarregar, suba até as abas e o +, e aperte A em cada um. Para baixo (ou B) volta para a página. Os botões de minimizar, maximizar e fechar ficam de fora de propósito. +7. **Permissão de microfone.** Abra um site que peça o microfone (ou o teste do WebRTC), percorra o prompt com o D-pad e escolha "permitir sempre". Confirme em Configurações que a permissão ficou salva para o site. +8. **Atualização.** Em Windows/Linux: com uma versão nova publicada, deixe o download acontecer, feche o app e reabra; no `.deb`/`.rpm` use **Reiniciar e instalar** em Configurações → Sobre e confirme o pedido de senha. No macOS, o app deve avisar e abrir a página da release. +9. **Streaming com DRM.** Abra a Netflix, entre na conta e toque um título. Registre a resolução que o player mostra e o que a linha "Widevine (DRM)" diz em Configurações → Sobre. Leitor de tela é o outro lado da mesma sessão: o roteiro de Orca, Narrator e VoiceOver está em [SCREEN-READER.md](SCREEN-READER.md).