diff --git a/app/src/assets/spatial.js b/app/src/assets/spatial.js index d5f08ed..5bff8da 100644 --- a/app/src/assets/spatial.js +++ b/app/src/assets/spatial.js @@ -254,41 +254,91 @@ })(); // The controller's pointer: the left stick moves it and A clicks where it points (real mouse events at the same spot, -// sent by the shell). It is a ring, not an arrow, so it never passes for the system mouse; the dot in the middle is the -// spot that gets clicked. +// sent by the shell). An arrow that turns into a pointing hand over anything clickable, like a mouse; white with a black +// edge and a shadow so it reads on any page. The tip of the arrow and of the finger is the spot that gets clicked. (() => { if (window.__joyCursor) return; - const SIZE = 34; - const ring = '' + - '' + - '' + - ''; + const svg = (box, body) => 'url("data:image/svg+xml;utf8,' + encodeURIComponent( + `${body}`) + '")'; + const SHAPES = { + arrow: { + width: 24, height: 34, tipX: 2.4, tipY: 2.4, + image: svg('0 0 20 28', ''), + }, + hand: { + width: 30, height: 35, tipX: 11.5, tipY: 2, + image: svg('0 0 26 30', '' + + ''), + }, + }; + // What a mouse would show a hand over: links, buttons and the like, and anything a page gives cursor: pointer. + const CLICKABLE = 'a[href], button, summary, label, select, [role="button"], [role="link"], [role="tab"], [role="menuitem"], ' + + '[role="option"], input[type="submit"], input[type="button"], input[type="checkbox"], input[type="radio"]'; + const overClickable = (x, y) => { + const el = document.elementFromPoint(x, y); + if (!el || el.tagName === 'IFRAME') return false; + return Boolean(el.closest(CLICKABLE)) || getComputedStyle(el).cursor === 'pointer'; + }; let el = null; + let shape = ''; let at = { x: 0, y: 0 }; const ensure = () => { if (el && document.contains(el)) return el; el = document.createElement('div'); el.id = '__joy-cursor'; el.setAttribute('aria-hidden', 'true'); - el.style.cssText = `position:fixed;left:0;top:0;width:${SIZE}px;height:${SIZE}px;z-index:2147483647;pointer-events:none;display:none;` + - 'filter:drop-shadow(0 1px 3px rgba(0,0,0,0.55));transition:scale 90ms ease-out;' + - 'background:url("data:image/svg+xml;utf8,' + encodeURIComponent(ring) + '") no-repeat center / contain;'; + el.style.cssText = 'position:fixed;left:0;top:0;z-index:2147483647;pointer-events:none;display:none;will-change:transform;' + + 'background-repeat:no-repeat;background-size:contain;filter:drop-shadow(0 2px 3px rgba(0,0,0,0.45));' + + 'transform-origin:var(--tip-x) var(--tip-y);transition:scale 90ms ease-out;'; + shape = ''; (document.body || document.documentElement).appendChild(el); return el; }; + const setShape = (c, name) => { + if (shape === name) return; + shape = name; + const s = SHAPES[name]; + c.style.width = `${s.width}px`; + c.style.height = `${s.height}px`; + c.style.backgroundImage = s.image; + c.style.setProperty('--tip-x', `${s.tipX}px`); + c.style.setProperty('--tip-y', `${s.tipY}px`); + }; + // Positions arrive from the shell a few dozen times a second, unevenly; the page draws the pointer on every frame of the + // screen instead, gliding the rest of the way to the newest one, so it moves smoothly whatever the arrival rate. A click + // goes to the newest position, never to the drawn one. + let drawn = null; + let frame = 0; + const paint = () => { + frame = 0; + if (!el || !drawn) return; + drawn.x += (at.x - drawn.x) * 0.5; + drawn.y += (at.y - drawn.y) * 0.5; + if (Math.abs(at.x - drawn.x) < 0.3 && Math.abs(at.y - drawn.y) < 0.3) drawn = { x: at.x, y: at.y }; + const s = SHAPES[shape]; + el.style.transform = `translate(${drawn.x - s.tipX}px, ${drawn.y - s.tipY}px)`; + if (drawn.x !== at.x || drawn.y !== at.y) frame = requestAnimationFrame(paint); + }; window.__joyCursor = { show(visible, x, y) { const c = ensure(); + const appearing = c.style.display !== 'block'; at = { x, y }; c.style.display = visible ? 'block' : 'none'; - // The click lands on (x, y): the ring is centred on it. - c.style.transform = `translate(${x - SIZE / 2}px, ${y - SIZE / 2}px)`; + if (!visible) return; + setShape(c, overClickable(x, y) ? 'hand' : 'arrow'); + // Appearing, it shows up where it is rather than sliding in from where it was last seen. + if (appearing || !drawn) drawn = { x, y }; + if (!frame) frame = requestAnimationFrame(paint); }, - // A short squeeze on every click, so a press from the sofa is seen to land. + // A short squeeze around the tip on every click, so a press from the sofa is seen to land. press() { const c = ensure(); if (c.style.display === 'none') this.show(true, at.x, at.y); - c.style.scale = '0.75'; + c.style.scale = '0.82'; setTimeout(() => { c.style.scale = '1'; }, 110); }, }; diff --git a/app/src/main/controllerActions.ts b/app/src/main/controllerActions.ts index de0ec86..6c12f9f 100644 --- a/app/src/main/controllerActions.ts +++ b/app/src/main/controllerActions.ts @@ -64,6 +64,8 @@ export class VirtualCursor { x = 200; y = 200; placed = false; + private drawing = false; + private redraw = false; place(x: number, y: number, width: number, height: number): void { this.x = Math.min(Math.max(0, x), Math.max(0, width - 1)); @@ -78,7 +80,28 @@ export class VirtualCursor { async moveBy(wc: WebContents, dx: number, dy: number, width: number, height: number): Promise { this.place(this.x + dx, this.y + dy, width, height); wc.sendInputEvent({ type: "mouseMove", x: Math.round(this.x), y: Math.round(this.y) }); - await this.show(wc, true); + await this.draw(wc); + } + + /** + * Draws the pointer where it is now. Moves arrive about 60 times a second; while one drawing is still on its way into + * the page, the next ones only ask for another, which then uses the latest position: the pointer never lags behind a + * queue of old ones. + */ + private async draw(wc: WebContents): Promise { + if (this.drawing) { + this.redraw = true; + return; + } + this.drawing = true; + try { + do { + this.redraw = false; + await this.show(wc, true); + } while (this.redraw); + } finally { + this.drawing = false; + } } click(wc: WebContents, button: "left" | "right" = "left"): void { diff --git a/app/src/main/services/navigation.ts b/app/src/main/services/navigation.ts index 3bb0d5a..e18cf8c 100644 --- a/app/src/main/services/navigation.ts +++ b/app/src/main/services/navigation.ts @@ -8,7 +8,8 @@ export interface NavigationHost { readonly tabs: TabManager; readonly settings: () => AppSettings; readonly httpAllowedHosts: Set; - openKeyboard(target: "address" | "page"): void; + /** The address field takes the focus with the controller keyboard open on it: where a new tab starts. */ + typeInAddress(): void; closeKeyboard(): void; } @@ -39,6 +40,7 @@ export class Navigation { newTab(isPrivate = false): void { const target = isPrivate ? Pages.newtab : newTabTarget(this.host.settings()); this.host.tabs.open(target, { isPrivate }); - if (target !== Pages.newtab) this.host.openKeyboard("address"); + // A new tab is for going somewhere: it opens on the address field with the keyboard up, whatever page it shows. + this.host.typeInAddress(); } } diff --git a/app/src/main/shell.ts b/app/src/main/shell.ts index ec23bca..1d2793e 100644 --- a/app/src/main/shell.ts +++ b/app/src/main/shell.ts @@ -115,7 +115,7 @@ export class Shell { tabs: this.tabs, settings: () => this.deps.store.current, httpAllowedHosts: this.httpAllowedHosts, - openKeyboard: (target) => this.openKeyboard(target), + typeInAddress: () => this.typeInAddress(), closeKeyboard: () => this.closeKeyboard(), }); this.wireTabs(); @@ -313,6 +313,17 @@ export class Shell { if (!this.wired.has(tab.webContents)) { this.wired.add(tab.webContents); this.attachShortcuts(tab.webContents); + // A view takes the window's focus as it is attached (a new tab, about 10 ms after the shell gave its address field the + // focus). While the keyboard types into that field the focus stays with the shell, or the D-pad walks dead keys. + // The tab hears of its focus before the shell hears it lost it, so the focus goes back once the swap is over. + tab.webContents.on("focus", () => { + if (this.keyboard?.target !== "address") return; + setImmediate(() => { + if (this.keyboard?.target !== "address" || this.window.isDestroyed()) return; + this.window.webContents.focus(); + this.chromeFocused = true; + }); + }); } if (!contentView.children.includes(tab.view)) contentView.addChildView(tab.view); tab.view.setVisible(tab === this.tabs.active && !this.pageCovered); @@ -563,6 +574,12 @@ export class Shell { this.window.webContents.send(Channels.shellFocusChrome); } + /** The toolbar takes the D-pad (see enterChrome) and the keyboard opens on the address field. */ + typeInAddress(): void { + this.enterChrome(); + this.openKeyboard("address"); + } + leaveChrome(): void { this.chromeFocused = false; this.tabs.activeWebContents?.focus(); diff --git a/app/src/renderer/shell/app.tsx b/app/src/renderer/shell/app.tsx index 2b267b8..d55c383 100644 --- a/app/src/renderer/shell/app.tsx +++ b/app/src/renderer/shell/app.tsx @@ -236,13 +236,17 @@ function AddressBar({ url, focused, setFocused, language, keyboard, children }: const fresh = keyboard && isInternal(url); const shown = fresh ? "" : url; const [value, setValue] = useState(shown); - useEffect(() => { if (!focused) setValue(shown); }, [shown, focused]); + // The field follows the tab's address until the user types in it: a new tab focuses the field, and a page that loads + // meanwhile must show its address, but text being typed is never overwritten. + const edited = useRef(false); + useEffect(() => { if (!focused || !edited.current) setValue(shown); }, [shown, focused]); const submit = () => cmd({ type: "search-or-navigate", text: value }); return (
{ e.preventDefault(); submit(); setFocused(false); (input.current as HTMLInputElement | null)?.blur(); }} + // Going somewhere hands the focus to the page that loads, as in Edge; the D-pad then walks the page, not the bar. + onSubmit={(e) => { e.preventDefault(); submit(); setFocused(false); (input.current as HTMLInputElement | null)?.blur(); cmd({ type: "focus-page" }); }} > cmd({ type: "keyboard-open", target: "address" })} - onFocus={() => { setFocused(true); (input.current as HTMLInputElement).select(); }} + onFocus={() => { setFocused(true); edited.current = false; (input.current as HTMLInputElement).select(); }} onBlur={() => setFocused(false)} onKeyDown={(e) => { // Down leaves the field for the suggestions the controller keyboard shows, so both inputs reach them. @@ -268,6 +272,7 @@ function AddressBar({ url, focused, setFocused, language, keyboard, children }: }} onInput={(e) => { const text = (e.target as HTMLInputElement).value; + edited.current = true; setValue(text); cmd({ type: "address-input", text }); }} diff --git a/app/src/renderer/shell/gamepad.ts b/app/src/renderer/shell/gamepad.ts index 0aed43c..149eb66 100644 --- a/app/src/renderer/shell/gamepad.ts +++ b/app/src/renderer/shell/gamepad.ts @@ -34,6 +34,7 @@ export function useGamepad(state: ShellState, overlay: Overlay, setOverlay: (o: const family = useRef(undefined); const lastDirection = useRef(0); const lastStick = useRef(0); + const lastPointerMove = useRef(0); /** * Who A answers to on a page: the stick's pointer once the stick moved it, the D-pad's focus once the D-pad moved. * The last one used wins, so both can be mixed on the same page. @@ -168,14 +169,20 @@ export function useGamepad(state: ShellState, overlay: Overlay, setOverlay: (o: const page = !overlayOpen && !chrome; const pointer = page && s.inputMode !== "Arrows"; - // Buttons follow the poll rate; the sticks keep their own, slower cadence so the pointer moves at the same speed. - if (page && now - lastStick.current >= STICK_INTERVAL_MS) { - lastStick.current = now; - const velocity = pointer ? stickToVelocity(lx, ly) : { dx: 0, dy: 0 }; + // The pointer moves on every poll, so it glides at the screen's rate instead of jumping 20 times a second; the step + // is scaled by the time since the last one, so the speed (set per STICK_INTERVAL_MS) does not depend on the rate. + const sinceMove = Math.min(now - lastPointerMove.current, STICK_INTERVAL_MS) / STICK_INTERVAL_MS; + lastPointerMove.current = now; + if (pointer) { + const velocity = stickToVelocity(lx, ly); if (velocity.dx !== 0 || velocity.dy !== 0) { pointing.current = true; - bridge().controller({ type: "cursor-move", dx: velocity.dx, dy: velocity.dy }); + bridge().controller({ type: "cursor-move", dx: velocity.dx * sinceMove, dy: velocity.dy * sinceMove }); } + } + // The wheel keeps the slower cadence: one notch per STICK_INTERVAL_MS is a readable scroll speed. + if (page && now - lastStick.current >= STICK_INTERVAL_MS) { + lastStick.current = now; const notches = stickToScroll(ry); const sideways = stickToScroll(rx); if (notches !== 0 || sideways !== 0) bridge().controller({ type: "scroll", notches, sideways }); diff --git a/app/tests/e2e/controller.spec.ts b/app/tests/e2e/controller.spec.ts index 0eeb558..8fdfb4c 100644 --- a/app/tests/e2e/controller.spec.ts +++ b/app/tests/e2e/controller.spec.ts @@ -320,3 +320,34 @@ test("the keyboard's own shortcuts: Y space, LT shift, X delete, LS the layer, a await step(10); await expect.poll(layerKey).toBe("abc"); }); + +test("a new tab opens on the address field with the keyboard up, and the D-pad types there", async () => { + await installPads([XBOX]); + await app.evaluate(() => globalThis.joy.command({ type: "new-tab" } as never)); + await expect(chrome.locator("#keyboard")).toBeVisible(); + await expect.poll(shellFocus).toBe("address"); + await expect.poll(shellHasWindowFocus).toBe(true); + // The first press goes into the keys, at "q"; A types it into the address. + await step(Pad.Right); + await expect.poll(shellFocus).toBe("q"); + await step(Pad.A); + await expect.poll(() => chrome.locator("#address").inputValue()).toBe("q"); +}); + +test("the pointer is an arrow, and a hand over what can be clicked", async () => { + await app.evaluate(({ protocol }, page) => { + protocol.handle("https", () => new Response(page, { headers: { "content-type": "text/html" } })); + }, POINTER_PAGE); + await app.evaluate(() => globalThis.joy.command({ type: "navigate", url: "https://pointer.example/" } as never)); + await expect.poll(async () => { + const tab = (await tabs()).find((t) => t.isActive); + return tab?.url === "https://pointer.example/" && !tab.isLoading; + }, { timeout: 30_000 }).toBe(true); + const shapeAt = async (where: string) => { + await app.evaluate(({}, code) => globalThis.joy.spatialEval(code), `(() => { const r = ${where}.getBoundingClientRect(); window.__joyCursor.show(true, r.left + r.width / 2, r.top + r.height / 2); })()`); + // The hand is the shape with fingers: its path starts at the fingertip. + return app.evaluate(() => globalThis.joy.pageEval("document.getElementById('__joy-cursor').style.backgroundImage.includes('M10%201.6') ? 'hand' : 'arrow'")); + }; + expect(await shapeAt("document.getElementById('area')")).toBe("arrow"); + expect(await shapeAt("document.getElementById('b')")).toBe("hand"); +}); diff --git a/app/tests/e2e/journeys.spec.ts b/app/tests/e2e/journeys.spec.ts index 11c8001..f0108bd 100644 --- a/app/tests/e2e/journeys.spec.ts +++ b/app/tests/e2e/journeys.spec.ts @@ -444,6 +444,11 @@ async function quickMenu(h: Harness, label: RegExp): Promise { await focusOn(h, "shell", `[...document.querySelectorAll('.menu .menu-item')].find((b) => ${label.toString()}.test((b.textContent || '').trim()))`, `menu item ${label}`); await press(h, Pad.A); await expect(h.chrome.locator(".overlay .menu")).toHaveCount(0); + // A new tab opens on the address field with the keyboard up; B puts it away to go on with the pad. + if (await h.chrome.locator("#keyboard").count()) { + await press(h, Pad.B); + await expect(h.chrome.locator("#keyboard")).toHaveCount(0); + } } test("three tabs, switching in the tab switcher and closing one, all from the pad", async () => {