From c03127a4958a7b0aecff57f6525f03e6ba3f6c4c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ka=C4=9Fan=20Ar=C4=B1ba=C5=9F?= <172207405+aribaskagan@users.noreply.github.com> Date: Sat, 14 Nov 2026 15:34:08 +0000 Subject: [PATCH 1/2] fix(scroll): clamp WebDriver and HarmonyOS swipes inside the container Directional scroll on those owners ignored containerRect, so a swipe could start or end outside the resolved list. The shared planner now shortens that gesture and reports the dispatched distance beside the requested one. --- .../webdriver-harmony-scroll-container.md | 5 + .../src/__tests__/input-actions.test.ts | 102 ++++++++++++++++ .../platform-harmonyos/src/input-actions.ts | 16 ++- .../src/webdriver-interactor-scroll.test.ts | 111 ++++++++++++++++++ .../src/webdriver-interactor.ts | 12 +- 5 files changed, 243 insertions(+), 3 deletions(-) create mode 100644 .changeset/webdriver-harmony-scroll-container.md create mode 100644 packages/provider-webdriver/src/webdriver-interactor-scroll.test.ts diff --git a/.changeset/webdriver-harmony-scroll-container.md b/.changeset/webdriver-harmony-scroll-container.md new file mode 100644 index 0000000000..ae0f984b7f --- /dev/null +++ b/.changeset/webdriver-harmony-scroll-container.md @@ -0,0 +1,5 @@ +--- +'agent-device': patch +--- + +WebDriver and HarmonyOS directional scrolls stay inside the resolved scroll container. A swipe that does not fit is shortened, and the response reports the distance that was dispatched beside the distance that was asked for. diff --git a/packages/platform-harmonyos/src/__tests__/input-actions.test.ts b/packages/platform-harmonyos/src/__tests__/input-actions.test.ts index 85fdf9a8fd..82db1091da 100644 --- a/packages/platform-harmonyos/src/__tests__/input-actions.test.ts +++ b/packages/platform-harmonyos/src/__tests__/input-actions.test.ts @@ -1,7 +1,9 @@ import assert from 'node:assert/strict'; import { beforeEach, test, vi } from 'vitest'; import type { GesturePlan } from '@agent-device/contracts/gesture-plan-types'; +import { buildScrollGesturePlan } from '@agent-device/contracts/scroll-gesture'; import type { DeviceInfo } from '@agent-device/kernel/device'; +import type { Rect } from '@agent-device/kernel/snapshot'; const { runHarmonyShell, sleep, readHarmonyGestureViewport, invalidateHarmonyGestureViewport } = vi.hoisted(() => ({ @@ -96,6 +98,106 @@ test('HarmonyOS scroll derives a viewport-aware swipe plan', async () => { ]); }); +const PHONE = { x: 0, y: 0, width: 402, height: 874 }; +/** Inset under a navigation bar and above a footer, shorter than a 0.75 swipe. */ +const INSET_LIST: Rect = { x: 18, y: 178, width: 366, height: 500 }; +/** The same list shifted into an application window that is not at the screen origin. */ +const OFFSET_WINDOW = { x: 40, y: 80, width: 402, height: 874 }; + +function strictlyInside(point: { x: number; y: number }, rect: Rect): boolean { + return ( + point.x > rect.x && + point.x < rect.x + rect.width && + point.y > rect.y && + point.y < rect.y + rect.height + ); +} + +function dispatchedSwipe(callIndex?: number): { x1: number; y1: number; x2: number; y2: number } { + const calls = runHarmonyShell.mock.calls; + const call = calls[callIndex ?? calls.length - 1]; + const args = call?.[1] as readonly unknown[]; + return { + x1: Number(args[3]), + y1: Number(args[4]), + x2: Number(args[5]), + y2: Number(args[6]), + }; +} + +test('HarmonyOS scroll keeps an inset list swipe inside the container and reports the shortened span', async () => { + readHarmonyGestureViewport.mockResolvedValue(PHONE); + const unclamped = await scrollHarmony(DEVICE, 'down', { amount: 0.75 }); + const outside = dispatchedSwipe(0); + const result = await scrollHarmony(DEVICE, 'down', { amount: 0.75, containerRect: INSET_LIST }); + const swipe = dispatchedSwipe(1); + + assert.equal( + strictlyInside({ x: outside.x1, y: outside.y1 }, INSET_LIST) && + strictlyInside({ x: outside.x2, y: outside.y2 }, INSET_LIST), + false, + 'the fixture must leave the list', + ); + assert.ok(strictlyInside({ x: swipe.x1, y: swipe.y1 }, INSET_LIST), JSON.stringify(swipe)); + assert.ok(strictlyInside({ x: swipe.x2, y: swipe.y2 }, INSET_LIST), JSON.stringify(swipe)); + assert.deepEqual(swipe, { x1: result.x1, y1: result.y1, x2: result.x2, y2: result.y2 }); + assert.equal(result.pixels, Math.abs(Number(result.y2) - Number(result.y1))); + assert.ok(Number(result.pixels) < Number(unclamped.pixels)); + assert.equal(result.requestedPixels, unclamped.pixels); +}); + +test('HarmonyOS scroll translates an offset window into the swipe frame before clamping', async () => { + readHarmonyGestureViewport.mockResolvedValue(OFFSET_WINDOW); + const list: Rect = { + x: OFFSET_WINDOW.x + INSET_LIST.x, + y: OFFSET_WINDOW.y + INSET_LIST.y, + width: INSET_LIST.width, + height: INSET_LIST.height, + }; + const result = await scrollHarmony(DEVICE, 'up', { amount: 0.75, containerRect: list }); + const swipe = dispatchedSwipe(); + const inSnapshotSpace = { + x1: swipe.x1 + OFFSET_WINDOW.x, + y1: swipe.y1 + OFFSET_WINDOW.y, + x2: swipe.x2 + OFFSET_WINDOW.x, + y2: swipe.y2 + OFFSET_WINDOW.y, + }; + + assert.ok(strictlyInside({ x: inSnapshotSpace.x1, y: inSnapshotSpace.y1 }, list)); + assert.ok(strictlyInside({ x: inSnapshotSpace.x2, y: inSnapshotSpace.y2 }, list)); + assert.ok(Number(result.pixels) < Number(result.requestedPixels)); +}); + +test('HarmonyOS scroll leaves a swipe that already fits its container unchanged', async () => { + const fitting: Rect = { x: 18, y: 62, width: 366, height: 729 }; + readHarmonyGestureViewport.mockResolvedValue(PHONE); + const unclamped = await scrollHarmony(DEVICE, 'down', { amount: 0.75 }); + const result = await scrollHarmony(DEVICE, 'down', { amount: 0.75, containerRect: fitting }); + + assert.deepEqual(result, unclamped); + assert.equal('requestedPixels' in result, false); +}); + +test('HarmonyOS scroll without a container keeps the unclamped swipe', async () => { + readHarmonyGestureViewport.mockResolvedValue(OFFSET_WINDOW); + const plan = await scrollHarmony(DEVICE, 'down', { amount: 0.75 }); + const expected = buildScrollGesturePlan({ + direction: 'down', + amount: 0.75, + referenceWidth: OFFSET_WINDOW.width, + referenceHeight: OFFSET_WINDOW.height, + }); + + assert.deepEqual(dispatchedSwipe(), { + x1: expected.x1, + y1: expected.y1, + x2: expected.x2, + y2: expected.y2, + }); + assert.equal(plan.pixels, expected.pixels); + assert.equal('requestedPixels' in plan, false); +}); + test('HarmonyOS gestures lower a single trajectory and reject multi-touch', async () => { const singlePlan: GesturePlan = { topology: 'single', diff --git a/packages/platform-harmonyos/src/input-actions.ts b/packages/platform-harmonyos/src/input-actions.ts index 1500465720..a6259e403b 100644 --- a/packages/platform-harmonyos/src/input-actions.ts +++ b/packages/platform-harmonyos/src/input-actions.ts @@ -1,11 +1,15 @@ import type { DeviceRotation } from '@agent-device/contracts/device'; import type { GesturePlan } from '@agent-device/contracts/gesture-plan-types'; -import { DEFAULT_MOBILE_SCROLL_DURATION_MS } from '@agent-device/contracts/scroll-command'; +import { + type ScrollContainerTarget, + DEFAULT_MOBILE_SCROLL_DURATION_MS, +} from '@agent-device/contracts/scroll-command'; import { type ScrollDirection, buildScrollGesturePlan, } from '@agent-device/contracts/scroll-gesture'; import type { DeviceInfo } from '@agent-device/kernel/device'; +import type { Rect } from '@agent-device/kernel/snapshot'; import { AppError } from '@agent-device/kernel/errors'; import { sleep } from '@agent-device/host-kit/retry'; import { runHarmonyShell } from './hdc.ts'; @@ -52,7 +56,7 @@ export async function fillHarmony( export async function scrollHarmony( device: DeviceInfo, direction: ScrollDirection, - options?: { amount?: number; pixels?: number; durationMs?: number }, + options?: { amount?: number; pixels?: number; durationMs?: number } & ScrollContainerTarget, ): Promise> { const viewport = await readHarmonyGestureViewport(device); const plan = buildScrollGesturePlan({ @@ -61,6 +65,9 @@ export async function scrollHarmony( pixels: options?.pixels, referenceWidth: viewport.width, referenceHeight: viewport.height, + ...(options?.containerRect === undefined + ? {} + : { container: relativeToViewport(options.containerRect, viewport) }), }); await runHarmonyShell(device, [ 'uitest', @@ -75,6 +82,11 @@ export async function scrollHarmony( return plan; } +/** The planner's box is the viewport size at the origin; a snapshot container is absolute. */ +function relativeToViewport(rect: Rect, viewport: Rect): Rect { + return { ...rect, x: rect.x - viewport.x, y: rect.y - viewport.y }; +} + /** Lowers HDC's public one-pointer primitives from a shared semantic plan. */ export async function performHarmonyGesture( device: DeviceInfo, diff --git a/packages/provider-webdriver/src/webdriver-interactor-scroll.test.ts b/packages/provider-webdriver/src/webdriver-interactor-scroll.test.ts new file mode 100644 index 0000000000..e0b59dbc4c --- /dev/null +++ b/packages/provider-webdriver/src/webdriver-interactor-scroll.test.ts @@ -0,0 +1,111 @@ +import assert from 'node:assert/strict'; +import { test } from 'vitest'; +import type { Rect } from '@agent-device/kernel/snapshot'; +import { createCloudWebDriverCapabilities } from './capabilities.ts'; +import type { + W3CActionSequence, + WebDriverClient, + WebDriverWindowRect, +} from './webdriver-client.ts'; +import { createWebDriverInteractor } from './webdriver-interactor.ts'; + +const WINDOW: WebDriverWindowRect = { x: 12, y: 30, width: 402, height: 874 }; +/** Inset under a navigation bar and above a footer, in the window's own coordinates. */ +const INSET_LIST: Rect = { x: 18, y: 178, width: 366, height: 500 }; +const FITTING_LIST: Rect = { x: 18, y: 62, width: 366, height: 729 }; + +function place(rect: Rect, frame: WebDriverWindowRect): Rect { + return { ...rect, x: rect.x + frame.x, y: rect.y + frame.y }; +} + +function strictlyInside(point: { x: number; y: number }, rect: Rect): boolean { + return ( + point.x > rect.x && + point.x < rect.x + rect.width && + point.y > rect.y && + point.y < rect.y + rect.height + ); +} + +function scrollWorld(frame: WebDriverWindowRect) { + const performed: W3CActionSequence[][] = []; + const client = { + hideKeyboard: async () => undefined, + source: async () => { + throw new Error('page source unavailable'); + }, + windowRect: async () => frame, + performActions: async (actions: W3CActionSequence[]) => { + performed.push(actions); + }, + releaseActions: async () => undefined, + } as unknown as WebDriverClient; + return { + performed, + interactor: createWebDriverInteractor({ + client, + backend: 'android', + capabilities: createCloudWebDriverCapabilities({ provider: 'test', platform: 'android' }), + }), + }; +} + +function swipeEndpoints(actions: W3CActionSequence[] | undefined): { + x1: number; + y1: number; + x2: number; + y2: number; +} { + const moves = actions?.[0]?.actions.filter((action) => action.type === 'pointerMove') ?? []; + const start = moves[0]; + const end = moves[1]; + if (start?.type !== 'pointerMove' || end?.type !== 'pointerMove') { + assert.fail('swipe is missing its pointer moves'); + } + return { x1: start.x, y1: start.y, x2: end.x, y2: end.y }; +} + +test('WebDriver scroll keeps an inset list swipe inside the container and reports the shortened span', async () => { + const { interactor, performed } = scrollWorld(WINDOW); + const list = place(INSET_LIST, WINDOW); + const unclamped = await interactor.scroll('down', { amount: 0.75 }); + const outside = swipeEndpoints(performed[0]); + const result = await interactor.scroll('down', { amount: 0.75, containerRect: list }); + const swipe = swipeEndpoints(performed[1]); + + assert.equal( + strictlyInside({ x: outside.x1, y: outside.y1 }, list) && + strictlyInside({ x: outside.x2, y: outside.y2 }, list), + false, + 'the fixture must leave the list', + ); + assert.ok(strictlyInside({ x: swipe.x1, y: swipe.y1 }, list), JSON.stringify(swipe)); + assert.ok(strictlyInside({ x: swipe.x2, y: swipe.y2 }, list), JSON.stringify(swipe)); + assert.deepEqual(swipe, { x1: result?.x1, y1: result?.y1, x2: result?.x2, y2: result?.y2 }); + assert.equal(result?.pixels, Math.abs(Number(result?.y2) - Number(result?.y1))); + assert.equal(result?.distance, result?.pixels); + assert.ok(Number(result?.pixels) < Number(unclamped?.pixels)); + assert.equal(result?.requestedPixels, unclamped?.pixels); +}); + +test('WebDriver scroll leaves a swipe that already fits its container unchanged', async () => { + const { interactor } = scrollWorld(WINDOW); + const unclamped = await interactor.scroll('down', { amount: 0.75 }); + const result = await interactor.scroll('down', { + amount: 0.75, + containerRect: place(FITTING_LIST, WINDOW), + }); + + assert.deepEqual(result, unclamped); + assert.equal(result !== undefined && 'requestedPixels' in result, false); +}); + +test('WebDriver scroll without a container keeps the window swipe', async () => { + const { interactor, performed } = scrollWorld(WINDOW); + const result = await interactor.scroll('up', { amount: 0.75 }); + const swipe = swipeEndpoints(performed[0]); + + assert.deepEqual(swipe, { x1: result?.x1, y1: result?.y1, x2: result?.x2, y2: result?.y2 }); + assert.equal(result !== undefined && 'requestedPixels' in result, false); + assert.equal(result?.distance, result?.pixels); +}); diff --git a/packages/provider-webdriver/src/webdriver-interactor.ts b/packages/provider-webdriver/src/webdriver-interactor.ts index 966234c37e..472bff3f4b 100644 --- a/packages/provider-webdriver/src/webdriver-interactor.ts +++ b/packages/provider-webdriver/src/webdriver-interactor.ts @@ -10,6 +10,7 @@ import type { KeyboardStatusResult, } from '@agent-device/contracts/interactor-types'; import type { GesturePlan } from '@agent-device/contracts/gesture-plan-types'; +import type { ScrollExecutionOptions } from '@agent-device/contracts/scroll-command'; import { type ScrollDirection, buildScrollGesturePlan, @@ -17,6 +18,7 @@ import { import type { TvRemoteButton } from '@agent-device/contracts/tv-remote'; import type { SettingOptions } from '@agent-device/contracts/settings'; import { snapshotViewportSizeFrom } from '@agent-device/kernel/rect'; +import type { Rect } from '@agent-device/kernel/snapshot'; import { AppError } from '@agent-device/kernel/errors'; import { capabilitySupported, @@ -84,6 +86,11 @@ function textEntryClearTargetUnavailableError(): AppError { ); } +/** The planner's box starts at the frame origin; a snapshot container is absolute. */ +function relativeToFrame(container: Rect, frame: WebDriverWindowRect): Rect { + return { ...container, x: container.x - frame.x, y: container.y - frame.y }; +} + function rectContains(rect: WebDriverWindowRect, x: number, y: number): boolean { return x >= rect.x && x <= rect.x + rect.width && y >= rect.y && y <= rect.y + rect.height; } @@ -238,7 +245,7 @@ class WebDriverInteractor implements Interactor { async scroll( direction: ScrollDirection, - options?: { amount?: number; pixels?: number; durationMs?: number }, + options?: ScrollExecutionOptions, ): Promise> { this.requireSupport('scroll'); const durationMs = options?.durationMs ?? 350; @@ -250,6 +257,9 @@ class WebDriverInteractor implements Interactor { pixels: options?.pixels, referenceWidth: rect.width, referenceHeight: rect.height, + ...(options?.containerRect === undefined + ? {} + : { container: relativeToFrame(options.containerRect, rect) }), }); const absolutePlan = { ...plan, From ed6665a4876435c284ac4fb640dc58120ee41cf2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ka=C4=9Fan=20Ar=C4=B1ba=C5=9F?= <172207405+aribaskagan@users.noreply.github.com> Date: Sun, 15 Nov 2026 11:42:08 +0000 Subject: [PATCH 2/2] fix(scroll): dispatch offset-window swipes in screen coordinates HarmonyOS translated the container into a zero-origin frame and then sent those local endpoints to hdc, which takes screen coordinates. The planner now accepts the frame origin, plans in that zero-origin frame, and returns absolute endpoints, so HarmonyOS, WebDriver, and Android dispatch the plan instead of translating it themselves. --- .../src/scroll-gesture-container.test.ts | 70 +++++++++++++++++++ packages/contracts/src/scroll-gesture.ts | 31 ++++++-- .../platform-android/src/input-actions.ts | 18 ++--- .../src/__tests__/input-actions.test.ts | 31 ++++---- .../platform-harmonyos/src/input-actions.ts | 11 +-- packages/platform-harmonyos/src/snapshot.ts | 2 +- .../src/webdriver-interactor.ts | 28 ++------ 7 files changed, 121 insertions(+), 70 deletions(-) diff --git a/packages/contracts/src/scroll-gesture-container.test.ts b/packages/contracts/src/scroll-gesture-container.test.ts index 591bf4df06..0dd541fa50 100644 --- a/packages/contracts/src/scroll-gesture-container.test.ts +++ b/packages/contracts/src/scroll-gesture-container.test.ts @@ -170,6 +170,76 @@ test.each<[ScrollDirection, string, Rect]>([ }, ); +test('a zero reference origin matches a plan that omits it', () => { + const omitted = buildScrollGesturePlan({ + direction: 'down', + amount: 0.75, + ...FRAME, + container: FOOTERED_CONTAINER, + }); + const explicit = buildScrollGesturePlan({ + direction: 'down', + amount: 0.75, + ...FRAME, + referenceOrigin: { x: 0, y: 0 }, + container: FOOTERED_CONTAINER, + }); + + assert.deepEqual(explicit, omitted); +}); + +test('an offset reference origin returns absolute endpoints inside the absolute container', () => { + const origin = { x: 40, y: 80 }; + const container: Rect = { + x: origin.x + FOOTERED_CONTAINER.x, + y: origin.y + FOOTERED_CONTAINER.y, + width: FOOTERED_CONTAINER.width, + height: FOOTERED_CONTAINER.height, + }; + const local = buildScrollGesturePlan({ + direction: 'up', + amount: 0.75, + ...FRAME, + container: FOOTERED_CONTAINER, + }); + const plan = buildScrollGesturePlan({ + direction: 'up', + amount: 0.75, + ...FRAME, + referenceOrigin: origin, + container, + }); + + assert.deepEqual( + [plan.x1, plan.y1, plan.x2, plan.y2], + [local.x1 + origin.x, local.y1 + origin.y, local.x2 + origin.x, local.y2 + origin.y], + ); + assert.ok(strictlyInside(plan, container), JSON.stringify(plan)); + assert.equal(plan.pixels, local.pixels); + assert.equal(plan.requestedPixels, local.requestedPixels); + assert.equal(plan.referenceWidth, FRAME.referenceWidth); + assert.equal(plan.referenceHeight, FRAME.referenceHeight); +}); + +test('an offset reference origin shifts an unclamped swipe into the dispatch frame', () => { + const origin = { x: 40, y: 80 }; + const local = buildScrollGesturePlan({ direction: 'down', amount: 0.75, ...FRAME }); + const plan = buildScrollGesturePlan({ + direction: 'down', + amount: 0.75, + ...FRAME, + referenceOrigin: origin, + }); + + assert.deepEqual(plan, { + ...local, + x1: local.x1 + origin.x, + y1: local.y1 + origin.y, + x2: local.x2 + origin.x, + y2: local.y2 + origin.y, + }); +}); + test('a container lying wholly in the padding of the scrolled axis leaves the padded swipe as it was', () => { const statusBand: Rect = { x: 0, y: 0, width: 402, height: PADDED_TOP - 1 }; const unclamped = buildScrollGesturePlan({ direction: 'up', ...FRAME }); diff --git a/packages/contracts/src/scroll-gesture.ts b/packages/contracts/src/scroll-gesture.ts index 04fc049b04..bdebb7a2b9 100644 --- a/packages/contracts/src/scroll-gesture.ts +++ b/packages/contracts/src/scroll-gesture.ts @@ -1,7 +1,7 @@ import { AppError } from '@agent-device/kernel/errors'; import { defineStringEnum } from './string-enum.ts'; import { isViewportRootNode } from './snapshot-visibility.ts'; -import type { Rect, SnapshotNode } from '@agent-device/kernel/snapshot'; +import type { Point, Rect, SnapshotNode } from '@agent-device/kernel/snapshot'; // What a caller may ASK for, as opposed to `ScrollDirection` (what the gesture resolves to): // `top`/`bottom` are scroll-to-extreme requests with no direction of their own. Declared here @@ -55,7 +55,14 @@ export type ScrollGestureOptions = { referenceWidth: number; referenceHeight: number; /** - * The scroll container the swipe is meant for, in the reference frame's own zero-origin space. + * Where the reference frame sits in the space the caller dispatches. The planner subtracts + * this from `container`, plans inside the zero-origin frame, then adds it to every endpoint. + * Omit it when that frame already starts at (0, 0). + */ + referenceOrigin?: Point; + /** + * The scroll container the swipe is meant for, in the same space as `referenceOrigin` + * (the frame's own zero-origin space when the origin is omitted). * Both endpoints stay strictly inside its intersection with the padded frame (#2714): a span * that already fits is unchanged, one that leaves it is moved in, and one longer than it is * shortened. A container the padding leaves nothing of along the scrolled axis gets the padded @@ -119,6 +126,16 @@ const SCROLL_CONTAINER_EDGE_INSET = 8; export function buildScrollGesturePlan(options: ScrollGestureOptions): ScrollGesturePlan { const direction = options.direction; + const originX = options.referenceOrigin?.x ?? 0; + const originY = options.referenceOrigin?.y ?? 0; + const container = + options.container === undefined + ? undefined + : { + ...options.container, + x: options.container.x - originX, + y: options.container.y - originY, + }; const vertical = direction === 'up' || direction === 'down'; const axisLength = vertical ? options.referenceHeight : options.referenceWidth; const crossLength = vertical ? options.referenceWidth : options.referenceHeight; @@ -138,8 +155,8 @@ export function buildScrollGesturePlan(options: ScrollGestureOptions): ScrollGes cross: Math.round(crossLength / 2), }; const crossPadding = scrollEdgePadding(crossLength); - const lane = options.container - ? keepSwipeInContainer(unclamped, options.container, vertical, { + const lane = container + ? keepSwipeInContainer(unclamped, container, vertical, { axis: { min: edgePadding, max: axisLength - edgePadding }, cross: { min: crossPadding, max: crossLength - crossPadding }, crossFrame: { min: 0, max: crossLength }, @@ -148,9 +165,13 @@ export function buildScrollGesturePlan(options: ScrollGestureOptions): ScrollGes const laneLength = lane.axisEnd - lane.axisStart; // Centring rounds an odd span up by one, so only a lane shorter than the travel itself is short. const shortened = laneLength < travelPixels; + const endpoints = laneEndpoints(direction, lane); return { direction, - ...laneEndpoints(direction, lane), + x1: endpoints.x1 + originX, + y1: endpoints.y1 + originY, + x2: endpoints.x2 + originX, + y2: endpoints.y2 + originY, referenceWidth: options.referenceWidth, referenceHeight: options.referenceHeight, amount: options.amount, diff --git a/packages/platform-android/src/input-actions.ts b/packages/platform-android/src/input-actions.ts index 2d0b5656d9..df41cb5573 100644 --- a/packages/platform-android/src/input-actions.ts +++ b/packages/platform-android/src/input-actions.ts @@ -324,26 +324,21 @@ export async function scrollAndroid( helperSessionScope: options?.helperSessionScope, }); const swipeSurface = resolveAndroidScrollSurface(direction, viewport, keyboard); - const relativePlan = buildScrollGesturePlan({ + const plan = buildScrollGesturePlan({ direction, amount: options?.amount, pixels: options?.pixels, referenceWidth: swipeSurface.viewport.width, referenceHeight: swipeSurface.viewport.height, - ...(options?.containerRect === undefined - ? {} - : { container: relativeToViewport(options.containerRect, swipeSurface.viewport) }), + referenceOrigin: { x: swipeSurface.viewport.x, y: swipeSurface.viewport.y }, + ...(options?.containerRect === undefined ? {} : { container: options.containerRect }), }); const scrollPlan = { - ...relativePlan, + ...plan, // Injected coordinates are absolute, so their zero-origin reference frame // must include the viewport offset as well as its dimensions. referenceWidth: swipeSurface.viewport.x + swipeSurface.viewport.width, referenceHeight: swipeSurface.viewport.y + swipeSurface.viewport.height, - x1: swipeSurface.viewport.x + relativePlan.x1, - y1: swipeSurface.viewport.y + relativePlan.y1, - x2: swipeSurface.viewport.x + relativePlan.x2, - y2: swipeSurface.viewport.y + relativePlan.y2, }; const durationMs = Math.max( options?.durationMs ?? DEFAULT_MOBILE_SCROLL_DURATION_MS, @@ -374,11 +369,6 @@ export async function scrollAndroid( }; } -/** The planner works in the swipe surface's zero-origin frame; snapshot rects are absolute. */ -function relativeToViewport(rect: Rect, viewport: Rect): Rect { - return { ...rect, x: rect.x - viewport.x, y: rect.y - viewport.y }; -} - /** * The band one Android scroll may swipe, refusing when the keyboard owns the window (#2500). * diff --git a/packages/platform-harmonyos/src/__tests__/input-actions.test.ts b/packages/platform-harmonyos/src/__tests__/input-actions.test.ts index 82db1091da..81eb769e45 100644 --- a/packages/platform-harmonyos/src/__tests__/input-actions.test.ts +++ b/packages/platform-harmonyos/src/__tests__/input-actions.test.ts @@ -146,7 +146,7 @@ test('HarmonyOS scroll keeps an inset list swipe inside the container and report assert.equal(result.requestedPixels, unclamped.pixels); }); -test('HarmonyOS scroll translates an offset window into the swipe frame before clamping', async () => { +test('HarmonyOS scroll dispatches an offset-window swipe inside the absolute container', async () => { readHarmonyGestureViewport.mockResolvedValue(OFFSET_WINDOW); const list: Rect = { x: OFFSET_WINDOW.x + INSET_LIST.x, @@ -156,15 +156,10 @@ test('HarmonyOS scroll translates an offset window into the swipe frame before c }; const result = await scrollHarmony(DEVICE, 'up', { amount: 0.75, containerRect: list }); const swipe = dispatchedSwipe(); - const inSnapshotSpace = { - x1: swipe.x1 + OFFSET_WINDOW.x, - y1: swipe.y1 + OFFSET_WINDOW.y, - x2: swipe.x2 + OFFSET_WINDOW.x, - y2: swipe.y2 + OFFSET_WINDOW.y, - }; - assert.ok(strictlyInside({ x: inSnapshotSpace.x1, y: inSnapshotSpace.y1 }, list)); - assert.ok(strictlyInside({ x: inSnapshotSpace.x2, y: inSnapshotSpace.y2 }, list)); + assert.ok(strictlyInside({ x: swipe.x1, y: swipe.y1 }, list), JSON.stringify(swipe)); + assert.ok(strictlyInside({ x: swipe.x2, y: swipe.y2 }, list), JSON.stringify(swipe)); + assert.deepEqual(swipe, { x1: result.x1, y1: result.y1, x2: result.x2, y2: result.y2 }); assert.ok(Number(result.pixels) < Number(result.requestedPixels)); }); @@ -178,23 +173,25 @@ test('HarmonyOS scroll leaves a swipe that already fits its container unchanged' assert.equal('requestedPixels' in result, false); }); -test('HarmonyOS scroll without a container keeps the unclamped swipe', async () => { +test('HarmonyOS scroll without a container keeps the unclamped swipe in screen coordinates', async () => { readHarmonyGestureViewport.mockResolvedValue(OFFSET_WINDOW); const plan = await scrollHarmony(DEVICE, 'down', { amount: 0.75 }); - const expected = buildScrollGesturePlan({ + const local = buildScrollGesturePlan({ direction: 'down', amount: 0.75, referenceWidth: OFFSET_WINDOW.width, referenceHeight: OFFSET_WINDOW.height, }); + const swipe = dispatchedSwipe(); - assert.deepEqual(dispatchedSwipe(), { - x1: expected.x1, - y1: expected.y1, - x2: expected.x2, - y2: expected.y2, + assert.deepEqual(swipe, { + x1: local.x1 + OFFSET_WINDOW.x, + y1: local.y1 + OFFSET_WINDOW.y, + x2: local.x2 + OFFSET_WINDOW.x, + y2: local.y2 + OFFSET_WINDOW.y, }); - assert.equal(plan.pixels, expected.pixels); + assert.deepEqual(swipe, { x1: plan.x1, y1: plan.y1, x2: plan.x2, y2: plan.y2 }); + assert.equal(plan.pixels, local.pixels); assert.equal('requestedPixels' in plan, false); }); diff --git a/packages/platform-harmonyos/src/input-actions.ts b/packages/platform-harmonyos/src/input-actions.ts index a6259e403b..35a60a697d 100644 --- a/packages/platform-harmonyos/src/input-actions.ts +++ b/packages/platform-harmonyos/src/input-actions.ts @@ -9,7 +9,6 @@ import { buildScrollGesturePlan, } from '@agent-device/contracts/scroll-gesture'; import type { DeviceInfo } from '@agent-device/kernel/device'; -import type { Rect } from '@agent-device/kernel/snapshot'; import { AppError } from '@agent-device/kernel/errors'; import { sleep } from '@agent-device/host-kit/retry'; import { runHarmonyShell } from './hdc.ts'; @@ -65,9 +64,8 @@ export async function scrollHarmony( pixels: options?.pixels, referenceWidth: viewport.width, referenceHeight: viewport.height, - ...(options?.containerRect === undefined - ? {} - : { container: relativeToViewport(options.containerRect, viewport) }), + referenceOrigin: { x: viewport.x, y: viewport.y }, + ...(options?.containerRect === undefined ? {} : { container: options.containerRect }), }); await runHarmonyShell(device, [ 'uitest', @@ -82,11 +80,6 @@ export async function scrollHarmony( return plan; } -/** The planner's box is the viewport size at the origin; a snapshot container is absolute. */ -function relativeToViewport(rect: Rect, viewport: Rect): Rect { - return { ...rect, x: rect.x - viewport.x, y: rect.y - viewport.y }; -} - /** Lowers HDC's public one-pointer primitives from a shared semantic plan. */ export async function performHarmonyGesture( device: DeviceInfo, diff --git a/packages/platform-harmonyos/src/snapshot.ts b/packages/platform-harmonyos/src/snapshot.ts index 2c6dddeaea..d5693f6856 100644 --- a/packages/platform-harmonyos/src/snapshot.ts +++ b/packages/platform-harmonyos/src/snapshot.ts @@ -19,7 +19,7 @@ import { readHarmonyKeyboardBand } from './keyboard-band.ts'; const MAX_NODES = 5_000; -// Gesture planning only reads the Application rect's width/height, but every +// Gesture planning reads the Application rect's origin and size, but every // read paid a full layout dump (`uitest dumpLayout` + `file recv` + `rm`). // Edge scrolls run several passes back to back, so a short memo collapses // them into one dump while staying far below the cadence where a foreground diff --git a/packages/provider-webdriver/src/webdriver-interactor.ts b/packages/provider-webdriver/src/webdriver-interactor.ts index 472bff3f4b..f88b206c23 100644 --- a/packages/provider-webdriver/src/webdriver-interactor.ts +++ b/packages/provider-webdriver/src/webdriver-interactor.ts @@ -18,7 +18,6 @@ import { import type { TvRemoteButton } from '@agent-device/contracts/tv-remote'; import type { SettingOptions } from '@agent-device/contracts/settings'; import { snapshotViewportSizeFrom } from '@agent-device/kernel/rect'; -import type { Rect } from '@agent-device/kernel/snapshot'; import { AppError } from '@agent-device/kernel/errors'; import { capabilitySupported, @@ -86,11 +85,6 @@ function textEntryClearTargetUnavailableError(): AppError { ); } -/** The planner's box starts at the frame origin; a snapshot container is absolute. */ -function relativeToFrame(container: Rect, frame: WebDriverWindowRect): Rect { - return { ...container, x: container.x - frame.x, y: container.y - frame.y }; -} - function rectContains(rect: WebDriverWindowRect, x: number, y: number): boolean { return x >= rect.x && x <= rect.x + rect.width && y >= rect.y && y <= rect.y + rect.height; } @@ -257,25 +251,11 @@ class WebDriverInteractor implements Interactor { pixels: options?.pixels, referenceWidth: rect.width, referenceHeight: rect.height, - ...(options?.containerRect === undefined - ? {} - : { container: relativeToFrame(options.containerRect, rect) }), + referenceOrigin: { x: rect.x, y: rect.y }, + ...(options?.containerRect === undefined ? {} : { container: options.containerRect }), }); - const absolutePlan = { - ...plan, - x1: plan.x1 + rect.x, - y1: plan.y1 + rect.y, - x2: plan.x2 + rect.x, - y2: plan.y2 + rect.y, - }; - await this.swipe( - absolutePlan.x1, - absolutePlan.y1, - absolutePlan.x2, - absolutePlan.y2, - durationMs, - ); - return { backend: 'webdriver', ...absolutePlan, distance: plan.pixels, durationMs }; + await this.swipe(plan.x1, plan.y1, plan.x2, plan.y2, durationMs); + return { backend: 'webdriver', ...plan, distance: plan.pixels, durationMs }; } async gestureViewport(): Promise {