diff --git a/.changeset/webdriver-harmony-scroll-container.md b/.changeset/webdriver-harmony-scroll-container.md new file mode 100644 index 0000000000..ae0f984b7f --- /dev/null +++ b/.changeset/webdriver-harmony-scroll-container.md @@ -0,0 +1,5 @@ +--- +'agent-device': patch +--- + +WebDriver and HarmonyOS directional scrolls stay inside the resolved scroll container. A swipe that does not fit is shortened, and the response reports the distance that was dispatched beside the distance that was asked for. diff --git a/packages/contracts/src/scroll-gesture-container.test.ts b/packages/contracts/src/scroll-gesture-container.test.ts index 591bf4df06..0dd541fa50 100644 --- a/packages/contracts/src/scroll-gesture-container.test.ts +++ b/packages/contracts/src/scroll-gesture-container.test.ts @@ -170,6 +170,76 @@ test.each<[ScrollDirection, string, Rect]>([ }, ); +test('a zero reference origin matches a plan that omits it', () => { + const omitted = buildScrollGesturePlan({ + direction: 'down', + amount: 0.75, + ...FRAME, + container: FOOTERED_CONTAINER, + }); + const explicit = buildScrollGesturePlan({ + direction: 'down', + amount: 0.75, + ...FRAME, + referenceOrigin: { x: 0, y: 0 }, + container: FOOTERED_CONTAINER, + }); + + assert.deepEqual(explicit, omitted); +}); + +test('an offset reference origin returns absolute endpoints inside the absolute container', () => { + const origin = { x: 40, y: 80 }; + const container: Rect = { + x: origin.x + FOOTERED_CONTAINER.x, + y: origin.y + FOOTERED_CONTAINER.y, + width: FOOTERED_CONTAINER.width, + height: FOOTERED_CONTAINER.height, + }; + const local = buildScrollGesturePlan({ + direction: 'up', + amount: 0.75, + ...FRAME, + container: FOOTERED_CONTAINER, + }); + const plan = buildScrollGesturePlan({ + direction: 'up', + amount: 0.75, + ...FRAME, + referenceOrigin: origin, + container, + }); + + assert.deepEqual( + [plan.x1, plan.y1, plan.x2, plan.y2], + [local.x1 + origin.x, local.y1 + origin.y, local.x2 + origin.x, local.y2 + origin.y], + ); + assert.ok(strictlyInside(plan, container), JSON.stringify(plan)); + assert.equal(plan.pixels, local.pixels); + assert.equal(plan.requestedPixels, local.requestedPixels); + assert.equal(plan.referenceWidth, FRAME.referenceWidth); + assert.equal(plan.referenceHeight, FRAME.referenceHeight); +}); + +test('an offset reference origin shifts an unclamped swipe into the dispatch frame', () => { + const origin = { x: 40, y: 80 }; + const local = buildScrollGesturePlan({ direction: 'down', amount: 0.75, ...FRAME }); + const plan = buildScrollGesturePlan({ + direction: 'down', + amount: 0.75, + ...FRAME, + referenceOrigin: origin, + }); + + assert.deepEqual(plan, { + ...local, + x1: local.x1 + origin.x, + y1: local.y1 + origin.y, + x2: local.x2 + origin.x, + y2: local.y2 + origin.y, + }); +}); + test('a container lying wholly in the padding of the scrolled axis leaves the padded swipe as it was', () => { const statusBand: Rect = { x: 0, y: 0, width: 402, height: PADDED_TOP - 1 }; const unclamped = buildScrollGesturePlan({ direction: 'up', ...FRAME }); diff --git a/packages/contracts/src/scroll-gesture.ts b/packages/contracts/src/scroll-gesture.ts index 04fc049b04..bdebb7a2b9 100644 --- a/packages/contracts/src/scroll-gesture.ts +++ b/packages/contracts/src/scroll-gesture.ts @@ -1,7 +1,7 @@ import { AppError } from '@agent-device/kernel/errors'; import { defineStringEnum } from './string-enum.ts'; import { isViewportRootNode } from './snapshot-visibility.ts'; -import type { Rect, SnapshotNode } from '@agent-device/kernel/snapshot'; +import type { Point, Rect, SnapshotNode } from '@agent-device/kernel/snapshot'; // What a caller may ASK for, as opposed to `ScrollDirection` (what the gesture resolves to): // `top`/`bottom` are scroll-to-extreme requests with no direction of their own. Declared here @@ -55,7 +55,14 @@ export type ScrollGestureOptions = { referenceWidth: number; referenceHeight: number; /** - * The scroll container the swipe is meant for, in the reference frame's own zero-origin space. + * Where the reference frame sits in the space the caller dispatches. The planner subtracts + * this from `container`, plans inside the zero-origin frame, then adds it to every endpoint. + * Omit it when that frame already starts at (0, 0). + */ + referenceOrigin?: Point; + /** + * The scroll container the swipe is meant for, in the same space as `referenceOrigin` + * (the frame's own zero-origin space when the origin is omitted). * Both endpoints stay strictly inside its intersection with the padded frame (#2714): a span * that already fits is unchanged, one that leaves it is moved in, and one longer than it is * shortened. A container the padding leaves nothing of along the scrolled axis gets the padded @@ -119,6 +126,16 @@ const SCROLL_CONTAINER_EDGE_INSET = 8; export function buildScrollGesturePlan(options: ScrollGestureOptions): ScrollGesturePlan { const direction = options.direction; + const originX = options.referenceOrigin?.x ?? 0; + const originY = options.referenceOrigin?.y ?? 0; + const container = + options.container === undefined + ? undefined + : { + ...options.container, + x: options.container.x - originX, + y: options.container.y - originY, + }; const vertical = direction === 'up' || direction === 'down'; const axisLength = vertical ? options.referenceHeight : options.referenceWidth; const crossLength = vertical ? options.referenceWidth : options.referenceHeight; @@ -138,8 +155,8 @@ export function buildScrollGesturePlan(options: ScrollGestureOptions): ScrollGes cross: Math.round(crossLength / 2), }; const crossPadding = scrollEdgePadding(crossLength); - const lane = options.container - ? keepSwipeInContainer(unclamped, options.container, vertical, { + const lane = container + ? keepSwipeInContainer(unclamped, container, vertical, { axis: { min: edgePadding, max: axisLength - edgePadding }, cross: { min: crossPadding, max: crossLength - crossPadding }, crossFrame: { min: 0, max: crossLength }, @@ -148,9 +165,13 @@ export function buildScrollGesturePlan(options: ScrollGestureOptions): ScrollGes const laneLength = lane.axisEnd - lane.axisStart; // Centring rounds an odd span up by one, so only a lane shorter than the travel itself is short. const shortened = laneLength < travelPixels; + const endpoints = laneEndpoints(direction, lane); return { direction, - ...laneEndpoints(direction, lane), + x1: endpoints.x1 + originX, + y1: endpoints.y1 + originY, + x2: endpoints.x2 + originX, + y2: endpoints.y2 + originY, referenceWidth: options.referenceWidth, referenceHeight: options.referenceHeight, amount: options.amount, diff --git a/packages/platform-android/src/input-actions.ts b/packages/platform-android/src/input-actions.ts index 2d0b5656d9..df41cb5573 100644 --- a/packages/platform-android/src/input-actions.ts +++ b/packages/platform-android/src/input-actions.ts @@ -324,26 +324,21 @@ export async function scrollAndroid( helperSessionScope: options?.helperSessionScope, }); const swipeSurface = resolveAndroidScrollSurface(direction, viewport, keyboard); - const relativePlan = buildScrollGesturePlan({ + const plan = buildScrollGesturePlan({ direction, amount: options?.amount, pixels: options?.pixels, referenceWidth: swipeSurface.viewport.width, referenceHeight: swipeSurface.viewport.height, - ...(options?.containerRect === undefined - ? {} - : { container: relativeToViewport(options.containerRect, swipeSurface.viewport) }), + referenceOrigin: { x: swipeSurface.viewport.x, y: swipeSurface.viewport.y }, + ...(options?.containerRect === undefined ? {} : { container: options.containerRect }), }); const scrollPlan = { - ...relativePlan, + ...plan, // Injected coordinates are absolute, so their zero-origin reference frame // must include the viewport offset as well as its dimensions. referenceWidth: swipeSurface.viewport.x + swipeSurface.viewport.width, referenceHeight: swipeSurface.viewport.y + swipeSurface.viewport.height, - x1: swipeSurface.viewport.x + relativePlan.x1, - y1: swipeSurface.viewport.y + relativePlan.y1, - x2: swipeSurface.viewport.x + relativePlan.x2, - y2: swipeSurface.viewport.y + relativePlan.y2, }; const durationMs = Math.max( options?.durationMs ?? DEFAULT_MOBILE_SCROLL_DURATION_MS, @@ -374,11 +369,6 @@ export async function scrollAndroid( }; } -/** The planner works in the swipe surface's zero-origin frame; snapshot rects are absolute. */ -function relativeToViewport(rect: Rect, viewport: Rect): Rect { - return { ...rect, x: rect.x - viewport.x, y: rect.y - viewport.y }; -} - /** * The band one Android scroll may swipe, refusing when the keyboard owns the window (#2500). * diff --git a/packages/platform-harmonyos/src/__tests__/input-actions.test.ts b/packages/platform-harmonyos/src/__tests__/input-actions.test.ts index 85fdf9a8fd..81eb769e45 100644 --- a/packages/platform-harmonyos/src/__tests__/input-actions.test.ts +++ b/packages/platform-harmonyos/src/__tests__/input-actions.test.ts @@ -1,7 +1,9 @@ import assert from 'node:assert/strict'; import { beforeEach, test, vi } from 'vitest'; import type { GesturePlan } from '@agent-device/contracts/gesture-plan-types'; +import { buildScrollGesturePlan } from '@agent-device/contracts/scroll-gesture'; import type { DeviceInfo } from '@agent-device/kernel/device'; +import type { Rect } from '@agent-device/kernel/snapshot'; const { runHarmonyShell, sleep, readHarmonyGestureViewport, invalidateHarmonyGestureViewport } = vi.hoisted(() => ({ @@ -96,6 +98,103 @@ test('HarmonyOS scroll derives a viewport-aware swipe plan', async () => { ]); }); +const PHONE = { x: 0, y: 0, width: 402, height: 874 }; +/** Inset under a navigation bar and above a footer, shorter than a 0.75 swipe. */ +const INSET_LIST: Rect = { x: 18, y: 178, width: 366, height: 500 }; +/** The same list shifted into an application window that is not at the screen origin. */ +const OFFSET_WINDOW = { x: 40, y: 80, width: 402, height: 874 }; + +function strictlyInside(point: { x: number; y: number }, rect: Rect): boolean { + return ( + point.x > rect.x && + point.x < rect.x + rect.width && + point.y > rect.y && + point.y < rect.y + rect.height + ); +} + +function dispatchedSwipe(callIndex?: number): { x1: number; y1: number; x2: number; y2: number } { + const calls = runHarmonyShell.mock.calls; + const call = calls[callIndex ?? calls.length - 1]; + const args = call?.[1] as readonly unknown[]; + return { + x1: Number(args[3]), + y1: Number(args[4]), + x2: Number(args[5]), + y2: Number(args[6]), + }; +} + +test('HarmonyOS scroll keeps an inset list swipe inside the container and reports the shortened span', async () => { + readHarmonyGestureViewport.mockResolvedValue(PHONE); + const unclamped = await scrollHarmony(DEVICE, 'down', { amount: 0.75 }); + const outside = dispatchedSwipe(0); + const result = await scrollHarmony(DEVICE, 'down', { amount: 0.75, containerRect: INSET_LIST }); + const swipe = dispatchedSwipe(1); + + assert.equal( + strictlyInside({ x: outside.x1, y: outside.y1 }, INSET_LIST) && + strictlyInside({ x: outside.x2, y: outside.y2 }, INSET_LIST), + false, + 'the fixture must leave the list', + ); + assert.ok(strictlyInside({ x: swipe.x1, y: swipe.y1 }, INSET_LIST), JSON.stringify(swipe)); + assert.ok(strictlyInside({ x: swipe.x2, y: swipe.y2 }, INSET_LIST), JSON.stringify(swipe)); + assert.deepEqual(swipe, { x1: result.x1, y1: result.y1, x2: result.x2, y2: result.y2 }); + assert.equal(result.pixels, Math.abs(Number(result.y2) - Number(result.y1))); + assert.ok(Number(result.pixels) < Number(unclamped.pixels)); + assert.equal(result.requestedPixels, unclamped.pixels); +}); + +test('HarmonyOS scroll dispatches an offset-window swipe inside the absolute container', async () => { + readHarmonyGestureViewport.mockResolvedValue(OFFSET_WINDOW); + const list: Rect = { + x: OFFSET_WINDOW.x + INSET_LIST.x, + y: OFFSET_WINDOW.y + INSET_LIST.y, + width: INSET_LIST.width, + height: INSET_LIST.height, + }; + const result = await scrollHarmony(DEVICE, 'up', { amount: 0.75, containerRect: list }); + const swipe = dispatchedSwipe(); + + assert.ok(strictlyInside({ x: swipe.x1, y: swipe.y1 }, list), JSON.stringify(swipe)); + assert.ok(strictlyInside({ x: swipe.x2, y: swipe.y2 }, list), JSON.stringify(swipe)); + assert.deepEqual(swipe, { x1: result.x1, y1: result.y1, x2: result.x2, y2: result.y2 }); + assert.ok(Number(result.pixels) < Number(result.requestedPixels)); +}); + +test('HarmonyOS scroll leaves a swipe that already fits its container unchanged', async () => { + const fitting: Rect = { x: 18, y: 62, width: 366, height: 729 }; + readHarmonyGestureViewport.mockResolvedValue(PHONE); + const unclamped = await scrollHarmony(DEVICE, 'down', { amount: 0.75 }); + const result = await scrollHarmony(DEVICE, 'down', { amount: 0.75, containerRect: fitting }); + + assert.deepEqual(result, unclamped); + assert.equal('requestedPixels' in result, false); +}); + +test('HarmonyOS scroll without a container keeps the unclamped swipe in screen coordinates', async () => { + readHarmonyGestureViewport.mockResolvedValue(OFFSET_WINDOW); + const plan = await scrollHarmony(DEVICE, 'down', { amount: 0.75 }); + const local = buildScrollGesturePlan({ + direction: 'down', + amount: 0.75, + referenceWidth: OFFSET_WINDOW.width, + referenceHeight: OFFSET_WINDOW.height, + }); + const swipe = dispatchedSwipe(); + + assert.deepEqual(swipe, { + x1: local.x1 + OFFSET_WINDOW.x, + y1: local.y1 + OFFSET_WINDOW.y, + x2: local.x2 + OFFSET_WINDOW.x, + y2: local.y2 + OFFSET_WINDOW.y, + }); + assert.deepEqual(swipe, { x1: plan.x1, y1: plan.y1, x2: plan.x2, y2: plan.y2 }); + assert.equal(plan.pixels, local.pixels); + assert.equal('requestedPixels' in plan, false); +}); + test('HarmonyOS gestures lower a single trajectory and reject multi-touch', async () => { const singlePlan: GesturePlan = { topology: 'single', diff --git a/packages/platform-harmonyos/src/input-actions.ts b/packages/platform-harmonyos/src/input-actions.ts index 1500465720..35a60a697d 100644 --- a/packages/platform-harmonyos/src/input-actions.ts +++ b/packages/platform-harmonyos/src/input-actions.ts @@ -1,6 +1,9 @@ import type { DeviceRotation } from '@agent-device/contracts/device'; import type { GesturePlan } from '@agent-device/contracts/gesture-plan-types'; -import { DEFAULT_MOBILE_SCROLL_DURATION_MS } from '@agent-device/contracts/scroll-command'; +import { + type ScrollContainerTarget, + DEFAULT_MOBILE_SCROLL_DURATION_MS, +} from '@agent-device/contracts/scroll-command'; import { type ScrollDirection, buildScrollGesturePlan, @@ -52,7 +55,7 @@ export async function fillHarmony( export async function scrollHarmony( device: DeviceInfo, direction: ScrollDirection, - options?: { amount?: number; pixels?: number; durationMs?: number }, + options?: { amount?: number; pixels?: number; durationMs?: number } & ScrollContainerTarget, ): Promise> { const viewport = await readHarmonyGestureViewport(device); const plan = buildScrollGesturePlan({ @@ -61,6 +64,8 @@ export async function scrollHarmony( pixels: options?.pixels, referenceWidth: viewport.width, referenceHeight: viewport.height, + referenceOrigin: { x: viewport.x, y: viewport.y }, + ...(options?.containerRect === undefined ? {} : { container: options.containerRect }), }); await runHarmonyShell(device, [ 'uitest', diff --git a/packages/platform-harmonyos/src/snapshot.ts b/packages/platform-harmonyos/src/snapshot.ts index 2c6dddeaea..d5693f6856 100644 --- a/packages/platform-harmonyos/src/snapshot.ts +++ b/packages/platform-harmonyos/src/snapshot.ts @@ -19,7 +19,7 @@ import { readHarmonyKeyboardBand } from './keyboard-band.ts'; const MAX_NODES = 5_000; -// Gesture planning only reads the Application rect's width/height, but every +// Gesture planning reads the Application rect's origin and size, but every // read paid a full layout dump (`uitest dumpLayout` + `file recv` + `rm`). // Edge scrolls run several passes back to back, so a short memo collapses // them into one dump while staying far below the cadence where a foreground diff --git a/packages/provider-webdriver/src/webdriver-interactor-scroll.test.ts b/packages/provider-webdriver/src/webdriver-interactor-scroll.test.ts new file mode 100644 index 0000000000..e0b59dbc4c --- /dev/null +++ b/packages/provider-webdriver/src/webdriver-interactor-scroll.test.ts @@ -0,0 +1,111 @@ +import assert from 'node:assert/strict'; +import { test } from 'vitest'; +import type { Rect } from '@agent-device/kernel/snapshot'; +import { createCloudWebDriverCapabilities } from './capabilities.ts'; +import type { + W3CActionSequence, + WebDriverClient, + WebDriverWindowRect, +} from './webdriver-client.ts'; +import { createWebDriverInteractor } from './webdriver-interactor.ts'; + +const WINDOW: WebDriverWindowRect = { x: 12, y: 30, width: 402, height: 874 }; +/** Inset under a navigation bar and above a footer, in the window's own coordinates. */ +const INSET_LIST: Rect = { x: 18, y: 178, width: 366, height: 500 }; +const FITTING_LIST: Rect = { x: 18, y: 62, width: 366, height: 729 }; + +function place(rect: Rect, frame: WebDriverWindowRect): Rect { + return { ...rect, x: rect.x + frame.x, y: rect.y + frame.y }; +} + +function strictlyInside(point: { x: number; y: number }, rect: Rect): boolean { + return ( + point.x > rect.x && + point.x < rect.x + rect.width && + point.y > rect.y && + point.y < rect.y + rect.height + ); +} + +function scrollWorld(frame: WebDriverWindowRect) { + const performed: W3CActionSequence[][] = []; + const client = { + hideKeyboard: async () => undefined, + source: async () => { + throw new Error('page source unavailable'); + }, + windowRect: async () => frame, + performActions: async (actions: W3CActionSequence[]) => { + performed.push(actions); + }, + releaseActions: async () => undefined, + } as unknown as WebDriverClient; + return { + performed, + interactor: createWebDriverInteractor({ + client, + backend: 'android', + capabilities: createCloudWebDriverCapabilities({ provider: 'test', platform: 'android' }), + }), + }; +} + +function swipeEndpoints(actions: W3CActionSequence[] | undefined): { + x1: number; + y1: number; + x2: number; + y2: number; +} { + const moves = actions?.[0]?.actions.filter((action) => action.type === 'pointerMove') ?? []; + const start = moves[0]; + const end = moves[1]; + if (start?.type !== 'pointerMove' || end?.type !== 'pointerMove') { + assert.fail('swipe is missing its pointer moves'); + } + return { x1: start.x, y1: start.y, x2: end.x, y2: end.y }; +} + +test('WebDriver scroll keeps an inset list swipe inside the container and reports the shortened span', async () => { + const { interactor, performed } = scrollWorld(WINDOW); + const list = place(INSET_LIST, WINDOW); + const unclamped = await interactor.scroll('down', { amount: 0.75 }); + const outside = swipeEndpoints(performed[0]); + const result = await interactor.scroll('down', { amount: 0.75, containerRect: list }); + const swipe = swipeEndpoints(performed[1]); + + assert.equal( + strictlyInside({ x: outside.x1, y: outside.y1 }, list) && + strictlyInside({ x: outside.x2, y: outside.y2 }, list), + false, + 'the fixture must leave the list', + ); + assert.ok(strictlyInside({ x: swipe.x1, y: swipe.y1 }, list), JSON.stringify(swipe)); + assert.ok(strictlyInside({ x: swipe.x2, y: swipe.y2 }, list), JSON.stringify(swipe)); + assert.deepEqual(swipe, { x1: result?.x1, y1: result?.y1, x2: result?.x2, y2: result?.y2 }); + assert.equal(result?.pixels, Math.abs(Number(result?.y2) - Number(result?.y1))); + assert.equal(result?.distance, result?.pixels); + assert.ok(Number(result?.pixels) < Number(unclamped?.pixels)); + assert.equal(result?.requestedPixels, unclamped?.pixels); +}); + +test('WebDriver scroll leaves a swipe that already fits its container unchanged', async () => { + const { interactor } = scrollWorld(WINDOW); + const unclamped = await interactor.scroll('down', { amount: 0.75 }); + const result = await interactor.scroll('down', { + amount: 0.75, + containerRect: place(FITTING_LIST, WINDOW), + }); + + assert.deepEqual(result, unclamped); + assert.equal(result !== undefined && 'requestedPixels' in result, false); +}); + +test('WebDriver scroll without a container keeps the window swipe', async () => { + const { interactor, performed } = scrollWorld(WINDOW); + const result = await interactor.scroll('up', { amount: 0.75 }); + const swipe = swipeEndpoints(performed[0]); + + assert.deepEqual(swipe, { x1: result?.x1, y1: result?.y1, x2: result?.x2, y2: result?.y2 }); + assert.equal(result !== undefined && 'requestedPixels' in result, false); + assert.equal(result?.distance, result?.pixels); +}); diff --git a/packages/provider-webdriver/src/webdriver-interactor.ts b/packages/provider-webdriver/src/webdriver-interactor.ts index 966234c37e..f88b206c23 100644 --- a/packages/provider-webdriver/src/webdriver-interactor.ts +++ b/packages/provider-webdriver/src/webdriver-interactor.ts @@ -10,6 +10,7 @@ import type { KeyboardStatusResult, } from '@agent-device/contracts/interactor-types'; import type { GesturePlan } from '@agent-device/contracts/gesture-plan-types'; +import type { ScrollExecutionOptions } from '@agent-device/contracts/scroll-command'; import { type ScrollDirection, buildScrollGesturePlan, @@ -238,7 +239,7 @@ class WebDriverInteractor implements Interactor { async scroll( direction: ScrollDirection, - options?: { amount?: number; pixels?: number; durationMs?: number }, + options?: ScrollExecutionOptions, ): Promise> { this.requireSupport('scroll'); const durationMs = options?.durationMs ?? 350; @@ -250,22 +251,11 @@ class WebDriverInteractor implements Interactor { pixels: options?.pixels, referenceWidth: rect.width, referenceHeight: rect.height, + referenceOrigin: { x: rect.x, y: rect.y }, + ...(options?.containerRect === undefined ? {} : { container: options.containerRect }), }); - const absolutePlan = { - ...plan, - x1: plan.x1 + rect.x, - y1: plan.y1 + rect.y, - x2: plan.x2 + rect.x, - y2: plan.y2 + rect.y, - }; - await this.swipe( - absolutePlan.x1, - absolutePlan.y1, - absolutePlan.x2, - absolutePlan.y2, - durationMs, - ); - return { backend: 'webdriver', ...absolutePlan, distance: plan.pixels, durationMs }; + await this.swipe(plan.x1, plan.y1, plan.x2, plan.y2, durationMs); + return { backend: 'webdriver', ...plan, distance: plan.pixels, durationMs }; } async gestureViewport(): Promise {