Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 5 additions & 0 deletions .changeset/webdriver-harmony-scroll-container.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
---
'agent-device': patch
---

WebDriver and HarmonyOS directional scrolls stay inside the resolved scroll container. A swipe that does not fit is shortened, and the response reports the distance that was dispatched beside the distance that was asked for.
70 changes: 70 additions & 0 deletions packages/contracts/src/scroll-gesture-container.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -170,6 +170,76 @@ test.each<[ScrollDirection, string, Rect]>([
},
);

test('a zero reference origin matches a plan that omits it', () => {
const omitted = buildScrollGesturePlan({
direction: 'down',
amount: 0.75,
...FRAME,
container: FOOTERED_CONTAINER,
});
const explicit = buildScrollGesturePlan({
direction: 'down',
amount: 0.75,
...FRAME,
referenceOrigin: { x: 0, y: 0 },
container: FOOTERED_CONTAINER,
});

assert.deepEqual(explicit, omitted);
});

test('an offset reference origin returns absolute endpoints inside the absolute container', () => {
const origin = { x: 40, y: 80 };
const container: Rect = {
x: origin.x + FOOTERED_CONTAINER.x,
y: origin.y + FOOTERED_CONTAINER.y,
width: FOOTERED_CONTAINER.width,
height: FOOTERED_CONTAINER.height,
};
const local = buildScrollGesturePlan({
direction: 'up',
amount: 0.75,
...FRAME,
container: FOOTERED_CONTAINER,
});
const plan = buildScrollGesturePlan({
direction: 'up',
amount: 0.75,
...FRAME,
referenceOrigin: origin,
container,
});

assert.deepEqual(
[plan.x1, plan.y1, plan.x2, plan.y2],
[local.x1 + origin.x, local.y1 + origin.y, local.x2 + origin.x, local.y2 + origin.y],
);
assert.ok(strictlyInside(plan, container), JSON.stringify(plan));
assert.equal(plan.pixels, local.pixels);
assert.equal(plan.requestedPixels, local.requestedPixels);
assert.equal(plan.referenceWidth, FRAME.referenceWidth);
assert.equal(plan.referenceHeight, FRAME.referenceHeight);
});

test('an offset reference origin shifts an unclamped swipe into the dispatch frame', () => {
const origin = { x: 40, y: 80 };
const local = buildScrollGesturePlan({ direction: 'down', amount: 0.75, ...FRAME });
const plan = buildScrollGesturePlan({
direction: 'down',
amount: 0.75,
...FRAME,
referenceOrigin: origin,
});

assert.deepEqual(plan, {
...local,
x1: local.x1 + origin.x,
y1: local.y1 + origin.y,
x2: local.x2 + origin.x,
y2: local.y2 + origin.y,
});
});

test('a container lying wholly in the padding of the scrolled axis leaves the padded swipe as it was', () => {
const statusBand: Rect = { x: 0, y: 0, width: 402, height: PADDED_TOP - 1 };
const unclamped = buildScrollGesturePlan({ direction: 'up', ...FRAME });
Expand Down
31 changes: 26 additions & 5 deletions packages/contracts/src/scroll-gesture.ts
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
import { AppError } from '@agent-device/kernel/errors';
import { defineStringEnum } from './string-enum.ts';
import { isViewportRootNode } from './snapshot-visibility.ts';
import type { Rect, SnapshotNode } from '@agent-device/kernel/snapshot';
import type { Point, Rect, SnapshotNode } from '@agent-device/kernel/snapshot';

// What a caller may ASK for, as opposed to `ScrollDirection` (what the gesture resolves to):
// `top`/`bottom` are scroll-to-extreme requests with no direction of their own. Declared here
Expand Down Expand Up @@ -55,7 +55,14 @@ export type ScrollGestureOptions = {
referenceWidth: number;
referenceHeight: number;
/**
* The scroll container the swipe is meant for, in the reference frame's own zero-origin space.
* Where the reference frame sits in the space the caller dispatches. The planner subtracts
* this from `container`, plans inside the zero-origin frame, then adds it to every endpoint.
* Omit it when that frame already starts at (0, 0).
*/
referenceOrigin?: Point;
/**
* The scroll container the swipe is meant for, in the same space as `referenceOrigin`
* (the frame's own zero-origin space when the origin is omitted).
* Both endpoints stay strictly inside its intersection with the padded frame (#2714): a span
* that already fits is unchanged, one that leaves it is moved in, and one longer than it is
* shortened. A container the padding leaves nothing of along the scrolled axis gets the padded
Expand Down Expand Up @@ -119,6 +126,16 @@ const SCROLL_CONTAINER_EDGE_INSET = 8;

export function buildScrollGesturePlan(options: ScrollGestureOptions): ScrollGesturePlan {
const direction = options.direction;
const originX = options.referenceOrigin?.x ?? 0;
const originY = options.referenceOrigin?.y ?? 0;
const container =
options.container === undefined
? undefined
: {
...options.container,
x: options.container.x - originX,
y: options.container.y - originY,
};
const vertical = direction === 'up' || direction === 'down';
const axisLength = vertical ? options.referenceHeight : options.referenceWidth;
const crossLength = vertical ? options.referenceWidth : options.referenceHeight;
Expand All @@ -138,8 +155,8 @@ export function buildScrollGesturePlan(options: ScrollGestureOptions): ScrollGes
cross: Math.round(crossLength / 2),
};
const crossPadding = scrollEdgePadding(crossLength);
const lane = options.container
? keepSwipeInContainer(unclamped, options.container, vertical, {
const lane = container

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

P2: When the cross-axis band is empty, keepSwipeInContainer returns the padded lane without applying the scrolled-axis clamp. A narrow container (under about 17px across) can therefore receive a swipe that starts outside its vertical range. Keep the axis clamp and fall back only the cross coordinate, or document this case explicitly.

Prompt for AI agents
Check if this issue is valid — if so, understand the root cause and fix it. When an issue isn't valid or won't be fixed in this PR, reply in its thread with the reason and then resolve the thread. At packages/contracts/src/scroll-gesture.ts, line 158:

<comment>When the cross-axis band is empty, `keepSwipeInContainer` returns the padded lane without applying the scrolled-axis clamp. A narrow container (under about 17px across) can therefore receive a swipe that starts outside its vertical range. Keep the axis clamp and fall back only the cross coordinate, or document this case explicitly.</comment>

<file context>
@@ -138,8 +155,8 @@ export function buildScrollGesturePlan(options: ScrollGestureOptions): ScrollGes
   const crossPadding = scrollEdgePadding(crossLength);
-  const lane = options.container
-    ? keepSwipeInContainer(unclamped, options.container, vertical, {
+  const lane = container
+    ? keepSwipeInContainer(unclamped, container, vertical, {
         axis: { min: edgePadding, max: axisLength - edgePadding },
</file context>

? keepSwipeInContainer(unclamped, container, vertical, {
axis: { min: edgePadding, max: axisLength - edgePadding },
cross: { min: crossPadding, max: crossLength - crossPadding },
crossFrame: { min: 0, max: crossLength },
Expand All @@ -148,9 +165,13 @@ export function buildScrollGesturePlan(options: ScrollGestureOptions): ScrollGes
const laneLength = lane.axisEnd - lane.axisStart;
// Centring rounds an odd span up by one, so only a lane shorter than the travel itself is short.
const shortened = laneLength < travelPixels;
const endpoints = laneEndpoints(direction, lane);
return {
direction,
...laneEndpoints(direction, lane),
x1: endpoints.x1 + originX,
y1: endpoints.y1 + originY,
x2: endpoints.x2 + originX,
y2: endpoints.y2 + originY,
referenceWidth: options.referenceWidth,
referenceHeight: options.referenceHeight,
amount: options.amount,
Expand Down
18 changes: 4 additions & 14 deletions packages/platform-android/src/input-actions.ts
Original file line number Diff line number Diff line change
Expand Up @@ -324,26 +324,21 @@ export async function scrollAndroid(
helperSessionScope: options?.helperSessionScope,
});
const swipeSurface = resolveAndroidScrollSurface(direction, viewport, keyboard);
const relativePlan = buildScrollGesturePlan({
const plan = buildScrollGesturePlan({
direction,
amount: options?.amount,
pixels: options?.pixels,
referenceWidth: swipeSurface.viewport.width,
referenceHeight: swipeSurface.viewport.height,
...(options?.containerRect === undefined
? {}
: { container: relativeToViewport(options.containerRect, swipeSurface.viewport) }),
referenceOrigin: { x: swipeSurface.viewport.x, y: swipeSurface.viewport.y },
...(options?.containerRect === undefined ? {} : { container: options.containerRect }),
});
const scrollPlan = {
...relativePlan,
...plan,
// Injected coordinates are absolute, so their zero-origin reference frame
// must include the viewport offset as well as its dimensions.
referenceWidth: swipeSurface.viewport.x + swipeSurface.viewport.width,
referenceHeight: swipeSurface.viewport.y + swipeSurface.viewport.height,
x1: swipeSurface.viewport.x + relativePlan.x1,
y1: swipeSurface.viewport.y + relativePlan.y1,
x2: swipeSurface.viewport.x + relativePlan.x2,
y2: swipeSurface.viewport.y + relativePlan.y2,
};
const durationMs = Math.max(
options?.durationMs ?? DEFAULT_MOBILE_SCROLL_DURATION_MS,
Expand Down Expand Up @@ -374,11 +369,6 @@ export async function scrollAndroid(
};
}

/** The planner works in the swipe surface's zero-origin frame; snapshot rects are absolute. */
function relativeToViewport(rect: Rect, viewport: Rect): Rect {
return { ...rect, x: rect.x - viewport.x, y: rect.y - viewport.y };
}

/**
* The band one Android scroll may swipe, refusing when the keyboard owns the window (#2500).
*
Expand Down
99 changes: 99 additions & 0 deletions packages/platform-harmonyos/src/__tests__/input-actions.test.ts
Original file line number Diff line number Diff line change
@@ -1,7 +1,9 @@
import assert from 'node:assert/strict';
import { beforeEach, test, vi } from 'vitest';
import type { GesturePlan } from '@agent-device/contracts/gesture-plan-types';
import { buildScrollGesturePlan } from '@agent-device/contracts/scroll-gesture';
import type { DeviceInfo } from '@agent-device/kernel/device';
import type { Rect } from '@agent-device/kernel/snapshot';

const { runHarmonyShell, sleep, readHarmonyGestureViewport, invalidateHarmonyGestureViewport } =
vi.hoisted(() => ({
Expand Down Expand Up @@ -96,6 +98,103 @@ test('HarmonyOS scroll derives a viewport-aware swipe plan', async () => {
]);
});

const PHONE = { x: 0, y: 0, width: 402, height: 874 };
/** Inset under a navigation bar and above a footer, shorter than a 0.75 swipe. */
const INSET_LIST: Rect = { x: 18, y: 178, width: 366, height: 500 };
/** The same list shifted into an application window that is not at the screen origin. */
const OFFSET_WINDOW = { x: 40, y: 80, width: 402, height: 874 };

function strictlyInside(point: { x: number; y: number }, rect: Rect): boolean {
return (
point.x > rect.x &&
point.x < rect.x + rect.width &&
point.y > rect.y &&
point.y < rect.y + rect.height
);
}

function dispatchedSwipe(callIndex?: number): { x1: number; y1: number; x2: number; y2: number } {
const calls = runHarmonyShell.mock.calls;
const call = calls[callIndex ?? calls.length - 1];
const args = call?.[1] as readonly unknown[];
return {
x1: Number(args[3]),
y1: Number(args[4]),
x2: Number(args[5]),
y2: Number(args[6]),
};
}

test('HarmonyOS scroll keeps an inset list swipe inside the container and reports the shortened span', async () => {
readHarmonyGestureViewport.mockResolvedValue(PHONE);
const unclamped = await scrollHarmony(DEVICE, 'down', { amount: 0.75 });
const outside = dispatchedSwipe(0);
const result = await scrollHarmony(DEVICE, 'down', { amount: 0.75, containerRect: INSET_LIST });
const swipe = dispatchedSwipe(1);

assert.equal(
strictlyInside({ x: outside.x1, y: outside.y1 }, INSET_LIST) &&
strictlyInside({ x: outside.x2, y: outside.y2 }, INSET_LIST),
false,
'the fixture must leave the list',
);
assert.ok(strictlyInside({ x: swipe.x1, y: swipe.y1 }, INSET_LIST), JSON.stringify(swipe));
assert.ok(strictlyInside({ x: swipe.x2, y: swipe.y2 }, INSET_LIST), JSON.stringify(swipe));
assert.deepEqual(swipe, { x1: result.x1, y1: result.y1, x2: result.x2, y2: result.y2 });
assert.equal(result.pixels, Math.abs(Number(result.y2) - Number(result.y1)));
assert.ok(Number(result.pixels) < Number(unclamped.pixels));
assert.equal(result.requestedPixels, unclamped.pixels);
});

test('HarmonyOS scroll dispatches an offset-window swipe inside the absolute container', async () => {
readHarmonyGestureViewport.mockResolvedValue(OFFSET_WINDOW);
const list: Rect = {
x: OFFSET_WINDOW.x + INSET_LIST.x,
y: OFFSET_WINDOW.y + INSET_LIST.y,
width: INSET_LIST.width,
height: INSET_LIST.height,
};
const result = await scrollHarmony(DEVICE, 'up', { amount: 0.75, containerRect: list });
const swipe = dispatchedSwipe();

assert.ok(strictlyInside({ x: swipe.x1, y: swipe.y1 }, list), JSON.stringify(swipe));
assert.ok(strictlyInside({ x: swipe.x2, y: swipe.y2 }, list), JSON.stringify(swipe));
assert.deepEqual(swipe, { x1: result.x1, y1: result.y1, x2: result.x2, y2: result.y2 });
assert.ok(Number(result.pixels) < Number(result.requestedPixels));
});

test('HarmonyOS scroll leaves a swipe that already fits its container unchanged', async () => {
const fitting: Rect = { x: 18, y: 62, width: 366, height: 729 };
readHarmonyGestureViewport.mockResolvedValue(PHONE);
const unclamped = await scrollHarmony(DEVICE, 'down', { amount: 0.75 });
const result = await scrollHarmony(DEVICE, 'down', { amount: 0.75, containerRect: fitting });

assert.deepEqual(result, unclamped);
assert.equal('requestedPixels' in result, false);
});

test('HarmonyOS scroll without a container keeps the unclamped swipe in screen coordinates', async () => {
readHarmonyGestureViewport.mockResolvedValue(OFFSET_WINDOW);
const plan = await scrollHarmony(DEVICE, 'down', { amount: 0.75 });
const local = buildScrollGesturePlan({
direction: 'down',
amount: 0.75,
referenceWidth: OFFSET_WINDOW.width,
referenceHeight: OFFSET_WINDOW.height,
});
const swipe = dispatchedSwipe();

assert.deepEqual(swipe, {
x1: local.x1 + OFFSET_WINDOW.x,
y1: local.y1 + OFFSET_WINDOW.y,
x2: local.x2 + OFFSET_WINDOW.x,
y2: local.y2 + OFFSET_WINDOW.y,
});
assert.deepEqual(swipe, { x1: plan.x1, y1: plan.y1, x2: plan.x2, y2: plan.y2 });
assert.equal(plan.pixels, local.pixels);
assert.equal('requestedPixels' in plan, false);
});

test('HarmonyOS gestures lower a single trajectory and reject multi-touch', async () => {
const singlePlan: GesturePlan = {
topology: 'single',
Expand Down
9 changes: 7 additions & 2 deletions packages/platform-harmonyos/src/input-actions.ts
Original file line number Diff line number Diff line change
@@ -1,6 +1,9 @@
import type { DeviceRotation } from '@agent-device/contracts/device';
import type { GesturePlan } from '@agent-device/contracts/gesture-plan-types';
import { DEFAULT_MOBILE_SCROLL_DURATION_MS } from '@agent-device/contracts/scroll-command';
import {
type ScrollContainerTarget,
DEFAULT_MOBILE_SCROLL_DURATION_MS,
} from '@agent-device/contracts/scroll-command';
import {
type ScrollDirection,
buildScrollGesturePlan,
Expand Down Expand Up @@ -52,7 +55,7 @@ export async function fillHarmony(
export async function scrollHarmony(
device: DeviceInfo,
direction: ScrollDirection,
options?: { amount?: number; pixels?: number; durationMs?: number },
options?: { amount?: number; pixels?: number; durationMs?: number } & ScrollContainerTarget,
): Promise<Record<string, unknown>> {
const viewport = await readHarmonyGestureViewport(device);
const plan = buildScrollGesturePlan({
Expand All @@ -61,6 +64,8 @@ export async function scrollHarmony(
pixels: options?.pixels,
referenceWidth: viewport.width,
referenceHeight: viewport.height,
referenceOrigin: { x: viewport.x, y: viewport.y },
...(options?.containerRect === undefined ? {} : { container: options.containerRect }),
});
await runHarmonyShell(device, [
'uitest',
Expand Down
2 changes: 1 addition & 1 deletion packages/platform-harmonyos/src/snapshot.ts
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@ import { readHarmonyKeyboardBand } from './keyboard-band.ts';

const MAX_NODES = 5_000;

// Gesture planning only reads the Application rect's width/height, but every
// Gesture planning reads the Application rect's origin and size, but every
// read paid a full layout dump (`uitest dumpLayout` + `file recv` + `rm`).
// Edge scrolls run several passes back to back, so a short memo collapses
// them into one dump while staying far below the cadence where a foreground
Expand Down
Loading