diff --git a/docs/evidence/android-densified-lowering-2026-11-13.md b/docs/evidence/android-densified-lowering-2026-11-13.md new file mode 100644 index 0000000000..77c0555b5e --- /dev/null +++ b/docs/evidence/android-densified-lowering-2026-11-13.md @@ -0,0 +1,52 @@ +# Android densified lowering through the shared densifyGesturePath, 2026-11-13 + +Contributor-reported evidence for #4239's Android side records one pan and one +scroll on hardware. The retained pan notes name the Android helper. The scroll +notes report success without identifying a backend. Raw CLI responses were not +preserved, so the excerpts below are not verbatim response artifacts. + +## Device and build + +- Redmi K60 (Android), adb serial `5p4hqglvpzxshiuw`, Settings app foreground +- Contributor-reported tested commit: `181935c951`, built in the contribution + worktree, daemon restarted clean; the Android automation helper (0.32.0 build, packaged under + this tree's version name) was installed with `adb install -t` before the run + +## Retained contributor notes + +The original pan notes included `executionProfile: timed-pan`. Android's command +does not emit that field. It is omitted below; it describes the plan selected by +the code, not a field observed in the response. + +```json +{ + "kind": "pan", + "backend": "android-helper", + "message": "Panned (500, 1400) by (0, -600)" +} +``` + +```json +{ + "message": "Scrolled down", + "success": true +} +``` + +The pan notes name `backend: android-helper`. For a pan plan, the shared planner +selects `timed-pan` and Android lowers its samples before helper dispatch; this is +an inference from the code. The scroll notes do not identify a backend or sampled +route. The contributor reported visible scrolling in both runs. + +## What this evidences + +The contributor reports Android helper pan dispatch and a successful scroll. +The retained notes cannot establish a response-reported execution profile or +which backend or lowering executed the scroll. Neither result shows the cadence +of individual timed samples. Cadence equivalence rests on the pure-function and +Android-lowering tests rather than these reported runs. + +The retained notes name `gesture pan --json` and `scroll down`, but do not +preserve the complete argument lists. The pan response records the start point +`(500, 1400)` and displacement `(0, -600)`. These are response facts, not a +reconstruction of the missing invocation. diff --git a/packages/contracts/src/gesture-plan.test.ts b/packages/contracts/src/gesture-plan.test.ts index bf734e8ef2..0afbff5079 100644 --- a/packages/contracts/src/gesture-plan.test.ts +++ b/packages/contracts/src/gesture-plan.test.ts @@ -1,6 +1,10 @@ import assert from 'node:assert/strict'; import { test } from 'vitest'; -import { buildDragGesturePlan, singlePointerPlanEndpoints } from './gesture-plan.ts'; +import { + buildDragGesturePlan, + densifyGesturePath, + singlePointerPlanEndpoints, +} from './gesture-plan.ts'; const DRAG_PROPERTY_RUNS = 100; @@ -77,5 +81,96 @@ test('every drag plan preserves ordered samples and exact hold/move endpoints', for (let index = 1; index < samples.length; index += 1) { assert.ok(samples[index]!.offsetMs >= samples[index - 1]!.offsetMs); } + const dense = densifyGesturePath(samples, 'android'); + assert.deepEqual(dense[0]?.point, from); + assert.deepEqual(dense.at(-1)?.point, to); } }); + +test('densifyGesturePath merges the profile grid into the canonical path and keeps its endpoints', () => { + const canonical = [ + { offsetMs: 0, point: { x: 0, y: 0 } }, + { offsetMs: 100, point: { x: 100, y: 0 } }, + { offsetMs: 200, point: { x: 100, y: 100 } }, + ]; + const dense = densifyGesturePath(canonical, 'default'); + + // The 16 ms grid and the canonical offsets merge into one ascending timeline. + const offsets = dense.map(({ offsetMs }) => offsetMs); + assert.equal(offsets[0], 0); + assert.equal(offsets.at(-1), 200); + assert.ok([...offsets].sort((left, right) => left - right).every((o, i) => o === offsets[i])); + assert.deepEqual(dense[0]?.point, { x: 0, y: 0 }); + assert.deepEqual(dense.at(-1)?.point, { x: 100, y: 100 }); + // A merged grid offset inside the first segment interpolates on that segment's line. + const at50 = dense.find(({ offsetMs }) => offsetMs === 50); + assert.deepEqual(at50?.point, { x: 50, y: 0 }); + const at150 = dense.find(({ offsetMs }) => offsetMs === 150); + assert.deepEqual(at150?.point, { x: 100, y: 50 }); + // Canonical offsets survive verbatim. + const at100 = dense.find(({ offsetMs }) => offsetMs === 100); + assert.deepEqual(at100?.point, { x: 100, y: 0 }); +}); + +test('densifyGesturePath hands within-segment pacing to the progress hook and keeps its endpoint rule', () => { + const canonical = [ + { offsetMs: 0, point: { x: 0, y: 0 } }, + { offsetMs: 100, point: { x: 100, y: 0 } }, + ]; + const seen: Array<[number, number]> = []; + const dense = densifyGesturePath(canonical, 'default', (progress, segmentDurationMs) => { + seen.push([progress, segmentDurationMs]); + return progress * progress; + }); + + const at50 = dense.find(({ offsetMs }) => offsetMs === 50); + assert.deepEqual(at50?.point, { x: 25, y: 0 }); + assert.ok( + seen.some(([progress, segmentDurationMs]) => progress === 0.5 && segmentDurationMs === 100), + ); + // The hook only shapes interior offsets: every progress it saw lies strictly + // between 0 and 1, so the endpoints never passed through it. + assert.ok(seen.length >= 1); + for (const [progress] of seen) { + assert.ok(progress > 0 && progress < 1, `hook saw progress ${progress}`); + } + assert.deepEqual(dense[0]?.point, { x: 0, y: 0 }); + assert.deepEqual(dense.at(-1)?.point, { x: 100, y: 0 }); +}); + +test('densifyGesturePath refuses a path of fewer than two timed samples', () => { + assert.throws(() => densifyGesturePath([{ offsetMs: 0, point: { x: 0, y: 0 } }], 'default'), { + code: 'INVALID_ARGS', + details: { reason: 'GESTURE_PATH_TOO_SHORT' }, + }); + assert.throws( + () => + densifyGesturePath( + [ + { offsetMs: 0, point: { x: 0, y: 0 } }, + { offsetMs: 0, point: { x: 1, y: 0 } }, + ], + 'default', + ), + { code: 'INVALID_ARGS', details: { reason: 'GESTURE_PATH_TOO_SHORT' } }, + ); +}); + +for (const [name, offsets] of [ + ['a nonzero first offset', [5, 100]], + ['tied first offsets', [0, 0, 100]], + ['tied last offsets', [0, 100, 100]], + ['reversed offsets', [0, 75, 50, 100]], + ['a non-finite interior offset', [0, Number.NaN, 100]], +] as const) { + test(`densifyGesturePath refuses ${name}`, () => { + const canonical = offsets.map((offsetMs, index) => ({ + offsetMs, + point: { x: index * 10, y: 0 }, + })); + assert.throws(() => densifyGesturePath(canonical, 'default'), { + code: 'INVALID_ARGS', + details: { reason: 'GESTURE_PATH_INVALID_TIMELINE' }, + }); + }); +} diff --git a/packages/contracts/src/gesture-plan.ts b/packages/contracts/src/gesture-plan.ts index 8a6db00b0e..dba1ad24a1 100644 --- a/packages/contracts/src/gesture-plan.ts +++ b/packages/contracts/src/gesture-plan.ts @@ -22,6 +22,7 @@ import type { GestureExecutionProfile, GestureSemanticInput, PointerTrajectory, + PointerTrajectorySample, SinglePointerGesturePlan, SinglePointerTrajectory, TapGesturePlan, @@ -467,6 +468,80 @@ export function sampleGestureOffsets( ); } +/** + * Densifies a plan's own timed path onto one merged sampling grid: the profile grid and the + * canonical offsets merge, every merged offset interpolates inside its canonical segment, and the + * endpoints and duration stay the plan's — the duration IS the last canonical offset, so a + * disagreeing duration cannot be written. This is the one owner of the sampled-path rule; every + * transport lowering calls it instead of keeping its own merge-and-interpolate loop. The optional + * progress hook lets a transport reshape within-segment pacing (Android's controlled scroll) while + * the segment walk and the endpoint rule stay shared. + */ +export function densifyGesturePath( + canonical: readonly PointerTrajectorySample[], + profile: GestureSamplingProfile, + progress: (baseProgress: number, segmentDurationMs: number) => number = (p) => p, +): PointerTrajectorySample[] { + // The walk reads each segment's end, so a path of fewer than two timed samples has no + // interpolable segment; the last offset is the duration the sampling grid builds on. + const durationMs = canonical.at(-1)?.offsetMs ?? 0; + if (canonical.length < 2 || !Number.isFinite(durationMs) || durationMs <= 0) { + throw new AppError('INVALID_ARGS', 'gesture path needs at least two timed samples', { + reason: 'GESTURE_PATH_TOO_SHORT', + }); + } + if ( + canonical[0]!.offsetMs !== 0 || + canonical.some( + ({ offsetMs }, index) => + !Number.isFinite(offsetMs) || (index > 0 && offsetMs <= canonical[index - 1]!.offsetMs), + ) + ) { + throw new AppError( + 'INVALID_ARGS', + 'gesture path needs a strictly increasing timeline from zero', + { + reason: 'GESTURE_PATH_INVALID_TIMELINE', + }, + ); + } + const offsets = [ + ...new Set([ + ...sampleGestureOffsets(durationMs, profile), + ...canonical.map(({ offsetMs }) => offsetMs), + ]), + ].sort((left, right) => left - right); + let segmentIndex = 0; + return offsets.map((offsetMs) => { + while ( + segmentIndex < canonical.length - 2 && + offsetMs > canonical[segmentIndex + 1]!.offsetMs + ) { + segmentIndex += 1; + } + const start = canonical[segmentIndex]!; + const end = canonical[segmentIndex + 1]!; + const segmentDurationMs = end.offsetMs - start.offsetMs; + const baseProgress = + segmentDurationMs === 0 ? 0 : (offsetMs - start.offsetMs) / segmentDurationMs; + return { + offsetMs, + point: + // Both endpoints and every zero-length segment keep the canonical + // point, so the hook only ever reshapes strictly interior offsets. + offsetMs === end.offsetMs || segmentDurationMs === 0 + ? end.point + : baseProgress === 0 + ? start.point + : interpolateGesturePoint( + start.point, + end.point, + progress(baseProgress, segmentDurationMs), + ), + }; + }); +} + function assertSamplesInViewport( samples: readonly { offsetMs: number; point: Point }[], viewport: Rect, diff --git a/packages/platform-android/src/touch-plan-lowering.ts b/packages/platform-android/src/touch-plan-lowering.ts index dc7e822dc8..59077b632e 100644 --- a/packages/platform-android/src/touch-plan-lowering.ts +++ b/packages/platform-android/src/touch-plan-lowering.ts @@ -1,8 +1,7 @@ import type { ScrollReleaseBehavior } from '@agent-device/contracts/scroll-command'; import { GESTURE_SAMPLE_INTERVAL_MS, - interpolateGesturePoint, - sampleGestureOffsets, + densifyGesturePath, } from '@agent-device/contracts/gesture-plan'; import type { GesturePlan, @@ -63,38 +62,11 @@ export function lowerAndroidTouchPlan(plan: AndroidTouchPlan): AndroidLoweredTou const { releaseBehavior, ...gesturePlan } = plan; const [{ pointerId, samples: canonicalSamples }] = gesturePlan.pointers; - const offsets = [ - ...new Set([ - ...sampleGestureOffsets(plan.durationMs, 'android'), - ...canonicalSamples.map(({ offsetMs }) => offsetMs), - ]), - ].sort((left, right) => left - right); - let segmentIndex = 0; - const dense = offsets.map((offsetMs) => { - while ( - segmentIndex < canonicalSamples.length - 2 && - offsetMs > canonicalSamples[segmentIndex + 1]!.offsetMs - ) { - segmentIndex += 1; - } - const start = canonicalSamples[segmentIndex]!; - const end = canonicalSamples[segmentIndex + 1]!; - const segmentDurationMs = end.offsetMs - start.offsetMs; - const progress = (offsetMs - start.offsetMs) / segmentDurationMs; - return { - offsetMs, - point: - offsetMs === end.offsetMs || segmentDurationMs === 0 - ? end.point - : interpolateGesturePoint( - start.point, - end.point, - releaseBehavior === 'controlled' - ? controlledScrollProgress(progress, segmentDurationMs) - : progress, - ), - }; - }); + const dense = densifyGesturePath(canonicalSamples, 'android', (progress, segmentDurationMs) => + releaseBehavior === 'controlled' + ? controlledScrollProgress(progress, segmentDurationMs) + : progress, + ); // The sampler floors the frame count at three, so `dense` always holds at least four samples. const samples: AndroidTransportSamples = [dense[0]!, dense[1]!, dense[2]!, ...dense.slice(3)]; diff --git a/src/commands/schema/cli-help-gestures.ts b/src/commands/schema/cli-help-gestures.ts index 8beb3dba63..e8debbd501 100644 --- a/src/commands/schema/cli-help-gestures.ts +++ b/src/commands/schema/cli-help-gestures.ts @@ -21,7 +21,7 @@ Shapes: agent-device gesture rotate 35 200 420 agent-device gesture transform 200 420 80 -40 2 35 700 agent-device gesture tap 80 300 200 450 310 600 --count 20 --pause-ms 100 - scroll