Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
55 changes: 54 additions & 1 deletion packages/capture-kit/src/snapshot-baseline.test.ts
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
import { expect, test } from 'vitest';
import type { SnapshotState } from '@agent-device/contracts/snapshot-model';
import { surfaceScopedNodes } from './snapshot-baseline.ts';
import type { SurfaceScopedNodes } from '@agent-device/contracts/interaction';
import { postActionBaseline, surfaceScopedNodes } from './snapshot-baseline.ts';

const nodes = [{ index: 0, type: 'Application', ref: 'e1' }];

Expand Down Expand Up @@ -33,6 +34,58 @@ test('a session with no stored snapshot still mints an empty baseline', () => {
expect(surfaceScopedNodes(undefined)).toEqual({ nodes: [] });
});

test('the mint carries the presentation the capture read', () => {
expect(surfaceScopedNodes(snapshot({ presentation: { interactiveOnly: true } }))).toEqual({
nodes,
presentation: { interactiveOnly: true },
});
});

const INTERACTIVE = { interactiveOnly: true };
const FULL = {};
const frame = (presentation: SurfaceScopedNodes['presentation']) => tree('frame', presentation);

// #4486: a settle diff compares one view, and the agent's view is its ref frame's presentation.
test.each([
['interactive frame, interactive candidate', INTERACTIVE, INTERACTIVE, 'candidate'],
['interactive frame, full candidate', INTERACTIVE, FULL, 'frame'],
['full frame, interactive candidate', FULL, INTERACTIVE, 'frame'],
['full frame, full candidate', FULL, FULL, 'candidate'],
[
'interactive frame, scoped interactive candidate',
INTERACTIVE,
{ interactiveOnly: true, scope: 'Composer' },
'frame',
],
['interactive frame, candidate with default full view', INTERACTIVE, undefined, 'frame'],
['full frame, candidate with default full view', FULL, undefined, 'candidate'],
] as const)(
'%s: the %s answers',
async (_case, framePresentation, candidatePresentation, expected) => {
const baseline = await postActionBaseline({
frame: frame(framePresentation),
candidates: [tree('candidate', candidatePresentation)],
});
expect(baseline.nodes[0]?.label).toBe(expected);
},
);

test('without a frame the first non-empty candidate answers, in its own view', async () => {
const baseline = await postActionBaseline({
frame: undefined,
candidates: [undefined, { nodes: [] }, tree('candidate', FULL)],
});
expect(baseline).toEqual(tree('candidate', FULL));
expect(await postActionBaseline({ frame: undefined, candidates: [] })).toEqual({ nodes: [] });
});

function tree(label: string, presentation: SurfaceScopedNodes['presentation']): SurfaceScopedNodes {
return {
nodes: [{ index: 0, type: 'Button', label, ref: 'e1' }],
...(presentation ? { presentation } : {}),
};
}

function snapshot(fields: Partial<SnapshotState>): SnapshotState {
return { nodes, createdAt: 0, presentationKey: '', ...fields } as SnapshotState;
}
46 changes: 44 additions & 2 deletions packages/capture-kit/src/snapshot-baseline.ts
Original file line number Diff line number Diff line change
@@ -1,9 +1,18 @@
import type { SnapshotState } from '@agent-device/contracts/snapshot-model';
import type { SurfaceScopedNodes } from '@agent-device/contracts/interaction';
import type { SnapshotPresentation } from '@agent-device/kernel/snapshot';

/** Reproduce a populated baseline's view; without one, capture the interactive view. */
export function postActionPresentation(
baseline: SurfaceScopedNodes | undefined,
): SnapshotPresentation {
return baseline?.presentation ?? (baseline?.nodes.length ? {} : { interactiveOnly: true });
}

/**
* Mints the one baseline value a post-action comparison carries: the captured nodes together with the
* surface they describe and the presentation the capture read (#2438, #3565).
* surface they describe, the presentation the capture read (#2438, #3565), and the view it was read
* in (#4486).
*
* Both settle routes take their baseline from here — the element-targeted interaction route and the
* generic daemon route (`scroll`/`back --settle`) — so a fact a capture established cannot be carried
Expand All @@ -16,7 +25,10 @@ import type { SurfaceScopedNodes } from '@agent-device/contracts/interaction';
*/
export function surfaceScopedNodes(
snapshot:
| Pick<SnapshotState, 'nodes' | 'iosSystemSurfaceBundleId' | 'presentedSurface'>
| Pick<
SnapshotState,
'nodes' | 'iosSystemSurfaceBundleId' | 'presentedSurface' | 'presentation'
>
| undefined,
): SurfaceScopedNodes {
return {
Expand All @@ -25,5 +37,35 @@ export function surfaceScopedNodes(
? { surfaceBundleId: snapshot.iosSystemSurfaceBundleId }
: {}),
...(snapshot?.presentedSurface ? { presentedSurface: snapshot.presentedSurface } : {}),
...(snapshot?.presentation ? { presentation: snapshot.presentation } : {}),
};
}

/**
* The pre-action tree a settle diff runs against, and through its `presentation` the view the settle
* captures in (#4486). Two trees compare only within one presentation, and the agent's view is the
* presentation of the ref frame it last read: `snapshot -i`, a full `snapshot`, a scoped one, or the
* refs a settle issued. So the first candidate read in that view answers. A candidate read in another
* view, such as a selector resolution that fell back to the full tree or a full tree a selector read
* stored, is skipped, and the frame itself answers when no candidate qualifies. Without a frame, the
* first candidate answers.
*
* Every settle route picks its baseline here, so none can compare an interactive read with a full one.
*/
export async function postActionBaseline(params: {
/** The ref frame, as the caller admits it as a baseline. */
frame: SurfaceScopedNodes | undefined;
/** Pre-action trees in the caller's order of preference. */
candidates: readonly (SurfaceScopedNodes | undefined)[];
}): Promise<SurfaceScopedNodes> {
const { isSameSnapshotPresentation } = await import('@agent-device/kernel/snapshot');
const { frame } = params;
const view = frame && frame.nodes.length > 0 ? frame.presentation : undefined;
const candidate = params.candidates.find(
(baseline) =>
baseline !== undefined &&
baseline.nodes.length > 0 &&
(!frame?.nodes.length || isSameSnapshotPresentation(baseline.presentation, view)),
);
return candidate ?? frame ?? { nodes: [] };
}
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@
type SnapshotDiagnosticsSummary,
} from '@agent-device/contracts/capture';
import { AppError } from '@agent-device/kernel/errors';
import { isSameSnapshotPresentation } from '@agent-device/kernel/snapshot';
import { normalizeType } from '@agent-device/contracts/snapshot';
import { snapshotProducerFacts } from '@agent-device/contracts/snapshot-producers';
import type {
Expand Down Expand Up @@ -135,7 +136,11 @@
}
const capture = await captureRuntimeSnapshot(runtime, options);
const flattenForDiff = options.interactiveOnly === true;
const previousSnapshot = capture.session?.snapshot;
const stored = capture.session?.snapshot;
const previousSnapshot =
stored && isSameSnapshotPresentation(stored.presentation, capture.snapshot.presentation)
? stored
: undefined;

Check warning on line 143 in packages/command-runtime/src/commands/capture/runtime/snapshot.ts

View check run for this annotation

apex-sentinel-staging / Apex Sentinel

minor: `diff snapshot` drops the stored baseline when the capture carries no presentation

`diff snapshot` against a runtime whose capture produces a `SnapshotState` without a `presentation` field always answers `baselineInitialized: true` with an empty diff. The guard compares `stored.presentation` with `capture.snapshot.presentation`; any stored tree that carries a `presentation` fails `isSameSnapshotPresentation` against `undefined`, which the kernel treats as the full tree. `surfaceScopedNodes` and `postActionPresentation` in this PR treat a missing presentation as the full view, but this call site reads it as "unknown view". ### Also raised here Also raised at this location by persona `javascript`: diff snapshot silently starts a new baseline when the stored tree has no presentation — A session whose stored snapshot carries no `presentation` (a pre-#​4486 session resumed from disk, or any capture path that builds SnapshotState without stamping it) now always diffs as `baselineInitialized: true` after this change, even against a stored tree read with the same flags. `isSameSnapshotPresentation(stored.presentation, capture.snapshot.presentation)` compares `buildSnapshotPresentationKey(undefined)` with the key of the current capture's `{}`/`{interactiveOnly:true}`, and an undefined argument yields `interactiveOnly: false`, so an interactive `diff snapshot -i` mismatches and the stored baseline is discarded — the opposite of how `postActionBaseline` treats absence, where `isSameSnapshotPresentation(baseline.presentation, view)` with `view === undefined` (an empty frame) accepts any candidate, and where an absent frame presentation equals `{}`. …
Comment thread
thymikee marked this conversation as resolved.
Resolved
const nextSession = nextSnapshotSession(options.session, capture);

if (!previousSnapshot) {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -6,8 +6,7 @@ import { toBackendContext } from '../../runtime-common.ts';
/**
* A leaf shared by every interaction runtime module that needs a fresh (or session-cached) tree:
* `resolution.ts`'s ref/selector resolution, `selector-target-resolution.ts`'s acting selector lookup,
* `moving-target-gate.ts`'s re-observation, `gestures.ts`'s viewport resolution, and
* `post-action-observation.ts`'s `--verify` baseline.
* `moving-target-gate.ts`'s re-observation, and `gestures.ts`'s viewport resolution.
* Deliberately self-contained — it imports no sibling of this directory, so those siblings can
* import it without ever risking a cycle back.
*/
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -5,9 +5,16 @@
SettleObservation,
SettleParams,
} from '@agent-device/contracts/interaction';
import { captureInteractionSnapshot } from './interaction-snapshot-capture.ts';
import { summarizePostActionEvidence } from './post-action-surface.ts';
import { surfaceScopedNodes } from '@agent-device/capture-kit/snapshot-baseline';
import { isSameSnapshotPresentation } from '@agent-device/kernel/snapshot';
import {
resolvePostActionSurfaceChange,
summarizePostActionEvidence,
} from './post-action-surface.ts';
import {
postActionPresentation,
surfaceScopedNodes,
} from '@agent-device/capture-kit/snapshot-baseline';
import { captureSelectorSnapshot } from './selector-read-shared.ts';
import { settleAfterInteraction, settleEvidence } from './settle.ts';

type ObservedResult<T extends object> = T & {
Expand Down Expand Up @@ -81,9 +88,17 @@
...params.settle,
resolved,
});
const evidence = params.verify
? settleEvidence(outcome.settledCapture, resolved.preAction)
: undefined;
// A surface transition is evidence in every view; node digests compare only within one view.
const baseline =
outcome.settledCapture &&
(resolvePostActionSurfaceChange(resolved.preAction, outcome.settledCapture) ||
isSameSnapshotPresentation(
resolved.preAction?.presentation,
outcome.settledCapture.presentation,
))
? resolved.preAction
: undefined;
const evidence = params.verify ? settleEvidence(outcome.settledCapture, baseline) : undefined;

Check warning on line 101 in packages/command-runtime/src/commands/interaction/runtime/post-action-observation.ts

View check run for this annotation

apex-sentinel-staging / Apex Sentinel

minor: --settle --verify loses changedFromBefore when preAction and frame views differ

`press label="Back" --settle --verify` after a full-frame `snapshot` produces no `evidence` at all (`changedFromBefore`, `digest`, `nodeCount` all absent) when selector resolution reads an interactive tree as `resolved.preAction`. The new gate at lines 92–100 compares `resolved.preAction` against the settled capture, but the settle engine diffed against a different, view-matched baseline (`postActionBaseline`'s frame).
Comment on lines +91 to +101

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

🆕 🟡 Minor · --settle --verify loses changedFromBefore when preAction and frame views differ

press label="Back" --settle --verify after a full-frame snapshot produces no evidence at all (changedFromBefore, digest, nodeCount all absent) when selector resolution reads an interactive tree as resolved.preAction. The new gate at lines 92–100 compares resolved.preAction against the settled capture, but the settle engine diffed against a different, view-matched baseline (postActionBaseline's frame).

Suggested change

Feed settleEvidence the baseline the settle loop actually diffed against (the postActionBaseline result on outcome) rather than gating resolved.preAction on its view; keep the surface-change escape hatch.

Details

persona: senior-developer, javascript

Useful? Reply @apex-sentinel-staging like, or @apex-sentinel-staging dislike <why> if it is not.

Reply @apex-sentinel-staging note ... to add context, or @apex-sentinel-staging pivot ... if it is wrong — I will re-check the cited code.

Copy link
Copy Markdown
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Partially valid at merged head 5b2c4cd: when selector preAction is interactive but settle substitutes a full ref-frame baseline, changedFromBefore is omitted. The claim that all evidence disappears is inaccurate: settleEvidence receives an undefined baseline and summarizePostActionEvidence still returns digest and node counts. Known surface transitions retain their verdict.

Local revision ed9ffd13a4277e507e73770621e82c23f3662e8f addresses the underlying substitution by keeping fresh selector pre-action captures and the latest stored observation for scroll/back, then reproducing that baseline presentation. It was not pushed before merge. The user then requested cleanup, which is complete. No worktree or follow-up PR has been recreated. Leaving this finding open for follow-up instead of marking an unmerged fix as resolved.

return { settle: outcome.observation, ...(evidence ? { evidence } : {}) };
}
if (!params.verify) return {};
Expand All @@ -92,8 +107,8 @@
}

/**
* Post-action side of `--verify` (#1047): one interactive-only capture through
* the same capture helper the resolution path already uses, digested and then
* Post-action side of `--verify` (#1047): one capture in the pre-action tree's presentation
* (interactive-only by default), so the comparison reads one view (#4486), digested and then
* discarded. The node tree itself is never attached to the result, only the
* cheap summary. A missing baseline yields no `changedFromBefore` claim —
* no baseline, no comparison.
Expand All @@ -104,7 +119,11 @@
resolved: ResolvedInteractionTarget,
): Promise<InteractionEvidence | undefined> {
try {
const capture = await captureInteractionSnapshot(runtime, options, true);
const capture = await captureSelectorSnapshot(runtime, options, {
updateSession: true,
includeRects: true,
presentation: postActionPresentation(resolved.preAction),
});
return summarizePostActionEvidence(surfaceScopedNodes(capture.snapshot), resolved.preAction);
} catch {
return undefined;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -117,12 +117,13 @@ test('press --settle attaches no diff across a sheet-to-app surface change and d
assert.match(settle.hint ?? '', /different surfaces/);
});

test('press --settle --verify reports one app-to-sheet surface change on both payloads', async () => {
const before = buttonSnapshot();
const sheet = webSignInSheetSnapshot(WEB_SIGN_IN_SHEET_LABELS);
test('press --settle --verify reports a surface change even when the selector and frame views differ', async () => {
const before = { ...buttonSnapshot(), presentation: { interactiveOnly: true } };
const frame = { ...before, presentation: {} };

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

P2: This fixture does not actually exercise the claimed selector-versus-frame view mismatch: the first selector capture overwrites session.snapshot, and no refFrameSnapshot is initialized. Preserve frame as refFrameSnapshot (or configure the helper to do so) so the test verifies the frame fallback and presentation selection rather than only repeating the existing cross-surface case.

Prompt for AI agents
Check if this issue is valid — if so, understand the root cause and fix it. When an issue isn't valid or won't be fixed in this PR, reply in its thread with the reason and then resolve the thread. At packages/command-runtime/src/commands/interaction/runtime/post-action-surface.test.ts, line 122:

<comment>This fixture does not actually exercise the claimed selector-versus-frame view mismatch: the first selector capture overwrites `session.snapshot`, and no `refFrameSnapshot` is initialized. Preserve `frame` as `refFrameSnapshot` (or configure the helper to do so) so the test verifies the frame fallback and presentation selection rather than only repeating the existing cross-surface case.</comment>

<file context>
@@ -117,12 +117,13 @@ test('press --settle attaches no diff across a sheet-to-app surface change and d
-  const sheet = webSignInSheetSnapshot(WEB_SIGN_IN_SHEET_LABELS);
+test('press --settle --verify reports a surface change even when the selector and frame views differ', async () => {
+  const before = { ...buttonSnapshot(), presentation: { interactiveOnly: true } };
+  const frame = { ...before, presentation: {} };
+  const sheet = { ...webSignInSheetSnapshot(WEB_SIGN_IN_SHEET_LABELS), presentation: {} };
   let captures = 0;
</file context>

const sheet = { ...webSignInSheetSnapshot(WEB_SIGN_IN_SHEET_LABELS), presentation: {} };
let captures = 0;
const device = createSettleDevice({
stored: before,
stored: frame,
captureSnapshot: () => {
captures += 1;
return { snapshot: captures === 1 ? before : sheet };
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@ import {
type SnapshotCommandOptionFields,
type SnapshotNode,
type SnapshotPreferredBackend,
type SnapshotPresentation,
} from '@agent-device/kernel/snapshot';
import type { SnapshotState } from '@agent-device/contracts/snapshot-model';
import {
Expand Down Expand Up @@ -72,6 +73,11 @@ export async function captureSelectorSnapshot(
interactiveOnly?: boolean;
includeHiddenContentHints?: boolean;
preferredBackend?: SnapshotPreferredBackend;
/**
* The whole presentation to capture in, overriding `interactiveOnly`, `scope` and the command's
* depth and raw flags: a post-action read reproduces its baseline's view (#4486).
*/
presentation?: SnapshotPresentation;
} = {
updateSession: true,
},
Expand All @@ -82,11 +88,22 @@ export async function captureSelectorSnapshot(
}
const sessionName = options.session ?? 'default';
const session = await runtime.sessions.get(sessionName);
const view = captureOptions.presentation;
const result = await captureSnapshot(toBackendContext(runtime, options), {
interactiveOnly: captureOptions.interactiveOnly ?? false,
depth: options.depth,
scope: captureOptions.scope ?? options.scope,
raw: options.raw,
...(view
? {
interactiveOnly: view.interactiveOnly === true,
depth: view.depth,
scope: view.scope,
raw: view.raw,
...(view.customActions === true ? { customActions: true } : {}),
}
: {
interactiveOnly: captureOptions.interactiveOnly ?? false,
depth: options.depth,
scope: captureOptions.scope ?? options.scope,
raw: options.raw,
}),
includeRects: captureOptions.includeRects,
...(captureOptions.preferredBackend
? { preferredBackend: captureOptions.preferredBackend }
Expand Down
Loading
Loading