From 924a0bd3570cb5ff45751d4b3b1a522415794064 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Sat, 14 Nov 2026 17:44:17 +0100 Subject: [PATCH] chore(gates): keep the managed Maestro migration atomic Land the consumer migration, isolated test installation and resolver setup, artifact retirement, dependency removal and matching enforcement as one self-contained layer. Preserve the existing plugin spine rank instead of lowering its boundary. --- .changeset/optional-maestro-engine.md | 7 + .fallowrc.json | 2 + .github/workflows/android.yml | 3 + .github/workflows/apple.yml | 3 + .../workflows/conformance-differential.yml | 7 + .github/workflows/replays-nightly.yml | 7 + docs/adr/0015-direct-maestro-engine.md | 1 + docs/adr/0034-provider-plugin-registry.md | 2 + package.json | 5 +- packages/client/src/index.ts | 5 +- .../src/flag-definitions-workflow.ts | 2 +- .../command-surface/src/batch/projection.ts | 6 +- .../command-surface/src/cli-grammar/types.ts | 6 + packages/command-surface/src/index.ts | 15 +- .../command-surface/src/replay/index.test.ts | 2 +- packages/command-surface/src/replay/index.ts | 59 ++-- packages/contracts/src/command-flags.ts | 1 + packages/contracts/src/plugin-inventory.ts | 15 +- packages/contracts/src/replay-backend.ts | 15 + packages/daemon/package.json | 4 +- .../support/replay-script-source-fixture.ts | 9 +- .../composite-resource-admission-ime.test.ts | 6 +- .../composite-resource-admission.fixtures.ts | 8 + .../http/daemon-runtime-bounded-close.test.ts | 44 +++ .../daemon-runtime-metadata-ownership.test.ts | 16 +- .../daemon/src/server/http/daemon-runtime.ts | 1 + .../daemon/src/server/http/server-shutdown.ts | 34 ++- .../sessions/handlers/__tests__/lease.test.ts | 1 + .../daemon/src/sessions/handlers/lease.ts | 4 +- packages/host-kit/src/internal/diagnostics.ts | 18 +- packages/maestro/package.json | 9 +- .../__tests__/daemon-runtime-port-fixtures.ts | 22 +- .../daemon-runtime-port-support.test.ts | 18 ++ .../__tests__/run-script-execution.test.ts | 18 +- .../wait-for-animation-to-end.test.ts | 6 + .../daemon-runtime-port-support.ts | 7 +- .../src/daemon-port/daemon-runtime-port.ts | 24 +- .../src/daemon-port/daemon-runtime-tap.ts | 6 +- .../maestro-screenshot-comparison.ts | 20 +- .../src/daemon-port/run-script-execution.ts | 24 +- .../daemon-port/wait-for-animation-to-end.ts | 11 +- packages/maestro/src/index.test.ts | 8 +- packages/maestro/src/index.ts | 23 +- .../maestro/src/internal/facade-support.ts | 4 - .../maestro/src/internal/support-matrix.ts | 33 --- packages/maestro/src/plugin.test.ts | 6 + .../__tests__/replay-maestro-backend.test.ts | 6 + .../session-replay-maestro-failure.test.ts | 4 + .../replay-maestro-declaration.ts | 21 -- .../session-replay-maestro-runtime.ts | 4 + packages/maestro/test/package-smoke.mjs | 135 +++++++++ packages/maestro/test/packed-runtime.mjs | 238 +++++++++++++++ packages/maestro/tsconfig.json | 3 + packages/plugins/src/registry.ts | 187 ++++++++++-- packages/plugins/src/replay-backend.test.ts | 178 +++++++++++ packages/plugins/src/replay-backend.ts | 2 + packages/plugins/src/store.ts | 1 + packages/plugins/src/validate.ts | 46 +++ packages/replay-port/package.json | 4 +- .../__tests__/script-source-bundle.test.ts | 17 +- .../__tests__/replay-backend-registry.test.ts | 81 +++-- .../replay-backend-declarations.ts | 52 +++- .../src/daemon-port/replay-backend-host.ts | 11 +- .../daemon-port/replay-backend-registry.ts | 153 ++++++---- .../src/replay-script-source.fixtures.ts | 1 + .../replay-port/src/script-source-bundle.ts | 25 +- packages/replay-port/tsconfig.json | 52 +++- pnpm-lock.yaml | 58 ++-- scripts/build-entries.ts | 2 - scripts/fuzz/execute.ts | 22 +- scripts/fuzz/harness.test.ts | 31 +- scripts/fuzz/target-types.ts | 2 + scripts/fuzz/targets.ts | 15 +- .../validation-arbitraries-maestro.test.ts | 9 +- scripts/fuzz/worker.ts | 15 +- scripts/install-workspace-maestro.ts | 47 +++ scripts/layering/architecture-ownership.ts | 7 +- scripts/layering/package-boundaries.test.ts | 7 +- scripts/layering/project-references.test.ts | 7 +- scripts/vitest-tmpdir-global-setup.ts | 7 + src/__tests__/cli-maestro-suite.test.ts | 3 +- .../cli-replay-script-sources.test.ts | 9 +- src/__tests__/cli-test-env.ts | 7 + src/__tests__/replay-plugin-setup.ts | 34 +++ .../replay-script-source.fixtures.ts | 9 +- .../__tests__/replay-maestro-export.test.ts | 10 +- src/cli/commands/daemon.ts | 2 +- src/cli/commands/plugins.ts | 2 +- src/cli/commands/replay.ts | 16 +- .../args-maestro-support-matrix.test.ts | 2 +- src/daemon-composition.test.ts | 3 + src/daemon-composition.ts | 10 +- src/host-replay.test.ts | 277 ++++++++++++++++++ src/host.ts | 9 +- .../__tests__/command-tools-parity.test.ts | 1 + src/provider-device-runtimes.ts | 2 + src/sdk/host-types.ts | 3 +- .../installed-package-metro.test.ts | 40 +-- .../integration/provider-scenarios/harness.ts | 1 + test/repo-guards/npm-package-scripts.test.ts | 2 +- vitest.config.ts | 1 + vitest.mutation.config.ts | 1 + website/docs/docs/plugins.md | 14 +- website/docs/docs/run-maestro-flows.md | 12 + 104 files changed, 2016 insertions(+), 451 deletions(-) create mode 100644 .changeset/optional-maestro-engine.md delete mode 100644 packages/maestro/src/internal/facade-support.ts delete mode 100644 packages/maestro/src/internal/support-matrix.ts delete mode 100644 packages/maestro/src/replay-backend/replay-maestro-declaration.ts create mode 100644 packages/maestro/test/package-smoke.mjs create mode 100644 packages/maestro/test/packed-runtime.mjs create mode 100644 packages/plugins/src/replay-backend.test.ts create mode 100644 scripts/install-workspace-maestro.ts create mode 100644 src/__tests__/replay-plugin-setup.ts create mode 100644 src/host-replay.test.ts diff --git a/.changeset/optional-maestro-engine.md b/.changeset/optional-maestro-engine.md new file mode 100644 index 0000000000..18d4c6ff90 --- /dev/null +++ b/.changeset/optional-maestro-engine.md @@ -0,0 +1,7 @@ +--- +'agent-device': minor +--- + +Release prerequisite: the plugin remains private and unavailable from npm until a usable package is published through the configured release workflow. + +Maestro YAML replay, suites, and export now require the optional `@agent-device/maestro` plugin. After publication, install it on each CLI and store-backed daemon host that uses these flows. Finish sessions and stop the matching daemon. Embedded hosts supply the factory through `startAgentDevice({ plugins })` and close/recreate the returned runtime; they do not use the CLI store or daemon stop command. Retry your original command to start a daemon with the updated plugin set; flow syntax and arguments stay the same. Native `.ad` replay remains available without this plugin. diff --git a/.fallowrc.json b/.fallowrc.json index 8a18a8bda2..715d1e01f2 100644 --- a/.fallowrc.json +++ b/.fallowrc.json @@ -10,6 +10,8 @@ "packages/maestro/test/packed-clickability.mjs", "packages/maestro/test/packed-screen-origin.mjs", "packages/maestro/test/package-test-support.mjs", + "packages/maestro/test/package-smoke.mjs", + "packages/maestro/test/packed-runtime.mjs", "packages/maestro/tsdown.config.ts", "packages/maestro/src/plugin.ts", "packages/provider-testmu/tsdown.config.ts", diff --git a/.github/workflows/android.yml b/.github/workflows/android.yml index 01e594915e..8ca5192eb0 100644 --- a/.github/workflows/android.yml +++ b/.github/workflows/android.yml @@ -30,6 +30,8 @@ jobs: if: ${{ !github.event.pull_request.draft }} runs-on: blacksmith-4vcpu-ubuntu-2404 timeout-minutes: 80 + env: + AGENT_DEVICE_HOME: ${{ github.workspace }}/.tmp/agent-device-android-smoke-home steps: - name: Checkout uses: useblacksmith/checkout@2b9fe87d796feea153b455ec2f3125b471a7b0e7 # v1.8.2 @@ -91,6 +93,7 @@ jobs: ANDROID_SERIAL="$(adb devices | awk 'NR > 1 && $2 == "device" { print $1; exit }')" test -n "$ANDROID_SERIAL" pnpm gate build + node --experimental-strip-types scripts/install-workspace-maestro.ts pnpm clean:daemon AGENT_DEVICE_ANDROID_E2E=1 AGENT_DEVICE_ANDROID_E2E_TIER=smoke AGENT_DEVICE_ANDROID_SERIAL="$ANDROID_SERIAL" AGENT_DEVICE_FIXTURE_APP_PATH="${{ steps.fixture-app.outputs.apk-path }}" AGENT_DEVICE_FIXTURE_APP_ID="${{ steps.fixture-app.outputs.app-id }}" node --experimental-strip-types scripts/node-test-tmpdir.ts --test test/integration/smoke-android-emulator.test.ts node --experimental-strip-types src/bin.ts test test/integration/replays/android/emulator/01-settings.ad --retries 2 --report-junit test/artifacts/replays-android-smoke.junit.xml diff --git a/.github/workflows/apple.yml b/.github/workflows/apple.yml index df10a2ef56..89da9b8ed6 100644 --- a/.github/workflows/apple.yml +++ b/.github/workflows/apple.yml @@ -36,6 +36,7 @@ jobs: runs-on: blacksmith-6vcpu-macos-26 timeout-minutes: 90 env: + AGENT_DEVICE_HOME: ${{ github.workspace }}/.tmp/agent-device-apple-smoke-home BASE_SHA: ${{ github.event.pull_request.base.sha }} IOS_STATE_DIR: ${{ github.workspace }}/.tmp/ios-agent-device-state MACOS_STATE_DIR: ${{ github.workspace }}/.tmp/macos-agent-device-state @@ -83,6 +84,8 @@ jobs: uses: ./.github/actions/run-gate with: gate: build + - if: steps.build.outcome == 'success' + run: node --experimental-strip-types scripts/install-workspace-maestro.ts - name: iOS smoke tests id: ios uses: ./.github/actions/smoke-ios-run diff --git a/.github/workflows/conformance-differential.yml b/.github/workflows/conformance-differential.yml index fd04fe7c74..a18b7cb5b5 100644 --- a/.github/workflows/conformance-differential.yml +++ b/.github/workflows/conformance-differential.yml @@ -64,6 +64,13 @@ jobs: - name: Setup toolchain uses: ./.github/actions/setup-node-pnpm + - name: Build and install the workspace Maestro plugin + run: | + pnpm --filter @agent-device/maestro build + export AGENT_DEVICE_HOME="$RUNNER_TEMP/agent-device-maestro-home" + node --experimental-strip-types scripts/install-workspace-maestro.ts + echo "AGENT_DEVICE_HOME=$AGENT_DEVICE_HOME" >> "$GITHUB_ENV" + - name: Restore and build iOS XCTest runner uses: ./.github/actions/setup-apple-runner-build with: diff --git a/.github/workflows/replays-nightly.yml b/.github/workflows/replays-nightly.yml index 0ee0e41b44..67812c3118 100644 --- a/.github/workflows/replays-nightly.yml +++ b/.github/workflows/replays-nightly.yml @@ -53,6 +53,13 @@ jobs: - name: Setup toolchain uses: ./.github/actions/setup-node-pnpm + - name: Build and install the workspace Maestro plugin + run: | + pnpm --filter @agent-device/maestro build + export AGENT_DEVICE_HOME="$RUNNER_TEMP/agent-device-fuzz-home" + node --experimental-strip-types scripts/install-workspace-maestro.ts + echo "AGENT_DEVICE_HOME=$AGENT_DEVICE_HOME" >> "$GITHUB_ENV" + # A lane whose classifier or watchdog regressed would pass forever. This runs the # broken-on-purpose targets first and fails unless each violation is caught. - name: Self-check the harness diff --git a/docs/adr/0015-direct-maestro-engine.md b/docs/adr/0015-direct-maestro-engine.md index 1fd2ad4c02..55ebd036ab 100644 --- a/docs/adr/0015-direct-maestro-engine.md +++ b/docs/adr/0015-direct-maestro-engine.md @@ -6,6 +6,7 @@ Accepted ## Rules at a glance +- Load the optional `@agent-device/maestro` engine through the managed version 2 replay-backend plugin contract. Keep help declarations in core. Keep session authority, request cancellation, sensitive-value registration, and device dispatch with the host. - Parse supported Maestro YAML into a source-preserving typed program. Execute it directly through a narrow compatibility runtime port. - Do not lower the program through `SessionAction`, private command names, positional JSON, or a recursive replay dispatcher. - One explicit execution context owns variables and the current observation generation. diff --git a/docs/adr/0034-provider-plugin-registry.md b/docs/adr/0034-provider-plugin-registry.md index 5b97037db5..58e4a2228b 100644 --- a/docs/adr/0034-provider-plugin-registry.md +++ b/docs/adr/0034-provider-plugin-registry.md @@ -7,6 +7,7 @@ Status: accepted. Umbrella #3747; contract and registry #3755. - One contract. A provider plugin is a factory that returns `{ apiVersion: 2, provider, credentials, connection?, setup(host) }`. `setup` returns `{ runtime, platformModule, dispose? }` or `{ webDriver, dispose? }`. A `{ webDriver }` result stays a data descriptor the host interprets. +- A replay plugin uses the same version 2 factory with `kind: 'replay-backend'`, `backend`, and `setup(host)`. Setup returns `{ replayBackend: { create(host) }, dispose? }`. Provider and backend ids have separate claim sets in this registry. A replay plugin registers no provider runtime. - One registry. Composition selects built-in, store, or embedder sources. Built-in providers use the same shape as an installed plugin, and one function validates what every factory returns. - Plugins import only types from agent-device. The type-only contract is `agent-device/plugins`; @@ -22,6 +23,7 @@ Status: accepted. Umbrella #3747; contract and registry #3755. - The registry receives candidates. Composition chooses the sources. - `daemon.json` has one reader. Its states are `absent`, `unreadable`, and `invalid`. A missing or stale record reads as a stopped daemon; an unreadable or invalid one never does. +- Keep replay setup engine-free. Load the engine only when its backend is selected. The runtime's registry memoizes creation and owns disposal after requests drain. Client-side source collection and export close their scoped registry. - The plugin set is fixed for a daemon's lifetime. A plugin change takes effect on the next daemon. - A store package is discovered through its package targets: the `./agent-device` export, else the root export, which is `main` for a package that declares no `exports`. Nothing is read from diff --git a/package.json b/package.json index 5385e36e07..30a0ddc100 100644 --- a/package.json +++ b/package.json @@ -86,7 +86,7 @@ "agent-device": "bin/agent-device.mjs" }, "scripts": { - "build": "tsdown", + "build": "pnpm --filter @agent-device/maestro build && tsdown", "clean:daemon": "node --experimental-strip-types scripts/clean-daemon.ts", "daemon:state-dir": "node --experimental-strip-types scripts/print-daemon-state-dir.ts", "explain:command": "node --experimental-strip-types scripts/explain-command.ts", @@ -165,7 +165,7 @@ "depgraph:test": "node --experimental-strip-types scripts/node-test-tmpdir.ts --experimental-strip-types --test scripts/depgraph/model.test.ts scripts/depgraph/entries.test.ts scripts/depgraph/import-graph.test.ts scripts/depgraph/affected.test.ts scripts/depgraph/structure-summary.test.ts tools/depgraph-view/*.test.mjs", "check:production-exports": "fallow dead-code --config fallow-production-exports.json --production --unused-exports --fail-on-issues", "check:bundle-owner-files": "node --experimental-strip-types scripts/check-bundle-owner-files.ts", - "check:package": "node --experimental-strip-types scripts/check-package.ts && node --experimental-strip-types scripts/node-test-tmpdir.ts --test packages/maestro/test/package-artifact.test.mjs", + "check:package": "node --experimental-strip-types scripts/check-package.ts && node --experimental-strip-types scripts/node-test-tmpdir.ts --test packages/maestro/test/package-artifact.test.mjs packages/maestro/test/package-smoke.mjs", "check:command-docs": "vitest run --project unit-core test/repo-guards/command-doc-coverage.test.ts", "check:adr-citations": "vitest run --project unit-core test/repo-guards/adr-citations.test.ts", "check:capabilities-matrix": "vitest run --project unit-core test/repo-guards/capability-matrix.test.ts", @@ -317,7 +317,6 @@ "@agent-device/device-selection": "workspace:*", "@agent-device/host-kit": "workspace:*", "@agent-device/kernel": "workspace:*", - "@agent-device/maestro": "workspace:*", "@agent-device/platform-android": "workspace:*", "@agent-device/platform-apple": "workspace:*", "@agent-device/platform-harmonyos": "workspace:*", diff --git a/packages/client/src/index.ts b/packages/client/src/index.ts index c03bb9c1ce..6a9ea5672c 100644 --- a/packages/client/src/index.ts +++ b/packages/client/src/index.ts @@ -1,3 +1,4 @@ +import type { ReplayBackendResolver } from '@agent-device/contracts/replay-backend'; import type { CommandFlags, InternalRequestOptions } from '@agent-device/contracts/command'; import { readSerializedSnapshotCaptureAnnotations, @@ -81,6 +82,8 @@ export function createAgentDeviceClient( config: AgentDeviceClientConfig = {}, deps: { transport?: AgentDeviceDaemonTransport; + /** The embedded runtime registry, shared by source preparation and execution. */ + resolveReplayBackend?: ReplayBackendResolver; host?: HostEnvironment; /** The operator-configured device selector enforced on every request. */ devicePin?: DevicePin; @@ -182,7 +185,7 @@ export function createAgentDeviceClient( command: DaemonCommandName, options: InternalRequestOptions = {}, ): Promise => { - const request = await prepareDaemonCommandRequest(command, options, host); + const request = await prepareDaemonCommandRequest(command, options, host, deps); return (await execute( request.command, request.positionals, diff --git a/packages/command-registry/src/flag-definitions-workflow.ts b/packages/command-registry/src/flag-definitions-workflow.ts index a3ce7f70cd..220a3936d4 100644 --- a/packages/command-registry/src/flag-definitions-workflow.ts +++ b/packages/command-registry/src/flag-definitions-workflow.ts @@ -53,7 +53,7 @@ export const WORKFLOW_FLAG_DEFINITIONS: readonly FlagDefinition[] = [ usageLabel: '--maestro', usageDescription: 'Replay: treat input as a supported Maestro YAML subset; unsupported syntax fails loudly. ' + - 'See agent-device help maestro for commands and boundaries', + 'Requires the @agent-device/maestro plugin, install separately. See agent-device help maestro for commands and boundaries', projectConfig: true, recorded: false, }, diff --git a/packages/command-surface/src/batch/projection.ts b/packages/command-surface/src/batch/projection.ts index c258401a27..245f3dd6bc 100644 --- a/packages/command-surface/src/batch/projection.ts +++ b/packages/command-surface/src/batch/projection.ts @@ -14,6 +14,7 @@ import { AppError } from '@agent-device/kernel/errors'; import { request } from '../cli-grammar/common.ts'; import type { AsyncDaemonWriter, + DaemonWriterDependencies, CommandInput, DaemonCommandRequest, } from '../cli-grammar/types.ts'; @@ -26,6 +27,7 @@ type PrepareDaemonCommandRequest = ( input: CommandInput, stepNumber: number, host: HostEnvironment, + deps?: DaemonWriterDependencies, ) => Promise; type PrepareBatchStep = ( @@ -37,11 +39,11 @@ type PrepareBatchStep = ( export function createBatchDaemonWriter( prepareDaemonCommandRequest: PrepareDaemonCommandRequest, ): AsyncDaemonWriter { - return async (input, host) => + return async (input, host, deps) => request(PUBLIC_COMMANDS.batch, [], { ...input, batchSteps: await readBatchDaemonSteps(input.steps, (command, input, stepNumber) => - prepareDaemonCommandRequest(command, input, stepNumber, host), + prepareDaemonCommandRequest(command, input, stepNumber, host, deps), ), batchOnError: input.onError, batchMaxSteps: input.maxSteps, diff --git a/packages/command-surface/src/cli-grammar/types.ts b/packages/command-surface/src/cli-grammar/types.ts index a83175f7f7..48bb9655ad 100644 --- a/packages/command-surface/src/cli-grammar/types.ts +++ b/packages/command-surface/src/cli-grammar/types.ts @@ -1,3 +1,4 @@ +import type { ReplayBackendResolver } from '@agent-device/contracts/replay-backend'; import type { HostEnvironment } from '@agent-device/kernel/source-value'; import type { CliFlags, @@ -91,6 +92,10 @@ export type CliReader = (positionals: string[], flags: CliFlags) => CliInput; /** Builds the daemon request for one command. Most writers are pure projections of their input. */ export type DaemonWriter = (input: CommandInput) => DaemonCommandRequest; +export type DaemonWriterDependencies = Readonly<{ + resolveReplayBackend?: ReplayBackendResolver; +}>; + /** * A writer that has caller-side work to do before the request exists: reading the replay script * files the request carries (#1802), which for a Maestro flow also loads the engine that walks its @@ -100,6 +105,7 @@ export type DaemonWriter = (input: CommandInput) => DaemonCommandRequest; export type AsyncDaemonWriter = ( input: CommandInput, host: HostEnvironment, + deps?: DaemonWriterDependencies, ) => Promise; export type AnyDaemonWriter = DaemonWriter | AsyncDaemonWriter; diff --git a/packages/command-surface/src/index.ts b/packages/command-surface/src/index.ts index f0a824cfc5..0a2c8d6bc8 100644 --- a/packages/command-surface/src/index.ts +++ b/packages/command-surface/src/index.ts @@ -1,6 +1,11 @@ import type { HostEnvironment } from '@agent-device/kernel/source-value'; import { createBatchDaemonWriter } from './batch/projection.ts'; -import type { AnyDaemonWriter, CommandInput, DaemonCommandRequest } from './cli-grammar/types.ts'; +import type { + AnyDaemonWriter, + CommandInput, + DaemonCommandRequest, + DaemonWriterDependencies, +} from './cli-grammar/types.ts'; import { findCommandMetadata } from './command-metadata.ts'; import { refuseRetiredCommonInput } from './common-input-fields.ts'; import { readMetadataCommandFlags } from './command-flags.ts'; @@ -54,6 +59,7 @@ async function prepareBatchDaemonCommandRequest( input: CommandInput, stepNumber: number, host: HostEnvironment, + deps?: DaemonWriterDependencies, ): Promise { const writer = (daemonWriters as Readonly>)[command]; if (!writer) { @@ -69,6 +75,7 @@ async function prepareBatchDaemonCommandRequest( metadata, metadata.readInput(input) as CommandInput, host, + deps, ); } catch (error) { const message = error instanceof Error ? error.message : String(error); @@ -86,6 +93,7 @@ export async function prepareDaemonCommandRequest( command: DaemonCommandName, input: CommandInput, host: HostEnvironment, + deps?: DaemonWriterDependencies, ): Promise { const writer = daemonWriters[command]; if (!writer) { @@ -93,7 +101,7 @@ export async function prepareDaemonCommandRequest( } refuseRetiredCommonInput(input as Record); const metadata = findCommandMetadata(command); - return await prepareRequestWithMetadataFlags(writer, metadata, input, host); + return await prepareRequestWithMetadataFlags(writer, metadata, input, host, deps); } async function prepareRequestWithMetadataFlags( @@ -101,8 +109,9 @@ async function prepareRequestWithMetadataFlags( metadata: ReturnType, input: CommandInput, host: HostEnvironment, + deps?: DaemonWriterDependencies, ): Promise { - const request = await writer(input, host); + const request = await writer(input, host, deps); return { ...request, ...(metadata ? { metadataFlags: readMetadataCommandFlags(metadata, request.options) } : {}), diff --git a/packages/command-surface/src/replay/index.test.ts b/packages/command-surface/src/replay/index.test.ts index f21d7fb236..471dfdf76a 100644 --- a/packages/command-surface/src/replay/index.test.ts +++ b/packages/command-surface/src/replay/index.test.ts @@ -161,7 +161,7 @@ describe('replay command interface', () => { test('validates -e keys with the grammar of the resolved replay format', async () => { fs.writeFileSync(path.join(SCRIPT_ROOT, 'flow.yaml'), 'appId: demo.app\n---\n- back\n'); - const host = { env: {}, cwd: SCRIPT_ROOT }; + const host = { env: { AGENT_DEVICE_HOME: process.env.AGENT_DEVICE_HOME }, cwd: SCRIPT_ROOT }; await expect( replayDaemonWriter({ path: './checkout.ad', backend: 'maestro', env: ['user=x'] }, host), ).rejects.toMatchObject({ code: 'INVALID_ARGS' }); diff --git a/packages/command-surface/src/replay/index.ts b/packages/command-surface/src/replay/index.ts index d8204a9565..bb37aaae3b 100644 --- a/packages/command-surface/src/replay/index.ts +++ b/packages/command-surface/src/replay/index.ts @@ -245,7 +245,7 @@ export const testCliReader: CliReader = (positionals, flags) => ({ testIme: flags.testIme, }); -export const replayDaemonWriter: AsyncDaemonWriter = async (input, host) => { +export const replayDaemonWriter: AsyncDaemonWriter = async (input, host, deps) => { const inputPath = requiredDaemonString(input.path, 'replay requires path'); const replayBackend = readReplayBackend(input); const replayShellEnv = collectReplayClientShellEnv(host.env); @@ -259,6 +259,8 @@ export const replayDaemonWriter: AsyncDaemonWriter = async (input, host) => { // client-collects-local-input move `replayShellEnv` already makes — and // the daemon executes only what arrives in this bundle. replayScriptSource: await loadReplayScriptSourceBundle({ + hostEnv: host.env, + resolveReplayBackend: deps?.resolveReplayBackend, inputPath, cwd: readReplayClientCwd(input, host.cwd), replayBackend, @@ -275,35 +277,52 @@ export const replayDaemonWriter: AsyncDaemonWriter = async (input, host) => { }); }; -export const testDaemonWriter: AsyncDaemonWriter = async (input, host) => { +export const testDaemonWriter: AsyncDaemonWriter = async (input, host, deps) => { const inputs = input.paths ?? []; const replayBackend = readReplayBackend(input); const cwd = readReplayClientCwd(input, host.cwd); const replayShellEnv = collectReplayClientShellEnv(host.env); + // #1802: `test` inputs belong to the caller, including expansion and source collection. + const sourcePaths = discoverReplaySourcePaths({ inputs, cwd, replayBackend }); + const collectSources = async (resolveReplayBackend = deps?.resolveReplayBackend) => { + const sources = []; + for (const inputPath of sourcePaths) { + sources.push( + await loadReplayScriptSourceBundle({ + hostEnv: host.env, + resolveReplayBackend, + inputPath, + cwd, + replayBackend, + env: readReplayScriptSourceEnv( + input, + replayShellEnv, + resolveReplayFormat(inputPath, replayBackend), + ), + }), + ); + } + return sources; + }; + const replayScriptSources = sourcePaths.some( + (inputPath) => resolveReplayFormat(inputPath, replayBackend) === 'maestro', + ) + ? await ( + await import('@agent-device/replay-port/replay-backend-registry') + ).withReplayBackend( + 'maestro', + host.env, + (backend) => collectSources(async () => backend), + deps?.resolveReplayBackend, + ) + : await collectSources(); return request(TEST_COMMAND_NAME, inputs, { ...stripReplayTestPresentationInput(input), replayUpdate: input.update, replayBackend, replayEnv: input.env, replayShellEnv, - // #1802: `test` inputs are paths, directories, and globs on the CALLER's - // filesystem. Expanding them here keeps discovery order identical for a - // local and a remote daemon and ships each discovered source's text. - replayScriptSources: await Promise.all( - discoverReplaySourcePaths({ inputs, cwd, replayBackend }).map( - async (inputPath) => - await loadReplayScriptSourceBundle({ - inputPath, - cwd, - replayBackend, - env: readReplayScriptSourceEnv( - input, - replayShellEnv, - resolveReplayFormat(inputPath, replayBackend), - ), - }), - ), - ), + replayScriptSources, }); }; diff --git a/packages/contracts/src/command-flags.ts b/packages/contracts/src/command-flags.ts index 872a9fd63a..5b6b3fd454 100644 --- a/packages/contracts/src/command-flags.ts +++ b/packages/contracts/src/command-flags.ts @@ -18,6 +18,7 @@ import { // wire cannot enforce a shape. This is the typed view of the same field, one rank above the wire // and below every consumer. +/** Shared replay-compatibility hints; these remain core runtime behavior without the flow engine. */ export type MaestroRuntimeFlags = { allowNonHittableCoordinateFallback?: boolean; expectedTapPoint?: Point; diff --git a/packages/contracts/src/plugin-inventory.ts b/packages/contracts/src/plugin-inventory.ts index 77e68c6fbd..4a697c4515 100644 --- a/packages/contracts/src/plugin-inventory.ts +++ b/packages/contracts/src/plugin-inventory.ts @@ -16,6 +16,7 @@ export type PluginInventoryCredentials = Readonly<{ type PluginInventoryIdentity = Readonly<{ provider?: string; + backend?: string; source: PluginSource; package?: string; version?: string; @@ -36,12 +37,13 @@ export type PluginCleanupStep = * What closing the registry did for one registered plugin. A plugin without `dispose` has nothing to * dispose, which reads as `ok`. */ -export type PluginCleanupOutcome = Readonly<{ - provider: string; - source: PluginSource; - shutdown: PluginCleanupStep; - dispose: PluginCleanupStep; -}>; +export type PluginCleanupOutcome = Readonly< + ({ provider: string; backend?: never } | { backend: string; provider?: never }) & { + source: PluginSource; + shutdown: PluginCleanupStep; + dispose: PluginCleanupStep; + } +>; export const NO_PLUGIN_CREDENTIALS: PluginInventoryCredentials = Object.freeze({ variables: Object.freeze([]), @@ -79,6 +81,7 @@ function readEntry(value: unknown): PluginInventoryEntry | undefined { if (!credentials) return undefined; const identity = { ...(typeof value.provider === 'string' ? { provider: value.provider } : {}), + ...(typeof value.backend === 'string' ? { backend: value.backend } : {}), source: value.source, ...(typeof value.package === 'string' ? { package: value.package } : {}), ...(typeof value.version === 'string' ? { version: value.version } : {}), diff --git a/packages/contracts/src/replay-backend.ts b/packages/contracts/src/replay-backend.ts index 43c2bb0e54..6f11641fa6 100644 --- a/packages/contracts/src/replay-backend.ts +++ b/packages/contracts/src/replay-backend.ts @@ -1,3 +1,4 @@ +import type { HostCommandRequest, HostCommandResult } from './platform-runtime-host.ts'; import type { DiagnosticEventInput } from './host-diagnostics.ts'; import type { SnapshotClickabilityEvidence } from './snapshot-private-evidence.ts'; import type { DaemonError } from '@agent-device/kernel/errors'; @@ -237,6 +238,13 @@ export type ReplayBackendHost = Readonly<{ readSnapshotClickabilityEvidence(owner: object): SnapshotClickabilityEvidence | undefined; /** Refuses capture-derived input when the host has no established screen origin. */ assertSnapshotScreenOrigin(owner: object, label: string): Promise; + /** Executes a backend child with the host's subprocess lifecycle and diagnostic scope. */ + runCommand( + request: Omit & + Readonly<{ host: HostEnvironment; stdin?: string; detached?: boolean }>, + signal?: AbortSignal, + ): Promise; + registerSensitiveValue(value: string): void; /** Appends one record to the run's trace file, if any; failures are diagnostics, never thrown. */ appendTraceEvent(tracePath: string | undefined, event: Record): void; /** The `AD_*` variables every replay run is seeded with. */ @@ -272,9 +280,16 @@ export type ReplayBackendHost = Readonly<{ */ export type ReplayBackendDeclaration = Readonly<{ id: TId; + package: string; help: Readonly<{ summary: string; body: string }>; }>; export type ReplayBackendPlugin = Readonly<{ create(host: ReplayBackendHost): ReplayBackend | Promise>; }>; + +export type ReplayBackendResolver = ( + id: string, + host: ReplayBackendHost, + packageName?: string, +) => Promise; diff --git a/packages/daemon/package.json b/packages/daemon/package.json index 2558eb9069..09a1aa211a 100644 --- a/packages/daemon/package.json +++ b/packages/daemon/package.json @@ -14,7 +14,6 @@ "@agent-device/device-selection": "workspace:*", "@agent-device/host-kit": "workspace:*", "@agent-device/kernel": "workspace:*", - "@agent-device/maestro": "workspace:*", "@agent-device/platform-android": "workspace:*", "@agent-device/platform-apple": "workspace:*", "@agent-device/platform-web": "workspace:*", @@ -31,7 +30,8 @@ "fast-check": "^4.9.0", "oxc-parser": "^0.146.0", "vitest": "5.0.4", - "yaml": "^2.9.0" + "yaml": "^2.9.0", + "@agent-device/maestro": "workspace:*" }, "exports": { ".": { diff --git a/packages/daemon/src/__tests__/support/replay-script-source-fixture.ts b/packages/daemon/src/__tests__/support/replay-script-source-fixture.ts index 8e23bf18ce..6126f50c76 100644 --- a/packages/daemon/src/__tests__/support/replay-script-source-fixture.ts +++ b/packages/daemon/src/__tests__/support/replay-script-source-fixture.ts @@ -16,6 +16,7 @@ export async function maestroScriptSourceBundleFor( filePath: string, ): Promise { return await loadReplayScriptSourceBundle({ + hostEnv: process.env, inputPath: filePath, cwd: process.cwd(), replayBackend: 'maestro', @@ -48,6 +49,7 @@ export async function withClientReplayScriptSources( if (req.flags?.replayScriptSource) return req; return withFlags(req, { replayScriptSource: await loadReplayScriptSourceBundle({ + hostEnv: process.env, inputPath: entryPath, cwd, replayBackend, @@ -59,7 +61,12 @@ export async function withClientReplayScriptSources( replayScriptSources: await Promise.all( discoverReplaySourcePaths({ inputs, cwd, replayBackend }).map( async (inputPath) => - await loadReplayScriptSourceBundle({ inputPath, cwd, replayBackend }), + await loadReplayScriptSourceBundle({ + hostEnv: process.env, + inputPath, + cwd, + replayBackend, + }), ), ), }); diff --git a/packages/daemon/src/core/composite-resource-admission-ime.test.ts b/packages/daemon/src/core/composite-resource-admission-ime.test.ts index 587629c4ba..f27ad683ac 100644 --- a/packages/daemon/src/core/composite-resource-admission-ime.test.ts +++ b/packages/daemon/src/core/composite-resource-admission-ime.test.ts @@ -1,3 +1,4 @@ +import { getReplayBackend } from '@agent-device/replay-port/replay-backend-registry'; import fs from 'node:fs'; import path from 'node:path'; import { afterEach, expect, test, vi } from 'vitest'; @@ -75,7 +76,10 @@ test.each(['batch', 'replay'] as const)( test('Maestro stop keeps the IME and app log; capture still bars caller close', async () => { const helper = androidHelperWorld(); - const world = await resourceAdmissionWorld('capture', { ime: helper.ime }); + const world = await resourceAdmissionWorld('capture', { + ime: helper.ime, + replayBackend: await getReplayBackend('maestro'), + }); try { assertRpcOk(await world.call('open', ['com.android.settings'], { testIme: true })); assertRpcOk(await world.call('logs', ['start'])); diff --git a/packages/daemon/src/core/composite-resource-admission.fixtures.ts b/packages/daemon/src/core/composite-resource-admission.fixtures.ts index e852c3ef04..363f585a11 100644 --- a/packages/daemon/src/core/composite-resource-admission.fixtures.ts +++ b/packages/daemon/src/core/composite-resource-admission.fixtures.ts @@ -1,3 +1,4 @@ +import type { ReplayBackend } from '@agent-device/contracts/replay-backend'; import path from 'node:path'; import { LeaseRegistry } from '../resources/lease-registry.ts'; import { applicationLifecycleRuntimeFixture } from '../__tests__/support/application-lifecycle-runtime-fixture.ts'; @@ -29,6 +30,7 @@ export async function resourceAdmissionWorld( audioMode: 'capture' | 'query' = 'capture', options: Readonly<{ device?: DeviceInfo; + replayBackend?: ReplayBackend; providerOwned?: boolean; withLifecycleProvider?: boolean; ime?: { @@ -188,6 +190,12 @@ export async function resourceAdmissionWorld( stdout: { write: () => {} }, stderr: { write: () => {} }, composeDeps: async (input) => { + const backend = options.replayBackend; + if (backend) { + const { bindReplayBackendResolver } = + await import('@agent-device/replay-port/replay-backend-registry'); + await bindReplayBackendResolver(async () => backend); + } const base = await daemonRuntimeDepsFixture()(input); return { ...base, diff --git a/packages/daemon/src/server/http/daemon-runtime-bounded-close.test.ts b/packages/daemon/src/server/http/daemon-runtime-bounded-close.test.ts index d87635bed5..0358ddc376 100644 --- a/packages/daemon/src/server/http/daemon-runtime-bounded-close.test.ts +++ b/packages/daemon/src/server/http/daemon-runtime-bounded-close.test.ts @@ -19,6 +19,7 @@ import { daemonPlatformServicesFixture } from '../../__tests__/support/platform- import { providerCredentialsFixture } from '../../__tests__/support/provider-credentials-fixture.ts'; import { startDaemonRuntime } from './daemon-runtime.ts'; import { + createPluginClose, describeUnfinishedShutdownWork, reportDaemonClose, type ShutdownReport, @@ -258,3 +259,46 @@ test('a clean close disposes the state composition declared in the entered handl expect(report.platformOwners).toBe('closed'); expect(disposeState).toHaveBeenCalledOnce(); }); + +test('only registered cleanup owners stay pending after a lazy backend failure', async () => { + vi.useFakeTimers(); + try { + const inventory: import('@agent-device/contracts/plugin-inventory').PluginInventoryEntry[] = [ + { + backend: 'maestro', + source: 'store', + state: 'active', + credentials: { variables: [], byBackend: {} }, + }, + { + backend: 'maestro', + source: 'embedder', + state: 'failed', + error: { code: 'INVALID_ARGS', message: 'duplicate', reason: 'duplicate_replay_backend' }, + credentials: { variables: [], byBackend: {} }, + }, + { + backend: 'rejected', + source: 'store', + state: 'failed', + error: { code: 'COMMAND_FAILED', message: 'setup failed', reason: 'plugin_setup_failed' }, + credentials: { variables: [], byBackend: {} }, + }, + ]; + const owner = createPluginClose({ + close: () => new Promise(() => {}), + timeoutMs: 50, + inventory, + }); + inventory[0] = { + ...inventory[0]!, + state: 'failed', + error: { code: 'COMMAND_FAILED', message: 'load failed', reason: 'plugin_load_failed' }, + }; + const close = owner.close(); + await vi.advanceTimersByTimeAsync(50); + expect(await close).toEqual({ disposed: [], failed: [], pending: ['replay-backend:maestro'] }); + } finally { + vi.useRealTimers(); + } +}); diff --git a/packages/daemon/src/server/http/daemon-runtime-metadata-ownership.test.ts b/packages/daemon/src/server/http/daemon-runtime-metadata-ownership.test.ts index 084dd43c04..99262f486a 100644 --- a/packages/daemon/src/server/http/daemon-runtime-metadata-ownership.test.ts +++ b/packages/daemon/src/server/http/daemon-runtime-metadata-ownership.test.ts @@ -288,6 +288,15 @@ test('a plugin whose cleanup failed at shutdown is recorded in daemon.log', asyn shutdown: { status: 'ok' }, dispose: { status: 'ok' }, }, + { + backend: 'maestro', + source: 'store', + shutdown: { status: 'ok' }, + dispose: { + status: 'failed', + error: { code: 'COMMAND_FAILED', message: 'backend disposal broke' }, + }, + }, ]; const runtime = await startRuntime(stateDir); try { @@ -295,12 +304,17 @@ test('a plugin whose cleanup failed at shutdown is recorded in daemon.log', asyn const events = logEvents(stateDir).filter( ({ phase }) => phase === 'provider_plugin_cleanup_failed', ); - expect(events).toHaveLength(1); + expect(events).toHaveLength(2); expect(events[0]?.data).toMatchObject({ provider: 'example', source: 'store', shutdown: { code: 'COMMAND_FAILED' }, }); + expect(events[1]?.data).toMatchObject({ + backend: 'maestro', + source: 'store', + dispose: { code: 'COMMAND_FAILED' }, + }); } finally { fs.rmSync(stateDir, { recursive: true, force: true }); } diff --git a/packages/daemon/src/server/http/daemon-runtime.ts b/packages/daemon/src/server/http/daemon-runtime.ts index 7054039ae3..55a27df8cf 100644 --- a/packages/daemon/src/server/http/daemon-runtime.ts +++ b/packages/daemon/src/server/http/daemon-runtime.ts @@ -825,6 +825,7 @@ async function startDaemonRuntimeOnHost( for (const failure of plugins.failed) { await emitDaemonDiagnostic(logPath, 'provider_plugin_cleanup_failed', { provider: failure.provider, + backend: failure.backend, source: failure.source, [failure.step]: failure.error, }); diff --git a/packages/daemon/src/server/http/server-shutdown.ts b/packages/daemon/src/server/http/server-shutdown.ts index 62afb2cc37..d269471eaa 100644 --- a/packages/daemon/src/server/http/server-shutdown.ts +++ b/packages/daemon/src/server/http/server-shutdown.ts @@ -69,7 +69,8 @@ export type ShutdownReport = { plugins: { disposed: string[]; failed: { - provider: string; + provider?: string; + backend?: string; source?: PluginCleanupOutcome['source']; step: 'shutdown' | 'dispose'; error: NormalizedError; @@ -106,7 +107,7 @@ export function describeUnfinishedShutdownWork(report: ShutdownReport): string[] } for (const failure of report.plugins.failed) { lines.push( - `Daemon shutdown: plugin ${failure.provider} failed to ${failure.step === 'shutdown' ? 'shut down' : 'dispose'}: ${failure.error.message}`, + `Daemon shutdown: plugin ${failure.provider ?? `replay backend ${failure.backend}`} failed to ${failure.step === 'shutdown' ? 'shut down' : 'dispose'}: ${failure.error.message}`, ); } const { pending, unknown } = report.providerReleases; @@ -151,7 +152,7 @@ export type PluginClose = Readonly<{ /** * The daemon's one plugin cleanup: the registry closes each plugin (runtime shutdown, then dispose) * and answers with a per-plugin outcome. Bounded, because one plugin that never finishes must not - * hold the whole close: past the budget every active plugin is reported pending while its cleanup + * hold the whole close: past the budget every registered plugin is reported pending while its cleanup * keeps running. */ export function createPluginClose(params: { @@ -160,10 +161,9 @@ export function createPluginClose(params: { timeoutMs?: number; }): PluginClose { let closing: Promise | undefined; - const activeProviders = (): string[] => - params.inventory.flatMap((entry) => - entry.state === 'active' && entry.provider !== undefined ? [entry.provider] : [], - ); + const cleanupOwners = params.inventory.flatMap((entry) => + entry.state === 'active' ? [{ provider: entry.provider, backend: entry.backend }] : [], + ); return { close: () => (closing ??= (async () => { @@ -172,14 +172,20 @@ export function createPluginClose(params: { params.timeoutMs ?? DAEMON_PLUGIN_SHUTDOWN_TIMEOUT_MS, ); if (outcome.kind === 'deadline') { - return { disposed: [], failed: [], pending: activeProviders() }; + return { + disposed: [], + failed: [], + pending: cleanupOwners.map( + ({ provider, backend }) => provider ?? `replay-backend:${backend}`, + ), + }; } if (outcome.kind === 'rejected') { const error = normalizeError(outcome.error); return { disposed: [], - failed: activeProviders().map((provider) => ({ - provider, + failed: cleanupOwners.map((identity) => ({ + ...identity, step: 'shutdown' as const, error, })), @@ -221,7 +227,8 @@ function reportCleanupOutcomes( outcomes: readonly PluginCleanupOutcome[], ): ShutdownReport['plugins'] { const report: ShutdownReport['plugins'] = { disposed: [], failed: [], pending: [] }; - for (const { provider, source, shutdown, dispose } of outcomes) { + for (const { provider, backend, source, shutdown, dispose } of outcomes) { + const identity = provider === undefined ? { backend } : { provider }; for (const [step, result] of [ ['shutdown', shutdown], ['dispose', dispose], @@ -229,13 +236,14 @@ function reportCleanupOutcomes( if (result.status !== 'failed') continue; const { code, message, hint } = result.error; report.failed.push({ - provider, + ...identity, source, step, error: { code, message, ...(hint === undefined ? {} : { hint }) }, }); } - if (shutdown.status === 'ok' && dispose.status === 'ok') report.disposed.push(provider); + if (shutdown.status === 'ok' && dispose.status === 'ok') + report.disposed.push(provider ?? `replay-backend:${backend}`); } return report; } diff --git a/packages/daemon/src/sessions/handlers/__tests__/lease.test.ts b/packages/daemon/src/sessions/handlers/__tests__/lease.test.ts index f080327d15..3ba3e45355 100644 --- a/packages/daemon/src/sessions/handlers/__tests__/lease.test.ts +++ b/packages/daemon/src/sessions/handlers/__tests__/lease.test.ts @@ -881,6 +881,7 @@ test('lease_allocate names the plugin that failed to register when it could have failed('testmu', '@agent-device/testmu'), failed(undefined, 'unloadable-plugin'), failed('other', 'other-plugin'), + { ...failed(undefined, '@agent-device/maestro'), backend: 'maestro' }, ], }), (error: unknown) => { diff --git a/packages/daemon/src/sessions/handlers/lease.ts b/packages/daemon/src/sessions/handlers/lease.ts index 7f4c7ba238..bd3186db08 100644 --- a/packages/daemon/src/sessions/handlers/lease.ts +++ b/packages/daemon/src/sessions/handlers/lease.ts @@ -437,7 +437,9 @@ function assertProviderRuntimeRegistered( if (provider === undefined || registeredProviders.includes(provider)) return; // A plugin that failed before its provider id was known could have been this provider. const failures = failedPlugins.flatMap((plugin) => - plugin.state === 'failed' && (plugin.provider === provider || plugin.provider === undefined) + plugin.state === 'failed' && + plugin.backend === undefined && + (plugin.provider === provider || plugin.provider === undefined) ? [ { ...(plugin.provider === undefined ? {} : { provider: plugin.provider }), diff --git a/packages/host-kit/src/internal/diagnostics.ts b/packages/host-kit/src/internal/diagnostics.ts index 7820de0fd0..0f8d291ac0 100644 --- a/packages/host-kit/src/internal/diagnostics.ts +++ b/packages/host-kit/src/internal/diagnostics.ts @@ -6,9 +6,14 @@ import path from 'node:path'; import { redactDiagnosticData } from '@agent-device/kernel/redaction'; import type { JsonValue } from '@agent-device/contracts/client'; import type { DiagnosticsRecordRef } from '@agent-device/kernel/errors'; - -/** The severity an event is recorded with. */ -export type DiagnosticLevel = 'info' | 'warn' | 'error' | 'debug'; +import type { + DiagnosticEventInput, + DiagnosticLevel, +} from '@agent-device/contracts/host-diagnostics'; +export type { + DiagnosticEventInput, + DiagnosticLevel, +} from '@agent-device/contracts/host-diagnostics'; type DiagnosticEvent = { ts: string; @@ -136,13 +141,6 @@ export type ResourceDiagnostic = Readonly<{ data: Readonly>; }>; -export type DiagnosticEventInput = { - level?: DiagnosticLevel; - phase: string; - durationMs?: number; - data?: Record; -}; - /** * Encodes diagnostic data for JSON-only ports, preserving bigints as decimal strings. * Performs no redaction; the receiving diagnostics port owns redaction. diff --git a/packages/maestro/package.json b/packages/maestro/package.json index 95c8ae8984..932f53933e 100644 --- a/packages/maestro/package.json +++ b/packages/maestro/package.json @@ -10,14 +10,6 @@ "types": "./src/index.ts", "default": "./src/index.ts" }, - "./replay-backend": { - "types": "./src/replay-backend/replay-maestro-backend.ts", - "default": "./src/replay-backend/replay-maestro-backend.ts" - }, - "./replay-backend-declaration": { - "types": "./src/replay-backend/replay-maestro-declaration.ts", - "default": "./src/replay-backend/replay-maestro-declaration.ts" - }, "./run-script-http": { "types": "./src/daemon-port/run-script-http.ts", "default": "./src/daemon-port/run-script-http.ts" @@ -30,6 +22,7 @@ "devDependencies": { "undici": "7.30.1", "yaml": "^2.9.0", + "@agent-device/test-kit": "workspace:*", "@agent-device/ad-script": "workspace:*", "@agent-device/capture-kit": "workspace:*", "@agent-device/contracts": "workspace:*", diff --git a/packages/maestro/src/daemon-port/__tests__/daemon-runtime-port-fixtures.ts b/packages/maestro/src/daemon-port/__tests__/daemon-runtime-port-fixtures.ts index c3c444e32d..9480f62aea 100644 --- a/packages/maestro/src/daemon-port/__tests__/daemon-runtime-port-fixtures.ts +++ b/packages/maestro/src/daemon-port/__tests__/daemon-runtime-port-fixtures.ts @@ -1,3 +1,9 @@ +import { runCmd, withCommandHost } from '@agent-device/host-kit/command'; +import { createTestHostEnvironment } from '@agent-device/test-kit'; +import { + emitDiagnostic, + registerDiagnosticSensitiveValue, +} from '@agent-device/host-kit/diagnostics'; import type { CommandFlags } from '@agent-device/contracts/command'; import { assertSnapshotScreenOrigin, @@ -27,11 +33,25 @@ export function makeRuntimeEnvelope( overrides: { flags?: CommandFlags; publicNetworkOnly?: boolean } = {}, ): Pick< CreateDaemonMaestroRuntimeOperationsOptions, - 'flags' | 'publicNetworkOnly' | 'readSnapshotClickabilityEvidence' | 'assertSnapshotScreenOrigin' + | 'readSnapshotClickabilityEvidence' + | 'assertSnapshotScreenOrigin' + | 'flags' + | 'publicNetworkOnly' + | 'host' + | 'registerSensitiveValue' + | 'emitDiagnostic' + | 'runCommand' > { return { readSnapshotClickabilityEvidence, assertSnapshotScreenOrigin, + host: createTestHostEnvironment(), + emitDiagnostic, + runCommand: ({ executable, args, host, ...options }, signal) => + withCommandHost({ host, requestEnv: undefined }, () => + runCmd(executable, args, { ...options, ...host, signal }), + ), + registerSensitiveValue: registerDiagnosticSensitiveValue, publicNetworkOnly: false, ...overrides, }; diff --git a/packages/maestro/src/daemon-port/__tests__/daemon-runtime-port-support.test.ts b/packages/maestro/src/daemon-port/__tests__/daemon-runtime-port-support.test.ts index ad1d7a5cfc..22fd587e3e 100644 --- a/packages/maestro/src/daemon-port/__tests__/daemon-runtime-port-support.test.ts +++ b/packages/maestro/src/daemon-port/__tests__/daemon-runtime-port-support.test.ts @@ -1,3 +1,7 @@ +import { + commandHostEnvironment, + withCommandExecutorOverride, +} from '@agent-device/host-kit/command'; import { expect, test, vi } from 'vitest'; import { invokeMaestroPublicOperation } from '../daemon-runtime-port-support.ts'; import { makeRuntimeEnvelope, makeDependencies } from './daemon-runtime-port-fixtures.ts'; @@ -123,3 +127,17 @@ test('preserves diagnostic metadata carried inside daemon error details', async details: { hint: 'Enable developer tools.', diagnosticId: 'diag-1' }, }); }); + +test('the runtime command adapter enters the supplied host scope', async () => { + const envelope = makeRuntimeEnvelope(); + const host = { ...envelope.host, env: { ...envelope.host.env, DEVELOPER_DIR: '/fixture/Xcode' } }; + await withCommandExecutorOverride( + async () => { + expect(commandHostEnvironment().env.DEVELOPER_DIR).toBe('/fixture/Xcode'); + return { stdout: '', stderr: '', exitCode: 0 }; + }, + async () => { + await envelope.runCommand({ executable: 'fixture', args: [], host }); + }, + ); +}); diff --git a/packages/maestro/src/daemon-port/__tests__/run-script-execution.test.ts b/packages/maestro/src/daemon-port/__tests__/run-script-execution.test.ts index fed45f972f..e138e87814 100644 --- a/packages/maestro/src/daemon-port/__tests__/run-script-execution.test.ts +++ b/packages/maestro/src/daemon-port/__tests__/run-script-execution.test.ts @@ -1,9 +1,11 @@ +import { makeRuntimeEnvelope } from './daemon-runtime-port-fixtures.ts'; +import { createTestHostEnvironment } from '@agent-device/test-kit'; import fs from 'node:fs'; import http from 'node:http'; import path from 'node:path'; import { expect, test, vi } from 'vitest'; import { AppError, isRequestCanceledError } from '@agent-device/kernel/errors'; -import { executeRunScriptFile } from '../run-script-execution.ts'; +import { executeRunScriptFile as runScriptFile } from '../run-script-execution.ts'; import { MAX_RUN_SCRIPT_HTTP_RESPONSE_BYTES } from '../run-script-http.ts'; import { mkdtempForTestSync } from '../../tmp-dir.fixtures.ts'; import { @@ -12,6 +14,10 @@ import { skipWhenLoopbackUnavailable, } from './loopback.fixtures.ts'; +function executeRunScriptFile(request: Parameters[0]) { + return runScriptFile(request, createTestHostEnvironment(), makeRuntimeEnvelope().runCommand); +} + async function withLoopbackServer( handler: http.RequestListener, run: (url: string) => Promise, @@ -237,3 +243,13 @@ test('executeRunScriptFile refuses remote scripts before reading or running host fs.rmSync(root, { recursive: true, force: true }); } }); + +test('script children resolve paths against the supplied embedded host working directory', async () => { + const { root } = writeScript('output.location = "embedded-host";'); + const output = await runScriptFile( + { scriptPath: 'setup.js', env: {}, publicNetworkOnly: false }, + createTestHostEnvironment({ cwd: root, env: {} }), + makeRuntimeEnvelope().runCommand, + ); + expect(output).toEqual({ 'output.location': 'embedded-host' }); +}); diff --git a/packages/maestro/src/daemon-port/__tests__/wait-for-animation-to-end.test.ts b/packages/maestro/src/daemon-port/__tests__/wait-for-animation-to-end.test.ts index 4741c9b43a..0db39d83b4 100644 --- a/packages/maestro/src/daemon-port/__tests__/wait-for-animation-to-end.test.ts +++ b/packages/maestro/src/daemon-port/__tests__/wait-for-animation-to-end.test.ts @@ -1,3 +1,4 @@ +import { emitDiagnostic } from '@agent-device/host-kit/diagnostics'; import { promises as fs } from 'node:fs'; import path from 'node:path'; import { expect, test } from 'vitest'; @@ -10,6 +11,7 @@ test('captures an explicit zero-timeout pair and accepts exactly 0.005% RGB diff let tempRoot: string | undefined; const stable = await waitForMaestroAnimationToEnd({ + emitDiagnostic, timeoutMs: 0, now: () => 0, capture: async (screenshotPath) => { @@ -29,6 +31,7 @@ test('retries changed pairs immediately until a matching pair is captured', asyn let tempRoot: string | undefined; const stable = await waitForMaestroAnimationToEnd({ + emitDiagnostic, timeoutMs: 2, now: () => now, capture: async (screenshotPath) => { @@ -58,6 +61,7 @@ test('treats dimension and capture failures as nonmatches before retrying', asyn let tempRoot: string | undefined; const stable = await waitForMaestroAnimationToEnd({ + emitDiagnostic, timeoutMs: 10, now: () => now, capture: async (screenshotPath) => { @@ -85,6 +89,7 @@ test('reports an expired deadline when no screenshot pair is stable', async () = let captureCount = 0; const stable = await waitForMaestroAnimationToEnd({ + emitDiagnostic, timeoutMs: 0, now: () => 0, capture: async (screenshotPath) => { @@ -106,6 +111,7 @@ test('propagates cancellation and cleans temporary screenshots', async () => { await expect( waitForMaestroAnimationToEnd({ + emitDiagnostic, timeoutMs: 10, now: () => 0, signal: controller.signal, diff --git a/packages/maestro/src/daemon-port/daemon-runtime-port-support.ts b/packages/maestro/src/daemon-port/daemon-runtime-port-support.ts index 38c823f9d3..a831484019 100644 --- a/packages/maestro/src/daemon-port/daemon-runtime-port-support.ts +++ b/packages/maestro/src/daemon-port/daemon-runtime-port-support.ts @@ -1,5 +1,6 @@ -import type { CommandFlags } from '@agent-device/contracts/command'; import type { ReplayBackendHost } from '@agent-device/contracts/replay-backend'; +import type { HostEnvironment } from '@agent-device/kernel/source-value'; +import type { CommandFlags } from '@agent-device/contracts/command'; import path from 'node:path'; import type { MaestroObservation, @@ -27,6 +28,10 @@ export type CreateDaemonMaestroRuntimeOperationsOptions = Pick< ReplayBackendHost, 'readSnapshotClickabilityEvidence' | 'assertSnapshotScreenOrigin' > & { + readonly host: HostEnvironment; + readonly runCommand: ReplayBackendHost['runCommand']; + readonly emitDiagnostic: ReplayBackendHost['emitDiagnostic']; + readonly registerSensitiveValue: (value: string) => void; readonly invoke: MaestroDaemonOperationInvoke; /** The runtime envelope every operation carries: platform, target, device, and recording flags. */ readonly flags?: CommandFlags; diff --git a/packages/maestro/src/daemon-port/daemon-runtime-port.ts b/packages/maestro/src/daemon-port/daemon-runtime-port.ts index 9e7fb91342..86d5bc94fe 100644 --- a/packages/maestro/src/daemon-port/daemon-runtime-port.ts +++ b/packages/maestro/src/daemon-port/daemon-runtime-port.ts @@ -8,7 +8,6 @@ import { type MaestroRuntimeOperations, type MaestroRuntimePort, } from '@agent-device/maestro'; -import { registerDiagnosticSensitiveValue } from '@agent-device/host-kit/diagnostics'; import { AppError } from '@agent-device/kernel/errors'; import { stripUndefined } from '@agent-device/kernel/record'; import { executeRunScriptFile } from './run-script-execution.ts'; @@ -137,7 +136,7 @@ function createDaemonMaestroRuntimeParts(options: CreateDaemonMaestroRuntimeOper context: MaestroRuntimeOperationContext, delayMs?: number, ): Promise => { - registerDiagnosticSensitiveValue(text); + options.registerSensitiveValue(text); await invokeMutation( { kind: 'typeText', text, ...(delayMs === undefined ? {} : { delayMs }) }, context, @@ -342,6 +341,7 @@ function createDaemonMaestroRuntimeParts(options: CreateDaemonMaestroRuntimeOper }, waitForAnimationToEnd: async (input, context) => { const visualStabilityReached = await waitForMaestroAnimationToEnd({ + emitDiagnostic: options.emitDiagnostic, timeoutMs: input.timeoutMs ?? MAESTRO_RUNTIME_ADAPTER_POLICY.animationWaitTimeoutMs, now: options.dependencies.now, signal: context.signal, @@ -356,15 +356,19 @@ function createDaemonMaestroRuntimeParts(options: CreateDaemonMaestroRuntimeOper artifactPaths: artifactPathsFromData(await invoke({ kind: 'screenshot', path: input.path })), }), runScript: async (input, context) => ({ - outputEnv: await executeRunScriptFile({ - scriptPath: resolveScriptPath(input.file, context, options.sourcePath), - publicNetworkOnly: options.publicNetworkOnly, - env: { - ...context.env, - ...(input.env ? stringifyEnvironment(input.env) : {}), + outputEnv: await executeRunScriptFile( + { + scriptPath: resolveScriptPath(input.file, context, options.sourcePath), + publicNetworkOnly: options.publicNetworkOnly, + env: { + ...context.env, + ...(input.env ? stringifyEnvironment(input.env) : {}), + }, + signal: context.signal, }, - signal: context.signal, - }), + options.host, + options.runCommand, + ), }), }; return { diff --git a/packages/maestro/src/daemon-port/daemon-runtime-tap.ts b/packages/maestro/src/daemon-port/daemon-runtime-tap.ts index ed241456ea..6b420b8efe 100644 --- a/packages/maestro/src/daemon-port/daemon-runtime-tap.ts +++ b/packages/maestro/src/daemon-port/daemon-runtime-tap.ts @@ -11,7 +11,6 @@ import { type MaestroTargetMatch, type MaestroTargetQuery, } from '@agent-device/maestro'; -import { emitDiagnostic } from '@agent-device/host-kit/diagnostics'; import { pointInsideRect } from '@agent-device/kernel/rect-center'; import { withMaestroScreenshotBaseline, @@ -92,6 +91,7 @@ export async function tapTargetAndSettle( if (options.platform === 'ios') { await withMaestroScreenshotBaseline({ + emitDiagnostic: options.emitDiagnostic, signal: context.signal, capture: async (path) => { metrics.screenshotCaptures += 1; @@ -231,7 +231,7 @@ async function clickSelector( expectedPoint: { x: number; y: number }, flags: MaestroClickOptions, ): Promise { - emitDiagnostic({ + options.emitDiagnostic({ level: 'debug', phase: 'maestro_tap_dispatch', data: { kind: 'selector', selectorKey: selector.key, expectedPoint }, @@ -260,7 +260,7 @@ export async function clickMaestroTargetPoint( flags: MaestroClickOptions, ): Promise { if (!point) throw new AppError('COMMAND_FAILED', 'Maestro target did not resolve to a point.'); - emitDiagnostic({ + options.emitDiagnostic({ level: 'debug', phase: 'maestro_tap_dispatch', data: { kind: 'point', point }, diff --git a/packages/maestro/src/daemon-port/maestro-screenshot-comparison.ts b/packages/maestro/src/daemon-port/maestro-screenshot-comparison.ts index 167b03b4bc..86d048eee2 100644 --- a/packages/maestro/src/daemon-port/maestro-screenshot-comparison.ts +++ b/packages/maestro/src/daemon-port/maestro-screenshot-comparison.ts @@ -3,7 +3,7 @@ import os from 'node:os'; import path from 'node:path'; import { MAESTRO_RUNTIME_ADAPTER_POLICY } from '@agent-device/maestro'; import { createRequestCanceledError, isRequestCanceledError } from '@agent-device/kernel/errors'; -import { emitDiagnostic } from '@agent-device/host-kit/diagnostics'; +import type { ReplayBackendHost } from '@agent-device/contracts/replay-backend'; import { computePngRgbDifferenceAsync } from '@agent-device/capture-kit/png-worker-client'; import type { PngRgbDifferenceResult } from '@agent-device/capture-kit/png-rgb-difference'; @@ -17,6 +17,7 @@ type MaestroScreenshotCaptureOptions = { }; export async function withMaestroScreenshotBaseline(options: { + readonly emitDiagnostic: ReplayBackendHost['emitDiagnostic']; readonly signal?: AbortSignal; readonly capture: (path: string) => Promise; readonly run: (baseline: MaestroScreenshotBaseline) => Promise; @@ -30,7 +31,13 @@ export async function withMaestroScreenshotBaseline(options: { if (!baselineAvailable || !(await captureMaestroScreenshot(options, afterPath))) { return undefined; } - return await compareMaestroScreenshotFiles(beforePath, afterPath, options.signal, 'tap'); + return await compareMaestroScreenshotFiles( + beforePath, + afterPath, + options.signal, + 'tap', + options.emitDiagnostic, + ); }, }); }); @@ -55,6 +62,7 @@ export async function compareMaestroScreenshotFiles( secondPath: string, signal: AbortSignal | undefined, purpose: 'animation' | 'tap', + emitDiagnostic: ReplayBackendHost['emitDiagnostic'], ): Promise { try { const [firstBuffer, secondBuffer] = await Promise.all([ @@ -68,7 +76,7 @@ export async function compareMaestroScreenshotFiles( 'Maestro screenshot', ); throwIfMaestroScreenshotAborted(signal); - return screenshotsMatch(comparison, purpose); + return screenshotsMatch(comparison, purpose, emitDiagnostic); } catch (error) { rethrowCancellation(error, signal); emitDiagnostic({ @@ -103,14 +111,15 @@ export async function captureMaestroScreenshot( function screenshotsMatch( comparison: PngRgbDifferenceResult, purpose: 'animation' | 'tap', + emitDiagnostic: ReplayBackendHost['emitDiagnostic'], ): boolean { if (comparison.status !== 'compared') { - emitComparison(purpose, comparison.status, comparison); + emitComparison(purpose, comparison.status, comparison, emitDiagnostic); return false; } const differencePercent = comparison.differencePercent; const matches = differencePercent <= MAESTRO_RUNTIME_ADAPTER_POLICY.animationDifferencePercent; - emitComparison(purpose, matches ? 'stable' : 'changed', comparison); + emitComparison(purpose, matches ? 'stable' : 'changed', comparison, emitDiagnostic); return matches; } @@ -118,6 +127,7 @@ function emitComparison( purpose: 'animation' | 'tap', result: 'stable' | 'changed' | 'dimension_mismatch' | 'data_length_mismatch', comparison: PngRgbDifferenceResult, + emitDiagnostic: ReplayBackendHost['emitDiagnostic'], ): void { emitDiagnostic({ level: 'debug', diff --git a/packages/maestro/src/daemon-port/run-script-execution.ts b/packages/maestro/src/daemon-port/run-script-execution.ts index 552097ca7a..042dbaeb62 100644 --- a/packages/maestro/src/daemon-port/run-script-execution.ts +++ b/packages/maestro/src/daemon-port/run-script-execution.ts @@ -1,8 +1,9 @@ +import type { HostEnvironment } from '@agent-device/kernel/source-value'; import fs from 'node:fs'; import path from 'node:path'; import { fileURLToPath } from 'node:url'; import { AppError } from '@agent-device/kernel/errors'; -import { runCmd } from '@agent-device/host-kit/command'; +import type { ReplayBackendHost } from '@agent-device/contracts/replay-backend'; import { assertMaestroScriptsTrusted } from '../internal/engine-flow.ts'; import type { RunScriptChildOutput, RunScriptRequest } from './run-script-vm.ts'; @@ -20,17 +21,24 @@ const RUN_SCRIPT_CHILD_TIMEOUT_MS = 2 * RUN_SCRIPT_TIMEOUT_MS + 5_000; */ export async function executeRunScriptFile( request: RunScriptRequest & { signal?: AbortSignal }, + host: HostEnvironment, + runCommand: ReplayBackendHost['runCommand'], ): Promise> { const { signal, ...childRequest } = request; assertMaestroScriptsTrusted({ trustedScripts: !childRequest.publicNetworkOnly }, 'runScript'); - const result = await runCmd(process.execPath, resolveRunScriptChildArgs('run-script-child'), { - stdin: JSON.stringify(childRequest), + const result = await runCommand( + { + executable: process.execPath, + args: resolveRunScriptChildArgs('run-script-child'), + host, + stdin: JSON.stringify(childRequest), + timeoutMs: RUN_SCRIPT_CHILD_TIMEOUT_MS, + // Its own process group, so a cancel or timeout also reaps an in-flight `http.post` child. + detached: true, + allowFailure: true, + }, signal, - timeoutMs: RUN_SCRIPT_CHILD_TIMEOUT_MS, - // Its own process group, so a cancel or timeout also reaps an in-flight `http.post` child. - detached: true, - allowFailure: true, - }); + ); const output = readScriptChildOutput(result.stdout); if (output && 'outputEnv' in output) return output.outputEnv; if (output) throw new AppError(output.error.code, output.error.message, output.error.details); diff --git a/packages/maestro/src/daemon-port/wait-for-animation-to-end.ts b/packages/maestro/src/daemon-port/wait-for-animation-to-end.ts index ee8db1e97e..8c077d5e24 100644 --- a/packages/maestro/src/daemon-port/wait-for-animation-to-end.ts +++ b/packages/maestro/src/daemon-port/wait-for-animation-to-end.ts @@ -1,4 +1,5 @@ import path from 'node:path'; +import type { ReplayBackendHost } from '@agent-device/contracts/replay-backend'; import { AppError } from '@agent-device/kernel/errors'; import { captureMaestroScreenshot, @@ -8,6 +9,7 @@ import { } from './maestro-screenshot-comparison.ts'; export type MaestroAnimationWaitOptions = { + readonly emitDiagnostic: ReplayBackendHost['emitDiagnostic']; readonly timeoutMs: number; readonly now: () => number; readonly signal?: AbortSignal; @@ -47,8 +49,13 @@ async function capturePairMatches( ]; if (!captures[0] || !captures[1]) return false; return ( - (await compareMaestroScreenshotFiles(firstPath, secondPath, options.signal, 'animation')) === - true + (await compareMaestroScreenshotFiles( + firstPath, + secondPath, + options.signal, + 'animation', + options.emitDiagnostic, + )) === true ); } diff --git a/packages/maestro/src/index.test.ts b/packages/maestro/src/index.test.ts index 204c3602ef..c6cf0e6abb 100644 --- a/packages/maestro/src/index.test.ts +++ b/packages/maestro/src/index.test.ts @@ -27,13 +27,7 @@ describe('@agent-device/maestro facade', () => { publishConfig: { exports: Record }; sideEffects?: boolean; }; - expect(Object.keys(manifest.exports)).toEqual([ - '.', - './replay-backend', - './replay-backend-declaration', - './run-script-http', - './agent-device', - ]); + expect(Object.keys(manifest.exports)).toEqual(['.', './run-script-http', './agent-device']); expect(Object.keys(manifest.publishConfig.exports)).toEqual(['./agent-device']); expect(manifest.sideEffects).toBe(false); }); diff --git a/packages/maestro/src/index.ts b/packages/maestro/src/index.ts index 94d51c0fa4..37a5638570 100644 --- a/packages/maestro/src/index.ts +++ b/packages/maestro/src/index.ts @@ -5,7 +5,6 @@ export { type MaestroActionEvent, type MaestroCompletedActionEvent, type MaestroExecutionObserver, - type MaestroExecutionOptions, type MaestroExecutionOutcome, type MaestroFailedAction, type MaestroFlow, @@ -13,26 +12,12 @@ export { export type { MaestroSourceReader } from './internal/program-loader.ts'; -export { - collectMaestroFlowSources, - type MaestroOptionalSourceReader, -} from './internal/source-closure.ts'; +export { collectMaestroFlowSources } from './internal/source-closure.ts'; -export { - exportReplayActionsToMaestro, - type MaestroExportOptions, - type MaestroExportResult, - type MaestroExportWarning, - type MaestroSelectorProjection, -} from './internal/export-flow.ts'; +export { exportReplayActionsToMaestro } from './internal/export-flow.ts'; export { MAESTRO_SELECTOR_PROJECTION } from './internal/selector-vocabulary.ts'; -export { - MAESTRO_COMPAT_LIMITATIONS, - MAESTRO_COMPAT_SUPPORTED_CAPABILITIES, -} from './internal/facade-support.ts'; - export { createMaestroRuntimePort, literalFromMaestroRegex, @@ -50,16 +35,12 @@ export { type MaestroRuntimeMetrics, type MaestroRuntimeCommand, type MaestroRuntimeOperationContext, - type MaestroRuntimeOperationResult, type MaestroRuntimeOperations, type MaestroRuntimePort, - type MaestroRuntimePortLifecycle, type MaestroRuntimeReadContext, type MaestroSelector, type MaestroSinglePointerGestureInput, type MaestroSnapshotTargetQuery, - type MaestroTargetFailureReason, type MaestroTargetMatch, type MaestroTargetQuery, - type MaestroTargetResolution, } from './internal/facade-runtime-port.ts'; diff --git a/packages/maestro/src/internal/facade-support.ts b/packages/maestro/src/internal/facade-support.ts deleted file mode 100644 index ae1d71f7dd..0000000000 --- a/packages/maestro/src/internal/facade-support.ts +++ /dev/null @@ -1,4 +0,0 @@ -export { - MAESTRO_COMPAT_LIMITATIONS, - MAESTRO_COMPAT_SUPPORTED_CAPABILITIES, -} from './support-matrix.ts'; diff --git a/packages/maestro/src/internal/support-matrix.ts b/packages/maestro/src/internal/support-matrix.ts deleted file mode 100644 index dc72bcc5ac..0000000000 --- a/packages/maestro/src/internal/support-matrix.ts +++ /dev/null @@ -1,33 +0,0 @@ -export const MAESTRO_COMPAT_SUPPORTED_CAPABILITIES = [ - 'Flows: launchApp (with clearState, permissions, and Apple-only launch arguments; permissions apply after state clearing but before launch, and a launchApp without permissions touches nothing — there is no silent all: allow default); setPermissions (mid-flow permission grants, denials, and resets; all resolves in the backend — one simctl call on iOS, the declared permissions on Android — with specific entries overriding after it); runFlow file/inline; runFlow.when and repeat.while conditions (platform, visible, notVisible, and true, all re-evaluated before every repeat iteration); onFlowStart/onFlowComplete; repeat with times, while, or both; and retry.', - 'Interactions: tapOn, doubleTapOn, longPressOn, inputText on the focused element, eraseText (up to 10000 characters), openLink, hideKeyboard, basic pressKey, and back; selector targets poll until available and support recursive index, childOf, above, below, leftOf, rightOf, containsChild, containsDescendants, points, and optional; outer command labels are metadata, not target selectors.', - 'Assertions and navigation: assertVisible, assertNotVisible, assertTrue (literal values and ${VAR} lookups only; "", "false", "0", "null", and "undefined" are falsy, everything else is truthy), extendedWaitUntil, scroll, scrollUntilVisible, absolute/percentage/target swipe, takeScreenshot, waitForAnimationToEnd, clearState, stopApp, and killApp (system-initiated process death: am kill on ADB-backed Android devices, which refuse a foreground target with android-kill-requires-background-app — background the app first, for example pressKey: Home — and fail a target whose process survives with android-kill-process-survived; iOS, Limrun, and WebDriver-provider backends run it as stopApp).', - 'Scripts: ordered runScript file/env scripts with http.post, json, and output variables; evalScript inline expressions run flow-scoped JavaScript and write output.* leaves for later steps.', -] as const; - -export const MAESTRO_COMPAT_LIMITATIONS = [ - 'Permissions: every entry is one settings permission call, applied in order with all first; the step stops at the first entry the selected platform refuses, earlier entries stay applied, and the error names what landed. Android’s only allow level is while-in-use, so location: inuse and location: never mean allow and deny there, while location: always and photos: limited are Apple-only and fail. On iOS, which service a runtime changes is simctl privacy’s own verdict: current runtimes refuse a targeted notifications change and leave notifications untouched under all.', - 'Runtime: iOS and Android only; launchApp.clearState and standalone clearState support Android and iOS simulators, launch arguments are Apple-only (a map-form argument key gains a leading - unless its value is a boolean, as in Maestro), and other standalone device utility/state commands are unsupported.', - 'Expressions: evalScript and condition true: fields (runFlow.when and repeat.while share one evaluator) are evaluated as JavaScript (flow env and prior output leaves are string-typed); a true: field that is a boolean, a maestro.platform comparison, or plain literal text after ${VAR} lookups is decided without JavaScript, with the assertTrue falsy table for literal text. Other fields stay literal or ${VAR} lookup-only — assertTrue supports literals and bare lookups, and other expression-shaped payloads fail loud.', - 'Environment: flow env is the default, AD_VAR_* overrides it, and CLI -e KEY=VALUE wins over both; override keys follow the flow variable grammar (mixed case and dots allowed), and AD_* stays reserved.', - 'Failure diagnostics: resolved targets and runFlow paths are rendered, while inputText payloads remain hidden; do not place secrets in diagnostic identifiers.', - 'Trust: runScript, evalScript, and JavaScript condition true: fields execute flow scripts via node:vm, which is not a security sandbox. runScript runs in a daemon child process with a 30 s script budget. It may make http.post network requests (30 s each, response bodies up to 8 MiB), and its output keys cannot contain a dot. Remote flows refuse runScript before reading its file, including files supplied in the replay bundle, and refuse evalScript and true: fields that need JavaScript. Run these scripts only in trusted local flows. Local runScript paths refer to the daemon filesystem: relative paths resolve from the containing flow, and absolute paths and symlinks are allowed.', - 'Remote screenshots: takeScreenshot uses a daemon-generated path and reports that path in artifactPaths. The flow cannot choose a file destination on the daemon host. Animation waits and tap retry comparisons use temporary screenshots that are removed after comparison.', - 'Errors and tracking: unsupported commands and fields fail with source context when available; open a focused issue only when implementation work is planned.', -] as const; - -export const MAESTRO_COMPATIBILITY_ADR_URL = - 'https://github.com/callstack/agent-device/blob/main/docs/adr/0015-direct-maestro-engine.md'; - -export const MAESTRO_COMPATIBILITY_ISSUE_URL = - 'https://github.com/callstack/agent-device/issues/new'; - -export function formatMaestroCompatibilityReference(): string { - return [ - 'Supported subset:', - ...MAESTRO_COMPAT_SUPPORTED_CAPABILITIES.map((capability) => ` - ${capability}`), - '', - 'Boundaries:', - ...MAESTRO_COMPAT_LIMITATIONS.map((limitation) => ` - ${limitation}`), - ].join('\n'); -} diff --git a/packages/maestro/src/plugin.test.ts b/packages/maestro/src/plugin.test.ts index 72f462e562..dce803e2ef 100644 --- a/packages/maestro/src/plugin.test.ts +++ b/packages/maestro/src/plugin.test.ts @@ -58,6 +58,12 @@ test('selected creation retains one worker, forwards its diagnostic to the host }); const host: ReplayBackendHost = { emitDiagnostic: vi.fn(), + runCommand: async () => { + throw new Error('subprocess unused'); + }, + registerSensitiveValue: () => { + throw new Error('sensitive values unused'); + }, appendTraceEvent: () => { throw new Error('trace unused'); }, diff --git a/packages/maestro/src/replay-backend/__tests__/replay-maestro-backend.test.ts b/packages/maestro/src/replay-backend/__tests__/replay-maestro-backend.test.ts index 8c2d989b41..7e3c5931ab 100644 --- a/packages/maestro/src/replay-backend/__tests__/replay-maestro-backend.test.ts +++ b/packages/maestro/src/replay-backend/__tests__/replay-maestro-backend.test.ts @@ -7,9 +7,15 @@ import { maestroReplayBackendPlugin } from '../replay-maestro-backend.ts'; // each test the behavior through the registry; what is pinned here is the adapter's own mapping. // None of these capabilities reach host services, so the host is never called. const unusedHost: ReplayBackendHost = { + runCommand: async () => { + throw new Error('Unexpected subprocess'); + }, emitDiagnostic: () => { throw new Error('host unused'); }, + registerSensitiveValue: () => { + throw new Error('host unused'); + }, appendTraceEvent: () => { throw new Error('host unused'); }, diff --git a/packages/maestro/src/replay-backend/__tests__/session-replay-maestro-failure.test.ts b/packages/maestro/src/replay-backend/__tests__/session-replay-maestro-failure.test.ts index 4987caa3bc..4a59396c5b 100644 --- a/packages/maestro/src/replay-backend/__tests__/session-replay-maestro-failure.test.ts +++ b/packages/maestro/src/replay-backend/__tests__/session-replay-maestro-failure.test.ts @@ -78,7 +78,11 @@ test('the failure response hands the host the step, its warnings, and its sugges await buildTypedMaestroFailureResponse( { + runCommand: async () => { + throw new Error('Unexpected subprocess'); + }, emitDiagnostic: () => {}, + registerSensitiveValue: () => {}, appendTraceEvent: () => {}, buildBuiltinVars: () => ({}), readScriptSourceFile: () => '', diff --git a/packages/maestro/src/replay-backend/replay-maestro-declaration.ts b/packages/maestro/src/replay-backend/replay-maestro-declaration.ts deleted file mode 100644 index e169ef4a08..0000000000 --- a/packages/maestro/src/replay-backend/replay-maestro-declaration.ts +++ /dev/null @@ -1,21 +0,0 @@ -import type { ReplayBackendDeclaration } from '@agent-device/contracts/replay-backend'; -import { - formatMaestroCompatibilityReference, - MAESTRO_COMPATIBILITY_ADR_URL, - MAESTRO_COMPATIBILITY_ISSUE_URL, -} from '../internal/support-matrix.ts'; - -export const maestroReplayBackendDeclaration: ReplayBackendDeclaration<'maestro'> = { - id: 'maestro', - help: { - summary: 'Supported Maestro YAML commands, grammar, and runtime boundaries', - body: `agent-device help maestro - -Run Maestro compatibility flows with replay --maestro or test --maestro. Bind an iOS or Android target with --platform or an existing session. - -${formatMaestroCompatibilityReference()} - -Unsupported syntax fails loudly rather than being skipped. Architecture, performance tradeoffs, and declared conformance divergences: ${MAESTRO_COMPATIBILITY_ADR_URL} -Focused compatibility request: ${MAESTRO_COMPATIBILITY_ISSUE_URL}`, - }, -}; diff --git a/packages/maestro/src/replay-backend/session-replay-maestro-runtime.ts b/packages/maestro/src/replay-backend/session-replay-maestro-runtime.ts index 37b5e9fea0..a0d795f9cf 100644 --- a/packages/maestro/src/replay-backend/session-replay-maestro-runtime.ts +++ b/packages/maestro/src/replay-backend/session-replay-maestro-runtime.ts @@ -353,6 +353,10 @@ function createMaestroReplayPort(params: { return createDaemonMaestroRuntimePort({ readSnapshotClickabilityEvidence: params.host.readSnapshotClickabilityEvidence, assertSnapshotScreenOrigin: params.host.assertSnapshotScreenOrigin, + host: command.host, + runCommand: params.host.runCommand, + emitDiagnostic: params.host.emitDiagnostic, + registerSensitiveValue: params.host.registerSensitiveValue, invoke: (operation) => invoke(maestroOperationDispatchRequest(replay, operation)), flags: maestroRuntimeDeviceFlags(device, platform, req.flags, publicNetworkOnly), publicNetworkOnly, diff --git a/packages/maestro/test/package-smoke.mjs b/packages/maestro/test/package-smoke.mjs new file mode 100644 index 0000000000..c8dc9cb609 --- /dev/null +++ b/packages/maestro/test/package-smoke.mjs @@ -0,0 +1,135 @@ +import { run } from './package-test-support.mjs'; +import assert from 'node:assert/strict'; +import crypto from 'node:crypto'; +import fs from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; +import { test } from 'node:test'; + +const root = path.resolve(import.meta.dirname, '../../..'); +function files(directory) { + return fs.readdirSync(directory, { withFileTypes: true }).flatMap((entry) => { + const file = path.join(directory, entry.name); + return entry.isDirectory() ? files(file) : [file]; + }); +} + +test('packed core excludes the engine and the managed plugin works without workspace dependencies', async (t) => { + const scratch = fs.mkdtempSync(path.join(os.tmpdir(), 'agent-device-maestro-package-')); + try { + const coreTarball = path.join( + scratch, + JSON.parse( + run('npm', ['pack', '--ignore-scripts', '--json', '--pack-destination', scratch], root), + )[0].filename, + ); + const pluginTarball = path.join(scratch, 'maestro.tgz'); + run('pnpm', ['pack', '--out', pluginTarball], path.join(root, 'packages/maestro')); + const consumer = path.join(scratch, 'consumer'); + const home = path.join(scratch, 'home'); + const installation = crypto.randomUUID(); + const store = path.join(home, 'plugins', installation); + for (const directory of [consumer, store]) { + fs.mkdirSync(directory, { recursive: true }); + fs.writeFileSync(path.join(directory, 'package.json'), '{"private":true,"type":"module"}'); + } + run('npm', ['install', '--ignore-scripts', '--no-audit', '--no-fund', coreTarball], consumer); + const core = path.join(consumer, 'node_modules/agent-device'); + const cli = path.join(core, 'bin/agent-device.mjs'); + const env = { + ...process.env, + AGENT_DEVICE_HOME: home, + AGENT_DEVICE_STATE_DIR: path.join(scratch, 'state'), + AGENT_DEVICE_NO_UPDATE: '1', + }; + const coreCode = files(path.join(core, 'dist')) + .filter((file) => file.endsWith('.js')) + .map((file) => fs.readFileSync(file, 'utf8')) + .join('\n'); + assert.ok(!coreCode.includes('Maestro runScript process exited without a result')); + assert.ok( + !files(path.join(core, 'dist')).some((file) => /run-script-(?:http-)?child\./.test(file)), + ); + const script = path.join(scratch, 'flow.ad'); + fs.writeFileSync(script, 'open com.example.app\n'); + assert.match( + run(process.execPath, [cli, 'help', 'maestro'], consumer, env), + /plugins add @agent-device\/maestro/, + ); + const missing = run( + process.execPath, + [ + '--input-type=module', + '-e', + ` + import { spawnSync } from 'node:child_process'; + const result = spawnSync(process.execPath, ${JSON.stringify([cli, 'replay', 'export', script, '--json'])}, { encoding: 'utf8' }); + if (result.status === 0) throw new Error('missing plugin succeeded'); + process.stdout.write(result.stdout + result.stderr); + `, + ], + consumer, + env, + ); + assert.match(missing, /plugin-not-installed/); + assert.match(missing, /agent-device plugins add @agent-device\/maestro/); + run('npm', ['install', '--ignore-scripts', '--no-audit', '--no-fund', pluginTarball], store); + fs.writeFileSync( + path.join(home, 'config.json'), + JSON.stringify({ plugins: { '@agent-device/maestro': { installation } } }), + ); + const plugin = path.join(store, 'node_modules/@agent-device/maestro'); + const manifest = JSON.parse(fs.readFileSync(path.join(plugin, 'package.json'), 'utf8')); + const exported = JSON.parse( + run(process.execPath, [cli, 'replay', 'export', script, '--json'], consumer, env), + ); + assert.equal(exported.success, true); + assert.match(exported.data.yaml, /com.example.app/); + for (const mode of [ + 'diagnostics', + 'worker-ownership', + 'embedded-worker', + 'run-script-debug', + 'run-script-env', + ]) { + await t.test(`packed runtime ${mode}`, () => { + const output = run( + process.execPath, + [ + '--experimental-strip-types', + path.join(root, 'packages/maestro/test/packed-runtime.mjs'), + plugin, + scratch, + mode, + core, + ], + consumer, + env, + ); + process.stdout.write(output); + }); + } + const entry = manifest.exports['./agent-device'].import; + const broken = path.join(plugin, entry); + fs.renameSync(broken, `${broken}.missing`); + const failed = run( + process.execPath, + [ + '--input-type=module', + '-e', + ` + import { spawnSync } from 'node:child_process'; + const result = spawnSync(process.execPath, ${JSON.stringify([cli, 'replay', 'export', script, '--json'])}, { encoding: 'utf8' }); + if (result.status === 0) throw new Error('broken plugin succeeded'); + process.stdout.write(result.stdout + result.stderr); + `, + ], + consumer, + env, + ); + assert.match(failed, /plugin_load_failed/); + assert.ok(!failed.includes('plugin-not-installed')); + } finally { + fs.rmSync(scratch, { recursive: true, force: true }); + } +}); diff --git a/packages/maestro/test/packed-runtime.mjs b/packages/maestro/test/packed-runtime.mjs new file mode 100644 index 0000000000..daaf015833 --- /dev/null +++ b/packages/maestro/test/packed-runtime.mjs @@ -0,0 +1,238 @@ +import assert from 'node:assert/strict'; +import fs from 'node:fs'; +import path from 'node:path'; +import { pathToFileURL } from 'node:url'; +import workerThreads from 'node:worker_threads'; +import { syncBuiltinESMExports } from 'node:module'; +import { Session } from 'node:inspector/promises'; +import { PNG } from 'pngjs'; +import { resolveProviderPlugins, startProviderRegistry } from '../../plugins/src/registry.ts'; +import { withDiagnosticsScope } from '../../host-kit/src/diagnostics.ts'; +import { createReplayBackendHost } from '../../replay-port/src/daemon-port/replay-backend-host.ts'; + +const [plugin, scratch, mode, core] = process.argv.slice(2); +const scriptTrace = mode.startsWith('run-script-'); +const secret = 'packed-script-secret-value'; +const workers = []; +const loadedPluginModules = []; +const imports = new Session(); +imports.connect(); +await imports.post('Debugger.enable'); +imports.on('Debugger.scriptParsed', ({ params }) => { + if (params.url.startsWith(pathToFileURL(fs.realpathSync(plugin)).href)) + loadedPluginModules.push(path.basename(params.url)); +}); +const OriginalWorker = workerThreads.Worker; +workerThreads.Worker = class extends OriginalWorker { + constructor(url, options) { + super(url, options); + if (String(url).includes('png-worker') && String(url).startsWith(fs.realpathSync(plugin))) { + const record = { url: String(url), exited: false, terminations: 0 }; + workers.push(record); + this.on('exit', () => { + record.exited = true; + }); + const terminate = this.terminate.bind(this); + this.terminate = () => { + record.terminations += 1; + return terminate(); + }; + } + } +}; +syncBuiltinESMExports(); +const { default: maestroPlugin } = await import( + pathToFileURL(path.join(plugin, 'dist/agent-device.mjs')) +); +async function registry() { + const registry = await startProviderRegistry( + await resolveProviderPlugins([{ source: 'embedder', load: () => maestroPlugin() }]), + {}, + { + adaptWebDriver: async () => { + throw new Error('not a provider'); + }, + }, + 'replay-backends', + ); + return { + close: registry.close, + select: (host) => registry.resolveReplayBackend('maestro', host), + }; +} +async function embedded(label) { + const { startAgentDevice } = await import(pathToFileURL(path.join(core, 'dist/src/index.js'))); + let backend; + const plugin = maestroPlugin(); + const runtime = await startAgentDevice({ + stateDir: path.join(scratch, `embedded-${label}`), + host: { + env: { + AGENT_DEVICE_HOME: path.join(scratch, 'empty-embedded-home'), + AGENT_DEVICE_DAEMON_IDLE_TIMEOUT_MS: '0', + }, + cwd: scratch, + }, + plugins: [ + { + ...plugin, + setup: async (host) => { + const setup = await plugin.setup(host); + return { + ...setup, + replayBackend: { + create: async (host) => { + backend = await setup.replayBackend.create(host); + return backend; + }, + }, + }; + }, + }, + ], + }); + return { + close: runtime.close, + select: async () => { + const entry = path.join(scratch, `embedded-${label}.yaml`); + fs.writeFileSync( + entry, + 'appId: com.example.app\n---\n- evalScript: ${output.value = "ok"}\n', + ); + const response = await runtime.client.replay.run({ + path: entry, + backend: 'maestro', + platform: 'ios', + }); + assert.equal(response.replayed, 1); + return backend; + }, + }; +} +const first = mode === 'embedded-worker' ? await embedded('first') : await registry(); +const second = mode === 'embedded-worker' ? await embedded('second') : await registry(); +assert.equal(workers.length, 0, 'unused plugin setup must not load/spawn the worker'); +assert.deepEqual( + loadedPluginModules, + ['agent-device.mjs'], + 'unused setup evaluates only the engine-free managed entry', +); +imports.disconnect(); +const traceLogPath = path.join(scratch, `host-${mode}.ndjson`); +const host = { + ...createReplayBackendHost(), + resolveTargetDevice: () => { + throw new Error('explicit platform needs no device discovery'); + }, + buildActionFailureResponse: (failure) => { + throw new Error(JSON.stringify(failure.error)); + }, +}; +const backend = await first.select(host); +const other = await second.select(host); +const entry = path.join(scratch, 'packed-runtime.yaml'); +const png = PNG.sync.write(new PNG({ width: 2, height: 2 })); +const calls = []; +async function run(selected) { + fs.writeFileSync(path.join(scratch, 'setup.js'), `output.value = SECRET === '${secret}'`); + const source = scriptTrace + ? `appId: com.example.app\n---\n- runScript:\n file: setup.js\n env:\n SECRET: ${secret}\n- assertTrue: \${output.value}\n` + : 'appId: com.example.app\n---\n- tapOn:\n point: "10,20"\n- waitForAnimationToEnd:\n timeout: 0\n'; + const debug = mode !== 'run-script-env'; + const env = { AGENT_DEVICE_EXEC_TRACE: debug ? undefined : '1' }; + const result = await withDiagnosticsScope({ debug, traceLogPath }, () => + selected.runReplay({ + request: { + command: 'replay', + positionals: [entry], + flags: { + platform: 'android', + replayBackend: 'maestro', + replayScriptSource: { entry, files: { [entry]: source } }, + }, + }, + host: { env, cwd: scratch }, + session: { + name: 'packed', + logPath: path.join(scratch, 'log'), + store: { + get: () => undefined, + lookup: () => undefined, + resolveOpenRuntimeHints: () => undefined, + }, + }, + invoke: async (request) => { + calls.push(request.command); + if (request.command === 'runtime') + return { ok: true, data: { viewport: { x: 0, y: 0, width: 400, height: 800 } } }; + if (request.command === 'screenshot') fs.writeFileSync(request.positionals[0], png); + return { ok: true, data: {} }; + }, + }), + ); + assert.equal(result.ok, true, JSON.stringify(result)); + assert.equal(result.data.replayed, 2); +} +try { + await run(backend); + if (!scriptTrace) { + assert.equal(workers.length, 1, 'comparison must use the actual packed PNG worker'); + assert.ok(workers[0].url.startsWith(fs.realpathSync(plugin)), workers[0].url); + assert.equal(workers[0].exited, false); + } + if (scriptTrace) { + const log = fs.readFileSync(traceLogPath, 'utf8'); + const events = log + .trim() + .split('\n') + .map((line) => JSON.parse(line)) + .filter((event) => event.phase === 'exec_command'); + for (const event of ['spawn_start', 'spawn', 'exit', 'close', 'settled']) + assert.ok( + events.some((row) => row.data.event === event), + `missing host exec_command ${event}`, + ); + assert.equal(new Set(events.map((row) => row.data.execId)).size, 1); + assert.equal(events.find((row) => row.data.event === 'settled').data.reason, 'success'); + assert.ok(events.find((row) => row.data.event === 'spawn').data.pid > 0); + assert.ok(!log.includes(secret), 'host trace must not expose subprocess secrets'); + assert.deepEqual(calls, [], 'runScript needs no device dispatch'); + } else if (mode === 'diagnostics') { + const events = fs + .readFileSync(traceLogPath, 'utf8') + .trim() + .split('\n') + .map((line) => JSON.parse(line)); + const tap = events.find((event) => event.phase === 'maestro_tap_dispatch'); + assert.equal(tap?.level, 'debug'); + assert.deepEqual(tap?.data, { kind: 'point', point: { x: 10, y: 20 } }); + const comparison = events.find((event) => event.phase === 'maestro_screenshot_compare'); + assert.equal(comparison?.data.purpose, 'animation'); + assert.equal(comparison?.data.result, 'stable'); + assert.equal(comparison?.data.differencePercent, 0); + assert.equal(calls.filter((command) => command === 'screenshot').length, 2); + assert.equal(calls.filter((command) => command === 'click').length, 1); + } else { + await first.close(); + await first.close(); + assert.equal(workers[0].exited, false, 'another registry still owns this worker'); + assert.equal(workers[0].terminations, 0); + await run(other); + assert.equal(workers.length, 1, 'remaining owner reuses the live worker'); + await second.close(); + await second.close(); + assert.equal(workers[0].exited, true, 'last owner waits for real worker exit'); + assert.equal(workers[0].terminations, 1, 'last owner terminates once'); + } + console.log( + JSON.stringify({ + mode, + hostLog: fs.existsSync(traceLogPath) ? fs.readFileSync(traceLogPath, 'utf8') : '', + workers, + calls, + }), + ); +} finally { + await first.close(); + await second.close(); +} diff --git a/packages/maestro/tsconfig.json b/packages/maestro/tsconfig.json index 8c7b3a73e0..7f2044a219 100644 --- a/packages/maestro/tsconfig.json +++ b/packages/maestro/tsconfig.json @@ -34,6 +34,9 @@ }, { "path": "../selectors" + }, + { + "path": "../test-kit" } ] } diff --git a/packages/plugins/src/registry.ts b/packages/plugins/src/registry.ts index 00acf6328f..ad2502de23 100644 --- a/packages/plugins/src/registry.ts +++ b/packages/plugins/src/registry.ts @@ -1,3 +1,9 @@ +import type { ReplayBackend } from '@agent-device/contracts/replay-backend'; +import type { + ReplayEnginePlugin, + ReplayEngineSetup, + ReplayBackendResolver, +} from './replay-backend.ts'; import { AppError } from '@agent-device/kernel/errors'; import { isRecord } from '@agent-device/kernel/record'; import type { ProviderDeviceRuntime } from '@agent-device/contracts/device'; @@ -21,7 +27,14 @@ import { import { describePluginFailure } from './inventory.ts'; import { isProviderId } from './discovery.ts'; import { ROUTING_PROVIDER_IDENTITIES } from './contract.ts'; -import { assertProviderRuntime, invalidPlugin, validateAgentDevicePlugin } from './validate.ts'; +import { + assertProviderRuntime, + invalidPlugin, + validateAgentDevicePlugin, + validateReplayEnginePlugin, + validateReplayEngineSetup, + validateReplayBackend, +} from './validate.ts'; /** * One plugin a source offers the registry. Sources differ only in how a candidate loads; the @@ -75,6 +88,11 @@ type Slot = { candidate: PluginCandidate; provider?: string; plugin?: AgentDevicePlugin; + backend?: string; + replayPlugin?: ReplayEnginePlugin; + replaySetup?: ReplayEngineSetup; + replayLoading?: Promise; + replayFailure?: unknown; failure?: unknown; registration?: ProviderRegistration; }; @@ -92,6 +110,7 @@ export type ResolvedProviderPlugins = Readonly<{ export type ProviderRegistry = Readonly<{ registrations: readonly ProviderRegistration[]; + resolveReplayBackend: ReplayBackendResolver; claimedProviders: readonly string[]; inventory: readonly PluginInventoryEntry[]; /** @@ -112,6 +131,7 @@ export async function resolveProviderPlugins( ): Promise { const slots: Slot[] = []; const claimed = new Set(); + const claimedBackends = new Set(); for (const candidate of candidates) { const slot: Slot = { candidate, provider: candidate.provider }; slots.push(slot); @@ -119,23 +139,11 @@ export async function resolveProviderPlugins( if (candidate.provider !== undefined) claimed.add(candidate.provider); continue; } - try { - const raw = await candidate.load(); - slot.provider = readProviderId(raw) ?? candidate.provider; - slot.plugin = validateAgentDevicePlugin(raw); - } catch (error) { - slot.failure = asPluginFailure(error, 'plugin_load_failed'); - } - if (slot.provider === undefined || ROUTING_PROVIDER_IDENTITIES.includes(slot.provider)) - continue; - if (claimed.has(slot.provider)) { - slot.plugin = undefined; - slot.failure ??= new AppError('INVALID_ARGS', `Duplicate provider plugin: ${slot.provider}`, { - reason: 'duplicate_provider', - provider: slot.provider, - }); - } else claimed.add(slot.provider); + await loadSlot(slot); + claimBackend(slot, claimedBackends); + claimProvider(slot, claimed); } + return Object.freeze({ connections: new Map( slots.flatMap(({ plugin, candidate }) => @@ -163,11 +171,13 @@ export async function startProviderRegistry( resolved: ResolvedProviderPlugins, env: NodeJS.ProcessEnv, policy: PluginRegistryPolicy, + scope: 'all' | 'replay-backends' = 'all', ): Promise { for (const slot of resolved.slots) { - if (!slot.plugin || slot.failure !== undefined) continue; + if (slot.failure !== undefined) continue; try { - await setUpSlot(slot, slot.plugin, env, policy); + if (slot.replayPlugin) await setUpReplaySlot(slot, slot.replayPlugin, env); + else if (slot.plugin && scope === 'all') await setUpSlot(slot, slot.plugin, env, policy); } catch (error) { slot.failure = asPluginFailure(error, 'plugin_setup_failed'); slot.registration = undefined; @@ -176,14 +186,136 @@ export async function startProviderRegistry( const registrations = resolved.slots.flatMap(({ registration }) => registration ? [registration] : [], ); + let closed = false; + const inventory = [...resolved.inventory()]; return Object.freeze({ registrations, claimedProviders: resolved.claimedProviders, - inventory: resolved.inventory(), - close: once(async () => await Promise.all(registrations.map(cleanUp))), + inventory, + resolveReplayBackend: async (id, host, packageName) => { + if (closed) + throw new AppError('COMMAND_FAILED', 'Plugin registry is closed', { + reason: 'plugin-registry-closed', + }); + return await resolveRegisteredReplayBackend(resolved, inventory, id, host, packageName); + }, + close: once(async () => { + closed = true; + const [providers, backends] = await Promise.all([ + Promise.all(registrations.map(cleanUp)), + Promise.all(resolved.slots.map(cleanUpReplaySlot)), + ]); + return [...providers, ...backends.flat()]; + }), }); } +async function loadSlot(slot: Slot): Promise { + try { + const raw = await slot.candidate.load(); + if (isRecord(raw) && raw.kind === 'replay-backend') { + slot.provider = undefined; + slot.backend = typeof raw.backend === 'string' ? raw.backend : undefined; + slot.replayPlugin = validateReplayEnginePlugin(raw); + } else { + slot.provider = readProviderId(raw) ?? slot.candidate.provider; + slot.plugin = validateAgentDevicePlugin(raw); + } + } catch (error) { + slot.failure = asPluginFailure(error, 'plugin_load_failed'); + } +} + +function claimBackend(slot: Slot, claimed: Set): void { + if (slot.backend === undefined) return; + if (!claimed.has(slot.backend)) { + claimed.add(slot.backend); + return; + } + slot.replayPlugin = undefined; + slot.failure ??= new AppError('INVALID_ARGS', `Duplicate replay backend: ${slot.backend}`, { + reason: 'duplicate_replay_backend', + backend: slot.backend, + }); +} + +function claimProvider(slot: Slot, claimed: Set): void { + if (slot.provider === undefined || ROUTING_PROVIDER_IDENTITIES.includes(slot.provider)) return; + if (!claimed.has(slot.provider)) { + claimed.add(slot.provider); + return; + } + slot.plugin = undefined; + slot.failure ??= new AppError('INVALID_ARGS', `Duplicate provider plugin: ${slot.provider}`, { + reason: 'duplicate_provider', + provider: slot.provider, + }); +} + +async function setUpReplaySlot( + slot: Slot, + plugin: ReplayEnginePlugin, + env: NodeJS.ProcessEnv, +): Promise { + const result = await plugin.setup(createPluginHost(env, slot.candidate.hostOptions)); + try { + slot.replaySetup = validateReplayEngineSetup(result); + } catch (error) { + await cleanUpRejectedSetup(result); + throw error; + } +} + +async function resolveRegisteredReplayBackend( + resolved: ResolvedProviderPlugins, + inventory: PluginInventoryEntry[], + id: string, + host: import('@agent-device/contracts/replay-backend').ReplayBackendHost, + packageName?: string, +): Promise { + const slot = + resolved.slots.find((slot) => slot.backend === id) ?? + resolved.slots.find( + (slot) => packageName !== undefined && slot.candidate.package === packageName, + ); + if (!slot) + throw new AppError('INVALID_ARGS', `Replay backend is not installed: ${id}`, { + reason: 'plugin-not-installed', + backend: id, + }); + if (slot.failure !== undefined) throw slot.failure; + const setup = slot.replaySetup; + if (!setup) throw invalidPlugin(`Replay backend was not set up: ${id}`); + return await (slot.replayLoading ??= Promise.resolve().then(async () => { + try { + const backend = validateReplayBackend(id, await setup.replayBackend.create(host)); + slot.replayFailure = undefined; + inventory.splice(0, inventory.length, ...resolved.inventory()); + return backend; + } catch (error) { + slot.replayLoading = undefined; + slot.replayFailure = asPluginFailure(error, 'plugin_load_failed'); + inventory.splice(0, inventory.length, ...resolved.inventory()); + throw slot.replayFailure; + } + })); +} + +async function cleanUpReplaySlot(slot: Slot): Promise { + const setup = slot.replaySetup; + const plugin = slot.replayPlugin; + if (!setup || !plugin) return []; + await slot.replayLoading?.catch(() => {}); + return [ + { + backend: plugin.backend, + source: slot.candidate.source, + shutdown: { status: 'ok' }, + dispose: await attempt(async () => await setup.dispose?.()), + }, + ]; +} + function withConnectionHost( connection: PluginConnection, hostOptions: Record | undefined, @@ -287,16 +419,21 @@ function describeSlot(slot: Slot): PluginInventoryEntry[] { if (candidate.dormant) return []; const base = { ...(slot.provider !== undefined ? { provider: slot.provider } : {}), + ...(slot.backend !== undefined ? { backend: slot.backend } : {}), source: candidate.source, ...(candidate.package !== undefined ? { package: candidate.package } : {}), ...(candidate.version !== undefined ? { version: candidate.version } : {}), }; - if (slot.failure !== undefined || !plugin) { + if ( + slot.failure !== undefined || + slot.replayFailure !== undefined || + (!plugin && !slot.replayPlugin) + ) { return [ { ...base, state: 'failed', - error: describePluginFailure(slot.failure), + error: describePluginFailure(slot.failure ?? slot.replayFailure), credentials: NO_PLUGIN_CREDENTIALS, }, ]; @@ -306,9 +443,9 @@ function describeSlot(slot: Slot): PluginInventoryEntry[] { ...base, state: 'active', credentials: { - variables: [...plugin.credentials.variables], + variables: [...(plugin?.credentials.variables ?? [])], byBackend: Object.fromEntries( - Object.entries(plugin.credentials.byBackend ?? {}).map(([backend, names]) => [ + Object.entries(plugin?.credentials.byBackend ?? {}).map(([backend, names]) => [ backend, [...names], ]), diff --git a/packages/plugins/src/replay-backend.test.ts b/packages/plugins/src/replay-backend.test.ts new file mode 100644 index 0000000000..6580069a02 --- /dev/null +++ b/packages/plugins/src/replay-backend.test.ts @@ -0,0 +1,178 @@ +import { expect, test, vi } from 'vitest'; +import { resolveProviderPlugins, startProviderRegistry } from './registry.ts'; +import type { PluginCandidate, PluginRegistryPolicy } from './registry.ts'; +import type { ReplayBackend, ReplayBackendHost } from '@agent-device/contracts/replay-backend'; +import type { ProviderPluginHost } from './host.ts'; +import type { ReplayEnginePlugin } from './replay-backend.ts'; + +const host = {} as ReplayBackendHost; +const policy: PluginRegistryPolicy = { + adaptWebDriver: async () => { + throw new Error('no provider'); + }, +}; +function candidate(plugin: unknown): PluginCandidate { + return { source: 'embedder', load: () => plugin }; +} +function engine(backend = 'maestro') { + const runtime: ReplayBackend = { + id: backend, + collectSourceFiles: ({ entryPath, entrySource }) => ({ [entryPath]: entrySource }), + inspectSource: () => ({ title: 'flow', appTarget: 'com.example.app' }), + runReplay: vi.fn(async () => ({ ok: true as const, data: { replayed: 1 } })), + exportReplayScript: () => ({ yaml: '- launchApp\n', warnings: [] }), + }; + const create = vi.fn(async () => runtime); + const dispose = vi.fn(); + const setup = vi.fn((_host: ProviderPluginHost) => ({ replayBackend: { create }, dispose })); + return { + plugin: { apiVersion: 2, kind: 'replay-backend', backend, setup } satisfies ReplayEnginePlugin, + runtime, + create, + dispose, + setup, + }; +} + +test('managed replay setup stays lazy and retains one backend and one cleanup owner', async () => { + const e = engine(); + const resolved = await resolveProviderPlugins([candidate(e.plugin)]); + const registry = await startProviderRegistry(resolved, { TOKEN: 'host-value' }, policy); + expect(e.create).not.toHaveBeenCalled(); + expect(e.setup.mock.calls[0]?.[0]).toMatchObject({ env: { TOKEN: 'host-value' } }); + expect(registry.registrations).toEqual([]); + expect(registry.inventory).toMatchObject([{ backend: 'maestro', state: 'active' }]); + const [one, two] = await Promise.all([ + registry.resolveReplayBackend('maestro', host), + registry.resolveReplayBackend('maestro', host), + ]); + expect(one).toBe(e.runtime); + expect(two).toBe(one); + expect(e.create).toHaveBeenCalledExactlyOnceWith(host); + expect(await one.runReplay({} as never)).toMatchObject({ + ok: true as const, + data: { replayed: 1 }, + }); + expect(await registry.close()).toEqual([ + { + backend: 'maestro', + source: 'embedder', + shutdown: { status: 'ok' }, + dispose: { status: 'ok' }, + }, + ]); + await registry.close(); + expect(e.dispose).toHaveBeenCalledTimes(1); + await expect(registry.resolveReplayBackend('maestro', host)).rejects.toMatchObject({ + details: { reason: 'plugin-registry-closed' }, + }); +}); + +test('backend identity is separate from provider identity and first failure claims the backend', async () => { + const bad = { ...engine().plugin, apiVersion: 99 }; + const other = engine('other'); + const resolved = await resolveProviderPlugins([ + candidate(bad), + candidate(engine().plugin), + candidate(other.plugin), + ]); + const registry = await startProviderRegistry(resolved, {}, policy); + expect(registry.claimedProviders).toEqual([]); + expect(registry.inventory).toMatchObject([ + { backend: 'maestro', state: 'failed', error: { reason: 'incompatible_plugin' } }, + { backend: 'maestro', state: 'failed', error: { reason: 'duplicate_replay_backend' } }, + { backend: 'other', state: 'active' }, + ]); + await expect(registry.resolveReplayBackend('maestro', host)).rejects.toMatchObject({ + details: { reason: 'incompatible_plugin' }, + }); + expect((await registry.resolveReplayBackend('other', host)).inspectSource('', '')).toEqual({ + title: 'flow', + appTarget: 'com.example.app', + }); + await registry.close(); +}); + +test('setup failures stay permanent while failed lazy loads can retry without fallback', async () => { + const setup = engine('setup'); + setup.setup.mockImplementation(() => { + throw new Error('setup refused'); + }); + const load = engine('load'); + load.create.mockRejectedValueOnce(new Error('engine failed')); + const registry = await startProviderRegistry( + await resolveProviderPlugins([candidate(setup.plugin), candidate(load.plugin)]), + {}, + policy, + ); + await expect(registry.resolveReplayBackend('absent', host)).rejects.toMatchObject({ + details: { reason: 'plugin-not-installed' }, + }); + await expect(registry.resolveReplayBackend('setup', host)).rejects.toMatchObject({ + details: { reason: 'plugin_setup_failed' }, + }); + await expect(registry.resolveReplayBackend('load', host)).rejects.toMatchObject({ + details: { reason: 'plugin_load_failed' }, + }); + expect(registry.inventory[1]).toMatchObject({ + state: 'failed', + error: { reason: 'plugin_load_failed' }, + }); + const [one, two] = await Promise.all([ + registry.resolveReplayBackend('load', host), + registry.resolveReplayBackend('load', host), + ]); + expect(one).toBe(load.runtime); + expect(two).toBe(one); + expect(await registry.resolveReplayBackend('load', host)).toBe(one); + expect(load.create).toHaveBeenCalledTimes(2); + expect(registry.inventory[1]).toMatchObject({ state: 'active' }); + await expect(registry.resolveReplayBackend('setup', host)).rejects.toMatchObject({ + details: { reason: 'plugin_setup_failed' }, + }); + expect(setup.setup).toHaveBeenCalledTimes(1); + await registry.close(); + expect(load.dispose).toHaveBeenCalledTimes(1); +}); + +test('the registry rejects the wrong runtime id and cleans up an invalid setup', async () => { + const wrong = engine('wrong'); + wrong.create.mockResolvedValue({ ...wrong.runtime, id: 'different' }); + const dispose = vi.fn(); + const invalid = { ...engine('invalid').plugin, setup: () => ({ replayBackend: {}, dispose }) }; + const registry = await startProviderRegistry( + await resolveProviderPlugins([candidate(wrong.plugin), candidate(invalid)]), + {}, + policy, + ); + await expect(registry.resolveReplayBackend('wrong', host)).rejects.toMatchObject({ + details: { reason: 'invalid_plugin' }, + }); + await expect(registry.resolveReplayBackend('invalid', host)).rejects.toMatchObject({ + details: { reason: 'invalid_plugin' }, + }); + expect(dispose).toHaveBeenCalledTimes(1); + await registry.close(); +}); + +test('one backend disposer cannot delay another backend disposer', async () => { + const first = engine('first'); + const second = engine('second'); + let release!: () => void; + first.dispose.mockImplementation( + () => + new Promise((resolve) => { + release = resolve; + }), + ); + const registry = await startProviderRegistry( + await resolveProviderPlugins([candidate(first.plugin), candidate(second.plugin)]), + {}, + policy, + ); + const closing = registry.close(); + await Promise.resolve(); + expect(second.dispose).toHaveBeenCalledTimes(1); + release(); + expect(await closing).toHaveLength(2); +}); diff --git a/packages/plugins/src/replay-backend.ts b/packages/plugins/src/replay-backend.ts index 55944fe4fc..85884e878c 100644 --- a/packages/plugins/src/replay-backend.ts +++ b/packages/plugins/src/replay-backend.ts @@ -13,3 +13,5 @@ export type ReplayEngineSetup = Readonly<{ replayBackend: ReplayBackendPlugin; dispose?(): Promise | void; }>; + +export type { ReplayBackendResolver } from '@agent-device/contracts/replay-backend'; diff --git a/packages/plugins/src/store.ts b/packages/plugins/src/store.ts index 8faf7ec77b..0a572a4bb7 100644 --- a/packages/plugins/src/store.ts +++ b/packages/plugins/src/store.ts @@ -176,6 +176,7 @@ export async function listPlugins(env: NodeJS.ProcessEnv, builtinProviders: read name: entry.package, ...(entry.version !== undefined ? { version: entry.version } : {}), ...(entry.provider !== undefined ? { provider: entry.provider } : {}), + ...(entry.backend !== undefined ? { backend: entry.backend } : {}), compatible: entry.state === 'active', ...(entry.error ? { error: entry.error } : {}), })); diff --git a/packages/plugins/src/validate.ts b/packages/plugins/src/validate.ts index 6fe6008f84..10ffdeae9a 100644 --- a/packages/plugins/src/validate.ts +++ b/packages/plugins/src/validate.ts @@ -1,3 +1,5 @@ +import type { ReplayBackend } from '@agent-device/contracts/replay-backend'; +import type { ReplayEnginePlugin, ReplayEngineSetup } from './replay-backend.ts'; import { AppError } from '@agent-device/kernel/errors'; import { isRecord } from '@agent-device/kernel/record'; import { isProviderId } from './discovery.ts'; @@ -134,3 +136,47 @@ function hasProviderFacets(runtime: ProviderDeviceRuntime): boolean { export function invalidPlugin(message: string): AppError { return new AppError('INVALID_ARGS', message, { reason: 'invalid_plugin' }); } + +export function validateReplayEnginePlugin(value: unknown): ReplayEnginePlugin { + if (!isRecord(value) || value.kind !== 'replay-backend') + throw invalidPlugin('Plugin must declare kind: replay-backend'); + if (value.apiVersion !== 2) + throw new AppError('INVALID_ARGS', 'Replay plugin requires apiVersion 2', { + reason: 'incompatible_plugin', + apiVersion: 2, + }); + if ( + value.provider !== undefined || + value.connection !== undefined || + value.credentials !== undefined + ) + throw invalidPlugin( + 'Replay plugins cannot declare provider identity, connection or credentials', + ); + if (!isProviderId(value.backend) || typeof value.setup !== 'function') + throw invalidPlugin('Replay plugin must declare a backend id and setup(host)'); + return value as ReplayEnginePlugin; +} + +export function validateReplayEngineSetup(value: unknown): ReplayEngineSetup { + if ( + !isRecord(value) || + !isRecord(value.replayBackend) || + typeof value.replayBackend.create !== 'function' || + (value.dispose !== undefined && typeof value.dispose !== 'function') + ) + throw invalidPlugin('Replay plugin setup must return replayBackend.create(host)'); + return value as ReplayEngineSetup; +} + +export function validateReplayBackend(id: string, value: unknown): ReplayBackend { + if ( + !isRecord(value) || + value.id !== id || + (['collectSourceFiles', 'inspectSource', 'runReplay', 'exportReplayScript'] as const).some( + (name) => typeof value[name] !== 'function', + ) + ) + throw invalidPlugin(`Replay plugin runtime does not match its backend: ${id}`); + return value as ReplayBackend; +} diff --git a/packages/replay-port/package.json b/packages/replay-port/package.json index 44b00d2069..181859b5cd 100644 --- a/packages/replay-port/package.json +++ b/packages/replay-port/package.json @@ -14,10 +14,10 @@ "@agent-device/device-selection": "workspace:*", "@agent-device/host-kit": "workspace:*", "@agent-device/kernel": "workspace:*", - "@agent-device/maestro": "workspace:*", "@agent-device/replay-test": "workspace:*", "@agent-device/selectors": "workspace:*", - "@agent-device/session-journal": "workspace:*" + "@agent-device/session-journal": "workspace:*", + "@agent-device/plugins": "workspace:*" }, "exports": { "./command-types": { diff --git a/packages/replay-port/src/__tests__/script-source-bundle.test.ts b/packages/replay-port/src/__tests__/script-source-bundle.test.ts index 0ab47f9274..9664ad4d92 100644 --- a/packages/replay-port/src/__tests__/script-source-bundle.test.ts +++ b/packages/replay-port/src/__tests__/script-source-bundle.test.ts @@ -16,7 +16,11 @@ test('a native .ad bundle carries the entry script resolved against the caller c const script = 'open "Demo"\nclick "Save"\n'; fs.writeFileSync(path.join(root, 'flow.ad'), script); - const bundle = await loadReplayScriptSourceBundle({ inputPath: './flow.ad', cwd: root }); + const bundle = await loadReplayScriptSourceBundle({ + hostEnv: process.env, + inputPath: './flow.ad', + cwd: root, + }); expect(bundle.entry).toBe(path.join(root, 'flow.ad')); expect(bundle.files).toEqual({ [path.join(root, 'flow.ad')]: script }); @@ -33,6 +37,7 @@ test('a Maestro bundle carries the entry flow and its transitive runFlow include fs.writeFileSync(path.join(root, 'tail.yaml'), '---\n- back\n'); const bundle = await loadReplayScriptSourceBundle({ + hostEnv: process.env, inputPath: './flow.yaml', cwd: root, replayBackend: 'maestro', @@ -54,7 +59,11 @@ test('a missing entry script fails as INVALID_ARGS naming the path as typed', as const root = mkdtempForTestSync('agent-device-bundle-missing-'); await expect( - loadReplayScriptSourceBundle({ inputPath: './flows/login.ad', cwd: root }), + loadReplayScriptSourceBundle({ + hostEnv: process.env, + inputPath: './flows/login.ad', + cwd: root, + }), ).rejects.toThrow( expect.objectContaining({ code: 'INVALID_ARGS', @@ -70,7 +79,9 @@ test('an oversized bundle is refused with the limit named', async () => { `open "Demo"\n${'#'.repeat(REPLAY_SCRIPT_SOURCE_BUNDLE_MAX_BYTES)}\n`, ); - await expect(loadReplayScriptSourceBundle({ inputPath: './flow.ad', cwd: root })).rejects.toThrow( + await expect( + loadReplayScriptSourceBundle({ hostEnv: process.env, inputPath: './flow.ad', cwd: root }), + ).rejects.toThrow( expect.objectContaining({ code: 'INVALID_ARGS', message: expect.stringContaining(`over the ${REPLAY_SCRIPT_SOURCE_BUNDLE_MAX_BYTES}-byte`), diff --git a/packages/replay-port/src/daemon-port/__tests__/replay-backend-registry.test.ts b/packages/replay-port/src/daemon-port/__tests__/replay-backend-registry.test.ts index a907928087..3abae7b59c 100644 --- a/packages/replay-port/src/daemon-port/__tests__/replay-backend-registry.test.ts +++ b/packages/replay-port/src/daemon-port/__tests__/replay-backend-registry.test.ts @@ -1,50 +1,49 @@ -import { expect, test, vi } from 'vitest'; -import { maestroReplayBackendDeclaration } from '@agent-device/maestro/replay-backend-declaration'; +import { expect, test } from 'vitest'; +import { createRuntimeStateHandle } from '@agent-device/host-kit/runtime-slots'; import { getReplayBackendDeclaration } from '../replay-backend-declarations.ts'; -import { getReplayBackend } from '../replay-backend-registry.ts'; +import { getReplayBackend, openReplayBackend } from '../replay-backend-registry.ts'; +import { mkdtempForTestSync } from '@agent-device/test-kit'; -// The #3377 extraction seam: the registry is the only host-side door to a non-native engine, so -// what it promises is exactly what the plugin swap must keep — the load-once contract and the -// capability shape every host call site routes through. - -test('lists the Maestro declaration for the maestro id', () => { - expect(getReplayBackendDeclaration('maestro')).toBe(maestroReplayBackendDeclaration); +test('help declares the exact optional package without requiring a working plugin', () => { + expect(getReplayBackendDeclaration('maestro')).toMatchObject({ + id: 'maestro', + package: '@agent-device/maestro', + }); }); -test('memoizes the backend so a process evaluates the engine once', async () => { - const first = getReplayBackend('maestro'); - const second = getReplayBackend('maestro'); - expect(second).toBe(first); - expect(await second).toBe(await first); +test('the runtime registry memoizes the installed backend capabilities', async () => { + const first = await getReplayBackend('maestro'); + expect(await getReplayBackend('maestro')).toBe(first); + expect(first.inspectSource('appId: com.example.app\n---\n- launchApp\n', 'flow.yaml')).toEqual({ + title: undefined, + appTarget: 'com.example.app', + }); }); -test('a backend carries every capability the host routes through it', async () => { - const backend = await getReplayBackend('maestro'); - expect(backend.id).toBe('maestro'); - expect(Object.keys(backend).sort()).toEqual( - ['collectSourceFiles', 'exportReplayScript', 'id', 'inspectSource', 'runReplay'].sort(), - ); +test('absence gives a vetted install command and preserves the host installation boundary', async () => { + await expect( + openReplayBackend('maestro', { + AGENT_DEVICE_HOME: mkdtempForTestSync('missing-replay-plugin-'), + }), + ).rejects.toMatchObject({ + details: { + reason: 'plugin-not-installed', + package: '@agent-device/maestro', + installCommand: 'agent-device plugins add @agent-device/maestro', + }, + }); }); -test('a failed load is not cached, so the next call retries it', async () => { - // The mutation that makes this fail: without the registry's delete-on-rejection, the rejected - // promise stays memoized and this second resolution rejects forever, where the pre-registry - // per-call `import()` recovered from a one-off load failure. A fresh registry instance keeps - // the memo map free of the successes above. - vi.resetModules(); - let loadAttempts = 0; - vi.doMock('@agent-device/maestro/replay-backend', () => ({ - // The loader reads the plugin export right after the `import()`, so failing the first read - // fails the load exactly like a rejecting module evaluation would. - get maestroReplayBackendPlugin() { - loadAttempts += 1; - if (loadAttempts === 1) throw new Error('transient load failure'); - return { create: () => ({ id: 'maestro' }) }; - }, - })); - const { getReplayBackend: freshGet } = await import('../replay-backend-registry.ts'); - await expect(freshGet('maestro')).rejects.toThrow('transient load failure'); - await expect(freshGet('maestro')).resolves.toMatchObject({ id: 'maestro' }); - expect(loadAttempts).toBe(2); - vi.doUnmock('@agent-device/maestro/replay-backend'); +test('an unbound daemon runtime never borrows another runtime or the client store', async () => { + const isolated = createRuntimeStateHandle(); + try { + await isolated.run( + async () => + await expect(getReplayBackend('maestro')).rejects.toMatchObject({ + details: { reason: 'plugin-not-installed' }, + }), + ); + } finally { + isolated.dispose(); + } }); diff --git a/packages/replay-port/src/daemon-port/replay-backend-declarations.ts b/packages/replay-port/src/daemon-port/replay-backend-declarations.ts index d49fddf315..0e1d2eb738 100644 --- a/packages/replay-port/src/daemon-port/replay-backend-declarations.ts +++ b/packages/replay-port/src/daemon-port/replay-backend-declarations.ts @@ -1,6 +1,56 @@ import type { ReplayBackendId } from '@agent-device/ad-script'; import type { ReplayBackendDeclaration } from '@agent-device/contracts/replay-backend'; -import { maestroReplayBackendDeclaration } from '@agent-device/maestro/replay-backend-declaration'; + +export const MAESTRO_COMPAT_SUPPORTED_CAPABILITIES = [ + 'Flows: launchApp (with clearState, permissions, and Apple-only launch arguments; permissions apply after state clearing but before launch, and a launchApp without permissions touches nothing — there is no silent all: allow default); setPermissions (mid-flow permission grants, denials, and resets; all resolves in the backend — one simctl call on iOS, the declared permissions on Android — with specific entries overriding after it); runFlow file/inline; runFlow.when and repeat.while conditions (platform, visible, notVisible, and true, all re-evaluated before every repeat iteration); onFlowStart/onFlowComplete; repeat with times, while, or both; and retry.', + 'Interactions: tapOn, doubleTapOn, longPressOn, inputText on the focused element, eraseText (up to 10000 characters), openLink, hideKeyboard, basic pressKey, and back; selector targets poll until available and support recursive index, childOf, above, below, leftOf, rightOf, containsChild, containsDescendants, points, and optional; outer command labels are metadata, not target selectors.', + 'Assertions and navigation: assertVisible, assertNotVisible, assertTrue (literal values and ${VAR} lookups only; "", "false", "0", "null", and "undefined" are falsy, everything else is truthy), extendedWaitUntil, scroll, scrollUntilVisible, absolute/percentage/target swipe, takeScreenshot, waitForAnimationToEnd, clearState, stopApp, and killApp (system-initiated process death: am kill on ADB-backed Android devices, which refuse a foreground target with android-kill-requires-background-app — background the app first, for example pressKey: Home — and fail a target whose process survives with android-kill-process-survived; iOS, Limrun, and WebDriver-provider backends run it as stopApp).', + 'Scripts: ordered runScript file/env scripts with http.post, json, and output variables; evalScript inline expressions run flow-scoped JavaScript and write output.* leaves for later steps.', +] as const; + +export const MAESTRO_COMPAT_LIMITATIONS = [ + 'Permissions: every entry is one settings permission call, applied in order with all first; the step stops at the first entry the selected platform refuses, earlier entries stay applied, and the error names what landed. Android’s only allow level is while-in-use, so location: inuse and location: never mean allow and deny there, while location: always and photos: limited are Apple-only and fail. On iOS, which service a runtime changes is simctl privacy’s own verdict: current runtimes refuse a targeted notifications change and leave notifications untouched under all.', + 'Runtime: iOS and Android only; launchApp.clearState and standalone clearState support Android and iOS simulators, launch arguments are Apple-only (a map-form argument key gains a leading - unless its value is a boolean, as in Maestro), and other standalone device utility/state commands are unsupported.', + 'Expressions: evalScript and condition true: fields (runFlow.when and repeat.while share one evaluator) are evaluated as JavaScript (flow env and prior output leaves are string-typed); a true: field that is a boolean, a maestro.platform comparison, or plain literal text after ${VAR} lookups is decided without JavaScript, with the assertTrue falsy table for literal text. Other fields stay literal or ${VAR} lookup-only — assertTrue supports literals and bare lookups, and other expression-shaped payloads fail loud.', + 'Environment: flow env is the default, AD_VAR_* overrides it, and CLI -e KEY=VALUE wins over both; override keys follow the flow variable grammar (mixed case and dots allowed), and AD_* stays reserved.', + 'Failure diagnostics: resolved targets and runFlow paths are rendered, while inputText payloads remain hidden; do not place secrets in diagnostic identifiers.', + 'Trust: runScript, evalScript, and JavaScript condition true: fields execute flow scripts via node:vm, which is not a security sandbox. runScript runs in a daemon child process with a 30 s script budget. It may make http.post network requests (30 s each, response bodies up to 8 MiB), and its output keys cannot contain a dot. Remote flows refuse runScript before reading its file, including files supplied in the replay bundle, and refuse evalScript and true: fields that need JavaScript. Run these scripts only in trusted local flows. Local runScript paths refer to the daemon filesystem: relative paths resolve from the containing flow, and absolute paths and symlinks are allowed.', + 'Remote screenshots: takeScreenshot uses a daemon-generated path and reports that path in artifactPaths. The flow cannot choose a file destination on the daemon host. Animation waits and tap retry comparisons use temporary screenshots that are removed after comparison.', + 'Errors and tracking: unsupported commands and fields fail with source context when available; open a focused issue only when implementation work is planned.', +] as const; + +const MAESTRO_COMPATIBILITY_ADR_URL = + 'https://github.com/callstack/agent-device/blob/main/docs/adr/0015-direct-maestro-engine.md'; + +const MAESTRO_COMPATIBILITY_ISSUE_URL = 'https://github.com/callstack/agent-device/issues/new'; + +function formatMaestroCompatibilityReference(): string { + return [ + 'Supported subset:', + ...MAESTRO_COMPAT_SUPPORTED_CAPABILITIES.map((capability) => ` - ${capability}`), + '', + 'Boundaries:', + ...MAESTRO_COMPAT_LIMITATIONS.map((limitation) => ` - ${limitation}`), + ].join('\n'); +} + +const maestroReplayBackendDeclaration: ReplayBackendDeclaration<'maestro'> = { + id: 'maestro', + package: '@agent-device/maestro', + help: { + summary: 'Supported Maestro YAML commands, grammar, and runtime boundaries', + body: `agent-device help maestro + +Install with agent-device plugins add @agent-device/maestro on the client and daemon host. Restart the daemon after installation. + +Run Maestro compatibility flows with replay --maestro or test --maestro. Bind an iOS or Android target with --platform or an existing session. + +${formatMaestroCompatibilityReference()} + +Unsupported syntax fails loudly rather than being skipped. Architecture, performance tradeoffs, and declared conformance divergences: ${MAESTRO_COMPATIBILITY_ADR_URL} +Focused compatibility request: ${MAESTRO_COMPATIBILITY_ISSUE_URL}`, + }, +}; /** * The engine-free half of every registered replay backend (#3377), as `help` lists it. Kept apart diff --git a/packages/replay-port/src/daemon-port/replay-backend-host.ts b/packages/replay-port/src/daemon-port/replay-backend-host.ts index 67c0ae8ce7..4729556108 100644 --- a/packages/replay-port/src/daemon-port/replay-backend-host.ts +++ b/packages/replay-port/src/daemon-port/replay-backend-host.ts @@ -1,4 +1,8 @@ -import { emitDiagnostic } from '@agent-device/host-kit/diagnostics'; +import { runCmd, withCommandHost } from '@agent-device/host-kit/command'; +import { + emitDiagnostic, + registerDiagnosticSensitiveValue, +} from '@agent-device/host-kit/diagnostics'; import type { ReplayBackendHost } from '@agent-device/contracts/replay-backend'; import { assertSnapshotScreenOrigin, @@ -24,6 +28,11 @@ export function createReplayBackendHost(): ReplayBackendHost { emitDiagnostic, readSnapshotClickabilityEvidence, assertSnapshotScreenOrigin, + runCommand: ({ executable, args, host, ...options }, signal) => + withCommandHost({ host, requestEnv: undefined }, () => + runCmd(executable, args, { ...options, ...host, signal }), + ), + registerSensitiveValue: registerDiagnosticSensitiveValue, appendTraceEvent: appendReplayTraceEvent, buildBuiltinVars: (params) => buildReplayBuiltinVars({ ...params, metadata: {} }), readScriptSourceFile: readReplayScriptSourceFile, diff --git a/packages/replay-port/src/daemon-port/replay-backend-registry.ts b/packages/replay-port/src/daemon-port/replay-backend-registry.ts index e5d2cc3d42..289116cb47 100644 --- a/packages/replay-port/src/daemon-port/replay-backend-registry.ts +++ b/packages/replay-port/src/daemon-port/replay-backend-registry.ts @@ -1,64 +1,109 @@ import type { ReplayBackendId } from '@agent-device/ad-script'; -import type { ReplayBackend, ReplayBackendPlugin } from '@agent-device/contracts/replay-backend'; +import type { ReplayBackend, ReplayBackendResolver } from '@agent-device/contracts/replay-backend'; +import type { AppError } from '@agent-device/kernel/errors'; -/** - * The replay backend registry (#3377). - * - * The host (the replay command, `test` discovery, request device binding, and the client-side - * replay surface) reaches a non-native engine only through this module. A backend id resolves to - * one plugin (`ReplayBackendPlugin`, `create(host)`) whose loading is memoized per process; this - * module and `replay-backend-declarations.ts` are the only host code that names an engine package. Backend ids come from the format grammar's vocabulary - * (`@agent-device/ad-script`), so the registry's keys and the `--maestro` flag's accepted values - * can never drift apart. The Maestro plugin extraction (#3377) swaps the builtin - * table for an installed-plugin lookup and changes no caller. - * - * A backend is a capability bundle over scripts: it expands a source closure, inspects a flow for - * scheduling metadata, runs a replay, and converts a native `.ad` script into its own format. - * Dispatch, session, and wire policy stay on this side — a backend receives an already-prepared - * command and returns an ordinary `DaemonResponse`, so the daemon never re-enters engine parsing - * or compatibility dispatch (ADR 0015). - * - * A hub eager-closure rule of this repository (`scripts/__tests__/eager-closure-budgets.ts`) - * holds `src/cli.ts` and `src/daemon.ts` to "no new module evaluated at startup", so the host's - * value imports of this module are function-scoped `await import()` calls at the few call sites; - * the plugin, its declaration, and the host behind it are reached the same way. Type imports are - * free. - */ +const backendResolverKey = Symbol('replay-backend-resolver'); +async function binding() { + const { runtimeSlot } = await import('@agent-device/host-kit/runtime-slots'); + return runtimeSlot(backendResolverKey, () => ({ + resolve: undefined as ReplayBackendResolver | undefined, + })); +} -/** - * Builtin registrations: the ONE extraction seam. Each loader is a function-scoped `import()` so - * no host entry evaluates an engine eagerly (the `cli-startup-import-closure` pins). Extraction - * swaps this table for the installed-plugin lookup; the host call sites never change. - */ -const BUILTIN_REPLAY_BACKEND_PLUGINS: { - readonly [TId in ReplayBackendId]: () => Promise>; -} = { - maestro: async () => - (await import('@agent-device/maestro/replay-backend')).maestroReplayBackendPlugin, -}; +/** Binds the fixed plugin set of the current daemon runtime. Its registry owns disposal. */ +export async function bindReplayBackendResolver(resolve: ReplayBackendResolver): Promise { + (await binding()).resolve = resolve; +} -const loadedBackends = new Map>>(); +/** Resolves against the current daemon's plugins, without consulting another host's store. */ +export async function getReplayBackend(id: ReplayBackendId): Promise { + const resolve = (await binding()).resolve; + if (!resolve) throw await missingReplayBackend(id); + return await resolveBackend(id, resolve); +} -async function createReplayBackend(id: ReplayBackendId): Promise> { - const plugin = await BUILTIN_REPLAY_BACKEND_PLUGINS[id](); +async function resolveBackend( + id: ReplayBackendId, + resolve: ReplayBackendResolver, +): Promise { + const { AppError } = await import('@agent-device/kernel/errors'); const { createReplayBackendHost } = await import('./replay-backend-host.ts'); - return plugin.create(createReplayBackendHost()); + try { + return await resolve(id, createReplayBackendHost(), await backendPackage(id)); + } catch (error) { + if (error instanceof AppError && error.details?.reason === 'plugin-not-installed') + throw await missingReplayBackend(id); + throw error; + } +} + +/** Uses the embedded registry, or owns a client-side store scope until the task settles. */ +export async function withReplayBackend( + id: ReplayBackendId, + env: NodeJS.ProcessEnv, + run: (backend: ReplayBackend) => Promise | T, + resolve?: ReplayBackendResolver, +): Promise { + if (resolve) return await run(await resolveBackend(id, resolve)); + const scope = await openReplayBackend(id, env); + try { + return await run(scope.backend); + } finally { + await scope.close(); + } } -/** - * Resolves a backend once per process, memoizing the engine's single evaluation. A load failure - * is NOT cached — a failed `import()` stays retryable, as it was at the pre-registry call sites - * that imported fresh per invocation. The id's type is the grammar's `isReplayBackendId` guard, so - * an unregistered value is rejected by the caller before it reaches here; engines never fall back - * to one another. - */ -export function getReplayBackend(id: ReplayBackendId): Promise> { - const cached = loadedBackends.get(id); - if (cached) return cached; - const loading = createReplayBackend(id).catch((error: unknown) => { - loadedBackends.delete(id); +/** A client/tooling scope owns its plugin registry until close. It never starts providers. */ +export async function openReplayBackend(id: ReplayBackendId, env: NodeJS.ProcessEnv) { + const { AppError } = await import('@agent-device/kernel/errors'); + const [{ storePluginCandidates }, { resolveProviderPlugins, startProviderRegistry }] = + await Promise.all([ + import('@agent-device/plugins/store'), + import('@agent-device/plugins/registry'), + ]); + const registry = await startProviderRegistry( + await resolveProviderPlugins(await storePluginCandidates(env)), + env, + { + adaptWebDriver: async () => { + throw new AppError('INVALID_ARGS', 'Client replay scope cannot start a provider'); + }, + }, + 'replay-backends', + ); + try { + const { createReplayBackendHost } = await import('./replay-backend-host.ts'); + const backend = await registry.resolveReplayBackend( + id, + createReplayBackendHost(), + await backendPackage(id), + ); + return { backend, close: registry.close }; + } catch (error) { + await registry.close(); + if (error instanceof AppError && error.details?.reason === 'plugin-not-installed') + throw await missingReplayBackend(id); throw error; - }); - loadedBackends.set(id, loading); - return loading; + } +} + +async function missingReplayBackend(id: ReplayBackendId): Promise { + const { AppError } = await import('@agent-device/kernel/errors'); + const packageName = await backendPackage(id); + const installCommand = `agent-device plugins add ${packageName}`; + return new AppError( + 'INVALID_ARGS', + `Replay backend ${id} requires ${packageName}. Run ${installCommand}, then retry the original command with all its arguments.`, + { + reason: 'plugin-not-installed', + backend: id, + package: packageName, + installCommand, + hint: 'Install on the client for source collection/export and on the daemon host for execution. Restart the daemon after changing plugins; retry the original command with all its arguments.', + }, + ); +} + +async function backendPackage(id: ReplayBackendId): Promise { + return (await import('./replay-backend-declarations.ts')).getReplayBackendDeclaration(id).package; } diff --git a/packages/replay-port/src/replay-script-source.fixtures.ts b/packages/replay-port/src/replay-script-source.fixtures.ts index 0b826d84b1..48f7efe52d 100644 --- a/packages/replay-port/src/replay-script-source.fixtures.ts +++ b/packages/replay-port/src/replay-script-source.fixtures.ts @@ -12,6 +12,7 @@ export async function maestroScriptSourceBundleFor( filePath: string, ): Promise { return await loadReplayScriptSourceBundle({ + hostEnv: process.env, inputPath: filePath, cwd: process.cwd(), replayBackend: 'maestro', diff --git a/packages/replay-port/src/script-source-bundle.ts b/packages/replay-port/src/script-source-bundle.ts index 57cf3b05c0..97d4715610 100644 --- a/packages/replay-port/src/script-source-bundle.ts +++ b/packages/replay-port/src/script-source-bundle.ts @@ -1,9 +1,9 @@ import fs from 'node:fs'; +import type { ReplayBackendResolver } from '@agent-device/contracts/replay-backend'; import { AppError } from '@agent-device/kernel/errors'; import type { ReplayScriptSourceBundle } from '@agent-device/contracts/replay'; import { resolveUserPath } from '@agent-device/host-kit/file'; import { resolveReplayFormat } from '@agent-device/ad-script'; -import type { ReplayBackend } from '@agent-device/contracts/replay-backend'; export const REPLAY_SCRIPT_SOURCE_BUNDLE_MAX_BYTES = 2 * 1024 * 1024; @@ -11,6 +11,8 @@ export type ReplayScriptSourceRequest = { inputPath: string; cwd: string; replayBackend?: string; + hostEnv: NodeJS.ProcessEnv; + resolveReplayBackend?: ReplayBackendResolver; env?: Readonly>; }; @@ -26,15 +28,20 @@ export async function loadReplayScriptSourceBundle( // Function-scoped on purpose: the eager-closure budget gate counts a static import here as a // module every warm CLI command evaluates, for a format most runs never use (#1802), and the // backend's engine sits one thunk behind it (#3377). - const backend: ReplayBackend = await ( + const files = await ( await import('@agent-device/replay-port/replay-backend-registry') - ).getReplayBackend(backendId); - const files = backend.collectSourceFiles({ - entryPath: entry, - entrySource, - env: params.env, - readSource: tryReadScriptFile, - }); + ).withReplayBackend( + backendId, + params.hostEnv, + (backend) => + backend.collectSourceFiles({ + entryPath: entry, + entrySource, + env: params.env, + readSource: tryReadScriptFile, + }), + params.resolveReplayBackend, + ); return finishBundle(entry, files, params.inputPath); } diff --git a/packages/replay-port/tsconfig.json b/packages/replay-port/tsconfig.json index 92d1911f45..062682eb66 100644 --- a/packages/replay-port/tsconfig.json +++ b/packages/replay-port/tsconfig.json @@ -11,18 +11,44 @@ }, "include": ["src", "../command-registry/src/global.d.ts"], "references": [ - { "path": "../ad-replay" }, - { "path": "../ad-script" }, - { "path": "../capture-kit" }, - { "path": "../command-registry" }, - { "path": "../contracts" }, - { "path": "../device-selection" }, - { "path": "../host-kit" }, - { "path": "../kernel" }, - { "path": "../maestro" }, - { "path": "../replay-test" }, - { "path": "../selectors" }, - { "path": "../session-journal" }, - { "path": "../test-kit" } + { + "path": "../ad-replay" + }, + { + "path": "../ad-script" + }, + { + "path": "../capture-kit" + }, + { + "path": "../command-registry" + }, + { + "path": "../contracts" + }, + { + "path": "../device-selection" + }, + { + "path": "../host-kit" + }, + { + "path": "../kernel" + }, + { + "path": "../replay-test" + }, + { + "path": "../selectors" + }, + { + "path": "../session-journal" + }, + { + "path": "../test-kit" + }, + { + "path": "../plugins" + } ] } diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index e50157475c..331f2ad435 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -71,9 +71,6 @@ importers: '@agent-device/kernel': specifier: workspace:* version: link:packages/kernel - '@agent-device/maestro': - specifier: workspace:* - version: link:packages/maestro '@agent-device/platform-android': specifier: workspace:* version: link:packages/platform-android @@ -376,7 +373,7 @@ importers: version: link:../test-kit vitest: specifier: 5.0.4 - version: 5.0.4(@types/node@22.20.1)(@vitest/coverage-v8@5.0.4)(vite@8.3.5(@types/node@26.2.0)(yaml@2.9.1)) + version: 5.0.4(@types/node@22.20.1)(@vitest/coverage-v8@5.0.4)(vite@8.3.5(@types/node@22.20.1)(yaml@2.9.1)) packages/command-output: dependencies: @@ -491,7 +488,7 @@ importers: version: link:../test-kit vitest: specifier: 5.0.4 - version: 5.0.4(@types/node@22.20.1)(@vitest/coverage-v8@5.0.4)(vite@8.3.5(@types/node@26.2.0)(yaml@2.9.1)) + version: 5.0.4(@types/node@22.20.1)(@vitest/coverage-v8@5.0.4)(vite@8.3.5(@types/node@22.20.1)(yaml@2.9.1)) packages/contracts: dependencies: @@ -532,9 +529,6 @@ importers: '@agent-device/kernel': specifier: workspace:* version: link:../kernel - '@agent-device/maestro': - specifier: workspace:* - version: link:../maestro '@agent-device/platform-android': specifier: workspace:* version: link:../platform-android @@ -560,6 +554,9 @@ importers: '@agent-device/apple-runner': specifier: workspace:* version: link:../apple-runner + '@agent-device/maestro': + specifier: workspace:* + version: link:../maestro '@agent-device/proxy': specifier: workspace:* version: link:../proxy @@ -704,6 +701,9 @@ importers: '@agent-device/selectors': specifier: workspace:* version: link:../selectors + '@agent-device/test-kit': + specifier: workspace:* + version: link:../test-kit agent-device: specifier: workspace:* version: link:../.. @@ -1093,9 +1093,9 @@ importers: '@agent-device/kernel': specifier: workspace:* version: link:../kernel - '@agent-device/maestro': + '@agent-device/plugins': specifier: workspace:* - version: link:../maestro + version: link:../plugins '@agent-device/replay-test': specifier: workspace:* version: link:../replay-test @@ -5650,17 +5650,23 @@ packages: resolution: {integrity: sha512-9K2k72s4n7rV9s4bX0MyjbX9iBribvKZbBJKuEmTCZfeWJXs6Yh7bGpY4eoc7UufAjvpheBqwyZCOIPBvxCv0A==} peerDependencies: '@volar/language-service': ~2.4.0 + typescript: '*' peerDependenciesMeta: '@volar/language-service': optional: true + typescript: + optional: true volar-service-typescript@0.0.71: resolution: {integrity: sha512-yTtM/BVT6hoyEYnDtaCyAtNhdNeS/mhTTABlBOdw3NNiRBUin3IznFJpgfjer4c6RYopiPjjQjc9VFhxVl1mLw==} peerDependencies: '@volar/language-service': ~2.4.0 + typescript: '*' peerDependenciesMeta: '@volar/language-service': optional: true + typescript: + optional: true volar-service-yaml@0.0.71: resolution: {integrity: sha512-qYGWGuVpUTnZGu5P/CR4KLK4aIR8RrcVnmfZ2eRcj9q/I8VZCoC5yy9FtEvfNvnDp4MU17yhdJcvpQPIqhJS2Q==} @@ -5970,8 +5976,8 @@ snapshots: volar-service-emmet: 0.0.71(@volar/language-service@2.4.30(typescript@5.9.3)) volar-service-html: 0.0.71(@volar/language-service@2.4.30(typescript@5.9.3)) volar-service-prettier: 0.0.71(@volar/language-service@2.4.30(typescript@5.9.3))(prettier@3.9.10) - volar-service-typescript: 0.0.71(@volar/language-service@2.4.30(typescript@5.9.3)) - volar-service-typescript-twoslash-queries: 0.0.71(@volar/language-service@2.4.30(typescript@5.9.3)) + volar-service-typescript: 0.0.71(@volar/language-service@2.4.30(typescript@5.9.3))(typescript@5.9.3) + volar-service-typescript-twoslash-queries: 0.0.71(@volar/language-service@2.4.30(typescript@5.9.3))(typescript@5.9.3) volar-service-yaml: 0.0.71(@volar/language-service@2.4.30(typescript@5.9.3)) vscode-html-languageservice: 5.6.2 vscode-uri: 3.2.0 @@ -7483,14 +7489,14 @@ snapshots: optionalDependencies: vite: 8.3.5(@types/node@22.20.1)(yaml@2.9.0) - '@vitest/mocker@5.0.4(vite@8.3.5(@types/node@26.2.0)(yaml@2.9.1))': + '@vitest/mocker@5.0.4(vite@8.3.5(@types/node@22.20.1)(yaml@2.9.1))': dependencies: '@jridgewell/trace-mapping': 0.3.31 '@vitest/spy': 5.0.4 estree-walker: 3.0.3 magic-string: 1.4.3 optionalDependencies: - vite: 8.3.5(@types/node@26.2.0)(yaml@2.9.1) + vite: 8.3.5(@types/node@22.20.1)(yaml@2.9.1) '@vitest/spy@5.0.4': {} @@ -10101,18 +10107,6 @@ snapshots: fsevents: 2.3.3 yaml: 2.9.1 - vite@8.3.5(@types/node@26.2.0)(yaml@2.9.1): - dependencies: - lightningcss: 1.33.0 - picomatch: 4.0.7 - postcss: 8.5.23 - rolldown: 1.2.15 - tinyglobby: 0.2.17 - optionalDependencies: - '@types/node': 26.2.0 - fsevents: 2.3.3 - yaml: 2.9.1 - vitefu@1.1.3(vite@8.3.5(@types/node@22.20.1)(yaml@2.9.1)): optionalDependencies: vite: 8.3.5(@types/node@22.20.1)(yaml@2.9.1) @@ -10139,10 +10133,10 @@ snapshots: transitivePeerDependencies: - msw - vitest@5.0.4(@types/node@22.20.1)(@vitest/coverage-v8@5.0.4)(vite@8.3.5(@types/node@26.2.0)(yaml@2.9.1)): + vitest@5.0.4(@types/node@22.20.1)(@vitest/coverage-v8@5.0.4)(vite@8.3.5(@types/node@22.20.1)(yaml@2.9.1)): dependencies: '@types/chai': 5.2.3 - '@vitest/mocker': 5.0.4(vite@8.3.5(@types/node@26.2.0)(yaml@2.9.1)) + '@vitest/mocker': 5.0.4(vite@8.3.5(@types/node@22.20.1)(yaml@2.9.1)) chai: 6.3.0 es-module-lexer: 2.3.2 expect-type: 1.4.0 @@ -10153,7 +10147,7 @@ snapshots: tinybench: 6.2.2 tinyexec: 1.3.1 tinyglobby: 0.2.17 - vite: 8.3.5(@types/node@26.2.0)(yaml@2.9.1) + vite: 8.3.5(@types/node@22.20.1)(yaml@2.9.1) why-is-node-running: 3.2.1 optionalDependencies: '@types/node': 22.20.1 @@ -10193,13 +10187,14 @@ snapshots: '@volar/language-service': 2.4.30(typescript@5.9.3) prettier: 3.9.10 - volar-service-typescript-twoslash-queries@0.0.71(@volar/language-service@2.4.30(typescript@5.9.3)): + volar-service-typescript-twoslash-queries@0.0.71(@volar/language-service@2.4.30(typescript@5.9.3))(typescript@5.9.3): dependencies: vscode-uri: 3.2.0 optionalDependencies: '@volar/language-service': 2.4.30(typescript@5.9.3) + typescript: 5.9.3 - volar-service-typescript@0.0.71(@volar/language-service@2.4.30(typescript@5.9.3)): + volar-service-typescript@0.0.71(@volar/language-service@2.4.30(typescript@5.9.3))(typescript@5.9.3): dependencies: path-browserify: 1.0.1 semver: 7.8.5 @@ -10209,6 +10204,7 @@ snapshots: vscode-uri: 3.2.0 optionalDependencies: '@volar/language-service': 2.4.30(typescript@5.9.3) + typescript: 5.9.3 volar-service-yaml@0.0.71(@volar/language-service@2.4.30(typescript@5.9.3)): dependencies: diff --git a/scripts/build-entries.ts b/scripts/build-entries.ts index d97666fbaa..897ffccead 100644 --- a/scripts/build-entries.ts +++ b/scripts/build-entries.ts @@ -17,8 +17,6 @@ export const buildEntries = { 'internal/bin': 'src/bin.ts', 'internal/companion-tunnel': 'packages/react-native/src/companion/companion-tunnel.ts', 'internal/daemon': 'src/daemon.ts', - 'internal/run-script-child': 'packages/maestro/src/daemon-port/run-script-child.ts', - 'internal/run-script-http-child': 'packages/maestro/src/daemon-port/run-script-http-child.ts', 'internal/png-worker': 'packages/capture-kit/src/png-worker.ts', 'internal/update-check-entry': 'src/cli/update-check-entry.ts', } as const; diff --git a/scripts/fuzz/execute.ts b/scripts/fuzz/execute.ts index 385598c960..58a18f3827 100644 --- a/scripts/fuzz/execute.ts +++ b/scripts/fuzz/execute.ts @@ -187,11 +187,11 @@ export async function createCaseRunner( caseTimeoutMs: number, ): Promise { let session = await startSession(target.name); - let closed = false; + let closePromise: Promise | undefined; return { run: async (input) => { - if (closed) throw new Error('fuzz case runner is closed'); + if (closePromise) throw new Error('fuzz case runner is closed'); const { failure, usable } = await runCase(session, target, input, caseTimeoutMs); if (!usable) { await endSession(session); @@ -199,9 +199,21 @@ export async function createCaseRunner( } return failure; }, - close: async () => { - closed = true; - await endSession(session); + close: () => { + closePromise ??= (async () => { + try { + if (session.death() === null && session.child.connected) { + const answer = awaitWorker(session.child, caseTimeoutMs); + session.child.send({ kind: 'close' }); + const outcome = await answer; + if (outcome.kind !== 'message' || outcome.message.kind !== 'closed') + throw new Error(`fuzz target cleanup did not finish within ${caseTimeoutMs}ms`); + } + } finally { + await endSession(session); + } + })(); + return closePromise; }, }; } diff --git a/scripts/fuzz/harness.test.ts b/scripts/fuzz/harness.test.ts index c525064b8c..bc1f9245e8 100644 --- a/scripts/fuzz/harness.test.ts +++ b/scripts/fuzz/harness.test.ts @@ -14,7 +14,7 @@ import { fileURLToPath } from 'node:url'; import { describe, expect, it } from 'vitest'; import { LANE_ENVELOPE_SCHEMA_VERSION } from '@agent-device/test-kit'; import { CASE_GENERATION_INPUTS } from './envelope.ts'; -import { runCases } from './execute.ts'; +import { createCaseRunner, runCases } from './execute.ts'; import { checkCase } from './invariant.ts'; import { SELF_CHECK_TARGETS } from './self-check-targets.ts'; @@ -246,3 +246,32 @@ describe('artifact promotion', () => { fs.rmSync(dir, { recursive: true, force: true }); }); }); + +it.each([['--help'], ['--target', 'selector', '--iterations', '1']])( + 'fuzz entrypoint %j does not require an installed Maestro plugin', + (...args) => { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'fuzz-no-plugin-')); + try { + const result = runHarness([...args, '--artifact-dir', dir], { + AGENT_DEVICE_HOME: path.join(dir, 'empty-home'), + }); + expect(result.status).toBe(0); + } finally { + fs.rmSync(dir, { recursive: true, force: true }); + } + }, +); + +it('concurrent close waits for the worker cleanup and exit', async () => { + const runner = await createCaseRunner(targetNamed('self-check-untyped-throw'), 750); + let closed = false; + const first = runner.close().then(() => { + closed = true; + }); + try { + await runner.close(); + expect(closed).toBe(true); + } finally { + await first; + } +}); diff --git a/scripts/fuzz/target-types.ts b/scripts/fuzz/target-types.ts index 08542cdfde..e84faa4c36 100644 --- a/scripts/fuzz/target-types.ts +++ b/scripts/fuzz/target-types.ts @@ -50,6 +50,8 @@ export type FuzzFailure = { export type FuzzTarget = { name: FuzzTargetName; + /** Opens optional parser services for this target and returns their cleanup. */ + prepare?: () => Promise<() => Promise>; /** Human-readable description used in failure reports. */ description: string; /** Runs the parser on one case; may throw. */ diff --git a/scripts/fuzz/targets.ts b/scripts/fuzz/targets.ts index 75f9019076..d222bbd0a5 100644 --- a/scripts/fuzz/targets.ts +++ b/scripts/fuzz/targets.ts @@ -11,8 +11,19 @@ import { parseArgs } from '../../src/cli/parser/args.ts'; import { validateSelectorExpression } from '@agent-device/selectors'; import { parseReplayScriptDetailed } from '@agent-device/ad-script'; import { readCliBatchStepsJson } from '@agent-device/command-surface'; -import { inspectMaestroFlow } from '@agent-device/maestro'; +import { openReplayBackend } from '@agent-device/replay-port/replay-backend-registry'; + import type { FuzzTarget } from './target-types.ts'; +import type { ReplayBackend } from '@agent-device/contracts/replay-backend'; + +let inspectMaestroFlow: ReplayBackend['inspectSource']; +async function prepareMaestro(): Promise<() => Promise> { + const scope = await openReplayBackend('maestro', process.env); + inspectMaestroFlow = scope.backend.inspectSource; + return async () => { + await scope.close(); + }; +} import { acceptCase, makeValidationCheck, rejectCase } from './validation-case.ts'; // argv is carried as one string so a case (and its corpus entry, artifact, and repro @@ -123,6 +134,7 @@ export const FUZZ_TARGETS: readonly FuzzTarget[] = [ // Same contract for Maestro flows: shape-valid YAML with one planted violation, so the // failure surfaces in the command-shape validation behind the YAML tokenizer. name: 'maestro-validation', + prepare: prepareMaestro, description: 'inspectMaestroFlow via shape-derived flows with expected outcomes', run: (input) => void inspectMaestroFlow(JSON.parse(input).payload as string, 'fuzz.yaml'), check: makeValidationCheck( @@ -137,6 +149,7 @@ export const FUZZ_TARGETS: readonly FuzzTarget[] = [ }, { name: 'maestro', + prepare: prepareMaestro, description: 'parseMaestroProgram (Maestro compat)', run: (input) => void inspectMaestroFlow(input, 'fuzz.yaml'), seeds: [ diff --git a/scripts/fuzz/validation-arbitraries-maestro.test.ts b/scripts/fuzz/validation-arbitraries-maestro.test.ts index 68f2d18c52..f52ec51160 100644 --- a/scripts/fuzz/validation-arbitraries-maestro.test.ts +++ b/scripts/fuzz/validation-arbitraries-maestro.test.ts @@ -7,7 +7,7 @@ // than by the YAML tokenizer. import fc from 'fast-check'; -import { describe, expect, it } from 'vitest'; +import { afterAll, beforeAll, describe, expect, it } from 'vitest'; import { SUPPORTED_MAESTRO_COMMAND_NAMES } from '../../packages/maestro/src/internal/program-ir-command-parser.ts'; import { getFuzzTarget } from './registry.ts'; import { decodeValidationCase } from './validation-case.ts'; @@ -18,6 +18,13 @@ const SAMPLE_SIZE = 3_000; const SEED = 1; const target = getFuzzTarget('maestro-validation'); +let dispose: (() => Promise) | undefined; +beforeAll(async () => { + dispose = await target.prepare?.(); +}); +afterAll(async () => { + await dispose?.(); +}); const sample = fc.sample(maestroValidationArb, { numRuns: SAMPLE_SIZE, seed: SEED }); describeGeneratorContract({ diff --git a/scripts/fuzz/worker.ts b/scripts/fuzz/worker.ts index c642339572..46923b8c3b 100644 --- a/scripts/fuzz/worker.ts +++ b/scripts/fuzz/worker.ts @@ -11,14 +11,18 @@ import { beginStderrCapture } from '@agent-device/test-kit'; import { checkCase, type FuzzFailure } from './invariant.ts'; import { getFuzzTarget } from './registry.ts'; -export type FuzzWorkerRequest = { kind: 'case'; input: string }; +export type FuzzWorkerRequest = { kind: 'case'; input: string } | { kind: 'close' }; -export type FuzzWorkerMessage = { kind: 'ready' } | { kind: 'result'; failure: FuzzFailure | null }; +export type FuzzWorkerMessage = + | { kind: 'ready' } + | { kind: 'closed' } + | { kind: 'result'; failure: FuzzFailure | null }; const send = process.send?.bind(process); if (!send) throw new Error('scripts/fuzz/worker.ts must be run as a forked child process.'); const target = getFuzzTarget(process.argv[2] ?? ''); +const dispose = await target.prepare?.(); // The batch-steps parser warns on deprecated step shapes; a fuzz run would emit thousands of // those lines and bury the failure report. Only JavaScript writes go through here: a fatal @@ -26,7 +30,12 @@ const target = getFuzzTarget(process.argv[2] ?? ''); // piped rather than discarded. beginStderrCapture({ discard: true }); -process.on('message', (request: FuzzWorkerRequest) => { +process.on('message', async (request: FuzzWorkerRequest) => { + if (request.kind === 'close') { + await dispose?.(); + send({ kind: 'closed' } satisfies FuzzWorkerMessage); + return; + } const failure = checkCase(target, request.input); send({ kind: 'result', failure } satisfies FuzzWorkerMessage); }); diff --git a/scripts/install-workspace-maestro.ts b/scripts/install-workspace-maestro.ts new file mode 100644 index 0000000000..b972a2c0db --- /dev/null +++ b/scripts/install-workspace-maestro.ts @@ -0,0 +1,47 @@ +import crypto from 'node:crypto'; +import fs from 'node:fs'; +import path from 'node:path'; +import { resolveUserConfigPath } from '@agent-device/host-kit/file'; + +/** Installs the explicitly selected workspace build into an isolated validation plugin store. */ +export function installWorkspaceMaestro(env: NodeJS.ProcessEnv, source: boolean): void { + if (!env.AGENT_DEVICE_HOME) + throw new Error('Workspace plugin installation requires AGENT_DEVICE_HOME'); + const configPath = resolveUserConfigPath(env); + const home = path.dirname(configPath); + const installation = crypto.randomUUID(); + const directory = path.join( + home, + 'plugins', + installation, + 'node_modules', + '@agent-device/maestro', + ); + const workspace = path.resolve(import.meta.dirname, '../packages/maestro'); + fs.mkdirSync(path.dirname(directory), { recursive: true }); + if (source) fs.symlinkSync(workspace, directory, 'dir'); + else { + fs.mkdirSync(directory); + fs.cpSync(path.join(workspace, 'dist'), path.join(directory, 'dist'), { recursive: true }); + const manifest = JSON.parse(fs.readFileSync(path.join(workspace, 'package.json'), 'utf8')); + fs.writeFileSync( + path.join(directory, 'package.json'), + JSON.stringify({ + name: manifest.name, + version: manifest.version, + type: 'module', + exports: manifest.publishConfig.exports, + }), + ); + } + const config = fs.existsSync(configPath) ? JSON.parse(fs.readFileSync(configPath, 'utf8')) : {}; + fs.writeFileSync( + configPath, + JSON.stringify({ + ...config, + plugins: { ...config.plugins, '@agent-device/maestro': { installation } }, + }), + ); +} + +if (process.argv[1] === import.meta.filename) installWorkspaceMaestro(process.env, false); diff --git a/scripts/layering/architecture-ownership.ts b/scripts/layering/architecture-ownership.ts index bbfecadd20..ed660ad925 100644 --- a/scripts/layering/architecture-ownership.ts +++ b/scripts/layering/architecture-ownership.ts @@ -176,7 +176,12 @@ export const LOGICAL_MODULE_POLICIES = [ { name: 'replay-port', roots: ['packages/replay-port/src/'], - forbiddenTargetRoots: [DAEMON_SOURCE_ROOT, 'src/providers/', 'src/compat/'], + forbiddenTargetRoots: [ + DAEMON_SOURCE_ROOT, + 'src/providers/', + 'src/compat/', + 'packages/maestro/', + ], }, { name: 'daemon-session-lifecycle', diff --git a/scripts/layering/package-boundaries.test.ts b/scripts/layering/package-boundaries.test.ts index 063267faaf..eace49cc44 100644 --- a/scripts/layering/package-boundaries.test.ts +++ b/scripts/layering/package-boundaries.test.ts @@ -679,8 +679,6 @@ test('the real tree parses, declares, and passes R11', () => { assert.deepEqual([...maestroPackage.exportTargets.keys()].sort(), [ '@agent-device/maestro', '@agent-device/maestro/agent-device', - '@agent-device/maestro/replay-backend', - '@agent-device/maestro/replay-backend-declaration', '@agent-device/maestro/run-script-http', ]); assert.deepEqual([...maestroPackage.workspaceDependencies].sort(), [ @@ -691,6 +689,7 @@ test('the real tree parses, declares, and passes R11', () => { '@agent-device/kernel', '@agent-device/provision-kit', '@agent-device/selectors', + '@agent-device/test-kit', 'agent-device', ]); const adScriptPackage = packages.find((pkg) => pkg.name === '@agent-device/ad-script'); @@ -898,8 +897,8 @@ test('the real tree parses, declares, and passes R11', () => { 'root must declare the contracts workspace dependency', ); assert.ok( - rootWorkspaceDependencyNames(repoRoot).has('@agent-device/maestro'), - 'root must declare the maestro workspace dependency', + !rootWorkspaceDependencyNames(repoRoot).has('@agent-device/maestro'), + 'core must not depend on the optional Maestro plugin', ); assert.ok( rootWorkspaceDependencyNames(repoRoot).has('@agent-device/ad-script'), diff --git a/scripts/layering/project-references.test.ts b/scripts/layering/project-references.test.ts index e6836b5380..76e8949e58 100644 --- a/scripts/layering/project-references.test.ts +++ b/scripts/layering/project-references.test.ts @@ -115,7 +115,7 @@ test('the compiled-set predicate is licensed by the tsconfig include/exclude sha } }); -test('the root manifest declares every workspace package as a workspace dependency', () => { +test('root workspace dependencies cover every non-Maestro package and exclude the optional engine', () => { const manifest = JSON.parse(fs.readFileSync(path.join(repoRoot, 'package.json'), 'utf8')) as { dependencies?: Record; devDependencies?: Record; @@ -128,6 +128,7 @@ test('the root manifest declares every workspace package as a workspace dependen .map(([name]) => name) .sort(); const expected = readWorkspacePackages(repoRoot) + .filter((pkg) => pkg.name !== '@agent-device/maestro') .map((pkg) => pkg.name) .sort(); assert.deepEqual( @@ -137,6 +138,10 @@ test('the root manifest declares every workspace package as a workspace dependen 'manifest (its tsconfig references already name it; the root project resolves package ' + 'specifiers through these declarations)', ); + assert.ok( + !declared.includes('@agent-device/maestro'), + 'The optional engine must be installed through the plugin store. R11 rejects undeclared imports from core.', + ); }); test('every package tsconfig references exactly its manifest workspace dependencies', () => { diff --git a/scripts/vitest-tmpdir-global-setup.ts b/scripts/vitest-tmpdir-global-setup.ts index 48e4192398..09577bc850 100644 --- a/scripts/vitest-tmpdir-global-setup.ts +++ b/scripts/vitest-tmpdir-global-setup.ts @@ -1,3 +1,4 @@ +import { installWorkspaceMaestro } from './install-workspace-maestro.ts'; import fs from 'node:fs'; import os from 'node:os'; import path from 'node:path'; @@ -25,6 +26,7 @@ import { let testRunTmpDir: string; let previousTmpDir: string | undefined; +let previousPluginHome: string | undefined; let previousSwiftCacheDir: string | undefined; // setup/teardown are vitest's globalSetup contract: it imports this file by @@ -53,6 +55,9 @@ export function setup(): void { path.join(TEST_RUN_TMP_ROOT, `${TEST_RUN_TMP_PREFIX}${process.pid}-`), ); process.env.TMPDIR = testRunTmpDir; + previousPluginHome = process.env.AGENT_DEVICE_HOME; + process.env.AGENT_DEVICE_HOME = path.join(testRunTmpDir, 'plugin-home'); + installWorkspaceMaestro(process.env, true); } // fallow-ignore-next-line unused-export @@ -63,6 +68,8 @@ export async function teardown(): Promise { await stopRunDirectoryConsumers(testRunTmpDir), ); } finally { + if (previousPluginHome === undefined) delete process.env.AGENT_DEVICE_HOME; + else process.env.AGENT_DEVICE_HOME = previousPluginHome; fs.rmSync(testRunTmpDir, { recursive: true, force: true }); restoreEnv('TMPDIR', previousTmpDir); restoreEnv('AGENT_DEVICE_SWIFT_CACHE_DIR', previousSwiftCacheDir); diff --git a/src/__tests__/cli-maestro-suite.test.ts b/src/__tests__/cli-maestro-suite.test.ts index 542a33fa6e..2a524637a1 100644 --- a/src/__tests__/cli-maestro-suite.test.ts +++ b/src/__tests__/cli-maestro-suite.test.ts @@ -1,4 +1,5 @@ import { test } from 'vitest'; +import { installedMaestroCliTestEnv } from './cli-test-env.ts'; import assert from 'node:assert/strict'; import { promises as fs } from 'node:fs'; import path from 'node:path'; @@ -29,7 +30,7 @@ test('test --maestro forwards Maestro backend and platform for directory suites' tests: [], }, }), - { cwd: tmpDir }, + { cwd: tmpDir, env: installedMaestroCliTestEnv() }, ); assert.equal(result.code, null); diff --git a/src/__tests__/cli-replay-script-sources.test.ts b/src/__tests__/cli-replay-script-sources.test.ts index 7a28590105..42b2c4fc48 100644 --- a/src/__tests__/cli-replay-script-sources.test.ts +++ b/src/__tests__/cli-replay-script-sources.test.ts @@ -9,6 +9,7 @@ import path from 'node:path'; import assert from 'node:assert/strict'; import { test } from 'vitest'; import type { ReplayScriptSourceBundle } from '@agent-device/contracts/replay'; +import { installedMaestroCliTestEnv } from './cli-test-env.ts'; import { runCliCapture } from './cli-capture.ts'; import { mkdtempForTestSync } from '@agent-device/test-kit'; @@ -83,7 +84,11 @@ test('a Maestro replay against a remote daemon sends its runFlow includes too', const result = await runCliCapture( ['replay', './flows/checkout.yaml', '--maestro', '--platform', 'ios'], - { cwd, env: REMOTE_DAEMON_ENV, defaultResponse: { ok: true, data: { replayed: 1 } } }, + { + cwd, + env: { ...REMOTE_DAEMON_ENV, ...installedMaestroCliTestEnv() }, + defaultResponse: { ok: true, data: { replayed: 1 } }, + }, ); const bundle = result.calls[0]?.flags?.replayScriptSource as ReplayScriptSourceBundle | undefined; @@ -110,7 +115,7 @@ test.each(['native', 'maestro'])( ['test', maestro ? './*.yaml' : './*.ad', '--json', ...(maestro ? ['--maestro'] : [])], { cwd, - env: REMOTE_DAEMON_ENV, + env: { ...REMOTE_DAEMON_ENV, ...(maestro ? installedMaestroCliTestEnv() : {}) }, defaultResponse: { ok: true, data: { tests: [], total: 0, passed: 0, failed: 0 } }, }, ); diff --git a/src/__tests__/cli-test-env.ts b/src/__tests__/cli-test-env.ts index 188ea44d18..f2493c9099 100644 --- a/src/__tests__/cli-test-env.ts +++ b/src/__tests__/cli-test-env.ts @@ -1,3 +1,4 @@ +import { installWorkspaceMaestro } from '../../scripts/install-workspace-maestro.ts'; import fs from 'node:fs'; import path from 'node:path'; import { mkdtempForTestSync } from '@agent-device/test-kit'; @@ -44,3 +45,9 @@ export function installIsolatedCliTestEnv( fs.rmSync(tempRoot, { recursive: true, force: true }); }; } + +export function installedMaestroCliTestEnv(): Record { + const env = { AGENT_DEVICE_HOME: mkdtempForTestSync('agent-device-cli-maestro-') }; + installWorkspaceMaestro(env, true); + return env; +} diff --git a/src/__tests__/replay-plugin-setup.ts b/src/__tests__/replay-plugin-setup.ts new file mode 100644 index 0000000000..b59846bbf5 --- /dev/null +++ b/src/__tests__/replay-plugin-setup.ts @@ -0,0 +1,34 @@ +import { afterEach, beforeEach, vi } from 'vitest'; + +let registry: Awaited< + ReturnType +>; +beforeEach(async () => { + const [ + { bindReplayBackendResolver }, + { resolveProviderPlugins, startProviderRegistry }, + { storePluginCandidates }, + ] = await Promise.all([ + vi.importActual( + '@agent-device/replay-port/replay-backend-registry', + ), + vi.importActual( + '@agent-device/plugins/registry', + ), + vi.importActual('@agent-device/plugins/store'), + ]); + registry = await startProviderRegistry( + await resolveProviderPlugins(await storePluginCandidates(process.env)), + process.env, + { + adaptWebDriver: async () => { + throw new Error('Replay test setup cannot start providers'); + }, + }, + 'replay-backends', + ); + await bindReplayBackendResolver(registry.resolveReplayBackend); +}); +afterEach(async () => { + await registry?.close(); +}); diff --git a/src/__tests__/test-utils/replay-script-source.fixtures.ts b/src/__tests__/test-utils/replay-script-source.fixtures.ts index 322419ca41..cbd10585fe 100644 --- a/src/__tests__/test-utils/replay-script-source.fixtures.ts +++ b/src/__tests__/test-utils/replay-script-source.fixtures.ts @@ -26,6 +26,7 @@ export async function withClientReplayScriptSources( if (req.command === 'replay') { if (req.flags?.replayScriptSource) return req; const replayScriptSource = await loadReplayScriptSourceBundle({ + hostEnv: process.env, inputPath: inputs[0], cwd, replayBackend, @@ -35,7 +36,13 @@ export async function withClientReplayScriptSources( if (req.flags?.replayScriptSources) return req; const replayScriptSources = await Promise.all( discoverReplaySourcePaths({ inputs, cwd, replayBackend }).map( - async (inputPath) => await loadReplayScriptSourceBundle({ inputPath, cwd, replayBackend }), + async (inputPath) => + await loadReplayScriptSourceBundle({ + hostEnv: process.env, + inputPath, + cwd, + replayBackend, + }), ), ); return withFlags({ replayScriptSources }); diff --git a/src/cli/commands/__tests__/replay-maestro-export.test.ts b/src/cli/commands/__tests__/replay-maestro-export.test.ts index 363b867dab..911fa3b27e 100644 --- a/src/cli/commands/__tests__/replay-maestro-export.test.ts +++ b/src/cli/commands/__tests__/replay-maestro-export.test.ts @@ -1,8 +1,8 @@ +import { installedMaestroCliTestEnv } from '../../../__tests__/cli-test-env.ts'; import fs from 'node:fs'; import path from 'node:path'; import { parseAllDocuments } from 'yaml'; import { describe, expect, test } from 'vitest'; -import { inspectMaestroFlow } from '@agent-device/maestro'; import { runCliCapture } from '../../../__tests__/cli-capture.ts'; import { mkdtempForTestSync } from '@agent-device/test-kit'; @@ -19,7 +19,8 @@ describe('exportReplayScriptToMaestro', () => { const open = withApp ? `com.example.app ${link} --relaunch` : link; fs.writeFileSync(sourcePath, `open ${open}\n`); - const result = await runCliCapture(['replay', 'export', sourcePath, '--json']); + const env = installedMaestroCliTestEnv(); + const result = await runCliCapture(['replay', 'export', sourcePath, '--json'], { env }); expect(result.code).toBeNull(); expect(result.calls).toEqual([]); @@ -37,9 +38,10 @@ describe('exportReplayScriptToMaestro', () => { ] : [[{ openLink: link }]], ); - expect(() => inspectMaestroFlow(output.data.yaml, 'flow.yaml')).not.toThrow(); - const written = await runCliCapture(['replay', 'export', sourcePath, '--out', outPath]); + const written = await runCliCapture(['replay', 'export', sourcePath, '--out', outPath], { + env, + }); expect(written).toMatchObject({ code: null, calls: [], stderr: '', stdout: `${outPath}\n` }); expect(fs.readFileSync(outPath, 'utf8')).toBe(output.data.yaml); diff --git a/src/cli/commands/daemon.ts b/src/cli/commands/daemon.ts index 5ffc31c066..3dbbf37c67 100644 --- a/src/cli/commands/daemon.ts +++ b/src/cli/commands/daemon.ts @@ -135,7 +135,7 @@ function renderDaemonStatus( plugin.state === 'active' ? 'active' : `failed: ${plugin.error?.message ?? 'unknown'}`; - return ` ${plugin.provider ?? '(unknown provider)'} [${plugin.source}] ${origin} - ${outcome}`; + return ` ${plugin.provider ?? (plugin.backend ? `replay backend ${plugin.backend}` : '(unknown plugin)')} [${plugin.source}] ${origin} - ${outcome}`; }), ]), ].join('\n'); diff --git a/src/cli/commands/plugins.ts b/src/cli/commands/plugins.ts index 9b40e6d65a..dea8b696f6 100644 --- a/src/cli/commands/plugins.ts +++ b/src/cli/commands/plugins.ts @@ -41,7 +41,7 @@ export const pluginsCommand: ClientCommandHandler = async ({ positionals, flags [ ...plugins.map((plugin) => plugin.compatible - ? `${plugin.name}@${plugin.version} (${plugin.provider})` + ? `${plugin.name}@${plugin.version} (${plugin.provider ?? plugin.backend})` : plugin.name === undefined ? `Plugin config unavailable: ${plugin.error?.message}` : `${plugin.name}: unavailable; run plugins update ${plugin.name} or plugins remove ${plugin.name}`, diff --git a/src/cli/commands/replay.ts b/src/cli/commands/replay.ts index 96d8caf968..0002cf2bc7 100644 --- a/src/cli/commands/replay.ts +++ b/src/cli/commands/replay.ts @@ -3,7 +3,6 @@ import path from 'node:path'; import { AppError } from '@agent-device/kernel/errors'; import { parseReplayScriptDetailed, readReplayScriptMetadata } from '@agent-device/ad-script'; import { resolveUserPath } from '@agent-device/host-kit/file'; -import type { ReplayBackend } from '@agent-device/contracts/replay-backend'; import { writeCommandOutput } from './shared.ts'; import type { ClientCommandHandler } from './router-types.ts'; @@ -44,13 +43,14 @@ async function handleReplayExportCommand({ // `replay export` exports to Maestro, so the conversion goes through that backend's registry // entry rather than naming the engine package from the CLI (#3377). This handler is loaded // lazily by the router, and the engine sits one further thunk behind the backend. - const backend: ReplayBackend = await ( + const result = await ( await import('@agent-device/replay-port/replay-backend-registry') - ).getReplayBackend('maestro'); - const result = backend.exportReplayScript(parsed.actions, { - actionLines: parsed.actionLines, - metadata: readReplayScriptMetadata(script), - }); + ).withReplayBackend('maestro', host.env, (backend) => + backend.exportReplayScript(parsed.actions, { + actionLines: parsed.actionLines, + metadata: readReplayScriptMetadata(script), + }), + ); const outputPath = typeof flags.out === 'string' ? resolveUserPath(flags.out, { cwd: host.cwd }) : undefined; if (outputPath) { @@ -64,7 +64,7 @@ async function handleReplayExportCommand({ await writeCommandOutput( flags, { - format: backend.id, + format: 'maestro', sourcePath, ...(outputPath ? { path: outputPath } : { yaml: result.yaml }), warnings: result.warnings, diff --git a/src/cli/parser/__tests__/args-maestro-support-matrix.test.ts b/src/cli/parser/__tests__/args-maestro-support-matrix.test.ts index c84c5b0ec2..ce4af0cbfc 100644 --- a/src/cli/parser/__tests__/args-maestro-support-matrix.test.ts +++ b/src/cli/parser/__tests__/args-maestro-support-matrix.test.ts @@ -5,7 +5,7 @@ import { getFlagDefinitions } from '@agent-device/command-registry/flag-registry import { MAESTRO_COMPAT_LIMITATIONS, MAESTRO_COMPAT_SUPPORTED_CAPABILITIES, -} from '@agent-device/maestro'; +} from '@agent-device/replay-port/replay-backend-declarations'; const MAESTRO_COMPATIBILITY_ADR_URL = 'https://github.com/callstack/agent-device/blob/main/docs/adr/0015-direct-maestro-engine.md'; diff --git a/src/daemon-composition.test.ts b/src/daemon-composition.test.ts index 90115b19bb..9f66551bf0 100644 --- a/src/daemon-composition.test.ts +++ b/src/daemon-composition.test.ts @@ -47,6 +47,9 @@ vi.mock('./provider-device-runtimes.ts', () => ({ platformModules: [], shutdown: ports.shutdownProviders, inventory: [], + resolveReplayBackend: async () => { + throw new Error('no replay backend'); + }, }), })); diff --git a/src/daemon-composition.ts b/src/daemon-composition.ts index 798d90f016..d42f91ad9b 100644 --- a/src/daemon-composition.ts +++ b/src/daemon-composition.ts @@ -90,7 +90,11 @@ export const createDaemonRuntimeDeps: ComposeDaemonRuntimeDeps = async ( export async function composeDaemonRuntimeDeps( input: Parameters[0], providerCandidates?: readonly PluginCandidate[], -): Promise { +): Promise< + DaemonRuntimeDeps & { + resolveReplayBackend: import('@agent-device/contracts/replay-backend').ReplayBackendResolver; + } +> { const { host, baseDir, @@ -114,6 +118,9 @@ export async function composeDaemonRuntimeDeps( // The registry has set up every plugin. Until this function returns, nothing else owns its // cleanup, so any later failure closes it here; success transfers ownership to the daemon. try { + const { bindReplayBackendResolver } = + await import('@agent-device/replay-port/replay-backend-registry'); + await bindReplayBackendResolver(providerComposition.resolveReplayBackend); const providerDeviceRuntimes = [...providerComposition.runtimes]; const deviceRuntimeGateway = createPlatformRuntimeGateway({ host, @@ -162,6 +169,7 @@ export async function composeDaemonRuntimeDeps( ownerLifecycle: createPlatformDaemonLifecycleOwners(host), shutdownProviderRuntimes: providerComposition.shutdown, pluginInventory: providerComposition.inventory, + resolveReplayBackend: providerComposition.resolveReplayBackend, leaseLifecycleProvider: providerRuntimeProviders.leaseLifecycleProvider, providerRuntimeIds: providerRuntimeProviders.providerRuntimeIds, recoverExpiredLease: providerRuntimeProviders.recoverExpiredLease, diff --git a/src/host-replay.test.ts b/src/host-replay.test.ts new file mode 100644 index 0000000000..81c317a697 --- /dev/null +++ b/src/host-replay.test.ts @@ -0,0 +1,277 @@ +import crypto from 'node:crypto'; +import fs from 'node:fs'; +import path from 'node:path'; +import { expect, test, vi } from 'vitest'; +import { prepareDaemonCommandRequest } from '@agent-device/command-surface'; +import { createAgentDeviceClient } from '@agent-device/client'; +import type { ReplayEnginePlugin } from '@agent-device/plugins/replay-backend'; +import { resolveProviderPlugins } from '@agent-device/plugins/registry'; +import { storePluginCandidates } from '@agent-device/plugins/store'; +import { createRuntimeStateHandle } from '@agent-device/host-kit/runtime-slots'; +import { createTestHostEnvironment, mkdtempForTestSync } from '@agent-device/test-kit'; +import { startAgentDevice } from './host.ts'; + +async function trackSourceRegistries() { + const pluginRegistry = await import('@agent-device/plugins/registry'); + const original = pluginRegistry.startProviderRegistry; + const closed = vi.fn(); + const started = vi.spyOn(pluginRegistry, 'startProviderRegistry'); + started.mockImplementation(async (...args) => { + const registry = await original(...args); + return { + ...registry, + close: async () => { + closed(); + return await registry.close(); + }, + }; + }); + return { started, closed }; +} + +test('a multi-flow Maestro suite collects real includes in one disposable client scope', async () => { + const root = mkdtempForTestSync('maestro-suite-sources-'); + const child = path.join(root, 'child.yaml'); + const entries = ['first.yaml', 'second.yaml'].map((name) => path.join(root, name)); + const flow = 'appId: demo.app\n---\n- runFlow: child.yaml\n'; + const included = 'appId: demo.app\n---\n- back\n'; + for (const entry of entries) fs.writeFileSync(entry, flow); + fs.writeFileSync(child, included); + const scope = await trackSourceRegistries(); + try { + const result = await prepareDaemonCommandRequest( + 'test', + { paths: entries, backend: 'maestro' }, + { env: process.env, cwd: root }, + ); + expect(result.options.replayScriptSources).toEqual( + entries.map((entry) => ({ entry, files: { [entry]: flow, [child]: included } })), + ); + expect(scope.started).toHaveBeenCalledTimes(1); + expect(scope.closed).toHaveBeenCalledTimes(1); + } finally { + scope.started.mockRestore(); + } +}); + +test('a failed Maestro source bundle closes its client registry', async () => { + const root = mkdtempForTestSync('maestro-suite-large-source-'); + const entry = path.join(root, 'large.yaml'); + fs.writeFileSync(entry, `# ${'x'.repeat(2 * 1024 * 1024)}\n`); + const scope = await trackSourceRegistries(); + try { + await expect( + prepareDaemonCommandRequest( + 'test', + { paths: [entry], backend: 'maestro' }, + { env: process.env, cwd: root }, + ), + ).rejects.toMatchObject({ code: 'INVALID_ARGS', details: { limitBytes: 2 * 1024 * 1024 } }); + expect(scope.started).toHaveBeenCalledTimes(1); + expect(scope.closed).toHaveBeenCalledTimes(1); + } finally { + scope.started.mockRestore(); + } +}); + +test.each([undefined, 'maestro'])( + 'native-only source collection with backend %s never starts a plugin registry', + async (backend) => { + const root = mkdtempForTestSync('native-suite-sources-'); + const entry = path.join(root, 'suite.ad'); + const script = 'open demo.app\nclose\n'; + fs.writeFileSync(entry, script); + const scope = await trackSourceRegistries(); + try { + const result = await prepareDaemonCommandRequest( + 'test', + { paths: [entry], backend }, + { env: { AGENT_DEVICE_HOME: path.join(root, 'empty-home') }, cwd: root }, + ); + expect(result.options.replayScriptSources).toEqual([{ entry, files: { [entry]: script } }]); + expect(scope.started).not.toHaveBeenCalled(); + expect(scope.closed).not.toHaveBeenCalled(); + } finally { + scope.started.mockRestore(); + } + }, +); + +function fixture() { + const root = mkdtempForTestSync('embedded-replay-'); + const entry = path.join(root, 'flow.yaml'); + fs.writeFileSync(entry, 'appId: com.example.app\n---\n- runFlow: child.yaml\n'); + fs.writeFileSync( + path.join(root, 'child.yaml'), + 'appId: com.example.app\n---\n- evalScript: ${output.value = "ok"}\n', + ); + const host = createTestHostEnvironment({ + env: { + AGENT_DEVICE_HOME: path.join(root, 'home'), + AGENT_DEVICE_DAEMON_IDLE_TIMEOUT_MS: '0', + }, + }); + return { root, entry, host, stateDir: path.join(root, 'state') }; +} + +async function trackedPlugin() { + const resolved = await resolveProviderPlugins(await storePluginCandidates(process.env)); + const maestro = resolved.slots.find((slot) => slot.backend === 'maestro')?.replayPlugin; + if (!maestro) throw new Error('Maestro test installation is missing'); + const calls = { setup: 0, create: 0, collect: 0, run: 0, dispose: 0 }; + const plugin: ReplayEnginePlugin = { + ...maestro, + setup: async (host) => { + calls.setup += 1; + const setup = await maestro.setup(host); + return { + replayBackend: { + create: async (host) => { + calls.create += 1; + const backend = await setup.replayBackend.create(host); + return { + ...backend, + collectSourceFiles: (params) => { + calls.collect += 1; + return backend.collectSourceFiles(params); + }, + runReplay: async (command) => { + calls.run += 1; + return await backend.runReplay(command); + }, + }; + }, + }, + dispose: async () => { + calls.dispose += 1; + await setup.dispose?.(); + }, + }; + }, + }; + return { plugin, calls }; +} + +test.each(['replay', 'test', 'batch-test'] as const)( + 'the embedded client prepares %s includes with one runtime registry and an empty store', + async (command) => { + const state = createRuntimeStateHandle(); + try { + await state.run(async () => { + const f = fixture(); + const { plugin, calls } = await trackedPlugin(); + const host = await startAgentDevice({ ...f, plugins: [plugin] }); + const input = { path: f.entry, platform: 'ios' as const, backend: 'maestro' }; + const testInput = { paths: [f.entry], platform: 'ios' as const, backend: 'maestro' }; + try { + expect(JSON.parse(JSON.stringify(host.clientConfig))).toEqual(host.clientConfig); + if (command === 'replay') { + await expect(host.client.replay.run(input)).resolves.toMatchObject({ replayed: 1 }); + } else if (command === 'test') { + await expect(host.client.replay.test(testInput)).resolves.toMatchObject({ + passed: 1, + failed: 0, + }); + } else { + await expect( + host.client.batch.run({ steps: [{ command: 'test', input: testInput }] }), + ).resolves.toMatchObject({ + executed: 1, + results: [{ command: 'test', ok: true, data: { passed: 1, failed: 0 } }], + }); + } + expect(calls).toEqual({ setup: 1, create: 1, collect: 1, run: 1, dispose: 0 }); + } finally { + await host.close(); + } + expect(calls.dispose).toBe(1); + }); + } finally { + state.dispose(); + } + }, +); + +test('isolated embedded clients ignore a conflicting store and never borrow another runtime backend', async () => { + const f = fixture(); + const installation = crypto.randomUUID(); + const store = path.join( + f.host.env.AGENT_DEVICE_HOME!, + 'plugins', + installation, + 'node_modules', + '@agent-device/maestro', + ); + fs.mkdirSync(store, { recursive: true }); + fs.writeFileSync( + path.join(store, 'package.json'), + JSON.stringify({ + name: '@agent-device/maestro', + version: '0.0.0', + type: 'module', + exports: { './agent-device': './plugin.mjs' }, + }), + ); + fs.writeFileSync( + path.join(store, 'plugin.mjs'), + 'throw new Error("conflicting store must not be loaded by embedded client");', + ); + fs.writeFileSync( + path.join(f.host.env.AGENT_DEVICE_HOME!, 'config.json'), + JSON.stringify({ plugins: { '@agent-device/maestro': { installation } } }), + ); + const firstPlugin = await trackedPlugin(); + const secondPlugin = await trackedPlugin(); + const first = await startAgentDevice({ ...f, plugins: [firstPlugin.plugin] }); + let second: Awaited> | undefined; + try { + second = await startAgentDevice({ + ...f, + stateDir: path.join(f.root, 'second'), + plugins: [secondPlugin.plugin], + }); + await Promise.all( + [first, second].map(async ({ client }) => { + await expect( + client.replay.run({ path: f.entry, backend: 'maestro', platform: 'ios' }), + ).resolves.toMatchObject({ replayed: 1 }); + }), + ); + for (const { calls } of [firstPlugin, secondPlugin]) { + expect(calls).toEqual({ setup: 1, create: 1, collect: 1, run: 1, dispose: 0 }); + } + // An outside attacher retains its own caller-side installation requirement. + const outside = createAgentDeviceClient(first.clientConfig, { host: f.host }); + await expect(outside.replay.run({ path: f.entry, backend: 'maestro' })).rejects.toMatchObject({ + details: { reason: 'plugin_load_failed' }, + }); + await first.close(); + await expect( + second.client.replay.run({ path: f.entry, backend: 'maestro', platform: 'ios' }), + ).resolves.toMatchObject({ replayed: 1 }); + expect(firstPlugin.calls.dispose).toBe(1); + expect(secondPlugin.calls.dispose).toBe(0); + } finally { + await first.close(); + await second?.close(); + } + expect(secondPlugin.calls.dispose).toBe(1); +}); + +test('an embedded client without Maestro preserves the typed optional-plugin error', async () => { + const f = fixture(); + const host = await startAgentDevice(f); + try { + await expect( + host.client.replay.run({ path: f.entry, backend: 'maestro' }), + ).rejects.toMatchObject({ + details: { + reason: 'plugin-not-installed', + package: '@agent-device/maestro', + installCommand: 'agent-device plugins add @agent-device/maestro', + }, + }); + } finally { + await host.close(); + } +}); diff --git a/src/host.ts b/src/host.ts index 4519dee056..ce44a3e1b2 100644 --- a/src/host.ts +++ b/src/host.ts @@ -40,13 +40,18 @@ export async function startAgentDevice(options: StartAgentDeviceOptions): Promis ]); const codeIdentity = await resolveLocalDaemonCodeIdentity(); const output = { write: () => {} }; + const clientDeps: NonNullable[1]> = { host }; const runtime = await startDaemonRuntime({ host: runtimeHost, stdout: output, stderr: output, ownership: 'embedded', codeIdentity, - composeDeps: async (input) => await composeDaemonRuntimeDeps(input, candidates), + composeDeps: async (input) => { + const deps = await composeDaemonRuntimeDeps(input, candidates); + clientDeps.resolveReplayBackend = deps.resolveReplayBackend; + return deps; + }, }); try { const identity = runtime.identity; @@ -70,7 +75,7 @@ export async function startAgentDevice(options: StartAgentDeviceOptions): Promis }), }); return Object.freeze({ - client: createAgentDeviceClient(clientConfig, { host }), + client: createAgentDeviceClient(clientConfig, clientDeps), clientConfig, close: () => runtime.close(), }); diff --git a/src/mcp/__tests__/command-tools-parity.test.ts b/src/mcp/__tests__/command-tools-parity.test.ts index c8088c4e6d..ca7629f8c6 100644 --- a/src/mcp/__tests__/command-tools-parity.test.ts +++ b/src/mcp/__tests__/command-tools-parity.test.ts @@ -215,6 +215,7 @@ test('MCP applies config-backed command defaults; explicit operator input is ref JSON.stringify({ iosXctestrunFile: configuredXctestrun, appsFilter: 'all' }), ); vi.stubEnv('HOME', home); + vi.stubEnv('AGENT_DEVICE_HOME', path.join(home, '.agent-device')); const calls: Array[0]> = []; const transport: AgentDeviceDaemonTransport = async (request) => { diff --git a/src/provider-device-runtimes.ts b/src/provider-device-runtimes.ts index ee678d0c93..6035283c70 100644 --- a/src/provider-device-runtimes.ts +++ b/src/provider-device-runtimes.ts @@ -10,6 +10,7 @@ import { builtinProviderCandidates } from './builtin-provider-plugins.ts'; import { PROVIDER_PLUGIN_POLICY } from './provider-plugin-policy.ts'; export type ProviderRuntimeComposition = Readonly<{ + resolveReplayBackend: import('@agent-device/plugins/replay-backend').ReplayBackendResolver; runtimes: readonly ProviderDeviceRuntime[]; platformModules: readonly PlatformRuntimeProviderRegistration[]; /** Every plugin the registry saw, built-in or not, active or failed. */ @@ -53,6 +54,7 @@ export async function createEmbeddedProviderRuntimeComposition( })), ), inventory: registry.inventory, + resolveReplayBackend: registry.resolveReplayBackend, shutdown: registry.close, }); } diff --git a/src/sdk/host-types.ts b/src/sdk/host-types.ts index e3ded0cc93..85a0392152 100644 --- a/src/sdk/host-types.ts +++ b/src/sdk/host-types.ts @@ -2,6 +2,7 @@ import type { HostEnvironment } from '@agent-device/kernel/source-value'; import type { AgentDeviceClientConfig } from '@agent-device/contracts/client'; import type { AgentDeviceClient } from '@agent-device/command-surface'; import type { AgentDevicePlugin } from '@agent-device/plugins/contract'; +import type { ReplayEnginePlugin } from '@agent-device/plugins/replay-backend'; import type { ShutdownReport } from '@agent-device/daemon'; export type StartAgentDeviceOptions = Readonly<{ @@ -10,7 +11,7 @@ export type StartAgentDeviceOptions = Readonly<{ /** The environment and working directory to run with. Defaults to one captured copy of this process's. */ host?: HostEnvironment; /** Imported plugins to register after the built-ins. The plugin store is not read. */ - plugins?: readonly AgentDevicePlugin[]; + plugins?: readonly (AgentDevicePlugin | ReplayEnginePlugin)[]; }>; export type AgentDeviceHost = Readonly<{ diff --git a/test/integration/installed-package-metro.test.ts b/test/integration/installed-package-metro.test.ts index f69ac28a1a..036bb81abf 100644 --- a/test/integration/installed-package-metro.test.ts +++ b/test/integration/installed-package-metro.test.ts @@ -83,41 +83,6 @@ function linkRuntimeDependencies(installedPackageRoot: string, consumerRoot: str } } -/** - * Runs the packaged runScript child, whose `http.post` starts the packaged HTTP child. Both entries - * are found from the bundled chunk that holds the resolver, so this proves the shipped layout. - */ -async function assertPackagedRunScriptChild(installedPackageRoot: string, root: string) { - const childPath = fs.realpathSync( - path.join(installedPackageRoot, 'dist', 'src', 'internal', 'run-script-child.js'), - ); - assert.equal(fs.existsSync(childPath), true); - const aliasRoot = path.join(root, 'run-script-entry-alias'); - fs.symlinkSync(installedPackageRoot, aliasRoot, 'junction'); - const aliasChildPath = path.join(aliasRoot, 'dist', 'src', 'internal', 'run-script-child.js'); - const server = http.createServer((_req, res) => res.end('x'.repeat(2 * 1024 * 1024))); - try { - const port = await listenOnLoopback(server); - const scriptPath = path.join(root, 'run-script-setup.js'); - fs.writeFileSync(scriptPath, 'output.size = http.post(SERVER_URL).body.length'); - for (const entryPath of [childPath, aliasChildPath]) { - const result = await runCmd(process.execPath, [entryPath], { - stdin: JSON.stringify({ - scriptPath, - env: { SERVER_URL: `http://127.0.0.1:${port}/` }, - publicNetworkOnly: false, - }), - timeoutMs: SUBPROCESS_TIMEOUT_MS, - }); - assert.deepEqual(readJson(result.stdout), { - outputEnv: { 'output.size': String(2 * 1024 * 1024) }, - }); - } - } finally { - await closeLoopbackServer(server); - } -} - async function runNodeModuleJson(cwd: string, args: string[], script: string): Promise { const stdout = await execFileText(process.execPath, [...args, script], { cwd, @@ -318,7 +283,10 @@ test('installed package exposes Node APIs and packaged companion tunnel entrypoi fs.existsSync(path.join(installedPackageRoot, 'dist', 'src', 'companion-tunnel.js')), false, ); - await assertPackagedRunScriptChild(installedPackageRoot, root); + assert.equal( + fs.existsSync(path.join(installedPackageRoot, 'dist/src/internal/run-script-child.js')), + false, + ); const pluginTypes = fs.readFileSync(path.join(installedPackageRoot, 'dist/src/plugins.d.ts')); assert.ok(pluginTypes.length < 16 * 1024); diff --git a/test/integration/provider-scenarios/harness.ts b/test/integration/provider-scenarios/harness.ts index a0f86fd44c..57450a7348 100644 --- a/test/integration/provider-scenarios/harness.ts +++ b/test/integration/provider-scenarios/harness.ts @@ -109,6 +109,7 @@ export async function createProviderScenarioHarness( const env: NodeJS.ProcessEnv = { PATH: process.env.PATH, HOME: process.env.HOME, + AGENT_DEVICE_HOME: process.env.AGENT_DEVICE_HOME, TMPDIR: process.env.TMPDIR, // A scenario that seeds claims for its daemon names the directory through the process. AGENT_DEVICE_CLAIMS_DIR: process.env.AGENT_DEVICE_CLAIMS_DIR ?? path.join(testRoot, 'claims'), diff --git a/test/repo-guards/npm-package-scripts.test.ts b/test/repo-guards/npm-package-scripts.test.ts index a4379e8984..3e8d37a10b 100644 --- a/test/repo-guards/npm-package-scripts.test.ts +++ b/test/repo-guards/npm-package-scripts.test.ts @@ -100,7 +100,7 @@ test('publishing cannot skip the package gate', () => { // the floor that job covers — so it must still name the same entry point the script does. assert.deepEqual(script('check:package').split(' && '), [ 'node --experimental-strip-types scripts/check-package.ts', - 'node --experimental-strip-types scripts/node-test-tmpdir.ts --test packages/maestro/test/package-artifact.test.mjs', + 'node --experimental-strip-types scripts/node-test-tmpdir.ts --test packages/maestro/test/package-artifact.test.mjs packages/maestro/test/package-smoke.mjs', ]); assert.match(packagedCliWorkflow, /with: \{ gate: package \}/); assert.match( diff --git a/vitest.config.ts b/vitest.config.ts index 0335d9caac..a83ba6c6ae 100644 --- a/vitest.config.ts +++ b/vitest.config.ts @@ -87,6 +87,7 @@ export const SETUP_FILES = [ 'src/__tests__/hermetic-env-setup.ts', 'src/__tests__/hermetic-signal-setup.ts', 'src/__tests__/process-memo-setup.ts', + 'src/__tests__/replay-plugin-setup.ts', ]; export default defineConfig({ diff --git a/vitest.mutation.config.ts b/vitest.mutation.config.ts index ea62e42ed9..d036797710 100644 --- a/vitest.mutation.config.ts +++ b/vitest.mutation.config.ts @@ -22,6 +22,7 @@ export default defineConfig({ alias: workspaceAliases, }, test: { + globalSetup: [path.join(repoRoot, 'scripts/vitest-tmpdir-global-setup.ts')], include: scope, exclude: [ ...MUTATION_EXCLUDED_TESTS, diff --git a/website/docs/docs/plugins.md b/website/docs/docs/plugins.md index 72fa20b2e9..4c639112e4 100644 --- a/website/docs/docs/plugins.md +++ b/website/docs/docs/plugins.md @@ -1,5 +1,5 @@ --- -title: Provider plugins +title: Plugins --- Provider plugins add device providers that agent-device does not bundle, such as [TestMu AI](/docs/testmu), [LG webOS](/docs/webos), and [Samsung Tizen](/docs/tizen). Installing one requires npm on your host: @@ -56,6 +56,12 @@ When two plugins claim the same provider, the first one wins: bundled providers, A lease that names a provider this daemon has no runtime for refuses with `UNSUPPORTED_OPERATION` and reason `provider-not-registered`, before any lease is recorded. The error lists the registered providers and, as `failedPlugins`, any plugin that failed to register for that provider (or failed before its provider was known); restart the daemon with the provider's plugin installed. Leases with no provider, `proxy`, and `cloud` need no provider runtime. +## Install Maestro support + +The 0.22 Maestro plugin is private and unavailable from npm. After publication, `agent-device plugins add @agent-device/maestro` installs the optional Maestro engine for `replay --maestro`, `test --maestro`, and `replay export`. [Run Maestro flows](/docs/run-maestro-flows) explains client and daemon installation. `help maestro` remains available without the plugin. + +Replay backends appear in plugin inventories with `backend` instead of `provider`. A daemon keeps its plugin set until it stops. Installing a package on a client does not change a remote or embedded daemon's plugin set. + ## Create a plugin The interface is experimental. A plugin package exports a factory at `./agent-device`. Use `agent-device` as a development dependency; plugins import only types from it. @@ -160,3 +166,9 @@ To move a package: ``` A package that still ships the old manifest loads when it also exports `./agent-device`, and the manifest is then ignored. A plugin object with an `apiVersion` other than 2 is listed as `failed` with `incompatible_plugin`, and a default export that is not a plugin factory with `invalid_plugin`; the daemon starts without either. + +## Create a replay backend plugin + +A replay engine uses the same managed `./agent-device` factory entry and `apiVersion: 2`. Return a `ReplayEnginePlugin` from the type-only `agent-device/plugins` contract with `kind: 'replay-backend'`, a `backend` id, and `setup(host)`. Setup returns `{ replayBackend: { create(host) }, dispose? }`. Keep setup engine-free; `create` may asynchronously import the engine when a replay capability is selected. + +The backend implements source collection, source inspection, replay execution, and export. Its `ReplayBackendHost` supplies dispatch-related services and request cancellation; each replay command supplies its host environment, session access, and `invoke`. The daemon owns sessions, policy, selector/ref authority, and device cleanup. `dispose` runs once after requests drain. Client-side source collection and export dispose their scoped plugin registry when the operation ends. diff --git a/website/docs/docs/run-maestro-flows.md b/website/docs/docs/run-maestro-flows.md index ca3d4d080a..c4724f97d6 100644 --- a/website/docs/docs/run-maestro-flows.md +++ b/website/docs/docs/run-maestro-flows.md @@ -10,6 +10,18 @@ title: Run Maestro flows For recording and replaying native `.ad` scripts, see [Replay and E2E testing](/docs/replay-e2e). +## Install Maestro support + +The 0.22 plugin is private and not yet available from npm. After publication, Maestro flows and export require the optional `@agent-device/maestro` plugin: + +```bash +agent-device plugins add @agent-device/maestro +``` + +Install it on the client machine for source collection and export. When you use a remote daemon, install it on that daemon's host too. An embedded daemon uses the plugins supplied by its host application. Close your sessions and restart the daemon after changing its plugins, then retry your original command with all its arguments. Your flow syntax stays the same. + +If the plugin is missing, the error includes `plugin-not-installed` and the installation command. A selected package that fails loading or setup reports its own failure; agent-device does not install a replacement or switch engines. For a store-backed daemon, run `agent-device plugins list --json` on the relevant host. Run `agent-device daemon status --json --state-dir ` on the daemon host with its state directory. That command reads a local record; it cannot inspect a remote daemon from your client machine. + ## Run a Maestro flow Pass `--maestro` to `replay` or `test`: