From ce66416f9c516b3839d3bf8620eb0f77a612b725 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Sat, 14 Nov 2026 18:31:35 +0100 Subject: [PATCH] chore(gates): keep optional replay routing and plugin admission atomic Land the native plugin with its routing consumer, installation, workflows and package guards. Retain C3 rejected-load retry and inventory ownership with typed repair guidance; preserve lower-layer slot identity and refuse raw YAML suites before plugin or device admission. --- .changeset/optional-native-replay.md | 8 + .fallowrc.json | 8 +- .github/actions/run-gate/action.yml | 9 + .github/workflows/android.yml | 2 +- .github/workflows/apple.yml | 4 +- .../workflows/conformance-differential.yml | 3 +- .github/workflows/replays-nightly.yml | 7 - docs/adr/0012-interactive-replay.md | 4 + .../0016-active-session-script-publication.md | 2 + docs/adr/0034-provider-plugin-registry.md | 4 +- examples/sdk/tsconfig.json | 1 + package.json | 6 +- packages/ad-script/src/index.ts | 1 + packages/ad-script/src/internal/format.ts | 20 ++ packages/command-surface/src/replay/index.ts | 14 +- packages/contracts/src/replay-backend.ts | 1 + packages/daemon/package.json | 5 +- .../composite-resource-admission.fixtures.ts | 38 +++- .../request-router-replay-plugin.test.ts | 212 ++++++++++++++++++ packages/daemon/src/server/request-router.ts | 10 + .../session-test-suite-command-video.test.ts | 2 + packages/daemon/tsconfig.json | 9 +- packages/maestro/test/packed-runtime.mjs | 15 ++ packages/plugins/src/registry.ts | 67 ++++-- packages/plugins/src/replay-backend.test.ts | 28 +++ packages/plugins/src/store.test.ts | 73 +++++- packages/plugins/src/store.ts | 46 ++++ packages/replay-port/package.json | 6 +- .../__tests__/replay-backend-registry.test.ts | 8 +- .../src/daemon-port/native-command.ts | 6 +- .../replay-backend-declarations.ts | 56 ++--- .../daemon-port/replay-backend-registry.ts | 47 +++- .../daemon-port/replay-plugin-admission.ts | 48 ++++ .../session-replay-runtime-plan.ts | 11 +- .../src/daemon-port/test-command.ts | 6 +- packages/replay-port/tsconfig.json | 6 - packages/replay/README.md | 50 +++++ packages/replay/package.json | 37 +++ packages/replay/src/plugin.test.ts | 83 +++++++ packages/replay/src/plugin.ts | 32 +++ packages/replay/test/package-smoke.mjs | 88 ++++++++ packages/replay/test/packed-runtime.mjs | 96 ++++++++ packages/replay/tsconfig.json | 30 +++ packages/replay/tsdown.config.ts | 26 +++ pnpm-lock.yaml | 37 +-- .../install-workspace-replay-plugins.test.ts | 42 ++++ scripts/install-workspace-maestro.ts | 47 ---- scripts/install-workspace-replay-plugins.ts | 68 ++++++ scripts/layering/model.ts | 1 + scripts/layering/package-boundaries.test.ts | 17 +- scripts/layering/project-references.test.ts | 19 +- scripts/vitest-tmpdir-global-setup.ts | 4 +- src/__tests__/cli-maestro-suite.test.ts | 4 +- .../cli-replay-script-sources.test.ts | 6 +- .../cli-startup-import-closure.test.ts | 23 +- src/__tests__/cli-test-env.ts | 15 +- src/__tests__/replay-plugin-setup.ts | 2 +- .../__tests__/replay-maestro-export.test.ts | 4 +- .../args-maestro-support-matrix.test.ts | 3 + src/commands/schema/cli-help-overview.ts | 2 +- src/commands/schema/cli-help.ts | 21 +- ...aemon-composition-replay-selection.test.ts | 139 ++++++++++++ src/daemon-composition.ts | 7 +- src/host-replay.test.ts | 9 +- .../__tests__/command-tools-parity.test.ts | 14 ++ .../command-tools-replay-plugin.test.ts | 47 ++++ src/provider-device-runtimes.ts | 26 ++- .../installed-package-metro.test.ts | 21 +- .../active-session-script-publication.test.ts | 31 ++- test/repo-guards/npm-package-scripts.test.ts | 2 +- tsconfig.typecheck.json | 1 + vitest.config.ts | 1 + website/docs/docs/plugins.md | 20 ++ website/docs/docs/replay-e2e.md | 14 ++ 74 files changed, 1677 insertions(+), 205 deletions(-) create mode 100644 .changeset/optional-native-replay.md create mode 100644 packages/daemon/src/server/__tests__/request-router-replay-plugin.test.ts create mode 100644 packages/replay-port/src/daemon-port/replay-plugin-admission.ts create mode 100644 packages/replay/README.md create mode 100644 packages/replay/package.json create mode 100644 packages/replay/src/plugin.test.ts create mode 100644 packages/replay/src/plugin.ts create mode 100644 packages/replay/test/package-smoke.mjs create mode 100644 packages/replay/test/packed-runtime.mjs create mode 100644 packages/replay/tsconfig.json create mode 100644 packages/replay/tsdown.config.ts create mode 100644 scripts/__tests__/install-workspace-replay-plugins.test.ts delete mode 100644 scripts/install-workspace-maestro.ts create mode 100644 scripts/install-workspace-replay-plugins.ts create mode 100644 src/daemon-composition-replay-selection.test.ts create mode 100644 src/mcp/__tests__/command-tools-replay-plugin.test.ts diff --git a/.changeset/optional-native-replay.md b/.changeset/optional-native-replay.md new file mode 100644 index 0000000000..5625e2b66d --- /dev/null +++ b/.changeset/optional-native-replay.md @@ -0,0 +1,8 @@ +--- +'agent-device': minor +'@agent-device/replay': minor +--- + +Release prerequisite: the replay plugin remains private and unpublished. Its npm installation command below is unavailable until a usable package is published through the configured release workflow. + +Native `.ad` replay and all test suites require the optional `@agent-device/replay` plugin on the daemon host. Run `agent-device plugins add @agent-device/replay`, finish your sessions and restart that daemon, then retry the original invocation with all its arguments. Remote hosts need their own installation. Embedded hosts supply the version 2 factory through `startAgentDevice({ plugins: [replay()] })` and own runtime replacement. Existing scripts, resume and repair syntax stay the same. Ordinary script recording remains in core. diff --git a/.fallowrc.json b/.fallowrc.json index 715d1e01f2..7df7e8b253 100644 --- a/.fallowrc.json +++ b/.fallowrc.json @@ -11,9 +11,13 @@ "packages/maestro/test/packed-screen-origin.mjs", "packages/maestro/test/package-test-support.mjs", "packages/maestro/test/package-smoke.mjs", + "packages/replay/test/package-smoke.mjs", + "packages/replay/test/packed-runtime.mjs", "packages/maestro/test/packed-runtime.mjs", "packages/maestro/tsdown.config.ts", "packages/maestro/src/plugin.ts", + "packages/replay/src/plugin.ts", + "packages/replay/tsdown.config.ts", "packages/provider-testmu/tsdown.config.ts", "packages/provider-revyl/test/package-smoke.mjs", "packages/provider-revyl/tsdown.config.ts", @@ -159,7 +163,7 @@ }, { // A published plugin's default export is its contract; the plugin loader imports the installed package's resolved entry at runtime and reads `default`. - "file": "packages/{provider-*/src/index,maestro/src/plugin}.ts", + "file": "packages/{provider-*/src/index,maestro/src/plugin,replay/src/plugin}.ts", "exports": ["default"] }, { @@ -289,7 +293,7 @@ }, { // Tool config default exports, loaded by the tool rather than imported. - "file": "{oxlint.config.ts,tsdown.config.ts,packages/provider-*/tsdown.config.ts,packages/maestro/tsdown.config.ts,packages/proxy/tsdown.config.ts,vitest.mutation.config.ts}", + "file": "{oxlint.config.ts,tsdown.config.ts,packages/provider-*/tsdown.config.ts,packages/maestro/tsdown.config.ts,packages/replay/tsdown.config.ts,packages/proxy/tsdown.config.ts,vitest.mutation.config.ts}", "exports": ["default"] }, { diff --git a/.github/actions/run-gate/action.yml b/.github/actions/run-gate/action.yml index d387926703..05927133fe 100644 --- a/.github/actions/run-gate/action.yml +++ b/.github/actions/run-gate/action.yml @@ -15,6 +15,15 @@ outputs: runs: using: composite steps: + - name: Install first-party replay plugins for execution gates + if: contains(fromJSON('["unit", "unit-ci", "provider-integration", "integration-node", "macos-coverage", "web-smoke", "mutation", "fuzz-parsers", "maestro-differential", "replay-ios", "replay-ios-device", "replay-macos", "replay-linux", "replay-android", "linux-command-evidence", "concurrency-torture"]'), inputs.gate) + shell: bash + run: | + set -euo pipefail + pnpm --filter @agent-device/maestro --filter @agent-device/replay build + export AGENT_DEVICE_HOME="${AGENT_DEVICE_HOME:-$RUNNER_TEMP/agent-device-replay-plugins}" + node --experimental-strip-types scripts/install-workspace-replay-plugins.ts + echo "AGENT_DEVICE_HOME=$AGENT_DEVICE_HOME" >> "$GITHUB_ENV" - id: run shell: bash env: diff --git a/.github/workflows/android.yml b/.github/workflows/android.yml index 8ca5192eb0..be1d037ae9 100644 --- a/.github/workflows/android.yml +++ b/.github/workflows/android.yml @@ -93,7 +93,7 @@ jobs: ANDROID_SERIAL="$(adb devices | awk 'NR > 1 && $2 == "device" { print $1; exit }')" test -n "$ANDROID_SERIAL" pnpm gate build - node --experimental-strip-types scripts/install-workspace-maestro.ts + node --experimental-strip-types scripts/install-workspace-replay-plugins.ts pnpm clean:daemon AGENT_DEVICE_ANDROID_E2E=1 AGENT_DEVICE_ANDROID_E2E_TIER=smoke AGENT_DEVICE_ANDROID_SERIAL="$ANDROID_SERIAL" AGENT_DEVICE_FIXTURE_APP_PATH="${{ steps.fixture-app.outputs.apk-path }}" AGENT_DEVICE_FIXTURE_APP_ID="${{ steps.fixture-app.outputs.app-id }}" node --experimental-strip-types scripts/node-test-tmpdir.ts --test test/integration/smoke-android-emulator.test.ts node --experimental-strip-types src/bin.ts test test/integration/replays/android/emulator/01-settings.ad --retries 2 --report-junit test/artifacts/replays-android-smoke.junit.xml diff --git a/.github/workflows/apple.yml b/.github/workflows/apple.yml index 89da9b8ed6..c59ec2cded 100644 --- a/.github/workflows/apple.yml +++ b/.github/workflows/apple.yml @@ -84,8 +84,8 @@ jobs: uses: ./.github/actions/run-gate with: gate: build - - if: steps.build.outcome == 'success' - run: node --experimental-strip-types scripts/install-workspace-maestro.ts + - if: matrix.platform == 'ios' && steps.build.outcome == 'success' + run: node --experimental-strip-types scripts/install-workspace-replay-plugins.ts - name: iOS smoke tests id: ios uses: ./.github/actions/smoke-ios-run diff --git a/.github/workflows/conformance-differential.yml b/.github/workflows/conformance-differential.yml index a18b7cb5b5..8d3bbec263 100644 --- a/.github/workflows/conformance-differential.yml +++ b/.github/workflows/conformance-differential.yml @@ -67,8 +67,9 @@ jobs: - name: Build and install the workspace Maestro plugin run: | pnpm --filter @agent-device/maestro build + pnpm --filter @agent-device/replay build export AGENT_DEVICE_HOME="$RUNNER_TEMP/agent-device-maestro-home" - node --experimental-strip-types scripts/install-workspace-maestro.ts + node --experimental-strip-types scripts/install-workspace-replay-plugins.ts echo "AGENT_DEVICE_HOME=$AGENT_DEVICE_HOME" >> "$GITHUB_ENV" - name: Restore and build iOS XCTest runner diff --git a/.github/workflows/replays-nightly.yml b/.github/workflows/replays-nightly.yml index 67812c3118..0ee0e41b44 100644 --- a/.github/workflows/replays-nightly.yml +++ b/.github/workflows/replays-nightly.yml @@ -53,13 +53,6 @@ jobs: - name: Setup toolchain uses: ./.github/actions/setup-node-pnpm - - name: Build and install the workspace Maestro plugin - run: | - pnpm --filter @agent-device/maestro build - export AGENT_DEVICE_HOME="$RUNNER_TEMP/agent-device-fuzz-home" - node --experimental-strip-types scripts/install-workspace-maestro.ts - echo "AGENT_DEVICE_HOME=$AGENT_DEVICE_HOME" >> "$GITHUB_ENV" - # A lane whose classifier or watchdog regressed would pass forever. This runs the # broken-on-purpose targets first and fails unless each violation is caught. - name: Self-check the harness diff --git a/docs/adr/0012-interactive-replay.md b/docs/adr/0012-interactive-replay.md index b7beb86c10..18b0affcad 100644 --- a/docs/adr/0012-interactive-replay.md +++ b/docs/adr/0012-interactive-replay.md @@ -18,6 +18,10 @@ At acceptance, replay had three gaps: ## Decision +### Optional execution boundary (0.22) + +Native `.ad` execution and test-suite scheduling require the first-party `@agent-device/replay` version 2 plugin. Maestro suites additionally require `@agent-device/maestro`. The managed registry owns loading, backend claims, failure isolation and disposal. Core keeps session and repair-transaction authority, device dispatch, selectors, refs, recording, cancellation and cleanup. The existing explicit step and suite ports cross the package boundary without a second session store or request context. Shared `.ad` data and divergence wire functions remain in core. This extraction changes installation requirements, not the identity, divergence, resume or repair decisions below. A missing plugin is refused before device binding. A stale daemon requires an explicit host restart, never automatic installation or session termination. + ### 1. Retire `--update` healing as an actor; repurpose its candidate machinery as ranked suggestions `--update`/`-u` does not rewrite `.ad` files. diff --git a/docs/adr/0016-active-session-script-publication.md b/docs/adr/0016-active-session-script-publication.md index 6ab35733c8..da050d4ea5 100644 --- a/docs/adr/0016-active-session-script-publication.md +++ b/docs/adr/0016-active-session-script-publication.md @@ -9,6 +9,8 @@ Implementation owners are `packages/daemon/src/sessions/handlers/session-script- ## Rules at a glance +Ordinary recording and publication remain core capabilities without the replay plugin. Native `.ad` replay/test execution or replay repair requires `@agent-device/replay` on the daemon host. + This section is a normative summary. [Decision](#decision) contains the binding contracts and refusal cases. diff --git a/docs/adr/0034-provider-plugin-registry.md b/docs/adr/0034-provider-plugin-registry.md index 58e4a2228b..21dec648cb 100644 --- a/docs/adr/0034-provider-plugin-registry.md +++ b/docs/adr/0034-provider-plugin-registry.md @@ -7,7 +7,7 @@ Status: accepted. Umbrella #3747; contract and registry #3755. - One contract. A provider plugin is a factory that returns `{ apiVersion: 2, provider, credentials, connection?, setup(host) }`. `setup` returns `{ runtime, platformModule, dispose? }` or `{ webDriver, dispose? }`. A `{ webDriver }` result stays a data descriptor the host interprets. -- A replay plugin uses the same version 2 factory with `kind: 'replay-backend'`, `backend`, and `setup(host)`. Setup returns `{ replayBackend: { create(host) }, dispose? }`. Provider and backend ids have separate claim sets in this registry. A replay plugin registers no provider runtime. +- A replay plugin uses the same version 2 factory with `kind: 'replay-backend'`, `backend`, and `setup(host)`. Setup returns `{ replayBackend: { create(host) }, dispose? }` for a format backend. Provider and backend ids have separate claim sets in this registry. A replay plugin registers no provider runtime. - One registry. Composition selects built-in, store, or embedder sources. Built-in providers use the same shape as an installed plugin, and one function validates what every factory returns. - Plugins import only types from agent-device. The type-only contract is `agent-device/plugins`; @@ -24,6 +24,8 @@ Status: accepted. Umbrella #3747; contract and registry #3755. - `daemon.json` has one reader. Its states are `absent`, `unreadable`, and `invalid`. A missing or stale record reads as a stopped daemon; an unreadable or invalid one never does. - Keep replay setup engine-free. Load the engine only when its backend is selected. The runtime's registry memoizes creation and owns disposal after requests drain. Client-side source collection and export close their scoped registry. +- Engine-free vetted declarations supply help and missing-plugin installation commands. They do not establish whether an engine is installed or working. Unknown names produce no inferred package. +- Missing replay execution returns `plugin-not-installed` before device binding. A selected package retains its load, validation, compatibility or setup reason. A package added after daemon startup returns `daemon-plugin-set-stale`; the caller finishes sessions and restarts that host. An embedded registry never consults the store for recovery. - The plugin set is fixed for a daemon's lifetime. A plugin change takes effect on the next daemon. - A store package is discovered through its package targets: the `./agent-device` export, else the root export, which is `main` for a package that declares no `exports`. Nothing is read from diff --git a/examples/sdk/tsconfig.json b/examples/sdk/tsconfig.json index a65fe38443..c913117ab3 100644 --- a/examples/sdk/tsconfig.json +++ b/examples/sdk/tsconfig.json @@ -71,6 +71,7 @@ { "path": "../../packages/proxy" }, { "path": "../../packages/react-native" }, { "path": "../../packages/remote" }, + { "path": "../../packages/replay" }, { "path": "../../packages/replay-port" }, { "path": "../../packages/replay-test" }, { "path": "../../packages/selectors" }, diff --git a/package.json b/package.json index 30a0ddc100..0ce4b7fcc0 100644 --- a/package.json +++ b/package.json @@ -86,7 +86,7 @@ "agent-device": "bin/agent-device.mjs" }, "scripts": { - "build": "pnpm --filter @agent-device/maestro build && tsdown", + "build": "pnpm --filter @agent-device/maestro build && pnpm --filter @agent-device/replay build && tsdown", "clean:daemon": "node --experimental-strip-types scripts/clean-daemon.ts", "daemon:state-dir": "node --experimental-strip-types scripts/print-daemon-state-dir.ts", "explain:command": "node --experimental-strip-types scripts/explain-command.ts", @@ -165,7 +165,7 @@ "depgraph:test": "node --experimental-strip-types scripts/node-test-tmpdir.ts --experimental-strip-types --test scripts/depgraph/model.test.ts scripts/depgraph/entries.test.ts scripts/depgraph/import-graph.test.ts scripts/depgraph/affected.test.ts scripts/depgraph/structure-summary.test.ts tools/depgraph-view/*.test.mjs", "check:production-exports": "fallow dead-code --config fallow-production-exports.json --production --unused-exports --fail-on-issues", "check:bundle-owner-files": "node --experimental-strip-types scripts/check-bundle-owner-files.ts", - "check:package": "node --experimental-strip-types scripts/check-package.ts && node --experimental-strip-types scripts/node-test-tmpdir.ts --test packages/maestro/test/package-artifact.test.mjs packages/maestro/test/package-smoke.mjs", + "check:package": "node --experimental-strip-types scripts/check-package.ts && node --experimental-strip-types scripts/node-test-tmpdir.ts --test packages/maestro/test/package-artifact.test.mjs packages/maestro/test/package-smoke.mjs packages/replay/test/package-smoke.mjs", "check:command-docs": "vitest run --project unit-core test/repo-guards/command-doc-coverage.test.ts", "check:adr-citations": "vitest run --project unit-core test/repo-guards/adr-citations.test.ts", "check:capabilities-matrix": "vitest run --project unit-core test/repo-guards/capability-matrix.test.ts", @@ -301,7 +301,6 @@ } }, "devDependencies": { - "@agent-device/ad-replay": "workspace:*", "@agent-device/ad-script": "workspace:*", "@agent-device/apple-runner": "workspace:*", "@agent-device/capture-kit": "workspace:*", @@ -331,7 +330,6 @@ "@agent-device/react-native": "workspace:*", "@agent-device/remote": "workspace:*", "@agent-device/replay-port": "workspace:*", - "@agent-device/replay-test": "workspace:*", "@agent-device/selectors": "workspace:*", "@agent-device/session-journal": "workspace:*", "@agent-device/test-kit": "workspace:*", diff --git a/packages/ad-script/src/index.ts b/packages/ad-script/src/index.ts index a3df8be6bc..c153e6e48c 100644 --- a/packages/ad-script/src/index.ts +++ b/packages/ad-script/src/index.ts @@ -61,6 +61,7 @@ export { } from './internal/vars.ts'; export { + getReplayPluginInstallation, isMaestroYamlPath, isReplayBackendId, maestroBackendRequiredMessage, diff --git a/packages/ad-script/src/internal/format.ts b/packages/ad-script/src/internal/format.ts index 96c5c23a6f..119871eff7 100644 --- a/packages/ad-script/src/internal/format.ts +++ b/packages/ad-script/src/internal/format.ts @@ -2,6 +2,26 @@ import path from 'node:path'; export type ReplayFormat = 'ad' | 'maestro'; +const REPLAY_PLUGIN_INSTALLATIONS = { + ad: { + package: '@agent-device/replay', + summary: 'Native .ad replay and test suites', + installationHint: + 'Install on the daemon host for native .ad replay and all test suites. Core clients can read and send native .ad sources without this plugin.', + }, + maestro: { + package: '@agent-device/maestro', + summary: 'Supported Maestro YAML commands, grammar, and runtime boundaries', + installationHint: + 'Install on the client for source collection/export and on the daemon host for Maestro execution.', + }, +} as const; + +/** Vetted installation facts for the replay format vocabulary, without loading an engine. */ +export function getReplayPluginInstallation(id: ReplayFormat) { + return { id, ...REPLAY_PLUGIN_INSTALLATIONS[id] }; +} + export function isMaestroYamlPath(sourcePath: string): boolean { const extension = path.extname(sourcePath).toLowerCase(); return extension === '.yaml' || extension === '.yml'; diff --git a/packages/command-surface/src/replay/index.ts b/packages/command-surface/src/replay/index.ts index bb37aaae3b..91eab7a2c6 100644 --- a/packages/command-surface/src/replay/index.ts +++ b/packages/command-surface/src/replay/index.ts @@ -28,6 +28,7 @@ import { METRO_RELOAD_FLAGS, REPLAY_FLAGS } from '@agent-device/command-registry import { withCommandRuntimeHints } from '../runtime-hints.ts'; import { collectReplayShellEnv, + getReplayPluginInstallation, parseReplayCliEnvEntries, readReplayCliEnvEntries, resolveReplayFormat, @@ -39,6 +40,13 @@ import { discoverReplaySourcePaths } from '@agent-device/replay-port/source-disc const REPLAY_COMMAND_NAME = 'replay'; const TEST_COMMAND_NAME = 'test'; +const replayPluginMcpDetail = (['ad', 'maestro'] as const) + .map((id) => { + const declaration = getReplayPluginInstallation(id); + return `${declaration.summary} (plugin, install separately). ${declaration.installationHint}`; + }) + .join(' '); + const REPLAY_SHELL_ENV_PREFIX = 'AD_VAR_'; const replayCommandDescription = @@ -330,7 +338,8 @@ export const replayCommandFacet = defineCommandFacet({ name: REPLAY_COMMAND_NAME, effect: 'destructive', text: { - summary: 'Replay a recorded session or Maestro flow', + summary: 'Replay a recorded session or Maestro flow (plugin, install separately)', + mcpDetail: replayPluginMcpDetail, cliDetail: 'Replay has a 60-second work budget inside a 90-second client timeout; --timeout sets a work budget of X and client timeout of X + 30000 ms. The running step receives ordinary request cancellation at the budget. When it settles, failure returns details.reason daemon_request_timeout and details.timeoutMs, preserving the failing script path and line. The daemon and other sessions stay available if it responds; an authored close may already have ended its session. A completed replay returns success even if it settles just after the deadline. Cancellation does not prove native device work stopped. A step that ignores cancellation beyond the 30-second margin may cause a transport timeout; an unresponsive daemon retains the client liveness-based reset. Direct test keeps its own timeout behavior. For Maestro YAML compatibility flows, use replay --maestro and keep the target binding such as --platform ios on the replay command. A script with no terminal close leaves its session (and daemon) running until you close it or it idle-reaps — no different from a session opened interactively. For native .ad scripts, --keep-session suppresses exactly an authored terminal close so you can continue interactively. replay export converts compatible actions to Maestro YAML locally, including app switches with explicit launchApp.appId targets, deep links (including tel: and mailto:) as openLink, and home as pressKey: Home.', }, @@ -345,7 +354,8 @@ export const testCommandFacet = defineCommandFacet({ name: TEST_COMMAND_NAME, effect: 'destructive', text: { - summary: 'Run replay test suites', + summary: 'Run replay test suites (plugin, install separately)', + mcpDetail: replayPluginMcpDetail, cliDetail: "Relative globs are expanded on the caller from its working directory, whose name is treated literally. Quote glob inputs to defer expansion to test. Copied diagnostic artifacts receive numbered filenames when needed to preserve other artifacts, replay sources, timing traces, and attempt manifests. JUnit reports (--reporter junit:) replace characters forbidden by XML 1.0 with U+FFFD and preserve legal Unicode and whitespace. JSON and other reporters retain the original suite values. Custom reporter getExitCode hooks must return an integer from 0 to 255 or undefined; the highest valid code wins and cannot lower a failing suite's exit code. A nested test stops after a bounded resource refusal even with --retries; other failures keep their configured retries.", }, diff --git a/packages/contracts/src/replay-backend.ts b/packages/contracts/src/replay-backend.ts index 6f11641fa6..07f6fc383c 100644 --- a/packages/contracts/src/replay-backend.ts +++ b/packages/contracts/src/replay-backend.ts @@ -281,6 +281,7 @@ export type ReplayBackendHost = Readonly<{ export type ReplayBackendDeclaration = Readonly<{ id: TId; package: string; + installationHint: string; help: Readonly<{ summary: string; body: string }>; }>; diff --git a/packages/daemon/package.json b/packages/daemon/package.json index 5351051cc2..2694d82ed7 100644 --- a/packages/daemon/package.json +++ b/packages/daemon/package.json @@ -17,7 +17,6 @@ "@agent-device/platform-apple": "workspace:*", "@agent-device/platform-web": "workspace:*", "@agent-device/replay-port": "workspace:*", - "@agent-device/replay-test": "workspace:*", "@agent-device/selectors": "workspace:*", "@agent-device/session-journal": "workspace:*" }, @@ -31,7 +30,9 @@ "vitest": "5.0.4", "yaml": "^2.9.0", "@agent-device/maestro": "workspace:*", - "@agent-device/ad-replay": "workspace:*" + "@agent-device/ad-replay": "workspace:*", + "@agent-device/plugins": "workspace:*", + "@agent-device/replay": "workspace:*" }, "exports": { ".": { diff --git a/packages/daemon/src/core/composite-resource-admission.fixtures.ts b/packages/daemon/src/core/composite-resource-admission.fixtures.ts index 363f585a11..8ba95b39db 100644 --- a/packages/daemon/src/core/composite-resource-admission.fixtures.ts +++ b/packages/daemon/src/core/composite-resource-admission.fixtures.ts @@ -1,3 +1,6 @@ +import replayPlugin from '@agent-device/replay/agent-device'; +import { resolveProviderPlugins, startProviderRegistry } from '@agent-device/plugins/registry'; +import { bindReplayBackendResolver } from '@agent-device/replay-port/replay-backend-registry'; import type { ReplayBackend } from '@agent-device/contracts/replay-backend'; import path from 'node:path'; import { LeaseRegistry } from '../resources/lease-registry.ts'; @@ -191,14 +194,39 @@ export async function resourceAdmissionWorld( stderr: { write: () => {} }, composeDeps: async (input) => { const backend = options.replayBackend; - if (backend) { - const { bindReplayBackendResolver } = - await import('@agent-device/replay-port/replay-backend-registry'); - await bindReplayBackendResolver(async () => backend); - } + const registry = await startProviderRegistry( + await resolveProviderPlugins([ + { source: 'embedder', load: replayPlugin }, + ...(backend + ? [ + { + source: 'embedder' as const, + load: () => ({ + apiVersion: 2, + kind: 'replay-backend', + backend: backend.id, + setup: () => ({ replayBackend: { create: () => backend } }), + }), + }, + ] + : []), + ]), + input.host.env, + { + adaptWebDriver: async () => { + throw new Error('Resource fixture cannot create providers'); + }, + }, + ); + await bindReplayBackendResolver( + registry.resolveReplayBackend, + registry.resolveReplayExecution, + ); const base = await daemonRuntimeDepsFixture()(input); return { ...base, + pluginInventory: registry.inventory, + shutdownProviderRuntimes: registry.close, ...((options.withLifecycleProvider ?? options.providerOwned) ? { leaseLifecycleProvider: { release: providerRelease } } : {}), diff --git a/packages/daemon/src/server/__tests__/request-router-replay-plugin.test.ts b/packages/daemon/src/server/__tests__/request-router-replay-plugin.test.ts new file mode 100644 index 0000000000..10c12915a4 --- /dev/null +++ b/packages/daemon/src/server/__tests__/request-router-replay-plugin.test.ts @@ -0,0 +1,212 @@ +import { createPluginSelectionGuard } from '@agent-device/plugins/store'; +import path from 'node:path'; +import { expect, test, vi } from 'vitest'; +import { createTestDeviceInventoryGateways } from '@agent-device/device-selection/inventory-gateways-fixtures'; +import { bindReplayBackendResolver } from '@agent-device/replay-port/replay-backend-registry'; +import { resolveProviderPlugins, startProviderRegistry } from '@agent-device/plugins/registry'; +import { + selectV2Plugin, + mkdtempForTestSync, + createTestHostEnvironment, +} from '@agent-device/test-kit'; +import { makeSessionStore } from '../../__tests__/support/store-factory.ts'; +import { LeaseRegistry } from '../../resources/lease-registry.ts'; +import { createRequestHandler } from '../../__tests__/support/test-device-runtime-gateway.ts'; +import { fakeCreateCommandSurface } from '../../__tests__/support/command-surface-fake.ts'; + +async function fixture(storeEnv?: NodeJS.ProcessEnv) { + const root = mkdtempForTestSync('replay-plugin-admission-'); + const discover = vi.fn(async () => []); + const registry = await startProviderRegistry( + await resolveProviderPlugins([]), + {}, + { + assertReplaySelectionCurrent: storeEnv ? createPluginSelectionGuard([], storeEnv) : undefined, + adaptWebDriver: async () => { + throw new Error('no provider'); + }, + }, + ); + await bindReplayBackendResolver(registry.resolveReplayBackend, registry.resolveReplayExecution); + const store = makeSessionStore('replay-plugin-admission-store-'); + const handler = createRequestHandler({ + host: createTestHostEnvironment(), + createCommandSurface: fakeCreateCommandSurface, + logPath: path.join(root, 'daemon.log'), + token: 'token', + sessionStore: store, + leaseRegistry: new LeaseRegistry(), + deviceInventoryGateways: createTestDeviceInventoryGateways({ local: discover }), + trackDownloadableArtifact: () => 'artifact-id', + }); + return { registry, discover, handler, store }; +} + +const source = { + entry: '/caller/flow.ad', + files: { '/caller/flow.ad': 'open com.example.app\nclose\n' }, +}; + +test.each([ + ['yaml', 'replay'], + ['yml', 'replay'], + ['yaml', 'test'], + ['yml', 'test'], +] as const)( + 'bare %s %s reports explicit routing before plugin or device admission', + async (extension, command) => { + const f = await fixture(); + const entry = `/caller/flow.${extension}`; + const bundle = { entry, files: { [entry]: 'appId: com.example.app' } }; + try { + const result = await f.handler({ + token: 'token', + command, + session: 'untouched', + positionals: [entry], + flags: + command === 'test' ? { replayScriptSources: [bundle] } : { replayScriptSource: bundle }, + }); + expect(result).toMatchObject({ + ok: false, + error: { + code: 'INVALID_ARGS', + message: `Maestro YAML requires explicit --maestro routing: ${command} ${entry} --maestro`, + }, + }); + if (result.ok) throw new Error('Bare YAML command succeeded'); + expect(result.error.details?.installCommand).toBeUndefined(); + expect(f.discover).not.toHaveBeenCalled(); + expect(f.store.get('untouched')).toBeUndefined(); + } finally { + await f.registry.close(); + } + }, +); + +test.each(['replay', 'test'] as const)( + 'missing %s execution is classified before device selection', + async (command) => { + const f = await fixture(); + try { + const result = await f.handler({ + token: 'token', + command, + session: 'untouched', + positionals: [source.entry], + flags: { platform: 'ios', replayScriptSource: source, replayScriptSources: [source] }, + }); + expect(result).toMatchObject({ + ok: false, + error: { + code: 'INVALID_ARGS', + details: { + reason: 'plugin-not-installed', + package: '@agent-device/replay', + installCommand: 'agent-device plugins add @agent-device/replay', + }, + }, + }); + expect(f.discover).not.toHaveBeenCalled(); + expect(f.store.get('untouched')).toBeUndefined(); + } finally { + await f.registry.close(); + } + }, +); + +test.each(['replay', 'test'] as const)( + 'explicit Maestro %s retains its required plugin admission', + async (command) => { + const f = await fixture(); + const entry = '/caller/flow.yaml'; + const bundle = { entry, files: { [entry]: 'appId: com.example.app' } }; + try { + const result = await f.handler({ + token: 'token', + command, + session: 'untouched', + positionals: [entry], + flags: { + replayBackend: 'maestro', + ...(command === 'test' + ? { replayScriptSources: [bundle] } + : { replayScriptSource: bundle }), + }, + }); + expect(result).toMatchObject({ + ok: false, + error: { + details: { + reason: 'plugin-not-installed', + package: command === 'test' ? '@agent-device/replay' : '@agent-device/maestro', + }, + }, + }); + expect(f.discover).not.toHaveBeenCalled(); + expect(f.store.get('untouched')).toBeUndefined(); + } finally { + await f.registry.close(); + } + }, +); + +test('a changed store needs a daemon restart and embedded runtimes never consult it', async () => { + const home = mkdtempForTestSync('replay-plugin-store-'); + const env = { AGENT_DEVICE_HOME: home }; + const f = await fixture(env); + try { + selectV2Plugin( + home, + '@agent-device/replay', + 'throw new Error("must not import while detecting stale daemon");', + ); + const request = { + token: 'token', + command: 'replay', + session: 'untouched', + positionals: [source.entry], + flags: { replayScriptSource: source }, + }; + expect(await f.handler(request)).toMatchObject({ + ok: false, + error: { details: { reason: 'daemon-plugin-set-stale' } }, + }); + const embedded = await fixture(); + try { + expect(await embedded.handler(request)).toMatchObject({ + ok: false, + error: { details: { reason: 'plugin-not-installed' } }, + }); + expect(embedded.discover).not.toHaveBeenCalled(); + } finally { + await embedded.registry.close(); + } + expect(f.discover).not.toHaveBeenCalled(); + } finally { + await f.registry.close(); + } +}); + +test.each(['unknown-engine', ''])( + 'backend %j receives no inferred installation command', + async (backend) => { + const f = await fixture(); + try { + const result = await f.handler({ + token: 'token', + command: 'replay', + session: 'untouched', + positionals: [source.entry], + flags: { replayBackend: backend, replayScriptSource: source }, + }); + expect(result).toMatchObject({ ok: false, error: { code: 'INVALID_ARGS' } }); + if (result.ok) throw new Error('Unknown backend succeeded'); + expect(result.error.details?.installCommand).toBeUndefined(); + expect(result.error.message).toContain('Unsupported replay backend'); + expect(f.discover).not.toHaveBeenCalled(); + } finally { + await f.registry.close(); + } + }, +); diff --git a/packages/daemon/src/server/request-router.ts b/packages/daemon/src/server/request-router.ts index 3cac1975ff..bc7b4a51e3 100644 --- a/packages/daemon/src/server/request-router.ts +++ b/packages/daemon/src/server/request-router.ts @@ -275,6 +275,16 @@ export function createRequestHandler(deps: RequestRouterDeps): DaemonInvokeFn { const unsupportedDeviceSpec = deviceSpecFlagsResponse(req); if (unsupportedDeviceSpec) return unsupportedDeviceSpec; + if (req.command === 'replay' || req.command === 'test') { + try { + const { requireReplayPlugins } = + await import('@agent-device/replay-port/replay-plugin-admission'); + await requireReplayPlugins(req); + } catch (error) { + return finalizeThrownRequestError(error); + } + } + let scope: RequestExecutionScope | undefined; let response: DaemonResponse; const platformRequestScope = createPlatformRequestScope(req); diff --git a/packages/daemon/src/sessions/__tests__/replay-suite/session-test-suite-command-video.test.ts b/packages/daemon/src/sessions/__tests__/replay-suite/session-test-suite-command-video.test.ts index 45822f8a6f..c2a79121ec 100644 --- a/packages/daemon/src/sessions/__tests__/replay-suite/session-test-suite-command-video.test.ts +++ b/packages/daemon/src/sessions/__tests__/replay-suite/session-test-suite-command-video.test.ts @@ -1,3 +1,4 @@ +import { getReplayExecution } from '@agent-device/replay-port/replay-backend-registry'; import type { DaemonResponse } from '@agent-device/kernel/contracts'; import { daemonPlatformServicesFixture } from '../../../__tests__/support/platform-services-fixture.ts'; /** Per-attempt recording is finalized exactly once after cancellation. */ @@ -217,6 +218,7 @@ function expectRecordRuntimeCall( } test('test finalizes replay video exactly once when cancellation arrives after start', async () => { + await getReplayExecution(); vi.useFakeTimers({ now: 1_000 }); const { root, replayPath, sessionStore, nestedRequests, events } = createRecordVideoFixture(); const finish = vi.fn(async () => ({ diff --git a/packages/daemon/tsconfig.json b/packages/daemon/tsconfig.json index d0e51a2557..fb3d123454 100644 --- a/packages/daemon/tsconfig.json +++ b/packages/daemon/tsconfig.json @@ -11,6 +11,9 @@ }, "include": ["src"], "references": [ + { + "path": "../plugins" + }, { "path": "../ad-replay" }, @@ -59,9 +62,6 @@ { "path": "../replay-port" }, - { - "path": "../replay-test" - }, { "path": "../selectors" }, @@ -70,6 +70,9 @@ }, { "path": "../test-kit" + }, + { + "path": "../replay" } ] } diff --git a/packages/maestro/test/packed-runtime.mjs b/packages/maestro/test/packed-runtime.mjs index daaf015833..77e832e622 100644 --- a/packages/maestro/test/packed-runtime.mjs +++ b/packages/maestro/test/packed-runtime.mjs @@ -99,6 +99,21 @@ async function embedded(label) { entry, 'appId: com.example.app\n---\n- evalScript: ${output.value = "ok"}\n', ); + assert.deepEqual(await runtime.client.sessions.list(), []); + await assert.rejects( + runtime.client.replay.test({ paths: [entry], backend: 'maestro', platform: 'ios' }), + (error) => { + assert.equal(error.code, 'INVALID_ARGS'); + assert.equal(error.details.reason, 'plugin-not-installed'); + assert.equal(error.details.package, '@agent-device/replay'); + assert.equal( + error.details.installCommand, + 'agent-device plugins add @agent-device/replay', + ); + return true; + }, + ); + assert.deepEqual(await runtime.client.sessions.list(), []); const response = await runtime.client.replay.run({ path: entry, backend: 'maestro', diff --git a/packages/plugins/src/registry.ts b/packages/plugins/src/registry.ts index 11c9c137cc..d5c4e47b2a 100644 --- a/packages/plugins/src/registry.ts +++ b/packages/plugins/src/registry.ts @@ -50,6 +50,7 @@ export type PluginCandidate = Readonly<{ source: PluginSource; package?: string; version?: string; + selectionStamp?: string; /** Claims `provider` before its load settles, so a later candidate with that id is a duplicate. Dormant candidates claim it without loading. */ provider?: string; /** Claims `provider` without registering or listing: a built-in that is not configured. */ @@ -75,6 +76,7 @@ export function claimProviderIds(providers: readonly string[]): PluginCandidate[ * plugin's `{ webDriver }` descriptor becomes a runtime. */ export type PluginRegistryPolicy = Readonly<{ + assertReplaySelectionCurrent?(packageName: string): void; adaptWebDriver( descriptor: WebDriverDescriptor, context: Readonly<{ clientVersion: string }>, @@ -205,15 +207,26 @@ export async function startProviderRegistry( throw new AppError('COMMAND_FAILED', 'Plugin registry is closed', { reason: 'plugin-registry-closed', }); - return await resolveRegisteredReplayBackend(resolved, inventory, id, host, packageName); + const slot = requireReplaySlot( + resolved, + id, + packageName, + policy.assertReplaySelectionCurrent, + ); + return await resolveRegisteredReplayBackend(resolved, slot, inventory, id, host); }, resolveReplayExecution: async (packageName) => { if (closed) throw new AppError('COMMAND_FAILED', 'Plugin registry is closed', { reason: 'plugin-registry-closed', }); - const slot = requireReplaySlot(resolved, NATIVE_REPLAY_BACKEND, packageName); - if (slot.failure !== undefined) throw slot.failure; + const slot = requireReplaySlot( + resolved, + NATIVE_REPLAY_BACKEND, + packageName, + policy.assertReplaySelectionCurrent, + ); + if (slot.failure !== undefined) throw replaySlotError(slot); const execution = slot.replaySetup?.execution; if (!execution) throw invalidPlugin('Native replay plugin must provide execution.create()'); return await (slot.executionLoading ??= Promise.resolve().then(async () => { @@ -226,7 +239,7 @@ export async function startProviderRegistry( slot.executionLoading = undefined; slot.replayFailure = asPluginFailure(error, 'plugin_load_failed'); inventory.splice(0, inventory.length, ...resolved.inventory()); - throw slot.replayFailure; + throw replaySlotError(slot); } })); }, @@ -299,13 +312,12 @@ async function setUpReplaySlot( async function resolveRegisteredReplayBackend( resolved: ResolvedProviderPlugins, + slot: Slot, inventory: PluginInventoryEntry[], id: string, host: import('@agent-device/contracts/replay-backend').ReplayBackendHost, - packageName?: string, ): Promise { - const slot = requireReplaySlot(resolved, id, packageName); - if (slot.failure !== undefined) throw slot.failure; + if (slot.failure !== undefined) throw replaySlotError(slot); const replayBackend = slot.replaySetup?.replayBackend; if (!replayBackend) throw invalidPlugin(`Replay backend was not set up: ${id}`); return await (slot.replayLoading ??= Promise.resolve().then(async () => { @@ -318,7 +330,7 @@ async function resolveRegisteredReplayBackend( slot.replayLoading = undefined; slot.replayFailure = asPluginFailure(error, 'plugin_load_failed'); inventory.splice(0, inventory.length, ...resolved.inventory()); - throw slot.replayFailure; + throw replaySlotError(slot); } })); } @@ -327,6 +339,7 @@ function requireReplaySlot( resolved: ResolvedProviderPlugins, id: string, packageName: string | undefined, + assertSelectionCurrent?: (packageName: string) => void, ): Slot { const slot = resolved.slots.find((slot) => slot.backend === id) ?? @@ -337,12 +350,38 @@ function requireReplaySlot( slot.backend === undefined && slot.failure !== undefined, ); - if (!slot) - throw new AppError('INVALID_ARGS', `Replay backend is not installed: ${id}`, { - reason: 'plugin-not-installed', - backend: id, - }); - return slot; + if (slot) { + if (slot.candidate.source === 'store' && slot.candidate.package) + assertSelectionCurrent?.(slot.candidate.package); + return slot; + } + if (packageName) assertSelectionCurrent?.(packageName); + throw new AppError('INVALID_ARGS', `Replay backend is not installed: ${id}`, { + reason: 'plugin-not-installed', + backend: id, + }); +} + +function replaySlotError(slot: Slot): AppError { + const error = asPluginFailure(slot.failure ?? slot.replayFailure, 'plugin_load_failed'); + const packageName = slot.candidate.package; + return new AppError( + error.code, + error.message, + { + ...error.details, + ...(packageName + ? { + package: packageName, + updateCommand: `agent-device plugins update ${packageName}`, + hint: `Repair or update ${packageName} on this host, then restart the daemon and retry the original invocation with all its arguments.`, + } + : { + hint: 'Repair the supplied replay plugin, then close and recreate the embedded runtime. Retry the original invocation with all its arguments.', + }), + }, + error, + ); } async function cleanUpReplaySlot(slot: Slot): Promise { diff --git a/packages/plugins/src/replay-backend.test.ts b/packages/plugins/src/replay-backend.test.ts index 8ab8afddac..1db62c668e 100644 --- a/packages/plugins/src/replay-backend.test.ts +++ b/packages/plugins/src/replay-backend.test.ts @@ -114,6 +114,7 @@ test('setup failures stay permanent while failed lazy loads can retry without fa details: { reason: 'plugin_setup_failed' }, }); await expect(registry.resolveReplayBackend('load', host)).rejects.toMatchObject({ + message: 'engine failed', details: { reason: 'plugin_load_failed' }, }); expect(registry.inventory[1]).toMatchObject({ @@ -351,3 +352,30 @@ test('a package name cannot select an active foreign backend for native executio await registry.close(); } }); + +test('a selected native package load failure cannot masquerade as absence by its message', async () => { + const registry = await startProviderRegistry( + await resolveProviderPlugins([ + { + source: 'store', + package: '@agent-device/replay', + load: () => { + throw new Error('Replay backend is not installed: ad'); + }, + }, + ]), + {}, + policy, + ); + try { + await expect(registry.resolveReplayExecution('@agent-device/replay')).rejects.toMatchObject({ + details: { + reason: 'plugin_load_failed', + package: '@agent-device/replay', + updateCommand: 'agent-device plugins update @agent-device/replay', + }, + }); + } finally { + await registry.close(); + } +}); diff --git a/packages/plugins/src/store.test.ts b/packages/plugins/src/store.test.ts index 6e30bbcf85..756ac1ee30 100644 --- a/packages/plugins/src/store.test.ts +++ b/packages/plugins/src/store.test.ts @@ -3,7 +3,12 @@ import fs from 'node:fs'; import path from 'node:path'; import { afterEach, test, vi } from 'vitest'; import { runCmd } from '@agent-device/host-kit/command'; -import { changePlugin, listPlugins } from './store.ts'; +import { + changePlugin, + listPlugins, + createPluginSelectionGuard, + storePluginCandidates, +} from './store.ts'; import { pluginHome, selectV2Plugin, v2RegistrationSource } from '@agent-device/test-kit'; vi.mock('@agent-device/host-kit/command', () => ({ runCmd: vi.fn() })); @@ -246,3 +251,69 @@ test('broken siblings do not block pinned updates or unrelated additions', async ); } }); + +test('a fixed daemon detects replacement of a selected replay installation without importing it', async () => { + const { home, env } = pluginHome(); + const name = '@agent-device/replay'; + selectV2Plugin(home, name, 'throw new Error("selection checks must not import this package");'); + const guard = createPluginSelectionGuard(await storePluginCandidates(env), env); + assert.doesNotThrow(() => guard(name)); + selectV2Plugin(home, '@example/unrelated', 'throw new Error("unrelated");'); + assert.doesNotThrow(() => guard(name)); + selectV2Plugin(home, name, 'throw new Error("replacement, same npm version");'); + assert.throws( + () => guard(name), + (error: unknown) => { + assert.equal( + (error as { details?: { reason?: string } }).details?.reason, + 'daemon-plugin-set-stale', + ); + return true; + }, + ); +}); + +test('a null replay selection is invalid configuration rather than an absent package', () => { + const { home, env } = pluginHome(); + fs.writeFileSync( + path.join(home, 'config.json'), + JSON.stringify({ plugins: { '@agent-device/replay': null } }), + ); + const guard = createPluginSelectionGuard([], env); + assert.throws( + () => guard('@agent-device/replay'), + (error: unknown) => { + assert.equal((error as { details?: { reason?: string } }).details?.reason, 'invalid_plugin'); + return true; + }, + ); +}); + +test('equivalent nested option key order keeps the daemon selection current', async () => { + const { home, env } = pluginHome(); + const name = '@agent-device/replay'; + selectV2Plugin(home, name, 'throw new Error("must not import");'); + const config = readPluginConfig(env); + const selection = config.plugins[name]; + selection.options = { region: 'eu', nested: { retries: 2, labels: ['a', 'b'] } }; + fs.writeFileSync(path.join(home, 'config.json'), JSON.stringify(config)); + const guard = createPluginSelectionGuard(await storePluginCandidates(env), env); + config.plugins[name] = { + options: { nested: { labels: ['a', 'b'], retries: 2 }, region: 'eu' }, + installation: selection.installation, + }; + fs.writeFileSync(path.join(home, 'config.json'), JSON.stringify(config, null, 2)); + assert.doesNotThrow(() => guard(name)); + config.plugins[name].options.nested.labels.reverse(); + fs.writeFileSync(path.join(home, 'config.json'), JSON.stringify(config)); + assert.throws( + () => guard(name), + (error: unknown) => { + assert.equal( + (error as { details?: { reason?: string } }).details?.reason, + 'daemon-plugin-set-stale', + ); + return true; + }, + ); +}); diff --git a/packages/plugins/src/store.ts b/packages/plugins/src/store.ts index 0a572a4bb7..635e1a8bea 100644 --- a/packages/plugins/src/store.ts +++ b/packages/plugins/src/store.ts @@ -3,6 +3,7 @@ import fs from 'node:fs'; import path from 'node:path'; import { pathToFileURL } from 'node:url'; import { isRecord } from '@agent-device/kernel/record'; +import { canonicalJson } from '@agent-device/kernel/collections'; import { AppError } from '@agent-device/kernel/errors'; import { runCmd } from '@agent-device/host-kit/command'; import { @@ -48,6 +49,49 @@ function validatePluginSelection(name: string, selection: PluginSelection): void } } +function selectionStamp(selection: PluginSelection | undefined): string { + return crypto + .createHash('sha256') + .update(canonicalJson(selection ?? null)) + .digest('hex'); +} + +/** Checks a daemon's fixed store selection without importing or replacing executable code. */ +export function createPluginSelectionGuard( + candidates: readonly PluginCandidate[], + env: NodeJS.ProcessEnv, +): (packageName: string) => void { + const selected = new Map( + candidates.flatMap((candidate) => + candidate.package ? [[candidate.package, candidate.selectionStamp] as const] : [], + ), + ); + return (packageName) => { + let current: PluginSelection | undefined; + try { + current = readPluginConfig(env).plugins?.[packageName]; + if (current !== undefined) validatePluginSelection(packageName, current); + } catch (error) { + throw new AppError( + 'INVALID_ARGS', + 'Cannot read the selected replay plugin configuration', + { + reason: 'invalid_plugin', + package: packageName, + hint: 'Repair the plugin configuration on the daemon host, then restart it after finishing your sessions.', + }, + error, + ); + } + if ((selected.get(packageName) ?? selectionStamp(undefined)) !== selectionStamp(current)) + throw new AppError('INVALID_ARGS', 'The running daemon retains an older plugin set.', { + reason: 'daemon-plugin-set-stale', + package: packageName, + hint: 'Finish your sessions and restart the daemon on its host, then retry the original invocation with all its arguments.', + }); + }; +} + function storeHome(env: NodeJS.ProcessEnv): string { return path.dirname(resolveUserConfigPath(env)); } @@ -132,6 +176,7 @@ export async function storePluginCandidates(env: NodeJS.ProcessEnv): Promise loadInstallation(installation), @@ -140,6 +185,7 @@ export async function storePluginCandidates(env: NodeJS.ProcessEnv): Promise Promise.reject(error), }; diff --git a/packages/replay-port/package.json b/packages/replay-port/package.json index 181859b5cd..6a9de03d98 100644 --- a/packages/replay-port/package.json +++ b/packages/replay-port/package.json @@ -6,7 +6,6 @@ "type": "module", "description": "Daemon-side replay port: the adapter over the ad-replay, ad-script and replay-test engines.", "dependencies": { - "@agent-device/ad-replay": "workspace:*", "@agent-device/ad-script": "workspace:*", "@agent-device/capture-kit": "workspace:*", "@agent-device/command-registry": "workspace:*", @@ -14,7 +13,6 @@ "@agent-device/device-selection": "workspace:*", "@agent-device/host-kit": "workspace:*", "@agent-device/kernel": "workspace:*", - "@agent-device/replay-test": "workspace:*", "@agent-device/selectors": "workspace:*", "@agent-device/session-journal": "workspace:*", "@agent-device/plugins": "workspace:*" @@ -139,6 +137,10 @@ "./source-discovery": { "types": "./src/source-discovery.ts", "default": "./src/source-discovery.ts" + }, + "./replay-plugin-admission": { + "types": "./src/daemon-port/replay-plugin-admission.ts", + "default": "./src/daemon-port/replay-plugin-admission.ts" } }, "devDependencies": { diff --git a/packages/replay-port/src/daemon-port/__tests__/replay-backend-registry.test.ts b/packages/replay-port/src/daemon-port/__tests__/replay-backend-registry.test.ts index 3abae7b59c..d6501c3505 100644 --- a/packages/replay-port/src/daemon-port/__tests__/replay-backend-registry.test.ts +++ b/packages/replay-port/src/daemon-port/__tests__/replay-backend-registry.test.ts @@ -1,3 +1,4 @@ +import { getReplayPluginInstallation } from '@agent-device/ad-script'; import { expect, test } from 'vitest'; import { createRuntimeStateHandle } from '@agent-device/host-kit/runtime-slots'; import { getReplayBackendDeclaration } from '../replay-backend-declarations.ts'; @@ -5,7 +6,12 @@ import { getReplayBackend, openReplayBackend } from '../replay-backend-registry. import { mkdtempForTestSync } from '@agent-device/test-kit'; test('help declares the exact optional package without requiring a working plugin', () => { - expect(getReplayBackendDeclaration('maestro')).toMatchObject({ + expect( + getReplayBackendDeclaration( + getReplayPluginInstallation('maestro'), + getReplayPluginInstallation('ad'), + ), + ).toMatchObject({ id: 'maestro', package: '@agent-device/maestro', }); diff --git a/packages/replay-port/src/daemon-port/native-command.ts b/packages/replay-port/src/daemon-port/native-command.ts index 607ae8e22a..e4371b85af 100644 --- a/packages/replay-port/src/daemon-port/native-command.ts +++ b/packages/replay-port/src/daemon-port/native-command.ts @@ -1,5 +1,5 @@ import { normalizeError } from '@agent-device/kernel/errors'; -import { runAdReplay } from '@agent-device/ad-replay'; +import { getReplayExecution } from './replay-backend-registry.ts'; import type { SnapshotTimingSample } from '@agent-device/contracts/capture'; import { summarizeSnapshotTimingSamples } from '@agent-device/contracts/capture'; import type { ReplayCommandResult } from '@agent-device/contracts/replay'; @@ -88,7 +88,9 @@ export async function runReplayCommand(command: ReplayCommand): Promise = { - id: 'maestro', - package: '@agent-device/maestro', - help: { - summary: 'Supported Maestro YAML commands, grammar, and runtime boundaries', - body: `agent-device help maestro +/** Full compatibility help is composed only by help consumers, from the vetted installation. */ +export function getReplayBackendDeclaration( + installation: ReturnType, + suiteInstallation: ReturnType, +): ReplayBackendDeclaration { + const body = + installation.id === 'ad' + ? `Run agent-device plugins add ${installation.package} on the daemon host for native .ad replay and all test suites. Existing scripts and arguments stay the same. Ordinary open/close --save-script recording stays in core. Restart the daemon after changing plugins, then retry the original invocation with all its arguments.` + : `agent-device help maestro + +Install with agent-device plugins add ${installation.package}. ${installation.installationHint} + +For test suites, also run agent-device plugins add ${suiteInstallation.package}. ${suiteInstallation.installationHint} -Install with agent-device plugins add @agent-device/maestro on the client and daemon host. Restart the daemon after installation. +Restart the daemon after installing the required plugins. Run Maestro compatibility flows with replay --maestro or test --maestro. Bind an iOS or Android target with --platform or an existing session. ${formatMaestroCompatibilityReference()} Unsupported syntax fails loudly rather than being skipped. Architecture, performance tradeoffs, and declared conformance divergences: ${MAESTRO_COMPATIBILITY_ADR_URL} -Focused compatibility request: ${MAESTRO_COMPATIBILITY_ISSUE_URL}`, - }, -}; - -/** - * The engine-free half of every registered replay backend (#3377), as `help` lists it. Kept apart - * from the registry so reading a declaration adds nothing to the closure of the daemon routes - * that only load backends. Keys come from the format grammar's vocabulary, and each declaration's - * `id` is typed to its key, as each plugin's is in the registry, so help cannot describe a backend - * other than the one its id loads. - */ -const REPLAY_BACKEND_DECLARATIONS: { - readonly [TId in ReplayBackendId]: ReplayBackendDeclaration; -} = { - maestro: maestroReplayBackendDeclaration, -}; +Focused compatibility request: ${MAESTRO_COMPATIBILITY_ISSUE_URL}`; + return { + id: installation.id, + package: installation.package, + installationHint: installation.installationHint, + help: { summary: installation.summary, body }, + }; +} -export function getReplayBackendDeclaration( - id: ReplayBackendId, -): ReplayBackendDeclaration { - return REPLAY_BACKEND_DECLARATIONS[id]; +/** Installation guidance uses the same vetted package declaration as missing-plugin errors. */ +export function replayPluginInstallHelp( + installation: ReturnType, +): string { + return `${installation.summary} (plugin, install separately). Run agent-device plugins add ${installation.package}. ${installation.installationHint}`; } diff --git a/packages/replay-port/src/daemon-port/replay-backend-registry.ts b/packages/replay-port/src/daemon-port/replay-backend-registry.ts index 289116cb47..706a42eb5b 100644 --- a/packages/replay-port/src/daemon-port/replay-backend-registry.ts +++ b/packages/replay-port/src/daemon-port/replay-backend-registry.ts @@ -1,4 +1,8 @@ -import type { ReplayBackendId } from '@agent-device/ad-script'; +import type { + ReplayExecution, + ReplayExecutionResolver, +} from '@agent-device/contracts/replay-execution'; +import type { ReplayBackendId, ReplayFormat } from '@agent-device/ad-script'; import type { ReplayBackend, ReplayBackendResolver } from '@agent-device/contracts/replay-backend'; import type { AppError } from '@agent-device/kernel/errors'; @@ -7,17 +11,39 @@ async function binding() { const { runtimeSlot } = await import('@agent-device/host-kit/runtime-slots'); return runtimeSlot(backendResolverKey, () => ({ resolve: undefined as ReplayBackendResolver | undefined, + execution: undefined as ReplayExecutionResolver | undefined, })); } /** Binds the fixed plugin set of the current daemon runtime. Its registry owns disposal. */ -export async function bindReplayBackendResolver(resolve: ReplayBackendResolver): Promise { - (await binding()).resolve = resolve; +export async function bindReplayBackendResolver( + resolve: ReplayBackendResolver, + execution?: ReplayExecutionResolver, +): Promise { + Object.assign(await binding(), { resolve, execution }); +} + +/** Resolves native replay and suite execution from this runtime's fixed plugin set. */ +export async function getReplayExecution(): Promise { + const state = await binding(); + if (!state.execution) throw await missingReplayBackend('ad'); + try { + return await state.execution(await backendPackage('ad')); + } catch (error) { + throw await recoverReplayPluginError('ad', error); + } +} + +async function recoverReplayPluginError(id: ReplayFormat, error: unknown): Promise { + const { AppError } = await import('@agent-device/kernel/errors'); + if (!(error instanceof AppError) || error.details?.reason !== 'plugin-not-installed') + return error; + return await missingReplayBackend(id); } /** Resolves against the current daemon's plugins, without consulting another host's store. */ export async function getReplayBackend(id: ReplayBackendId): Promise { - const resolve = (await binding()).resolve; + const { resolve } = await binding(); if (!resolve) throw await missingReplayBackend(id); return await resolveBackend(id, resolve); } @@ -26,14 +52,11 @@ async function resolveBackend( id: ReplayBackendId, resolve: ReplayBackendResolver, ): Promise { - const { AppError } = await import('@agent-device/kernel/errors'); const { createReplayBackendHost } = await import('./replay-backend-host.ts'); try { return await resolve(id, createReplayBackendHost(), await backendPackage(id)); } catch (error) { - if (error instanceof AppError && error.details?.reason === 'plugin-not-installed') - throw await missingReplayBackend(id); - throw error; + throw await recoverReplayPluginError(id, error); } } @@ -87,7 +110,7 @@ export async function openReplayBackend(id: ReplayBackendId, env: NodeJS.Process } } -async function missingReplayBackend(id: ReplayBackendId): Promise { +async function missingReplayBackend(id: ReplayFormat): Promise { const { AppError } = await import('@agent-device/kernel/errors'); const packageName = await backendPackage(id); const installCommand = `agent-device plugins add ${packageName}`; @@ -99,11 +122,11 @@ async function missingReplayBackend(id: ReplayBackendId): Promise { backend: id, package: packageName, installCommand, - hint: 'Install on the client for source collection/export and on the daemon host for execution. Restart the daemon after changing plugins; retry the original command with all its arguments.', + hint: `${(await import('@agent-device/ad-script')).getReplayPluginInstallation(id).installationHint} Restart the daemon after changing plugins; retry the original command with all its arguments.`, }, ); } -async function backendPackage(id: ReplayBackendId): Promise { - return (await import('./replay-backend-declarations.ts')).getReplayBackendDeclaration(id).package; +async function backendPackage(id: ReplayFormat): Promise { + return (await import('@agent-device/ad-script')).getReplayPluginInstallation(id).package; } diff --git a/packages/replay-port/src/daemon-port/replay-plugin-admission.ts b/packages/replay-port/src/daemon-port/replay-plugin-admission.ts new file mode 100644 index 0000000000..6ae7eed784 --- /dev/null +++ b/packages/replay-port/src/daemon-port/replay-plugin-admission.ts @@ -0,0 +1,48 @@ +import type { ReplayBackendId } from '@agent-device/ad-script'; +import type { DaemonWireRequest } from '@agent-device/contracts/command'; +import { AppError } from '@agent-device/kernel/errors'; +import { getReplayBackend, getReplayExecution } from './replay-backend-registry.ts'; + +/** Rejects unavailable execution before request binding can select or act on a device. */ +export async function requireReplayPlugins(request: DaemonWireRequest): Promise { + const { isReplayBackendId, unsupportedReplayBackendMessage } = + await import('@agent-device/ad-script'); + const backend = request.flags?.replayBackend; + if (backend !== undefined && !isReplayBackendId(backend)) + throw new AppError('INVALID_ARGS', unsupportedReplayBackendMessage(backend)); + if (request.command === 'test') await requireReplayTestPlugins(request, backend); + else if (request.command === 'replay') await requireReplayEntryPlugin(request, backend); +} + +async function requireReplayTestPlugins( + request: DaemonWireRequest, + backend: ReplayBackendId | undefined, +): Promise { + const { resolveReplayFormat, isMaestroYamlPath, maestroBackendRequiredMessage } = + await import('@agent-device/ad-script'); + const entry = request.flags?.replayScriptSources?.find((source) => + isMaestroYamlPath(source.entry), + )?.entry; + if (entry !== undefined && backend !== 'maestro') + throw new AppError('INVALID_ARGS', maestroBackendRequiredMessage('test', entry)); + await getReplayExecution(); + if ( + request.flags?.replayScriptSources?.some( + (source) => resolveReplayFormat(source.entry, backend) === 'maestro', + ) + ) + await getReplayBackend('maestro'); +} + +async function requireReplayEntryPlugin( + request: DaemonWireRequest, + backend: ReplayBackendId | undefined, +): Promise { + const { resolveReplayFormat, isMaestroYamlPath, maestroBackendRequiredMessage } = + await import('@agent-device/ad-script'); + const entry = request.flags?.replayScriptSource?.entry ?? request.positionals?.[0] ?? ''; + if (isMaestroYamlPath(entry) && backend !== 'maestro') + throw new AppError('INVALID_ARGS', maestroBackendRequiredMessage('replay', entry)); + if (resolveReplayFormat(entry, backend) === 'maestro') await getReplayBackend('maestro'); + else await getReplayExecution(); +} diff --git a/packages/replay-port/src/daemon-port/session-replay-runtime-plan.ts b/packages/replay-port/src/daemon-port/session-replay-runtime-plan.ts index 2cb6ad48a6..24986b1b82 100644 --- a/packages/replay-port/src/daemon-port/session-replay-runtime-plan.ts +++ b/packages/replay-port/src/daemon-port/session-replay-runtime-plan.ts @@ -1,4 +1,4 @@ -import { inspectAdReplay } from '@agent-device/ad-replay'; +import type { ReplayExecution } from '@agent-device/contracts/replay-execution'; import type { SessionAction } from '@agent-device/contracts/session'; import type { ReplayDispatchOptions } from '@agent-device/contracts/replay'; import type { CommandFlags, DaemonWireRequest } from '@agent-device/contracts/command'; @@ -89,6 +89,7 @@ export type PreparedReplayPlan = { }; export function prepareReplayPlan(params: { + inspectAdReplay: ReplayExecution['inspectAdReplay']; req: DaemonWireRequest; /** The command's runtime host; variable sources read environment and cwd from it. */ host: HostEnvironment; @@ -116,7 +117,12 @@ export function prepareReplayPlan(params: { const backendRejection = validateReplayBackendFlag(req); if (backendRejection) return { ok: false, response: backendRejection }; - const { manifest, replayReq } = inspectReplayPlanManifest(req, script, dispatch); + const { manifest, replayReq } = inspectReplayPlanManifest( + req, + script, + dispatch, + params.inspectAdReplay, + ); const { metadata, actions, actionLines, planDigest } = manifest; const preEntrySession = sessionStore.get(); const entryIndexResult = resolveReplayPlanEntryIndex({ @@ -179,6 +185,7 @@ function inspectReplayPlanManifest( req: DaemonWireRequest, script: string, dispatch: ReplayDispatchOptions | undefined, + inspectAdReplay: ReplayExecution['inspectAdReplay'], ): { manifest: AdReplayManifest; replayReq: ReplayDispatchRequest } { const manifest = inspectAdReplay(script, { platform: req.flags?.platform, diff --git a/packages/replay-port/src/daemon-port/test-command.ts b/packages/replay-port/src/daemon-port/test-command.ts index 28488f8951..d30b14c19b 100644 --- a/packages/replay-port/src/daemon-port/test-command.ts +++ b/packages/replay-port/src/daemon-port/test-command.ts @@ -1,4 +1,4 @@ -import { runReplayTestSuite } from '@agent-device/replay-test'; +import { getReplayExecution } from './replay-backend-registry.ts'; /** Runs the replay-test scheduler and its nested replay attempts. */ import { randomUUID } from 'node:crypto'; @@ -195,7 +195,9 @@ export async function runReplayTestCommand(command: ReplayTestCommand): Promise< return errorResponse(appErr.code, appErr.message); } const sourceBundlesByPath = new Map(sourceBundles.map((bundle) => [bundle.entry, bundle])); - const outcome = await runReplayTestSuite({ + const outcome = await ( + await getReplayExecution() + ).runReplayTestSuite({ request: suiteRequest, // The host owns the request-global progress sink; the scheduler receives only the // narrow emit capability (#1478 P3b). diff --git a/packages/replay-port/tsconfig.json b/packages/replay-port/tsconfig.json index 062682eb66..ea675e52e0 100644 --- a/packages/replay-port/tsconfig.json +++ b/packages/replay-port/tsconfig.json @@ -11,9 +11,6 @@ }, "include": ["src", "../command-registry/src/global.d.ts"], "references": [ - { - "path": "../ad-replay" - }, { "path": "../ad-script" }, @@ -35,9 +32,6 @@ { "path": "../kernel" }, - { - "path": "../replay-test" - }, { "path": "../selectors" }, diff --git a/packages/replay/README.md b/packages/replay/README.md new file mode 100644 index 0000000000..5b1b7b548d --- /dev/null +++ b/packages/replay/README.md @@ -0,0 +1,50 @@ +# Run native replay and test suites + +Install `@agent-device/replay` on the daemon host before running native `.ad` scripts or any `test` suite: + +```sh +agent-device plugins add @agent-device/replay +``` + +Finish your sessions, then restart the daemon on that host. Retry your original command with all its arguments. Existing `.ad` syntax, variables, target identity, divergence reports, resume and repair do not need a migration. + +The plugin uses API version 2 and supports core `>=0.22.0-0 <0.23.0`. The extraction's pinned development host, `0.21.38-dev`, is also accepted. Other core releases, including the next development bump, fail with `incompatible_plugin` until compatibility is verified and the declared range is updated. The managed API version 2 check also applies. Use matching core and plugin releases. The package remains private until a maintainer publishes a non-`latest` npm placeholder and configures trusted publishing for `.github/workflows/release.yml`, job `npm-publish`. + +## Choose the installation host + +| Workflow | Client host | Daemon host | +| --- | --- | --- | +| Native `.ad` replay | Core reads and sends the source | `@agent-device/replay` | +| Native `.ad` test suites | Core discovers and sends sources | `@agent-device/replay` | +| Maestro replay | `@agent-device/maestro` collects includes | `@agent-device/maestro` | +| Maestro test suites | `@agent-device/maestro` collects includes | Both plugins | +| `replay export` to Maestro | `@agent-device/maestro` | No daemon required | + +Installing locally does not install anything on a remote daemon. Run `plugins add` on each host that needs the package. + +For an embedded daemon, supply the factory explicitly: + +```js +import { startAgentDevice } from 'agent-device'; +import replay from '@agent-device/replay/agent-device'; + +const runtime = await startAgentDevice({ + stateDir: '/tmp/checkout-agent', + plugins: [replay()], +}); +try { + await runtime.client.replay.run({ path: './checkout.ad' }); +} finally { + await runtime.close(); +} +``` + +An embedded runtime uses its supplied plugins, including an empty list. It does not borrow a conflicting plugin store. Its host must close and recreate it to change that set. + +## Recover from an unavailable plugin + +`plugin-not-installed` includes the exact package and `installCommand`. Install it on the named host, restart that daemon, then retry the original invocation with all its arguments. + +`plugin_load_failed`, `invalid_plugin`, `incompatible_plugin` and `plugin_setup_failed` describe a selected package that failed. Repair or update that package; another backend is never selected automatically. `daemon-plugin-set-stale` means the running daemon has an older set than the store. Finish your sessions and restart it. No error installs a package or terminates your sessions automatically. + +Ordinary `open --save-script`, `session save-script` and `close` recording remain in core. Replay-specific `--save-script` repair requires this plugin. Core retains sessions, dispatch, selectors, refs, recording, cancellation and cleanup. The plugin receives the existing explicit step and suite ports; shared `.ad` data and divergence readers remain independent of execution. diff --git a/packages/replay/package.json b/packages/replay/package.json new file mode 100644 index 0000000000..26fdfe76dc --- /dev/null +++ b/packages/replay/package.json @@ -0,0 +1,37 @@ +{ + "name": "@agent-device/replay", + "version": "0.0.0", + "private": true, + "sideEffects": false, + "type": "module", + "description": "Optional native .ad replay and test-suite execution plugin for agent-device.", + "exports": { + "./agent-device": { + "types": "./src/plugin.ts", + "default": "./src/plugin.ts" + } + }, + "devDependencies": { + "@agent-device/ad-replay": "workspace:*", + "@agent-device/replay-test": "workspace:*", + "agent-device": "workspace:*", + "@agent-device/plugins": "workspace:*", + "@agent-device/test-kit": "workspace:*" + }, + "scripts": { + "build": "tsdown --config tsdown.config.ts", + "prepack": "pnpm build" + }, + "files": [ + "dist" + ], + "publishConfig": { + "exports": { + "./agent-device": { + "types": "./dist/agent-device.d.mts", + "import": "./dist/agent-device.mjs" + } + }, + "access": "public" + } +} diff --git a/packages/replay/src/plugin.test.ts b/packages/replay/src/plugin.test.ts new file mode 100644 index 0000000000..1e549a3e5d --- /dev/null +++ b/packages/replay/src/plugin.test.ts @@ -0,0 +1,83 @@ +import fs from 'node:fs'; +import path from 'node:path'; +import { mkdtempForTestSync } from '@agent-device/test-kit'; +import { expect, test, vi } from 'vitest'; +import { createPluginHost } from '@agent-device/plugins/host'; +import { resolveProviderPlugins, startProviderRegistry } from '@agent-device/plugins/registry'; +import replayPlugin from './plugin.ts'; + +test('the first-party factory uses the managed registry and preserves native script syntax', async () => { + const registry = await startProviderRegistry( + await resolveProviderPlugins([{ source: 'embedder', load: replayPlugin }]), + {}, + { + adaptWebDriver: async () => { + throw new Error('Replay cannot create providers'); + }, + }, + ); + try { + const execution = await registry.resolveReplayExecution(); + const manifest = execution.inspectAdReplay( + 'open com.example.app\nfill "id=password" "${PASSWORD}"\nclose', + ); + expect(manifest.actions.map((action) => action.command)).toEqual(['open', 'fill', 'close']); + expect(manifest.actions[1]?.positionals).toContain('${PASSWORD}'); + const artifactsDir = mkdtempForTestSync('replay-plugin-suite-'); + const sourcePath = path.join(artifactsDir, 'flow.ad'); + fs.writeFileSync(sourcePath, 'open com.example.app\nclose\n'); + const runReplay = vi.fn(async () => ({ + status: 'passed' as const, + replayed: 2, + healed: 0, + warnings: [], + artifactPaths: [], + })); + const cleanupSession = vi.fn(async () => {}); + const release = vi.fn(); + const suite = await execution.runReplayTestSuite({ + request: { + inputs: [sourcePath], + sessionName: 'package-suite', + artifactsDir, + }, + runReplay, + cleanupSession, + emitProgress: () => {}, + isCanceled: () => false, + emitDiagnostic: () => {}, + bindAttemptCancellation: () => ({ cancel: () => {}, release }), + discoverSources: () => [ + { path: sourcePath, manifest: { device: { platform: { kind: 'unspecified' } } } }, + ], + resolveShardTargets: async () => [], + }); + expect(suite, JSON.stringify(suite)).toMatchObject({ + status: 'completed', + data: { passed: 1, failed: 0 }, + }); + expect(runReplay).toHaveBeenCalledOnce(); + expect(cleanupSession).toHaveBeenCalledOnce(); + expect(release).toHaveBeenCalledOnce(); + } finally { + await registry.close(); + } +}); + +test.each(['0.23.0', '0.21.39-dev'])('the package rejects unverified core %s', (clientVersion) => { + const host = { ...createPluginHost({}, undefined), clientVersion }; + expect(() => replayPlugin().setup(host)).toThrow( + expect.objectContaining({ + details: expect.objectContaining({ reason: 'incompatible_plugin' }), + }), + ); +}); + +test.each(['0.21.38-dev', '0.22.0-beta.1', '0.22.0'])( + 'the package accepts declared core %s', + (clientVersion) => { + expect( + replayPlugin().setup({ ...createPluginHost({}, undefined), clientVersion }), + ).toMatchObject({ execution: { create: expect.any(Function) } }); + }, +); diff --git a/packages/replay/src/plugin.ts b/packages/replay/src/plugin.ts new file mode 100644 index 0000000000..c4029905b8 --- /dev/null +++ b/packages/replay/src/plugin.ts @@ -0,0 +1,32 @@ +import type { ReplayEnginePlugin } from 'agent-device/plugins'; + +/** Native .ad execution and suite scheduling use host-owned dispatch, sessions and cancellation. */ +function replayPlugin(): ReplayEnginePlugin<'ad'> { + return { + apiVersion: 2, + kind: 'replay-backend', + backend: 'ad', + setup: (host) => { + if (host.clientVersion !== '0.21.38-dev' && !host.clientVersion.startsWith('0.22.')) { + throw host.createError('INVALID_ARGS', 'The replay plugin requires agent-device 0.22', { + reason: 'incompatible_plugin', + supportedCoreVersions: '>=0.22.0-0 <0.23.0 || 0.21.38-dev', + hint: 'Install matching agent-device and @agent-device/replay releases, then restart the daemon.', + }); + } + return { + execution: { + create: async () => { + const [{ inspectAdReplay, runAdReplay }, { runReplayTestSuite }] = await Promise.all([ + import('@agent-device/ad-replay'), + import('@agent-device/replay-test'), + ]); + return { inspectAdReplay, runAdReplay, runReplayTestSuite }; + }, + }, + }; + }, + }; +} + +export { replayPlugin as default }; diff --git a/packages/replay/test/package-smoke.mjs b/packages/replay/test/package-smoke.mjs new file mode 100644 index 0000000000..efa58c9d4e --- /dev/null +++ b/packages/replay/test/package-smoke.mjs @@ -0,0 +1,88 @@ +import assert from 'node:assert/strict'; +import { runCmdSync } from '@agent-device/host-kit/command'; +import fs from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; +import { test } from 'node:test'; + +const root = path.resolve(import.meta.dirname, '../../..'); +function run(command, args, cwd) { + return runCmdSync(command, args, { + cwd, + env: process.env, + timeoutMs: 120_000, + maxBuffer: 16 * 1024 * 1024, + }).stdout; +} +function files(directory) { + return fs.readdirSync(directory, { withFileTypes: true }).flatMap((entry) => { + const file = path.join(directory, entry.name); + return entry.isDirectory() ? files(file) : [file]; + }); +} + +test('packed core excludes native execution and a standalone replay plugin uses the real host', () => { + const scratch = fs.mkdtempSync(path.join(os.tmpdir(), 'agent-device-replay-package-')); + try { + const coreTarball = path.join( + scratch, + JSON.parse( + run('npm', ['pack', '--ignore-scripts', '--json', '--pack-destination', scratch], root), + )[0].filename, + ); + const pluginTarball = path.join(scratch, 'replay.tgz'); + run('pnpm', ['pack', '--out', pluginTarball], path.join(root, 'packages/replay')); + const consumer = path.join(scratch, 'consumer'); + fs.mkdirSync(consumer); + fs.writeFileSync(path.join(consumer, 'package.json'), '{"private":true,"type":"module"}'); + run( + 'npm', + ['install', '--ignore-scripts', '--no-audit', '--no-fund', coreTarball, pluginTarball], + consumer, + ); + const core = path.join(consumer, 'node_modules/agent-device'); + const plugin = path.join(consumer, 'node_modules/@agent-device/replay'); + const manifest = JSON.parse(fs.readFileSync(path.join(plugin, 'package.json'), 'utf8')); + assert.deepEqual(Object.keys(manifest.exports), ['./agent-device']); + assert.equal(manifest.dependencies, undefined); + assert.equal(manifest.peerDependencies, undefined); + assert.ok(!fs.existsSync(path.join(plugin, 'node_modules/agent-device'))); + const coreCode = files(path.join(core, 'dist')) + .filter((file) => file.endsWith('.js')) + .map((file) => fs.readFileSync(file, 'utf8')) + .join('\n'); + for (const engineMarker of [ + 'Expected passing replay test outcome.', + 'replay accepts .ad script files. JSON replay payloads are no longer supported.', + ]) + assert.ok( + !coreCode.includes(engineMarker), + `Executable native engine leaked: ${engineMarker}`, + ); + const pluginCode = files(path.join(plugin, 'dist')) + .filter((file) => file.endsWith('.mjs')) + .map((file) => fs.readFileSync(file, 'utf8')) + .join('\n'); + assert.ok(pluginCode.includes('Expected passing replay test outcome.')); + assert.ok(pluginCode.includes('JSON replay payloads are no longer supported.')); + for (const command of ['replay', 'test']) { + const help = run( + process.execPath, + [path.join(core, 'bin/agent-device.mjs'), 'help', command], + consumer, + ); + assert.match(help, /plugin, install separately/); + assert.match(help, /agent-device plugins add @agent-device\/replay/); + } + assert.match( + run( + process.execPath, + [path.join(root, 'packages/replay/test/packed-runtime.mjs'), core, plugin, scratch], + consumer, + ), + /Packed native replay/, + ); + } finally { + fs.rmSync(scratch, { recursive: true, force: true }); + } +}); diff --git a/packages/replay/test/packed-runtime.mjs b/packages/replay/test/packed-runtime.mjs new file mode 100644 index 0000000000..e3fbc65bdd --- /dev/null +++ b/packages/replay/test/packed-runtime.mjs @@ -0,0 +1,96 @@ +import assert from 'node:assert/strict'; +import fs from 'node:fs'; +import path from 'node:path'; +import { pathToFileURL } from 'node:url'; + +const [core, plugin, scratch] = process.argv.slice(2); +const { startAgentDevice } = await import(pathToFileURL(path.join(core, 'dist/src/index.js'))); +const { default: replayPlugin } = await import( + pathToFileURL(path.join(plugin, 'dist/agent-device.mjs')) +); +const script = path.join(scratch, 'empty.ad'); +fs.writeFileSync(script, '# native replay entrypoint and suite transport\n'); +const host = { + cwd: scratch, + env: { + AGENT_DEVICE_HOME: path.join(scratch, 'conflicting-home'), + AGENT_DEVICE_DAEMON_IDLE_TIMEOUT_MS: '0', + }, +}; +fs.mkdirSync(host.env.AGENT_DEVICE_HOME, { recursive: true }); +fs.writeFileSync( + path.join(host.env.AGENT_DEVICE_HOME, 'config.json'), + JSON.stringify({ + plugins: { '@agent-device/replay': { installation: 'invalid-conflicting-store' } }, + }), +); +async function embedded(label, plugins) { + return await startAgentDevice({ host, plugins, stateDir: path.join(scratch, label) }); +} +const absent = await embedded('absent', []); +try { + for (const run of [ + () => absent.client.replay.run({ path: script }), + () => absent.client.replay.test({ paths: [script] }), + ]) + await assert.rejects(run(), (error) => { + assert.equal(error.code, 'INVALID_ARGS'); + assert.equal(error.details.reason, 'plugin-not-installed'); + assert.equal(error.details.package, '@agent-device/replay'); + assert.equal(error.details.installCommand, 'agent-device plugins add @agent-device/replay'); + return true; + }); +} finally { + await absent.close(); +} +for (const [label, plugin, reason] of [ + ['incompatible', { ...replayPlugin(), apiVersion: 99 }, 'incompatible_plugin'], + ['invalid', { ...replayPlugin(), setup: () => ({ execution: {} }) }, 'invalid_plugin'], + [ + 'setup', + { + ...replayPlugin(), + setup: () => { + throw new Error('selected setup failed'); + }, + }, + 'plugin_setup_failed', + ], +]) { + const runtime = await embedded(label, [plugin]); + try { + await assert.rejects(runtime.client.replay.run({ path: script }), (error) => { + assert.equal(error.details.reason, reason); + return true; + }); + } finally { + await runtime.close(); + } +} +const installed = await embedded('installed', [replayPlugin()]); +try { + const replay = await installed.client.replay.run({ path: script }); + assert.equal(replay.replayed, 0); + assert.equal(replay.sessionActive, false); + assert.deepEqual(replay.artifactPaths, []); + const suite = await installed.client.replay.test({ + paths: [script], + artifactsDir: path.join(scratch, 'artifacts'), + }); + assert.equal(suite.passed, 1); + assert.equal(suite.failed, 0); + assert.equal(suite.tests[0].status, 'passed'); + assert.ok(fs.existsSync(suite.artifactsDir)); + const malformed = path.join(scratch, 'legacy.ad'); + fs.writeFileSync(malformed, '[{"command":"open"}]'); + await assert.rejects(installed.client.replay.run({ path: malformed }), (error) => { + assert.equal(error.code, 'INVALID_ARGS'); + assert.match(error.message, /JSON replay payloads are no longer supported/); + return true; + }); +} finally { + await installed.close(); +} +process.stdout.write( + 'Packed native replay, suites, failure isolation and embedded scopes passed.\n', +); diff --git a/packages/replay/tsconfig.json b/packages/replay/tsconfig.json new file mode 100644 index 0000000000..9b836387f6 --- /dev/null +++ b/packages/replay/tsconfig.json @@ -0,0 +1,30 @@ +{ + "extends": "../../tsconfig.json", + "compilerOptions": { + "composite": true, + "noEmit": false, + "emitDeclarationOnly": true, + "declaration": true, + "declarationDir": "./dist-types", + "rootDir": ".", + "tsBuildInfoFile": "./dist-types/index.tsbuildinfo", + "paths": { + "agent-device/plugins": ["./node_modules/agent-device/src/sdk/plugins.ts"] + } + }, + "include": ["src"], + "references": [ + { + "path": "../ad-replay" + }, + { + "path": "../replay-test" + }, + { + "path": "../plugins" + }, + { + "path": "../test-kit" + } + ] +} diff --git a/packages/replay/tsdown.config.ts b/packages/replay/tsdown.config.ts new file mode 100644 index 0000000000..84fe67d349 --- /dev/null +++ b/packages/replay/tsdown.config.ts @@ -0,0 +1,26 @@ +import { defineConfig } from 'tsdown'; + +type BuildLog = Readonly<{ code?: string; message: string }>; + +function rejectUnresolvedWorkspaceImports( + level: string, + log: BuildLog, + handler: (level: string, log: BuildLog) => void, +): void { + if (log.code === 'UNRESOLVED_IMPORT') throw new Error(log.message); + handler(level, log); +} + +export default defineConfig({ + entry: { 'agent-device': 'src/plugin.ts' }, + outDir: 'dist', + tsconfig: '../../tsconfig.lib.json', + format: 'esm', + platform: 'node', + target: 'es2022', + minify: true, + dts: true, + hash: false, + deps: { alwaysBundle: [/^@agent-device\//] }, + inputOptions: { onLog: rejectUnresolvedWorkspaceImports }, +}); diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 146ca9b87e..50570dab81 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -23,9 +23,6 @@ importers: .: devDependencies: - '@agent-device/ad-replay': - specifier: workspace:* - version: link:packages/ad-replay '@agent-device/ad-script': specifier: workspace:* version: link:packages/ad-script @@ -113,9 +110,6 @@ importers: '@agent-device/replay-port': specifier: workspace:* version: link:packages/replay-port - '@agent-device/replay-test': - specifier: workspace:* - version: link:packages/replay-test '@agent-device/revyl': specifier: workspace:* version: link:packages/provider-revyl @@ -535,9 +529,6 @@ importers: '@agent-device/replay-port': specifier: workspace:* version: link:../replay-port - '@agent-device/replay-test': - specifier: workspace:* - version: link:../replay-test '@agent-device/selectors': specifier: workspace:* version: link:../selectors @@ -554,9 +545,15 @@ importers: '@agent-device/maestro': specifier: workspace:* version: link:../maestro + '@agent-device/plugins': + specifier: workspace:* + version: link:../plugins '@agent-device/proxy': specifier: workspace:* version: link:../proxy + '@agent-device/replay': + specifier: workspace:* + version: link:../replay '@agent-device/test-kit': specifier: workspace:* version: link:../test-kit @@ -1061,11 +1058,26 @@ importers: specifier: ^3.2.0 version: 3.2.0 - packages/replay-port: - dependencies: + packages/replay: + devDependencies: '@agent-device/ad-replay': specifier: workspace:* version: link:../ad-replay + '@agent-device/plugins': + specifier: workspace:* + version: link:../plugins + '@agent-device/replay-test': + specifier: workspace:* + version: link:../replay-test + '@agent-device/test-kit': + specifier: workspace:* + version: link:../test-kit + agent-device: + specifier: workspace:* + version: link:../.. + + packages/replay-port: + dependencies: '@agent-device/ad-script': specifier: workspace:* version: link:../ad-script @@ -1090,9 +1102,6 @@ importers: '@agent-device/plugins': specifier: workspace:* version: link:../plugins - '@agent-device/replay-test': - specifier: workspace:* - version: link:../replay-test '@agent-device/selectors': specifier: workspace:* version: link:../selectors diff --git a/scripts/__tests__/install-workspace-replay-plugins.test.ts b/scripts/__tests__/install-workspace-replay-plugins.test.ts new file mode 100644 index 0000000000..366527c79f --- /dev/null +++ b/scripts/__tests__/install-workspace-replay-plugins.test.ts @@ -0,0 +1,42 @@ +import assert from 'node:assert/strict'; +import fs from 'node:fs'; +import path from 'node:path'; +import { afterEach, test, vi } from 'vitest'; +import { mkdtempForTestSync } from '@agent-device/test-kit'; +import { installWorkspaceReplayPlugins } from '../install-workspace-replay-plugins.ts'; + +afterEach(() => vi.restoreAllMocks()); + +test('failed second staging keeps the previous config and removes both new installations', () => { + const home = mkdtempForTestSync('workspace-plugin-staging-'); + const config = '{"platform":"android","plugins":{"existing":{"installation":"kept"}}}'; + const file = path.join(home, 'config.json'); + fs.writeFileSync(file, config); + const symlink = fs.symlinkSync; + vi.spyOn(fs, 'symlinkSync').mockImplementation((target, destination, type) => { + if (path.basename(String(target)) === 'replay') throw new Error('second staging failed'); + symlink(target, destination, type); + }); + assert.throws( + () => installWorkspaceReplayPlugins({ AGENT_DEVICE_HOME: home }, true), + /second staging failed/, + ); + assert.equal(fs.readFileSync(file, 'utf8'), config); + assert.deepEqual(fs.readdirSync(path.join(home, 'plugins')), []); +}); + +test('failed publication keeps the previous config and removes staged installations', () => { + const home = mkdtempForTestSync('workspace-plugin-publish-'); + const config = '{"platform":"android"}'; + const file = path.join(home, 'config.json'); + fs.writeFileSync(file, config); + vi.spyOn(fs, 'renameSync').mockImplementation(() => { + throw new Error('publish failed'); + }); + assert.throws( + () => installWorkspaceReplayPlugins({ AGENT_DEVICE_HOME: home }, true), + /publish failed/, + ); + assert.equal(fs.readFileSync(file, 'utf8'), config); + assert.deepEqual(fs.readdirSync(path.join(home, 'plugins')), []); +}); diff --git a/scripts/install-workspace-maestro.ts b/scripts/install-workspace-maestro.ts deleted file mode 100644 index b972a2c0db..0000000000 --- a/scripts/install-workspace-maestro.ts +++ /dev/null @@ -1,47 +0,0 @@ -import crypto from 'node:crypto'; -import fs from 'node:fs'; -import path from 'node:path'; -import { resolveUserConfigPath } from '@agent-device/host-kit/file'; - -/** Installs the explicitly selected workspace build into an isolated validation plugin store. */ -export function installWorkspaceMaestro(env: NodeJS.ProcessEnv, source: boolean): void { - if (!env.AGENT_DEVICE_HOME) - throw new Error('Workspace plugin installation requires AGENT_DEVICE_HOME'); - const configPath = resolveUserConfigPath(env); - const home = path.dirname(configPath); - const installation = crypto.randomUUID(); - const directory = path.join( - home, - 'plugins', - installation, - 'node_modules', - '@agent-device/maestro', - ); - const workspace = path.resolve(import.meta.dirname, '../packages/maestro'); - fs.mkdirSync(path.dirname(directory), { recursive: true }); - if (source) fs.symlinkSync(workspace, directory, 'dir'); - else { - fs.mkdirSync(directory); - fs.cpSync(path.join(workspace, 'dist'), path.join(directory, 'dist'), { recursive: true }); - const manifest = JSON.parse(fs.readFileSync(path.join(workspace, 'package.json'), 'utf8')); - fs.writeFileSync( - path.join(directory, 'package.json'), - JSON.stringify({ - name: manifest.name, - version: manifest.version, - type: 'module', - exports: manifest.publishConfig.exports, - }), - ); - } - const config = fs.existsSync(configPath) ? JSON.parse(fs.readFileSync(configPath, 'utf8')) : {}; - fs.writeFileSync( - configPath, - JSON.stringify({ - ...config, - plugins: { ...config.plugins, '@agent-device/maestro': { installation } }, - }), - ); -} - -if (process.argv[1] === import.meta.filename) installWorkspaceMaestro(process.env, false); diff --git a/scripts/install-workspace-replay-plugins.ts b/scripts/install-workspace-replay-plugins.ts new file mode 100644 index 0000000000..2fcf3be114 --- /dev/null +++ b/scripts/install-workspace-replay-plugins.ts @@ -0,0 +1,68 @@ +import crypto from 'node:crypto'; +import fs from 'node:fs'; +import path from 'node:path'; +import { publishFileSync, resolveUserConfigPath } from '@agent-device/host-kit/file'; + +/** Installs the explicitly selected workspace build into an isolated validation plugin store. */ +export function installWorkspaceReplayPlugins(env: NodeJS.ProcessEnv, source: boolean): void { + if (!env.AGENT_DEVICE_HOME) + throw new Error('Workspace plugin installation requires AGENT_DEVICE_HOME'); + const configPath = resolveUserConfigPath(env); + const home = path.dirname(configPath); + const staged = ['maestro', 'replay'].map((packageId) => ({ + packageId, + installation: crypto.randomUUID(), + })); + try { + for (const { packageId, installation } of staged) + installWorkspacePlugin(source, home, packageId, installation); + const config = fs.existsSync(configPath) ? JSON.parse(fs.readFileSync(configPath, 'utf8')) : {}; + const selections = Object.fromEntries( + staged.map(({ packageId, installation }) => [`@agent-device/${packageId}`, { installation }]), + ); + publishFileSync({ + destination: configPath, + contents: JSON.stringify({ ...config, plugins: { ...config.plugins, ...selections } }), + mode: 0o600, + preserveExistingMode: true, + }); + } catch (error) { + for (const { installation } of staged) + fs.rmSync(path.join(home, 'plugins', installation), { recursive: true, force: true }); + throw error; + } +} + +function installWorkspacePlugin( + source: boolean, + home: string, + packageId: string, + installation: string, +): void { + const directory = path.join( + home, + 'plugins', + installation, + 'node_modules', + `@agent-device/${packageId}`, + ); + const workspace = path.resolve(import.meta.dirname, `../packages/${packageId}`); + fs.mkdirSync(path.dirname(directory), { recursive: true }); + if (source) fs.symlinkSync(workspace, directory, 'dir'); + else { + fs.mkdirSync(directory); + fs.cpSync(path.join(workspace, 'dist'), path.join(directory, 'dist'), { recursive: true }); + const manifest = JSON.parse(fs.readFileSync(path.join(workspace, 'package.json'), 'utf8')); + fs.writeFileSync( + path.join(directory, 'package.json'), + JSON.stringify({ + name: manifest.name, + version: manifest.version, + type: 'module', + exports: manifest.publishConfig.exports, + }), + ); + } +} + +if (process.argv[1] === import.meta.filename) installWorkspaceReplayPlugins(process.env, false); diff --git a/scripts/layering/model.ts b/scripts/layering/model.ts index c77ce379d4..6d1fd33090 100644 --- a/scripts/layering/model.ts +++ b/scripts/layering/model.ts @@ -62,6 +62,7 @@ const TARGET_DAG_RANK = new Map([ ['device-selection', 1], ['maestro', 1], ['replay-port', 1], + ['replay', 1], ['replay-test', 1], ['selectors', 1], ['session-journal', 1], diff --git a/scripts/layering/package-boundaries.test.ts b/scripts/layering/package-boundaries.test.ts index 66368a2faa..dc0c3b7ba1 100644 --- a/scripts/layering/package-boundaries.test.ts +++ b/scripts/layering/package-boundaries.test.ts @@ -6,6 +6,7 @@ import assert from 'node:assert/strict'; import { execFileSync } from 'node:child_process'; import fs from 'node:fs'; import os from 'node:os'; +import { createRequire } from 'node:module'; import path from 'node:path'; import { test } from 'node:test'; import { listSourceFiles } from './check.ts'; @@ -866,8 +867,9 @@ test('the real tree parses, declares, and passes R11', () => { 'root must declare the ad-script workspace dependency', ); assert.ok( - rootWorkspaceDependencyNames(repoRoot).has('@agent-device/ad-replay'), - 'root must declare the ad-replay workspace dependency', + !rootWorkspaceDependencyNames(repoRoot).has('@agent-device/ad-replay') && + !rootWorkspaceDependencyNames(repoRoot).has('@agent-device/replay-test'), + 'core must not depend on the optional native execution engines', ); assert.ok( rootWorkspaceDependencyNames(repoRoot).has('@agent-device/selectors'), @@ -889,6 +891,7 @@ test('the real tree parses, declares, and passes R11', () => { }); test('Node resolution enforces the exports map at runtime', () => { + const replayPackageRequire = createRequire(path.join(repoRoot, 'packages/replay/package.json')); // Real resolver, not the gate's model: a deep import is a resolution error, // and the legal subpath realpaths OUTSIDE node_modules (pnpm symlink), which // is what lets --experimental-strip-types load package sources in dev. @@ -915,11 +918,15 @@ test('Node resolution enforces the exports map at runtime', () => { '@agent-device/ad-replay/testing', '@agent-device/ad-replay/internal/target-verification.ts', '@agent-device/ad-replay/src/index.ts', + '@agent-device/replay-test/src/internal/session-test.ts', '@agent-device/selectors/internal/parse.ts', '@agent-device/selectors/src/index.ts', ]) { assert.throws( - () => import.meta.resolve(deep), + () => + deep.startsWith('@agent-device/ad-replay/') || deep.startsWith('@agent-device/replay-test/') + ? replayPackageRequire.resolve(deep) + : import.meta.resolve(deep), /ERR_PACKAGE_PATH_NOT_EXPORTED|Package subpath|No "exports" main/, `${deep} must not resolve`, ); @@ -952,7 +959,9 @@ test('Node resolution enforces the exports map at runtime', () => { assert.ok(xmlResolved.endsWith('packages/xml/src/index.ts'), xmlResolved); const adScriptResolved = import.meta.resolve('@agent-device/ad-script'); assert.ok(adScriptResolved.endsWith('packages/ad-script/src/index.ts'), adScriptResolved); - const adReplayResolved = import.meta.resolve('@agent-device/ad-replay'); + const adReplayResolved = replayPackageRequire.resolve('@agent-device/ad-replay'); + const replayTestResolved = replayPackageRequire.resolve('@agent-device/replay-test'); + assert.ok(replayTestResolved.endsWith('packages/replay-test/src/index.ts'), replayTestResolved); assert.ok(adReplayResolved.endsWith('packages/ad-replay/src/index.ts'), adReplayResolved); const selectorsResolved = import.meta.resolve('@agent-device/selectors'); assert.ok(selectorsResolved.endsWith('packages/selectors/src/index.ts'), selectorsResolved); diff --git a/scripts/layering/project-references.test.ts b/scripts/layering/project-references.test.ts index 76e8949e58..1bbeb147c0 100644 --- a/scripts/layering/project-references.test.ts +++ b/scripts/layering/project-references.test.ts @@ -115,7 +115,7 @@ test('the compiled-set predicate is licensed by the tsconfig include/exclude sha } }); -test('root workspace dependencies cover every non-Maestro package and exclude the optional engine', () => { +test('root workspace dependencies cover every core package and exclude optional engines', () => { const manifest = JSON.parse(fs.readFileSync(path.join(repoRoot, 'package.json'), 'utf8')) as { dependencies?: Record; devDependencies?: Record; @@ -128,7 +128,15 @@ test('root workspace dependencies cover every non-Maestro package and exclude th .map(([name]) => name) .sort(); const expected = readWorkspacePackages(repoRoot) - .filter((pkg) => pkg.name !== '@agent-device/maestro') + .filter( + (pkg) => + ![ + '@agent-device/maestro', + '@agent-device/replay', + '@agent-device/ad-replay', + '@agent-device/replay-test', + ].includes(pkg.name), + ) .map((pkg) => pkg.name) .sort(); assert.deepEqual( @@ -139,7 +147,12 @@ test('root workspace dependencies cover every non-Maestro package and exclude th 'specifiers through these declarations)', ); assert.ok( - !declared.includes('@agent-device/maestro'), + ![ + '@agent-device/maestro', + '@agent-device/replay', + '@agent-device/ad-replay', + '@agent-device/replay-test', + ].some((name) => declared.includes(name)), 'The optional engine must be installed through the plugin store. R11 rejects undeclared imports from core.', ); }); diff --git a/scripts/vitest-tmpdir-global-setup.ts b/scripts/vitest-tmpdir-global-setup.ts index 09577bc850..90659584ce 100644 --- a/scripts/vitest-tmpdir-global-setup.ts +++ b/scripts/vitest-tmpdir-global-setup.ts @@ -1,4 +1,4 @@ -import { installWorkspaceMaestro } from './install-workspace-maestro.ts'; +import { installWorkspaceReplayPlugins } from './install-workspace-replay-plugins.ts'; import fs from 'node:fs'; import os from 'node:os'; import path from 'node:path'; @@ -57,7 +57,7 @@ export function setup(): void { process.env.TMPDIR = testRunTmpDir; previousPluginHome = process.env.AGENT_DEVICE_HOME; process.env.AGENT_DEVICE_HOME = path.join(testRunTmpDir, 'plugin-home'); - installWorkspaceMaestro(process.env, true); + installWorkspaceReplayPlugins(process.env, true); } // fallow-ignore-next-line unused-export diff --git a/src/__tests__/cli-maestro-suite.test.ts b/src/__tests__/cli-maestro-suite.test.ts index 2a524637a1..e792ae7899 100644 --- a/src/__tests__/cli-maestro-suite.test.ts +++ b/src/__tests__/cli-maestro-suite.test.ts @@ -1,5 +1,5 @@ import { test } from 'vitest'; -import { installedMaestroCliTestEnv } from './cli-test-env.ts'; +import { installedReplayCliTestEnv } from './cli-test-env.ts'; import assert from 'node:assert/strict'; import { promises as fs } from 'node:fs'; import path from 'node:path'; @@ -30,7 +30,7 @@ test('test --maestro forwards Maestro backend and platform for directory suites' tests: [], }, }), - { cwd: tmpDir, env: installedMaestroCliTestEnv() }, + { cwd: tmpDir, env: installedReplayCliTestEnv() }, ); assert.equal(result.code, null); diff --git a/src/__tests__/cli-replay-script-sources.test.ts b/src/__tests__/cli-replay-script-sources.test.ts index 42b2c4fc48..4ca757191f 100644 --- a/src/__tests__/cli-replay-script-sources.test.ts +++ b/src/__tests__/cli-replay-script-sources.test.ts @@ -9,7 +9,7 @@ import path from 'node:path'; import assert from 'node:assert/strict'; import { test } from 'vitest'; import type { ReplayScriptSourceBundle } from '@agent-device/contracts/replay'; -import { installedMaestroCliTestEnv } from './cli-test-env.ts'; +import { installedReplayCliTestEnv } from './cli-test-env.ts'; import { runCliCapture } from './cli-capture.ts'; import { mkdtempForTestSync } from '@agent-device/test-kit'; @@ -86,7 +86,7 @@ test('a Maestro replay against a remote daemon sends its runFlow includes too', ['replay', './flows/checkout.yaml', '--maestro', '--platform', 'ios'], { cwd, - env: { ...REMOTE_DAEMON_ENV, ...installedMaestroCliTestEnv() }, + env: { ...REMOTE_DAEMON_ENV, ...installedReplayCliTestEnv() }, defaultResponse: { ok: true, data: { replayed: 1 } }, }, ); @@ -115,7 +115,7 @@ test.each(['native', 'maestro'])( ['test', maestro ? './*.yaml' : './*.ad', '--json', ...(maestro ? ['--maestro'] : [])], { cwd, - env: { ...REMOTE_DAEMON_ENV, ...(maestro ? installedMaestroCliTestEnv() : {}) }, + env: { ...REMOTE_DAEMON_ENV, ...(maestro ? installedReplayCliTestEnv() : {}) }, defaultResponse: { ok: true, data: { tests: [], total: 0, passed: 0, failed: 0 } }, }, ); diff --git a/src/__tests__/cli-startup-import-closure.test.ts b/src/__tests__/cli-startup-import-closure.test.ts index 19d668bc0c..a5ba6f846f 100644 --- a/src/__tests__/cli-startup-import-closure.test.ts +++ b/src/__tests__/cli-startup-import-closure.test.ts @@ -27,6 +27,9 @@ const LAZY_HTTP_MODULES = new Set(['node:http', 'node:https']); */ function isLazyEngineModule(specifier: string): boolean { return ( + ['@agent-device/replay', '@agent-device/ad-replay', '@agent-device/replay-test'].some( + (name) => specifier === name || specifier.startsWith(`${name}/`), + ) || specifier === '@agent-device/maestro' || (specifier.startsWith('@agent-device/maestro/') && specifier !== '@agent-device/maestro/replay-backend-declaration') @@ -34,6 +37,15 @@ function isLazyEngineModule(specifier: string): boolean { } test('the lazy engine set covers the plugin subpath but not its engine-free declaration', () => { + for (const name of [ + '@agent-device/replay', + '@agent-device/ad-replay', + '@agent-device/replay-test', + ]) { + expect(isLazyEngineModule(name)).toBe(true); + expect(isLazyEngineModule(`${name}/agent-device`)).toBe(true); + expect(isLazyEngineModule(`${name}ish`)).toBe(false); + } expect(isLazyEngineModule('@agent-device/maestro')).toBe(true); expect(isLazyEngineModule('@agent-device/maestro/replay-backend')).toBe(true); expect(isLazyEngineModule('@agent-device/maestro/replay-backend-declaration')).toBe(false); @@ -116,7 +128,7 @@ test('the CLI startup import closure never evaluates node:http or node:https', ( ).toEqual([]); }); -test('the CLI startup import closure never evaluates the Maestro engine', () => { +test('the CLI startup import closure never evaluates a lazy replay engine', () => { const offenders: string[] = []; for (const file of eagerClosureOf(path.join(srcRoot, 'cli.ts'))) { for (const specifier of eagerlyEvaluatedModules(file, fs.readFileSync(file, 'utf8'))) { @@ -128,13 +140,12 @@ test('the CLI startup import closure never evaluates the Maestro engine', () => expect( offenders, - 'Load @agent-device/maestro on demand instead: the command registry evaluates every command ' + - 'family on startup, so importing the engine here costs every warm CLI command ~28ms of ' + - 'YAML-parser evaluation and a 131 kB startup chunk, for a format most runs never use.', + 'Load replay engine packages on demand instead: the command registry evaluates every command ' + + 'family on startup, so eager engine imports add their parsing and execution code to every CLI invocation.', ).toEqual([]); }); -test('the replay command source closure keeps Maestro behind flow selection', () => { +test('the replay command source closure keeps replay engines behind execution or source selection', () => { const replayEntry = path.join(srcRoot, '../packages/command-surface/src/replay/index.ts'); const replayClosure = eagerClosureOf(replayEntry); const sourceBundle = path.join(srcRoot, '../packages/replay-port/src/script-source-bundle.ts'); @@ -149,7 +160,7 @@ test('the replay command source closure keeps Maestro behind flow selection', () expect( offenders, - 'The replay command may evaluate @agent-device/maestro only inside flow source collection.', + 'The replay command must load replay engines only for execution or flow source collection.', ).toEqual([]); }); diff --git a/src/__tests__/cli-test-env.ts b/src/__tests__/cli-test-env.ts index f2493c9099..97119c682d 100644 --- a/src/__tests__/cli-test-env.ts +++ b/src/__tests__/cli-test-env.ts @@ -1,4 +1,4 @@ -import { installWorkspaceMaestro } from '../../scripts/install-workspace-maestro.ts'; +import { installWorkspaceReplayPlugins } from '../../scripts/install-workspace-replay-plugins.ts'; import fs from 'node:fs'; import path from 'node:path'; import { mkdtempForTestSync } from '@agent-device/test-kit'; @@ -34,6 +34,13 @@ export function installIsolatedCliTestEnv( } } + if (!explicitKeys.has('AGENT_DEVICE_HOME') && !explicitKeys.has('HOME')) { + if (!previousEnv.has('AGENT_DEVICE_HOME')) + previousEnv.set('AGENT_DEVICE_HOME', process.env.AGENT_DEVICE_HOME); + process.env.AGENT_DEVICE_HOME = path.join(tempRoot, 'plugins'); + installWorkspaceReplayPlugins(process.env, true); + } + return () => { for (const [key, value] of previousEnv.entries()) { if (value === undefined) { @@ -46,8 +53,8 @@ export function installIsolatedCliTestEnv( }; } -export function installedMaestroCliTestEnv(): Record { - const env = { AGENT_DEVICE_HOME: mkdtempForTestSync('agent-device-cli-maestro-') }; - installWorkspaceMaestro(env, true); +export function installedReplayCliTestEnv(): Record { + const env = { AGENT_DEVICE_HOME: mkdtempForTestSync('agent-device-cli-replay-') }; + installWorkspaceReplayPlugins(env, true); return env; } diff --git a/src/__tests__/replay-plugin-setup.ts b/src/__tests__/replay-plugin-setup.ts index b59846bbf5..e92e4463d8 100644 --- a/src/__tests__/replay-plugin-setup.ts +++ b/src/__tests__/replay-plugin-setup.ts @@ -27,7 +27,7 @@ beforeEach(async () => { }, 'replay-backends', ); - await bindReplayBackendResolver(registry.resolveReplayBackend); + await bindReplayBackendResolver(registry.resolveReplayBackend, registry.resolveReplayExecution); }); afterEach(async () => { await registry?.close(); diff --git a/src/cli/commands/__tests__/replay-maestro-export.test.ts b/src/cli/commands/__tests__/replay-maestro-export.test.ts index 911fa3b27e..32057071d7 100644 --- a/src/cli/commands/__tests__/replay-maestro-export.test.ts +++ b/src/cli/commands/__tests__/replay-maestro-export.test.ts @@ -1,4 +1,4 @@ -import { installedMaestroCliTestEnv } from '../../../__tests__/cli-test-env.ts'; +import { installedReplayCliTestEnv } from '../../../__tests__/cli-test-env.ts'; import fs from 'node:fs'; import path from 'node:path'; import { parseAllDocuments } from 'yaml'; @@ -19,7 +19,7 @@ describe('exportReplayScriptToMaestro', () => { const open = withApp ? `com.example.app ${link} --relaunch` : link; fs.writeFileSync(sourcePath, `open ${open}\n`); - const env = installedMaestroCliTestEnv(); + const env = installedReplayCliTestEnv(); const result = await runCliCapture(['replay', 'export', sourcePath, '--json'], { env }); expect(result.code).toBeNull(); diff --git a/src/cli/parser/__tests__/args-maestro-support-matrix.test.ts b/src/cli/parser/__tests__/args-maestro-support-matrix.test.ts index ce4af0cbfc..7f32e6fef1 100644 --- a/src/cli/parser/__tests__/args-maestro-support-matrix.test.ts +++ b/src/cli/parser/__tests__/args-maestro-support-matrix.test.ts @@ -20,6 +20,9 @@ test('Maestro flag routes to the versioned compatibility help topic', () => { test('Maestro help covers the supported subset and operational boundaries', async () => { const help = await usageForCommand('maestro'); expect(help).not.toBeNull(); + expect(help).toContain('agent-device plugins add @agent-device/maestro'); + expect(help).toContain('agent-device plugins add @agent-device/replay'); + expect(help).toContain('on the daemon host for native .ad replay and all test suites'); for (const statement of [ ...MAESTRO_COMPAT_SUPPORTED_CAPABILITIES, ...MAESTRO_COMPAT_LIMITATIONS, diff --git a/src/commands/schema/cli-help-overview.ts b/src/commands/schema/cli-help-overview.ts index 51fa29cf6c..cddf5817c3 100644 --- a/src/commands/schema/cli-help-overview.ts +++ b/src/commands/schema/cli-help-overview.ts @@ -49,7 +49,7 @@ Rules: More commands (exact shapes: agent-device help ): open install devices apps boot close app and device lifecycle screenshot record logs network perf trace evidence and diagnostics - replay test batch session scripted flows + replay test batch session scripted flows (replay/test plugins, install separately) alert keyboard clipboard settings gesture system and input Guides (agent-device help ): diff --git a/src/commands/schema/cli-help.ts b/src/commands/schema/cli-help.ts index 35b397f26d..c7d16b5f4b 100644 --- a/src/commands/schema/cli-help.ts +++ b/src/commands/schema/cli-help.ts @@ -1,6 +1,10 @@ +import { getReplayPluginInstallation } from '@agent-device/ad-script'; import { gestureHelpTopic } from './cli-help-gestures.ts'; import { listCliCommandNames } from '@agent-device/command-registry/catalog'; -import { getReplayBackendDeclaration } from '@agent-device/replay-port/replay-backend-declarations'; +import { + getReplayBackendDeclaration, + replayPluginInstallHelp, +} from '@agent-device/replay-port/replay-backend-declarations'; import { normalizeCliCommandAlias } from '@agent-device/command-registry/cli-command-aliases'; import { helpBody } from '@agent-device/command-registry/command-text'; import { getCliCommandSchema, getCommandSchema } from '@agent-device/command-surface'; @@ -70,7 +74,10 @@ const HELP_TOPICS = { body: 'agent-device help commands', }, ...manualQaHelpTopics, - maestro: getReplayBackendDeclaration('maestro').help, + maestro: getReplayBackendDeclaration( + getReplayPluginInstallation('maestro'), + getReplayPluginInstallation('ad'), + ).help, workflow: { summary: 'Normal agent-device bootstrap, exploration, and validation loop', body: `agent-device help workflow @@ -875,6 +882,16 @@ export function buildCommandUsageText(commandName: string): string | null { schema.flagDescriptionOverrides, ); const sections: string[] = []; + if (commandName === 'replay' || commandName === 'test') { + sections.push(replayPluginInstallHelp(getReplayPluginInstallation('ad'))); + sections.push( + getReplayBackendDeclaration( + getReplayPluginInstallation('ad'), + getReplayPluginInstallation('ad'), + ).help.body, + ); + sections.push(replayPluginInstallHelp(getReplayPluginInstallation('maestro'))); + } if (commandName === 'connect') { sections.push(`Optional provider plugins:\n${providerPluginHelp()}`); } diff --git a/src/daemon-composition-replay-selection.test.ts b/src/daemon-composition-replay-selection.test.ts new file mode 100644 index 0000000000..e721d9d526 --- /dev/null +++ b/src/daemon-composition-replay-selection.test.ts @@ -0,0 +1,139 @@ +import fs from 'node:fs'; +import path from 'node:path'; +import { expect, test } from 'vitest'; +import { createRuntimeStateHandle } from '@agent-device/host-kit/runtime-slots'; +import { storePluginCandidates } from '@agent-device/plugins/store'; +import { + getReplayBackend, + getReplayExecution, +} from '@agent-device/replay-port/replay-backend-registry'; +import { + createTestHostEnvironment, + mkdtempForTestSync, + pluginHome, + selectV2Plugin, +} from '@agent-device/test-kit'; +import { composeDaemonRuntimeDeps } from './daemon-composition.ts'; + +function pluginSource(backend: 'ad' | 'maestro') { + const capability = + backend === 'ad' + ? `execution: { create: () => ({ + inspectAdReplay() {}, runAdReplay() {}, runReplayTestSuite() {} + }) }` + : `replayBackend: { create: () => ({ + id: 'maestro', collectSourceFiles() {}, inspectSource() {}, + runReplay() {}, exportReplayScript() {} + }) }`; + return `export default () => ({ + apiVersion: 2, kind: 'replay-backend', backend: '${backend}', + setup: () => ({ ${capability} }) + });`; +} + +async function composedRuntime(env: NodeJS.ProcessEnv, embedded = false) { + const state = createRuntimeStateHandle(); + const baseDir = mkdtempForTestSync('replay-selection-runtime-'); + const candidates = embedded ? await storePluginCandidates(env) : undefined; + try { + const deps = await state.run(() => + composeDaemonRuntimeDeps( + { + host: createTestHostEnvironment({ env }), + baseDir, + sessionsDir: path.join(baseDir, 'sessions'), + assertShutdownAllowed: undefined, + resolveSessionDir: (id) => path.join(baseDir, 'sessions', id), + resolveSessionArtifacts: () => { + throw new Error('Replay resolution must not create a session'); + }, + openWebSessionNames: () => [], + }, + candidates, + ), + ); + return { + resolve: (backend: 'ad' | 'maestro') => + state.run(() => (backend === 'ad' ? getReplayExecution() : getReplayBackend(backend))), + close: async () => { + await deps.shutdownProviderRuntimes(); + state.dispose(); + }, + }; + } catch (error) { + state.dispose(); + throw error; + } +} + +test.each([ + ['ad', 'replace'], + ['ad', 'remove'], + ['maestro', 'replace'], + ['maestro', 'remove'], +] as const)( + 'daemon refuses a retained custom %s implementation after %s', + async (backend, change) => { + const { home, env } = pluginHome(); + const name = `@example/${backend}`; + const source = pluginSource(backend); + selectV2Plugin(home, name, source); + const runtime = await composedRuntime(env); + try { + const first = await runtime.resolve(backend); + selectV2Plugin(home, '@example/unrelated', pluginSource(backend)); + expect(await runtime.resolve(backend)).toBe(first); + if (change === 'replace') { + selectV2Plugin(home, name, source); + } else { + const configPath = path.join(home, 'config.json'); + const config = JSON.parse(fs.readFileSync(configPath, 'utf8')); + delete config.plugins[name]; + fs.writeFileSync(configPath, JSON.stringify(config)); + } + await expect(runtime.resolve(backend)).rejects.toMatchObject({ + details: { reason: 'daemon-plugin-set-stale', package: name }, + }); + } finally { + await runtime.close(); + } + }, +); + +test.each(['ad', 'maestro'] as const)( + 'embedded %s implementations remain independent of store changes', + async (backend) => { + const { home, env } = pluginHome(); + selectV2Plugin(home, `@example/${backend}`, pluginSource(backend)); + const runtime = await composedRuntime(env, true); + try { + const first = await runtime.resolve(backend); + fs.writeFileSync(path.join(home, 'config.json'), '{broken'); + expect(await runtime.resolve(backend)).toBe(first); + } finally { + await runtime.close(); + } + }, +); + +test.each([ + ['ad', '@agent-device/replay'], + ['maestro', '@agent-device/maestro'], +] as const)( + 'a newly installed %s catalog package requires daemon restart', + async (backend, name) => { + const { home, env } = pluginHome(); + const runtime = await composedRuntime(env); + try { + await expect(runtime.resolve(backend)).rejects.toMatchObject({ + details: { reason: 'plugin-not-installed', package: name }, + }); + selectV2Plugin(home, name, pluginSource(backend)); + await expect(runtime.resolve(backend)).rejects.toMatchObject({ + details: { reason: 'daemon-plugin-set-stale', package: name }, + }); + } finally { + await runtime.close(); + } + }, +); diff --git a/src/daemon-composition.ts b/src/daemon-composition.ts index d42f91ad9b..2e9d935c70 100644 --- a/src/daemon-composition.ts +++ b/src/daemon-composition.ts @@ -93,6 +93,7 @@ export async function composeDaemonRuntimeDeps( ): Promise< DaemonRuntimeDeps & { resolveReplayBackend: import('@agent-device/contracts/replay-backend').ReplayBackendResolver; + resolveReplayExecution: import('@agent-device/contracts/replay-execution').ReplayExecutionResolver; } > { const { @@ -120,7 +121,10 @@ export async function composeDaemonRuntimeDeps( try { const { bindReplayBackendResolver } = await import('@agent-device/replay-port/replay-backend-registry'); - await bindReplayBackendResolver(providerComposition.resolveReplayBackend); + await bindReplayBackendResolver( + providerComposition.resolveReplayBackend, + providerComposition.resolveReplayExecution, + ); const providerDeviceRuntimes = [...providerComposition.runtimes]; const deviceRuntimeGateway = createPlatformRuntimeGateway({ host, @@ -170,6 +174,7 @@ export async function composeDaemonRuntimeDeps( shutdownProviderRuntimes: providerComposition.shutdown, pluginInventory: providerComposition.inventory, resolveReplayBackend: providerComposition.resolveReplayBackend, + resolveReplayExecution: providerComposition.resolveReplayExecution, leaseLifecycleProvider: providerRuntimeProviders.leaseLifecycleProvider, providerRuntimeIds: providerRuntimeProviders.providerRuntimeIds, recoverExpiredLease: providerRuntimeProviders.recoverExpiredLease, diff --git a/src/host-replay.test.ts b/src/host-replay.test.ts index 0174eadba8..e2159d48b9 100644 --- a/src/host-replay.test.ts +++ b/src/host-replay.test.ts @@ -114,6 +114,13 @@ function fixture() { return { root, entry, host, stateDir: path.join(root, 'state') }; } +async function nativePlugin() { + const resolved = await resolveProviderPlugins(await storePluginCandidates(process.env)); + const plugin = resolved.slots.find((slot) => slot.backend === 'ad')?.replayPlugin; + if (!plugin) throw new Error('Native replay test installation is missing'); + return plugin; +} + async function trackedPlugin() { const resolved = await resolveProviderPlugins(await storePluginCandidates(process.env)); const maestro = resolved.slots.find((slot) => slot.backend === 'maestro')?.replayPlugin; @@ -162,7 +169,7 @@ test.each(['replay', 'test', 'batch-test'] as const)( await state.run(async () => { const f = fixture(); const { plugin, calls } = await trackedPlugin(); - const host = await startAgentDevice({ ...f, plugins: [plugin] }); + const host = await startAgentDevice({ ...f, plugins: [plugin, await nativePlugin()] }); const input = { path: f.entry, platform: 'ios' as const, backend: 'maestro' }; const testInput = { paths: [f.entry], platform: 'ios' as const, backend: 'maestro' }; try { diff --git a/src/mcp/__tests__/command-tools-parity.test.ts b/src/mcp/__tests__/command-tools-parity.test.ts index ca7629f8c6..726ccfe90d 100644 --- a/src/mcp/__tests__/command-tools-parity.test.ts +++ b/src/mcp/__tests__/command-tools-parity.test.ts @@ -19,6 +19,20 @@ afterEach(() => { let temporaryDirectory: string | undefined; +test('MCP replay discovery identifies optional plugins and installation scopes', () => { + const tools = listCommandTools({ outputSchemas: true }); + for (const name of ['replay', 'test']) { + const tool = tools.find((candidate) => candidate.name === name); + assert.ok(tool); + assert.match(tool.description, /plugin, install separately/); + assert.match(tool.description, /Native \.ad replay and test suites/); + assert.match(tool.description, /Install on the daemon host/); + assert.match(tool.description, /Install on the client for source collection/); + assert.ok(tool.inputSchema.properties); + assert.ok(tool.outputSchema); + } +}); + test('MCP collection results use object envelopes without changing object results or text', async () => { const results = { devices: [ diff --git a/src/mcp/__tests__/command-tools-replay-plugin.test.ts b/src/mcp/__tests__/command-tools-replay-plugin.test.ts new file mode 100644 index 0000000000..0fcd8633a1 --- /dev/null +++ b/src/mcp/__tests__/command-tools-replay-plugin.test.ts @@ -0,0 +1,47 @@ +import fs from 'node:fs'; +import path from 'node:path'; +import { expect, test } from 'vitest'; +import { createTestHostEnvironment, mkdtempForTestSync } from '@agent-device/test-kit'; +import { startAgentDevice } from '../../host.ts'; +import { createCommandToolExecutor } from '../command-tools.ts'; + +test.each([undefined, 'declared'] as const)( + 'MCP preserves optional replay recovery through the real client/router in %s mode', + async (structuredContent) => { + const root = mkdtempForTestSync('mcp-replay-plugin-'); + const script = path.join(root, 'flow.ad'); + fs.writeFileSync(script, 'open com.example.app\nclose\n'); + const runtime = await startAgentDevice({ + stateDir: path.join(root, 'state'), + plugins: [], + host: createTestHostEnvironment({ env: { AGENT_DEVICE_HOME: path.join(root, 'home') } }), + }); + try { + const executor = createCommandToolExecutor({ + createClient: () => runtime.client, + structuredContent, + }); + for (const [command, input] of [ + ['replay', { path: script }], + ['test', { paths: [script] }], + ] as const) { + const result = await executor.execute(command, input); + expect(result.isError).toBe(true); + expect(result.content[0]?.text).toContain('agent-device plugins add @agent-device/replay'); + expect(result.content[0]?.text).toContain('original command with all its arguments'); + if (structuredContent) + expect(result.structuredContent).toMatchObject({ + code: 'INVALID_ARGS', + details: { + reason: 'plugin-not-installed', + package: '@agent-device/replay', + installCommand: 'agent-device plugins add @agent-device/replay', + }, + }); + else expect(result.structuredContent).toBeUndefined(); + } + } finally { + await runtime.close(); + } + }, +); diff --git a/src/provider-device-runtimes.ts b/src/provider-device-runtimes.ts index 6035283c70..72437a93d2 100644 --- a/src/provider-device-runtimes.ts +++ b/src/provider-device-runtimes.ts @@ -4,13 +4,14 @@ import type { PluginCleanupOutcome, PluginInventoryEntry, } from '@agent-device/contracts/plugin-inventory'; -import type { PluginCandidate } from '@agent-device/plugins/registry'; +import type { PluginCandidate, PluginRegistryPolicy } from '@agent-device/plugins/registry'; import type { PlatformRuntimeProviderRegistration } from './platform-runtime-gateway.ts'; import { builtinProviderCandidates } from './builtin-provider-plugins.ts'; import { PROVIDER_PLUGIN_POLICY } from './provider-plugin-policy.ts'; export type ProviderRuntimeComposition = Readonly<{ resolveReplayBackend: import('@agent-device/plugins/replay-backend').ReplayBackendResolver; + resolveReplayExecution: import('@agent-device/contracts/replay-execution').ReplayExecutionResolver; runtimes: readonly ProviderDeviceRuntime[]; platformModules: readonly PlatformRuntimeProviderRegistration[]; /** Every plugin the registry saw, built-in or not, active or failed. */ @@ -26,17 +27,27 @@ export type ProviderRuntimeComposition = Readonly<{ export async function createDaemonProviderRuntimeComposition( host: HostEnvironment, ): Promise { - const { storePluginCandidates } = await import('@agent-device/plugins/store'); - return await createEmbeddedProviderRuntimeComposition( - host, - await storePluginCandidates(host.env), - ); + const { storePluginCandidates, createPluginSelectionGuard } = + await import('@agent-device/plugins/store'); + const candidates = await storePluginCandidates(host.env); + return await composeProviderRuntime(host, candidates, { + ...PROVIDER_PLUGIN_POLICY, + assertReplaySelectionCurrent: createPluginSelectionGuard(candidates, host.env), + }); } /** Composes built-ins followed by supplied candidates without reading the plugin store. */ export async function createEmbeddedProviderRuntimeComposition( host: HostEnvironment, candidates: readonly PluginCandidate[], +): Promise { + return await composeProviderRuntime(host, candidates, PROVIDER_PLUGIN_POLICY); +} + +async function composeProviderRuntime( + host: HostEnvironment, + candidates: readonly PluginCandidate[], + policy: PluginRegistryPolicy, ): Promise { const { resolveProviderPlugins, startProviderRegistry } = await import('@agent-device/plugins/registry'); @@ -44,7 +55,7 @@ export async function createEmbeddedProviderRuntimeComposition( ...builtinProviderCandidates(host), ...candidates, ]); - const registry = await startProviderRegistry(resolved, host.env, PROVIDER_PLUGIN_POLICY); + const registry = await startProviderRegistry(resolved, host.env, policy); return Object.freeze({ runtimes: Object.freeze(registry.registrations.map(({ runtime }) => runtime)), platformModules: Object.freeze( @@ -55,6 +66,7 @@ export async function createEmbeddedProviderRuntimeComposition( ), inventory: registry.inventory, resolveReplayBackend: registry.resolveReplayBackend, + resolveReplayExecution: registry.resolveReplayExecution, shutdown: registry.close, }); } diff --git a/test/integration/installed-package-metro.test.ts b/test/integration/installed-package-metro.test.ts index 036bb81abf..1ffa91b772 100644 --- a/test/integration/installed-package-metro.test.ts +++ b/test/integration/installed-package-metro.test.ts @@ -54,7 +54,11 @@ function ensureBuiltPackage(): void { }); } -function extractInstalledPackage(tarballPath: string, consumerRoot: string): string { +function extractInstalledPackage( + tarballPath: string, + consumerRoot: string, + packageName = 'agent-device', +): string { const nodeModulesRoot = path.join(consumerRoot, 'node_modules'); fs.mkdirSync(nodeModulesRoot, { recursive: true }); runCmdSync('tar', ['-xzf', tarballPath, '-C', nodeModulesRoot], { @@ -62,7 +66,8 @@ function extractInstalledPackage(tarballPath: string, consumerRoot: string): str timeoutMs: SUBPROCESS_TIMEOUT_MS, }); const extractedPackageRoot = path.join(nodeModulesRoot, 'package'); - const installedPackageRoot = path.join(nodeModulesRoot, 'agent-device'); + const installedPackageRoot = path.join(nodeModulesRoot, packageName); + fs.mkdirSync(path.dirname(installedPackageRoot), { recursive: true }); fs.renameSync(extractedPackageRoot, installedPackageRoot); return installedPackageRoot; } @@ -92,14 +97,24 @@ async function runNodeModuleJson(cwd: string, args: string[], script: string): P /** A v2 plugin written against the packed `agent-device/plugins` types, type-checked as a consumer. */ async function typeCheckInstalledPluginContract(consumerRoot: string): Promise { + for (const name of ['maestro', 'replay']) { + const tarball = path.join(consumerRoot, `${name}.tgz`); + runCmdSync('pnpm', ['pack', '--out', tarball], { + cwd: path.join(repoRoot, 'packages', name), + timeoutMs: SUBPROCESS_TIMEOUT_MS, + }); + extractInstalledPackage(tarball, consumerRoot, `@agent-device/${name}`); + } const source = path.join(consumerRoot, 'plugin-contract-smoke.ts'); fs.writeFileSync( source, `import type { AgentDevicePluginFactory } from 'agent-device/plugins'; import { startAgentDevice, type AgentDeviceHost, type StartAgentDeviceOptions, type AgentDeviceClientConfig } from 'agent-device'; +import maestro from '@agent-device/maestro/agent-device'; +import replay from '@agent-device/replay/agent-device'; export async function hostConsumer(host: NonNullable) { - const options: StartAgentDeviceOptions = { stateDir: './state', host, plugins: [await example()] }; + const options: StartAgentDeviceOptions = { stateDir: './state', host, plugins: [await example(), maestro(), replay()] }; const running: AgentDeviceHost = await startAgentDevice(options); const config: AgentDeviceClientConfig = running.clientConfig; void config; diff --git a/test/integration/provider-scenarios/active-session-script-publication.test.ts b/test/integration/provider-scenarios/active-session-script-publication.test.ts index 3b7f5b4d97..d9185a3450 100644 --- a/test/integration/provider-scenarios/active-session-script-publication.test.ts +++ b/test/integration/provider-scenarios/active-session-script-publication.test.ts @@ -4,7 +4,7 @@ import os from 'node:os'; import path from 'node:path'; import { emitDiagnostic } from '@agent-device/host-kit/diagnostics'; import { INTERNAL_COMMANDS } from '@agent-device/command-registry/catalog'; -import { test } from 'vitest'; +import { test, vi } from 'vitest'; import { assertRpcError, assertRpcOk } from './assertions.ts'; import { androidSettingsXml, createAndroidSettingsWorld } from './android-world.ts'; import { withProviderScenarioResource } from './harness.ts'; @@ -496,3 +496,32 @@ test('#1398: a read-only wait recorded after a parameterized fill never re-seria }, ); }, 20_000); + +test('ordinary open/close recording works with no replay plugin and replay refuses before another device call', async () => { + const root = fs.mkdtempSync(path.join(os.tmpdir(), 'core-recording-no-plugin-')); + vi.stubEnv('AGENT_DEVICE_HOME', path.join(root, 'empty-plugin-home')); + try { + await withProviderScenarioResource(createAndroidSettingsWorld, async (world) => { + const scriptPath = path.join(root, 'recorded.ad'); + const client = world.daemon.client(); + await client.apps.open({ app: 'settings', saveScript: scriptPath, ...world.selection }); + await client.command.wait({ selector: 'label="Search"', ...world.selection }); + await client.sessions.close(); + assert.match(fs.readFileSync(scriptPath, 'utf8'), /agent-device:target-v1/); + const deviceCalls = world.adbCalls.length; + await assert.rejects( + client.replay.run({ path: scriptPath, ...world.selection }), + (error: unknown) => { + assert.equal( + (error as { details?: { reason?: string } }).details?.reason, + 'plugin-not-installed', + ); + return true; + }, + ); + assert.equal(world.adbCalls.length, deviceCalls); + }); + } finally { + fs.rmSync(root, { recursive: true, force: true }); + } +}); diff --git a/test/repo-guards/npm-package-scripts.test.ts b/test/repo-guards/npm-package-scripts.test.ts index 3e8d37a10b..95b46953b3 100644 --- a/test/repo-guards/npm-package-scripts.test.ts +++ b/test/repo-guards/npm-package-scripts.test.ts @@ -100,7 +100,7 @@ test('publishing cannot skip the package gate', () => { // the floor that job covers — so it must still name the same entry point the script does. assert.deepEqual(script('check:package').split(' && '), [ 'node --experimental-strip-types scripts/check-package.ts', - 'node --experimental-strip-types scripts/node-test-tmpdir.ts --test packages/maestro/test/package-artifact.test.mjs packages/maestro/test/package-smoke.mjs', + 'node --experimental-strip-types scripts/node-test-tmpdir.ts --test packages/maestro/test/package-artifact.test.mjs packages/maestro/test/package-smoke.mjs packages/replay/test/package-smoke.mjs', ]); assert.match(packagedCliWorkflow, /with: \{ gate: package \}/); assert.match( diff --git a/tsconfig.typecheck.json b/tsconfig.typecheck.json index 8380463560..122d4298d9 100644 --- a/tsconfig.typecheck.json +++ b/tsconfig.typecheck.json @@ -39,6 +39,7 @@ { "path": "packages/proxy" }, { "path": "packages/react-native" }, { "path": "packages/remote" }, + { "path": "packages/replay" }, { "path": "packages/replay-port" }, { "path": "packages/replay-test" }, { "path": "packages/selectors" }, diff --git a/vitest.config.ts b/vitest.config.ts index a83ba6c6ae..e4a1f959bc 100644 --- a/vitest.config.ts +++ b/vitest.config.ts @@ -220,6 +220,7 @@ export default defineConfig({ 'scripts/__tests__/release-version.test.ts', 'scripts/__tests__/release-plan.test.ts', 'scripts/__tests__/release-publish.test.ts', + 'scripts/__tests__/install-workspace-replay-plugins.test.ts', 'scripts/ios-snapshot-benchmark/*.test.ts', // The perf suite's pure logic, without a device. 'scripts/perf/*.test.ts', diff --git a/website/docs/docs/plugins.md b/website/docs/docs/plugins.md index 4c639112e4..1f82d3057b 100644 --- a/website/docs/docs/plugins.md +++ b/website/docs/docs/plugins.md @@ -29,6 +29,26 @@ Install only packages you trust: plugin factories run with the daemon's host per A failed install keeps the previous selection. A running daemon keeps the plugin set and files it started with, including removed installations. To apply changes, close sessions and run `agent-device daemon stop` with the matching `--state-dir`; the next device command uses the new set. Use separate state directories for different plugin homes. +## Install replay and test support + +The 0.22 `@agent-device/replay` and `@agent-device/maestro` packages are private and not yet published. The npm commands below apply after publication. Native replay and suites cannot be installed from npm until then. + +In 0.22, install the native replay plugin on the daemon host: + +```bash +agent-device plugins add @agent-device/replay +``` + +Finish your sessions and restart that daemon, then retry the original invocation with all its arguments. Native `.ad` scripts keep their syntax, variables, target verification, resume and repair behavior. Ordinary script recording stays in core. + +The plugin uses API version 2 and supports core `>=0.22.0-0 <0.23.0`, plus the pinned `0.21.38-dev` extraction host. A mismatched release returns `incompatible_plugin`. + +A remote daemon needs its own installation. A local plugin store does not make an engine available remotely. An embedded host imports the factory from `@agent-device/replay/agent-device` and supplies `plugins: [replay()]` to `startAgentDevice`. Supplied plugins are authoritative; the embedded runtime does not read the store. + +Maestro YAML replay requires `@agent-device/maestro` on both client and daemon hosts. Maestro suites also need `@agent-device/replay` on the daemon host. `replay export` needs the Maestro plugin on the client only. + +A missing installation returns `plugin-not-installed` with the package and installation command. Selected package failures keep `plugin_load_failed`, `invalid_plugin`, `incompatible_plugin` or `plugin_setup_failed`; repair or update that selected package. `daemon-plugin-set-stale` requires restarting the daemon after sessions finish. For an embedded daemon, its host closes and recreates the runtime. No recovery installs packages or ends sessions automatically. + ## See which providers are registered `agent-device daemon status` lists every provider the running daemon registered: the bundled ones (BrowserStack, AWS Device Farm, and Limrun when it is configured), then installed plugins. With `--json`, each entry has: diff --git a/website/docs/docs/replay-e2e.md b/website/docs/docs/replay-e2e.md index 6c32202c5e..9e1ab68964 100644 --- a/website/docs/docs/replay-e2e.md +++ b/website/docs/docs/replay-e2e.md @@ -40,6 +40,20 @@ and the same original-input guidance. To run Maestro YAML flows or export `.ad` scripts to Maestro YAML, see [Run Maestro flows](/docs/run-maestro-flows). +## Install replay support + +For 0.22, native `.ad` replay and all `test` suites require a separately installed first-party plugin on the daemon host: + +The 0.22 `@agent-device/replay` and `@agent-device/maestro` packages are private and not yet published. The npm commands below apply after publication. Native replay and suites cannot be installed from npm until then. + +```bash +agent-device plugins add @agent-device/replay +``` + +Finish your sessions and restart that daemon, then retry your original command with all its arguments. Existing scripts, variables, resume and repair syntax stay the same. A local installation does not install the plugin on a remote daemon. Ordinary `open`/`close --save-script` recording stays available in core. + +For Maestro YAML, install `@agent-device/maestro` on the client and daemon host as well. `replay export` uses the client-side Maestro plugin. See [plugin installation and recovery](./plugins.md). + ## How it works You work in two passes: