Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions packages/contracts/package.json
Original file line number Diff line number Diff line change
Expand Up @@ -571,6 +571,10 @@
"./replay-test-execution": {
"types": "./src/replay-test-execution.ts",
"default": "./src/replay-test-execution.ts"
},
"./replay-execution": {
"types": "./src/replay-execution.ts",
"default": "./src/replay-execution.ts"
}
},
"devDependencies": {
Expand Down
34 changes: 34 additions & 0 deletions packages/contracts/src/replay-execution.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,34 @@
/** The managed backend identity that owns native replay and suite execution. */
export const NATIVE_REPLAY_BACKEND = 'ad';

import type {
AdReplayDigestFlags,
AdReplayManifest,
AdReplayRunRequest,
AdReplayRunOutcome,
AdReplayStepRuntime,
} from './ad-replay-execution.ts';
import type {
ReplayTestSuiteRequest,
ReplayTestSuiteOutcome,
ReplayTestRuntimeDependencies,
} from './replay-test-execution.ts';

/** Native execution receives host-owned inspection, step and suite inputs. */
export type ReplayExecution = Readonly<{
inspectAdReplay(script: string, digestFlags?: AdReplayDigestFlags): AdReplayManifest;
runAdReplay(
request: AdReplayRunRequest,
runtime: AdReplayStepRuntime,
): Promise<AdReplayRunOutcome>;
runReplayTestSuite(
params: { request: ReplayTestSuiteRequest } & ReplayTestRuntimeDependencies,
): Promise<ReplayTestSuiteOutcome>;
}>;

/** The managed registry creates one native engine and owns its lifetime. */
export type ReplayExecutionPlugin = Readonly<{
create(): ReplayExecution | Promise<ReplayExecution>;
}>;

export type ReplayExecutionResolver = (packageName?: string) => Promise<ReplayExecution>;
2 changes: 1 addition & 1 deletion packages/maestro/src/plugin.ts
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
import type { ReplayEnginePlugin } from 'agent-device/plugins';

/** The managed entry is engine-free. Only a selected backend evaluates the implementation. */
function maestroPlugin(): ReplayEnginePlugin {
function maestroPlugin(): ReplayEnginePlugin<'maestro'> {
return {
apiVersion: 2,
kind: 'replay-backend',
Expand Down
74 changes: 59 additions & 15 deletions packages/plugins/src/registry.ts
Original file line number Diff line number Diff line change
@@ -1,3 +1,8 @@
import { NATIVE_REPLAY_BACKEND } from '@agent-device/contracts/replay-execution';
import type {
ReplayExecution,
ReplayExecutionResolver,
} from '@agent-device/contracts/replay-execution';
import type { ReplayBackend } from '@agent-device/contracts/replay-backend';
import type {
ReplayEnginePlugin,
Expand Down Expand Up @@ -34,6 +39,7 @@ import {
validateReplayEnginePlugin,
validateReplayEngineSetup,
validateReplayBackend,
validateReplayExecution,
} from './validate.ts';

/**
Expand Down Expand Up @@ -93,6 +99,7 @@ type Slot = {
replaySetup?: ReplayEngineSetup;
replayLoading?: Promise<ReplayBackend>;
replayFailure?: unknown;
executionLoading?: Promise<ReplayExecution>;
failure?: unknown;
registration?: ProviderRegistration;
};
Expand All @@ -111,6 +118,7 @@ export type ResolvedProviderPlugins = Readonly<{
export type ProviderRegistry = Readonly<{
registrations: readonly ProviderRegistration[];
resolveReplayBackend: ReplayBackendResolver;
resolveReplayExecution: ReplayExecutionResolver;
claimedProviders: readonly string[];
inventory: readonly PluginInventoryEntry[];
/**
Expand Down Expand Up @@ -199,6 +207,29 @@ export async function startProviderRegistry(
});
return await resolveRegisteredReplayBackend(resolved, inventory, id, host, packageName);
},
resolveReplayExecution: async (packageName) => {
if (closed)
throw new AppError('COMMAND_FAILED', 'Plugin registry is closed', {
reason: 'plugin-registry-closed',
});
const slot = requireReplaySlot(resolved, NATIVE_REPLAY_BACKEND, packageName);
if (slot.failure !== undefined) throw slot.failure;
const execution = slot.replaySetup?.execution;
if (!execution) throw invalidPlugin('Native replay plugin must provide execution.create()');
return await (slot.executionLoading ??= Promise.resolve().then(async () => {
try {
const runtime = validateReplayExecution(await execution.create());
slot.replayFailure = undefined;
inventory.splice(0, inventory.length, ...resolved.inventory());
return runtime;
} catch (error) {
slot.executionLoading = undefined;
slot.replayFailure = asPluginFailure(error, 'plugin_load_failed');
inventory.splice(0, inventory.length, ...resolved.inventory());
throw slot.replayFailure;
}
}));
Comment thread
thymikee marked this conversation as resolved.
Resolved
},
Comment thread
thymikee marked this conversation as resolved.
Resolved
close: once(async () => {
closed = true;
const [providers, backends] = await Promise.all([
Expand Down Expand Up @@ -259,7 +290,7 @@ async function setUpReplaySlot(
): Promise<void> {
const result = await plugin.setup(createPluginHost(env, slot.candidate.hostOptions));
try {
slot.replaySetup = validateReplayEngineSetup(result);
slot.replaySetup = validateReplayEngineSetup(result, plugin.backend);
} catch (error) {
await cleanUpRejectedSetup(result);
throw error;
Expand All @@ -273,22 +304,13 @@ async function resolveRegisteredReplayBackend(
host: import('@agent-device/contracts/replay-backend').ReplayBackendHost,
packageName?: string,
): Promise<ReplayBackend> {
const slot =
resolved.slots.find((slot) => slot.backend === id) ??
resolved.slots.find(
(slot) => packageName !== undefined && slot.candidate.package === packageName,
);
if (!slot)
throw new AppError('INVALID_ARGS', `Replay backend is not installed: ${id}`, {
reason: 'plugin-not-installed',
backend: id,
});
const slot = requireReplaySlot(resolved, id, packageName);
if (slot.failure !== undefined) throw slot.failure;
const setup = slot.replaySetup;
if (!setup) throw invalidPlugin(`Replay backend was not set up: ${id}`);
const replayBackend = slot.replaySetup?.replayBackend;
if (!replayBackend) throw invalidPlugin(`Replay backend was not set up: ${id}`);
return await (slot.replayLoading ??= Promise.resolve().then(async () => {
try {
const backend = validateReplayBackend(id, await setup.replayBackend.create(host));
const backend = validateReplayBackend(id, await replayBackend.create(host));
slot.replayFailure = undefined;
inventory.splice(0, inventory.length, ...resolved.inventory());
return backend;
Expand All @@ -301,11 +323,33 @@ async function resolveRegisteredReplayBackend(
}));
}

function requireReplaySlot(
resolved: ResolvedProviderPlugins,
id: string,
packageName: string | undefined,
): Slot {
const slot =
resolved.slots.find((slot) => slot.backend === id) ??
resolved.slots.find(
(slot) =>
packageName !== undefined &&
slot.candidate.package === packageName &&
slot.backend === undefined &&
slot.failure !== undefined,
);
if (!slot)
throw new AppError('INVALID_ARGS', `Replay backend is not installed: ${id}`, {
reason: 'plugin-not-installed',
backend: id,
});
return slot;
}

async function cleanUpReplaySlot(slot: Slot): Promise<PluginCleanupOutcome[]> {
const setup = slot.replaySetup;
const plugin = slot.replayPlugin;
if (!setup || !plugin) return [];
await slot.replayLoading?.catch(() => {});
await (slot.replayLoading ?? slot.executionLoading)?.catch(() => {});
return [
{
backend: plugin.backend,
Expand Down
175 changes: 175 additions & 0 deletions packages/plugins/src/replay-backend.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,8 @@ import { expect, test, vi } from 'vitest';
import { resolveProviderPlugins, startProviderRegistry } from './registry.ts';
import type { PluginCandidate, PluginRegistryPolicy } from './registry.ts';
import type { ReplayBackend, ReplayBackendHost } from '@agent-device/contracts/replay-backend';
import { validateReplayEngineSetup } from './validate.ts';
import type { ReplayExecution } from '@agent-device/contracts/replay-execution';
import type { ProviderPluginHost } from './host.ts';
import type { ReplayEnginePlugin } from './replay-backend.ts';

Expand Down Expand Up @@ -176,3 +178,176 @@ test('one backend disposer cannot delay another backend disposer', async () => {
release();
expect(await closing).toHaveLength(2);
});

test('native execution retries failed lazy loads while sharing claims, caching and disposal', async () => {
const execution: ReplayExecution = {
inspectAdReplay: vi.fn(() => {
throw new Error('inspection is outside registry ownership');
}),
runAdReplay: vi.fn(async () => {
throw new Error('dispatch is outside registry ownership');
}),
runReplayTestSuite: vi.fn(async () => {
throw new Error('scheduling is outside registry ownership');
}),
};
const create = vi
.fn(async () => execution)
.mockRejectedValueOnce(new Error('native engine failed'));
const competing = { setup: vi.fn(() => ({ execution: { create } })) };
const competingPlugin = {
apiVersion: 2,
kind: 'replay-backend',
backend: 'ad',
setup: competing.setup,
};
const dispose = vi.fn();
const registry = await startProviderRegistry(
await resolveProviderPlugins([
candidate({
apiVersion: 2,
kind: 'replay-backend',
backend: 'ad',
setup: () => ({ execution: { create }, dispose }),
}),
candidate(competingPlugin),
]),
{},
policy,
);
expect(registry.inventory).toMatchObject([
{ backend: 'ad', state: 'active' },
{ backend: 'ad', state: 'failed', error: { reason: 'duplicate_replay_backend' } },
]);
expect(competing.setup).not.toHaveBeenCalled();
await expect(registry.resolveReplayBackend('ad', host)).rejects.toMatchObject({
details: { reason: 'invalid_plugin' },
});
expect(registry.inventory[0]).toMatchObject({ backend: 'ad', state: 'active' });
expect(create).not.toHaveBeenCalled();
await expect(registry.resolveReplayExecution()).rejects.toMatchObject({
message: 'native engine failed',
details: { reason: 'plugin_load_failed' },
});
expect(create).toHaveBeenCalledTimes(1);
expect(registry.inventory[0]).toMatchObject({
state: 'failed',
error: { reason: 'plugin_load_failed' },
});
const [one, two] = await Promise.all([
registry.resolveReplayExecution(),
registry.resolveReplayExecution(),
]);
expect(one).toBe(two);
expect(one).toBe(execution);
expect(await registry.resolveReplayExecution()).toBe(one);
expect(create).toHaveBeenCalledTimes(2);
expect(registry.inventory[0]).toMatchObject({ backend: 'ad', state: 'active' });
expect(registry.inventory[0]?.error).toBeUndefined();
await registry.close();
expect(dispose).toHaveBeenCalledTimes(1);
await expect(registry.resolveReplayExecution()).rejects.toMatchObject({
details: { reason: 'plugin-registry-closed' },
});
});

test.each([
[{ apiVersion: 99 }, 'incompatible_plugin'],
[{ setup: () => ({ execution: {} }) }, 'invalid_plugin'],
[
{
setup: () => {
throw new Error('failed setup');
},
},
'plugin_setup_failed',
],
[{ setup: () => ({ execution: { create: () => ({}) } }) }, 'invalid_plugin'],
])(
'selected native execution failures preserve their typed cause: %s',
async (override, reason) => {
const registry = await startProviderRegistry(
await resolveProviderPlugins([
candidate({
apiVersion: 2,
kind: 'replay-backend',
backend: 'ad',
setup: () => ({ execution: { create: () => ({}) } }),
...override,
}),
]),
{},
policy,
);
await expect(registry.resolveReplayExecution()).rejects.toMatchObject({ details: { reason } });
await registry.close();
},
);

test.each([
['ad', { replayBackend: { create: vi.fn() } }],
['maestro', { execution: { create: vi.fn() } }],
['custom', { execution: { create: vi.fn() } }],
])('setup validates the factory against backend %s', (backend, setup) => {
expect(() => validateReplayEngineSetup(setup, backend)).toThrow(
expect.objectContaining({ details: { reason: 'invalid_plugin' } }),
);
});

test('wrong native capability fails inventory, disposes setup and keeps its first claim', async () => {
const wrong = engine('ad');
const laterSetup = vi.fn(() => ({ execution: { create: vi.fn() } }));
const registry = await startProviderRegistry(
await resolveProviderPlugins([
candidate(wrong.plugin),
candidate({ apiVersion: 2, kind: 'replay-backend', backend: 'ad', setup: laterSetup }),
candidate(engine('maestro').plugin),
]),
{},
policy,
);
expect(registry.inventory).toMatchObject([
{ backend: 'ad', state: 'failed', error: { reason: 'invalid_plugin' } },
{ backend: 'ad', state: 'failed', error: { reason: 'duplicate_replay_backend' } },
{ backend: 'maestro', state: 'active' },
]);
expect(wrong.dispose).toHaveBeenCalledOnce();
expect(wrong.create).not.toHaveBeenCalled();
expect(laterSetup).not.toHaveBeenCalled();
await expect(registry.resolveReplayExecution()).rejects.toMatchObject({
details: { reason: 'invalid_plugin' },
});
await registry.close();
expect(wrong.dispose).toHaveBeenCalledOnce();
});

test('a package name cannot select an active foreign backend for native execution', async () => {
const maestro = engine();
const packageName = '@agent-device/maestro';
const registry = await startProviderRegistry(
await resolveProviderPlugins([
{ ...candidate(maestro.plugin), package: packageName },
{
source: 'store',
package: '@agent-device/replay',
load: () => {
throw new Error('selected native package is unreadable');
},
},
]),
{},
policy,
);
try {
await expect(registry.resolveReplayExecution(packageName)).rejects.toMatchObject({
details: { reason: 'plugin-not-installed', backend: 'ad' },
});
await expect(registry.resolveReplayExecution('@agent-device/replay')).rejects.toMatchObject({
details: { reason: 'plugin_load_failed' },
});
expect(maestro.create).not.toHaveBeenCalled();
expect(await registry.resolveReplayBackend('maestro', host, packageName)).toBe(maestro.runtime);
} finally {
await registry.close();
}
});
Loading
Loading