From aa44c95f9abcd97e3e09ecc1e364a60f31ee5f91 Mon Sep 17 00:00:00 2001 From: DongYun Kang Date: Fri, 13 Nov 2026 13:31:40 +0900 Subject: [PATCH 1/2] refactor(e2e): split the session half of a worker from the runner TargetExecutor's engine init, dispose, and per-attempt session launch and close move to an EngineSessions base class. The standalone worker takes a WorkerPolicy and an AttemptPolicy per kind; the hosted and one-shot kinds live in standalone-kinds.ts. resolveProjectConfig resolves what is not reporters or test filters, which resolveConfig in config/resolve-run.ts adds, with their types in run-types.ts. No behavior changes: this is the seam the next change moves into a package of its own. Co-Authored-By: Claude Opus 5.5 --- .changeset/project-config.md | 5 + packages/e2e/src/cli/cache.ts | 2 +- packages/e2e/src/cli/doctor.ts | 3 +- packages/e2e/src/cli/index.ts | 3 +- packages/e2e/src/collect/collect.ts | 21 +- packages/e2e/src/collect/select.ts | 42 +- packages/e2e/src/config/load.ts | 2 +- packages/e2e/src/config/resolve-run.ts | 144 ++++++ packages/e2e/src/config/resolve.ts | 152 +----- packages/e2e/src/explore/index.ts | 6 +- packages/e2e/src/explore/public-explore.ts | 2 +- packages/e2e/src/index.ts | 1 + packages/e2e/src/mcp/config.ts | 3 +- packages/e2e/src/mcp/run-code.ts | 2 +- packages/e2e/src/mcp/session.ts | 5 +- packages/e2e/src/report/build.ts | 37 +- packages/e2e/src/report/builtin.ts | 2 +- packages/e2e/src/report/json.ts | 2 +- packages/e2e/src/report/junit.ts | 2 +- packages/e2e/src/report/left-out.ts | 2 +- packages/e2e/src/report/list.ts | 4 +- packages/e2e/src/report/markdown.ts | 2 +- packages/e2e/src/report/outcome.ts | 2 +- packages/e2e/src/report/write.ts | 2 +- packages/e2e/src/run-types.ts | 186 ++++++++ packages/e2e/src/run/app-ports.ts | 6 +- packages/e2e/src/run/changes.ts | 2 +- packages/e2e/src/run/engine-sessions.ts | 438 +++++++++++++++++ packages/e2e/src/run/execute.ts | 444 +----------------- packages/e2e/src/run/host-trace.ts | 2 +- packages/e2e/src/run/open-session.ts | 10 +- packages/e2e/src/run/provision.ts | 63 ++- packages/e2e/src/run/public-run.ts | 2 +- packages/e2e/src/run/records.ts | 50 +- packages/e2e/src/run/runner.ts | 43 +- packages/e2e/src/run/scheduler.ts | 3 +- packages/e2e/src/run/serial.ts | 6 +- packages/e2e/src/run/standalone-attempt.ts | 221 ++++----- packages/e2e/src/run/standalone-kinds.ts | 139 ++++++ packages/e2e/src/run/standalone-test.ts | 6 +- packages/e2e/src/run/standalone.ts | 85 ++-- packages/e2e/src/run/test-filters.ts | 8 +- packages/e2e/src/run/units.ts | 3 +- packages/e2e/src/run/worker/entry.ts | 3 +- packages/e2e/src/run/worker/protocol.ts | 4 +- packages/e2e/src/telemetry/events.ts | 6 +- packages/e2e/src/types.ts | 179 +------ .../e2e/tests/integration/cli-init.test.ts | 4 +- .../integration/mcp-session-host.test.ts | 4 +- .../tests/integration/mcp-test-run.test.ts | 4 +- .../integration/shared-app-processes.test.ts | 4 +- .../tests/integration/test-filters.test.ts | 2 +- .../tests/unit/agent-assert-explore.test.ts | 4 +- packages/e2e/tests/unit/agent-config.test.ts | 5 +- .../e2e/tests/unit/agent-entry-prompt.test.ts | 4 +- .../tests/unit/agent-model-refusal.test.ts | 2 +- packages/e2e/tests/unit/app-ports.test.ts | 3 +- packages/e2e/tests/unit/changes.test.ts | 2 +- packages/e2e/tests/unit/collect.test.ts | 2 +- packages/e2e/tests/unit/config.test.ts | 6 +- .../e2e/tests/unit/declared-processes.test.ts | 3 +- .../e2e/tests/unit/engine-protocol.test.ts | 2 +- packages/e2e/tests/unit/engine.test.ts | 2 +- .../e2e/tests/unit/fixture-runtime.test.ts | 5 +- packages/e2e/tests/unit/init-gateways.test.ts | 2 +- .../tests/unit/model-request-headers.test.ts | 2 +- .../e2e/tests/unit/press-sequentially.test.ts | 2 +- .../unit/provider-failure-message.test.ts | 4 +- packages/e2e/tests/unit/report-junit.test.ts | 2 +- .../e2e/tests/unit/report-markdown.test.ts | 2 +- packages/e2e/tests/unit/report-status.test.ts | 3 +- packages/e2e/tests/unit/scheduler.test.ts | 2 +- packages/e2e/tests/unit/select.test.ts | 7 +- .../e2e/tests/unit/target-identity.test.ts | 3 +- .../e2e/tests/unit/telemetry-events.test.ts | 2 +- packages/e2e/tests/unit/test-filters.test.ts | 5 +- .../e2e/tests/unit/tool-loop-overflow.test.ts | 4 +- .../e2e/tests/unit/tool-loop-prefix.test.ts | 5 +- .../e2e/tests/unit/tool-loop-stall.test.ts | 4 +- .../tests/unit/tool-loop-tool-choice.test.ts | 4 +- .../e2e/tests/unit/trace-cache-config.test.ts | 5 +- packages/e2e/tests/unit/worker-handle.test.ts | 2 +- packages/e2e/tests/unit/worker-models.test.ts | 2 +- 83 files changed, 1339 insertions(+), 1138 deletions(-) create mode 100644 .changeset/project-config.md create mode 100644 packages/e2e/src/config/resolve-run.ts create mode 100644 packages/e2e/src/run-types.ts create mode 100644 packages/e2e/src/run/engine-sessions.ts create mode 100644 packages/e2e/src/run/standalone-kinds.ts diff --git a/.changeset/project-config.md b/.changeset/project-config.md new file mode 100644 index 0000000000..83a1d16ffa --- /dev/null +++ b/.changeset/project-config.md @@ -0,0 +1,5 @@ +--- +'e2e': patch +--- + +`e2e` exports `ProjectConfig`, the part of `E2EConfig` without `reporters` and `filters`. diff --git a/packages/e2e/src/cli/cache.ts b/packages/e2e/src/cli/cache.ts index e5189fb658..86869ea26e 100644 --- a/packages/e2e/src/cli/cache.ts +++ b/packages/e2e/src/cli/cache.ts @@ -17,7 +17,7 @@ import type { Command } from 'commander'; import picocolors from 'picocolors'; import { FileCacheStore, MAX_CACHE_WIRE_BYTES } from '../cache/store.ts'; import { discoverConfig, loadConfigModule, missingConfigError } from '../config/load.ts'; -import { resolveConfig } from '../config/resolve.ts'; +import { resolveConfig } from '../config/resolve-run.ts'; import { errorMessage } from '../internal/errors.ts'; import { visibleWidth } from '../report/format.ts'; import { docsLine, examples } from './help.ts'; diff --git a/packages/e2e/src/cli/doctor.ts b/packages/e2e/src/cli/doctor.ts index d92668a7d0..5359dee2c1 100644 --- a/packages/e2e/src/cli/doctor.ts +++ b/packages/e2e/src/cli/doctor.ts @@ -14,7 +14,8 @@ import { AgentError, isAgentError } from '../agent/error.ts'; import { engineAppInfo } from '../config/app.ts'; import { modelLabel, type ResolvedAgentConfig, type ResolvedModel } from '../config/agent.ts'; import { discoverConfig, loadConfigModule, missingConfigError } from '../config/load.ts'; -import { resolveConfig, type ResolvedConfig } from '../config/resolve.ts'; +import { type ResolvedConfig } from '../config/resolve.ts'; +import { resolveConfig } from '../config/resolve-run.ts'; import type { ResolvedTarget } from '../config/targets.ts'; import { selectTargets } from '../collect/select.ts'; import type { EngineDoctorCheck } from '../engine/index.ts'; diff --git a/packages/e2e/src/cli/index.ts b/packages/e2e/src/cli/index.ts index 89ac21e623..a69c9c6df6 100644 --- a/packages/e2e/src/cli/index.ts +++ b/packages/e2e/src/cli/index.ts @@ -20,7 +20,8 @@ import { writeTerminal } from '../run/process-output.ts'; import { STEP_BOUNDS, TIMEOUT_BOUNDS } from '../explore/bounds.ts'; import { BUILTIN_REPORTERS, isBuiltinReporter } from '../report/builtin.ts'; import { bounded, failureLine } from '../report/format.ts'; -import type { BuiltinReporter, RecordingMode } from '../types.ts'; +import type { RecordingMode } from '../types.ts'; +import type { BuiltinReporter } from '../run-types.ts'; import { isRecordingMode, legacyTraceSpelling, RECORDING_MODES } from '../internal/recording-modes.ts'; import { discoverConfig } from '../config/load.ts'; import { isCiMode } from '../config/resolve.ts'; diff --git a/packages/e2e/src/collect/collect.ts b/packages/e2e/src/collect/collect.ts index e6bac528de..0d6a8bb1fd 100644 --- a/packages/e2e/src/collect/collect.ts +++ b/packages/e2e/src/collect/collect.ts @@ -18,26 +18,7 @@ import { type RegisteredTest, } from './registry.ts'; import type { SourceLocation } from '../internal/source.ts'; - -/** - * Everything needed to name and report a test, with no executable or realm - * state attached. Fully JSON-serializable, so it is what result records carry - * and what crosses the runner<->worker IPC channel. - */ -export interface TestIdentity { - readonly kind: 'test' | 'setup'; - readonly title: string; - readonly titlePath: readonly string[]; - readonly declarationIndex: number; - readonly sessions: readonly string[]; - /** The tags the test declares, outermost describe first, each once; what `--tag` selects on. */ - readonly tags: readonly string[]; - readonly source: SourceLocation | undefined; - /** Normalized project-root-relative file path with `/` separators. */ - readonly file: string; - readonly id: string; - readonly serialId: string | undefined; -} +import type { TestIdentity } from '../run/records.ts'; export interface CollectedTest extends RegisteredTest, TestIdentity { /** Outermost serial group, when the test is a serial-group member. */ diff --git a/packages/e2e/src/collect/select.ts b/packages/e2e/src/collect/select.ts index 2113047834..fffe4f3a74 100644 --- a/packages/e2e/src/collect/select.ts +++ b/packages/e2e/src/collect/select.ts @@ -6,10 +6,12 @@ import { resultId } from '../internal/ids.ts'; import { expectedFailureOption } from '../internal/expected-failure.ts'; import { didYouMean, suggestionNote } from '../internal/suggest.ts'; import type { ResolvedConfig, ResolvedTarget } from '../config/resolve.ts'; -import { attemptKeep, attemptVideo, type AttemptRecordings, type RecordingKind, type ResolvedRecording } from '../internal/recording-modes.ts'; +import { attemptKeep, attemptVideo, type AttemptRecordings } from '../internal/recording-modes.ts'; import type { Capability, RecordingMode, TestOptions } from '../types.ts'; import { excludingEntry, type Collection, type CollectedTest, type UncollectedFile } from './collect.ts'; import { groupChain } from './registry.ts'; +import { pairRecording } from '../run/provision.ts'; +import type { SkipInfo } from '../run/records.ts'; export interface ResolvedTestOptions { readonly timeout: number; @@ -39,33 +41,6 @@ export interface ResolvedTestOptions { readonly video: RecordingMode | undefined; } -/** Why a pair did not run; only a `filtered` one can name the test filter that left it out. */ -export type SkipInfo = - | (SkipDetail & { - readonly cause: 'filtered'; - /** The name of the test filter that left the test out; absent when the selection did, as with `--tag`. */ - readonly filter?: string; - }) - | (SkipDetail & { - readonly cause: - | 'explicit' - | 'platform-unavailable' - | 'capability-unavailable' - | 'setup-failed' - | 'serial-predecessor-failed' - | 'hook-failed' - | 'infrastructure-unavailable' - | 'failure-limit' - | 'time-limit'; - readonly filter?: never; - }); - -/** What a skip says whatever its cause: why, and the id of a related test, such as the setup that failed. */ -interface SkipDetail { - readonly reason: string; - readonly relatedId?: string; -} - export interface TestTargetPair { readonly test: CollectedTest; readonly target: ResolvedTarget; @@ -230,17 +205,6 @@ export function resolveOptions(test: CollectedTest, config: ResolvedConfig): Res }; } -/** - * Which attempts of a pair record `kind`, and where the mode came from: the - * test's own, else its target's (which the flag and the config already - * decided). One resolver for both recordings, so `trace` and `video` can - * never disagree about precedence. - */ -export function pairRecording(pair: Pick, kind: RecordingKind): ResolvedRecording { - const own = pair.options[kind]; - return own === undefined ? pair.target[kind] : { mode: own, source: 'test' }; -} - /** What an attempt of a pair at `attemptIndex` captures: one keep decision for the trace and the video alike. */ export function pairRecordings(pair: Pick, attemptIndex: number): AttemptRecordings { return { diff --git a/packages/e2e/src/config/load.ts b/packages/e2e/src/config/load.ts index 55bf291caa..881ff5bbe5 100644 --- a/packages/e2e/src/config/load.ts +++ b/packages/e2e/src/config/load.ts @@ -5,7 +5,7 @@ import path from 'node:path'; import { ConfigurationError, InfrastructureError, isE2EError } from '../internal/errors.ts'; import { onBun } from '../internal/runtime.ts'; import { withConfigSecretReferences } from '../secrets.ts'; -import type { E2EConfig } from '../types.ts'; +import type { E2EConfig } from '../run-types.ts'; import { explainModuleError } from './diagnose.ts'; import { freshModuleURL, registerLoader } from './esm-hooks.ts'; import { CONFIG_NAMES, JS_CONFIG_NAMES, TS_CONFIG_NAMES } from './names.ts'; diff --git a/packages/e2e/src/config/resolve-run.ts b/packages/e2e/src/config/resolve-run.ts new file mode 100644 index 0000000000..468ec0b2ec --- /dev/null +++ b/packages/e2e/src/config/resolve-run.ts @@ -0,0 +1,144 @@ +/** + * The runner's config: the project config (`resolveProjectConfig`), plus + * the reporters and test filters only a run reads. + */ + +import { ConfigurationError } from '../internal/errors.ts'; +import { didYouMean } from '../internal/suggest.ts'; +import { BUILTIN_REPORTER_LIST, BUILTIN_REPORTERS, isBuiltinReporter } from '../report/builtin.ts'; +import { ellipsize } from '../report/format.ts'; +import type { BuiltinReporter, E2EConfig, Reporter, TestFilter } from '../run-types.ts'; +import { resolveProjectConfig, type CliOverrides, type ResolvedConfig } from './resolve.ts'; +import { TARGET_NAME_PATTERN } from './targets.ts'; + +/** A resolved config with what only a run reads. */ +export interface ResolvedRunConfig extends ResolvedConfig { + /** The built-in renderers in force: `--reporter` when given, else the config's ids. */ + readonly reporters: readonly BuiltinReporter[]; + /** The reporter objects the config names; `--reporter` never removes one. */ + readonly customReporters: readonly Reporter[]; + /** The test filters, in order; empty when unset. A run with `--changed-since` applies them. */ + readonly filters: readonly TestFilter[]; +} + +/** The flags that replace config keys, `--reporter` among them. */ +export interface RunCliOverrides extends CliOverrides { + reporters?: readonly BuiltinReporter[]; +} + +/** Resolves a config file's export for the runner: the project config, then its reporters and filters. */ +export function resolveConfig( + raw: E2EConfig, + options: { + projectRoot: string; + configPath?: string; + env?: NodeJS.ProcessEnv; + cli?: RunCliOverrides; + }, +): ResolvedRunConfig { + const resolved = resolveProjectConfig(raw, options); + const { reporters, customReporters } = resolveReporters(raw, options.cli ?? {}); + return { ...resolved, reporters, customReporters, filters: resolveTestFilters(raw.filters) }; +} + +/** + * Splits `reporters` into the built-in ids and the reporter objects. + * `--reporter` replaces the ids only: an object has no id to name on the + * command line, so a CI flag choosing `list,junit` never drops the upload a + * config asked for. + */ +function resolveReporters( + raw: E2EConfig, + cli: RunCliOverrides, +): { reporters: readonly BuiltinReporter[]; customReporters: readonly Reporter[] } { + const configured = raw.reporters ?? ['list']; + if (!Array.isArray(configured)) { + throw new ConfigurationError( + 'INVALID_CONFIG', + 'reporters must be an array of reporter ids and reporter objects', + ); + } + const ids: BuiltinReporter[] = []; + const customReporters: Reporter[] = []; + for (const reporter of configured as readonly unknown[]) { + if (typeof reporter === 'string') { + if (!isBuiltinReporter(reporter)) { + throw new ConfigurationError( + 'INVALID_CONFIG', + `unknown reporter "${reporter}"; reporters are ${BUILTIN_REPORTER_LIST}${didYouMean(reporter, BUILTIN_REPORTERS)}`, + ); + } + ids.push(reporter); + } else if (isReporter(reporter)) { + customReporters.push(reporter); + } else { + throw new ConfigurationError( + 'INVALID_CONFIG', + `a reporter is a built-in id (${BUILTIN_REPORTERS.join(', ')}) or an object with a name and an onEvent or onRunFinished method`, + ); + } + } + const reporters = cli.reporters ?? ids; + if (reporters.includes('json') && reporters.includes('list')) { + throw new ConfigurationError('INVALID_CONFIG', 'the json renderer cannot be combined with list'); + } + return { reporters, customReporters }; +} + +const MAX_FILTER_NAME = 256; + +/** + * Resolves `filters`: a list of test filters, each checked structurally, + * never with `instanceof`, since the config may import another copy of + * `e2e`. A filter's name is how the report and errors tell it apart, so + * names are unique. Workers resolve it too and never call a filter, so the + * check stays synchronous. + */ +function resolveTestFilters(raw: E2EConfig['filters']): readonly TestFilter[] { + if (raw === undefined) return []; + if (!Array.isArray(raw)) { + throw new ConfigurationError('INVALID_CONFIG', 'filters must be a list of test filters: filters: [{ name, filter(tests, context) }]'); + } + const names = new Set(); + return Array.from(raw as readonly unknown[], (value, index) => { + const label = `filters[${index}]`; + if (typeof value === 'function') { + throw new ConfigurationError('INVALID_CONFIG', `${label} is ${value.name ? `the function ${value.name}` : 'a function'}; call it, such as filters: [decisionFilter({ model })]`); + } + if (!isTestFilter(value)) { + throw new ConfigurationError('INVALID_CONFIG', `${label} must be a TestFilter: { name, filter(tests, context) }`); + } + const { name } = value; + if (!TARGET_NAME_PATTERN.test(name)) { + throw new ConfigurationError( + 'INVALID_CONFIG', + `invalid filter name ${JSON.stringify(name)}: names are ASCII letters, numbers, "_", "-", or ".", and cannot be only dots`, + ); + } + if (name.length > MAX_FILTER_NAME) { + throw new ConfigurationError('INVALID_CONFIG', `filter name ${JSON.stringify(ellipsize(name, 32))} is longer than ${MAX_FILTER_NAME} characters`); + } + if (names.has(name)) { + throw new ConfigurationError('INVALID_CONFIG', `filters has two filters named ${JSON.stringify(name)}; the report tells filters apart by name`); + } + names.add(name); + return value; + }); +} + +/** Structural test filter check: a string name and a `filter` function. */ +function isTestFilter(value: unknown): value is TestFilter { + if (typeof value !== 'object' || value === null) return false; + const { name, filter } = value as { name?: unknown; filter?: unknown }; + return typeof name === 'string' && typeof filter === 'function'; +} + +/** Structural reporter check: a non-empty name and at least one handler, each a function when present. */ +function isReporter(value: unknown): value is Reporter { + if (typeof value !== 'object' || value === null) return false; + const { name, onEvent, onRunFinished } = value as { name?: unknown; onEvent?: unknown; onRunFinished?: unknown }; + if (typeof name !== 'string' || name.length === 0) return false; + const optionalFunction = (handler: unknown): boolean => handler === undefined || typeof handler === 'function'; + return optionalFunction(onEvent) && optionalFunction(onRunFinished) && (onEvent ?? onRunFinished) !== undefined; +} + diff --git a/packages/e2e/src/config/resolve.ts b/packages/e2e/src/config/resolve.ts index f19ee21e2a..020321a43e 100644 --- a/packages/e2e/src/config/resolve.ts +++ b/packages/e2e/src/config/resolve.ts @@ -9,26 +9,11 @@ import { canonicalDigest, sha256Hex } from '../internal/ids.ts'; import { realpathOfExisting } from '../internal/paths.ts'; import { didYouMean } from '../internal/suggest.ts'; import { isRecordingMode, legacyTraceSpelling, RECORDING_MODES, type RecordingKind, type ResolvedRecording } from '../internal/recording-modes.ts'; -import { BUILTIN_REPORTER_LIST, BUILTIN_REPORTERS, isBuiltinReporter } from '../report/builtin.ts'; -import { ellipsize } from '../report/format.ts'; import { isStepExecutor } from '../agent/executor.ts'; import { compileGlob, compileGlobList, literalPrefix } from '../internal/globs.ts'; import { isRecord } from './command.ts'; import { boundedInt, describeValue, timerMs } from './validate.ts'; -import type { - ArtifactStore, - ArtifactsConfig, - BuiltinReporter, - CacheMode, - E2EConfig, - ModelInstance, - Reporter, - SecretProvider, - SecretPurpose, - RecordingMode, - CacheStore, - TestFilter, -} from '../types.ts'; +import type { ArtifactStore, ArtifactsConfig, CacheMode, ModelInstance, SecretProvider, SecretPurpose, RecordingMode, CacheStore, ProjectConfig } from '../types.ts'; import { isModelInstance, resolveAgentConfig, runLimits, type ResolvedAgentConfig, type ResolvedLimits } from './agent.ts'; import { bindTargets, digestTargets, resolveTargets, TARGET_NAME_PATTERN, type PortAssignments, type ResolvedTarget } from './targets.ts'; import { credentialNamed, credentialSecretName, envName, isSecretValue, secretValueProblem } from './secrets.ts'; @@ -76,12 +61,6 @@ export interface ResolvedConfig { readonly artifactStore: ArtifactStore | undefined; /** Absolute directory the run writes its results to: `--output`, else the config's `output`, else `/.e2e`. */ readonly output: string; - /** The built-in renderers in force: `--reporter` when given, else the config's ids. */ - readonly reporters: readonly BuiltinReporter[]; - /** The reporter objects the config names; `--reporter` never removes one. */ - readonly customReporters: readonly Reporter[]; - /** The test filters, in order; empty when unset. A run with `--changed-since` applies them. */ - readonly filters: readonly TestFilter[]; /** * The agents unpinned tests run as, one result each: `agents.default`, or * the names `--agent` gave, in order and deduplicated. Never empty. @@ -143,7 +122,6 @@ export interface CliOverrides { actionTimeout?: number; /** `--assertion-timeout `: the default `expect` deadline, over the config's `assertionTimeout`. */ assertionTimeout?: number; - reporters?: readonly BuiltinReporter[]; /** Replay cache mode override; `--no-cache` maps to `'off'`. */ cache?: CacheMode; /** `--strict-cache`: turns `cache.strict` on for the run. */ @@ -234,8 +212,8 @@ export function isCiMode(env: NodeJS.ProcessEnv = process.env): boolean { * is what identities and the digest key on. `assignPorts` puts a run's * ports in. */ -export function resolveConfig( - raw: E2EConfig, +export function resolveProjectConfig( + raw: ProjectConfig, options: { projectRoot: string; configPath?: string; @@ -299,8 +277,6 @@ export function resolveConfig( } const artifactStore = resolveArtifactStore(raw); - const { reporters, customReporters } = resolveReporters(raw, cli); - const filters = resolveTestFilters(raw.filters); const projectId = resolveProjectId(raw.projectId, options.projectRoot); const { credentials, secrets, allSecrets } = resolveSecrets(raw, env); @@ -329,9 +305,6 @@ export function resolveConfig( workers, artifactStore, output, - reporters, - customReporters, - filters, agentNames, agent, agents, @@ -352,7 +325,7 @@ export function resolveConfig( * from the same config and ports; the digest and the identities stand, * because the ports never enter them. */ -export function assignPorts(config: ResolvedConfig, ports: PortAssignments): ResolvedConfig { +export function assignPorts(config: Config, ports: PortAssignments): Config { return { ...config, targets: bindTargets(config.targets, config.projectRoot, ports), ports }; } @@ -369,7 +342,7 @@ export function assignPorts(config: ResolvedConfig, ports: PortAssignments): Res * `resolveCacheDir`, with what named it. */ function resolveCacheConfig( - raw: E2EConfig, + raw: ProjectConfig, ci: boolean, projectRoot: string, cli: CliOverrides, @@ -588,7 +561,7 @@ function removedRecordingReplacement(kinds: readonly unknown[] | undefined, trac * list (bare or as `kinds`) and the `trace` block are refused with the * `trace` mode they meant, and `video` with the option that replaced it. */ -function resolveArtifactStore(raw: E2EConfig): ArtifactStore | undefined { +function resolveArtifactStore(raw: ProjectConfig): ArtifactStore | undefined { const value: unknown = raw.artifacts; if (value === undefined) return undefined; if (Array.isArray(value)) { @@ -652,7 +625,7 @@ interface RunRecording { * attempt keeps its trace by default, locally and in CI alike: the trace is * the runner's own text, cheap to write and worth reading on every failure. */ -function runRecordings(raw: E2EConfig, cli: CliOverrides): Readonly> { +function runRecordings(raw: ProjectConfig, cli: CliOverrides): Readonly> { return { trace: { cli: recordingMode(cli.trace, '--trace', 'trace'), config: recordingMode(raw.trace, 'trace', 'trace'), fallback: 'retain-on-failure' }, video: { cli: recordingMode(cli.video, '--video', 'video'), config: recordingMode(raw.video, 'video', 'video'), fallback: 'off' }, @@ -674,107 +647,6 @@ function targetRecording(run: RunRecording, own: unknown, where: string, kind: R } -/** - * Splits `reporters` into the built-in ids and the reporter objects. - * `--reporter` replaces the ids only: an object has no id to name on the - * command line, so a CI flag choosing `list,junit` never drops the upload a - * config asked for. - */ -function resolveReporters( - raw: E2EConfig, - cli: CliOverrides, -): { reporters: readonly BuiltinReporter[]; customReporters: readonly Reporter[] } { - const configured = raw.reporters ?? ['list']; - if (!Array.isArray(configured)) { - throw new ConfigurationError( - 'INVALID_CONFIG', - 'reporters must be an array of reporter ids and reporter objects', - ); - } - const ids: BuiltinReporter[] = []; - const customReporters: Reporter[] = []; - for (const reporter of configured as readonly unknown[]) { - if (typeof reporter === 'string') { - if (!isBuiltinReporter(reporter)) { - throw new ConfigurationError( - 'INVALID_CONFIG', - `unknown reporter "${reporter}"; reporters are ${BUILTIN_REPORTER_LIST}${didYouMean(reporter, BUILTIN_REPORTERS)}`, - ); - } - ids.push(reporter); - } else if (isReporter(reporter)) { - customReporters.push(reporter); - } else { - throw new ConfigurationError( - 'INVALID_CONFIG', - `a reporter is a built-in id (${BUILTIN_REPORTERS.join(', ')}) or an object with a name and an onEvent or onRunFinished method`, - ); - } - } - const reporters = cli.reporters ?? ids; - if (reporters.includes('json') && reporters.includes('list')) { - throw new ConfigurationError('INVALID_CONFIG', 'the json renderer cannot be combined with list'); - } - return { reporters, customReporters }; -} - -const MAX_FILTER_NAME = 256; - -/** - * Resolves `filters`: a list of test filters, each checked structurally, - * never with `instanceof`, since the config may import another copy of - * `e2e`. A filter's name is how the report and errors tell it apart, so - * names are unique. Workers resolve it too and never call a filter, so the - * check stays synchronous. - */ -function resolveTestFilters(raw: E2EConfig['filters']): readonly TestFilter[] { - if (raw === undefined) return []; - if (!Array.isArray(raw)) { - throw new ConfigurationError('INVALID_CONFIG', 'filters must be a list of test filters: filters: [{ name, filter(tests, context) }]'); - } - const names = new Set(); - return Array.from(raw as readonly unknown[], (value, index) => { - const label = `filters[${index}]`; - if (typeof value === 'function') { - throw new ConfigurationError('INVALID_CONFIG', `${label} is ${value.name ? `the function ${value.name}` : 'a function'}; call it, such as filters: [decisionFilter({ model })]`); - } - if (!isTestFilter(value)) { - throw new ConfigurationError('INVALID_CONFIG', `${label} must be a TestFilter: { name, filter(tests, context) }`); - } - const { name } = value; - if (!TARGET_NAME_PATTERN.test(name)) { - throw new ConfigurationError( - 'INVALID_CONFIG', - `invalid filter name ${JSON.stringify(name)}: names are ASCII letters, numbers, "_", "-", or ".", and cannot be only dots`, - ); - } - if (name.length > MAX_FILTER_NAME) { - throw new ConfigurationError('INVALID_CONFIG', `filter name ${JSON.stringify(ellipsize(name, 32))} is longer than ${MAX_FILTER_NAME} characters`); - } - if (names.has(name)) { - throw new ConfigurationError('INVALID_CONFIG', `filters has two filters named ${JSON.stringify(name)}; the report tells filters apart by name`); - } - names.add(name); - return value; - }); -} - -/** Structural test filter check: a string name and a `filter` function. */ -function isTestFilter(value: unknown): value is TestFilter { - if (typeof value !== 'object' || value === null) return false; - const { name, filter } = value as { name?: unknown; filter?: unknown }; - return typeof name === 'string' && typeof filter === 'function'; -} - -/** Structural reporter check: a non-empty name and at least one handler, each a function when present. */ -function isReporter(value: unknown): value is Reporter { - if (typeof value !== 'object' || value === null) return false; - const { name, onEvent, onRunFinished } = value as { name?: unknown; onEvent?: unknown; onRunFinished?: unknown }; - if (typeof name !== 'string' || name.length === 0) return false; - const optionalFunction = (handler: unknown): boolean => handler === undefined || typeof handler === 'function'; - return optionalFunction(onEvent) && optionalFunction(onRunFinished) && (onEvent ?? onRunFinished) !== undefined; -} - function isArtifactStore(value: unknown): value is ArtifactStore { return ( typeof value === 'object' && @@ -881,7 +753,7 @@ function resolveProjectId(explicit: string | undefined, projectRoot: string): st * like a password handle collides. */ function resolveSecrets( - raw: E2EConfig, + raw: ProjectConfig, env: NodeJS.ProcessEnv, ): Pick { const credentials = new Map(); @@ -1038,7 +910,7 @@ const AGENT_NAME_PATTERN = TARGET_NAME_PATTERN; * unknown name is a config error before anything starts. */ function resolveAgents( - raw: E2EConfig['agents'], + raw: ProjectConfig['agents'], selected: readonly string[] | undefined, ): { agents: ReadonlyMap; agentNames: readonly string[]; agent: ResolvedAgentConfig } { if (raw !== undefined && (typeof raw !== 'object' || raw === null || Array.isArray(raw) || isStepExecutor(raw))) { @@ -1073,7 +945,7 @@ function resolveAgents( } function computeConfigDigest( - raw: E2EConfig, + raw: ProjectConfig & { readonly reporters?: unknown; readonly filters?: unknown }, projectId: string, targets: readonly ResolvedTarget[], ): string { @@ -1146,7 +1018,7 @@ function computeConfigDigest( * so it is a plain options object: its plain options are cloned, its tools * named, and its executor reduced to what identifies it. */ -function digestAgents(agents: NonNullable): unknown { +function digestAgents(agents: NonNullable): unknown { return Object.fromEntries( Object.entries(agents).map(([name, entry]) => { const { tools, executor, ...options } = entry ?? {}; @@ -1173,7 +1045,7 @@ function digestAgents(agents: NonNullable): unknown { } /** `cache` as the digest records it: a mode string as is, an options object with its store reduced to whether it writes. */ -function digestCache(cache: NonNullable): unknown { +function digestCache(cache: NonNullable): unknown { if (typeof cache !== 'object' || cache === null) return cache; const { store, ...options } = cache; return { ...options, ...(store === undefined ? {} : { store: { writable: store.writable } }) }; diff --git a/packages/e2e/src/explore/index.ts b/packages/e2e/src/explore/index.ts index 5715422dc8..b2c1517ac7 100644 --- a/packages/e2e/src/explore/index.ts +++ b/packages/e2e/src/explore/index.ts @@ -13,7 +13,8 @@ import type { ModuleRegistration, RegisteredTest } from '../collect/registry.ts' import { selectTargets } from '../collect/select.ts'; import { discoverConfig, loadConfigModule, missingConfigError } from '../config/load.ts'; import { missingModelError, resolveAgentConfig } from '../config/agent.ts'; -import { resolveConfig, type ResolvedConfig, type ResolvedCredential, type ResolvedTarget } from '../config/resolve.ts'; +import { type ResolvedConfig, type ResolvedCredential, type ResolvedTarget } from '../config/resolve.ts'; +import { resolveConfig } from '../config/resolve-run.ts'; import { launchesPinnedApp } from '../config/targets.ts'; import { ConfigurationError } from '../internal/errors.ts'; import type { ReportExplore } from '../report/build.ts'; @@ -22,7 +23,8 @@ import type { RunEventSink } from '../run/events.ts'; import type { RunnerOutput } from '../run/process-output.ts'; import { run, type RunOutcome } from '../run/runner.ts'; import { registerStaticSecrets, staticSecretLedger } from '../run/secrecy.ts'; -import type { AgentConfig, BuiltinReporter, E2EConfig, RecordingMode } from '../types.ts'; +import type { AgentConfig, RecordingMode } from '../types.ts'; +import type { BuiltinReporter, E2EConfig } from '../run-types.ts'; import { createExploreBody } from './body.ts'; import { STEP_BOUNDS, TIMEOUT_BOUNDS } from './bounds.ts'; import { changeContext, readExploreChange, type ChangeBrief, type ExploreChange } from './change.ts'; diff --git a/packages/e2e/src/explore/public-explore.ts b/packages/e2e/src/explore/public-explore.ts index f49916a5e3..0efbf9c873 100644 --- a/packages/e2e/src/explore/public-explore.ts +++ b/packages/e2e/src/explore/public-explore.ts @@ -9,7 +9,7 @@ import type { ReportExplore } from '../report/build.ts'; import type { RunEventSink } from '../run/events.ts'; import { ownedRun, runResult, type RunResult } from '../run/public-run.ts'; import { toSelectionRunOptions } from '../run/runner.ts'; -import type { E2EConfig } from '../types.ts'; +import type { E2EConfig } from '../run-types.ts'; /** Options for one exploration through the public runner API. Recording and output settings come from the config. */ export interface ExplorationOptions { diff --git a/packages/e2e/src/index.ts b/packages/e2e/src/index.ts index a2b0c5d065..cde33aec38 100644 --- a/packages/e2e/src/index.ts +++ b/packages/e2e/src/index.ts @@ -11,6 +11,7 @@ export { renderMarkdownReport } from './report/markdown.ts'; export type { MarkdownReportOptions } from './report/markdown.ts'; export type * from './types.ts'; +export type * from './run-types.ts'; export type { ExecutorActions, ExecutorAttempt, diff --git a/packages/e2e/src/mcp/config.ts b/packages/e2e/src/mcp/config.ts index a4089d4d91..1592f1d3fb 100644 --- a/packages/e2e/src/mcp/config.ts +++ b/packages/e2e/src/mcp/config.ts @@ -2,7 +2,8 @@ import path from 'node:path'; import { discoverConfig, loadConfigModule, missingConfigError } from '../config/load.ts'; -import { resolveConfig, type ResolvedConfig } from '../config/resolve.ts'; +import { type ResolvedConfig } from '../config/resolve.ts'; +import { resolveConfig } from '../config/resolve-run.ts'; import { ConfigurationError } from '../internal/errors.ts'; import type { TargetApp } from '../types.ts'; import { loadEngine, type EngineLookup, type FactoryRef } from './default-engine.ts'; diff --git a/packages/e2e/src/mcp/run-code.ts b/packages/e2e/src/mcp/run-code.ts index 9dc9e4214f..e64a0254e7 100644 --- a/packages/e2e/src/mcp/run-code.ts +++ b/packages/e2e/src/mcp/run-code.ts @@ -33,7 +33,7 @@ import { isPlainObject, rejectUnknownOptions } from '../internal/options.ts'; import { describeExpression } from '../locator/expression.ts'; import { createScreen, locatorInternals, type LocatorInternals } from '../locator/screen.ts'; import { createScreenContext } from '../run/fixtures.ts'; -import type { StandaloneAttempt } from '../run/standalone.ts'; +import type { StandaloneAttempt } from '../run/standalone-attempt.ts'; import { secretHandle } from '../secrets.ts'; import type { App, Screen, Secret } from '../types.ts'; import { parseCode, type Call, type Chain, type Statement, type Value } from './code-parse.ts'; diff --git a/packages/e2e/src/mcp/session.ts b/packages/e2e/src/mcp/session.ts index 14a034057a..9c984c9fa9 100644 --- a/packages/e2e/src/mcp/session.ts +++ b/packages/e2e/src/mcp/session.ts @@ -27,7 +27,8 @@ import { selectTarget } from '../run/select-target.ts'; import { SharedAppProcesses } from '../run/process-pool.ts'; import { outputLayout } from '../run/output.ts'; import { processSecrets, registerStaticSecrets } from '../run/secrecy.ts'; -import { openStandaloneAttempt, type StandaloneAttempt } from '../run/standalone.ts'; +import { openStandaloneAttempt, type OneShotAttempt } from '../run/standalone-kinds.ts'; +import type { StandaloneAttempt } from '../run/standalone-attempt.ts'; import { selectStandaloneTest, testLocation } from '../run/standalone-test.ts'; import type { AgentParams } from '../types.ts'; import { createSessionCatalog, isGrammarVerb, type SessionCatalog } from './catalog.ts'; @@ -301,7 +302,7 @@ export class SessionHost { const cancel = (): void => abort.abort(); if (opening.aborted) cancel(); else opening.addEventListener('abort', cancel, { once: true }); - let attempt: StandaloneAttempt | undefined; + let attempt: OneShotAttempt | undefined; let step: InteractiveStep | undefined; try { attempt = await openStandaloneAttempt({ diff --git a/packages/e2e/src/report/build.ts b/packages/e2e/src/report/build.ts index 4a8ec66515..8d660bcb30 100644 --- a/packages/e2e/src/report/build.ts +++ b/packages/e2e/src/report/build.ts @@ -2,7 +2,6 @@ import os from 'node:os'; import path from 'node:path'; -import { ENGINE_SPI_VERSION, type EngineSpiVersion } from '../engine/contract.ts'; import { BLOCKABLE_CODES } from '../agent/executor.ts'; import { DEFAULT_LIMITS } from '../config/agent.ts'; import type { ResolvedConfig, ResolvedLimits, ResolvedTarget } from '../config/resolve.ts'; @@ -14,8 +13,10 @@ import { packageVersion } from '../internal/package-version.ts'; import { onBun } from '../internal/runtime.ts'; import { projectRelativePath } from '../internal/source.ts'; import type { VcsInfo } from '../internal/vcs.ts'; -import type { SkipInfo } from '../collect/select.ts'; +import type { SkipInfo } from '../run/records.ts'; import { expectedFailureOf, memberRan, someSkippedAfterFailure } from '../run/records.ts'; +import { describeTarget, type TargetProvenance } from '../run/provision.ts'; +import type { EngineSpiVersion } from '../engine/contract.ts'; import type { ArtifactRecord, AttemptRecord, @@ -43,38 +44,6 @@ export interface ReportSource { column: number; } -export interface TargetProvenance { - engine: { name: string; version: string; spiVersion: EngineSpiVersion }; - capabilities: string[]; - /** What the engine can capture: screenshots (a failure's among them), a video. */ - artifactCapabilities: ('screenshot' | 'video')[]; - stateCapability: boolean; -} - -/** - * What a target's engine declaration says about it: the one description the - * runner grades against and the report records. A target without an engine is - * agent-tools-only and honestly reports no capabilities. - */ -export function describeTarget(target: ResolvedTarget): TargetProvenance { - const engine = target.engine; - const artifactCapabilities: TargetProvenance['artifactCapabilities'] = []; - if (engine?.artifacts !== undefined) { - artifactCapabilities.push('screenshot'); - if (engine.artifacts.startVideo !== undefined) artifactCapabilities.push('video'); - } - return { - engine: { - name: engine?.name ?? 'none', - version: engine?.version ?? 'unversioned', - spiVersion: engine?.spiVersion ?? ENGINE_SPI_VERSION, - }, - capabilities: [...(engine?.capabilities ?? [])].toSorted(), - artifactCapabilities, - stateCapability: engine?.state !== undefined, - }; -} - export interface BuildReportOptions { runId: string; config: ResolvedConfig | undefined; diff --git a/packages/e2e/src/report/builtin.ts b/packages/e2e/src/report/builtin.ts index 3fc3827ac6..51b0fd24c9 100644 --- a/packages/e2e/src/report/builtin.ts +++ b/packages/e2e/src/report/builtin.ts @@ -6,7 +6,7 @@ * drift apart on what the ids are. */ -import type { BuiltinReporter, Reporter } from '../types.ts'; +import type { BuiltinReporter, Reporter } from '../run-types.ts'; import { jsonReporter } from './json.ts'; import { junitReporter } from './junit.ts'; import { markdownReporter } from './markdown.ts'; diff --git a/packages/e2e/src/report/json.ts b/packages/e2e/src/report/json.ts index 3fc67a3cf6..d5a5ab2b11 100644 --- a/packages/e2e/src/report/json.ts +++ b/packages/e2e/src/report/json.ts @@ -1,6 +1,6 @@ /** The built-in `json` reporter: the report document to stdout, and nothing else there. */ -import type { Reporter } from '../types.ts'; +import type { Reporter } from '../run-types.ts'; import { writeTerminal } from '../run/process-output.ts'; export const jsonReporter: Reporter = { diff --git a/packages/e2e/src/report/junit.ts b/packages/e2e/src/report/junit.ts index da77094cec..7df5df8164 100644 --- a/packages/e2e/src/report/junit.ts +++ b/packages/e2e/src/report/junit.ts @@ -4,7 +4,7 @@ import path from 'node:path'; import { envValue } from '../internal/env.ts'; import { sanitizeText } from '../internal/errors.ts'; import { realpathOfExisting, relativeToProjectRoot } from '../internal/paths.ts'; -import type { FinishedRun } from '../types.ts'; +import type { FinishedRun } from '../run-types.ts'; import type { Report1Document, ReportError, ReportResult, ReportSerialGroup } from './build.ts'; import { repeatSuffix } from './format.ts'; import { failureScreenshot, toldAttempt } from './failure-text.ts'; diff --git a/packages/e2e/src/report/left-out.ts b/packages/e2e/src/report/left-out.ts index e7ada1d115..e455a90c20 100644 --- a/packages/e2e/src/report/left-out.ts +++ b/packages/e2e/src/report/left-out.ts @@ -5,7 +5,7 @@ * or repeats counts once. */ -import type { SkipInfo } from '../collect/select.ts'; +import type { SkipInfo } from '../run/records.ts'; /** How many tests each test filter left out of `results`, by its name in the order first seen. */ export function leftOutTests(results: Iterable<{ readonly testId: string; readonly skip?: SkipInfo | undefined }>): Map { diff --git a/packages/e2e/src/report/list.ts b/packages/e2e/src/report/list.ts index 8e3944c2d7..81d6808a6d 100644 --- a/packages/e2e/src/report/list.ts +++ b/packages/e2e/src/report/list.ts @@ -9,7 +9,7 @@ import path from 'node:path'; import picocolors from 'picocolors'; -import type { SkipInfo } from '../collect/select.ts'; +import type { SkipInfo } from '../run/records.ts'; import type { SerializedError } from '../internal/errors.ts'; import { resultId } from '../internal/ids.ts'; import { packageVersion } from '../internal/package-version.ts'; @@ -17,7 +17,7 @@ import type { RunEvent, RunEventFact, RunEventOf, RunEventResult, SetupStep } fr import type { StepPrompt } from '../run/step-hold.ts'; import type { AppLogRecord, StepRecord } from '../run/steps.ts'; import { cleanupErrorsOf, expectedFailureOf, failureBeforeSkip, type ArtifactRecord, type AttemptRecord, type AttemptStatus, type FailureEvidence, type ResultStatus, type SerialGroupRecord } from '../run/records.ts'; -import type { Reporter, ReporterSummary } from '../types.ts'; +import type { Reporter, ReporterSummary } from '../run-types.ts'; import { projectSource } from './build.ts'; import { codeFrame, userFrame } from './code-frame.ts'; import { appErrorsOf, type AppErrors } from './failure-text.ts'; diff --git a/packages/e2e/src/report/markdown.ts b/packages/e2e/src/report/markdown.ts index 14d9528138..574eb661be 100644 --- a/packages/e2e/src/report/markdown.ts +++ b/packages/e2e/src/report/markdown.ts @@ -45,7 +45,7 @@ import { cell, code, formatDuration, link, MAX_ID_CHARS, MAX_PATH_CHARS, MAX_TIT import { outcome, type Outcome } from './outcome.ts'; import { isExploreVerdict } from './traces.ts'; import { posixRelative, writeTextReport } from './write.ts'; -import type { Reporter, ReporterSummary } from '../types.ts'; +import type { Reporter, ReporterSummary } from '../run-types.ts'; type ReportRun = Report1Document['run']; type ReportArtifact = ReportResult['attempts'][number]['artifacts'][number]; diff --git a/packages/e2e/src/report/outcome.ts b/packages/e2e/src/report/outcome.ts index c8166ee5e8..5dd497276a 100644 --- a/packages/e2e/src/report/outcome.ts +++ b/packages/e2e/src/report/outcome.ts @@ -6,7 +6,7 @@ * (JUnit, markdown) share this one reading. */ -import type { SkipInfo } from '../collect/select.ts'; +import type { SkipInfo } from '../run/records.ts'; import type { FailureEvidence } from '../run/records.ts'; import type { AppLogRecord } from '../run/steps.ts'; import { isCutSerialRetry } from '../run/retry.ts'; diff --git a/packages/e2e/src/report/write.ts b/packages/e2e/src/report/write.ts index 149946eb7c..de8e2cfb3b 100644 --- a/packages/e2e/src/report/write.ts +++ b/packages/e2e/src/report/write.ts @@ -3,7 +3,7 @@ import { mkdirSync } from 'node:fs'; import path from 'node:path'; import { writeFileAtomic } from '../internal/atomic-write.ts'; -import type { FinishedRun, Reporter } from '../types.ts'; +import type { FinishedRun, Reporter } from '../run-types.ts'; /** Atomically writes one text document, creating the parent directory when it is missing. */ export async function writeTextReport(filePath: string, content: string): Promise { diff --git a/packages/e2e/src/run-types.ts b/packages/e2e/src/run-types.ts new file mode 100644 index 0000000000..6fd23d8f48 --- /dev/null +++ b/packages/e2e/src/run-types.ts @@ -0,0 +1,186 @@ +/** + * The test runner's own public types: the config file's shape, reporters, + * and test filters. Everything else a config names is in `types.ts`. + */ + +import type { RunEvent, RunExitCode, RunStatus } from './run/events.ts'; +import type { Report1Document } from './report/build.ts'; +import type { ProjectConfig } from './types.ts'; + +/** The reporters the runner ships, named by id; each is a `Reporter` on the same contract. */ +export type BuiltinReporter = 'list' | 'json' | 'junit' | 'markdown'; + +/** + * The run's report document, the one `.e2e/report.json` holds. Its + * `schemaVersion` names the wire format (`report-1`), pinned by + * `schema/report-v1.schema.json` in the package. + */ +export type Report = Report1Document; + +/** What a reporter receives once the run is over and its report is on disk. */ +export interface FinishedRun { + /** The report-1 document, as written. */ + readonly report: Report; + /** The same status `report.run.status` carries. */ + readonly status: RunStatus; + /** The exit code the process will end with. */ + readonly exitCode: RunExitCode; + /** Absolute project root, what the terminal shows paths relative to. */ + readonly projectRoot: string; + /** Where `report.json` was written; undefined when the write failed or config never loaded. */ + readonly reportPath: string | undefined; + /** Absolute directory the report's artifact paths are relative to: `/results`, a directory per test. */ + readonly artifactsRoot: string; + /** Where `--ai-trace` wrote the run's model calls, when it was requested. */ + readonly aiTracePath: string | undefined; + /** + * The trace the runner wrote for each test that kept one (by its `trace` + * mode, a failed one by default), by report result id, as a path from the + * project root (`.e2e/results/checkout-applies-the-coupon-1a2b3c4d5e6f7a8b/trace.md`). + */ + readonly traces: ReadonlyMap; + /** + * The report `--last-failed` selected from, when the run was given that + * flag: the run before this one, whose tests that did not fail were left + * out here. A reporter that keeps one place current folds this run into it. + */ + readonly lastRun?: Report; + /** + * Replaces every fixed-value secret and credential password in `text`, and + * any run of 8 or more consecutive characters of one. The report is + * redacted already; this is for text a reporter adds from elsewhere, such + * as lines of a test file. + */ + redact(text: string): string; +} + +/** Rows a reporter hands back for the terminal summary: a label and its text, a URL or a path. */ +export type ReporterSummary = readonly { readonly label: string; readonly text: string }[]; + +/** + * A reporter in `reporters`, the contract the built-in `list`, `json`, + * `junit`, and `markdown` reporters implement too. `onEvent` sees every run event as it + * happens, exactly what the `list` reporter renders, and must not block: a + * throw quarantines it for the rest of the run. Event types are added over + * time; a reporter handles the ones it knows and ignores the rest. `onRunFinished` runs once + * `report.json` is written and the summary has printed; it is awaited within + * a fixed budget, and the rows it resolves with print under the summary. Its + * `signal` aborts when that budget runs out or the run is forced to stop, so + * a reporter hands it to its requests and leaves nothing running behind. A + * reporter can never change the run's status or exit code: a failure or a + * timeout is one line on stderr. Like every live value, a reporter never + * crosses a process boundary; workers construct their own copy when they + * load the config module and never call it, so constructing one must have no + * side effects. + */ +export interface Reporter { + /** Label in diagnostics; required. */ + readonly name: string; + /** Every run event as it happens. Must not block or throw. */ + onEvent?(event: RunEvent): void; + /** Runs once the report is written and the summary printed; awaited one minute. Returned rows print under the summary. */ + onRunFinished?(run: FinishedRun, signal: AbortSignal): Promise; +} + +/** + * Leaves collected tests out of a run before anything starts. Called at most + * once, in the runner process; workers load the config too and never call + * it, so constructing one must have no side effects: read git, the + * environment, or the network inside `filter`. A call that throws, returns + * an invalid result, or runs past 5 minutes leaves nothing out. A failed call + * scores nothing either, unless the error it throws has a `relevance` + * property shaped like `TestFilterResult.relevance`: the run checks it as it + * checks a result's, starts the likeliest tests first, and still warns of the + * failure. A call past 5 minutes is abandoned, and nothing it throws after + * that is read. + */ +export interface TestFilter { + /** Names the filter in the report and in errors: ASCII letters, numbers, `_`, `-`, and `.`, not only dots, at most 256 characters, unique among `filters`. */ + readonly name: string; + /** Decides which of `tests` to leave out; every test the result does not name runs. */ + readonly filter: (tests: readonly TestFilterCandidate[], context: TestFilterContext) => TestFilterResult | Promise; +} + +/** One test a filter may leave out: what names it, never its body. */ +export interface TestFilterCandidate { + /** Stable test id (`::`), the report's `testId`, and the key `exclude` and `relevance` take. */ + readonly id: string; + /** Project-relative POSIX path of the test file. */ + readonly file: string; + /** The describe titles, then the test title. */ + readonly titlePath: readonly string[]; + /** The describe chain's tags, outermost first, then the test's own. */ + readonly tags: readonly string[]; + /** Project-relative POSIX path of the setup test file that saves the test's session; absent when the test uses none. */ + readonly setupFile?: string; +} + +/** What a run's test filters judge: every file that differs from the base commit. */ +export interface TestFilterChanges { + /** The commit the change is measured from: the merge base of the base ref and HEAD. */ + readonly base: string; + /** Absolute path of the git repository root, spelled as on disk like `projectRoot`; `files` are relative to it. */ + readonly root: string; + /** + * Repository-relative POSIX paths of every file added, modified, or + * deleted since `base`, both paths of a rename, uncommitted and untracked + * files included; sorted, never empty. Spelled as git prints them, which + * on macOS is NFC while a test's `file` keeps the bytes on disk, so + * compare a non-ASCII path with `.normalize('NFC')` on both sides. + */ + readonly files: readonly string[]; +} + +/** What the runner hands a filter besides its tests. */ +export interface TestFilterContext { + /** Absolute project root, spelled as on disk like `changes.root`: symlinks resolved, names in on-disk case; a test's `file` is relative to it. */ + readonly projectRoot: string; + /** The change the filter judges; the same for every filter of the run. */ + readonly changes: TestFilterChanges; + /** + * The config file this run loaded, whatever its name, as a POSIX path + * relative to `changes.root` like `changes.files`, in its on-disk bytes + * (compare in NFC, as with a test's `file`); `undefined` when the run was + * given a config object or loaded one outside the repository. + */ + readonly configFile?: string | undefined; + /** + * Aborts when the run is interrupted, or with a `TimeoutError` reason after + * 5 minutes; hand it to every model and network call. + */ + readonly signal: AbortSignal; + /** Prints one progress line as a run notice. */ + log(line: string): void; + /** Replaces every fixed-value secret and credential password in `text`, and any run of 8 or more consecutive characters of one. */ + redact(text: string): string; +} + +/** A filter's answer: every test it does not name runs. */ +export interface TestFilterResult { + /** The tests to leave out, by `TestFilterCandidate.id`, each with the reason the report shows; a test of a serial group still runs when another test of its group does. */ + readonly exclude: Readonly>; + /** + * How likely each test is to fail on this change, from 0 to 1, by + * `TestFilterCandidate.id`. Each target starts the files of the likeliest + * tests first, in bands at 75%, 50%, and 25%, and targets go by the band of + * their best file, ties in config order. A test no filter scores counts as + * 0, and one several filters score keeps its highest. It never leaves a + * test out. + */ + readonly relevance?: Readonly>; + /** One line about the whole decision, such as `the diff is too large to judge`. */ + readonly note?: string; +} + +/** The config an `e2e.config.ts` default-exports, through `defineConfig`. */ +export interface E2EConfig extends ProjectConfig { + /** + * Output renderers and reporter objects. `junit` writes `/junit.xml`, + * `markdown` writes `/summary.md`, `json` prints the report and + * excludes `list`; a `Reporter` object runs + * beside them and `--reporter` never removes it. + */ + reporters?: readonly (BuiltinReporter | Reporter)[]; + /** Test filters, in order, run with `--changed-since`: each judges the tests the one before it kept. */ + filters?: readonly TestFilter[]; +} diff --git a/packages/e2e/src/run/app-ports.ts b/packages/e2e/src/run/app-ports.ts index dd1a07bbc8..12ad4a27dc 100644 --- a/packages/e2e/src/run/app-ports.ts +++ b/packages/e2e/src/run/app-ports.ts @@ -16,7 +16,7 @@ import { ConfigurationError, errorMessage } from '../internal/errors.ts'; * loopback host this machine cannot bind is a bad app URL, reported before * anything starts. */ -export async function allocateAppPorts(config: ResolvedConfig): Promise { +export async function allocateAppPorts(config: Config): Promise { return allocatePorts(config, config.targets); } @@ -24,12 +24,12 @@ export async function allocateAppPorts(config: ResolvedConfig): Promise { +export async function allocateTargetPorts(config: Config, target: ResolvedTarget): Promise<{ config: Config; target: ResolvedTarget }> { const allocated = await allocatePorts(config, [target]); return { config: allocated, target: allocated.targets.find((candidate) => candidate.name === target.name)! }; } -async function allocatePorts(config: ResolvedConfig, among: readonly ResolvedTarget[]): Promise { +async function allocatePorts(config: Config, among: readonly ResolvedTarget[]): Promise { const pending = among.filter((target) => target.app.portRequest !== undefined && !Object.hasOwn(config.ports, target.name)); if (pending.length === 0) return config; diff --git a/packages/e2e/src/run/changes.ts b/packages/e2e/src/run/changes.ts index c0feb97530..ed0b38e5b5 100644 --- a/packages/e2e/src/run/changes.ts +++ b/packages/e2e/src/run/changes.ts @@ -12,7 +12,7 @@ import { compareCodePoints } from '../internal/globs.ts'; import { onDiskPath } from '../internal/paths.ts'; import { budgetText } from '../internal/time.ts'; import { ellipsize } from '../report/format.ts'; -import type { TestFilterChanges } from '../types.ts'; +import type { TestFilterChanges } from '../run-types.ts'; /** * A `changedSince` that is not a string, is empty, or reads as a git option diff --git a/packages/e2e/src/run/engine-sessions.ts b/packages/e2e/src/run/engine-sessions.ts new file mode 100644 index 0000000000..a9aba46625 --- /dev/null +++ b/packages/e2e/src/run/engine-sessions.ts @@ -0,0 +1,438 @@ +/** + * The engine side of one worker: the engine's init and dispose, and each + * attempt's session from launch to close. The runner's `TargetExecutor` runs + * tests on top of it; a standalone worker drives it directly. + */ + +import { rm } from 'node:fs/promises'; +import path from 'node:path'; +import type { EngineDisposeContext, EngineSnapshot, ResolveSecretOptions } from '../engine/index.ts'; +import type { TargetSession, OperationContext, VideoSegment } from '../engine/surface.ts'; +import type { ResolvedConfig, ResolvedTarget } from '../config/resolve.ts'; +import { classifyError, ConfigurationError, InfrastructureError, serializeError, translateEngineError, type SerializedError } from '../internal/errors.ts'; +import { DebugTrace } from '../internal/debug.ts'; +import { engineAppInfo } from '../config/app.ts'; +import { NEVER_ABORTS, withScopedBudget } from '../internal/time.ts'; +import { keeps, type AttemptRecordings, type VideoRecording } from '../internal/recording-modes.ts'; +import type { Secret } from '../types.ts'; +import { createEngineSession } from '../engine/session.ts'; +import type { ArtifactSink } from './fixtures.ts'; +import type { AttemptRecord, RunError } from './records.ts'; +import { adoptSecrets, registerDerivedSecrets, resolveSecretValue, sessionLedger } from './secrecy.ts'; +import { isSecret } from '../secrets.ts'; +import { targetIdentity, type SessionIdentity, type SessionStore } from './sessions.ts'; +import { EngineError } from '../engine/contract.ts'; +import { hostedVideoUrl } from '../engine/recording.ts'; +import { sessionRecordings } from './recordings.ts'; + +/** What a worker's engine sessions need: the target, its config, and the worker's place in the run. */ +export interface EngineSessionsOptions { + readonly config: ResolvedConfig; + readonly target: ResolvedTarget; + readonly runId: string; + readonly sessionStore: SessionStore; + readonly headed: boolean; + /** This worker's slot among the target's workers; see `EngineInitInfo.workerSlot`. */ + readonly workerSlot: number; + /** This worker's environment; see `EngineInitInfo.env`. */ + readonly env: Readonly>; + readonly interruptSignal: AbortSignal; + readonly debug?: DebugTrace; + readonly events?: { onNotice?(message: string): void }; +} + +/** + * The slice of an attempt record a session close reads and writes: the + * verdict the caller has already reached, and the cleanup outcome the close + * reports. + */ +export interface ClosingRecord { + readonly status: AttemptRecord['status']; + cleanup: AttemptRecord['cleanup']; +} + +/** + * What one attempt's session opens with: the saved session to restore, and + * what the attempt captures (`pairRecordings`): the video it records, and + * whether it keeps a trace, which decides whether screens are taken at all. + */ +export interface SessionPlan { + readonly session: string | undefined; + readonly recordings: AttemptRecordings; +} + +/** What closing an attempt's session needs besides its verdict: the video it started. */ +export interface SessionClose { + readonly attemptId: string; + readonly video: VideoRecording | undefined; +} + +/** One worker's engine: init once, then one session per attempt. */ +export class EngineSessions { + readonly target: ResolvedTarget; + get runId(): string { + return this.options.runId; + } + readonly interruptSignal: AbortSignal; + readonly debug: DebugTrace; + + protected readonly runErrors: RunError[] = []; + protected readonly sessionIdentity: SessionIdentity; + /** Resolves once the engine's init hook completed for this worker. */ + private engineReady: Promise | undefined; + + constructor(protected readonly options: EngineSessionsOptions) { + this.target = options.target; + this.interruptSignal = options.interruptSignal; + this.debug = options.debug ?? new DebugTrace(false); + this.sessionIdentity = targetIdentity(options.target); + } + + protected get config(): ResolvedConfig { + return this.options.config; + } + + /** Builds one engine operation context. */ + protected op(attemptId: string, timeoutMs: number, signal: AbortSignal): OperationContext { + return { signal, timeoutMs, runId: this.options.runId, attemptId, origin: 'test' }; + } + + /** Run-level errors recorded so far, in order. */ + collectedRunErrors(): readonly RunError[] { + return this.runErrors; + } + + /** + * Runs the engine's init hook once per worker, before the first session. + * Boot work (simulators, device leases) is bounded by the launch timeout + * but never charged against a step budget. + */ + initEngineOnce(): Promise { + const engine = this.target.engine; + const init = engine?.init?.bind(engine); + if (init === undefined) return Promise.resolve(); + // Memoized for the worker's lifetime, failure included: an engine that + // could not boot fails every attempt on this worker with the same cause + // instead of re-running a broken boot per test. Init outlives any single + // attempt, so it aborts on worker interrupt, not on one test's deadline. + this.engineReady ??= this.debug.time('engine.init', () => + this.lifecycle( + `initializing engine ${engine?.name ?? 'none'}`, + this.config.launchTimeout, + 'LAUNCH_TIMEOUT', + this.interruptSignal, + (signal) => + init({ + runId: this.options.runId, + targetName: this.target.name, + projectRoot: this.config.projectRoot, + app: engineAppInfo(this.target.app), + env: this.options.env, + headed: this.options.headed, + workerSlot: this.options.workerSlot, + signal, + log: (line) => this.options.events?.onNotice?.(`${this.target.name} worker ${String(this.options.workerSlot)}: ${line}`), + }), + ), + ); + return this.engineReady; + } + + /** + * Runs one lifecycle call on the engine seam within the given budget. The + * signal handed to `run` follows `parent` and is aborted the moment the call + * fails, timeout included, so a hook that outlived its budget is told to + * stop instead of running on into the retry. Synchronous throws are caught, + * and every failure is translated onto the runner taxonomy: an engine + * failing outside its contract is infrastructure, never a test error. + */ + protected async lifecycle( + label: string, + timeoutMs: number, + code: 'LAUNCH_TIMEOUT' | 'CLEANUP_TIMEOUT', + parent: AbortSignal, + run: (signal: AbortSignal) => Promise, + ): Promise { + try { + return await withScopedBudget(timeoutMs, parent, () => new InfrastructureError(code, `${label} timed out`), run); + } catch (cause) { + throw translateEngineError(cause, ` while ${label}`); + } + } + + /** + * Disposes the engine at worker end of life, bounded by the cleanup budget. + * Every engine runs over the protocol, whose client forwards `dispose` only + * once `init` was called: a worker that never initialized opened no + * connection, so the engine has nothing of it to release. + */ + async dispose(options: Pick = {}): Promise { + const disposeEngine = this.target.engine?.dispose?.bind(this.target.engine); + if (disposeEngine === undefined) return; + try { + await this.lifecycle( + 'disposing the engine', + this.config.cleanupTimeout, + 'CLEANUP_TIMEOUT', + NEVER_ABORTS, + (signal) => disposeEngine({ signal, timeoutMs: this.config.cleanupTimeout, ...options }), + ); + } catch (cause) { + this.runErrors.push({ error: serializeError(classifyError(cause), { phase: 'cleanup' }) }); + } + } + + /** Starts one attempt on the engine, restores the planned session, and starts the planned video. */ + async launchSession( + plan: SessionPlan, + attemptId: string, + artifactsDir: string, + signal: AbortSignal, + ): Promise { + // The engine booted in init() once per worker; the adapter is per-attempt + // so refs never cross attempts. + await this.initEngineOnce(); + const engine = this.target.engine; + // Session restore rides the engine's neutral state capability. Checked + // before any per-attempt isolation opens, so a misconfigured session never + // orphans a started attempt. + const sessionName = plan.session; + if (sessionName !== undefined && engine?.state === undefined) { + throw new ConfigurationError( + 'UNSUPPORTED_CAPABILITY', + `target "${this.target.name}" has no engine state capability for session restore`, + ); + } + const { video } = plan.recordings; + const session = createEngineSession({ engine, targetName: this.target.name, traced: plan.recordings.trace !== undefined }); + const launch = (label: string, run: (launchSignal: AbortSignal) => Promise) => + this.lifecycle(label, this.config.launchTimeout, 'LAUNCH_TIMEOUT', signal, run); + const launchOp = (launchSignal: AbortSignal) => + this.op(attemptId, this.config.launchTimeout, launchSignal); + try { + const startAttempt = engine?.startAttempt?.bind(engine); + if (startAttempt !== undefined) { + await this.debug.time('session.launch', () => + launch(`starting an attempt on engine ${engine?.name ?? 'none'}`, (launchSignal) => + startAttempt({ + attemptId, + artifactsDir, + signal: launchSignal, + resolveSecret: (secret, options) => this.resolveEngineSecret(session, secret, options), + appLog: (entry) => session.appLog.push(entry), + navigation: (line) => session.appLog.navigated(line), + ...(session.screens.traced ? { screen: (snapshot: EngineSnapshot) => session.screens.push(snapshot) } : {}), + environment: (facts) => session.environment.push(facts), + }), + ), + ); + } + if (sessionName !== undefined) { + const saved = await this.options.sessionStore.load(sessionName, this.sessionIdentity); + // The values the saving session learned come back with its state, so + // a stored value the app echoes is redacted here too. + adoptSecrets(sessionLedger(session, this.config.allSecrets), saved.secrets); + await launch('restoring the session', (launchSignal) => + session.restoreState!(saved.state, launchOp(launchSignal)), + ); + } + // A video the attempt asks for starts here, when the engine has it: an + // engine without it was either refused before the run (a mode a target + // or test set) or skipped with a notice (a run-wide mode). A required + // video's failure fails the launch; a default one's is swallowed. + if (video !== undefined && session.artifacts.startVideo !== undefined) { + const recordings = sessionRecordings(session, this.target.name); + const starting = launch('starting the video', (launchSignal) => recordings.startAttemptVideo(launchOp(launchSignal))); + if (video.policy === 'required') await starting; + else await starting.catch(() => undefined); + } + } catch (cause) { + // The attempt's isolation is open, or a timed-out startAttempt may still + // open it: end it within the cleanup budget, or the retry opens a second + // one. An engine without startAttempt makes this a no-op. + await this.endAttempt(session, attemptId).catch(() => undefined); + throw cause; + } + return session; + } + + /** + * The plaintext of a secret the target's engine declared, for an option it + * hands the app (basic-auth credentials). Registered for the session's and + * the process's redaction by `resolveSecretValue`, with every value the + * engine derives from it (the base64 credential an `Authorization` header + * carries) under the same name, so its text and text downloads are + * redacted of them. Its pixels stay as they are, since nothing was typed. + */ + private async resolveEngineSecret(session: TargetSession, secret: Secret, options?: ResolveSecretOptions): Promise { + const engine = this.target.engine; + if (!isSecret(secret) || !(engine?.secrets ?? []).some((declared) => declared.name === secret.name)) { + const name = isSecret(secret) ? `"${secret.name}"` : 'a value that is not a secrets.get() handle'; + throw new ConfigurationError( + 'SECRET_UNAVAILABLE', + `engine ${engine?.name ?? 'none'} asked for secret ${name}, which it did not declare in its secrets`, + ); + } + const ledger = sessionLedger(session, this.config.allSecrets); + const plaintext = await resolveSecretValue(secret, this.config.allSecrets, ledger); + registerDerivedSecrets(secret.name, (await options?.derived?.(plaintext)) ?? [], ledger); + return plaintext; + } + + /** + * Collects a failure the engine holds from a path no step awaited, within + * the cleanup budget; the interrupt is the one thing that cuts it short. + * The error is the test's, thrown as it was so `classifyError` keeps its + * code: `lifecycle` would name every non-engine error an engine failure. + */ + settleAttempt(session: TargetSession, attemptId: string): Promise { + return withScopedBudget( + this.config.cleanupTimeout, + this.interruptSignal, + () => new InfrastructureError('CLEANUP_TIMEOUT', 'settling the attempt timed out'), + (signal) => session.settle(this.op(attemptId, this.config.cleanupTimeout, signal)), + ); + } + + /** Ends one attempt's isolation within the cleanup budget. */ + private endAttempt(session: TargetSession, attemptId: string): Promise { + return this.lifecycle( + 'ending the attempt', + this.config.cleanupTimeout, + 'CLEANUP_TIMEOUT', + NEVER_ABORTS, + (signal) => session.close(this.op(attemptId, this.config.cleanupTimeout, signal)), + ); + } + + /** + * Finalizes the video, then ends the attempt with a fresh cleanup + * budget. The caller has classified the attempt by now: `record.status` + * decides whether an `on-failure` recording is kept. + */ + async closeSession( + session: TargetSession, + close: SessionClose, + record: ClosingRecord, + artifactSink: ArtifactSink, + secondaryErrors: SerializedError[], + ): Promise { + const { attemptId, video } = close; + await this.stopRunningRecording(session, attemptId, record, artifactSink, secondaryErrors); + if (session.artifacts.stopVideo !== undefined && video !== undefined) { + const recordings = sessionRecordings(session, this.target.name); + await this.stopVideo(video, attemptId, record, secondaryErrors, async (operation) => { + const segments = await recordings.stopAttemptVideo(operation); + // Recorded so a failure could be looked at; a pass has nothing to show. + if (!keeps(video.keep, record.status)) { + await Promise.all( + segments.flatMap((segment) => ('path' in segment ? [rm(path.join(artifactSink.dir, segment.path), { force: true })] : [])), + ); + return; + } + registerVideos(artifactSink, segments); + }); + } + try { + await this.debug.time('session.close', () => this.endAttempt(session, attemptId)); + } catch (cause) { + record.cleanup = 'failed'; + secondaryErrors.push(serializeError(classifyError(cause), { phase: 'cleanup' })); + } + } + + /** + * Stops a recording the test started with `app.startRecording` and left + * running, within the cleanup budget, and keeps it as the attempt's + * artifact. One that cannot be finalized is a cleanup failure. + */ + protected async stopRunningRecording( + session: TargetSession, + attemptId: string, + record: ClosingRecord, + artifactSink: ArtifactSink, + secondaryErrors: SerializedError[], + ): Promise { + const recordings = sessionRecordings(session, this.target.name); + if (!recordings.spanRunning) return; + try { + await this.lifecycle( + 'stopping the recording', + this.config.cleanupTimeout, + 'CLEANUP_TIMEOUT', + NEVER_ABORTS, + (signal) => recordings.stopSpan(this.op(attemptId, this.config.cleanupTimeout, signal), artifactSink), + ); + } catch (cause) { + record.cleanup = 'failed'; + secondaryErrors.push(serializeError(classifyError(cause), { phase: 'cleanup' })); + } + } + + /** + * Stops the video the attempt started, within the cleanup budget. A + * required video that cannot be finalized is a cleanup failure the report + * must show; a best-effort one fails quietly. + */ + private async stopVideo( + recording: VideoRecording, + attemptId: string, + record: ClosingRecord, + secondaryErrors: SerializedError[], + stop: (operation: OperationContext) => Promise, + ): Promise { + try { + await this.lifecycle( + 'stopping the video', + this.config.cleanupTimeout, + 'CLEANUP_TIMEOUT', + NEVER_ABORTS, + (signal) => stop(this.op(attemptId, this.config.cleanupTimeout, signal)), + ); + } catch (cause) { + if (recording.policy !== 'required') return; + record.cleanup = 'failed'; + secondaryErrors.push(serializeError(classifyError(cause), { phase: 'cleanup' })); + } + } + +} + +/** + * The status of a failed attempt. An interrupt outranks everything except a + * timeout that had already fired: the attempt then timed out, whatever + * arrived after. + */ +export function classifyAttemptStatus( + failure: { readonly code: string }, + timedOut: boolean, + interrupted: boolean, +): 'failed' | 'timed-out' | 'interrupted' { + if (interrupted && !timedOut) return 'interrupted'; + if (timedOut || failure.code === 'TEST_TIMEOUT') { + return 'timed-out'; + } + return 'failed'; +} + +/** + * Registers an attempt's kept recordings, in order: a file under its path, a + * link by URL. An engine's own link is held to what a provider's is, an + * http(s) URL; one that is not fails the cleanup once every other recording + * is registered, so it costs no other video. + */ +function registerVideos(sink: ArtifactSink, segments: readonly VideoSegment[]): void { + const rejected: string[] = []; + for (const segment of segments) { + if ('path' in segment) { + sink.register('video', segment.path, { startedAt: segment.startedAt }); + continue; + } + const url = hostedVideoUrl(segment.url); + if (url === undefined) rejected.push(JSON.stringify(segment.url)); + else sink.link(url, { mediaType: segment.mediaType, startedAt: segment.startedAt }); + } + if (rejected.length > 0) { + throw new EngineError('ENGINE_FAILURE', `the engine returned a video link that is not an http(s) URL: ${rejected.join(', ')}`, { retryable: false }); + } +} \ No newline at end of file diff --git a/packages/e2e/src/run/execute.ts b/packages/e2e/src/run/execute.ts index 1431656ea9..06f525b15c 100644 --- a/packages/e2e/src/run/execute.ts +++ b/packages/e2e/src/run/execute.ts @@ -1,45 +1,27 @@ /** Test-target execution engine. */ -import { rm } from 'node:fs/promises'; import path from 'node:path'; -import type { EngineDisposeContext, EngineSnapshot, ResolveSecretOptions } from '../engine/index.ts'; -import type { TargetSession, OperationContext, VideoSegment } from '../engine/surface.ts'; -import type { ResolvedConfig, ResolvedTarget } from '../config/resolve.ts'; -import { - classifyError, - ConfigurationError, - E2EError, - InfrastructureError, - serializeError, - sanitizeText, - truncateUtf8, - TestError, - TestTimeoutError, - translateEngineError, - type SerializedError, -} from '../internal/errors.ts'; +import type { TargetSession } from '../engine/surface.ts'; +import type { ResolvedConfig } from '../config/resolve.ts'; +import { classifyError, ConfigurationError, E2EError, serializeError, sanitizeText, truncateUtf8, TestError, TestTimeoutError, type SerializedError } from '../internal/errors.ts'; import { isRuntimeSkip, type RuntimeSkip } from '../internal/skip.ts'; import { lateExpectedFailureError } from '../internal/expected-failure.ts'; import { settleExpectedFailure } from './expected-failure.ts'; import type { ExecutorAttempt } from '../agent/executor.ts'; import { isModelUnreachable } from '../agent/error.ts'; import { withAiTraceScope } from '../internal/ai-trace.ts'; -import { DebugTrace } from '../internal/debug.ts'; import { resultId, timestamp, uuidv7 } from '../internal/ids.ts'; -import { engineAppInfo } from '../config/app.ts'; -import { Deadline, NEVER_ABORTS, withAbort, withScopedBudget, withTimeout } from '../internal/time.ts'; +import { Deadline, withAbort, withTimeout } from '../internal/time.ts'; import { createAgentCacheContext, settleStagedTraces } from '../cache/context.ts'; import { storedRecordingsFor, type StoredRecordings } from '../cache/rekeyed.ts'; import type { ModuleRegistration, RegisteredTest } from '../collect/registry.ts'; import { pairRecordings, type TestTargetPair } from '../collect/select.ts'; -import { keeps, type AttemptRecordings, type VideoRecording } from '../internal/recording-modes.ts'; -import type { ArtifactStore, Secret } from '../types.ts'; +import type { ArtifactStore } from '../types.ts'; import { attemptSegments, createAttemptArtifacts, resultSegment } from './artifacts.ts'; import { AttemptBudget } from './budget.ts'; -import { createEngineSession } from '../engine/session.ts'; import { createExtendedFixtures } from './extended-fixtures.ts'; import { captureFailureEvidence } from './failure-evidence.ts'; -import { createFixtures, type ArtifactSink } from './fixtures.ts'; +import { createFixtures } from './fixtures.ts'; import { WrittenScreenshots } from './screenshots.ts'; import { publishAttempt } from '../expect/attempt.ts'; import { PollScope, runInPollScope } from '../expect/poll-scope.ts'; @@ -58,15 +40,12 @@ import { traceScreen } from './step-screens.ts'; import { runWithRetries } from './retry.ts'; import { runSerialUnit, type SerialAttemptStart, type SerialHost, type SharedSerialSession } from './serial.ts'; import { interruptedInPhase, interruptedSkip, pairKey, pairResult, unstartedResult } from './units.ts'; -import { adoptSecrets, carriedSecrets, processSecrets, registerDerivedSecrets, redactForSession, resolveSecretValue, sessionLedger, staticSecretLedger } from './secrecy.ts'; -import { isSecret } from '../secrets.ts'; -import { SessionStaging, SessionStore, targetIdentity, type SessionIdentity } from './sessions.ts'; +import { carriedSecrets, processSecrets, redactForSession, sessionLedger, staticSecretLedger } from './secrecy.ts'; +import { SessionStaging } from './sessions.ts'; import { StepRecorder, type StepHoldRequest, type StepProgress } from './steps.ts'; import { AttemptHolds, type HeldStep } from './step-hold.ts'; -import { EngineError } from '../engine/contract.ts'; -import { hostedVideoUrl } from '../engine/recording.ts'; -import { sessionRecordings } from './recordings.ts'; import { WorkerModels } from './worker-models.ts'; +import { classifyAttemptStatus, EngineSessions, type EngineSessionsOptions } from './engine-sessions.ts'; import type { SetupFn } from '../types.ts'; export interface ExecutionEvents { @@ -105,10 +84,7 @@ export interface ExecutionEvents { onRunAbort?(error: RunError): void; } -export interface TargetExecutorOptions { - readonly config: ResolvedConfig; - readonly target: ResolvedTarget; - readonly runId: string; +export interface TargetExecutorOptions extends EngineSessionsOptions { readonly artifactsRoot: string; /** * The directory under `artifactsRoot` a `--last-failed` rerun's attempts @@ -116,12 +92,6 @@ export interface TargetExecutorOptions { * undefined when the attempts fill the root themselves. */ readonly rerunDir?: string | undefined; - readonly sessionStore: SessionStore; - readonly headed: boolean; - /** This worker's slot among the target's workers; see `EngineInitInfo.workerSlot`. */ - readonly workerSlot: number; - /** This worker's environment; see `EngineInitInfo.env`. */ - readonly env: Readonly>; /** * Whether this executor runs in a process of its own that ends with its * work. Only then can an interrupted test body be abandoned mid-flight: @@ -130,7 +100,6 @@ export interface TargetExecutorOptions { * interrupt waits for it to reach a harness call or its timeout. */ readonly isolated: boolean; - readonly interruptSignal: AbortSignal; /** * Holds each top-level step of a test's first attempt until the runner * lets it run (`e2e run --step`); it resolves to run the step. Setup tests @@ -139,7 +108,6 @@ export interface TargetExecutorOptions { readonly holdStep?: ((step: HeldStep, signal: AbortSignal) => Promise) | undefined; /** Whether each hold reads the screen it starts on into `HeldStep.screen`. */ readonly heldScreens?: boolean | undefined; - readonly debug?: DebugTrace; readonly events?: ExecutionEvents; /** The virtual file of tests registered in memory, and the agents they run as; see `InMemoryAttempts`. */ readonly inMemory?: InMemoryAttempts | undefined; @@ -158,32 +126,6 @@ export interface InMemoryAttempts { type AttemptPhase = 'launch' | 'beforeEach' | 'body' | 'afterEach'; -/** - * The slice of an attempt record a session close reads and writes: the - * verdict the caller has already reached, and the cleanup outcome the close - * reports. - */ -export interface ClosingRecord { - readonly status: AttemptRecord['status']; - cleanup: AttemptRecord['cleanup']; -} - -/** - * What one attempt's session opens with: the saved session to restore, and - * what the attempt captures (`pairRecordings`): the video it records, and - * whether it keeps a trace, which decides whether screens are taken at all. - */ -export interface SessionPlan { - readonly session: string | undefined; - readonly recordings: AttemptRecordings; -} - -/** What closing an attempt's session needs besides its verdict: the video it started. */ -export interface SessionClose { - readonly attemptId: string; - readonly video: VideoRecording | undefined; -} - /** How one attempt acquires its session and session-staging hooks. */ export type AttemptContext = | { readonly kind: 'ordinary' } @@ -191,41 +133,29 @@ export type AttemptContext = | { readonly kind: 'serial'; readonly shared: SharedSerialSession }; /** Executes every selected pair for one target sequentially. */ -export class TargetExecutor implements SerialHost { - readonly target: ResolvedTarget; - get runId(): string { - return this.options.runId; - } +export class TargetExecutor extends EngineSessions implements SerialHost { get artifactStore(): ArtifactStore | undefined { return this.config.artifactStore; } readonly artifactsRoot: string; readonly rerunDir: string | undefined; - readonly interruptSignal: AbortSignal; readonly realms: RealmManager; - readonly debug: DebugTrace; - private readonly runErrors: RunError[] = []; /** Fails the attempt in flight from outside it, while one is; see `strayRejection`. */ private strayFailure: ((cause: unknown) => void) | undefined; /** The last test whose attempt ended here: the likely source of a rejection that surfaces between attempts. */ private lastAttemptTestId: string | undefined; private readonly models: WorkerModels; - private readonly sessionIdentity: SessionIdentity; /** The file store's listing under `cache.strict`, read once for every attempt here. */ private readonly storedRecordings: StoredRecordings | undefined; - /** Resolves once the engine's init hook completed for this worker. */ - private engineReady: Promise | undefined; /** The stored screenshots the run wrote, so no comparison passes against one nobody reviewed. */ private readonly writtenScreenshots: WrittenScreenshots; - constructor(private readonly options: TargetExecutorOptions) { - this.target = options.target; + constructor(protected override readonly options: TargetExecutorOptions) { + super(options); this.artifactsRoot = options.artifactsRoot; this.writtenScreenshots = new WrittenScreenshots(options.artifactsRoot); this.rerunDir = options.rerunDir; - this.interruptSignal = options.interruptSignal; - this.debug = options.debug ?? new DebugTrace(false); this.models = new WorkerModels((error) => { options.events?.onRunAbort?.({ error: serializeError(error) }); }); @@ -238,14 +168,9 @@ export class TargetExecutor implements SerialHost { redactTitle: staticSecretLedger(options.config.allSecrets).redact, debug: this.debug, }); - this.sessionIdentity = targetIdentity(options.target); this.storedRecordings = storedRecordingsFor(options.config.cache); } - private get config(): ResolvedConfig { - return this.options.config; - } - /** Emits one final result record (SerialHost). Records are not retained. */ emit(result: ResultRecord): void { this.options.events?.onResult?.(result); @@ -271,11 +196,6 @@ export class TargetExecutor implements SerialHost { this.options.events?.onSerialAttempt?.(groupId, attempt); } - /** Builds one engine operation context. */ - private op(attemptId: string, timeoutMs: number, signal: AbortSignal): OperationContext { - return { signal, timeoutMs, runId: this.options.runId, attemptId, origin: 'test' }; - } - /** * The screen a held step waits on, as the trace page lists one: its * location, then a line per node, redacted with the session's ledger. Read @@ -297,11 +217,6 @@ export class TargetExecutor implements SerialHost { } } - /** Run-level errors recorded so far, in order. */ - collectedRunErrors(): readonly RunError[] { - return this.runErrors; - } - /** * Takes a rejection nobody in this process caught: a promise a test started * and never awaited. Node cannot say which test created it, so it is @@ -331,85 +246,6 @@ export class TargetExecutor implements SerialHost { return true; } - /** - * Runs the engine's init hook once per worker, before the first session. - * Boot work (simulators, device leases) is bounded by the launch timeout - * but never charged against a step budget. - */ - initEngineOnce(): Promise { - const engine = this.target.engine; - const init = engine?.init?.bind(engine); - if (init === undefined) return Promise.resolve(); - // Memoized for the worker's lifetime, failure included: an engine that - // could not boot fails every attempt on this worker with the same cause - // instead of re-running a broken boot per test. Init outlives any single - // attempt, so it aborts on worker interrupt, not on one test's deadline. - this.engineReady ??= this.debug.time('engine.init', () => - this.lifecycle( - `initializing engine ${engine?.name ?? 'none'}`, - this.config.launchTimeout, - 'LAUNCH_TIMEOUT', - this.interruptSignal, - (signal) => - init({ - runId: this.options.runId, - targetName: this.target.name, - projectRoot: this.config.projectRoot, - app: engineAppInfo(this.target.app), - env: this.options.env, - headed: this.options.headed, - workerSlot: this.options.workerSlot, - signal, - log: (line) => this.options.events?.onNotice?.(`${this.target.name} worker ${String(this.options.workerSlot)}: ${line}`), - }), - ), - ); - return this.engineReady; - } - - /** - * Runs one lifecycle call on the engine seam within the given budget. The - * signal handed to `run` follows `parent` and is aborted the moment the call - * fails, timeout included, so a hook that outlived its budget is told to - * stop instead of running on into the retry. Synchronous throws are caught, - * and every failure is translated onto the runner taxonomy: an engine - * failing outside its contract is infrastructure, never a test error. - */ - private async lifecycle( - label: string, - timeoutMs: number, - code: 'LAUNCH_TIMEOUT' | 'CLEANUP_TIMEOUT', - parent: AbortSignal, - run: (signal: AbortSignal) => Promise, - ): Promise { - try { - return await withScopedBudget(timeoutMs, parent, () => new InfrastructureError(code, `${label} timed out`), run); - } catch (cause) { - throw translateEngineError(cause, ` while ${label}`); - } - } - - /** - * Disposes the engine at worker end of life, bounded by the cleanup budget. - * Every engine runs over the protocol, whose client forwards `dispose` only - * once `init` was called: a worker that never initialized opened no - * connection, so the engine has nothing of it to release. - */ - async dispose(options: Pick = {}): Promise { - const disposeEngine = this.target.engine?.dispose?.bind(this.target.engine); - if (disposeEngine === undefined) return; - try { - await this.lifecycle( - 'disposing the engine', - this.config.cleanupTimeout, - 'CLEANUP_TIMEOUT', - NEVER_ABORTS, - (signal) => disposeEngine({ signal, timeoutMs: this.config.cleanupTimeout, ...options }), - ); - } catch (cause) { - this.runErrors.push({ error: serializeError(classifyError(cause), { phase: 'cleanup' }) }); - } - } /** * Runs one file-target unit: runnable pairs of a single file, in declaration @@ -624,221 +460,6 @@ export class TargetExecutor implements SerialHost { this.emitUnitResult(pair, finalStatus, attempts, hookFailure); } - // --- attempt core --- - - /** Starts one attempt on the engine, restores the planned session, and starts the planned video. */ - async launchSession( - plan: SessionPlan, - attemptId: string, - artifactsDir: string, - signal: AbortSignal, - ): Promise { - // The engine booted in init() once per worker; the adapter is per-attempt - // so refs never cross attempts. - await this.initEngineOnce(); - const engine = this.target.engine; - // Session restore rides the engine's neutral state capability. Checked - // before any per-attempt isolation opens, so a misconfigured session never - // orphans a started attempt. - const sessionName = plan.session; - if (sessionName !== undefined && engine?.state === undefined) { - throw new ConfigurationError( - 'UNSUPPORTED_CAPABILITY', - `target "${this.target.name}" has no engine state capability for session restore`, - ); - } - const { video } = plan.recordings; - const session = createEngineSession({ engine, targetName: this.target.name, traced: plan.recordings.trace !== undefined }); - const launch = (label: string, run: (launchSignal: AbortSignal) => Promise) => - this.lifecycle(label, this.config.launchTimeout, 'LAUNCH_TIMEOUT', signal, run); - const launchOp = (launchSignal: AbortSignal) => - this.op(attemptId, this.config.launchTimeout, launchSignal); - try { - const startAttempt = engine?.startAttempt?.bind(engine); - if (startAttempt !== undefined) { - await this.debug.time('session.launch', () => - launch(`starting an attempt on engine ${engine?.name ?? 'none'}`, (launchSignal) => - startAttempt({ - attemptId, - artifactsDir, - signal: launchSignal, - resolveSecret: (secret, options) => this.resolveEngineSecret(session, secret, options), - appLog: (entry) => session.appLog.push(entry), - navigation: (line) => session.appLog.navigated(line), - ...(session.screens.traced ? { screen: (snapshot: EngineSnapshot) => session.screens.push(snapshot) } : {}), - environment: (facts) => session.environment.push(facts), - }), - ), - ); - } - if (sessionName !== undefined) { - const saved = await this.options.sessionStore.load(sessionName, this.sessionIdentity); - // The values the saving session learned come back with its state, so - // a stored value the app echoes is redacted here too. - adoptSecrets(sessionLedger(session, this.config.allSecrets), saved.secrets); - await launch('restoring the session', (launchSignal) => - session.restoreState!(saved.state, launchOp(launchSignal)), - ); - } - // A video the attempt asks for starts here, when the engine has it: an - // engine without it was either refused before the run (a mode a target - // or test set) or skipped with a notice (a run-wide mode). A required - // video's failure fails the launch; a default one's is swallowed. - if (video !== undefined && session.artifacts.startVideo !== undefined) { - const recordings = sessionRecordings(session, this.target.name); - const starting = launch('starting the video', (launchSignal) => recordings.startAttemptVideo(launchOp(launchSignal))); - if (video.policy === 'required') await starting; - else await starting.catch(() => undefined); - } - } catch (cause) { - // The attempt's isolation is open, or a timed-out startAttempt may still - // open it: end it within the cleanup budget, or the retry opens a second - // one. An engine without startAttempt makes this a no-op. - await this.endAttempt(session, attemptId).catch(() => undefined); - throw cause; - } - return session; - } - - /** - * The plaintext of a secret the target's engine declared, for an option it - * hands the app (basic-auth credentials). Registered for the session's and - * the process's redaction by `resolveSecretValue`, with every value the - * engine derives from it (the base64 credential an `Authorization` header - * carries) under the same name, so its text and text downloads are - * redacted of them. Its pixels stay as they are, since nothing was typed. - */ - private async resolveEngineSecret(session: TargetSession, secret: Secret, options?: ResolveSecretOptions): Promise { - const engine = this.target.engine; - if (!isSecret(secret) || !(engine?.secrets ?? []).some((declared) => declared.name === secret.name)) { - const name = isSecret(secret) ? `"${secret.name}"` : 'a value that is not a secrets.get() handle'; - throw new ConfigurationError( - 'SECRET_UNAVAILABLE', - `engine ${engine?.name ?? 'none'} asked for secret ${name}, which it did not declare in its secrets`, - ); - } - const ledger = sessionLedger(session, this.config.allSecrets); - const plaintext = await resolveSecretValue(secret, this.config.allSecrets, ledger); - registerDerivedSecrets(secret.name, (await options?.derived?.(plaintext)) ?? [], ledger); - return plaintext; - } - - /** - * Collects a failure the engine holds from a path no step awaited, within - * the cleanup budget; the interrupt is the one thing that cuts it short. - * The error is the test's, thrown as it was so `classifyError` keeps its - * code: `lifecycle` would name every non-engine error an engine failure. - */ - settleAttempt(session: TargetSession, attemptId: string): Promise { - return withScopedBudget( - this.config.cleanupTimeout, - this.interruptSignal, - () => new InfrastructureError('CLEANUP_TIMEOUT', 'settling the attempt timed out'), - (signal) => session.settle(this.op(attemptId, this.config.cleanupTimeout, signal)), - ); - } - - /** Ends one attempt's isolation within the cleanup budget. */ - private endAttempt(session: TargetSession, attemptId: string): Promise { - return this.lifecycle( - 'ending the attempt', - this.config.cleanupTimeout, - 'CLEANUP_TIMEOUT', - NEVER_ABORTS, - (signal) => session.close(this.op(attemptId, this.config.cleanupTimeout, signal)), - ); - } - - /** - * Finalizes the video, then ends the attempt with a fresh cleanup - * budget. The caller has classified the attempt by now: `record.status` - * decides whether an `on-failure` recording is kept. - */ - async closeSession( - session: TargetSession, - close: SessionClose, - record: ClosingRecord, - artifactSink: ArtifactSink, - secondaryErrors: SerializedError[], - ): Promise { - const { attemptId, video } = close; - await this.stopRunningRecording(session, attemptId, record, artifactSink, secondaryErrors); - if (session.artifacts.stopVideo !== undefined && video !== undefined) { - const recordings = sessionRecordings(session, this.target.name); - await this.stopVideo(video, attemptId, record, secondaryErrors, async (operation) => { - const segments = await recordings.stopAttemptVideo(operation); - // Recorded so a failure could be looked at; a pass has nothing to show. - if (!keeps(video.keep, record.status)) { - await Promise.all( - segments.flatMap((segment) => ('path' in segment ? [rm(path.join(artifactSink.dir, segment.path), { force: true })] : [])), - ); - return; - } - registerVideos(artifactSink, segments); - }); - } - try { - await this.debug.time('session.close', () => this.endAttempt(session, attemptId)); - } catch (cause) { - record.cleanup = 'failed'; - secondaryErrors.push(serializeError(classifyError(cause), { phase: 'cleanup' })); - } - } - - /** - * Stops a recording the test started with `app.startRecording` and left - * running, within the cleanup budget, and keeps it as the attempt's - * artifact. One that cannot be finalized is a cleanup failure. - */ - private async stopRunningRecording( - session: TargetSession, - attemptId: string, - record: ClosingRecord, - artifactSink: ArtifactSink, - secondaryErrors: SerializedError[], - ): Promise { - const recordings = sessionRecordings(session, this.target.name); - if (!recordings.spanRunning) return; - try { - await this.lifecycle( - 'stopping the recording', - this.config.cleanupTimeout, - 'CLEANUP_TIMEOUT', - NEVER_ABORTS, - (signal) => recordings.stopSpan(this.op(attemptId, this.config.cleanupTimeout, signal), artifactSink), - ); - } catch (cause) { - record.cleanup = 'failed'; - secondaryErrors.push(serializeError(classifyError(cause), { phase: 'cleanup' })); - } - } - - /** - * Stops the video the attempt started, within the cleanup budget. A - * required video that cannot be finalized is a cleanup failure the report - * must show; a best-effort one fails quietly. - */ - private async stopVideo( - recording: VideoRecording, - attemptId: string, - record: ClosingRecord, - secondaryErrors: SerializedError[], - stop: (operation: OperationContext) => Promise, - ): Promise { - try { - await this.lifecycle( - 'stopping the video', - this.config.cleanupTimeout, - 'CLEANUP_TIMEOUT', - NEVER_ABORTS, - (signal) => stop(this.op(attemptId, this.config.cleanupTimeout, signal)), - ); - } catch (cause) { - if (recording.policy !== 'required') return; - record.cleanup = 'failed'; - secondaryErrors.push(serializeError(classifyError(cause), { phase: 'cleanup' })); - } - } /** Runs one attempt to completion (SerialHost). */ /** @@ -1405,42 +1026,3 @@ export class TargetExecutor implements SerialHost { return record; } } - -/** - * The status of a failed attempt. An interrupt outranks everything except a - * timeout that had already fired: the attempt then timed out, whatever - * arrived after. - */ -export function classifyAttemptStatus( - failure: { readonly code: string }, - timedOut: boolean, - interrupted: boolean, -): 'failed' | 'timed-out' | 'interrupted' { - if (interrupted && !timedOut) return 'interrupted'; - if (timedOut || failure.code === 'TEST_TIMEOUT') { - return 'timed-out'; - } - return 'failed'; -} - -/** - * Registers an attempt's kept recordings, in order: a file under its path, a - * link by URL. An engine's own link is held to what a provider's is, an - * http(s) URL; one that is not fails the cleanup once every other recording - * is registered, so it costs no other video. - */ -function registerVideos(sink: ArtifactSink, segments: readonly VideoSegment[]): void { - const rejected: string[] = []; - for (const segment of segments) { - if ('path' in segment) { - sink.register('video', segment.path, { startedAt: segment.startedAt }); - continue; - } - const url = hostedVideoUrl(segment.url); - if (url === undefined) rejected.push(JSON.stringify(segment.url)); - else sink.link(url, { mediaType: segment.mediaType, startedAt: segment.startedAt }); - } - if (rejected.length > 0) { - throw new EngineError('ENGINE_FAILURE', `the engine returned a video link that is not an http(s) URL: ${rejected.join(', ')}`, { retryable: false }); - } -} diff --git a/packages/e2e/src/run/host-trace.ts b/packages/e2e/src/run/host-trace.ts index 5cb219fc6a..17438f04e0 100644 --- a/packages/e2e/src/run/host-trace.ts +++ b/packages/e2e/src/run/host-trace.ts @@ -3,7 +3,7 @@ import { readFileSync } from 'node:fs'; import path from 'node:path'; import type { ResolvedConfig, ResolvedTarget } from '../config/resolve.ts'; -import type { Report } from '../types.ts'; +import type { Report } from '../run-types.ts'; import { buildReport } from '../report/build.ts'; import { renderTracePage } from '../report/failure-text.ts'; import { outcome } from '../report/outcome.ts'; diff --git a/packages/e2e/src/run/open-session.ts b/packages/e2e/src/run/open-session.ts index 99346ea8c8..44dbca8fd9 100644 --- a/packages/e2e/src/run/open-session.ts +++ b/packages/e2e/src/run/open-session.ts @@ -1,13 +1,15 @@ /** Hosted test runners keep a worker alive while opening isolated per-test attempts. */ import { discoverConfig, loadConfigModule, missingConfigError } from '../config/load.ts'; -import { resolveConfig, type ResolvedConfig } from '../config/resolve.ts'; +import { type ResolvedConfig } from '../config/resolve.ts'; +import { resolveConfig } from '../config/resolve-run.ts'; import { classifyError, ConfigurationError, E2EError, serializeError, type SerializedError } from '../internal/errors.ts'; -import type { E2EConfig, Report, TestFixtures } from '../types.ts'; +import type { TestFixtures } from '../types.ts'; +import type { E2EConfig, Report } from '../run-types.ts'; import { allocateTargetPorts } from './app-ports.ts'; import { processSecrets, registerStaticSecrets } from './secrecy.ts'; import { selectTarget } from './select-target.ts'; -import { openStandaloneHost } from './standalone.ts'; +import { openStandaloneHost } from './standalone-kinds.ts'; import type { StepProgress, StepRecord } from './steps.ts'; import { reserveHostedRun } from './public-ownership.ts'; @@ -110,7 +112,7 @@ async function openHostWorker(options: OpenHostOptions) runId: attempt.runId, attemptId: attempt.attemptId, target: target.name, fixtures: attempt.fixtures as TestFixtures & Fixtures, artifactsDir: attempt.artifactsDir, tracePath: attempt.tracePath, - steps: () => attempt.steps.all(), report: attempt.report, close: attempt.close, + steps: () => attempt.steps.all(), report: () => attempt.extra.report && structuredClone(attempt.extra.report), close: attempt.close, [Symbol.asyncDispose]: () => dispose(attempt.close, 'session'), }; }), diff --git a/packages/e2e/src/run/provision.ts b/packages/e2e/src/run/provision.ts index 485484fd40..62b708281e 100644 --- a/packages/e2e/src/run/provision.ts +++ b/packages/e2e/src/run/provision.ts @@ -6,19 +6,70 @@ * provision differently. */ -import { pairRecording, type TestTargetPair } from '../collect/select.ts'; import { engineAppInfo } from '../config/app.ts'; import type { ResolvedTarget } from '../config/resolve.ts'; import type { EnginePrepareResult } from '../engine/index.ts'; +import { ENGINE_SPI_VERSION, type EngineSpiVersion } from '../engine/contract.ts'; import type { DebugTrace } from '../internal/debug.ts'; import { ConfigurationError, InfrastructureError, translateProvisioningError } from '../internal/errors.ts'; import { Deadline, NEVER_ABORTS, withScopedBudget } from '../internal/time.ts'; -import { describeTarget, type TargetProvenance } from '../report/build.ts'; import { isRetryMode, recordsOnSomeAttempt, type RecordingKind, type ResolvedRecording } from '../internal/recording-modes.ts'; +import type { RecordingMode } from '../types.ts'; import { declaredCommands } from './declared-processes.ts'; import { ManagedProcess, type AppProcesses, type ManagedProcessHooks } from './managed-process.ts'; import { UNSHARED, type ProcessPool } from './process-pool.ts'; +/** What grading reads of one test-target pair a run selected. */ +export interface GradedPair { + readonly disposition: string; + readonly target: ResolvedTarget; + readonly test: { readonly titlePath: readonly string[]; readonly file: string }; + readonly options: { readonly trace: RecordingMode | undefined; readonly video: RecordingMode | undefined; readonly retries: number }; +} + +export interface TargetProvenance { + engine: { name: string; version: string; spiVersion: EngineSpiVersion }; + capabilities: string[]; + /** What the engine can capture: screenshots (a failure's among them), a video. */ + artifactCapabilities: ('screenshot' | 'video')[]; + stateCapability: boolean; +} + +/** + * What a target's engine declaration says about it: the one description the + * runner grades against and the report records. A target without an engine is + * agent-tools-only and honestly reports no capabilities. + */ +export function describeTarget(target: ResolvedTarget): TargetProvenance { + const engine = target.engine; + const artifactCapabilities: TargetProvenance['artifactCapabilities'] = []; + if (engine?.artifacts !== undefined) { + artifactCapabilities.push('screenshot'); + if (engine.artifacts.startVideo !== undefined) artifactCapabilities.push('video'); + } + return { + engine: { + name: engine?.name ?? 'none', + version: engine?.version ?? 'unversioned', + spiVersion: engine?.spiVersion ?? ENGINE_SPI_VERSION, + }, + capabilities: [...(engine?.capabilities ?? [])].toSorted(), + artifactCapabilities, + stateCapability: engine?.state !== undefined, + }; +} + +/** + * Which attempts of a pair record `kind`, and where the mode came from: the + * test's own, else its target's (which the flag and the config already + * decided). One resolver for both recordings, so `trace` and `video` can + * never disagree about precedence. + */ +export function pairRecording(pair: Pick, kind: RecordingKind): ResolvedRecording { + const own = pair.options[kind]; + return own === undefined ? pair.target[kind] : { mode: own, source: 'test' }; +} + /** A target graded against its engine declaration, before any test starts. */ export interface EngineGrade { readonly provenance: TargetProvenance; @@ -41,10 +92,10 @@ export interface EngineGrade { * engine can record and is skipped with a notice elsewhere, and a default * one is skipped quietly. */ -export function validateEngine(target: ResolvedTarget, pairs?: readonly TestTargetPair[]): EngineGrade { +export function validateEngine(target: ResolvedTarget, pairs?: readonly GradedPair[]): EngineGrade { const provenance = describeTarget(target); if (provenance.artifactCapabilities.includes('video')) return { provenance, videoSkipped: false }; - const asked: { readonly recording: ResolvedRecording; readonly pair?: TestTargetPair }[] = + const asked: { readonly recording: ResolvedRecording; readonly pair?: GradedPair }[] = pairs === undefined ? recordsOnSomeAttempt(target.video.mode, 0) ? [{ recording: target.video }] : [] : pairs.flatMap((pair) => { @@ -72,7 +123,7 @@ export function validateEngine(target: ResolvedTarget, pairs?: readonly TestTarg * to pass the flag instead of setting retries. */ export function recordingNotices( - perTarget: readonly { readonly target: ResolvedTarget; readonly pairs: readonly TestTargetPair[] }[], + perTarget: readonly { readonly target: ResolvedTarget; readonly pairs: readonly GradedPair[] }[], grades: ReadonlyMap, options: { readonly explore?: boolean } = {}, ): string[] { @@ -97,7 +148,7 @@ export function recordingNotices( */ function retryOnlyNotice( kind: RecordingKind, - perTarget: readonly { readonly target: ResolvedTarget; readonly pairs: readonly TestTargetPair[] }[], + perTarget: readonly { readonly target: ResolvedTarget; readonly pairs: readonly GradedPair[] }[], captures: (target: ResolvedTarget) => boolean, explore: boolean, ): string | undefined { diff --git a/packages/e2e/src/run/public-run.ts b/packages/e2e/src/run/public-run.ts index e8e4543099..b18ceb4f24 100644 --- a/packages/e2e/src/run/public-run.ts +++ b/packages/e2e/src/run/public-run.ts @@ -4,7 +4,7 @@ import { packageVersion } from '../internal/package-version.ts'; import { runsFromCheckout } from '../telemetry/checkout.ts'; import { runCompletedEvent } from '../telemetry/events.ts'; import { Telemetry } from '../telemetry/telemetry.ts'; -import type { Report } from '../types.ts'; +import type { Report } from '../run-types.ts'; import type { RunEventSink, RunExitCode, RunStatus } from './events.ts'; import { withProcessOutput, type RunnerOutput } from './process-output.ts'; import { reserveSuiteRun } from './public-ownership.ts'; diff --git a/packages/e2e/src/run/records.ts b/packages/e2e/src/run/records.ts index d710fa035b..d8d364959a 100644 --- a/packages/e2e/src/run/records.ts +++ b/packages/e2e/src/run/records.ts @@ -3,12 +3,58 @@ import type { SerializedError } from '../internal/errors.ts'; import type { ExpectedFailureInfo } from '../internal/expected-failure.ts'; import { resultId } from '../internal/ids.ts'; -import type { TestIdentity } from '../collect/collect.ts'; -import type { SkipInfo } from '../collect/select.ts'; +import type { SourceLocation } from '../internal/source.ts'; import { keeps, type Keep } from '../internal/recording-modes.ts'; import type { ResolvedTarget } from '../config/resolve.ts'; import type { AppLogRecord, StepRecord } from './steps.ts'; +/** + * Everything needed to name and report a test, with no executable or realm + * state attached. Fully JSON-serializable, so it is what result records carry + * and what crosses the runner<->worker IPC channel. + */ +export interface TestIdentity { + readonly kind: 'test' | 'setup'; + readonly title: string; + readonly titlePath: readonly string[]; + readonly declarationIndex: number; + readonly sessions: readonly string[]; + /** The tags the test declares, outermost describe first, each once; what `--tag` selects on. */ + readonly tags: readonly string[]; + readonly source: SourceLocation | undefined; + /** Normalized project-root-relative file path with `/` separators. */ + readonly file: string; + readonly id: string; + readonly serialId: string | undefined; +} + +/** Why a pair did not run; only a `filtered` one can name the test filter that left it out. */ +export type SkipInfo = + | (SkipDetail & { + readonly cause: 'filtered'; + /** The name of the test filter that left the test out; absent when the selection did, as with `--tag`. */ + readonly filter?: string; + }) + | (SkipDetail & { + readonly cause: + | 'explicit' + | 'platform-unavailable' + | 'capability-unavailable' + | 'setup-failed' + | 'serial-predecessor-failed' + | 'hook-failed' + | 'infrastructure-unavailable' + | 'failure-limit' + | 'time-limit'; + readonly filter?: never; + }); + +/** What a skip says whatever its cause: why, and the id of a related test, such as the setup that failed. */ +interface SkipDetail { + readonly reason: string; + readonly relatedId?: string; +} + export type ArtifactProducer = { kind: 'step'; stepId: string } | { kind: 'attempt' }; export interface ArtifactRecord { diff --git a/packages/e2e/src/run/runner.ts b/packages/e2e/src/run/runner.ts index b4f22a7132..eb159da5cf 100644 --- a/packages/e2e/src/run/runner.ts +++ b/packages/e2e/src/run/runner.ts @@ -3,13 +3,8 @@ import { rm } from 'node:fs/promises'; import path from 'node:path'; import { discoverConfig, loadConfigModule, missingConfigError } from '../config/load.ts'; -import { - isCiMode, - resolveConfig, - type CliOverrides, - type ResolvedConfig, - type ResolvedTarget, -} from '../config/resolve.ts'; +import { isCiMode, type ResolvedTarget } from '../config/resolve.ts'; +import { resolveConfig, type ResolvedRunConfig, type RunCliOverrides } from '../config/resolve-run.ts'; import { collect, collectInMemory, collectSetups, type Collection } from '../collect/collect.ts'; import type { ModuleRegistration } from '../collect/registry.ts'; import { @@ -40,7 +35,8 @@ import { DebugTrace } from '../internal/debug.ts'; import { printableUrl } from '../internal/urls.ts'; import { resultId, timestamp, uuidv7 } from '../internal/ids.ts'; import type { ExploreProgress } from '../explore/progress.ts'; -import { buildReport, projectSource, type Report1Document, type ReportChanges, type ReportExplore, type ReportFilter, type TargetProvenance } from '../report/build.ts'; +import { buildReport, projectSource, type Report1Document, type ReportChanges, type ReportExplore, type ReportFilter } from '../report/build.ts'; +import type { TargetProvenance } from './provision.ts'; import { agentStepTable } from '../report/debug-steps.ts'; import { writeTracePages, type TracePages } from '../report/traces.ts'; import { STATELESS_REPORTERS } from '../report/builtin.ts'; @@ -66,7 +62,8 @@ import { loadTimings, writeTimings } from './timings.ts'; import { childProcessSpawner } from './worker/handle.ts'; import { setSecretRegistry } from '../secrets.ts'; import { budgetText, withAbort } from '../internal/time.ts'; -import type { BuiltinReporter, E2EConfig, FinishedRun, RecordingMode, Reporter, ReporterSummary } from '../types.ts'; +import type { RecordingMode } from '../types.ts'; +import type { BuiltinReporter, E2EConfig, FinishedRun, Reporter, ReporterSummary } from '../run-types.ts'; import { modelLabel } from '../config/agent.ts'; import { boundedInt, describeValue, positiveInt, timerMs } from '../config/validate.ts'; import { validateChangedSince } from './changes.ts'; @@ -264,7 +261,7 @@ export interface InMemoryTests { * explorer. Setup tests they need run as the config's own agents, with the * config's cache and retries; the in-memory tests run with the trace cache off. */ - readonly agents?: ResolvedConfig['agents'] | undefined; + readonly agents?: ResolvedRunConfig['agents'] | undefined; /** * The exploration behind `e2e explore`: its progress becomes `explore` run * events, and its record is read once the run is over, as `run.explore`. @@ -394,7 +391,7 @@ export async function listSelection(options: ListOptions, notice: (message: stri const cwd = options.cwd ?? process.cwd(); const env = options.env ?? process.env; const runOptions = toSelectionRunOptions(options); - const loadConfig = async (): Promise => { + const loadConfig = async (): Promise => { const loaded = await loadRunConfig(runOptions, cwd, env, runOptions.output === undefined ? {} : { output: runOptions.output }); registerStaticSecrets(loaded.allSecrets); return loaded; @@ -425,7 +422,7 @@ export interface RunOutcome { aiTracePath: string | undefined; results: readonly ResultRecord[]; /** The config the run resolved; undefined when it failed to load. */ - config: ResolvedConfig | undefined; + config: ResolvedRunConfig | undefined; } /** @@ -468,7 +465,7 @@ export async function run(options: RunOptions = {}): Promise { // Hoisted: workers re-resolve the same config file and need these overrides, // or a flag would apply in the runner and be dropped in every worker. - const cli: CliOverrides = {}; + const cli: RunCliOverrides = {}; if (options.retries !== undefined) cli.retries = options.retries; if (options.workers !== undefined) cli.workers = options.workers; if (options.actionTimeout !== undefined) cli.actionTimeout = options.actionTimeout; @@ -500,7 +497,7 @@ export async function run(options: RunOptions = {}): Promise { const keepStdoutFor = (reporters: readonly BuiltinReporter[] | undefined): void => { if (reporters?.includes('json') === true) processOutput?.keepStdoutForReport(); }; - const loadConfig = async (): Promise => { + const loadConfig = async (): Promise => { const config = await loadRunConfig(options, cwd, env, cli, graph); registerStaticSecrets(config.allSecrets); keepStdoutFor(config.reporters); @@ -651,7 +648,7 @@ export async function run(options: RunOptions = {}): Promise { * returned in-memory document becomes the only complete record. The file * is not retried — the destination just failed. */ - const writeCanonicalReport = async (config: ResolvedConfig, document: Report1Document): Promise => { + const writeCanonicalReport = async (config: ResolvedRunConfig, document: Report1Document): Promise => { const target = outputLayout(config.output).report; try { await writeJsonReport(target, document); @@ -672,7 +669,7 @@ export async function run(options: RunOptions = {}): Promise { /** The timings file: `--timings` against the project root, else the one in the output directory. */ - const timingsPath = (config: ResolvedConfig): string => + const timingsPath = (config: ResolvedRunConfig): string => options.timings === undefined ? outputLayout(config.output).timings : path.resolve(config.projectRoot, options.timings); /** @@ -681,7 +678,7 @@ export async function run(options: RunOptions = {}): Promise { * error. Runs before the report is written, because an absent timings file * falls back to the previous report as the history to merge into. */ - const writeTimingsHistory = async (config: ResolvedConfig, document: Report1Document): Promise => { + const writeTimingsHistory = async (config: ResolvedRunConfig, document: Report1Document): Promise => { try { await writeTimings(document, { timingsPath: timingsPath(config), @@ -700,7 +697,7 @@ export async function run(options: RunOptions = {}): Promise { * error. The in-process recorder is drained here; child-process workers * already shipped theirs over the worker channel. */ - const writeAiTrace = async (config: ResolvedConfig): Promise => { + const writeAiTrace = async (config: ResolvedRunConfig): Promise => { if (aiTrace === undefined) return undefined; if (aiTraceRecorder !== undefined) { aiTrace.merge(aiTraceRecorder.drain({ all: true })); @@ -731,7 +728,7 @@ export async function run(options: RunOptions = {}): Promise { * page that could not be written is a line on stderr, never a run error * that would disagree with the report already on disk. */ - const writeTraces = async (config: ResolvedConfig, document: Report1Document): Promise => { + const writeTraces = async (config: ResolvedRunConfig, document: Report1Document): Promise => { try { return await writeTracePages(document, tracedResultIds(results, serialGroups), { projectRoot: config.projectRoot, @@ -1381,7 +1378,7 @@ function toListedPair(projectRoot: string, pair: TestTargetPair): ListedPair { * report is a collection-phase failure like any other selection error; the * report itself reaches the reporters as `FinishedRun.lastRun`. */ -async function selectionInputs(options: SelectionRunOptions, config: ResolvedConfig): Promise { +async function selectionInputs(options: SelectionRunOptions, config: ResolvedRunConfig): Promise { const lastRun = options.lastFailed === true ? await readLastRun(outputLayout(config.output).report) : undefined; const filters: SelectionFilters = { @@ -1430,9 +1427,9 @@ async function loadRunConfig( options: SelectionRunOptions, cwd: string, env: NodeJS.ProcessEnv, - cli: CliOverrides, + cli: RunCliOverrides, graph?: string, -): Promise { +): Promise { validateRunOptions(options); if (options.rawConfig !== undefined) { return resolveConfig(options.rawConfig, { projectRoot: cwd, env, cli }); @@ -1528,7 +1525,7 @@ function verdictExitCodes(results: readonly ResultRecord[], serialGroups: readon class ReporterAbandoned extends Error {} /** `FinishedRun.redact`: the config's fixed secret values; a config that never loaded declared none. */ -function redactStaticSecrets(secrets: ResolvedConfig['allSecrets'] | undefined): (text: string) => string { +function redactStaticSecrets(secrets: ResolvedRunConfig['allSecrets'] | undefined): (text: string) => string { if (secrets === undefined) return (text) => String(text); const { redactFragments } = staticSecretLedger(secrets); return (text) => redactFragments(String(text)); diff --git a/packages/e2e/src/run/scheduler.ts b/packages/e2e/src/run/scheduler.ts index 604f2b83e7..89ea630631 100644 --- a/packages/e2e/src/run/scheduler.ts +++ b/packages/e2e/src/run/scheduler.ts @@ -8,7 +8,8 @@ * execution. */ -import type { SkipInfo, TestTargetPair } from '../collect/select.ts'; +import type { TestTargetPair } from '../collect/select.ts'; +import type { SkipInfo } from './records.ts'; import type { ResolvedTarget } from '../config/resolve.ts'; import type { AiTraceSnapshot } from '../internal/ai-trace.ts'; import type { DebugSnapshot } from '../internal/debug.ts'; diff --git a/packages/e2e/src/run/serial.ts b/packages/e2e/src/run/serial.ts index e09696c626..fffe8050fe 100644 --- a/packages/e2e/src/run/serial.ts +++ b/packages/e2e/src/run/serial.ts @@ -10,11 +10,13 @@ import { import { canonicalDigest, timestamp, uuidv7 } from '../internal/ids.ts'; import type { CollectedTest } from '../collect/collect.ts'; import { groupTitles, type RegisteredTest } from '../collect/registry.ts'; -import { pairRecordings, type SkipInfo, type TestTargetPair } from '../collect/select.ts'; +import { pairRecordings, type TestTargetPair } from '../collect/select.ts'; +import type { SkipInfo } from './records.ts'; import type { ResolvedTarget } from '../config/resolve.ts'; import type { ArtifactStore } from '../types.ts'; import { attemptSegments, createAttemptArtifacts, resultSegment } from './artifacts.ts'; -import type { AttemptContext, ClosingRecord, SessionClose, SessionPlan } from './execute.ts'; +import type { AttemptContext } from './execute.ts'; +import type { ClosingRecord, SessionClose, SessionPlan } from './engine-sessions.ts'; import type { ArtifactSink } from './fixtures.ts'; import type { StepRecord } from './steps.ts'; import { findRegistered, type FileRef, type Realm, RealmManager } from './realm.ts'; diff --git a/packages/e2e/src/run/standalone-attempt.ts b/packages/e2e/src/run/standalone-attempt.ts index f15706334a..d7f97a92fb 100644 --- a/packages/e2e/src/run/standalone-attempt.ts +++ b/packages/e2e/src/run/standalone-attempt.ts @@ -1,62 +1,41 @@ -/** One attempt on a prepared standalone worker: reservation, launch, fixtures, deadline, and close. */ +/** + * One attempt on a prepared standalone worker: reservation, launch, fixtures, + * deadline, and close. What differs by kind (the runner's hosted and one-shot + * attempts) comes in through `AttemptPolicy`. + */ import path from 'node:path'; import type { ExecutorAttempt } from '../agent/executor.ts'; import type { AgentContext } from '../agent/invocation.ts'; import type { ResolvedConfig, ResolvedTarget } from '../config/resolve.ts'; import type { TargetSession } from '../engine/surface.ts'; -import { publishAttempt } from '../expect/attempt.ts'; import { PollScope } from '../expect/poll-scope.ts'; -import { SoftFailures } from '../expect/soft.ts'; import type { DebugTrace } from '../internal/debug.ts'; import { classifyError, ConfigurationError, InfrastructureError, serializeError, TestTimeoutError, type SerializedError } from '../internal/errors.ts'; -import { resultId, timestamp, uuidv7 } from '../internal/ids.ts'; +import { timestamp, uuidv7 } from '../internal/ids.ts'; import { Deadline, MAX_TIMER_MS, withTimeout } from '../internal/time.ts'; import { attemptVideo } from '../internal/recording-modes.ts'; -import type { Report, TestFixtures } from '../types.ts'; -import { createAttemptArtifacts, resultSegment } from './artifacts.ts'; +import type { TestFixtures } from '../types.ts'; +import { createAttemptArtifacts, type AttemptArtifacts } from './artifacts.ts'; import { AttemptBudget } from './budget.ts'; -import { classifyAttemptStatus, type ClosingRecord, type TargetExecutor } from './execute.ts'; -import { createFixtures } from './fixtures.ts'; -import { writeHostTrace } from './host-trace.ts'; +import { classifyAttemptStatus, type ClosingRecord, type EngineSessions } from './engine-sessions.ts'; +import { createFixtures, type AttemptEnvironment } from './fixtures.ts'; import type { AttemptRecord } from './records.ts'; -import { holdSecretRegistry } from '../secrets.ts'; -import { createAgentCacheContext } from '../cache/context.ts'; -import { storedRecordingsFor } from '../cache/rekeyed.ts'; -import { WrittenScreenshots } from './screenshots.ts'; -import { processSecrets, redactForSession, sessionLedger } from './secrecy.ts'; -import { targetIdentity } from './sessions.ts'; -import { createStandaloneTestRun, type StandaloneTest, type StandaloneTestRun } from './standalone-test.ts'; +import { redactForSession, sessionLedger } from './secrecy.ts'; import { StepRecorder, type StepProgress } from './steps.ts'; import { WorkerModels } from './worker-models.ts'; -/** - * Who drives the attempt. A hosted attempt is a test in another runner: it - * publishes itself for `expect.soft` and `expect.poll`, enforces its deadline, - * and writes a trace under `results/host/`. A one-shot attempt is `e2e mcp`'s - * live session, with artifacts under `results/mcp/`: no test body unless the - * session runs one (`StandaloneSessionOptions.test`), which publishes itself - * only while its code runs. - */ -export type AttemptKind = 'hosted' | 'one-shot'; - export interface StandaloneSessionOptions { readonly signal: AbortSignal; /** Operation deadline from launch, positive and finite. */ readonly timeoutMs: number; readonly agent?: string | undefined; readonly onProgress?: (progress: StepProgress) => void; - /** The test title of a hosted attempt; ignored by a one-shot attempt. */ + /** The test title of a hosted attempt; its policy decides whether the attempt reads it. */ readonly name?: string; - /** - * The collected test a one-shot attempt runs (`open_session {test}`): the - * attempt takes its identity, agent context, and replay cache, and exposes - * the run as `test`. - */ - readonly test?: StandaloneTest | undefined; } -export interface StandaloneAttempt { +export interface StandaloneAttempt { readonly runId: string; readonly attemptId: string; readonly fixtures: TestFixtures; @@ -66,19 +45,74 @@ export interface StandaloneAttempt { readonly budget: AttemptBudget; readonly artifactsDir: string; readonly tracePath: string; - /** The test the attempt runs, when its options named one. */ - readonly test: StandaloneTestRun | undefined; - report(): Report | undefined; + /** What the policy's `launched` built on the attempt. */ + readonly extra: Extra; close(): Promise; } +/** Where an attempt's results go, and the test id its records carry. */ +export interface AttemptIdentity { + readonly testId: string; + /** Directory segments under the results root. */ + readonly segments: readonly string[]; +} + +/** What a policy's `reserve` publishes for the attempt, withdrawn by `release`. */ +export interface AttemptPublication { + /** Failures the publication holds at close, such as soft assertions; called once. */ + close(): unknown; + release(): void; +} + +/** What a policy's `launched` reads of the open attempt. */ +export interface LaunchedAttempt { + readonly worker: AttemptWorker; + readonly attemptId: string; + readonly session: TargetSession; + readonly steps: StepRecorder; + readonly artifacts: AttemptArtifacts; + /** The inputs the attempt's own fixtures were built from, for fixtures of another budget. */ + readonly fixtureBase: Omit; + /** The worker's, the caller's, and the attempt's own cancellation together. */ + readonly signal: AbortSignal; + readonly redact: (text: string) => string; +} + +/** What a policy's `closed` reads once the attempt's session is closed. */ +export interface ClosedAttempt { + readonly worker: AttemptWorker; + /** What `launched` returned. */ + readonly extra: Extra; + readonly identity: AttemptIdentity; + readonly agent: string; + readonly options: StandaloneSessionOptions; + readonly tracePath: string; + readonly redact: (text: string) => string; + readonly record: Omit; +} + +/** + * What one kind of attempt adds to the shared lifecycle. Every hook but + * `identity` is optional, so a kind that publishes nothing, runs no test, and + * writes no trace needs none of them. + */ +export interface AttemptPolicy { + /** The attempt's test id and results directory. */ + identity(worker: AttemptWorker, attemptId: string, agent: string, redactedName: string): AttemptIdentity; + /** Runs synchronously at reservation, before any engine work, so nothing can publish over it. */ + reserve?(worker: AttemptWorker, reserved: { readonly attemptId: string; readonly budget: AttemptBudget; readonly polls: PollScope }): AttemptPublication; + /** Runs once the attempt's fixtures exist; a throw closes the attempt and fails the launch. */ + launched?(attempt: LaunchedAttempt): Extra; + /** Runs once the session is closed and its artifacts settled; a throw is a cleanup failure. */ + closed?(attempt: ClosedAttempt): Promise; +} + /** The prepared worker an attempt runs on. */ -export interface AttemptWorker { - readonly kind: AttemptKind; +export interface AttemptWorker { readonly config: ResolvedConfig; readonly target: ResolvedTarget; readonly runId: string; - readonly executor: TargetExecutor; + readonly executor: Sessions; readonly resultsDir: string; readonly debug: DebugTrace; /** Aborts when the host closes or its caller cancels it. */ @@ -94,10 +128,8 @@ export interface AttemptReservation { /** The worker's, the caller's, and the attempt's own cancellation together. */ readonly signal: AbortSignal; readonly budget: AttemptBudget; - readonly soft: SoftFailures; readonly polls: PollScope; - /** Withdraws the publication and the registry hold once cleanup has settled. */ - readonly release: () => void; + readonly publication: AttemptPublication | undefined; } /** What the host learns when an attempt's close settles. */ @@ -106,10 +138,15 @@ export interface AttemptClosed { } /** - * Validates the options and publishes a hosted attempt, synchronously, so - * `expect` finds it before launch and no second attempt can publish over it. + * Validates the options and runs the policy's `reserve`, synchronously, so + * what it publishes is there before launch and no second attempt can publish + * over it. */ -export function reserveAttempt(worker: AttemptWorker, options: StandaloneSessionOptions): AttemptReservation { +export function reserveAttempt( + worker: AttemptWorker, + policy: AttemptPolicy, + options: StandaloneSessionOptions, +): AttemptReservation { if (worker.signal.aborted || options.signal.aborted) throw new InfrastructureError('CANCELLED', 'opening the e2e session was cancelled'); if (!Number.isFinite(options.timeoutMs) || options.timeoutMs <= 0) { throw new ConfigurationError('INVALID_CONFIG', 'session timeout must be a positive, finite number of milliseconds'); @@ -118,15 +155,9 @@ export function reserveAttempt(worker: AttemptWorker, options: StandaloneSession const end = new AbortController(); const signal = AbortSignal.any([worker.signal, options.signal, end.signal]); const budget = new AttemptBudget(signal, new Deadline(options.timeoutMs)); - const soft = new SoftFailures(); const polls = new PollScope('the hosted session'); - if (worker.kind === 'one-shot') return { attemptId, options, end, signal, budget, soft, polls, release: () => undefined }; - const { config } = worker; - const publication = new AbortController(); - publishAttempt({ attemptId, owner: 'host', testKind: 'test', assertionTimeout: config.assertionTimeout, budget, soft, polls }, publication.signal); - // The live attempt's secrets resolve against its own config while other hosts hold theirs. - const releaseRegistry = holdSecretRegistry(config, { live: true }); - return { attemptId, options, end, signal, budget, soft, polls, release: () => { publication.abort(); releaseRegistry(); } }; + const publication = policy.reserve?.(worker, { attemptId, budget, polls }); + return { attemptId, options, end, signal, budget, polls, publication }; } /** @@ -135,19 +166,19 @@ export function reserveAttempt(worker: AttemptWorker, options: StandaloneSession * on; the worker stays usable. `onClosed` runs once when `close` settles, * before `close` returns, so the host can retire a worker whose cleanup failed. */ -export async function launchAttempt( - worker: AttemptWorker, +export async function launchAttempt( + worker: AttemptWorker, + policy: AttemptPolicy, reservation: AttemptReservation, onClosed: (closed: AttemptClosed) => Promise, -): Promise { +): Promise> { const { config, target, executor, runId } = worker; - const { attemptId, options, budget, soft, polls, end } = reservation; + const { attemptId, options, budget, polls, end } = reservation; const attemptSignal = reservation.signal; let session: TargetSession | undefined; const redact = (text: string): string => redactForSession(session, config.allSecrets, text); const agent = options.agent ?? config.agentNames[0] ?? ''; - const test = worker.kind === 'one-shot' ? options.test : undefined; - const identity = attemptIdentity(worker, attemptId, agent, redact(options.name ?? target.name), test?.pair.test.id); + const identity = policy.identity(worker, attemptId, agent, redact(options.name ?? target.name)); const steps = new StepRecorder(attemptId, { maxEventsPerStep: config.limits.maxEventsPerStep, projectRoot: config.projectRoot, redact, signal: attemptSignal, ...(options.onProgress === undefined ? {} : { onProgress: options.onProgress }), @@ -188,31 +219,12 @@ export async function launchAttempt( models: new WorkerModels((error) => recordFailure(error, 'body')), debug: worker.debug, }; let acquired: ReturnType; - let testRun: StandaloneTestRun | undefined; + let extra: Extra; try { acquired = createFixtures({ ...base, budget, agentContext: undefined }); - if (test !== undefined) { - // The test's fixtures have a budget of their own, so a body its timeout - // cut short cannot drive the session the agent uses afterwards. Its - // agent steps replay and record as on a run's first attempt, and its - // screenshots count as written by a run of their own: a baseline one - // session wrote is compared against in the next, as the next run would. - // Its agent steps read the test's title, as they do in a run. - const testBudget = new AttemptBudget(attemptSignal, new Deadline(test.pair.options.timeout)); - const cache = createAgentCacheContext({ - cache: config.cache, projectRoot: config.projectRoot, projectId: config.projectId, testId: test.pair.test.id, - target: targetIdentity(target), attemptIndex: 0, recordings: storedRecordingsFor(config.cache), - }); - const { fixtures } = createFixtures({ - ...base, attempt: { ...attempt, titlePath: test.pair.test.titlePath }, budget: testBudget, agentContext: test.pair.options.agentContext, - ...(cache === undefined ? {} : { cache }), - test: { file: test.pair.test.file, titlePath: test.pair.test.titlePath, writtenScreenshots: new WrittenScreenshots(artifacts.dir) }, - }); - testRun = createStandaloneTestRun({ - config, target, executor, runId, attemptId, session: opened, steps, budget: testBudget, fixtures, - artifacts: artifacts.sink, cache, signal: attemptSignal, redact, - }, test); - } + extra = policy.launched === undefined + ? undefined as Extra + : policy.launched({ worker, attemptId, session: opened, steps, artifacts, fixtureBase: base, signal: attemptSignal, redact }); } catch (cause) { end.abort(); await closeOpened({ status: 'failed', cleanup: 'complete' }); @@ -239,7 +251,6 @@ export async function launchAttempt( const startedMs = Date.now(); const tracePath = path.join(artifacts.dir, 'trace.md'); let ending: Promise | undefined; - let report: Report | undefined; const close = (): Promise => { ending ??= (async () => { clearTimeout(expiry); @@ -248,7 +259,7 @@ export async function launchAttempt( const abandonedPoll = polls.close(); const abandonedStep = steps.abandonRunning(); end.abort(); - for (const error of [abandonedStep, abandonedPoll, soft.close()]) if (error !== undefined) recordFailure(error, 'body'); + for (const error of [abandonedStep, abandonedPoll, reservation.publication?.close()]) if (error !== undefined) recordFailure(error, 'body'); try { await withTimeout(steps.settleAbandoned(), config.cleanupTimeout, () => new InfrastructureError('CLEANUP_TIMEOUT', 'abandoned session steps did not settle')); } catch (cause) { recordFailure(cause, 'cleanup'); } @@ -261,15 +272,13 @@ export async function launchAttempt( await closeOpened(record); } catch (cause) { recordFailure(cause, 'cleanup'); record.cleanup = 'failed'; } try { await artifacts.settle(); } catch (cause) { recordFailure(cause, 'cleanup'); } - if (worker.kind === 'hosted') { + if (policy.closed !== undefined) { try { const { error, status } = verdict(); - report = await writeHostTrace({ - config, target, runId, testId: identity.testId, agent, tracePath, redact, - title: processSecrets.redact(redact(options.name ?? target.name)), - resultsDir: worker.resultsDir, segments: identity.segments, - attempt: { - id: attemptId, index: 0, status, startedAt, durationMs: Date.now() - startedMs, + await policy.closed({ + worker, extra, identity, agent, options, tracePath, redact, + record: { + id: attemptId, status, startedAt, durationMs: Date.now() - startedMs, steps: [...steps.all()], appLog: [...steps.appLog()], artifacts: artifacts.records, ...(error === undefined ? {} : { error }), secondaryErrors: attemptErrors.slice(1), cleanup: record.cleanup, }, @@ -279,7 +288,7 @@ export async function launchAttempt( try { await onClosed({ cleanup: record.cleanup }); } finally { - reservation.release(); + reservation.publication?.release(); } return attemptErrors; })(); @@ -287,30 +296,6 @@ export async function launchAttempt( }; return { runId, attemptId, fixtures: acquired.fixtures, agentRuntime: acquired.agentRuntime, session: opened, steps, budget, - artifactsDir: artifacts.dir, tracePath, test: testRun, close, - report: () => report && structuredClone(report), + artifactsDir: artifacts.dir, tracePath, extra, close, }; } - -interface AttemptIdentity { - readonly testId: string; - /** Directory segments under the results root. */ - readonly segments: readonly string[]; -} - -/** - * A hosted attempt's directory is `host/--<digest>`: the digest - * covers the unique attempt id, so repeated names never share a directory, - * and the title slug is dropped while a provider's values are unknown, since - * no caller text can then be vouched for as a path. A one-shot attempt keeps - * `mcp/<target>-<attemptId>`, and the id of the test it runs, if any. - */ -function attemptIdentity(worker: AttemptWorker, attemptId: string, agent: string, redactedName: string, runsTest: string | undefined): AttemptIdentity { - const { kind, target, config } = worker; - if (kind === 'one-shot') return { testId: runsTest ?? `session:${target.name}`, segments: ['mcp', `${target.name}-${attemptId}`] }; - const testId = `host:${attemptId}`; - const providerSecrets = [...config.allSecrets.values()].some((secret) => typeof secret.value === 'function'); - const title = providerSecrets ? 'session' : processSecrets.redact(redactedName); - const segment = resultSegment({ id: resultId(testId, target.name, agent, 0), file: 'host', titlePath: [title] }); - return { testId, segments: ['host', segment] }; -} diff --git a/packages/e2e/src/run/standalone-kinds.ts b/packages/e2e/src/run/standalone-kinds.ts new file mode 100644 index 0000000000..fe0d9b9cc6 --- /dev/null +++ b/packages/e2e/src/run/standalone-kinds.ts @@ -0,0 +1,139 @@ +/** + * The runner's two kinds of standalone worker. A hosted attempt is a test in + * another runner (`openHost` on `e2e/runner`): it publishes itself for + * `expect.soft` and `expect.poll`, holds the live secret registry, and writes + * a trace under `results/host/`. A one-shot attempt is `e2e mcp`'s live + * session, with artifacts under `results/mcp/`, which may run one collected + * test (`open_session {test}`). + */ + +import { createAgentCacheContext } from '../cache/context.ts'; +import { storedRecordingsFor } from '../cache/rekeyed.ts'; +import { publishAttempt } from '../expect/attempt.ts'; +import { SoftFailures } from '../expect/soft.ts'; +import type { SerializedError } from '../internal/errors.ts'; +import { resultId } from '../internal/ids.ts'; +import { Deadline } from '../internal/time.ts'; +import { holdSecretRegistry } from '../secrets.ts'; +import type { Report } from '../run-types.ts'; +import { resultSegment } from './artifacts.ts'; +import { AttemptBudget } from './budget.ts'; +import { TargetExecutor } from './execute.ts'; +import { createFixtures } from './fixtures.ts'; +import { writeHostTrace } from './host-trace.ts'; +import { WrittenScreenshots } from './screenshots.ts'; +import { processSecrets } from './secrecy.ts'; +import { targetIdentity } from './sessions.ts'; +import { openWorker, type StandaloneHost, type StandaloneHostOptions, type WorkerPolicy } from './standalone.ts'; +import type { StandaloneAttempt, StandaloneSessionOptions } from './standalone-attempt.ts'; +import { createStandaloneTestRun, type StandaloneTest, type StandaloneTestRun } from './standalone-test.ts'; + +/** What a hosted attempt keeps beside the shared ones: the report its trace was written from. */ +export interface HostedReport { + report: Report | undefined; +} + +/** The runner's sessions on a standalone worker: a `TargetExecutor` that runs no units. */ +const runnerSessions: WorkerPolicy<TargetExecutor, never>['sessions'] = (options) => new TargetExecutor({ ...options, isolated: false }); + +/** Prepares, starts app processes, and initializes one worker for a hosted test runner. */ +export function openStandaloneHost(options: StandaloneHostOptions): Promise<StandaloneHost<HostedReport>> { + return openWorker<TargetExecutor, HostedReport>(options, { + claimsEngine: true, + sessions: runnerSessions, + identity: ({ target, config }, attemptId, agent, redactedName) => { + // The directory is `host/<file>-<title>-<digest>`: the digest covers the + // unique attempt id, so repeated names never share a directory, and the + // title slug is dropped while a provider's values are unknown, since no + // caller text can then be vouched for as a path. + const testId = `host:${attemptId}`; + const providerSecrets = [...config.allSecrets.values()].some((secret) => typeof secret.value === 'function'); + const title = providerSecrets ? 'session' : processSecrets.redact(redactedName); + const segment = resultSegment({ id: resultId(testId, target.name, agent, 0), file: 'host', titlePath: [title] }); + return { testId, segments: ['host', segment] }; + }, + reserve: ({ config }, { attemptId, budget, polls }) => { + const soft = new SoftFailures(); + const publication = new AbortController(); + publishAttempt({ attemptId, owner: 'host', testKind: 'test', assertionTimeout: config.assertionTimeout, budget, soft, polls }, publication.signal); + // The live attempt's secrets resolve against its own config while other hosts hold theirs. + const releaseRegistry = holdSecretRegistry(config, { live: true }); + return { close: () => soft.close(), release: () => { publication.abort(); releaseRegistry(); } }; + }, + launched: () => ({ report: undefined }), + closed: async ({ worker, extra, identity, agent, options: opened, tracePath, redact, record }) => { + const { config, target, runId, resultsDir } = worker; + extra.report = await writeHostTrace({ + config, target, runId, testId: identity.testId, agent, tracePath, redact, + title: processSecrets.redact(redact(opened.name ?? target.name)), + resultsDir, segments: identity.segments, + attempt: { ...record, index: 0 }, + }); + }, + }); +} + +export interface StandaloneAttemptOptions extends StandaloneHostOptions, StandaloneSessionOptions { + /** + * The collected test the attempt runs (`open_session {test}`): the attempt + * takes its identity, agent context, and replay cache, and exposes the run + * as `test`. + */ + readonly test?: StandaloneTest | undefined; +} + +/** MCP's attempt, with the test it runs when its options named one. */ +export interface OneShotAttempt extends StandaloneAttempt<StandaloneTestRun | undefined> { + readonly test: StandaloneTestRun | undefined; +} + +/** + * MCP's one attempt on a worker of its own, closed together. MCP keeps its own + * engine ownership. The teardown of a test the attempt ran drives the app, so + * it runs before the host's close cancels the attempt's operations. + */ +export async function openStandaloneAttempt(options: StandaloneAttemptOptions): Promise<OneShotAttempt> { + const { test } = options; + const host = await openWorker<TargetExecutor, StandaloneTestRun | undefined>(options, { + claimsEngine: false, + sessions: runnerSessions, + identity: ({ target }, attemptId) => ({ testId: test?.pair.test.id ?? `session:${target.name}`, segments: ['mcp', `${target.name}-${attemptId}`] }), + launched: ({ worker, attemptId, session, steps, artifacts, fixtureBase, signal, redact }) => { + if (test === undefined) return undefined; + const { config, target, executor, runId } = worker; + // The test's fixtures have a budget of their own, so a body its timeout + // cut short cannot drive the session the agent uses afterwards. Its + // agent steps replay and record as on a run's first attempt, and its + // screenshots count as written by a run of their own: a baseline one + // session wrote is compared against in the next, as the next run would. + // Its agent steps read the test's title, as they do in a run. + const testBudget = new AttemptBudget(signal, new Deadline(test.pair.options.timeout)); + const cache = createAgentCacheContext({ + cache: config.cache, projectRoot: config.projectRoot, projectId: config.projectId, testId: test.pair.test.id, + target: targetIdentity(target), attemptIndex: 0, recordings: storedRecordingsFor(config.cache), + }); + const { fixtures } = createFixtures({ + ...fixtureBase, attempt: { ...fixtureBase.attempt, titlePath: test.pair.test.titlePath }, budget: testBudget, agentContext: test.pair.options.agentContext, + ...(cache === undefined ? {} : { cache }), + test: { file: test.pair.test.file, titlePath: test.pair.test.titlePath, writtenScreenshots: new WrittenScreenshots(artifacts.dir) }, + }); + return createStandaloneTestRun({ + config, target, executor, runId, attemptId, session, steps, budget: testBudget, fixtures, + artifacts: artifacts.sink, cache, signal, redact, + }, test); + }, + }); + try { + const attempt = await host.openAttempt(options); + const close = async (): Promise<readonly SerializedError[]> => { + const torn = (await attempt.extra?.teardown()) ?? []; + // A test's `beforeAll` failure is a run error too, and was its outcome when the session opened. + const closed = (await host.close()).filter((error) => attempt.extra === undefined || error.phase !== 'beforeAll'); + return [...torn, ...closed]; + }; + return { ...attempt, test: attempt.extra, close }; + } catch (cause) { + await host.close(); + throw cause; + } +} diff --git a/packages/e2e/src/run/standalone-test.ts b/packages/e2e/src/run/standalone-test.ts index 9dcd75bb54..7ed72ee8f1 100644 --- a/packages/e2e/src/run/standalone-test.ts +++ b/packages/e2e/src/run/standalone-test.ts @@ -11,7 +11,8 @@ import { isModelUnreachable } from '../agent/error.ts'; import { settleStagedTraces, type AgentCacheContext } from '../cache/context.ts'; -import { collect, type CollectedFile, type TestIdentity } from '../collect/collect.ts'; +import { collect, type CollectedFile } from '../collect/collect.ts'; +import type { TestIdentity } from './records.ts'; import { select, type TestTargetPair } from '../collect/select.ts'; import type { ResolvedConfig, ResolvedTarget } from '../config/resolve.ts'; import type { OperationContext, TargetSession } from '../engine/surface.ts'; @@ -27,7 +28,8 @@ import { settleExpectedFailure } from './expected-failure.ts'; import { withAbort, withTimeout } from '../internal/time.ts'; import type { SetupFn } from '../types.ts'; import type { AttemptBudget } from './budget.ts'; -import { classifyAttemptStatus, type TargetExecutor } from './execute.ts'; +import type { TargetExecutor } from './execute.ts'; +import { classifyAttemptStatus } from './engine-sessions.ts'; import { createExtendedFixtures, type ExtendedFixtures } from './extended-fixtures.ts'; import { captureFailureEvidence } from './failure-evidence.ts'; import type { ArtifactSink, AttemptFixtures } from './fixtures.ts'; diff --git a/packages/e2e/src/run/standalone.ts b/packages/e2e/src/run/standalone.ts index e4b2896493..bb5c96658a 100644 --- a/packages/e2e/src/run/standalone.ts +++ b/packages/e2e/src/run/standalone.ts @@ -1,4 +1,8 @@ -/** The worker a hosted test runner or the one-shot MCP path keeps alive between attempts. */ +/** + * The worker a hosted test runner or the one-shot MCP path keeps alive + * between attempts. What differs by kind is a `WorkerPolicy`; the runner's + * kinds are in `standalone-kinds.ts`. + */ import type { ResolvedConfig, ResolvedTarget } from '../config/resolve.ts'; import { DebugTrace } from '../internal/debug.ts'; @@ -6,7 +10,7 @@ import { classifyError, InfrastructureError, serializeError, type SerializedErro import { realmSlot } from '../internal/realm-slot.ts'; import { uuidv7 } from '../internal/ids.ts'; import { holdSecretRegistry } from '../secrets.ts'; -import { TargetExecutor } from './execute.ts'; +import type { EngineSessions, EngineSessionsOptions } from './engine-sessions.ts'; import type { AppProcesses } from './managed-process.ts'; import { outputLayout } from './output.ts'; import type { ProcessPool } from './process-pool.ts'; @@ -14,11 +18,9 @@ import { PreparedEngines, recordingNotices, startDeclaredProcesses, validateEngi import { processSecrets } from './secrecy.ts'; import { SessionStore } from './sessions.ts'; import { - launchAttempt, reserveAttempt, type AttemptKind, type AttemptReservation, type StandaloneAttempt, type StandaloneSessionOptions, + launchAttempt, reserveAttempt, type AttemptPolicy, type AttemptReservation, type StandaloneAttempt, type StandaloneSessionOptions, } from './standalone-attempt.ts'; -export type { StandaloneAttempt, StandaloneSessionOptions } from './standalone-attempt.ts'; - export interface StandaloneHostOptions { readonly config: ResolvedConfig; readonly target: ResolvedTarget; @@ -30,24 +32,30 @@ export interface StandaloneHostOptions { readonly debug?: DebugTrace; } -export interface StandaloneAttemptOptions extends StandaloneHostOptions, StandaloneSessionOptions {} - -export interface StandaloneHost { +export interface StandaloneHost<Extra = unknown> { readonly runId: string; /** One attempt at a time: the next may open once the previous close settled. */ - openAttempt(options: StandaloneSessionOptions): Promise<StandaloneAttempt>; + openAttempt(options: StandaloneSessionOptions): Promise<StandaloneAttempt<Extra>>; close(): Promise<readonly SerializedError[]>; } +/** What a kind of worker adds: who owns the engine, and what runs its sessions. */ +export interface WorkerPolicy<Sessions extends EngineSessions = EngineSessions, Extra = undefined> extends AttemptPolicy<Sessions, Extra> { + /** Whether the worker claims its engine instance against other hosts until a clean teardown. */ + readonly claimsEngine: boolean; + /** The worker's engine sessions, given where its results go. */ + sessions(options: EngineSessionsOptions & { readonly artifactsRoot: string }): Sessions; +} + /** * Where the worker is between attempts. `retired` follows an attempt whose * engine cleanup failed: its isolation is uncertain, so no later attempt may * reuse the worker. */ -type WorkerState = +type WorkerState<Extra> = | { readonly phase: 'idle' } - | { readonly phase: 'opening'; readonly pending: Promise<StandaloneAttempt> } - | { readonly phase: 'live'; readonly attempt: StandaloneAttempt } + | { readonly phase: 'opening'; readonly pending: Promise<StandaloneAttempt<Extra>> } + | { readonly phase: 'live'; readonly attempt: StandaloneAttempt<Extra> } | { readonly phase: 'retired' }; const engineOwners = realmSlot<WeakSet<object>>('e2e.standalone.engines.v1'); @@ -73,34 +81,11 @@ function claimEngine(engine: object | undefined): () => void { return () => owners().delete(engine); } -/** Prepares, starts app processes, and initializes one worker for a hosted test runner. */ -export function openStandaloneHost(options: StandaloneHostOptions): Promise<StandaloneHost> { - return openWorker(options, 'hosted'); -} - -/** - * MCP's one attempt on a worker of its own, closed together. MCP keeps its own - * engine ownership. The teardown of a test the attempt ran drives the app, so - * it runs before the host's close cancels the attempt's operations. - */ -export async function openStandaloneAttempt(options: StandaloneAttemptOptions): Promise<StandaloneAttempt> { - const host = await openWorker(options, 'one-shot'); - try { - const attempt = await host.openAttempt(options); - const close = async (): Promise<readonly SerializedError[]> => { - const torn = (await attempt.test?.teardown()) ?? []; - // A test's `beforeAll` failure is a run error too, and was its outcome when the session opened. - const closed = (await host.close()).filter((error) => attempt.test === undefined || error.phase !== 'beforeAll'); - return [...torn, ...closed]; - }; - return { ...attempt, close }; - } catch (cause) { - await host.close(); - throw cause; - } -} - -async function openWorker(options: StandaloneHostOptions, kind: AttemptKind): Promise<StandaloneHost> { +/** Prepares, starts app processes, and initializes one worker of the policy's kind. */ +export async function openWorker<Sessions extends EngineSessions, Extra>( + options: StandaloneHostOptions, + policy: WorkerPolicy<Sessions, Extra>, +): Promise<StandaloneHost<Extra>> { const { config, target } = options; const hostEnd = new AbortController(); const signal = AbortSignal.any([options.signal, hostEnd.signal]); @@ -112,7 +97,7 @@ async function openWorker(options: StandaloneHostOptions, kind: AttemptKind): Pr const grade = validateEngine(target); for (const message of recordingNotices([{ target, pairs: [] }], new Map([[target.name, grade]]))) notice(target.name, message); refuseOwnedEngine(target.engine); - const releaseEngine = kind === 'hosted' ? claimEngine(target.engine) : () => undefined; + const releaseEngine = policy.claimsEngine ? claimEngine(target.engine) : () => undefined; const releaseRegistry = holdSecretRegistry(config); const engines = new PreparedEngines(); const layout = outputLayout(config.output); @@ -122,7 +107,7 @@ async function openWorker(options: StandaloneHostOptions, kind: AttemptKind): Pr errors.push(serializeError(classifyError(cause), { phase: 'cleanup', projectRoot: config.projectRoot, redact: processSecrets.redact })); }; let processes: AppProcesses | undefined; - let executor: TargetExecutor | undefined; + let executor: Sessions | undefined; let tearingDown: Promise<void> | undefined; /** Releases every owner once, whichever of host close or a retiring attempt gets there first. */ const teardown = (): Promise<void> => { @@ -151,10 +136,10 @@ async function openWorker(options: StandaloneHostOptions, kind: AttemptKind): Pr const hooks = { ci: config.ci, env: options.env, notice: (message: string) => notice('app', message) }; processes = await startDeclaredProcesses([target], config.projectRoot, hooks, signal, debug, options.processes); if (signal.aborted) throw new InfrastructureError('CANCELLED', `opening a host on target "${target.name}" was cancelled`); - executor = new TargetExecutor({ + executor = policy.sessions({ config, target, runId, artifactsRoot: layout.results, sessionStore, headed: options.headed, workerSlot: 0, env: { ...prepared?.env, ...options.env }, - isolated: false, interruptSignal: signal, debug, + interruptSignal: signal, debug, events: { onNotice: (message) => notice(target.name, message) }, }); await executor.initEngineOnce(); @@ -165,8 +150,8 @@ async function openWorker(options: StandaloneHostOptions, kind: AttemptKind): Pr for (const error of errors) notice(target.name, error.message); throw cause; } - const worker = { kind, config, target, runId, executor, resultsDir: layout.results, debug, signal }; - let state: WorkerState = { phase: 'idle' }; + const worker = { config, target, runId, executor, resultsDir: layout.results, debug, signal }; + let state: WorkerState<Extra> = { phase: 'idle' }; let closing: Promise<readonly SerializedError[]> | undefined; /** Cancels a launch before awaiting it, then ends the attempt before disposing its worker. */ @@ -186,14 +171,14 @@ async function openWorker(options: StandaloneHostOptions, kind: AttemptKind): Pr }; /** Reserves the attempt synchronously, before any engine work can run. */ - const openAttempt = (sessionOptions: StandaloneSessionOptions): Promise<StandaloneAttempt> => { + const openAttempt = (sessionOptions: StandaloneSessionOptions): Promise<StandaloneAttempt<Extra>> => { if (closing !== undefined || state.phase === 'retired') return Promise.reject(new InfrastructureError('WORKER_PROTOCOL', 'the e2e host is closed')); if (state.phase !== 'idle') return Promise.reject(new InfrastructureError('WORKER_PROTOCOL', 'the e2e host already has an opening, live, or closing session')); let reservation: AttemptReservation; try { - reservation = reserveAttempt(worker, sessionOptions); + reservation = reserveAttempt(worker, policy, sessionOptions); } catch (cause) { return Promise.reject(cause); } - const pending = launchAttempt(worker, reservation, async ({ cleanup }) => { + const pending = launchAttempt(worker, policy, reservation, async ({ cleanup }) => { if (cleanup === 'complete') { state = { phase: 'idle' }; return; @@ -207,7 +192,7 @@ async function openWorker(options: StandaloneHostOptions, kind: AttemptKind): Pr return attempt; }, (cause: unknown) => { - reservation.release(); + reservation.publication?.release(); state = { phase: 'idle' }; throw cause; }, diff --git a/packages/e2e/src/run/test-filters.ts b/packages/e2e/src/run/test-filters.ts index 65ba6ff791..a48b124d96 100644 --- a/packages/e2e/src/run/test-filters.ts +++ b/packages/e2e/src/run/test-filters.ts @@ -3,7 +3,7 @@ import path from 'node:path'; import type { Collection } from '../collect/collect.ts'; import { select, type Selection, type SelectionFilters, type TestExclusion, type TestTargetPair } from '../collect/select.ts'; -import type { ResolvedConfig } from '../config/resolve.ts'; +import type { ResolvedRunConfig } from '../config/resolve-run.ts'; import { describeValue } from '../config/validate.ts'; import { leaveWorkBehind } from '../internal/abandoned.ts'; import { E2EError, errorMessage } from '../internal/errors.ts'; @@ -15,14 +15,14 @@ import { compareTests, type FilterOutcome, type ReportChanges, type ReportFilter import { ellipsize, filterDecision, shortCommit } from '../report/format.ts'; import { plural } from '../report/markdown-text.ts'; import { leftOutTests } from '../report/left-out.ts'; -import type { TestFilter, TestFilterCandidate, TestFilterChanges, TestFilterContext } from '../types.ts'; +import type { TestFilter, TestFilterCandidate, TestFilterChanges, TestFilterContext } from '../run-types.ts'; import { findChanges } from './changes.ts'; import { staticSecretLedger } from './secrecy.ts'; import { relevanceBand } from './units.ts'; /** What `filterSelection` judges a selection with, and where each line about what the gate and the filters decided goes. */ export interface FilterSelectionOptions { - readonly config: ResolvedConfig; + readonly config: ResolvedRunConfig; /** The base ref `--changed-since` named. */ readonly changedSince: string; readonly signal: AbortSignal; @@ -60,7 +60,7 @@ export interface TestFilterScope { readonly changes: TestFilterChanges; /** The loaded config file relative to `changes.root`, POSIX; none for a config object. */ readonly configFile: string | undefined; - readonly secrets: ResolvedConfig['allSecrets']; + readonly secrets: ResolvedRunConfig['allSecrets']; /** The id of every collected test, setup tests included: an `exclude` naming one a filter was not given is ignored, since that test's fate is settled. */ readonly collectedIds: ReadonlySet<string>; /** How long each filter call may take before the run stops waiting for it. */ diff --git a/packages/e2e/src/run/units.ts b/packages/e2e/src/run/units.ts index 74b1d58218..7d1423dc55 100644 --- a/packages/e2e/src/run/units.ts +++ b/packages/e2e/src/run/units.ts @@ -8,7 +8,8 @@ import path from 'node:path'; import type { Collection } from '../collect/collect.ts'; import { testIdentity } from '../collect/collect.ts'; -import type { Selection, SkipInfo, TestTargetPair } from '../collect/select.ts'; +import type { Selection, TestTargetPair } from '../collect/select.ts'; +import type { SkipInfo } from './records.ts'; import type { ResolvedTarget } from '../config/resolve.ts'; import type { ResultRecord } from './records.ts'; import type { WirePair } from './worker/protocol.ts'; diff --git a/packages/e2e/src/run/worker/entry.ts b/packages/e2e/src/run/worker/entry.ts index 43589def81..ce97a999e4 100644 --- a/packages/e2e/src/run/worker/entry.ts +++ b/packages/e2e/src/run/worker/entry.ts @@ -9,7 +9,8 @@ import { collectFromRegistration } from '../../collect/collect.ts'; import { collectModule } from '../../collect/registry.ts'; import type { TestTargetPair } from '../../collect/select.ts'; import { importModule, loadConfigModule } from '../../config/load.ts'; -import { assignPorts, resolveConfig } from '../../config/resolve.ts'; +import { assignPorts } from '../../config/resolve.ts'; +import { resolveConfig } from '../../config/resolve-run.ts'; import { setSecretRegistry } from '../../secrets.ts'; import { loadAiSdk } from '../../agent/ai-sdk.ts'; import { AiTraceRecorder, registerAiTraceRecorder } from '../../internal/ai-trace.ts'; diff --git a/packages/e2e/src/run/worker/protocol.ts b/packages/e2e/src/run/worker/protocol.ts index 759b780a4b..dfa5478d4b 100644 --- a/packages/e2e/src/run/worker/protocol.ts +++ b/packages/e2e/src/run/worker/protocol.ts @@ -7,8 +7,8 @@ * each identity with a locally resolved test function. */ -import type { TestIdentity } from '../../collect/collect.ts'; -import type { ResolvedTestOptions, SkipInfo } from '../../collect/select.ts'; +import type { ResolvedTestOptions } from '../../collect/select.ts'; +import type { SkipInfo, TestIdentity } from '../records.ts'; import type { CliOverrides, PortAssignments, ResolvedTarget } from '../../config/resolve.ts'; import type { AiTraceSnapshot } from '../../internal/ai-trace.ts'; import type { DebugSnapshot } from '../../internal/debug.ts'; diff --git a/packages/e2e/src/telemetry/events.ts b/packages/e2e/src/telemetry/events.ts index 6268b9777e..5211dcff41 100644 --- a/packages/e2e/src/telemetry/events.ts +++ b/packages/e2e/src/telemetry/events.ts @@ -14,7 +14,7 @@ import { GRAMMAR_ACTION_NAMES, GRAMMAR_TOOL_NAMES, HARNESS_TOOL_NAMES, MCP_SESSION_SURFACE, PROJECT_TOOL_EVENT_PREFIX } from '../agent/action-names.ts'; import { modelLabel } from '../config/agent.ts'; -import type { ResolvedConfig } from '../config/resolve.ts'; +import type { ResolvedRunConfig } from '../config/resolve-run.ts'; import type { McpClient } from '../mcp/server.ts'; import type { McpSessionSummary } from '../mcp/usage.ts'; import type { Report1Document, ReportError, ReportExplore, ReportStep, ReportUsage } from '../report/build.ts'; @@ -369,14 +369,14 @@ export interface RunContext { /** The flag names given, never their values. */ readonly flags: readonly string[]; /** The config the run resolved; undefined when it failed to load. */ - readonly config: ResolvedConfig | undefined; + readonly config: ResolvedRunConfig | undefined; } /** * Which config features a run used: counts, booleans, and the runner's own * option ids, never a name, a path, or a value the project wrote. */ -function configFeatures(config: ResolvedConfig): Record<string, JsonValue> { +function configFeatures(config: ResolvedRunConfig): Record<string, JsonValue> { const agents = [...config.agents.values()]; return { config_workers: config.workers, diff --git a/packages/e2e/src/types.ts b/packages/e2e/src/types.ts index cdd84a6340..57ef6022c0 100644 --- a/packages/e2e/src/types.ts +++ b/packages/e2e/src/types.ts @@ -13,8 +13,6 @@ import type { StepCacheInfo } from './run/steps.ts'; import type { EngineHandle } from './engine/index.ts'; import type { KeyModifier, Momentum, ScrollDirection, SelectOption, ViewportPoint } from './engine/contract.ts'; import type { CacheStore } from './cache/store.ts'; -import type { RunEvent, RunExitCode, RunStatus } from './run/events.ts'; -import type { Report1Document } from './report/build.ts'; export type { KeyModifier, Momentum, ScrollDirection, SelectOption } from './engine/contract.ts'; export type { CacheReadResult, CacheStore } from './cache/store.ts'; @@ -1548,172 +1546,12 @@ export type AgentConfig = AgentOptions & } ); -/** The reporters the runner ships, named by id; each is a `Reporter` on the same contract. */ -export type BuiltinReporter = 'list' | 'json' | 'junit' | 'markdown'; - -/** - * The run's report document, the one `.e2e/report.json` holds. Its - * `schemaVersion` names the wire format (`report-1`), pinned by - * `schema/report-v1.schema.json` in the package. - */ -export type Report = Report1Document; - -/** What a reporter receives once the run is over and its report is on disk. */ -export interface FinishedRun { - /** The report-1 document, as written. */ - readonly report: Report; - /** The same status `report.run.status` carries. */ - readonly status: RunStatus; - /** The exit code the process will end with. */ - readonly exitCode: RunExitCode; - /** Absolute project root, what the terminal shows paths relative to. */ - readonly projectRoot: string; - /** Where `report.json` was written; undefined when the write failed or config never loaded. */ - readonly reportPath: string | undefined; - /** Absolute directory the report's artifact paths are relative to: `<output>/results`, a directory per test. */ - readonly artifactsRoot: string; - /** Where `--ai-trace` wrote the run's model calls, when it was requested. */ - readonly aiTracePath: string | undefined; - /** - * The trace the runner wrote for each test that kept one (by its `trace` - * mode, a failed one by default), by report result id, as a path from the - * project root (`.e2e/results/checkout-applies-the-coupon-1a2b3c4d5e6f7a8b/trace.md`). - */ - readonly traces: ReadonlyMap<string, string>; - /** - * The report `--last-failed` selected from, when the run was given that - * flag: the run before this one, whose tests that did not fail were left - * out here. A reporter that keeps one place current folds this run into it. - */ - readonly lastRun?: Report; - /** - * Replaces every fixed-value secret and credential password in `text`, and - * any run of 8 or more consecutive characters of one. The report is - * redacted already; this is for text a reporter adds from elsewhere, such - * as lines of a test file. - */ - redact(text: string): string; -} - -/** Rows a reporter hands back for the terminal summary: a label and its text, a URL or a path. */ -export type ReporterSummary = readonly { readonly label: string; readonly text: string }[]; - /** - * A reporter in `reporters`, the contract the built-in `list`, `json`, - * `junit`, and `markdown` reporters implement too. `onEvent` sees every run event as it - * happens, exactly what the `list` reporter renders, and must not block: a - * throw quarantines it for the rest of the run. Event types are added over - * time; a reporter handles the ones it knows and ignores the rest. `onRunFinished` runs once - * `report.json` is written and the summary has printed; it is awaited within - * a fixed budget, and the rows it resolves with print under the summary. Its - * `signal` aborts when that budget runs out or the run is forced to stop, so - * a reporter hands it to its requests and leaves nothing running behind. A - * reporter can never change the run's status or exit code: a failure or a - * timeout is one line on stderr. Like every live value, a reporter never - * crosses a process boundary; workers construct their own copy when they - * load the config module and never call it, so constructing one must have no - * side effects. + * The project half of `E2EConfig`: targets, agents, the replay cache, + * secrets, budgets, and output, which `resolveProjectConfig` resolves; + * `E2EConfig` adds the runner's reporters and test filters. */ -export interface Reporter { - /** Label in diagnostics; required. */ - readonly name: string; - /** Every run event as it happens. Must not block or throw. */ - onEvent?(event: RunEvent): void; - /** Runs once the report is written and the summary printed; awaited one minute. Returned rows print under the summary. */ - onRunFinished?(run: FinishedRun, signal: AbortSignal): Promise<ReporterSummary | void>; -} - -/** - * Leaves collected tests out of a run before anything starts. Called at most - * once, in the runner process; workers load the config too and never call - * it, so constructing one must have no side effects: read git, the - * environment, or the network inside `filter`. A call that throws, returns - * an invalid result, or runs past 5 minutes leaves nothing out. A failed call - * scores nothing either, unless the error it throws has a `relevance` - * property shaped like `TestFilterResult.relevance`: the run checks it as it - * checks a result's, starts the likeliest tests first, and still warns of the - * failure. A call past 5 minutes is abandoned, and nothing it throws after - * that is read. - */ -export interface TestFilter { - /** Names the filter in the report and in errors: ASCII letters, numbers, `_`, `-`, and `.`, not only dots, at most 256 characters, unique among `filters`. */ - readonly name: string; - /** Decides which of `tests` to leave out; every test the result does not name runs. */ - readonly filter: (tests: readonly TestFilterCandidate[], context: TestFilterContext) => TestFilterResult | Promise<TestFilterResult>; -} - -/** One test a filter may leave out: what names it, never its body. */ -export interface TestFilterCandidate { - /** Stable test id (`<file>::<titles>`), the report's `testId`, and the key `exclude` and `relevance` take. */ - readonly id: string; - /** Project-relative POSIX path of the test file. */ - readonly file: string; - /** The describe titles, then the test title. */ - readonly titlePath: readonly string[]; - /** The describe chain's tags, outermost first, then the test's own. */ - readonly tags: readonly string[]; - /** Project-relative POSIX path of the setup test file that saves the test's session; absent when the test uses none. */ - readonly setupFile?: string; -} - -/** What a run's test filters judge: every file that differs from the base commit. */ -export interface TestFilterChanges { - /** The commit the change is measured from: the merge base of the base ref and HEAD. */ - readonly base: string; - /** Absolute path of the git repository root, spelled as on disk like `projectRoot`; `files` are relative to it. */ - readonly root: string; - /** - * Repository-relative POSIX paths of every file added, modified, or - * deleted since `base`, both paths of a rename, uncommitted and untracked - * files included; sorted, never empty. Spelled as git prints them, which - * on macOS is NFC while a test's `file` keeps the bytes on disk, so - * compare a non-ASCII path with `.normalize('NFC')` on both sides. - */ - readonly files: readonly string[]; -} - -/** What the runner hands a filter besides its tests. */ -export interface TestFilterContext { - /** Absolute project root, spelled as on disk like `changes.root`: symlinks resolved, names in on-disk case; a test's `file` is relative to it. */ - readonly projectRoot: string; - /** The change the filter judges; the same for every filter of the run. */ - readonly changes: TestFilterChanges; - /** - * The config file this run loaded, whatever its name, as a POSIX path - * relative to `changes.root` like `changes.files`, in its on-disk bytes - * (compare in NFC, as with a test's `file`); `undefined` when the run was - * given a config object or loaded one outside the repository. - */ - readonly configFile?: string | undefined; - /** - * Aborts when the run is interrupted, or with a `TimeoutError` reason after - * 5 minutes; hand it to every model and network call. - */ - readonly signal: AbortSignal; - /** Prints one progress line as a run notice. */ - log(line: string): void; - /** Replaces every fixed-value secret and credential password in `text`, and any run of 8 or more consecutive characters of one. */ - redact(text: string): string; -} - -/** A filter's answer: every test it does not name runs. */ -export interface TestFilterResult { - /** The tests to leave out, by `TestFilterCandidate.id`, each with the reason the report shows; a test of a serial group still runs when another test of its group does. */ - readonly exclude: Readonly<Record<string, string>>; - /** - * How likely each test is to fail on this change, from 0 to 1, by - * `TestFilterCandidate.id`. Each target starts the files of the likeliest - * tests first, in bands at 75%, 50%, and 25%, and targets go by the band of - * their best file, ties in config order. A test no filter scores counts as - * 0, and one several filters score keeps its highest. It never leaves a - * test out. - */ - readonly relevance?: Readonly<Record<string, number>>; - /** One line about the whole decision, such as `the diff is too large to judge`. */ - readonly note?: string; -} - -export interface E2EConfig { +export interface ProjectConfig { /** Stable project id, 1 through 256 characters; defaults to the root `package.json` name. */ projectId?: string; /** The surfaces tests run on; required, at least one, each naming its engine. */ @@ -1769,13 +1607,6 @@ export interface E2EConfig { * directory the run clears. */ output?: string; - /** - * Output renderers and reporter objects. `junit` writes `<output>/junit.xml`, - * `markdown` writes `<output>/summary.md`, `json` prints the report and - * excludes `list`; a `Reporter` object runs - * beside them and `--reporter` never removes it. - */ - reporters?: readonly (BuiltinReporter | Reporter)[]; /** * The agents by name, each one plain object: the built-in agent's options, * or `{ executor }` for a custom brain. `default` is the one tests run @@ -1785,8 +1616,6 @@ export interface E2EConfig { * construct their own, exactly like model instances. */ agents?: Readonly<Record<string, AgentConfig>>; - /** Test filters, in order, run with `--changed-since`: each judges the tests the one before it kept. */ - filters?: readonly TestFilter[]; /** * The adaptive replay cache. Opt-out: unset means * `read-write`, and `'off'` disables it, as does the `--no-cache` flag, diff --git a/packages/e2e/tests/integration/cli-init.test.ts b/packages/e2e/tests/integration/cli-init.test.ts index 3571fd54c9..2ecfa3e9a4 100644 --- a/packages/e2e/tests/integration/cli-init.test.ts +++ b/packages/e2e/tests/integration/cli-init.test.ts @@ -14,8 +14,8 @@ import { dependencyRange } from '../../src/cli/init/versions.ts'; // Use the built modules, just as config and test imports use the installed packages. const loaderModule = '../../dist/config/load.js'; const { loadConfigModule } = await import(loaderModule) as typeof import('../../src/config/load.ts'); -const resolveModule = '../../dist/config/resolve.js'; -const { resolveConfig } = await import(resolveModule) as typeof import('../../src/config/resolve.ts'); +const resolveModule = '../../dist/config/resolve-run.js'; +const { resolveConfig } = await import(resolveModule) as typeof import('../../src/config/resolve-run.ts'); const collectModule = '../../dist/collect/collect.js'; const { collect } = await import(collectModule) as typeof import('../../src/collect/collect.ts'); const scaffoldModule = '../../dist/cli/init/scaffold.js'; diff --git a/packages/e2e/tests/integration/mcp-session-host.test.ts b/packages/e2e/tests/integration/mcp-session-host.test.ts index 0e3a420288..8295678d4c 100644 --- a/packages/e2e/tests/integration/mcp-session-host.test.ts +++ b/packages/e2e/tests/integration/mcp-session-host.test.ts @@ -26,8 +26,8 @@ import type { RecordingMode, TargetApp } from '../../src/types.ts'; const sessionModule = new URL('../../dist/mcp/session.js', import.meta.url).href; const { SessionHost } = (await import(sessionModule)) as typeof import('../../src/mcp/session.ts'); -const resolveModule = new URL('../../dist/config/resolve.js', import.meta.url).href; -const { resolveConfig } = (await import(resolveModule)) as typeof import('../../src/config/resolve.ts'); +const resolveModule = new URL('../../dist/config/resolve-run.js', import.meta.url).href; +const { resolveConfig } = (await import(resolveModule)) as typeof import('../../src/config/resolve-run.ts'); const secretsModule = new URL('../../dist/secrets.js', import.meta.url).href; const { credentials, secrets } = (await import(secretsModule)) as typeof import('../../src/secrets.ts'); const agentModule = new URL('../../dist/agent/public.js', import.meta.url).href; diff --git a/packages/e2e/tests/integration/mcp-test-run.test.ts b/packages/e2e/tests/integration/mcp-test-run.test.ts index b6f3bccebd..5ad9503e61 100644 --- a/packages/e2e/tests/integration/mcp-test-run.test.ts +++ b/packages/e2e/tests/integration/mcp-test-run.test.ts @@ -31,8 +31,8 @@ import { cacheConfig, entriesFor, expectReplayed, FLOWS, flowsModel, flowsSuite, const sessionModule = new URL('../../dist/mcp/session.js', import.meta.url).href; const { SessionHost } = (await import(sessionModule)) as typeof import('../../src/mcp/session.ts'); -const resolveModule = new URL('../../dist/config/resolve.js', import.meta.url).href; -const { resolveConfig } = (await import(resolveModule)) as typeof import('../../src/config/resolve.ts'); +const resolveModule = new URL('../../dist/config/resolve-run.js', import.meta.url).href; +const { resolveConfig } = (await import(resolveModule)) as typeof import('../../src/config/resolve-run.ts'); const CLI = fileURLToPath(new URL('../../dist/cli/bin.js', import.meta.url)); diff --git a/packages/e2e/tests/integration/shared-app-processes.test.ts b/packages/e2e/tests/integration/shared-app-processes.test.ts index f02d9403d1..5696677251 100644 --- a/packages/e2e/tests/integration/shared-app-processes.test.ts +++ b/packages/e2e/tests/integration/shared-app-processes.test.ts @@ -21,8 +21,8 @@ const provisionModule = new URL('../../dist/run/provision.js', import.meta.url). const { startDeclaredProcesses } = (await import(provisionModule)) as typeof import('../../src/run/provision.ts'); const poolModule = new URL('../../dist/run/process-pool.js', import.meta.url).href; const { SharedAppProcesses } = (await import(poolModule)) as typeof import('../../src/run/process-pool.ts'); -const resolveModule = new URL('../../dist/config/resolve.js', import.meta.url).href; -const { resolveConfig } = (await import(resolveModule)) as typeof import('../../src/config/resolve.ts'); +const resolveModule = new URL('../../dist/config/resolve-run.js', import.meta.url).href; +const { resolveConfig } = (await import(resolveModule)) as typeof import('../../src/config/resolve-run.ts'); const debugModule = new URL('../../dist/internal/debug.js', import.meta.url).href; const { DebugTrace } = (await import(debugModule)) as typeof import('../../src/internal/debug.ts'); diff --git a/packages/e2e/tests/integration/test-filters.test.ts b/packages/e2e/tests/integration/test-filters.test.ts index 9dc9413d66..9e450e15e3 100644 --- a/packages/e2e/tests/integration/test-filters.test.ts +++ b/packages/e2e/tests/integration/test-filters.test.ts @@ -20,7 +20,7 @@ import type { Report1Document } from '../../src/report/build.ts'; import { renderMarkdownReport } from '../../src/report/markdown.ts'; import type { ListedPair, ListOptions } from '../../src/run/runner.ts'; import type { RunEvent } from '../../src/run/events.ts'; -import type { E2EConfig, TestFilter, TestFilterCandidate, TestFilterContext } from '../../src/types.ts'; +import type { E2EConfig, TestFilter, TestFilterCandidate, TestFilterContext } from '../../src/run-types.ts'; import { createFakeEngine, FAKE_APP } from '../helpers/fake-engine.ts'; import { freePort } from '../helpers/free-port.ts'; import { commitFiles, initRepo, runGit } from '../helpers/git-repo.ts'; diff --git a/packages/e2e/tests/unit/agent-assert-explore.test.ts b/packages/e2e/tests/unit/agent-assert-explore.test.ts index 104e217fd2..88ed210548 100644 --- a/packages/e2e/tests/unit/agent-assert-explore.test.ts +++ b/packages/e2e/tests/unit/agent-assert-explore.test.ts @@ -1,7 +1,7 @@ /** The built-in agent's exploring assertion loop: its rules, its evidence policy, and its verdict vocabulary. */ import { expect, it } from 'vitest'; -import { resolveConfig } from '../../src/config/resolve.ts'; +import { resolveConfig } from '../../src/config/resolve-run.ts'; import { defineEngine, type EngineHandle } from '../../src/engine/index.ts'; import { createEngineSession } from '../../src/engine/session.ts'; import { Deadline } from '../../src/internal/time.ts'; @@ -9,7 +9,7 @@ import { AttemptBudget } from '../../src/run/budget.ts'; import { createFixtures } from '../../src/run/fixtures.ts'; import { StepRecorder } from '../../src/run/steps.ts'; import { WorkerModels } from '../../src/run/worker-models.ts'; -import type { E2EConfig } from '../../src/types.ts'; +import type { E2EConfig } from '../../src/run-types.ts'; import { runAgentStepsOnFakeTime } from '../helpers/agent-fake-time.ts'; import { installFakeLoopModel, loopCalls } from '../helpers/fake-loop-model.ts'; import { snapshot } from '../helpers/snapshot.ts'; diff --git a/packages/e2e/tests/unit/agent-config.test.ts b/packages/e2e/tests/unit/agent-config.test.ts index f4bf15a288..b2d533f039 100644 --- a/packages/e2e/tests/unit/agent-config.test.ts +++ b/packages/e2e/tests/unit/agent-config.test.ts @@ -1,12 +1,13 @@ import { APICallError, createGateway } from 'ai'; import { describe, expect, it, vi } from 'vitest'; -import { resolveConfig, type CliOverrides } from '../../src/config/resolve.ts'; +import { type CliOverrides } from '../../src/config/resolve.ts'; +import { resolveConfig } from '../../src/config/resolve-run.ts'; import { z } from 'zod'; import { defineTool } from '../../src/agent/tool.ts'; import { createToolLoopExecutor } from '../../src/agent/tool-loop.ts'; import { TRANSPORT_RETRIES } from '../../src/agent/model/sdk.ts'; import type { SdkLanguageModel } from '../../src/agent/ai-sdk.ts'; -import type { E2EConfig } from '../../src/types.ts'; +import type { E2EConfig } from '../../src/run-types.ts'; import { HARNESS_TOOL_NAMES } from '../../src/agent/action-names.ts'; import { FINDING_TOOL_NAME } from '../../src/explore/executor.ts'; import { recordingTools } from '../../src/mcp/recording.ts'; diff --git a/packages/e2e/tests/unit/agent-entry-prompt.test.ts b/packages/e2e/tests/unit/agent-entry-prompt.test.ts index 9ccacad23d..5cc9d99f20 100644 --- a/packages/e2e/tests/unit/agent-entry-prompt.test.ts +++ b/packages/e2e/tests/unit/agent-entry-prompt.test.ts @@ -1,5 +1,5 @@ import { describe, expect, it } from 'vitest'; -import { resolveConfig } from '../../src/config/resolve.ts'; +import { resolveConfig } from '../../src/config/resolve-run.ts'; import { defineEngine } from '../../src/engine/index.ts'; import { createEngineSession } from '../../src/engine/session.ts'; import { Deadline } from '../../src/internal/time.ts'; @@ -7,7 +7,7 @@ import { AttemptBudget } from '../../src/run/budget.ts'; import { createFixtures } from '../../src/run/fixtures.ts'; import { StepRecorder } from '../../src/run/steps.ts'; import { WorkerModels } from '../../src/run/worker-models.ts'; -import type { E2EConfig } from '../../src/types.ts'; +import type { E2EConfig } from '../../src/run-types.ts'; import { runAgentStepsOnFakeTime } from '../helpers/agent-fake-time.ts'; import { installFakeLoopModel, loopCalls } from '../helpers/fake-loop-model.ts'; import { fakeCalls, installFakeModel, judgment } from '../helpers/fake-model.ts'; diff --git a/packages/e2e/tests/unit/agent-model-refusal.test.ts b/packages/e2e/tests/unit/agent-model-refusal.test.ts index 855b1e4e42..166d5d6979 100644 --- a/packages/e2e/tests/unit/agent-model-refusal.test.ts +++ b/packages/e2e/tests/unit/agent-model-refusal.test.ts @@ -2,7 +2,7 @@ import { describe, expect, it } from 'vitest'; import { createModelAdapter } from '../../src/agent/model/sdk.ts'; import { defineEngine } from '../../src/engine/index.ts'; import { createEngineSession } from '../../src/engine/session.ts'; -import { resolveConfig } from '../../src/config/resolve.ts'; +import { resolveConfig } from '../../src/config/resolve-run.ts'; import { Deadline } from '../../src/internal/time.ts'; import { AttemptBudget } from '../../src/run/budget.ts'; import { createFixtures } from '../../src/run/fixtures.ts'; diff --git a/packages/e2e/tests/unit/app-ports.test.ts b/packages/e2e/tests/unit/app-ports.test.ts index b168b76fc2..ff2779a6d2 100644 --- a/packages/e2e/tests/unit/app-ports.test.ts +++ b/packages/e2e/tests/unit/app-ports.test.ts @@ -1,6 +1,7 @@ import { describe, expect, it } from 'vitest'; import { engineAppInfo } from '../../src/config/app.ts'; -import { assignPorts, resolveConfig, type PortAssignments } from '../../src/config/resolve.ts'; +import { assignPorts, type PortAssignments } from '../../src/config/resolve.ts'; +import { resolveConfig } from '../../src/config/resolve-run.ts'; import { defineEngine } from '../../src/engine/index.ts'; import type { TargetApp } from '../../src/types.ts'; import { allocateAppPorts, allocateTargetPorts } from '../../src/run/app-ports.ts'; diff --git a/packages/e2e/tests/unit/changes.test.ts b/packages/e2e/tests/unit/changes.test.ts index 54f80c60f1..ee077fe5fd 100644 --- a/packages/e2e/tests/unit/changes.test.ts +++ b/packages/e2e/tests/unit/changes.test.ts @@ -9,7 +9,7 @@ import { mkdirSync, realpathSync, rmSync, writeFileSync } from 'node:fs'; import path from 'node:path'; import { describe, expect, it, vi } from 'vitest'; import { findChanges } from '../../src/run/changes.ts'; -import type { TestFilterChanges } from '../../src/types.ts'; +import type { TestFilterChanges } from '../../src/run-types.ts'; import { commitFiles, initRepo, runGit, useTempDirs } from '../helpers/git-repo.ts'; const tempDir = useTempDirs(); diff --git a/packages/e2e/tests/unit/collect.test.ts b/packages/e2e/tests/unit/collect.test.ts index e82d8d8e2c..8de1b8befa 100644 --- a/packages/e2e/tests/unit/collect.test.ts +++ b/packages/e2e/tests/unit/collect.test.ts @@ -3,7 +3,7 @@ import { tmpdir } from 'node:os'; import path from 'node:path'; import { afterAll, beforeAll, describe, expect, it } from 'vitest'; import { collect, excludingEntry, relativeToRoot, selectPositionals } from '../../src/collect/collect.ts'; -import { resolveConfig } from '../../src/config/resolve.ts'; +import { resolveConfig } from '../../src/config/resolve-run.ts'; // What the config globs discovered, in discovery order. `tests/agent/notes.md` // exists on disk but is not in this list: positionals only narrow it. diff --git a/packages/e2e/tests/unit/config.test.ts b/packages/e2e/tests/unit/config.test.ts index 7bb919da2d..104555dd3c 100644 --- a/packages/e2e/tests/unit/config.test.ts +++ b/packages/e2e/tests/unit/config.test.ts @@ -3,11 +3,13 @@ import os from 'node:os'; import path from 'node:path'; import { describe, expect, it } from 'vitest'; import { discoverConfig } from '../../src/config/load.ts'; -import { isCiMode, resolveConfig, type CliOverrides } from '../../src/config/resolve.ts'; +import { isCiMode, type CliOverrides } from '../../src/config/resolve.ts'; +import { resolveConfig } from '../../src/config/resolve-run.ts'; import { ConfigurationError, defineEngine } from '../../src/engine/index.ts'; import { protocolEngine } from '../../src/engine/protocol/client.ts'; import { secrets } from '../../src/secrets.ts'; -import type { CommandConfig, E2EConfig, Target, TargetApp, TestFilter } from '../../src/types.ts'; +import type { CommandConfig, Target, TargetApp } from '../../src/types.ts'; +import type { E2EConfig, TestFilter } from '../../src/run-types.ts'; import { snapshot } from '../helpers/snapshot.ts'; import { invalid } from '../helpers/invalid.ts'; diff --git a/packages/e2e/tests/unit/declared-processes.test.ts b/packages/e2e/tests/unit/declared-processes.test.ts index 80d7d4e46a..6729dfc8b0 100644 --- a/packages/e2e/tests/unit/declared-processes.test.ts +++ b/packages/e2e/tests/unit/declared-processes.test.ts @@ -1,6 +1,7 @@ import { describe, expect, it } from 'vitest'; import { defineEngine } from '../../src/engine/index.ts'; -import { assignPorts, resolveConfig } from '../../src/config/resolve.ts'; +import { assignPorts } from '../../src/config/resolve.ts'; +import { resolveConfig } from '../../src/config/resolve-run.ts'; import { declaredCommands } from '../../src/run/declared-processes.ts'; import type { TargetApp } from '../../src/types.ts'; import { snapshot } from '../helpers/snapshot.ts'; diff --git a/packages/e2e/tests/unit/engine-protocol.test.ts b/packages/e2e/tests/unit/engine-protocol.test.ts index be8a21267e..e0668d60db 100644 --- a/packages/e2e/tests/unit/engine-protocol.test.ts +++ b/packages/e2e/tests/unit/engine-protocol.test.ts @@ -24,7 +24,7 @@ import { type OperationContext, type SemanticNode, } from '../../src/engine/index.ts'; -import { resolveConfig } from '../../src/config/resolve.ts'; +import { resolveConfig } from '../../src/config/resolve-run.ts'; import { ConfigurationError, TestTimeoutError } from '../../src/internal/errors.ts'; import { protocolEngine } from '../../src/engine/protocol/client.ts'; import { Connection } from '../../src/engine/protocol/connection.ts'; diff --git a/packages/e2e/tests/unit/engine.test.ts b/packages/e2e/tests/unit/engine.test.ts index 4c9959aaf0..86c4d2a609 100644 --- a/packages/e2e/tests/unit/engine.test.ts +++ b/packages/e2e/tests/unit/engine.test.ts @@ -9,7 +9,7 @@ import { type EngineSnapshot, } from '../../src/engine/index.ts'; import { createEngineSession } from '../../src/engine/session.ts'; -import { resolveConfig } from '../../src/config/resolve.ts'; +import { resolveConfig } from '../../src/config/resolve-run.ts'; import { secrets } from '../../src/secrets.ts'; import { snapshot } from '../helpers/snapshot.ts'; import type { OperationContext, SemanticNode } from '../../src/engine/surface.ts'; diff --git a/packages/e2e/tests/unit/fixture-runtime.test.ts b/packages/e2e/tests/unit/fixture-runtime.test.ts index 26d3fbacfa..7b10161020 100644 --- a/packages/e2e/tests/unit/fixture-runtime.test.ts +++ b/packages/e2e/tests/unit/fixture-runtime.test.ts @@ -2,7 +2,7 @@ import { describe, expect, it, vi } from 'vitest'; import { z } from 'zod'; import { defineEngine, EngineError, type EngineHandle, LOCATOR_ACTION_KINDS, TestError } from '../../src/engine/index.ts'; import { createEngineSession } from '../../src/engine/session.ts'; -import { resolveConfig } from '../../src/config/resolve.ts'; +import { resolveConfig } from '../../src/config/resolve-run.ts'; import { credentials, secrets, setSecretRegistry } from '../../src/secrets.ts'; import { expect as expectFixture } from '../../src/expect/index.ts'; import { Deadline } from '../../src/internal/time.ts'; @@ -12,7 +12,8 @@ import { StepRecorder } from '../../src/run/steps.ts'; import { WorkerModels } from '../../src/run/worker-models.ts'; import type { StepExecutor } from '../../src/agent/executor.ts'; import { defineTool, getToolContext } from '../../src/agent/tool.ts'; -import type { E2EConfig, TargetApp } from '../../src/types.ts'; +import type { TargetApp } from '../../src/types.ts'; +import type { E2EConfig } from '../../src/run-types.ts'; import { installFakeLoopModel } from '../helpers/fake-loop-model.ts'; import { extracted, fakeCalls, installFakeModel, judgment, notFound } from '../helpers/fake-model.ts'; import { snapshot } from '../helpers/snapshot.ts'; diff --git a/packages/e2e/tests/unit/init-gateways.test.ts b/packages/e2e/tests/unit/init-gateways.test.ts index 37de9a4c8e..ea9f56dc40 100644 --- a/packages/e2e/tests/unit/init-gateways.test.ts +++ b/packages/e2e/tests/unit/init-gateways.test.ts @@ -8,7 +8,7 @@ import { readFileSync } from 'node:fs'; import { describe, expect, it } from 'vitest'; import { GATEWAYS } from '../../src/cli/init/gateways.ts'; -import { resolveConfig } from '../../src/config/resolve.ts'; +import { resolveConfig } from '../../src/config/resolve-run.ts'; const EXPORTS = (JSON.parse(readFileSync(new URL('../../package.json', import.meta.url), 'utf8')) as { exports: Record<string, { default: string }>; diff --git a/packages/e2e/tests/unit/model-request-headers.test.ts b/packages/e2e/tests/unit/model-request-headers.test.ts index 998b099382..f5f58aea73 100644 --- a/packages/e2e/tests/unit/model-request-headers.test.ts +++ b/packages/e2e/tests/unit/model-request-headers.test.ts @@ -3,7 +3,7 @@ import { z } from 'zod'; import { afterAll, beforeAll, beforeEach, describe, expect, it } from 'vitest'; import { createModelAdapter } from '../../src/agent/model/sdk.ts'; import { defineTool } from '../../src/agent/tool.ts'; -import { resolveConfig } from '../../src/config/resolve.ts'; +import { resolveConfig } from '../../src/config/resolve-run.ts'; import { defineEngine } from '../../src/engine/index.ts'; import { createEngineSession } from '../../src/engine/session.ts'; import { USER_AGENT } from '../../src/internal/client-identity.ts'; diff --git a/packages/e2e/tests/unit/press-sequentially.test.ts b/packages/e2e/tests/unit/press-sequentially.test.ts index 5305ee5b16..a5492399f2 100644 --- a/packages/e2e/tests/unit/press-sequentially.test.ts +++ b/packages/e2e/tests/unit/press-sequentially.test.ts @@ -6,7 +6,7 @@ import { afterEach, describe, expect, it, vi } from 'vitest'; import { LOCATOR_ACTION_KINDS, type LocatorActionKind } from '../../src/engine/index.ts'; -import { resolveConfig } from '../../src/config/resolve.ts'; +import { resolveConfig } from '../../src/config/resolve-run.ts'; import { secrets, setSecretRegistry } from '../../src/secrets.ts'; import { screenOver } from '../helpers/screen-over.ts'; import { snapshot } from '../helpers/snapshot.ts'; diff --git a/packages/e2e/tests/unit/provider-failure-message.test.ts b/packages/e2e/tests/unit/provider-failure-message.test.ts index 5b93e7dddc..649aab2aaf 100644 --- a/packages/e2e/tests/unit/provider-failure-message.test.ts +++ b/packages/e2e/tests/unit/provider-failure-message.test.ts @@ -1,7 +1,7 @@ import { createOpenAICompatible } from '@ai-sdk/openai-compatible'; import { afterEach, describe, expect, it } from 'vitest'; import { loadAiSdk } from '../../src/agent/ai-sdk.ts'; -import { resolveConfig } from '../../src/config/resolve.ts'; +import { resolveConfig } from '../../src/config/resolve-run.ts'; import { defineEngine } from '../../src/engine/index.ts'; import { createEngineSession } from '../../src/engine/session.ts'; import { providerFailureMessage, setErrorRedactor } from '../../src/internal/errors.ts'; @@ -10,7 +10,7 @@ import { AttemptBudget } from '../../src/run/budget.ts'; import { createFixtures } from '../../src/run/fixtures.ts'; import { StepRecorder } from '../../src/run/steps.ts'; import { WorkerModels } from '../../src/run/worker-models.ts'; -import type { E2EConfig } from '../../src/types.ts'; +import type { E2EConfig } from '../../src/run-types.ts'; import { runAgentStepsOnFakeTime } from '../helpers/agent-fake-time.ts'; import { snapshot } from '../helpers/snapshot.ts'; diff --git a/packages/e2e/tests/unit/report-junit.test.ts b/packages/e2e/tests/unit/report-junit.test.ts index be48f03965..ae01e8708a 100644 --- a/packages/e2e/tests/unit/report-junit.test.ts +++ b/packages/e2e/tests/unit/report-junit.test.ts @@ -2,7 +2,7 @@ import fs from 'node:fs'; import os from 'node:os'; import path from 'node:path'; import { afterEach, describe, expect, it, vi } from 'vitest'; -import type { FinishedRun } from '../../src/types.ts'; +import type { FinishedRun } from '../../src/run-types.ts'; import type { Report1Document, ReportSerialGroup } from '../../src/report/build.ts'; import { junitReporter, renderJunitReport, type JunitReportOptions } from '../../src/report/junit.ts'; import { REPORT_AT as AT, reportAttempt, reportDocument, reportError, reportResult } from '../helpers/report.ts'; diff --git a/packages/e2e/tests/unit/report-markdown.test.ts b/packages/e2e/tests/unit/report-markdown.test.ts index 604054d7f3..1928f61990 100644 --- a/packages/e2e/tests/unit/report-markdown.test.ts +++ b/packages/e2e/tests/unit/report-markdown.test.ts @@ -13,7 +13,7 @@ import path from 'node:path'; import { afterAll, describe, expect, it } from 'vitest'; import type { Report1Document, ReportAttempt, ReportExplore, ReportExploreFinding, ReportFilter, ReportResult, ReportSerialGroup, ReportStep } from '../../src/report/build.ts'; import { markdownReporter, renderMarkdownReport } from '../../src/report/markdown.ts'; -import type { FinishedRun } from '../../src/types.ts'; +import type { FinishedRun } from '../../src/run-types.ts'; import { REPORT_AT, reportAttempt, reportDocument, reportError, reportResult, reportStep, reportTarget } from '../helpers/report.ts'; const FINISHED_AT = '2026-01-01T00:00:08.400Z'; diff --git a/packages/e2e/tests/unit/report-status.test.ts b/packages/e2e/tests/unit/report-status.test.ts index 6ed9e44e72..d4a12cff3f 100644 --- a/packages/e2e/tests/unit/report-status.test.ts +++ b/packages/e2e/tests/unit/report-status.test.ts @@ -5,7 +5,8 @@ */ import { describe, expect, it } from 'vitest'; -import { resolveConfig, type ResolvedConfig, type ResolvedTarget } from '../../src/config/resolve.ts'; +import { type ResolvedConfig, type ResolvedTarget } from '../../src/config/resolve.ts'; +import { resolveConfig } from '../../src/config/resolve-run.ts'; import type { SerializedError } from '../../src/internal/errors.ts'; import { uuidv7 } from '../../src/internal/ids.ts'; import { buildReport, type BuildReportOptions } from '../../src/report/build.ts'; diff --git a/packages/e2e/tests/unit/scheduler.test.ts b/packages/e2e/tests/unit/scheduler.test.ts index 50dbbc47ce..ce69a23292 100644 --- a/packages/e2e/tests/unit/scheduler.test.ts +++ b/packages/e2e/tests/unit/scheduler.test.ts @@ -10,9 +10,9 @@ import type { CollectedFile, CollectedTest, Collection } from '../../src/collect import type { ResolvedTestOptions, Selection, - SkipInfo, TestTargetPair, } from '../../src/collect/select.ts'; +import type { SkipInfo } from '../../src/run/records.ts'; import type { ResolvedTarget } from '../../src/config/resolve.ts'; import { defineEngine, type EngineHandle } from '../../src/engine/index.ts'; import { classifyError, serializeError } from '../../src/internal/errors.ts'; diff --git a/packages/e2e/tests/unit/select.test.ts b/packages/e2e/tests/unit/select.test.ts index b592e20d1d..10ed31d11c 100644 --- a/packages/e2e/tests/unit/select.test.ts +++ b/packages/e2e/tests/unit/select.test.ts @@ -4,11 +4,12 @@ import { fileURLToPath } from 'node:url'; import { describe, expect, it } from 'vitest'; import { collectFromRegistration, type Collection } from '../../src/collect/collect.ts'; import { collectModule, test } from '../../src/collect/registry.ts'; -import { pairRecording, pairRecordings, repeatEach, resolveOptions, select, type SelectionFilters } from '../../src/collect/select.ts'; -import { resolveConfig } from '../../src/config/resolve.ts'; +import { pairRecordings, repeatEach, resolveOptions, select, type SelectionFilters } from '../../src/collect/select.ts'; +import { pairRecording } from '../../src/run/provision.ts'; +import { resolveConfig } from '../../src/config/resolve-run.ts'; import { defineEngine } from '../../src/engine/index.ts'; import { resultId } from '../../src/internal/ids.ts'; -import type { E2EConfig } from '../../src/types.ts'; +import type { E2EConfig } from '../../src/run-types.ts'; const noop = async () => {}; /** A project root that does not exist, so the diagnostics that stat a positional find nothing, on every host. */ diff --git a/packages/e2e/tests/unit/target-identity.test.ts b/packages/e2e/tests/unit/target-identity.test.ts index 8599369887..969d98bfef 100644 --- a/packages/e2e/tests/unit/target-identity.test.ts +++ b/packages/e2e/tests/unit/target-identity.test.ts @@ -9,7 +9,8 @@ import fs from 'node:fs'; import os from 'node:os'; import path from 'node:path'; import { describe, expect, it } from 'vitest'; -import { assignPorts, resolveConfig } from '../../src/config/resolve.ts'; +import { assignPorts } from '../../src/config/resolve.ts'; +import { resolveConfig } from '../../src/config/resolve-run.ts'; import { defineEngine } from '../../src/engine/index.ts'; import { targetIdentity } from '../../src/run/sessions.ts'; import type { Target } from '../../src/types.ts'; diff --git a/packages/e2e/tests/unit/telemetry-events.test.ts b/packages/e2e/tests/unit/telemetry-events.test.ts index 1835d17334..412f118ab4 100644 --- a/packages/e2e/tests/unit/telemetry-events.test.ts +++ b/packages/e2e/tests/unit/telemetry-events.test.ts @@ -1,5 +1,5 @@ import { describe, expect, it } from 'vitest'; -import { resolveConfig } from '../../src/config/resolve.ts'; +import { resolveConfig } from '../../src/config/resolve-run.ts'; import { defineEngine } from '../../src/engine/index.ts'; import type { McpSessionSummary } from '../../src/mcp/usage.ts'; import type { ReportExplore } from '../../src/report/build.ts'; diff --git a/packages/e2e/tests/unit/test-filters.test.ts b/packages/e2e/tests/unit/test-filters.test.ts index 626a3c76ca..0b4a4c02b0 100644 --- a/packages/e2e/tests/unit/test-filters.test.ts +++ b/packages/e2e/tests/unit/test-filters.test.ts @@ -15,7 +15,8 @@ import { AgentError } from '../../src/agent/error.ts'; import { collectFromRegistration, type Collection } from '../../src/collect/collect.ts'; import { collectModule, test } from '../../src/collect/registry.ts'; import { select } from '../../src/collect/select.ts'; -import { resolveConfig, type ResolvedConfig } from '../../src/config/resolve.ts'; +import type { ResolvedRunConfig as ResolvedConfig } from '../../src/config/resolve-run.ts'; +import { resolveConfig } from '../../src/config/resolve-run.ts'; import { defineEngine } from '../../src/engine/index.ts'; import { ConfigurationError, type E2EError } from '../../src/internal/errors.ts'; import { @@ -29,7 +30,7 @@ import { type FilterSelectionOptions, type TestFilterScope, } from '../../src/run/test-filters.ts'; -import type { E2EConfig, TestFilter, TestFilterCandidate, TestFilterContext, TestFilterResult } from '../../src/types.ts'; +import type { E2EConfig, TestFilter, TestFilterCandidate, TestFilterContext, TestFilterResult } from '../../src/run-types.ts'; import { invalid } from '../helpers/invalid.ts'; /** What `leaveWorkBehind` was told: a timeout records the work it left behind, for the CLI to exit on. */ diff --git a/packages/e2e/tests/unit/tool-loop-overflow.test.ts b/packages/e2e/tests/unit/tool-loop-overflow.test.ts index 2c3d302bb1..51e6a941f8 100644 --- a/packages/e2e/tests/unit/tool-loop-overflow.test.ts +++ b/packages/e2e/tests/unit/tool-loop-overflow.test.ts @@ -3,7 +3,7 @@ import { z } from 'zod'; import { createToolLoopExecutor } from '../../src/agent/tool-loop.ts'; import { projectTools } from '../../src/agent/default-agent.ts'; import { defineTool } from '../../src/agent/tool.ts'; -import { resolveConfig } from '../../src/config/resolve.ts'; +import { resolveConfig } from '../../src/config/resolve-run.ts'; import { defineEngine, type EngineHandle } from '../../src/engine/index.ts'; import { createEngineSession } from '../../src/engine/session.ts'; import { Deadline } from '../../src/internal/time.ts'; @@ -11,7 +11,7 @@ import { AttemptBudget } from '../../src/run/budget.ts'; import { createFixtures } from '../../src/run/fixtures.ts'; import { StepRecorder } from '../../src/run/steps.ts'; import { WorkerModels } from '../../src/run/worker-models.ts'; -import type { E2EConfig } from '../../src/types.ts'; +import type { E2EConfig } from '../../src/run-types.ts'; import { runAgentStepsOnFakeTime } from '../helpers/agent-fake-time.ts'; import { installFakeLoopModel, loopCalls } from '../helpers/fake-loop-model.ts'; import { snapshot } from '../helpers/snapshot.ts'; diff --git a/packages/e2e/tests/unit/tool-loop-prefix.test.ts b/packages/e2e/tests/unit/tool-loop-prefix.test.ts index f0702d58fc..74205aa816 100644 --- a/packages/e2e/tests/unit/tool-loop-prefix.test.ts +++ b/packages/e2e/tests/unit/tool-loop-prefix.test.ts @@ -13,7 +13,7 @@ import { tool } from 'ai'; import { afterAll, beforeAll, describe, expect, it } from 'vitest'; import { z } from 'zod'; import { createToolLoopExecutor } from '../../src/agent/tool-loop.ts'; -import { resolveConfig } from '../../src/config/resolve.ts'; +import { resolveConfig } from '../../src/config/resolve-run.ts'; import { defineEngine, type EngineHandle } from '../../src/engine/index.ts'; import { createEngineSession } from '../../src/engine/session.ts'; import { Deadline } from '../../src/internal/time.ts'; @@ -21,7 +21,8 @@ import { AttemptBudget } from '../../src/run/budget.ts'; import { createFixtures } from '../../src/run/fixtures.ts'; import { StepRecorder } from '../../src/run/steps.ts'; import { WorkerModels } from '../../src/run/worker-models.ts'; -import type { E2EConfig, ModelInstance } from '../../src/types.ts'; +import type { ModelInstance } from '../../src/types.ts'; +import type { E2EConfig } from '../../src/run-types.ts'; import { runAgentStepsOnFakeTime } from '../helpers/agent-fake-time.ts'; import { installFakeLoopModel, loopCalls } from '../helpers/fake-loop-model.ts'; import { snapshot } from '../helpers/snapshot.ts'; diff --git a/packages/e2e/tests/unit/tool-loop-stall.test.ts b/packages/e2e/tests/unit/tool-loop-stall.test.ts index 377abc8344..5c2e30350c 100644 --- a/packages/e2e/tests/unit/tool-loop-stall.test.ts +++ b/packages/e2e/tests/unit/tool-loop-stall.test.ts @@ -5,7 +5,7 @@ import { z } from 'zod'; import { withStallGuard } from '../../src/agent/model/stall.ts'; import { asSdkLanguageModel } from '../../src/agent/ai-sdk.ts'; import { defineTool } from '../../src/agent/tool.ts'; -import { resolveConfig } from '../../src/config/resolve.ts'; +import { resolveConfig } from '../../src/config/resolve-run.ts'; import { defineEngine } from '../../src/engine/index.ts'; import { createEngineSession } from '../../src/engine/session.ts'; import { Deadline } from '../../src/internal/time.ts'; @@ -13,7 +13,7 @@ import { AttemptBudget } from '../../src/run/budget.ts'; import { createFixtures } from '../../src/run/fixtures.ts'; import { StepRecorder } from '../../src/run/steps.ts'; import { WorkerModels } from '../../src/run/worker-models.ts'; -import type { E2EConfig } from '../../src/types.ts'; +import type { E2EConfig } from '../../src/run-types.ts'; import { runAgentStepsOnFakeTime } from '../helpers/agent-fake-time.ts'; import { installFakeLoopModel } from '../helpers/fake-loop-model.ts'; import { judgment } from '../helpers/fake-model.ts'; diff --git a/packages/e2e/tests/unit/tool-loop-tool-choice.test.ts b/packages/e2e/tests/unit/tool-loop-tool-choice.test.ts index dac709acf3..567d2cde01 100644 --- a/packages/e2e/tests/unit/tool-loop-tool-choice.test.ts +++ b/packages/e2e/tests/unit/tool-loop-tool-choice.test.ts @@ -2,7 +2,7 @@ import { afterAll, beforeAll, describe, expect, it } from 'vitest'; import { z } from 'zod'; import { isForcedToolCallSkipped, isForcedToolChoiceRejected } from '../../src/agent/model/tool-choice.ts'; import { defineTool } from '../../src/agent/tool.ts'; -import { resolveConfig } from '../../src/config/resolve.ts'; +import { resolveConfig } from '../../src/config/resolve-run.ts'; import { defineEngine } from '../../src/engine/index.ts'; import { createEngineSession } from '../../src/engine/session.ts'; import { Deadline } from '../../src/internal/time.ts'; @@ -10,7 +10,7 @@ import { AttemptBudget } from '../../src/run/budget.ts'; import { createFixtures } from '../../src/run/fixtures.ts'; import { StepRecorder } from '../../src/run/steps.ts'; import { WorkerModels } from '../../src/run/worker-models.ts'; -import type { E2EConfig } from '../../src/types.ts'; +import type { E2EConfig } from '../../src/run-types.ts'; import { runAgentStepsOnFakeTime } from '../helpers/agent-fake-time.ts'; import { installFakeLoopModel, loopCalls, type LoopResponder } from '../helpers/fake-loop-model.ts'; import { snapshot } from '../helpers/snapshot.ts'; diff --git a/packages/e2e/tests/unit/trace-cache-config.test.ts b/packages/e2e/tests/unit/trace-cache-config.test.ts index 577063b841..78a940fdb1 100644 --- a/packages/e2e/tests/unit/trace-cache-config.test.ts +++ b/packages/e2e/tests/unit/trace-cache-config.test.ts @@ -9,8 +9,9 @@ import { createAgentCacheContext, settleStagedTraces } from '../../src/cache/con import { FileCacheStore } from '../../src/cache/store.ts'; import { buildTraceEntry } from '../../src/cache/trace.ts'; import { readTraceEntry } from '../../src/cache/trace-read.ts'; -import { resolveConfig } from '../../src/config/resolve.ts'; -import type { ActionTrace, E2EConfig, CacheStore } from '../../src/types.ts'; +import { resolveConfig } from '../../src/config/resolve-run.ts'; +import type { ActionTrace, CacheStore } from '../../src/types.ts'; +import type { E2EConfig } from '../../src/run-types.ts'; const ROOT = path.resolve('/tmp/e2e-cache-config-tests'); const BASE_ENV = {} as NodeJS.ProcessEnv; diff --git a/packages/e2e/tests/unit/worker-handle.test.ts b/packages/e2e/tests/unit/worker-handle.test.ts index 5ab9ae3e2d..0908f6cae6 100644 --- a/packages/e2e/tests/unit/worker-handle.test.ts +++ b/packages/e2e/tests/unit/worker-handle.test.ts @@ -9,7 +9,7 @@ import { tmpdir } from 'node:os'; import path from 'node:path'; import { fileURLToPath, pathToFileURL } from 'node:url'; import { afterAll, describe, expect, it, vi } from 'vitest'; -import { resolveConfig } from '../../src/config/resolve.ts'; +import { resolveConfig } from '../../src/config/resolve-run.ts'; import { defineEngine, type EngineCleanupContext, type EngineHandle } from '../../src/engine/index.ts'; import { SessionStore } from '../../src/run/sessions.ts'; import { TargetWorker } from '../../src/run/worker/session.ts'; diff --git a/packages/e2e/tests/unit/worker-models.test.ts b/packages/e2e/tests/unit/worker-models.test.ts index a6613a4644..94a257e425 100644 --- a/packages/e2e/tests/unit/worker-models.test.ts +++ b/packages/e2e/tests/unit/worker-models.test.ts @@ -1,7 +1,7 @@ import { describe, expect, it } from 'vitest'; import type { SdkLanguageModel } from '../../src/agent/ai-sdk.ts'; import { AgentError } from '../../src/agent/error.ts'; -import { resolveConfig } from '../../src/config/resolve.ts'; +import { resolveConfig } from '../../src/config/resolve-run.ts'; import { WorkerModels } from '../../src/run/worker-models.ts'; const ENV = { APP_URL: 'http://localhost:3000' } as NodeJS.ProcessEnv; From 7dd8ace57d5ce2823dbc57fe906ab8d75ac8c7cc Mon Sep 17 00:00:00 2001 From: DongYun Kang <kdy.1997.dev@gmail.com> Date: Fri, 13 Nov 2026 16:01:27 +0900 Subject: [PATCH 2/2] refactor(e2e): keep pairRecording beside the recording modes collect/select.ts read it from run/provision.ts, which loads app processes and engine provisioning; internal/recording-modes.ts already holds the modes it resolves. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> --- packages/e2e/src/collect/select.ts | 3 +-- packages/e2e/src/internal/recording-modes.ts | 17 +++++++++++++++++ packages/e2e/src/run/provision.ts | 13 +------------ packages/e2e/tests/unit/select.test.ts | 2 +- 4 files changed, 20 insertions(+), 15 deletions(-) diff --git a/packages/e2e/src/collect/select.ts b/packages/e2e/src/collect/select.ts index fffe4f3a74..d6ae54092c 100644 --- a/packages/e2e/src/collect/select.ts +++ b/packages/e2e/src/collect/select.ts @@ -6,11 +6,10 @@ import { resultId } from '../internal/ids.ts'; import { expectedFailureOption } from '../internal/expected-failure.ts'; import { didYouMean, suggestionNote } from '../internal/suggest.ts'; import type { ResolvedConfig, ResolvedTarget } from '../config/resolve.ts'; -import { attemptKeep, attemptVideo, type AttemptRecordings } from '../internal/recording-modes.ts'; +import { attemptKeep, attemptVideo, pairRecording, type AttemptRecordings } from '../internal/recording-modes.ts'; import type { Capability, RecordingMode, TestOptions } from '../types.ts'; import { excludingEntry, type Collection, type CollectedTest, type UncollectedFile } from './collect.ts'; import { groupChain } from './registry.ts'; -import { pairRecording } from '../run/provision.ts'; import type { SkipInfo } from '../run/records.ts'; export interface ResolvedTestOptions { diff --git a/packages/e2e/src/internal/recording-modes.ts b/packages/e2e/src/internal/recording-modes.ts index a01a219b64..54ad2d2849 100644 --- a/packages/e2e/src/internal/recording-modes.ts +++ b/packages/e2e/src/internal/recording-modes.ts @@ -27,6 +27,23 @@ export interface ResolvedRecording { readonly source: RecordingSource; } +/** + * Which attempts of a pair record `kind`, and where the mode came from: the + * test's own, else its target's (which the flag and the config already + * decided). One resolver for both recordings, so `trace` and `video` can + * never disagree about precedence. + */ +export function pairRecording( + pair: { + readonly options: { readonly [Kind in RecordingKind]: RecordingMode | undefined }; + readonly target: { readonly [Kind in RecordingKind]: ResolvedRecording }; + }, + kind: RecordingKind, +): ResolvedRecording { + const own = pair.options[kind]; + return own === undefined ? pair.target[kind] : { mode: own, source: 'test' }; +} + /** Which attempts keep what they captured: every one, or only one that ran and did not pass. */ export type Keep = 'always' | 'on-failure'; diff --git a/packages/e2e/src/run/provision.ts b/packages/e2e/src/run/provision.ts index 62b708281e..cc3a6c13ea 100644 --- a/packages/e2e/src/run/provision.ts +++ b/packages/e2e/src/run/provision.ts @@ -13,7 +13,7 @@ import { ENGINE_SPI_VERSION, type EngineSpiVersion } from '../engine/contract.ts import type { DebugTrace } from '../internal/debug.ts'; import { ConfigurationError, InfrastructureError, translateProvisioningError } from '../internal/errors.ts'; import { Deadline, NEVER_ABORTS, withScopedBudget } from '../internal/time.ts'; -import { isRetryMode, recordsOnSomeAttempt, type RecordingKind, type ResolvedRecording } from '../internal/recording-modes.ts'; +import { isRetryMode, pairRecording, recordsOnSomeAttempt, type RecordingKind, type ResolvedRecording } from '../internal/recording-modes.ts'; import type { RecordingMode } from '../types.ts'; import { declaredCommands } from './declared-processes.ts'; import { ManagedProcess, type AppProcesses, type ManagedProcessHooks } from './managed-process.ts'; @@ -59,17 +59,6 @@ export function describeTarget(target: ResolvedTarget): TargetProvenance { }; } -/** - * Which attempts of a pair record `kind`, and where the mode came from: the - * test's own, else its target's (which the flag and the config already - * decided). One resolver for both recordings, so `trace` and `video` can - * never disagree about precedence. - */ -export function pairRecording(pair: Pick<GradedPair, 'options' | 'target'>, kind: RecordingKind): ResolvedRecording { - const own = pair.options[kind]; - return own === undefined ? pair.target[kind] : { mode: own, source: 'test' }; -} - /** A target graded against its engine declaration, before any test starts. */ export interface EngineGrade { readonly provenance: TargetProvenance; diff --git a/packages/e2e/tests/unit/select.test.ts b/packages/e2e/tests/unit/select.test.ts index 10ed31d11c..8125a16383 100644 --- a/packages/e2e/tests/unit/select.test.ts +++ b/packages/e2e/tests/unit/select.test.ts @@ -5,7 +5,7 @@ import { describe, expect, it } from 'vitest'; import { collectFromRegistration, type Collection } from '../../src/collect/collect.ts'; import { collectModule, test } from '../../src/collect/registry.ts'; import { pairRecordings, repeatEach, resolveOptions, select, type SelectionFilters } from '../../src/collect/select.ts'; -import { pairRecording } from '../../src/run/provision.ts'; +import { pairRecording } from '../../src/internal/recording-modes.ts'; import { resolveConfig } from '../../src/config/resolve-run.ts'; import { defineEngine } from '../../src/engine/index.ts'; import { resultId } from '../../src/internal/ids.ts';