Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 5 additions & 0 deletions .changeset/project-config.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
---
'e2e': patch
---

`e2e` exports `ProjectConfig`, the part of `E2EConfig` without `reporters` and `filters`.
2 changes: 1 addition & 1 deletion packages/e2e/src/cli/cache.ts
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@ import type { Command } from 'commander';
import picocolors from 'picocolors';
import { FileCacheStore, MAX_CACHE_WIRE_BYTES } from '../cache/store.ts';
import { discoverConfig, loadConfigModule, missingConfigError } from '../config/load.ts';
import { resolveConfig } from '../config/resolve.ts';
import { resolveConfig } from '../config/resolve-run.ts';
import { errorMessage } from '../internal/errors.ts';
import { visibleWidth } from '../report/format.ts';
import { docsLine, examples } from './help.ts';
Expand Down
3 changes: 2 additions & 1 deletion packages/e2e/src/cli/doctor.ts
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,8 @@ import { AgentError, isAgentError } from '../agent/error.ts';
import { engineAppInfo } from '../config/app.ts';
import { modelLabel, type ResolvedAgentConfig, type ResolvedModel } from '../config/agent.ts';
import { discoverConfig, loadConfigModule, missingConfigError } from '../config/load.ts';
import { resolveConfig, type ResolvedConfig } from '../config/resolve.ts';
import { type ResolvedConfig } from '../config/resolve.ts';
import { resolveConfig } from '../config/resolve-run.ts';
import type { ResolvedTarget } from '../config/targets.ts';
import { selectTargets } from '../collect/select.ts';
import type { EngineDoctorCheck } from '../engine/index.ts';
Expand Down
3 changes: 2 additions & 1 deletion packages/e2e/src/cli/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,8 @@ import { writeTerminal } from '../run/process-output.ts';
import { STEP_BOUNDS, TIMEOUT_BOUNDS } from '../explore/bounds.ts';
import { BUILTIN_REPORTERS, isBuiltinReporter } from '../report/builtin.ts';
import { bounded, failureLine } from '../report/format.ts';
import type { BuiltinReporter, RecordingMode } from '../types.ts';
import type { RecordingMode } from '../types.ts';
import type { BuiltinReporter } from '../run-types.ts';
import { isRecordingMode, legacyTraceSpelling, RECORDING_MODES } from '../internal/recording-modes.ts';
import { discoverConfig } from '../config/load.ts';
import { isCiMode } from '../config/resolve.ts';
Expand Down
21 changes: 1 addition & 20 deletions packages/e2e/src/collect/collect.ts
Original file line number Diff line number Diff line change
Expand Up @@ -18,26 +18,7 @@ import {
type RegisteredTest,
} from './registry.ts';
import type { SourceLocation } from '../internal/source.ts';

/**
* Everything needed to name and report a test, with no executable or realm
* state attached. Fully JSON-serializable, so it is what result records carry
* and what crosses the runner<->worker IPC channel.
*/
export interface TestIdentity {
readonly kind: 'test' | 'setup';
readonly title: string;
readonly titlePath: readonly string[];
readonly declarationIndex: number;
readonly sessions: readonly string[];
/** The tags the test declares, outermost describe first, each once; what `--tag` selects on. */
readonly tags: readonly string[];
readonly source: SourceLocation | undefined;
/** Normalized project-root-relative file path with `/` separators. */
readonly file: string;
readonly id: string;
readonly serialId: string | undefined;
}
import type { TestIdentity } from '../run/records.ts';

export interface CollectedTest extends RegisteredTest, TestIdentity {
/** Outermost serial group, when the test is a serial-group member. */
Expand Down
41 changes: 2 additions & 39 deletions packages/e2e/src/collect/select.ts
Original file line number Diff line number Diff line change
Expand Up @@ -6,10 +6,11 @@ import { resultId } from '../internal/ids.ts';
import { expectedFailureOption } from '../internal/expected-failure.ts';
import { didYouMean, suggestionNote } from '../internal/suggest.ts';
import type { ResolvedConfig, ResolvedTarget } from '../config/resolve.ts';
import { attemptKeep, attemptVideo, type AttemptRecordings, type RecordingKind, type ResolvedRecording } from '../internal/recording-modes.ts';
import { attemptKeep, attemptVideo, pairRecording, type AttemptRecordings } from '../internal/recording-modes.ts';
import type { Capability, RecordingMode, TestOptions } from '../types.ts';
import { excludingEntry, type Collection, type CollectedTest, type UncollectedFile } from './collect.ts';
import { groupChain } from './registry.ts';
import type { SkipInfo } from '../run/records.ts';

export interface ResolvedTestOptions {
readonly timeout: number;
Expand Down Expand Up @@ -39,33 +40,6 @@ export interface ResolvedTestOptions {
readonly video: RecordingMode | undefined;
}

/** Why a pair did not run; only a `filtered` one can name the test filter that left it out. */
export type SkipInfo =
| (SkipDetail & {
readonly cause: 'filtered';
/** The name of the test filter that left the test out; absent when the selection did, as with `--tag`. */
readonly filter?: string;
})
| (SkipDetail & {
readonly cause:
| 'explicit'
| 'platform-unavailable'
| 'capability-unavailable'
| 'setup-failed'
| 'serial-predecessor-failed'
| 'hook-failed'
| 'infrastructure-unavailable'
| 'failure-limit'
| 'time-limit';
readonly filter?: never;
});

/** What a skip says whatever its cause: why, and the id of a related test, such as the setup that failed. */
interface SkipDetail {
readonly reason: string;
readonly relatedId?: string;
}

export interface TestTargetPair {
readonly test: CollectedTest;
readonly target: ResolvedTarget;
Expand Down Expand Up @@ -230,17 +204,6 @@ export function resolveOptions(test: CollectedTest, config: ResolvedConfig): Res
};
}

/**
* Which attempts of a pair record `kind`, and where the mode came from: the
* test's own, else its target's (which the flag and the config already
* decided). One resolver for both recordings, so `trace` and `video` can
* never disagree about precedence.
*/
export function pairRecording(pair: Pick<TestTargetPair, 'options' | 'target'>, kind: RecordingKind): ResolvedRecording {
const own = pair.options[kind];
return own === undefined ? pair.target[kind] : { mode: own, source: 'test' };
}

/** What an attempt of a pair at `attemptIndex` captures: one keep decision for the trace and the video alike. */
export function pairRecordings(pair: Pick<TestTargetPair, 'options' | 'target'>, attemptIndex: number): AttemptRecordings {
return {
Expand Down
2 changes: 1 addition & 1 deletion packages/e2e/src/config/load.ts
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@ import path from 'node:path';
import { ConfigurationError, InfrastructureError, isE2EError } from '../internal/errors.ts';
import { onBun } from '../internal/runtime.ts';
import { withConfigSecretReferences } from '../secrets.ts';
import type { E2EConfig } from '../types.ts';
import type { E2EConfig } from '../run-types.ts';
import { explainModuleError } from './diagnose.ts';
import { freshModuleURL, registerLoader } from './esm-hooks.ts';
import { CONFIG_NAMES, JS_CONFIG_NAMES, TS_CONFIG_NAMES } from './names.ts';
Expand Down
144 changes: 144 additions & 0 deletions packages/e2e/src/config/resolve-run.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,144 @@
/**
* The runner's config: the project config (`resolveProjectConfig`), plus
* the reporters and test filters only a run reads.
*/

import { ConfigurationError } from '../internal/errors.ts';
import { didYouMean } from '../internal/suggest.ts';
import { BUILTIN_REPORTER_LIST, BUILTIN_REPORTERS, isBuiltinReporter } from '../report/builtin.ts';
import { ellipsize } from '../report/format.ts';
import type { BuiltinReporter, E2EConfig, Reporter, TestFilter } from '../run-types.ts';
import { resolveProjectConfig, type CliOverrides, type ResolvedConfig } from './resolve.ts';
import { TARGET_NAME_PATTERN } from './targets.ts';

/** A resolved config with what only a run reads. */
export interface ResolvedRunConfig extends ResolvedConfig {
/** The built-in renderers in force: `--reporter` when given, else the config's ids. */
readonly reporters: readonly BuiltinReporter[];
/** The reporter objects the config names; `--reporter` never removes one. */
readonly customReporters: readonly Reporter[];
/** The test filters, in order; empty when unset. A run with `--changed-since` applies them. */
readonly filters: readonly TestFilter[];
}

/** The flags that replace config keys, `--reporter` among them. */
export interface RunCliOverrides extends CliOverrides {
reporters?: readonly BuiltinReporter[];
}

/** Resolves a config file's export for the runner: the project config, then its reporters and filters. */
export function resolveConfig(
raw: E2EConfig,
options: {
projectRoot: string;
configPath?: string;
env?: NodeJS.ProcessEnv;
cli?: RunCliOverrides;
},
): ResolvedRunConfig {
const resolved = resolveProjectConfig(raw, options);
const { reporters, customReporters } = resolveReporters(raw, options.cli ?? {});
return { ...resolved, reporters, customReporters, filters: resolveTestFilters(raw.filters) };
}

/**
* Splits `reporters` into the built-in ids and the reporter objects.
* `--reporter` replaces the ids only: an object has no id to name on the
* command line, so a CI flag choosing `list,junit` never drops the upload a
* config asked for.
*/
function resolveReporters(
raw: E2EConfig,
cli: RunCliOverrides,
): { reporters: readonly BuiltinReporter[]; customReporters: readonly Reporter[] } {
const configured = raw.reporters ?? ['list'];
if (!Array.isArray(configured)) {
throw new ConfigurationError(
'INVALID_CONFIG',
'reporters must be an array of reporter ids and reporter objects',
);
}
const ids: BuiltinReporter[] = [];
const customReporters: Reporter[] = [];
for (const reporter of configured as readonly unknown[]) {
if (typeof reporter === 'string') {
if (!isBuiltinReporter(reporter)) {
throw new ConfigurationError(
'INVALID_CONFIG',
`unknown reporter "${reporter}"; reporters are ${BUILTIN_REPORTER_LIST}${didYouMean(reporter, BUILTIN_REPORTERS)}`,
);
}
ids.push(reporter);
} else if (isReporter(reporter)) {
customReporters.push(reporter);
} else {
throw new ConfigurationError(
'INVALID_CONFIG',
`a reporter is a built-in id (${BUILTIN_REPORTERS.join(', ')}) or an object with a name and an onEvent or onRunFinished method`,
);
}
}
const reporters = cli.reporters ?? ids;
if (reporters.includes('json') && reporters.includes('list')) {
throw new ConfigurationError('INVALID_CONFIG', 'the json renderer cannot be combined with list');
}
return { reporters, customReporters };
}

const MAX_FILTER_NAME = 256;

/**
* Resolves `filters`: a list of test filters, each checked structurally,
* never with `instanceof`, since the config may import another copy of
* `e2e`. A filter's name is how the report and errors tell it apart, so
* names are unique. Workers resolve it too and never call a filter, so the
* check stays synchronous.
*/
function resolveTestFilters(raw: E2EConfig['filters']): readonly TestFilter[] {
if (raw === undefined) return [];
if (!Array.isArray(raw)) {
throw new ConfigurationError('INVALID_CONFIG', 'filters must be a list of test filters: filters: [{ name, filter(tests, context) }]');
}
const names = new Set<string>();
return Array.from(raw as readonly unknown[], (value, index) => {
const label = `filters[${index}]`;
if (typeof value === 'function') {
throw new ConfigurationError('INVALID_CONFIG', `${label} is ${value.name ? `the function ${value.name}` : 'a function'}; call it, such as filters: [decisionFilter({ model })]`);
}
if (!isTestFilter(value)) {
throw new ConfigurationError('INVALID_CONFIG', `${label} must be a TestFilter: { name, filter(tests, context) }`);
}
const { name } = value;
if (!TARGET_NAME_PATTERN.test(name)) {
throw new ConfigurationError(
'INVALID_CONFIG',
`invalid filter name ${JSON.stringify(name)}: names are ASCII letters, numbers, "_", "-", or ".", and cannot be only dots`,
);
}
if (name.length > MAX_FILTER_NAME) {
throw new ConfigurationError('INVALID_CONFIG', `filter name ${JSON.stringify(ellipsize(name, 32))} is longer than ${MAX_FILTER_NAME} characters`);
}
if (names.has(name)) {
throw new ConfigurationError('INVALID_CONFIG', `filters has two filters named ${JSON.stringify(name)}; the report tells filters apart by name`);
}
names.add(name);
return value;
});
}

/** Structural test filter check: a string name and a `filter` function. */
function isTestFilter(value: unknown): value is TestFilter {
if (typeof value !== 'object' || value === null) return false;
const { name, filter } = value as { name?: unknown; filter?: unknown };
return typeof name === 'string' && typeof filter === 'function';
}

/** Structural reporter check: a non-empty name and at least one handler, each a function when present. */
function isReporter(value: unknown): value is Reporter {
if (typeof value !== 'object' || value === null) return false;
const { name, onEvent, onRunFinished } = value as { name?: unknown; onEvent?: unknown; onRunFinished?: unknown };
if (typeof name !== 'string' || name.length === 0) return false;
const optionalFunction = (handler: unknown): boolean => handler === undefined || typeof handler === 'function';
return optionalFunction(onEvent) && optionalFunction(onRunFinished) && (onEvent ?? onRunFinished) !== undefined;
}

Loading
Loading