From 64f710002ea63da384ef8fe25574ed85aeef7c5f Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 16 Nov 2026 17:25:54 +0000 Subject: [PATCH 1/3] Port createAgent for a core session with the replay cache by cacheKey (upstream #1734) Closes #1676 Co-Authored-By: Claude Sonnet 5.5 Claude-Session: https://claude.ai/code/session_01Ls2e1zqvVudTEe6MWBzFXR --- COMPATIBILITY.md | 8 +- .../public/api/E2E.AgentCacheOptions.html | 13 + .../public/api/E2E.CachingActOptions.html | 13 + docs-site/public/api/E2E.CoreAgent.html | 13 + docs-site/public/api/E2E.CoreAgents.html | 13 + .../public/api/E2E.CreateAgentOptions.html | 13 + ...ngine.LocatorAction.PressSequentially.html | 13 + docs-site/public/api/E2E.ModelUsage.html | 13 + .../docs/api/E2E.Engine/EngineStartOptions.md | 3 +- .../docs/api/E2E.Engine/IBrowserProvider.md | 4 +- .../LocatorAction.PressSequentially.md | 78 ++++++ .../src/content/docs/api/E2E.Engine/index.md | 1 + .../src/content/docs/api/E2E/ActOptions.md | 4 +- .../content/docs/api/E2E/AgentCacheOptions.md | 88 +++++++ .../src/content/docs/api/E2E/AgentOptions.md | 4 +- .../content/docs/api/E2E/CachingActOptions.md | 61 +++++ .../src/content/docs/api/E2E/CoreAgent.md | 131 ++++++++++ .../src/content/docs/api/E2E/CoreAgents.md | 52 ++++ .../docs/api/E2E/CreateAgentOptions.md | 43 ++++ .../content/docs/api/E2E/E2ESessionOptions.md | 2 +- .../src/content/docs/api/E2E/LaunchOptions.md | 5 +- .../src/content/docs/api/E2E/ModelUsage.md | 97 +++++++ .../src/content/docs/api/E2E/ScriptedModel.md | 2 +- docs-site/src/content/docs/api/E2E/Session.md | 20 ++ docs-site/src/content/docs/api/E2E/index.md | 6 + docs-site/src/content/docs/docs/core.mdx | 60 ++++- skills/e2e-dotnet/SKILL.md | 2 +- skills/e2e-dotnet/references/host.md | 4 +- src/E2E/Agent/Act.cs | 23 +- src/E2E/Agent/Agent.cs | 66 ++--- src/E2E/Agent/AgentError.cs | 3 + src/E2E/Agent/AgentOptions.cs | 2 +- src/E2E/Cache/Context.cs | 107 +++++++- src/E2E/Core/CoreAgent.cs | 240 ++++++++++++++++++ src/E2E/Core/Internals.cs | 68 +++++ src/E2E/Core/Launch.cs | 32 ++- src/E2E/Run/E2ESession.cs | 12 +- tests/E2E.Tests/Core/CoreFakeEngine.cs | 22 +- tests/E2E.Tests/Core/CoreTests.cs | 26 +- tests/E2E.Tests/Core/CreateAgentTests.cs | 204 +++++++++++++++ 40 files changed, 1497 insertions(+), 74 deletions(-) create mode 100644 docs-site/public/api/E2E.AgentCacheOptions.html create mode 100644 docs-site/public/api/E2E.CachingActOptions.html create mode 100644 docs-site/public/api/E2E.CoreAgent.html create mode 100644 docs-site/public/api/E2E.CoreAgents.html create mode 100644 docs-site/public/api/E2E.CreateAgentOptions.html create mode 100644 docs-site/public/api/E2E.Engine.LocatorAction.PressSequentially.html create mode 100644 docs-site/public/api/E2E.ModelUsage.html create mode 100644 docs-site/src/content/docs/api/E2E.Engine/LocatorAction.PressSequentially.md create mode 100644 docs-site/src/content/docs/api/E2E/AgentCacheOptions.md create mode 100644 docs-site/src/content/docs/api/E2E/CachingActOptions.md create mode 100644 docs-site/src/content/docs/api/E2E/CoreAgent.md create mode 100644 docs-site/src/content/docs/api/E2E/CoreAgents.md create mode 100644 docs-site/src/content/docs/api/E2E/CreateAgentOptions.md create mode 100644 docs-site/src/content/docs/api/E2E/ModelUsage.md create mode 100644 src/E2E/Core/CoreAgent.cs create mode 100644 src/E2E/Core/Internals.cs create mode 100644 tests/E2E.Tests/Core/CreateAgentTests.cs diff --git a/COMPATIBILITY.md b/COMPATIBILITY.md index 7b87c7f3c..ce5d3dc0a 100644 --- a/COMPATIBILITY.md +++ b/COMPATIBILITY.md @@ -90,7 +90,7 @@ Names are C# versions of the JavaScript API: `agent.act` is `ActAsync`, `screen. - Truncated recordings (tester-army/e2e `trace-recorder`, `trace-cache` and `step-cache` tests, #1187). A recording that cannot be stored faithfully is flagged `CacheEntry.Truncated` (upstream's `truncated`) and never replays: `Decide.DecideTraceReplay` answers `miss` with the reason `truncated` (before the start-path check), so the executor runs the step from the top with nothing replayed, also under `cache.strict` (`truncated` is not a stale reason, as upstream's `STALE_REASONS` omits it), and `FileStepCache.UnderAnotherKey` never counts a truncated entry as the recording a changed key hid (upstream's `rekeyed.ts`). The entry is still written, as upstream writes it. `TraceRecorder` (upstream's `TraceRecorder`) poisons the trace in four cases: an action past the cap (`Trace.MaxTraceActions`, 50, upstream's `MAX_TRACE_ACTIONS`; the constructor's `maxActions` is upstream's option of that name, and a scroll that folds into the one before it does not count), a replay input the redactor would alter (a typed, selected or navigated value, a key, an upload path, the text a `scrollUntil` pages toward, and the start and end routes: a registered secret in it) or one over `Trace.MaxTraceInputChars` (4,096; upstream's `verbatim`, which keeps a redacted, bounded copy for documentation and never replays it), and a targeted action whose descriptor no replay could re-find (`Relocate.IsRelocatableDescriptor`: an unnamed twin that no named row or lone place tells apart, upstream's `require`). A secret fill holds its marker (``), a name, so it is not a replay input and never poisons the trace. A secret value typed through `Fill` is poisoned, not stored: the entry holds the redacted copy and `Truncated`. The store's read also refuses an entry over a cap, as `invalid-entry` (upstream's `readActionTrace`, `readAnchors`, `readStates`): more than 50 actions, a typed value, URL, scroll text or route over 4,096 characters, more than `Trace.MaxTraceAnchors` (8) anchors on a side of the delta, and anchor states that are not `Trace.TraceAnchorStates` (`checked`, `expanded`, `pressed`, `selected`), each once, in sorted order, never empty. `Decide.OpensWithNavigate` skips leading project-tool gaps as well as a resize, as upstream's `opensWithNavigate` does. Differences: `CacheEntry.HasValidCaps`, `RecordedAction.HasValidInputs` and `RecordedTarget.HasValidStates` are .NET-only names for the read checks, which sit beside the port's other `HasValid*` checks (the port validates at the replay, not in a reader); `TraceRecorder.Record(RecordedAction)` returns a copy, since a poisoned input is stored in its redacted form; `Finalize` takes the routes unredacted and redacts them itself (upstream's `anchorPath`), so the caller no longer does; `Trace.IsSecretMarker` is internal now, for the recorder. Not ported: the checks of fields the port has no counterpart of (verdict summary, executor, `createdAt`, an action with no target). Tests: `TraceRecorder/TraceRecorderTests` (the three recorder `it`s), `TraceCache/DecideTraceReplayTests` (the two `decideTraceReplay` `it`s), `TraceCache/Trace1EntryTests` (the cap and anchor state rejections, at the session seam like the resize ones), `StepCache/StepTraceSessionTests` ("misses a truncated entry"), and the `cache.strict` tests of `StepCache/` ("still runs live ... a recording too long to replay", and the truncated half of "runs live when the only recording is truncated or records no step"). `.NET-only`: `Keeps_a_secret_fill_marker_without_poisoning_the_trace` (the port's marker is a `fill` value, upstream's `typeSecret` has its own field). - Secret fill policy events (#1331, upstream's `authorizeSecretFill`, `agent/secrets.ts`). `SecretFill.AuthorizeSecretFillAsync` (`Agent/SecretFill.cs`, upstream's `agent/secrets.ts`) records one policy event on the step for each decision, through `Phases.RecordPolicyEvent` (the mechanism of `upload.path`): `secret.registered` (denied, `AUTH_CREDENTIAL_UNAVAILABLE` for a password, else `SECRET_UNAVAILABLE`), `secret.sink` (denied, `POLICY_DENIED`: a disabled field or one that is not editable), `secret.purpose` (denied, `POLICY_DENIED`) and `secret.fill` (allowed). A live fill records them on each authorization (the stale retry authorizes again); a replayed fill (`ActReplayHost`) authorizes once, before its fill, so it records one set. An event holds a name, a decision and a code, never the value. Upstream passes a `SecretFillHost` and the `AgentContext` and resolves the value inside `authorizeSecretFill` (#1382); the port's `ISecretFillHost` and `ISecretFillRuntime` (the `config.allSecrets.get` and `secrets.resolve` members, and the redactor) are those two, `AttemptScope` is the runtime, and the function returns the resolved `Secret` rather than the plaintext string. The plaintext is read last, after every check, so a refused fill never asks the provider. The purpose is the registered secret's, not the handle's, as upstream's. Upstream's `unavailableCode` is `Secret.UnavailableCode`. `AgentInputStale/SecretFillStaleRetryTests` asserts the `secret.fill` count and stays `Partial_` (no action origin, no `credentials`, no replay of a cached fill through a project runner). `AgentSecrets/AuthorizeSecretFillTests` ports the four `it`s of `describe('authorizeSecretFill')` whole, calling `SecretFill.AuthorizeSecretFillAsync` directly; the credential's password is `ResolvedCredential.Password` rather than `Credentials.User("admin").Password`, whose registry is process-wide. - A replayed secret fill looks the secret up among the secrets the step's params declare, through the dispatcher's own lookup (`ActionDispatcher.DeclaredSecret`, upstream's `this.options.secrets.get(name)` in `typeSecret`, `agent/action-dispatcher.ts`), so a marker the step did not declare fails with `POLICY_DENIED` (`secret "" was not declared in this step's params; only declared secrets can be filled`), as a live `type_secret` does (#1330). It was `AUTH_CREDENTIAL_UNAVAILABLE` (`Replay needs secret ...`). The replay misses as `action-failed`. Test: `TraceReplay/ReplayThroughTheActTests.A_secret_marker_the_step_did_not_declare_misses_as_action_failed` (.NET-only: no upstream `it` covers a replayed undeclared secret). -- `cache.mode` `read-only` replays but never writes or evicts, including the `end-mismatch` eviction. `off` neither replays nor records. Under `cache.strict`, a stale recording fails the act with `REPLAY_STALE` and stays in place; a strict run never writes or evicts, and its retries replay. A `no-entry` miss fails too when the cache directory holds a recording of the same step under another key. Entries record the step as the test, the engine, the instruction, the params digest, the repeat index, and the agent; the port stores the target id (`TargetId`) as upstream does and also the engine, which upstream does not, and the instruction text where upstream stores its digest. Only an entry that records the whole step counts, and, outside `cache.strict`, a verified whole replay in `read-write` mode writes the missing fields into an older entry. A custom `IStepCache` is not checked. The message names what changed between the entry's `keyedBy` and this run's key context, as upstream's `keyContextChanges` does, and says "the runner, the engine, the app, or the agent's context" for an entry with no `keyedBy`. The message ends with upstream's advice (`staleAdvice` in `cache/context.ts`, built by `CacheContexts.CreateAgentCacheContext` into `AgentCacheContext.Strict.Advice` and handed to the step cache as `AttemptScope.StrictAdvice`): `re-record it with a read-write run with cache.strict set to false`, then `and commit the changed entry under ` for a cache directory inside the project (shown relative to the project root, with `/` separators) or `; the run writes the new entry under ` for one outside it (absolute), or `; the run writes the new entry to the configured cache.store` when a store replaces the directory. `cache.dir` and `E2E_CACHE_DIR` change the directory it names. Differences: the port has no `--strict-cache` flag (the internal `CliOverrides.CacheStrict` stands in for it, so only code can turn it on and the advice then reads `without --strict-cache`, as upstream's), so `cache.strict` is the only knob a user can turn on; `AgentCacheContext` has no `claimKey`, `staged`, or `strict.recordings` (`AttemptScope` claims keys, the step session lists the cache directory). `TraceCacheConfig/ReplayStaleAdviceTests` ports the three `REPLAY_STALE advice` its; the second names a `--cache-dir` outside the project through `CliOverrides.CacheDir`. +- `cache.mode` `read-only` replays but never writes or evicts, including the `end-mismatch` eviction. `off` neither replays nor records. Under `cache.strict`, a stale recording fails the act with `REPLAY_STALE` and stays in place; a strict run never writes or evicts, and its retries replay. A `no-entry` miss fails too when the cache directory holds a recording of the same step under another key. Entries record the step as the test, the engine, the instruction, the params digest, the repeat index, and the agent; the port stores the target id (`TargetId`) as upstream does and also the engine, which upstream does not, and the instruction text where upstream stores its digest. Only an entry that records the whole step counts, and, outside `cache.strict`, a verified whole replay in `read-write` mode writes the missing fields into an older entry. A custom `IStepCache` is not checked. The message names what changed between the entry's `keyedBy` and this run's key context, as upstream's `keyContextChanges` does, and says "the runner, the engine, the app, or the agent's context" for an entry with no `keyedBy`. The message ends with upstream's advice (`staleAdvice` in `cache/context.ts`, built by `CacheContexts.CreateAgentCacheContext` into `AgentCacheContext.Strict.Advice` and handed to the step cache as `AttemptScope.StrictAdvice`): `re-record it with a read-write run with cache.strict set to false`, then `and commit the changed entry under ` for a cache directory inside the project (shown relative to the project root, with `/` separators) or `; the run writes the new entry under ` for one outside it (absolute), or `; the run writes the new entry to the configured cache.store` when a store replaces the directory. `cache.dir` and `E2E_CACHE_DIR` change the directory it names. Differences: the port has no `--strict-cache` flag (the internal `CliOverrides.CacheStrict` stands in for it, so only code can turn it on and the advice then reads `without --strict-cache`, as upstream's), so `cache.strict` is the only knob a user can turn on; `AgentCacheContext` has no `strict.recordings` (the step session lists the cache directory), and a test's attempt claims its keys on `AttemptScope` (`ActCache` builds the context per act) while a context a program creates (`CoreAgents.CreateAgent`) holds its own `ClaimKey` and `Staged`. `TraceCacheConfig/ReplayStaleAdviceTests` ports the three `REPLAY_STALE advice` its; the second names a `--cache-dir` outside the project through `CliOverrides.CacheDir`. - The entry's file name is upstream's `traceCacheKeyHash` (`cache/identity.ts`): the SHA-256 of the RFC 8785 canonical JSON of a `trace-1` key (`buildTraceCacheKey`) of the test, target, engine name and contract version, app identity, instruction digest (`instructionDigest`, normalized by `normalizeInstruction`), params digest (`paramsDigest`), repeat index (`createCallIndexer`, counted per agent name and redacted context), agent, agent context digest, and replay policy version. The port's key differs from upstream's in these parts only: `project` is the digest of the empty `projectId` (the port has no `projectId`); `appIdentity` is upstream's `targetIdentity` (`Sessions.TargetIdentity`): the digest of the target app's resolved `identity` (the declared `app.identity`, else the URL's origin and path as declared, a free port still `:0`, else the bundle id, else the build path) and `environment`; an app that declares neither keys on its URL and its host's default environment, so recordings made before `app.identity` and `environment` (which keyed on the empty declaration) miss as `no-entry` and must be re-recorded; a fixture that overrides `BaseUrl` keys on that URL; `testId` is the test title, not `::`; `targetId` is the target's name and `engineName` the engine's `IEngine.Name` (the port has no engine manifest). `IEngine.Name` (upstream's `engine.name`) is a default interface member that returns `Platform`; `MobileEngine` names itself `mobile` as upstream's does, so its key and the report's `run.targets[].engine` read `mobile` where they read `ios` or `android`, and an earlier mobile recording misses as `no-entry`. The replay policy version is not bumped, as upstream did not bump it for a key change (`e77c17a` dropped the engine version from the key with a changeset only). `WebEngine` is `web`, `DocumentEngine` `document`, `WinAppEngine` `windows`; `policyVersion` is the port's own `6`: version 5 followed the placeholder grammar of `cache/template.ts`, so an entry written with the earlier U+0001 slots is never read, and its step records again; version 6 keys a secret param as upstream's `validateParams` projects it, so the next run records those steps again. Params are projected before the digest as the port's cache has always projected them: a `Values.Unique` value is its placeholder, `{{param:<pointer>}}` (upstream's `templateParams`, `CacheKeys.TemplateParams`, over the tree `ActParams.ValidateParams` projects; no params digest as the empty object) and a `Secret` is `{"kind":"secret","name":<name>,"purpose":<purpose>}` (its `purpose` only when it has one), never its value; a number is written by .NET's shortest round-trip form, which matches ECMAScript's for the values a param holds. The entry stores `keyedBy` (`TraceKeyContext`) as upstream does; an entry with a malformed `keyedBy` still reads, without it. **Recordings made before this key (the port's former text-hash key) are not read and are not migrated:** they miss as `no-entry` and must be re-recorded. Under `cache.strict` they do not fail as `REPLAY_STALE` either: the check matches the step by its recorded params digest, which the old entries hold in the old form, so the step runs live. - The entry stores `targetId` (`CacheEntry.TargetId`, upstream's `recordedFor.targetId`; bounded to `MAX_TRACE_DESCRIPTOR_CHARS` on write like `Test` and `Agent`, and a blank, over-long, or non-text value makes the entry `invalid-entry` as upstream's `readProvenance` does) next to its flat `Test`, `Instruction`, `ParamsDigest`, `CallIndex`, and `Agent` (upstream nests them under `recordedFor`): the target's name, masked like the test title, provenance a replay never reads; an entry recorded before it was stored holds none and reads as before. The port has no `e2e cache ls`, which lists it. The occurrence fields (`ParamsDigest`, `CallIndex`, `Agent`, upstream's `recordedFor.paramsDigest`, `callIndex`, `agent`) are optional on read, and a confirmed whole replay of an entry that lacks them writes them in (`E2ESession.CompleteProvenance`, upstream's `completeProvenance`); a read-only run leaves it untouched. `TraceCacheModesTests.Caches_by_default_and_CI_demotes_...` (#1287) now starts from such an entry and proves the CI-demoted replay leaves the file as it was; it runs through `RunProject.RunExistingAsync` (#1336). @@ -167,12 +167,14 @@ Differences: Differences: -- A session is one `Session` class with `Screen`, `App`, `Platform`, `Browser`, `SaveStateAsync`, `SetDefaultTimeout`, `CloseAsync`, and `DisposeAsync`, where upstream's is a flat object typed by the engine's fixtures (`Host<Fixtures>`). There is no `Device` member on it yet (#1678). It has no `agent`: `createAgent` is not ported (#1676). +- A session is one `Session` class with `Screen`, `App`, `Platform`, `Browser`, `SaveStateAsync`, `SetDefaultTimeout`, `CloseAsync`, and `DisposeAsync`, where upstream's is a flat object typed by the engine's fixtures (`Host<Fixtures>`). There is no `Device` member on it yet (#1678). It has no `agent` member: `CoreAgents.CreateAgent(session, options)` (`Core/CoreAgent.cs`) is upstream's `createAgent` (`core/agent.ts`, #1676), a function over the session as upstream's is. - `LaunchOptions` has no `Headed` (the web engine takes its window mode through its own options), no `Signal` or `HandleSignals` (the token passed to `LaunchAsync` and `NewSessionAsync` is the signal; closing on SIGINT, SIGTERM, and SIGHUP is #1677), and `Timeout` is a `TimeSpan`. `Env` is a dictionary and `OnNotice` takes the line only. - `RedactThrown` rebuilds an error that carries a secret value (type and code kept, causes redacted) where upstream edits the thrown error in place, which a .NET exception's read-only message does not allow. An error with no secret is thrown as it is. - `SessionState.ExportAsync` and `ImportAsync` are `Task`-returning like upstream's `async` methods, so a bad key rejects instead of throwing. The key is `SessionStateKeyOptions.Key`, a `byte[]` of 32 bytes; there is no unknown-option check (`plaintext: true` cannot be passed). - `app.linkBase` (`AppConfig.LinkBase`, `ResolvedApp.LinkBase`, `LaunchApp.LinkBase`, checked by `TargetApps.CheckLinkBase`, upstream's `checkLinkBase`) and the engine `openLink` hook (`IEngineSession.OpenLinkAsync`, `EngineSessionSpec.OpenLink`, `ISessionApp.OpenLinkAsync`, `session/openLink` in the manifest schema) are ported, with `App.OpenAsync(path)` joining and opening the link as upstream's `launchPinnedApp` does. `EngineCapabilities.OpenLink` is .NET-only: C# has no optional member to test, so an engine declares the hook the way it declares `Resize`. `TargetApps.JoinLink`, `TargetApps.RefusesLink`, and `Urls.IsRefusedLinkScheme` (upstream's `joinLink`, `refusesLink`, `isRefusedLinkScheme`) are shared with `DeviceLinks`. Not ported: the unknown-key message of a `defineEngine` session, which a typed `EngineSessionSpec` cannot receive (`engine.test.ts`). -- Tests: `Core/CoreSessionStateTests` is `core-state.test.ts`; `Core/CoreTests` is the ported part of `core.test.ts` (partial, and renamed `Partial_...`: 'opens sessions one at a time' (no `gadget` fixture or key list), 'refuses what is not an engine' (no non-engine or unknown app key to pass), 'says a closed session is closed' (the redaction half only), 'bounds each operation by its default timeout' (no agent step, which needs `createAgent`, #1676)); `AppLinkBase/` is `app-link-base.test.ts`; `Core/CoreTests` also holds 'opens a path as a link in the pinned app on a device target with a link base' (`Opens_a_path_as_a_link_...`); `SchemaFixtures/SessionStateV1SchemaTests`; `StepVerification/StepRecorderLastVerifiedStepIndexTests` has the two new `it` (`RedactThrown` is partial: the caught error is checked, not the original in place); `TraceCache/Trace1EntryTests` has the space-press `it`. Not ported: the `it` of `createAgent`, the link-base `it` of `core.test.ts`, `core-signals.test.ts`, and the `engine.test.ts` fixture-name check (#1676 to #1678). +- `CoreAgents.CreateAgent` returns one `CoreAgent` where upstream types `CachingAgent`, `CoreAgent`, and `UncheckedAgent` by the options (a TypeScript union C# overloads cannot express): `ActAsync` takes `CachingActOptions` (an `ActOptions` plus `CacheKey` and `Cache`, which can only be `false`) and refuses at runtime what upstream's types refuse at compile time (`INVALID_ARGUMENT`). `CreateAgentOptions` is an `AgentOptions` plus `Cache` (`AgentCacheOptions`: `Dir`, `Mode`, `Store`, `Strict`); `Mode` reuses `CacheMode` and `Off` is `INVALID_ARGUMENT`. `ActOptions` and `AgentOptions` are no longer sealed so these can derive from them. `CoreAgents` (.NET-only name) holds the function, which has no class in upstream; `CoreAgentBinding` (.NET-only) stands for the `AgentContext` override (`select`, `cache`) `createAgent` builds, since the port's `Agent` reads an `AttemptScope`. Upstream's `WorkerModels` (preflight of the model at the first call) has no counterpart: a model is called, and refused, as in a test. `newSession` takes no signal (see above), so there is no cancelled-session settlement (`settlement()` of `SessionInternals` always settles); the options `createAgent` rejects as unknown keys (`rejectUnknownKeys`) cannot be passed in C#. +- `FailureWatch`, `SessionInternals`, `AttemptSettlement`, `SurvivedFailure`, and `CacheContexts.SettleStagedTraces` (`Core/Internals.cs`, `Cache/Context.cs`) are upstream's `FailureWatch`, `SessionInternals`, `AttemptSettlement`, `SurvivedFailure`, and `settleStagedTraces`. `AgentCacheContext` now has `TestId`, `ClaimKey`, and `Staged` (upstream's `claimKey` and `staged`) for a context a program creates; a test's attempt still keeps them on `AttemptScope`, which builds its context per act (`AttemptScope.ActCache`). A program's settlement confirms an act by its step index against the last verified step, as upstream does; `E2ESession.Flush` takes the per-act `preserve` it needs. `AgentError.ModelUnreachableCodes` is upstream's `MODEL_UNREACHABLE_CODES`. A recording's key uses the project id `@e2e-dev/core` and the test id `core:<cacheKey>`, as upstream's. +- Tests: `Core/CoreSessionStateTests` is `core-state.test.ts`; `Core/CoreTests` is the ported part of `core.test.ts` (partial, and renamed `Partial_...`: 'opens sessions one at a time' (no `gadget` fixture or key list), 'refuses what is not an engine' (no non-engine or unknown app key to pass), 'says a closed session is closed' (the redaction half only)); `AppLinkBase/` is `app-link-base.test.ts`; `Core/CoreTests` also holds 'opens a path as a link in the pinned app on a device target with a link base' (`Opens_a_path_as_a_link_...`); `SchemaFixtures/SessionStateV1SchemaTests`; `StepVerification/StepRecorderLastVerifiedStepIndexTests` has the two new `it` (`RedactThrown` is partial: the caught error is checked, not the original in place); `TraceCache/Trace1EntryTests` has the space-press `it`. `Core/CreateAgentTests` is `describe('createAgent')` of `core.test.ts` (all four `it`; 'requires a cacheKey' also checks the typed-options refusal at runtime, since C# has no `@ts-expect-error`). Not ported: `core-signals.test.ts` and the `engine.test.ts` fixture-name check (#1677, #1678). ## Not ported diff --git a/docs-site/public/api/E2E.AgentCacheOptions.html b/docs-site/public/api/E2E.AgentCacheOptions.html new file mode 100644 index 000000000..cc07a2995 --- /dev/null +++ b/docs-site/public/api/E2E.AgentCacheOptions.html @@ -0,0 +1,13 @@ +<!DOCTYPE html> +<html lang="en"> +<head> +<meta charset="utf-8"> +<link rel="canonical" href="https://www.hardkoded.com/e2e-dotnet/api/E2E/AgentCacheOptions/"> +<meta http-equiv="refresh" content="0; url=/e2e-dotnet/api/E2E/AgentCacheOptions/"> +<title>Redirecting… + + + +

Continue

+ + diff --git a/docs-site/public/api/E2E.CachingActOptions.html b/docs-site/public/api/E2E.CachingActOptions.html new file mode 100644 index 000000000..d1fc2651a --- /dev/null +++ b/docs-site/public/api/E2E.CachingActOptions.html @@ -0,0 +1,13 @@ + + + + + + +Redirecting… + + + +

Continue

+ + diff --git a/docs-site/public/api/E2E.CoreAgent.html b/docs-site/public/api/E2E.CoreAgent.html new file mode 100644 index 000000000..3a781e741 --- /dev/null +++ b/docs-site/public/api/E2E.CoreAgent.html @@ -0,0 +1,13 @@ + + + + + + +Redirecting… + + + +

Continue

+ + diff --git a/docs-site/public/api/E2E.CoreAgents.html b/docs-site/public/api/E2E.CoreAgents.html new file mode 100644 index 000000000..420545ab4 --- /dev/null +++ b/docs-site/public/api/E2E.CoreAgents.html @@ -0,0 +1,13 @@ + + + + + + +Redirecting… + + + +

Continue

+ + diff --git a/docs-site/public/api/E2E.CreateAgentOptions.html b/docs-site/public/api/E2E.CreateAgentOptions.html new file mode 100644 index 000000000..89bfaf820 --- /dev/null +++ b/docs-site/public/api/E2E.CreateAgentOptions.html @@ -0,0 +1,13 @@ + + + + + + +Redirecting… + + + +

Continue

+ + diff --git a/docs-site/public/api/E2E.Engine.LocatorAction.PressSequentially.html b/docs-site/public/api/E2E.Engine.LocatorAction.PressSequentially.html new file mode 100644 index 000000000..fdc3b15de --- /dev/null +++ b/docs-site/public/api/E2E.Engine.LocatorAction.PressSequentially.html @@ -0,0 +1,13 @@ + + + + + + +Redirecting… + + + +

Continue

+ + diff --git a/docs-site/public/api/E2E.ModelUsage.html b/docs-site/public/api/E2E.ModelUsage.html new file mode 100644 index 000000000..381683ca8 --- /dev/null +++ b/docs-site/public/api/E2E.ModelUsage.html @@ -0,0 +1,13 @@ + + + + + + +Redirecting… + + + +

Continue

+ + diff --git a/docs-site/src/content/docs/api/E2E.Engine/EngineStartOptions.md b/docs-site/src/content/docs/api/E2E.Engine/EngineStartOptions.md index 5d9922854..1d46e9477 100644 --- a/docs-site/src/content/docs/api/E2E.Engine/EngineStartOptions.md +++ b/docs-site/src/content/docs/api/E2E.Engine/EngineStartOptions.md @@ -178,7 +178,8 @@ public bool Headed { get; init; } ``` -Whether the run asked for a visible surface (`--headed`). .NET-only place: upstream hands it to `init` ([Headed](/api/E2E.Engine/EngineInitInfo/#E2E_Engine_EngineInitInfo_Headed)), which the port does not call per worker, so a session start carries it. +Whether the run asked for a visible surface (`--headed`). .NET-only place: upstream hands it to `init` ([Headed](/api/E2E.Engine/EngineInitInfo/#E2E_Engine_EngineInitInfo_Headed)), +which the port does not call per worker, so a session start carries it. **Value:** [bool](https://learn.microsoft.com/dotnet/api/system.boolean) diff --git a/docs-site/src/content/docs/api/E2E.Engine/IBrowserProvider.md b/docs-site/src/content/docs/api/E2E.Engine/IBrowserProvider.md index 2d488b9e3..540385771 100644 --- a/docs-site/src/content/docs/api/E2E.Engine/IBrowserProvider.md +++ b/docs-site/src/content/docs/api/E2E.Engine/IBrowserProvider.md @@ -185,8 +185,8 @@ Task> SweepAsync(BrowserReleaseContext context, Cancellati Releases every lease of the run and target the provider still holds open, and resolves to their ids. Called once per target from finish, in the runner, after every worker exited and the leases prepare made were released: whatever it finds is a lease a worker acquired (a per-attempt browser, a replacement) and never gave back, -because the worker died before it could. Find them by what acquire tagged them with at the service: the run id -and the target. +because the worker died before it could. Find them by service tags or a shared journal scoped to the run id and +target. **Parameters** diff --git a/docs-site/src/content/docs/api/E2E.Engine/LocatorAction.PressSequentially.md b/docs-site/src/content/docs/api/E2E.Engine/LocatorAction.PressSequentially.md new file mode 100644 index 000000000..828196809 --- /dev/null +++ b/docs-site/src/content/docs/api/E2E.Engine/LocatorAction.PressSequentially.md @@ -0,0 +1,78 @@ +--- +title: "LocatorAction.PressSequentially" +description: "Focuses the node and types Text one character at a time." +editUrl: false +prev: false +next: false +sidebar: + label: "LocatorAction.PressSequentially" + hidden: true +--- + + + +Namespace: [E2E.Engine](/api/E2E.Engine/) + + +```csharp +public sealed record LocatorAction.PressSequentially : LocatorAction, IEquatable, IEquatable +``` + + +Focuses the node and types <code class="paramref">Text</code> one character at a time. + +**Inheritance:** [object](https://learn.microsoft.com/dotnet/api/system.object) → [LocatorAction](/api/E2E.Engine/LocatorAction/) + +**Implements:** [IEquatable<LocatorAction>](https://learn.microsoft.com/dotnet/api/system.iequatable-1), [IEquatable<LocatorAction.PressSequentially>](https://learn.microsoft.com/dotnet/api/system.iequatable-1) + +[View source](https://github.com/hardkoded/e2e-dotnet/blob/main/src/E2E/Engine/LocatorAction.cs) + +## Constructors + + + + +### PressSequentially(string, TimeSpan?) + + +```csharp +public PressSequentially(string Text, TimeSpan? Delay = null) +``` + + +Focuses the node and types <code class="paramref">Text</code> one character at a time. + +**Parameters** + +| Name | Type | Description | +| --- | --- | --- | +| `Text` | [string](https://learn.microsoft.com/dotnet/api/system.string) | | +| `Delay` | [TimeSpan?](https://learn.microsoft.com/dotnet/api/system.timespan) | | + +## Properties + + + + +### Delay + + +```csharp +public TimeSpan? Delay { get; init; } +``` + + +**Value:** [TimeSpan?](https://learn.microsoft.com/dotnet/api/system.timespan) + + + + +### Text + + +```csharp +public string Text { get; init; } +``` + + +**Value:** [string](https://learn.microsoft.com/dotnet/api/system.string) diff --git a/docs-site/src/content/docs/api/E2E.Engine/index.md b/docs-site/src/content/docs/api/E2E.Engine/index.md index a1ab006ab..ef0ca09b3 100644 --- a/docs-site/src/content/docs/api/E2E.Engine/index.md +++ b/docs-site/src/content/docs/api/E2E.Engine/index.md @@ -79,6 +79,7 @@ Public types in the `E2E.Engine` namespace. | [LocatorAction.Hover](/api/E2E.Engine/LocatorAction.Hover/) | Moves the pointer over the node. | | [LocatorAction.LongPress](/api/E2E.Engine/LocatorAction.LongPress/) | A press held on the node for Duration, or for the engine's own default hold when it is null. | | [LocatorAction.Press](/api/E2E.Engine/LocatorAction.Press/) | | +| [LocatorAction.PressSequentially](/api/E2E.Engine/LocatorAction.PressSequentially/) | Focuses the node and types Text one character at a time. | | [LocatorAction.ScrollIntoView](/api/E2E.Engine/LocatorAction.ScrollIntoView/) | Scrolls the node's nearest scrollable container until the node is in the viewport. | | [LocatorAction.SecondaryTap](/api/E2E.Engine/LocatorAction.SecondaryTap/) | A right click, or the platform's equivalent. Modifiers are held for the click, as for LocatorAction.Tap. | | [LocatorAction.Select](/api/E2E.Engine/LocatorAction.Select/) | Selects one option: the one whose value or label is Text, or the one Option names by its label, its value, or its index. Exactly one of the two is set. | diff --git a/docs-site/src/content/docs/api/E2E/ActOptions.md b/docs-site/src/content/docs/api/E2E/ActOptions.md index cc6dd3fcb..c95d8f590 100644 --- a/docs-site/src/content/docs/api/E2E/ActOptions.md +++ b/docs-site/src/content/docs/api/E2E/ActOptions.md @@ -15,7 +15,7 @@ Namespace: [E2E](/api/E2E/) ```csharp -public sealed class ActOptions +public class ActOptions ``` @@ -23,6 +23,8 @@ Values passed into one `ActAsync` goal. Put fresh data in [Unique(string)](/api/ **Inheritance:** [object](https://learn.microsoft.com/dotnet/api/system.object) +**Derived:** [CachingActOptions](/api/E2E/CachingActOptions/) + [View source](https://github.com/hardkoded/e2e-dotnet/blob/main/src/E2E/Agent/Agent.cs) ## Properties diff --git a/docs-site/src/content/docs/api/E2E/AgentCacheOptions.md b/docs-site/src/content/docs/api/E2E/AgentCacheOptions.md new file mode 100644 index 000000000..99a8e73e7 --- /dev/null +++ b/docs-site/src/content/docs/api/E2E/AgentCacheOptions.md @@ -0,0 +1,88 @@ +--- +title: "AgentCacheOptions" +description: "Where ActAsync keeps the flows it recorded, and whether it may write new ones. Upstream's AgentCacheOptions (core/agent.ts)." +editUrl: false +prev: false +next: false +sidebar: + label: "AgentCacheOptions" + hidden: true +--- + + + +Namespace: [E2E](/api/E2E/) + + +```csharp +public sealed class AgentCacheOptions +``` + + +Where `ActAsync` keeps the flows it recorded, and whether it may write new ones. Upstream's `AgentCacheOptions` (`core/agent.ts`). + +**Inheritance:** [object](https://learn.microsoft.com/dotnet/api/system.object) + +[View source](https://github.com/hardkoded/e2e-dotnet/blob/main/src/E2E/Core/CoreAgent.cs) + +## Properties + + + + +### Dir + + +```csharp +public string? Dir { get; init; } +``` + + +The file store's directory, from the [Cwd](/api/E2E/LaunchOptions/#E2E_LaunchOptions_Cwd) the host was launched with; default `.e2e/cache`. Ignored with [Store](#E2E_AgentCacheOptions_Store). + +**Value:** [string](https://learn.microsoft.com/dotnet/api/system.string) + + + + +### Mode + + +```csharp +public CacheMode Mode { get; init; } +``` + + +[ReadWrite](/api/E2E/CacheMode/#E2E_CacheMode_ReadWrite) (the default) replays and records; [ReadOnly](/api/E2E/CacheMode/#E2E_CacheMode_ReadOnly) replays and never writes. [Off](/api/E2E/CacheMode/#E2E_CacheMode_Off) is `INVALID_ARGUMENT`: leave [Cache](/api/E2E/CreateAgentOptions/#E2E_CreateAgentOptions_Cache) unset. + +**Value:** [CacheMode](/api/E2E/CacheMode/) + + + + +### Store + + +```csharp +public IStepCache? Store { get; init; } +``` + + +A custom store in place of the file store, the seam a shared cache plugs into. + +**Value:** [IStepCache](/api/E2E/IStepCache/) + + + + +### Strict + + +```csharp +public bool Strict { get; init; } +``` + + +Replays only: a recording that no longer replays fails its step with `REPLAY_STALE` instead of calling the model, and nothing is written. + +**Value:** [bool](https://learn.microsoft.com/dotnet/api/system.boolean) diff --git a/docs-site/src/content/docs/api/E2E/AgentOptions.md b/docs-site/src/content/docs/api/E2E/AgentOptions.md index 1fa70bf46..e7ed1e706 100644 --- a/docs-site/src/content/docs/api/E2E/AgentOptions.md +++ b/docs-site/src/content/docs/api/E2E/AgentOptions.md @@ -15,7 +15,7 @@ Namespace: [E2E](/api/E2E/) ```csharp -public sealed class AgentOptions +public class AgentOptions ``` @@ -24,6 +24,8 @@ defaults; no agent inherits another's values. A call picks one with its `Agent` **Inheritance:** [object](https://learn.microsoft.com/dotnet/api/system.object) +**Derived:** [CreateAgentOptions](/api/E2E/CreateAgentOptions/) + [View source](https://github.com/hardkoded/e2e-dotnet/blob/main/src/E2E/Agent/AgentOptions.cs) ## Properties diff --git a/docs-site/src/content/docs/api/E2E/CachingActOptions.md b/docs-site/src/content/docs/api/E2E/CachingActOptions.md new file mode 100644 index 000000000..bcee9a5a0 --- /dev/null +++ b/docs-site/src/content/docs/api/E2E/CachingActOptions.md @@ -0,0 +1,61 @@ +--- +title: "CachingActOptions" +description: "ActOptions for ActAsync: each call names the recording it replays and records, or runs live. Upstream's CachingActOptions (core/agent.ts), which TypeScript t..." +editUrl: false +prev: false +next: false +sidebar: + label: "CachingActOptions" + hidden: true +--- + + + +Namespace: [E2E](/api/E2E/) + + +```csharp +public sealed class CachingActOptions : ActOptions +``` + + +[ActOptions](/api/E2E/ActOptions/) for `ActAsync`: each call names the recording it replays and records, or runs live. Upstream's +`CachingActOptions` (`core/agent.ts`), which TypeScript types as a union; C# holds both members, and the agent refuses the one that +does not fit it at runtime. [Agent](/api/E2E/ActOptions/#E2E_ActOptions_Agent) is `INVALID_ARGUMENT`: create another agent for another configuration. + +**Inheritance:** [object](https://learn.microsoft.com/dotnet/api/system.object) → [ActOptions](/api/E2E/ActOptions/) + +[View source](https://github.com/hardkoded/e2e-dotnet/blob/main/src/E2E/Core/CoreAgent.cs) + +## Properties + + + + +### Cache + + +```csharp +public bool? Cache { get; init; } +``` + + +`false` runs this call live, neither replaying nor recording. Any other value is `INVALID_ARGUMENT`. + +**Value:** [bool?](https://learn.microsoft.com/dotnet/api/system.boolean) + + + + +### CacheKey + + +```csharp +public string? CacheKey { get; init; } +``` + + +The recording this call replays and records, unique within what the program does: `checkout`, `cart-best-coupon`. The key is +this name, the instruction, its params, and how many times the session made the same call, so a reworded instruction records afresh. + +**Value:** [string](https://learn.microsoft.com/dotnet/api/system.string) diff --git a/docs-site/src/content/docs/api/E2E/CoreAgent.md b/docs-site/src/content/docs/api/E2E/CoreAgent.md new file mode 100644 index 000000000..961c43c07 --- /dev/null +++ b/docs-site/src/content/docs/api/E2E/CoreAgent.md @@ -0,0 +1,131 @@ +--- +title: "CoreAgent" +description: "An agent over one session, configured in code: ActAsync, AssertAsync, WaitForAsync, and ExtractAsync behave as they do in a test. With a cache, ActAsync repl..." +editUrl: false +prev: false +next: false +sidebar: + label: "CoreAgent" + hidden: true +--- + + + +Namespace: [E2E](/api/E2E/) + + +```csharp +public sealed class CoreAgent +``` + + +An agent over one session, configured in code: `ActAsync`, `AssertAsync`, `WaitForAsync`, and `ExtractAsync` behave as they +do in a test. With a cache, `ActAsync` replays recorded flows, each keyed by the [CacheKey](/api/E2E/CachingActOptions/#E2E_CachingActOptions_CacheKey) the call names. +Upstream's `CachingAgent`, `CoreAgent`, and `UncheckedAgent` (`core/agent.ts`), which TypeScript tells apart by the options +`createAgent` was called with: C# has one type, and refuses at runtime the `ActAsync` call that does not fit the agent. + +**Inheritance:** [object](https://learn.microsoft.com/dotnet/api/system.object) + +[View source](https://github.com/hardkoded/e2e-dotnet/blob/main/src/E2E/Core/CoreAgent.cs) + +## Methods + + + + +### ActAsync(string, CachingActOptions?, CancellationToken) + + +```csharp +public Task ActAsync(string instruction, CachingActOptions? options = null, CancellationToken cancellationToken = default) +``` + + +Runs one goal. On an agent with a cache the call names its recording with [CacheKey](/api/E2E/CachingActOptions/#E2E_CachingActOptions_CacheKey), or runs live with +[Cache](/api/E2E/CachingActOptions/#E2E_CachingActOptions_Cache) set to `false`; without one it fails with `INVALID_ARGUMENT`. On an agent without a cache, +naming either fails with `INVALID_ARGUMENT`. + +**Parameters** + +| Name | Type | Description | +| --- | --- | --- | +| `instruction` | [string](https://learn.microsoft.com/dotnet/api/system.string) | | +| `options` | [CachingActOptions](/api/E2E/CachingActOptions/) | | +| `cancellationToken` | [CancellationToken](https://learn.microsoft.com/dotnet/api/system.threading.cancellationtoken) | | + +**Returns:** [Task<ActResult>](https://learn.microsoft.com/dotnet/api/system.threading.tasks.task-1) + + + + +### AssertAsync(string, AssertOptions?, CancellationToken) + + +```csharp +public Task AssertAsync(string statement, AssertOptions? options = null, CancellationToken cancellationToken = default) +``` + + +Checks that the statement holds on the current screen. See `AssertAsync`. + +**Parameters** + +| Name | Type | Description | +| --- | --- | --- | +| `statement` | [string](https://learn.microsoft.com/dotnet/api/system.string) | | +| `options` | [AssertOptions](/api/E2E/AssertOptions/) | | +| `cancellationToken` | [CancellationToken](https://learn.microsoft.com/dotnet/api/system.threading.cancellationtoken) | | + +**Returns:** [Task](https://learn.microsoft.com/dotnet/api/system.threading.tasks.task) + + + + +### `ExtractAsync(string, ExtractOptions?, CancellationToken)` + + +```csharp +public Task ExtractAsync(string instruction, ExtractOptions? options = null, CancellationToken cancellationToken = default) +``` + + +Reads data off the current screen. See `ExtractAsync<T>`. + +**Type parameters** + +| Name | Description | +| --- | --- | +| `T` | | + +**Parameters** + +| Name | Type | Description | +| --- | --- | --- | +| `instruction` | [string](https://learn.microsoft.com/dotnet/api/system.string) | | +| `options` | [ExtractOptions](/api/E2E/ExtractOptions/) | | +| `cancellationToken` | [CancellationToken](https://learn.microsoft.com/dotnet/api/system.threading.cancellationtoken) | | + +**Returns:** [Task<T>](https://learn.microsoft.com/dotnet/api/system.threading.tasks.task-1) + + + + +### WaitForAsync(string, WaitForOptions?, CancellationToken) + + +```csharp +public Task WaitForAsync(string statement, WaitForOptions? options = null, CancellationToken cancellationToken = default) +``` + + +Waits until the statement holds on screen. See `WaitForAsync`. + +**Parameters** + +| Name | Type | Description | +| --- | --- | --- | +| `statement` | [string](https://learn.microsoft.com/dotnet/api/system.string) | | +| `options` | [WaitForOptions](/api/E2E/WaitForOptions/) | | +| `cancellationToken` | [CancellationToken](https://learn.microsoft.com/dotnet/api/system.threading.cancellationtoken) | | + +**Returns:** [Task](https://learn.microsoft.com/dotnet/api/system.threading.tasks.task) diff --git a/docs-site/src/content/docs/api/E2E/CoreAgents.md b/docs-site/src/content/docs/api/E2E/CoreAgents.md new file mode 100644 index 000000000..a6eb4c2b9 --- /dev/null +++ b/docs-site/src/content/docs/api/E2E/CoreAgents.md @@ -0,0 +1,52 @@ +--- +title: "CoreAgents" +description: "createAgent from @e2e-dev/core/agent: the agent over one session, configured in code. Upstream's core/agent.ts." +editUrl: false +prev: false +next: false +sidebar: + label: "CoreAgents" + hidden: true +--- + + + +Namespace: [E2E](/api/E2E/) + + +```csharp +public static class CoreAgents +``` + + +`createAgent` from `@e2e-dev/core/agent`: the agent over one session, configured in code. Upstream's `core/agent.ts`. + +**Inheritance:** [object](https://learn.microsoft.com/dotnet/api/system.object) + +[View source](https://github.com/hardkoded/e2e-dotnet/blob/main/src/E2E/Core/CoreAgent.cs) + +## Methods + + + + +### CreateAgent(Session, CreateAgentOptions) + + +```csharp +public static CoreAgent CreateAgent(Session session, CreateAgentOptions options) +``` + + +Creates an agent over <code class="paramref">session</code>. With [Cache](/api/E2E/CreateAgentOptions/#E2E_CreateAgentOptions_Cache), its `ActAsync` replays flows, +each named by its [CacheKey](/api/E2E/CachingActOptions/#E2E_CachingActOptions_CacheKey), and records them when the session closes. A missing model fails the first call that +needs one as `MODEL_UNAVAILABLE`, never the creation. + +**Parameters** + +| Name | Type | Description | +| --- | --- | --- | +| `session` | [Session](/api/E2E/Session/) | | +| `options` | [CreateAgentOptions](/api/E2E/CreateAgentOptions/) | | + +**Returns:** [CoreAgent](/api/E2E/CoreAgent/) diff --git a/docs-site/src/content/docs/api/E2E/CreateAgentOptions.md b/docs-site/src/content/docs/api/E2E/CreateAgentOptions.md new file mode 100644 index 000000000..6acd01b06 --- /dev/null +++ b/docs-site/src/content/docs/api/E2E/CreateAgentOptions.md @@ -0,0 +1,43 @@ +--- +title: "CreateAgentOptions" +description: "The agent's model, budgets, and context, as an agents entry of a config takes them (AgentOptions), plus what only a program sets. Upstream's CreateAgentOptio..." +editUrl: false +prev: false +next: false +sidebar: + label: "CreateAgentOptions" + hidden: true +--- + + + +Namespace: [E2E](/api/E2E/) + + +```csharp +public sealed class CreateAgentOptions : AgentOptions +``` + + +The agent's model, budgets, and context, as an `agents` entry of a config takes them ([AgentOptions](/api/E2E/AgentOptions/)), plus what only a program sets. Upstream's `CreateAgentOptions` (`core/agent.ts`). + +**Inheritance:** [object](https://learn.microsoft.com/dotnet/api/system.object) → [AgentOptions](/api/E2E/AgentOptions/) + +[View source](https://github.com/hardkoded/e2e-dotnet/blob/main/src/E2E/Core/CoreAgent.cs) + +## Properties + + + + +### Cache + + +```csharp +public AgentCacheOptions? Cache { get; init; } +``` + + +Replays recorded `ActAsync` flows; every `ActAsync` then names its recording with [CacheKey](/api/E2E/CachingActOptions/#E2E_CachingActOptions_CacheKey). Unset, every `ActAsync` calls the model. + +**Value:** [AgentCacheOptions](/api/E2E/AgentCacheOptions/) diff --git a/docs-site/src/content/docs/api/E2E/E2ESessionOptions.md b/docs-site/src/content/docs/api/E2E/E2ESessionOptions.md index 105ec33ce..85d6e211e 100644 --- a/docs-site/src/content/docs/api/E2E/E2ESessionOptions.md +++ b/docs-site/src/content/docs/api/E2E/E2ESessionOptions.md @@ -424,7 +424,7 @@ public string? LinkBase { get; init; } ``` -The target's `app.linkBase`, which `App.OpenAsync(path)` joins a path to on a device. It applies only to a pinned app with no [BaseUrl](/api/E2E/E2ESessionOptions/#E2E_E2ESessionOptions_BaseUrl); with a base URL a path resolves against that URL. +The target's `app.linkBase`, which `App.OpenAsync(path)` joins a path to on a device. It applies only to a pinned app with no [BaseUrl](#E2E_E2ESessionOptions_BaseUrl); with a base URL a path resolves against that URL. **Value:** [string](https://learn.microsoft.com/dotnet/api/system.string) diff --git a/docs-site/src/content/docs/api/E2E/LaunchOptions.md b/docs-site/src/content/docs/api/E2E/LaunchOptions.md index e17b00e30..8aa6f2ffc 100644 --- a/docs-site/src/content/docs/api/E2E/LaunchOptions.md +++ b/docs-site/src/content/docs/api/E2E/LaunchOptions.md @@ -147,7 +147,8 @@ public TimeSpan? Timeout { get; init; } ``` -Default timeout of one operation: an action, a wait, a read. Without it an action gets 30 seconds. An agent step has its own, `ActAsync`'s -timeout. There is no session deadline. +Default timeout of one operation: an action, a wait, a read, and `App.OpenAsync` or `App.RestartAsync`. Without it an action, a wait, +or a read gets 30 seconds, and an app open or restart 120 seconds. A session changes it with [SetDefaultTimeout(TimeSpan)](/api/E2E/Session/#E2E_Session_SetDefaultTimeout_System_TimeSpan_). An agent +step has its own, `ActAsync`'s timeout. There is no session deadline. **Value:** [TimeSpan?](https://learn.microsoft.com/dotnet/api/system.timespan) diff --git a/docs-site/src/content/docs/api/E2E/ModelUsage.md b/docs-site/src/content/docs/api/E2E/ModelUsage.md new file mode 100644 index 000000000..e78bad60b --- /dev/null +++ b/docs-site/src/content/docs/api/E2E/ModelUsage.md @@ -0,0 +1,97 @@ +--- +title: "ModelUsage" +description: "ModelUsage in E2E." +editUrl: false +prev: false +next: false +sidebar: + label: "ModelUsage" + hidden: true +--- + + + +Namespace: [E2E](/api/E2E/) + + +```csharp +public sealed class ModelUsage +``` + + +**Inheritance:** [object](https://learn.microsoft.com/dotnet/api/system.object) + +[View source](https://github.com/hardkoded/e2e-dotnet/blob/main/src/E2E/Agent/Model.cs) + +## Properties + + + + +### CacheReadTokens + + +```csharp +public int? CacheReadTokens { get; init; } +``` + + +Input tokens the provider served from its prompt cache, when it reports the split. A part of [InputTokens](#E2E_ModelUsage_InputTokens). + +**Value:** [int?](https://learn.microsoft.com/dotnet/api/system.int32) + + + + +### CacheWriteTokens + + +```csharp +public int? CacheWriteTokens { get; init; } +``` + + +Input tokens the provider wrote to its prompt cache, when it reports the split. A part of [InputTokens](#E2E_ModelUsage_InputTokens). + +**Value:** [int?](https://learn.microsoft.com/dotnet/api/system.int32) + + + + +### EstimatedCostUsd + + +```csharp +public double? EstimatedCostUsd { get; init; } +``` + + +The billed cost of the call in US dollars, when the provider reports one. + +**Value:** [double?](https://learn.microsoft.com/dotnet/api/system.double) + + + + +### InputTokens + + +```csharp +public int InputTokens { get; init; } +``` + + +**Value:** [int](https://learn.microsoft.com/dotnet/api/system.int32) + + + + +### OutputTokens + + +```csharp +public int OutputTokens { get; init; } +``` + + +**Value:** [int](https://learn.microsoft.com/dotnet/api/system.int32) diff --git a/docs-site/src/content/docs/api/E2E/ScriptedModel.md b/docs-site/src/content/docs/api/E2E/ScriptedModel.md index 069c8036f..c2766ca71 100644 --- a/docs-site/src/content/docs/api/E2E/ScriptedModel.md +++ b/docs-site/src/content/docs/api/E2E/ScriptedModel.md @@ -29,7 +29,7 @@ and the judge may answer a judgment (a request with a [Schema](/api/E2E/ModelReq **Implements:** [IAgentModel](/api/E2E/IAgentModel/) -[View source](https://github.com/hardkoded/e2e-dotnet/blob/main/src/E2E/Agent/Model.cs) +[View source](https://github.com/hardkoded/e2e-dotnet/blob/main/src/E2E/obj/Release/net10.0/System.Text.RegularExpressions.Generator/System.Text.RegularExpressions.Generator.RegexGenerator/RegexGenerator.g.cs) ## Constructors diff --git a/docs-site/src/content/docs/api/E2E/Session.md b/docs-site/src/content/docs/api/E2E/Session.md index fcd7be457..92a9848b8 100644 --- a/docs-site/src/content/docs/api/E2E/Session.md +++ b/docs-site/src/content/docs/api/E2E/Session.md @@ -142,3 +142,23 @@ Captures the session's state (cookies, storage, app data) in memory. Persist it | `cancellationToken` | [CancellationToken](https://learn.microsoft.com/dotnet/api/system.threading.cancellationtoken) | | **Returns:** [Task<SessionState>](https://learn.microsoft.com/dotnet/api/system.threading.tasks.task-1) + + + + +### SetDefaultTimeout(TimeSpan) + + +```csharp +public void SetDefaultTimeout(TimeSpan timeout) +``` + + +The default timeout of operations started from now on: actions, waits, reads, and app opens and restarts. A running operation keeps the +deadline it started with. Throws `INVALID_ARGUMENT` unless <code class="paramref">timeout</code> is positive and finite. Upstream's `setDefaultTimeout`. + +**Parameters** + +| Name | Type | Description | +| --- | --- | --- | +| `timeout` | [TimeSpan](https://learn.microsoft.com/dotnet/api/system.timespan) | | diff --git a/docs-site/src/content/docs/api/E2E/index.md b/docs-site/src/content/docs/api/E2E/index.md index 7ba0819c7..5549a6a11 100644 --- a/docs-site/src/content/docs/api/E2E/index.md +++ b/docs-site/src/content/docs/api/E2E/index.md @@ -20,6 +20,7 @@ The core. Agent, Screen, Expect, the engines, model clients, config, and the rep | [ActResult](/api/E2E/ActResult/) | | | [Agent](/api/E2E/Agent/) | One goal (ActAsync) or a check against the current screen. | | [AgentAttribute](/api/E2E/AgentAttribute/) | Pins the agent that the calls of a test, or of every test in a class, use when they name none. Upstream's agent option of test and describe. The closest pin wins: a call's Agent, then the method, then the class, then an enclosing or base class, then the E2E_AGENT environment variable. Base and enclosing classes count although the attribute is not inherited: TestAgentOptions reads each layer itself. An unknown name fails the test with INVALID_ARGUMENT. | +| [AgentCacheOptions](/api/E2E/AgentCacheOptions/) | Where ActAsync keeps the flows it recorded, and whether it may write new ones. Upstream's AgentCacheOptions (core/agent.ts). | | [AgentConfig](/api/E2E/AgentConfig/) | One entry of agents. Upstream model and judge are AI SDK instances; here they are model ids for Provider, which share BaseUrl and ApiKeyEnv. Each entry starts from the defaults; none inherits another's values. | | [AgentContextAttribute](/api/E2E/AgentContextAttribute/) | Appends facts only this test knows to the context of every agent, after the agent's own context. Upstream's agentContext option of test and describe. Layers join outer to inner, one per line. Keep instructions out of it: it reaches the judges too. | | [AgentException](/api/E2E/AgentException/) | An agent step failed, was blocked, or could not talk to its model. | @@ -47,9 +48,13 @@ The core. Agent, Screen, Expect, the engines, model clients, config, and the rep | [CacheEntry](/api/E2E/CacheEntry/) | A recording of one verified act. Its file name is the hash of a trace cache key (TraceCacheKeyHash): the cache schema, the replay policy version, the project, the test, the target, the engine's name and contract version, the app's identity, the instruction's and params' digests, the agent's name, a digest of the agent's redacted context, and which repeat of that same call in the attempt it is. The model id and the engine's package version are not part of the key. | | [CacheInfo](/api/E2E/CacheInfo/) | | | [CacheLookup](/api/E2E/CacheLookup/) | | +| [CachingActOptions](/api/E2E/CachingActOptions/) | ActOptions for ActAsync: each call names the recording it replays and records, or runs live. Upstream's CachingActOptions (core/agent.ts), which TypeScript types as a union; C# holds both members, and the agent refuses the one that does not fit it at runtime. Agent is INVALID_ARGUMENT: create another agent for another configuration. | | [ClickOptions](/api/E2E/ClickOptions/) | Options for TapAsync, DoubleTapAsync, and SecondaryTapAsync. Upstream's ClickOptions. | | [CommandConfig](/api/E2E/CommandConfig/) | The command a target's app.command starts the app with (upstream's CommandConfig). It is a program and its arguments, never a shell line: the executable is started directly and each argument is passed as it is. | | [ConfigurationException](/api/E2E/ConfigurationException/) | e2e.config.json is invalid. The code is INVALID_CONFIG. | +| [CoreAgent](/api/E2E/CoreAgent/) | An agent over one session, configured in code: ActAsync, AssertAsync, WaitForAsync, and ExtractAsync behave as they do in a test. With a cache, ActAsync replays recorded flows, each keyed by the CacheKey the call names. Upstream's CachingAgent, CoreAgent, and UncheckedAgent (core/agent.ts), which TypeScript tells apart by the options createAgent was called with: C# has one type, and refuses at runtime the ActAsync call that does not fit the agent. | +| [CoreAgents](/api/E2E/CoreAgents/) | createAgent from @e2e-dev/core/agent: the agent over one session, configured in code. Upstream's core/agent.ts. | +| [CreateAgentOptions](/api/E2E/CreateAgentOptions/) | The agent's model, budgets, and context, as an agents entry of a config takes them (AgentOptions), plus what only a program sets. Upstream's CreateAgentOptions (core/agent.ts). | | [Credentials](/api/E2E/Credentials/) | The accounts of the config's credentials (upstream's credentials, secrets.ts): E2E_USER_{NAME}_USERNAME and E2E_USER_{NAME}_PASSWORD replace the config's values. While a session runs, only a configured name resolves. | | [DefaultAgent](/api/E2E/DefaultAgent/) | Upstream's createBuiltInAgent (agent/default-agent.ts): the tool loop with the built-in rules, tools, and opening prompt. | | [DragAtActionEvent](/api/E2E/DragAtActionEvent/) | A dragAt the step dispatched. Mobile engines have no dragAt. | @@ -119,6 +124,7 @@ The core. Agent, Screen, Expect, the engines, model clients, config, and the rep | [ModelTool](/api/E2E/ModelTool/) | | | [ModelToolCall](/api/E2E/ModelToolCall/) | | | [ModelToolChoice](/api/E2E/ModelToolChoice/) | The AI SDK's ToolChoice: auto lets the model answer in prose or call any offered tool, required makes it call some tool, and a named tool makes it call that one. Each provider sends its own form of it (Anthropic's any, Bedrock's any, OpenAI's required, Gemini's ANY). | +| [ModelUsage](/api/E2E/ModelUsage/) | | | [ModelWarning](/api/E2E/ModelWarning/) | One of upstream's call warnings (CallWarning): the provider changed the request instead of failing it. A provider that sends auto in place of the forced tool choice it was asked for says so with Type unsupported and Feature toolChoice. | | [NewSessionOptions](/api/E2E/NewSessionOptions/) | What NewSessionAsync is given. Upstream's NewSessionOptions (core/launch.ts). | | [OpenAiCompatibleModel](/api/E2E/OpenAiCompatibleModel/) | Chat-completions client for OpenAI and any server that speaks the same tool-call JSON. Set BaseUrl to a local server such as http://127.0.0.1:11434/v1. The API key is optional for servers that do not check one. ModelProviders has presets for OpenRouter, the Vercel AI Gateway, SpaceXAI, and Ollama. | diff --git a/docs-site/src/content/docs/docs/core.mdx b/docs-site/src/content/docs/docs/core.mdx index 3a884de81..1268e0e2c 100644 --- a/docs-site/src/content/docs/docs/core.mdx +++ b/docs-site/src/content/docs/docs/core.mdx @@ -40,9 +40,67 @@ foreach (var sku in new[] { "A-1001", "B-2040" }) | Closing | `CloseAsync` is idempotent and throws an `AggregateException` of the failures closing found. | +## Let the agent act + +`CoreAgents.CreateAgent(session, options)` puts an agent over one session. Its `ActAsync`, `AssertAsync`, `WaitForAsync`, and `ExtractAsync` work as they do in a [test](/reference/agent). + +```csharp +await using var shop = await Launch.LaunchAsync(new WebEngine(), new LaunchOptions { App = new LaunchApp { Url = "https://shop.example.com" } }); +await using var session = await shop.NewSessionAsync(); +await session.App.OpenAsync("/products/A-1001"); + +var agent = CoreAgents.CreateAgent(session, new CreateAgentOptions +{ + Model = model, + Cache = new AgentCacheOptions { Dir = ".shop-bot/cache" }, +}); + +await agent.ActAsync("Add the item to the cart and apply the coupon with the largest discount", new CachingActOptions { CacheKey = "cart-best-coupon" }); +// A check after the step lets the cache keep its recording. +await session.Screen.GetByRole("button", "Pay now").WaitForAsync(); + +var order = await agent.ExtractAsync("the order total"); +if (order.Total <= 39_000) +{ + try + { + await session.Screen.GetByRole("button", "Pay now").TapAsync(); + } + catch (E2EException) + { + // A failed tap may still have reached the server: check the order list, never retry the payment. + Console.Error.WriteLine("check /my/orders"); + throw; + } +} +else +{ + Console.WriteLine($"the total, {order.Total}, is over the limit: the order stays unpaid"); +} + +record Order(int Total); +``` + +`CreateAgentOptions` takes the options of an [`agents` entry](/reference/config#agents) (`AgentOptions`), such as `Model`, `Judge`, `MaxSteps`, and `Context`, plus `Cache`. There is no default model: an agent without one fails its first call with `MODEL_UNAVAILABLE`. An `ActAsync` step has its own timeout, 120000 ms: set it per call with `Timeout`. A call takes no `Agent` option: it is `INVALID_ARGUMENT`, so create another agent for another configuration. A session that did not come from `Host.NewSessionAsync` is `INVALID_ARGUMENT` too. + +### Replay recorded steps + +With `Cache`, `ActAsync` records each flow and replays it on the next run with no model call. Every `ActAsync` then names its recording with `CachingActOptions.CacheKey`, or runs live with `Cache = false`. An `ActAsync` without one fails with `INVALID_ARGUMENT`, and so does a `CacheKey` on an agent created without `Cache`. C# has one `CoreAgent` type where TypeScript types the agent by the options it was created with, so the refusal is at runtime. + +A recording is keyed by the `CacheKey`, the instruction, its params, the app, and how many times the session made the same call. Pass a value that changes between runs as a param: a value the agent reads off the screen is decided by the model again on each run. + +A recording is saved when the session closes, or its host does, and only when a check passed after the step, such as `Locator.WaitForAsync()`, `AssertAsync`, or `WaitForAsync`. When an agent step fails, the recordings not yet confirmed at that moment are dropped. A failure of your own code's locator call, such as a `WaitForAsync` that ran out, drops nothing. A step that fails only because no model answered (`MODEL_UNAVAILABLE`, `MODEL_PROVIDER_FAILED`) leaves the recordings as they are. Read [Replay cache](/cache) for how a replay checks the screen and hands off to the model. + +| Option | Default | Meaning | +| --- | --- | --- | +| `Dir` | `.e2e/cache` | The directory, from the `Cwd` the host was launched with. | +| `Mode` | `ReadWrite` | `ReadOnly` replays and never writes. `Off` is `INVALID_ARGUMENT`: leave `Cache` unset. | +| `Strict` | `false` | A recording that no longer replays fails with `REPLAY_STALE`. | +| `Store` | - | A custom `IStepCache` in place of the files. | + ## Keep a signed-in state `session.SaveStateAsync()` captures a browser session's cookies and storage in memory. Close that session, then start another from the state with `NewSessionAsync(new NewSessionOptions { State = state })`. An engine with no state capability fails with `UNSUPPORTED_CAPABILITY`. diff --git a/skills/e2e-dotnet/SKILL.md b/skills/e2e-dotnet/SKILL.md index 475d6772a..de360cfee 100644 --- a/skills/e2e-dotnet/SKILL.md +++ b/skills/e2e-dotnet/SKILL.md @@ -36,7 +36,7 @@ trip. The installed tool prints the same text with `e2e guide ` | Running tests: `dotnet test` filters, the `e2e` tool (`e2e doctor` checks that `e2e explore` and `e2e mcp` can start; `--json` prints the `doctor-1` report for a host, `--check-model` calls each model once; `e2e cache ls|stats|clear` lists, sizes, or empties the replay cache), `report.json` for scripts, exit codes, CI, test filters (NUnit, xUnit v3, and TUnit, mobile and Windows adapters included: `E2ERunFixture.Filters` and `ChangedSince`, or `E2E_CHANGED_SINCE`; `run.changes`, `run.filters`, and `filtered` skips in `report.json`), `--last-failed` reruns (`E2ERunFixture.LastFailed` or `E2E_LAST_FAILED=1`; `run.carried` in `report.json`, `FinishedRun.LastRun` for reporters, folded into the GitHub comment) | `running` | [references/running.md](references/running.md) | | Handling a failed run: the trace page, `report.json` for scripts, `E2ESession.Report()` for a session you run yourself, error codes and their fixes, a visible browser, replay misses, phase timings, the agent step table, every model call | `debugging` | [references/debugging.md](references/debugging.md) | | Exploring an app toward a goal without a test file, with `e2e explore` or `Explore.ExploreAsync` | `explore` | [references/explore.md](references/explore.md) | -| Driving an app from plain code with no test framework (`Launch.LaunchAsync(engine, new LaunchOptions { App, Secrets, Headed, Timeout, ArtifactsDir, Cwd })`, `Host.NewSessionAsync`, `Host.Secret`, `Session.SaveStateAsync`, and `SessionState.ExportAsync`/`ImportAsync` with a 32-byte key you keep), or running isolated per-test sessions from your own runner: `E2EHost.OpenAsync`, `OpenHostOptions` (config value or file, target, `EngineFactory`), `host.OpenSessionAsync`, `E2ESession.CloseAsync` and the failures it returns, `Steps()` | `host` | [references/host.md](references/host.md) | +| Driving an app from plain code with no test framework (`Launch.LaunchAsync(engine, new LaunchOptions { App, Secrets, Headed, Timeout, ArtifactsDir, Cwd })`, `Host.NewSessionAsync`, `Host.Secret`, `Session.SaveStateAsync`, `SessionState.ExportAsync`/`ImportAsync` with a 32-byte key you keep, and `CoreAgents.CreateAgent(session, CreateAgentOptions)` with `CachingActOptions.CacheKey` for an agent over a session), or running isolated per-test sessions from your own runner: `E2EHost.OpenAsync`, `OpenHostOptions` (config value or file, target, `EngineFactory`), `host.OpenSessionAsync`, `E2ESession.CloseAsync` and the failures it returns, `Steps()` | `host` | [references/host.md](references/host.md) | | Testing an iOS or Android app: `MobileEngine`, the `Device` fixture, picking a simulator (without `MobileOptions.Device`, `MobileEngine` skips booted devices another agent-device session holds, and `prepare` fails with `ENGINE_FAILURE` when every booted one is held, and with none booted agent-device boots a shut-down one: on iOS an iPhone over an iPad over an Apple TV, on Android a phone or tablet over a TV, then newest OS, then name order, and a reachable paired iPhone or iPad counts as booted; `prepare` logs a per-device timing line), video of the device screen, `Device.SetPermissionAsync` (the enums, or wire-name strings such as `"camera"` and `"grant"`; an unknown name or state is `INVALID_ARGUMENT` before any device command), `Device.UnstableRaw` (the test's agent-device session: name, device id and name, selection, and a raw `IDeviceClient` whose calls are not steps, replayed, or redacted, and that throws `NotSupportedException` for the `text-size` and `animations` settings, `apps.uninstall`, and `command.hardwareButton`; unstable; `INVALID_STATE` after the session is disposed; a custom agent tool gets the running attempt's from `MobileTools.UnstableDeviceSession(engines)`), a NativeScript `testID` or `accessibilityIdentifier` as the test id `GetByTestId` reads (an Android `accessibilityLabel` is put before the text), Limrun and BrowserStack hosted devices, a MAUI app through the DevFlow agent, known limits, `Locator.PressAsync("Enter")` tapping an unfocused field first, and the agent's `open_app` returning to the app under test when it names no app, a Compose Multiplatform field on iOS (for an empty (no value), editable, non-secure iOS field that reports no placeholder of its own, whose only child is a `StaticText` with no children, and whose label is a non-empty name, then `, `, then that child's text, that text is the field's `Placeholder`, the child is left out, and the name is the label before `, `; a non-empty `ContentDescription` stays the name, and that label is then the `Text`; any other field keeps its label and children), and `FillAsync` and `ClearAsync` refusing a node that is not an editable field with `UNSUPPORTED_CAPABILITY` (an iOS `other` that holds a value or only text is typed into), reading the field again before typing (a field the new capture no longer lists, an incomplete capture, or a changed input purpose or secure, disabled, or hidden state is a retryable `NODE_STALE` before anything is typed), looking at the screen again after the keyboard opens or closes (an unmoved control is acted on at once only when keyboard visibility is unchanged, else it waits out `TransitionMilliseconds`; a test's screen swipe waits out the remaining budget before it scrolls), and a `target_still_moving` refusal with `dispatched: no` being a retryable `NODE_STALE` | `mobile` | [references/mobile.md](references/mobile.md) | | Testing a Windows desktop app: `WinAppEngine`, `WinAppTest`, the `Window` fixture, the `windows` target in `e2e explore` and `e2e mcp`, the interactive-desktop requirement, known limits | `windows` | [references/windows.md](references/windows.md) | | Asked to bug bash or hunt for bugs: the bug-bash config, many `e2e explore` runs at once, proving each bug with a repro test | `bug-bash` | [references/bug-bash.md](references/bug-bash.md) | diff --git a/skills/e2e-dotnet/references/host.md b/skills/e2e-dotnet/references/host.md index 604d06c3b..a46f8ae25 100644 --- a/skills/e2e-dotnet/references/host.md +++ b/skills/e2e-dotnet/references/host.md @@ -100,4 +100,6 @@ await session.Screen.GetByLabel("Password").FillAsync(shop.Secret("SHOP_PASSWORD - `LaunchOptions.Timeout` bounds an action, a wait, a read, and `App.OpenAsync()` or `App.RestartAsync()` (30 seconds for the first three and 120 seconds for the last two without it). `session.SetDefaultTimeout(timeout)` moves all of them for the operations that start from then on; a zero or infinite span is `INVALID_ARGUMENT`. There is no session deadline. `Headed` shows the browser window (a device ignores it). An operation on a closed session fails with `CANCELLED` and the message `the session is closed`. `ArtifactsDir` keeps screenshots and downloads (a relative path resolves from `Cwd`); without it they go to a temp directory deleted when the host closes. - `await session.SaveStateAsync()` captures cookies and storage in memory. Pass it to `shop.NewSessionAsync(new NewSessionOptions { State = state })`. To keep it between runs, `state.ExportAsync(new SessionStateKeyOptions { Key = key })` with a 32-byte key you keep, and `SessionState.ImportAsync(bytes, options)`. A state restores only on the same engine, platform, and app (`SESSION_MISMATCH`; give two addresses of one app the same `LaunchApp.Identity`), a wrong key or changed byte is `SESSION_INVALID`, and an old one `SESSION_EXPIRED`. - `LaunchApp.LinkBase` (an absolute URL or a scheme such as `shopapp://`, with no query or fragment) makes `session.App.OpenAsync("/orders/42")` open `shopapp://orders/42` in the pinned app of a device engine. An absolute link opens as given; `file:`, `data:`, `javascript:`, `view-source:`, `blob:`, and `filesystem:` are `POLICY_DENIED`; an engine without a link hook, or a bad value, is `INVALID_CONFIG` at launch. -- Not ported yet: closing on signals and `createAgent` (issues 1676 to 1678). +- `CoreAgents.CreateAgent(session, new CreateAgentOptions { Model = model, Cache = new AgentCacheOptions { Dir = ".cache" } })` puts an agent over a session: `ActAsync`, `AssertAsync`, `WaitForAsync`, and `ExtractAsync` as in a test. `CreateAgentOptions` is an `AgentOptions` plus `Cache` (`Dir`, `Mode`, `Store`, `Strict`). No model fails the first call with `MODEL_UNAVAILABLE`; `ActAsync` takes no `Agent` option (`INVALID_ARGUMENT`). +- With `Cache`, each `ActAsync` names its recording, `new CachingActOptions { CacheKey = "checkout" }`, or runs live with `Cache = false`; without either, or with a `CacheKey` on an agent without a cache, it is `INVALID_ARGUMENT`. A recording is keyed by the `CacheKey`, the instruction, its params, the app, and how often the session made the same call. It is saved when the session or its host closes, and only if a check passed after it (`Locator.WaitForAsync`, `AssertAsync`, `WaitForAsync`). An agent step that fails drops the recordings not yet confirmed (not when only a missing model explains it); a failed locator call of your own code drops nothing. +- Not ported yet: closing on signals (issue 1677). diff --git a/src/E2E/Agent/Act.cs b/src/E2E/Agent/Act.cs index b948bb40c..6b433c75b 100644 --- a/src/E2E/Agent/Act.cs +++ b/src/E2E/Agent/Act.cs @@ -25,7 +25,7 @@ private Task RunActStepAsync(string instruction, ActOptions? options, ArgumentException.ThrowIfNullOrWhiteSpace(instruction); CheckInstruction(instruction); ActParams.Validate(options?.Params, "agent.act"); - var agent = _scope.Select(options?.Agent); + var agent = Select(options?.Agent); return StepAsync( "agent.act", instruction, @@ -188,20 +188,20 @@ private async Task DispatchCoreAsync(StepSpec spec, ResolvedAgent age try { // An act is cacheable unless its executor opted out. An assert must not change the app, so its recording would be empty. - if (spec.Kind == ExecutorStepKind.Act && executor.Cache != ExecutorCache.Off && _scope.CacheEnabled && _scope.Cache is { } cache) + if (spec.Kind == ExecutorStepKind.Act && executor.Cache != ExecutorCache.Off && (_binding is null ? _scope.ActCache() : _binding.Cache) is { } cache) { - var key = _scope.ClaimKey(instruction, spec.Params, agent, out var callIndex, out var keyContext); + var claimed = cache.ClaimKey(instruction, spec.Params, _scope.IdentityOf(agent)); session = new StepTraceSession( new ActReplayHost(_scope, feed, budget, dispatcher, token), new StepCacheOptions { - Cache = cache, + Cache = cache.Store, Replaying = active => budget.Replaying(active), - Write = _scope.CacheWrite, - Strict = _scope.CacheStrict, - StrictAdvice = _scope.CacheStrict ? _scope.StrictAdvice : "", - ReplayEligible = _scope.ReplayEligible, - Claim = new ClaimedKey(key, Provenance(instruction, spec.Params, callIndex, agent.Name, keyContext)), + Write = cache.Mode == CacheMode.ReadWrite, + Strict = cache.Strict is not null, + StrictAdvice = cache.Strict?.Advice ?? "", + ReplayEligible = cache.ReplayEligible, + Claim = new ClaimedKey(claimed.Key, Provenance(cache.TestId, instruction, spec.Params, claimed.CallIndex, agent.Name, claimed.Context)), StepIndex = _scope.Steps.CurrentStepIndex, Instruction = instruction, Params = spec.Params, @@ -209,6 +209,7 @@ private async Task DispatchCoreAsync(StepSpec spec, ResolvedAgent age AppOrigin = _scope.AppOrigin, }); _scope.Acts.Add(session.Pending); + cache.Staged?.Add(session.Pending); } // The baseline a recording's delta is read from, the screen a replay starts on, and the built-in agent's first look: @@ -520,12 +521,12 @@ private static void Settle(StepVerdict verdict, StepAccounting budget, StepSpec /// The step an entry is recorded for, as the entry stores it. None of it is replay input, so a /// secret value in the test title, instruction, or agent name is masked like any other recorded string. /// - private CacheEntry Provenance(string instruction, IReadOnlyDictionary? parameters, int callIndex, string agent, TraceKeyContext keyContext) + private CacheEntry Provenance(string testId, string instruction, IReadOnlyDictionary? parameters, int callIndex, string agent, TraceKeyContext keyContext) { return new CacheEntry { Schema = FileStepCache.SchemaVersion, - Test = TextRules.Bound(_scope.Redactor.Redact(_scope.TestTitle), Trace.MaxTraceDescriptorChars), + Test = TextRules.Bound(_scope.Redactor.Redact(testId), Trace.MaxTraceDescriptorChars), TargetId = TextRules.Bound(_scope.Redactor.Redact(_scope.TargetName), Trace.MaxTraceDescriptorChars), Instruction = _scope.Redactor.Redact(instruction.Trim()), Engine = _scope.EnginePlatform, diff --git a/src/E2E/Agent/Agent.cs b/src/E2E/Agent/Agent.cs index f84b13d1d..91a1f8836 100644 --- a/src/E2E/Agent/Agent.cs +++ b/src/E2E/Agent/Agent.cs @@ -71,12 +71,19 @@ public sealed partial class Agent /// The looks of the newest act. internal ObservationFeed Feed => _feed; - internal Agent(AttemptScope scope) + // The agent a program created over a core session, which brings its own configuration and replay cache; null for a test's agent. + private readonly CoreAgentBinding? _binding; + + internal Agent(AttemptScope scope, CoreAgentBinding? binding = null) { _scope = scope; + _binding = binding; _feed = new ObservationFeed(scope.Session, TimeProvider.System) { Scope = scope }; } + // Upstream's `select` of the agent context: the agent a call runs as. A core agent is the one it was created with. + private ResolvedAgent Select(string? name) => _binding is null ? _scope.Select(name) : _binding.Select(name); + public Task ActAsync(string instruction, ActOptions? options = null, CancellationToken cancellationToken = default) => _scope.Track(RunActStepAsync(instruction, options, cancellationToken)); @@ -88,7 +95,7 @@ public Task AssertAsync(string statement, AssertOptions? options = null, Cancell throw new TestException("INVALID_ARGUMENT", "assert budgets require Explore = true"); } - var agent = _scope.Select(options?.Agent); + var agent = Select(options?.Agent); // The brain that plans the flows judges the assertions too, so a custom executor takes the assert. So does an exploring one: // the judgment tier below looks once. if (agent.Executor is not null || options?.Explore == true) @@ -114,7 +121,7 @@ public Task WaitForAsync(string statement, WaitForOptions? options = null, Cance private async Task WaitForStepAsync(string statement, WaitForOptions? options, CancellationToken cancellationToken) { ArgumentException.ThrowIfNullOrWhiteSpace(statement); - var agent = _scope.Select(options?.Agent); + var agent = Select(options?.Agent); var parameters = JudgmentParams(options?.Params, "agent.waitFor"); await StepAsync("agent.waitFor", statement, agent, verifies: true, () => WaitForCoreAsync(statement, parameters, options, cancellationToken)).ConfigureAwait(false); } @@ -122,7 +129,7 @@ private async Task WaitForStepAsync(string statement, WaitForOptions? options, C private async Task WaitForCoreAsync(string statement, string? parameters, WaitForOptions? options, CancellationToken cancellationToken) { ArgumentException.ThrowIfNullOrWhiteSpace(statement); - var agent = _scope.Select(options?.Agent); + var agent = Select(options?.Agent); var interval = Interval(options?.Interval); var timeout = ResolveTimeout(options?.Timeout, agent.JudgmentTimeout); var maxCalls = Budget(options?.MaxModelCalls, agent.MaxModelCalls, "MaxModelCalls"); @@ -223,14 +230,14 @@ public Task ExtractAsync(string instruction, ExtractOptions? options = nul private async Task ExtractStepAsync(string instruction, ExtractOptions? options, CancellationToken cancellationToken) { ArgumentException.ThrowIfNullOrWhiteSpace(instruction); - var agent = _scope.Select(options?.Agent); + var agent = Select(options?.Agent); return await StepAsync("agent.extract", instruction, agent, verifies: false, () => ExtractCoreAsync(instruction, options, cancellationToken)).ConfigureAwait(false); } private async Task ExtractCoreAsync(string instruction, ExtractOptions? options, CancellationToken cancellationToken) { ArgumentException.ThrowIfNullOrWhiteSpace(instruction); - var agent = _scope.Select(options?.Agent); + var agent = Select(options?.Agent); var vision = ResolveVision(options?.Vision ?? VisionMode.Off); var extractTimeout = ResolveTimeout(options?.Timeout, agent.JudgmentTimeout); var guard = Guard("agent.extract", cancellationToken, extractTimeout); @@ -1055,7 +1062,7 @@ private CancellationTokenSource Link(CancellationToken cancellationToken, TimeSp } /// Values passed into one goal. Put fresh data in . -public sealed class ActOptions +public class ActOptions { public IReadOnlyDictionary? Params { get; init; } @@ -1337,7 +1344,6 @@ internal static class Vision internal sealed class AttemptScope : ISecretFillRuntime { - private readonly Dictionary> _callIndexers = new(StringComparer.Ordinal); private readonly List _secrets = []; private readonly List<(string Name, string Value)> _registered = []; private Built? _redactor; @@ -1735,30 +1741,30 @@ public void AttachArtifact(string kind, string relative) public string ClaimKey(string instruction, IReadOnlyDictionary? parameters, ResolvedAgent agent, out int callIndex, out TraceKeyContext context) { // The key reads the agent context as the model does, with the secrets known so far redacted. - var identity = new CacheAgentIdentity(agent.Name, agent.Context is null ? null : Redactor.Redact(agent.Context)); - var signature = Identity.TraceCallSignature(Identity.ActKind, instruction, CacheKeys.Project(parameters)); - - // One occurrence count per agent: the agent is part of the key, so another agent's call of the same instruction must not - // renumber this one's. Folded like the key folds it: no context is the empty one. - var agentKey = Ids.CanonicalJson(new JsonArray(identity.Name, identity.Context ?? "")); - if (!_callIndexers.TryGetValue(agentKey, out var indexer)) - { - _callIndexers[agentKey] = indexer = Identity.CreateCallIndexer(); - } - - callIndex = indexer(signature); - var key = Identity.BuildTraceCacheKey(new TraceCacheKeyParts( - Identity.ProjectIdentity(CacheProjectId), - TestTitle, - new CacheTargetIdentity(TargetName, EnginePlatform, EngineName, BuildReport.EngineSpiVersion, AppIdentity), - signature, - callIndex, - identity, - CacheKeys.ReplayPolicyVersion)); - context = Identity.KeyContext(key); - return Identity.TraceCacheKeyHash(key); + var claimed = Claimer()(instruction, parameters, IdentityOf(agent)); + callIndex = claimed.CallIndex; + context = claimed.Context; + return claimed.Key; } + /// The agent as a key sees it: its name and its context as the model reads it, with the secrets known so far redacted. + internal CacheAgentIdentity IdentityOf(ResolvedAgent agent) => new(agent.Name, agent.Context is null ? null : Redactor.Redact(agent.Context)); + + private Func?, CacheAgentIdentity, ClaimedTraceKey>? _claimKey; + + private Func?, CacheAgentIdentity, ClaimedTraceKey> Claimer() => + _claimKey ??= CacheContexts.KeyClaimer(CacheProjectId, TestTitle, new CacheTargetIdentity(TargetName, EnginePlatform, EngineName, BuildReport.EngineSpiVersion, AppIdentity)); + + /// The cache context of one act of this attempt, or null when its cache is off: the store and mode the attempt resolved, and its key claims. + internal AgentCacheContext? ActCache() => + CacheEnabled && Cache is { } store + ? new AgentCacheContext(CacheWrite ? CacheMode.ReadWrite : CacheMode.ReadOnly, store, ReplayEligible, CacheStrict ? new AgentCacheStrict(StrictAdvice) : null) + { + TestId = TestTitle, + ClaimKey = Claimer(), + } + : null; + /// /// Upstream's strict.advice of AgentCacheContext (cache/context.ts), built by CacheContexts.CreateAgentCacheContext: /// how to re-record a recording cache.strict failed on. Empty when the cache is not strict. diff --git a/src/E2E/Agent/AgentError.cs b/src/E2E/Agent/AgentError.cs index 442969bed..f0acad055 100644 --- a/src/E2E/Agent/AgentError.cs +++ b/src/E2E/Agent/AgentError.cs @@ -39,6 +39,9 @@ internal static class AgentError ["REPLAY_STALE"] = "configuration", }; + /// The codes that mean no model answered (upstream's MODEL_UNREACHABLE_CODES, derived from AGENT_CODE_TABLE). + public static IReadOnlySet ModelUnreachableCodes { get; } = new HashSet(StringComparer.Ordinal) { "MODEL_UNAVAILABLE", "MODEL_PROVIDER_FAILED" }; + /// /// Upstream's toAgentError: a failure on the closed agent code set. An agent error stays as it is. A runner or engine code the /// agent knows keeps it, UNSUPPORTED_CAPABILITY and INVALID_CONFIG are POLICY_DENIED, any other infrastructure diff --git a/src/E2E/Agent/AgentOptions.cs b/src/E2E/Agent/AgentOptions.cs index 905126606..2c54a1649 100644 --- a/src/E2E/Agent/AgentOptions.cs +++ b/src/E2E/Agent/AgentOptions.cs @@ -12,7 +12,7 @@ namespace E2E; /// One named agent, as upstream agents.<name>. Each entry starts from the built-in /// defaults; no agent inherits another's values. A call picks one with its Agent option. /// -public sealed class AgentOptions +public class AgentOptions { /// /// The model behind act. There is no implicit default. A custom that brought a model of its diff --git a/src/E2E/Cache/Context.cs b/src/E2E/Cache/Context.cs index 734722874..e2ebb6a41 100644 --- a/src/E2E/Cache/Context.cs +++ b/src/E2E/Cache/Context.cs @@ -14,10 +14,42 @@ namespace E2E; internal sealed record AgentCacheStrict(string Advice); /// -/// Upstream's AgentCacheContext (cache/context.ts), as far as the port has it: the resolved store, and whether the attempt -/// may replay. Upstream's claimKey and staged stay on AttemptScope and the step session. +/// Upstream's AgentCacheContext (cache/context.ts), as far as the port has it: the resolved store, whether the attempt +/// may replay, and, for a context a program created (), the claimKey that names its recordings and the +/// staged acts it settles. A test's attempt keeps both on AttemptScope, which builds its context per act. /// -internal sealed record AgentCacheContext(CacheMode Mode, IStepCache Store, bool ReplayEligible, AgentCacheStrict? Strict); +internal sealed record AgentCacheContext(CacheMode Mode, IStepCache Store, bool ReplayEligible, AgentCacheStrict? Strict) +{ + /// The test id its keys and entries carry: the test's title, or core:<cacheKey> for a program's agent. + public string TestId { get; init; } = ""; + + /// Upstream's claimKey: claims one act's key, advancing its occurrence count. + public Func?, CacheAgentIdentity, ClaimedTraceKey> ClaimKey { get; init; } = static (_, _, _) => throw new InvalidOperationException("the context claims no keys"); + + /// Upstream's staged: the acts this context's agent ran, which settle when the context's session ends. Null when the attempt settles them. + public List? Staged { get; init; } +} + +/// What claimed: the key's hash, the zero-based repeat of its signature, and the parts of the key outside the step. +internal sealed record ClaimedTraceKey(string Key, int CallIndex, TraceKeyContext Context); + +/// How an attempt ended, as the settlement of its staged entries reads it. Upstream's AttemptSettlement (cache/context.ts). +/// Entries staged before this step index were verified, and are confirmed. +/// +/// Whether the attempt implicates the entries it did not confirm. False when its failure is that no model answered, a verdict on the provider and +/// never on the app: unconfirmed entries are then left exactly as they are, neither written nor evicted. +/// +/// +/// Failures the attempt went on after, which only a program without a test around it does: each settles the entries it found unconfirmed as that +/// failure does, whatever passed later. Entries staged after it are judged by the checks that follow them. +/// +internal sealed record AttemptSettlement(int LastVerifiedStepIndex, bool ImplicatesUnconfirmed, IReadOnlyList Survived); + +/// One failed step an attempt went on after, and the staged entries it found unconfirmed. Upstream's SurvivedFailure (cache/context.ts). +/// The first step index no verification had confirmed when the step failed. +/// The failed step's index, the last the failure covers. +/// Whether the failure implicates those entries; see . +internal sealed record SurvivedFailure(int From, int Through, bool Implicates); /// Upstream's cache/context.ts: the per-attempt cache context. .NET-only name: Context would collide with E2ESession.Context. internal static class CacheContexts @@ -28,7 +60,7 @@ internal static class CacheContexts /// cache.strict is on, where every attempt replays, nothing is written or evicted, and the mode is . /// The port has no projectId, testId, or target parameter: AttemptScope claims the keys. /// - public static AgentCacheContext? CreateAgentCacheContext(CacheConfig cache, string projectRoot, int attemptIndex) + public static AgentCacheContext? CreateAgentCacheContext(CacheConfig cache, string projectRoot, int attemptIndex, string projectId = "", string testId = "", CacheTargetIdentity? target = null) { ArgumentNullException.ThrowIfNull(cache); if (cache.Mode == CacheMode.Off) @@ -43,7 +75,72 @@ internal static class CacheContexts mode, store, strict || attemptIndex == 0, - strict ? new AgentCacheStrict(StaleAdvice(cache, cache.StrictSource ?? new CacheStrictSource(true, false), projectRoot)) : null); + strict ? new AgentCacheStrict(StaleAdvice(cache, cache.StrictSource ?? new CacheStrictSource(true, false), projectRoot)) : null) + { + TestId = testId, + ClaimKey = target is null ? static (_, _, _) => throw new InvalidOperationException("the context claims no keys") : KeyClaimer(projectId, testId, target), + Staged = target is null ? null : [], + }; + } + + /// + /// Upstream's claimKey of createAgentCacheContext: one occurrence count per agent (the agent is part of the key, so another + /// agent's call of the same instruction must not renumber this one's), and the hash of the trace cache key the call claims. + /// + internal static Func?, CacheAgentIdentity, ClaimedTraceKey> KeyClaimer(string projectId, string testId, CacheTargetIdentity target) + { + var project = Identity.ProjectIdentity(projectId); + var indexers = new Dictionary>(StringComparer.Ordinal); + var gate = new Lock(); + return (instruction, parameters, agent) => + { + var signature = Identity.TraceCallSignature(Identity.ActKind, instruction, CacheKeys.Project(parameters)); + lock (gate) + { + // Folded like the key folds it: no context is the empty one. + var agentKey = Ids.CanonicalJson(new System.Text.Json.Nodes.JsonArray(agent.Name, agent.Context ?? "")); + if (!indexers.TryGetValue(agentKey, out var indexer)) + { + indexers[agentKey] = indexer = Identity.CreateCallIndexer(); + } + + var callIndex = indexer(signature); + var key = Identity.BuildTraceCacheKey(new TraceCacheKeyParts(project, testId, target, signature, callIndex, agent, CacheKeys.ReplayPolicyVersion)); + return new ClaimedTraceKey(Identity.TraceCacheKeyHash(key), callIndex, Identity.KeyContext(key)); + } + }; + } + + /// + /// Upstream's settleStagedTraces, for a context a program created: writes the acts the context staged that a check confirmed, and + /// evicts the ones a failure found unconfirmed. An act is confirmed when it ran before the last verification and no surviving failure covers + /// it. An unconfirmed one is evicted only when the failure covering it implicates the recording, or the settlement does; otherwise it is left + /// exactly as it was. Nothing settles in a read-only context, or without a settlement. + /// + public static void SettleStagedTraces(AgentCacheContext context, AttemptSettlement? settlement) + { + var staged = context.Staged; + if (staged is null) + { + return; + } + + var acts = new List(staged); + staged.Clear(); + if (settlement is null || context.Mode != CacheMode.ReadWrite) + { + return; + } + + var covers = new Dictionary(); + foreach (var act in acts) + { + var failure = act.StepIndex is { } index ? settlement.Survived.FirstOrDefault(survived => index >= survived.From && index <= survived.Through) : null; + covers[act] = failure; + act.Verified = failure is null && act.StepIndex is { } at && at < settlement.LastVerifiedStepIndex; + } + + E2ESession.Flush(context.Store, acts, act => !(covers[act]?.Implicates ?? settlement.ImplicatesUnconfirmed)); } /// diff --git a/src/E2E/Core/CoreAgent.cs b/src/E2E/Core/CoreAgent.cs new file mode 100644 index 000000000..b6db86084 --- /dev/null +++ b/src/E2E/Core/CoreAgent.cs @@ -0,0 +1,240 @@ +// Copyright 2026 TesterArmy. +// Modified by Dario Kondratiuk. +// SPDX-License-Identifier: Apache-2.0 + +using E2E.Engine; + +namespace E2E; + +/// Where keeps the flows it recorded, and whether it may write new ones. Upstream's AgentCacheOptions (core/agent.ts). +public sealed class AgentCacheOptions +{ + /// The file store's directory, from the the host was launched with; default .e2e/cache. Ignored with . + public string? Dir { get; init; } + + /// (the default) replays and records; replays and never writes. is INVALID_ARGUMENT: leave unset. + public CacheMode Mode { get; init; } = CacheMode.ReadWrite; + + /// A custom store in place of the file store, the seam a shared cache plugs into. + public IStepCache? Store { get; init; } + + /// Replays only: a recording that no longer replays fails its step with REPLAY_STALE instead of calling the model, and nothing is written. + public bool Strict { get; init; } +} + +/// The agent's model, budgets, and context, as an agents entry of a config takes them (), plus what only a program sets. Upstream's CreateAgentOptions (core/agent.ts). +public sealed class CreateAgentOptions : AgentOptions +{ + /// Replays recorded ActAsync flows; every ActAsync then names its recording with . Unset, every ActAsync calls the model. + public AgentCacheOptions? Cache { get; init; } +} + +/// +/// for : each call names the recording it replays and records, or runs live. Upstream's +/// CachingActOptions (core/agent.ts), which TypeScript types as a union; C# holds both members, and the agent refuses the one that +/// does not fit it at runtime. is INVALID_ARGUMENT: create another agent for another configuration. +/// +public sealed class CachingActOptions : ActOptions +{ + /// + /// The recording this call replays and records, unique within what the program does: checkout, cart-best-coupon. The key is + /// this name, the instruction, its params, and how many times the session made the same call, so a reworded instruction records afresh. + /// + public string? CacheKey { get; init; } + + /// false runs this call live, neither replaying nor recording. Any other value is INVALID_ARGUMENT. + public bool? Cache { get; init; } +} + +/// +/// What is, for a core agent: the agent a program created, not one a config names. It is the one agent, so a call that names +/// another is refused, and it has no model until the program passes one. Upstream's AgentContext override of createAgent. +/// +internal sealed class CoreAgentBinding(ResolvedAgent agent, AgentCacheContext? cache) +{ + /// The cache context of this agent's calls, or null for a call that runs live. + public AgentCacheContext? Cache { get; } = cache; + + public ResolvedAgent Select(string? requested) + { + if (requested is not null) + { + throw new TestException("INVALID_ARGUMENT", "an E2E core agent takes no Agent option: create another agent for another configuration"); + } + + if (agent.Model is null && agent.Executor is null) + { + throw new AgentException("MODEL_UNAVAILABLE", "the agent has no model: pass one to CreateAgent, an IAgentModel such as CoreAgents.CreateAgent(session, new CreateAgentOptions { Model = model })"); + } + + return agent; + } +} + +/// +/// An agent over one session, configured in code: ActAsync, AssertAsync, WaitForAsync, and ExtractAsync behave as they +/// do in a test. With a cache, ActAsync replays recorded flows, each keyed by the the call names. +/// Upstream's CachingAgent, CoreAgent, and UncheckedAgent (core/agent.ts), which TypeScript tells apart by the options +/// createAgent was called with: C# has one type, and refuses at runtime the ActAsync call that does not fit the agent. +/// +public sealed class CoreAgent +{ + private readonly AttemptScope _scope; + private readonly ResolvedAgent _resolved; + private readonly Agent _live; + private readonly Func? _cacheFor; + + internal CoreAgent(AttemptScope scope, ResolvedAgent resolved, Func? cacheFor) + { + _scope = scope; + _resolved = resolved; + _cacheFor = cacheFor; + _live = new Agent(scope, new CoreAgentBinding(resolved, null)); + } + + /// + /// Runs one goal. On an agent with a cache the call names its recording with , or runs live with + /// set to false; without one it fails with INVALID_ARGUMENT. On an agent without a cache, + /// naming either fails with INVALID_ARGUMENT. + /// + public async Task ActAsync(string instruction, CachingActOptions? options = null, CancellationToken cancellationToken = default) + { + ActOptions? rest = options; + var cacheKey = options?.CacheKey; + var perCall = options?.Cache; + if (_cacheFor is null) + { + if (cacheKey is not null || perCall is not null) + { + throw new TestException("INVALID_ARGUMENT", "ActAsync CacheKey and Cache need an agent created with CreateAgent(session, new CreateAgentOptions { Cache = ... })"); + } + + return await _live.ActAsync(instruction, rest, cancellationToken).ConfigureAwait(false); + } + + if (perCall == false && cacheKey is null) + { + return await _live.ActAsync(instruction, rest, cancellationToken).ConfigureAwait(false); + } + + if (string.IsNullOrWhiteSpace(cacheKey) || perCall is not null) + { + throw new TestException("INVALID_ARGUMENT", "ActAsync on an agent with a cache names its recording: pass a CacheKey, or Cache = false to run live"); + } + + var context = _cacheFor(cacheKey); + return await new Agent(_scope, new CoreAgentBinding(_resolved, context)).ActAsync(instruction, rest, cancellationToken).ConfigureAwait(false); + } + + /// Checks that the statement holds on the current screen. See . + public Task AssertAsync(string statement, AssertOptions? options = null, CancellationToken cancellationToken = default) => _live.AssertAsync(statement, options, cancellationToken); + + /// Waits until the statement holds on screen. See . + public Task WaitForAsync(string statement, WaitForOptions? options = null, CancellationToken cancellationToken = default) => _live.WaitForAsync(statement, options, cancellationToken); + + /// Reads data off the current screen. See . + public Task ExtractAsync(string instruction, ExtractOptions? options = null, CancellationToken cancellationToken = default) => _live.ExtractAsync(instruction, options, cancellationToken); +} + +/// createAgent from @e2e-dev/core/agent: the agent over one session, configured in code. Upstream's core/agent.ts. +public static class CoreAgents +{ + private const string AgentName = "agent"; + + /// + /// What stands in a recording's key where a run puts its project. A run reads it off the project, or hashes the directory when there is none; + /// a program's recordings must be found from any directory and on any machine, so a shared store works, and the cache key already names the flow. + /// + private const string CacheProjectId = "@e2e-dev/core"; + + /// + /// Creates an agent over . With , its replays flows, + /// each named by its , and records them when the session closes. A missing model fails the first call that + /// needs one as MODEL_UNAVAILABLE, never the creation. + /// + public static CoreAgent CreateAgent(Session session, CreateAgentOptions options) + { + var internals = InternalsOf(session); + if (options is null) + { + throw new TestException("INVALID_ARGUMENT", "CreateAgent options must be an object with a model"); + } + + var resolved = ResolvedAgent.From(AgentName, options); + var cache = options.Cache is null ? null : ResolveAgentCache(options.Cache, internals.ProjectRoot); + if (cache is null) + { + return new CoreAgent(session.E2E.Scope, resolved, null); + } + + // One cache context per key, each its own occurrence count; all settle when the session closes. + var scope = session.E2E.Scope; + var caches = new Dictionary(StringComparer.Ordinal); + var gate = new Lock(); + AgentCacheContext? CacheFor(string cacheKey) + { + lock (gate) + { + if (!caches.TryGetValue(cacheKey, out var context)) + { + caches[cacheKey] = context = CacheContexts.CreateAgentCacheContext( + cache, + internals.ProjectRoot, + 0, + CacheProjectId, + "core:" + cacheKey, + new CacheTargetIdentity(scope.TargetName, scope.EnginePlatform, scope.EngineName, BuildReport.EngineSpiVersion, scope.AppIdentity)); + } + + return context; + } + } + + internals.Closers.Add(() => + { + var settlement = internals.Settlement(); + AgentCacheContext?[] contexts; + lock (gate) + { + contexts = [.. caches.Values]; + } + + foreach (var context in contexts) + { + if (context is not null) + { + CacheContexts.SettleStagedTraces(context, settlement); + } + } + + return Task.CompletedTask; + }); + return new CoreAgent(scope, resolved, CacheFor); + } + + private static SessionInternals InternalsOf(Session session) => + session?.Internals ?? throw new TestException("INVALID_ARGUMENT", "CreateAgent takes a session from host.NewSessionAsync()"); + + /// The cache options as the replay cache reads them: the file store under the project root unless a store replaces it. + private static CacheConfig ResolveAgentCache(AgentCacheOptions cache, string projectRoot) + { + if (cache.Mode is not (CacheMode.ReadWrite or CacheMode.ReadOnly)) + { + throw new TestException("INVALID_ARGUMENT", "CreateAgent Cache.Mode must be ReadWrite or ReadOnly"); + } + + if (cache.Dir is not null && string.IsNullOrWhiteSpace(cache.Dir)) + { + throw new TestException("INVALID_ARGUMENT", "CreateAgent Cache.Dir must be a non-empty path"); + } + + return new CacheConfig + { + Mode = cache.Mode, + Store = cache.Store, + Directory = Path.GetFullPath(cache.Dir ?? Path.Combine(".e2e", "cache"), projectRoot), + Strict = cache.Strict, + StrictSource = cache.Strict ? new CacheStrictSource(true, false) : null, + }; + } +} diff --git a/src/E2E/Core/Internals.cs b/src/E2E/Core/Internals.cs new file mode 100644 index 000000000..d35cae180 --- /dev/null +++ b/src/E2E/Core/Internals.cs @@ -0,0 +1,68 @@ +// Copyright 2026 TesterArmy. +// Modified by Dario Kondratiuk. +// SPDX-License-Identifier: Apache-2.0 + +namespace E2E; + +/// +/// What reads off a session it did not create: the attempt behind it, the host's project root, and the +/// session's close hooks. Upstream's SessionInternals (core/internals.ts), which keeps it in a WeakMap so that a session +/// object carries none of it; the port's holds it in an internal member. +/// +internal sealed class SessionInternals(StandaloneAttempt attempt, string projectRoot, FailureWatch failures) +{ + public StandaloneAttempt Attempt { get; } = attempt; + + /// The directory the host was launched from, which a relative cache directory resolves against. Upstream's config.projectRoot. + public string ProjectRoot { get; } = projectRoot; + + /// Run in order when the session closes, before its attempt ends: an agent settles its replay cache here. + public List> Closers { get; } = []; + + /// How the session's steps went, as the replay cache settles what its agents recorded. + public AttemptSettlement Settlement() => failures.Settlement(Attempt.Session.Recorder); +} + +/// +/// Watches a session's steps for the agent steps that failed. A test ends at its failure; a program may catch one and go on. So each failure +/// settles only the recordings it found unconfirmed, as a run's failure does: dropped, unless only a missing model explains it. Recordings made +/// after it are confirmed by the checks that pass after them. Upstream's FailureWatch (core/internals.ts). +/// +internal sealed class FailureWatch +{ + private readonly List _failures = []; + private StepRecorder? _steps; + + /// The onProgress hook the session's attempt reports its steps to. + public void OnProgress(StepProgress progress) + { + // Only an agent step's failure implicates recordings: a program catches a locator's timeout on purpose, to branch. + if (progress.Phase != StepProgressPhase.End || progress.Kind != StepKind.Agent || progress.Status is null or StepStatus.Passed || _steps is null) + { + return; + } + + var code = progress.Error?.Code; + lock (_failures) + { + _failures.Add(new SurvivedFailure( + _steps.LastVerifiedStepIndex, + progress.Identity?.StepIndex ?? int.MaxValue, + code is null || !AgentError.ModelUnreachableCodes.Contains(code))); + } + } + + public void Watch(StepRecorder steps) => _steps = steps; + + /// + /// How the session's recordings settle when it closes. A close is not a verdict: a program may end, or be interrupted, between a replay and + /// the check after it, so what no check confirmed is left as it was, neither written nor evicted. + /// + public AttemptSettlement Settlement(StepRecorder steps) + { + lock (_failures) + { + return new AttemptSettlement(steps.LastVerifiedStepIndex, false, [.. _failures]); + } + } +} diff --git a/src/E2E/Core/Launch.cs b/src/E2E/Core/Launch.cs index 01462e902..1d853618f 100644 --- a/src/E2E/Core/Launch.cs +++ b/src/E2E/Core/Launch.cs @@ -91,8 +91,9 @@ public sealed class Session : IAsyncDisposable private readonly Action _onClosed; private readonly Lazy _closing; - internal Session(StandaloneAttempt attempt, StateIdentity identity, Action onClosed) + internal Session(StandaloneAttempt attempt, StateIdentity identity, SessionInternals internals, Action onClosed) { + Internals = internals; _attempt = attempt; _session = attempt.Session; _identity = identity; @@ -114,6 +115,9 @@ internal Session(StandaloneAttempt attempt, StateIdentity identity, Action onClo internal E2ESession E2E => _session; + /// What reads off the session; see . + internal SessionInternals Internals { get; } + /// Captures the session's state (cookies, storage, app data) in memory. Persist it only with . Upstream's saveState. public Task SaveStateAsync(CancellationToken cancellationToken = default) => OpeningRedaction.RedactOpening(async () => @@ -144,10 +148,23 @@ public void SetDefaultTimeout(TimeSpan timeout) private async Task CloseOnceAsync() { - IReadOnlyList errors; + // The closers run in order, before the attempt ends: an agent settles its replay cache here. + var errors = new List(); + foreach (var closer in Internals.Closers) + { + try + { + await closer().ConfigureAwait(false); + } + catch (Exception cause) when (cause is not OperationCanceledException) + { + errors.Add(cause); + } + } + try { - errors = await _attempt.CloseAsync().ConfigureAwait(false); + errors.AddRange(await _attempt.CloseAsync().ConfigureAwait(false)); } finally { @@ -219,10 +236,12 @@ public Task NewSessionAsync(NewSessionOptions? options = null, Cancella } var restored = options?.State is { } state ? SessionState.RestoredStateFor(state, _identity) : null; - var attempt = await _host.OpenAttemptAsync(SessionOptions(restored), cancellationToken, hosted: true).ConfigureAwait(false); + var failures = new FailureWatch(); + var attempt = await _host.OpenAttemptAsync(SessionOptions(restored, failures), cancellationToken, hosted: true).ConfigureAwait(false); try { - var session = new Session(attempt, _identity, () => Release(attempt)); + failures.Watch(attempt.Session.Recorder); + var session = new Session(attempt, _identity, new SessionInternals(attempt, _projectRoot, failures), () => Release(attempt)); if (_timeout is { } timeout) { session.SetDefaultTimeout(timeout); @@ -315,7 +334,7 @@ private async Task CloseOnceAsync() Launch.ThrowClosingErrors(errors, "closing the host"); } - private E2ESessionOptions SessionOptions(SavedSession? restored) => new() + private E2ESessionOptions SessionOptions(SavedSession? restored, FailureWatch failures) => new() { Engine = _engine, TargetName = _target, @@ -343,6 +362,7 @@ private async Task CloseOnceAsync() // No report stands between an operation's error and the program, so the error itself is classified and redacted. RedactThrown = true, RetainSteps = Launch.RetainedSteps, + OnProgress = failures.OnProgress, State = restored, }; diff --git a/src/E2E/Run/E2ESession.cs b/src/E2E/Run/E2ESession.cs index 800bf57c0..eea62a46e 100644 --- a/src/E2E/Run/E2ESession.cs +++ b/src/E2E/Run/E2ESession.cs @@ -106,6 +106,12 @@ private E2ESession(AttemptScope scope, CancellationTokenSource timeout, AttemptB public IReadOnlyList Steps() => [.. StepRecords.Select(StepProgressWire.Public)]; /// The attempt's step timeline, with the screen each step left when the attempt keeps a trace. + /// The attempt's step timeline, which a program's agent reads the verified steps off (upstream's attempt.steps). + internal StepRecorder Recorder => _scope.Steps; + + /// The attempt's scope, which a program's agent runs over (upstream's attempt.agentRuntime). + internal AttemptScope Scope => _scope; + internal IReadOnlyList StepRecords => _scope.Steps.All(); /// The hook steps run in from now on; null for the test body (upstream's enter). A test framework's hook methods name theirs themselves. @@ -1238,11 +1244,15 @@ private void Commit(Exception? error, CancellationToken cancellationToken) /// recorded or replayed is evicted, unless says the failure implicates nothing. /// Returns what became of each step's recording, by step index. /// - internal static IReadOnlyDictionary Flush(IStepCache cache, IEnumerable acts, bool preserve) + internal static IReadOnlyDictionary Flush(IStepCache cache, IEnumerable acts, bool preserve) => Flush(cache, acts, _ => preserve); + + /// As , with decided per act: a program's session settles each failure it went on after separately. + internal static IReadOnlyDictionary Flush(IStepCache cache, IEnumerable acts, Func preserveOf) { var outcomes = new Dictionary(); foreach (var act in acts) { + var preserve = preserveOf(act); var recorded = act.Completed && act.Entry is { Actions.Count: > 0 } && !act.ParamCollision; var outcome = act.Decided; try diff --git a/tests/E2E.Tests/Core/CoreFakeEngine.cs b/tests/E2E.Tests/Core/CoreFakeEngine.cs index c59c58818..80447b43a 100644 --- a/tests/E2E.Tests/Core/CoreFakeEngine.cs +++ b/tests/E2E.Tests/Core/CoreFakeEngine.cs @@ -87,11 +87,11 @@ public Task StartAsync(EngineStartOptions options, CancellationT return Task.FromResult(new Session(this, options.ArtifactsDir)); } - private void Record(string method, OperationContext context) + private void Record(string method, OperationContext context, TimeSpan? timeout = null) { lock (_operations) { - _operations.Add((method, context.Timeout)); + _operations.Add((method, timeout ?? context.Timeout)); } } @@ -127,6 +127,24 @@ public Task OpenLinkAsync(string url, OperationContext context, CancellationToke public Task ObserveAsync(CancellationToken cancellationToken) => Task.FromResult(new Observation { Route = "/", Roots = owner.Scene }); + public Task ObserveAsync(OperationContext context, CancellationToken cancellationToken) + { + owner.Record("observe", context); + return ObserveAsync(cancellationToken); + } + + public Task ObserveAsync(TimeSpan timeout, OperationContext context, CancellationToken cancellationToken) + { + owner.Record("observe", context, timeout); + return ObserveAsync(cancellationToken); + } + + public Task ObserveAsync(ObserveOptions? options, TimeSpan timeout, OperationContext context, CancellationToken cancellationToken) + { + owner.Record("observe", context, timeout); + return ObserveAsync(cancellationToken); + } + public Task PerformAsync(SemanticNode node, LocatorAction action, CancellationToken cancellationToken) { lock (owner._performed) diff --git a/tests/E2E.Tests/Core/CoreTests.cs b/tests/E2E.Tests/Core/CoreTests.cs index f7ed87825..3c790242f 100644 --- a/tests/E2E.Tests/Core/CoreTests.cs +++ b/tests/E2E.Tests/Core/CoreTests.cs @@ -5,12 +5,13 @@ using System.Text; using System.Text.Json.Nodes; using E2E.Engine; +using E2E.Tests.AgentAct; namespace E2E.Tests.Core; /// /// Upstream's describe('e2e/core') (tests/integration/core.test.ts): hosts, sessions, saved state, secrets, timeouts, link bases, and the -/// agent's cache. The it that need a feature the port has not ported (createAgent) are not here, or are partial. +/// agent's cache. The it that need a feature the port has not ported are not here, or are partial; the it of describe('createAgent') are in CreateAgentTests. /// public sealed class CoreTests : IAsyncLifetime { @@ -222,9 +223,8 @@ public async Task Hands_secrets_out_by_name_and_redacts_their_values_from_what_t await session.CloseAsync(); } - // Partial: upstream also runs an agent step (createAgent, #1676) and checks that its observations take the new default; the port has no createAgent yet. [Fact] - public async Task Partial_Bounds_each_operation_by_its_default_timeout_which_setDefaultTimeout_moves_with_no_session_deadline() + public async Task Bounds_each_operation_by_its_default_timeout_which_setDefaultTimeout_moves_with_no_session_deadline() { var fake = new CoreFakeEngine(); var opened = await HostOf(fake, new LaunchOptions { App = FakeApp, Timeout = TimeSpan.FromSeconds(10) }); @@ -238,6 +238,26 @@ public async Task Partial_Bounds_each_operation_by_its_default_timeout_which_set Assert.Equal("LOCATOR_NOT_FOUND", failure.Code); Assert.True(DateTime.UtcNow - started < TimeSpan.FromSeconds(5)); Assert.Equal("INVALID_ARGUMENT", Assert.Throws(() => session.SetDefaultTimeout(TimeSpan.Zero)).Code); + + // The agent's engine operations take the same default. + session.SetDefaultTimeout(TimeSpan.FromMilliseconds(1_234)); + var before = fake.Operations.Count; + var observer = new TestExecutor("observer", async (context, _) => + { + await context.ObserveAsync(); + return ExecutorKit.Passed("looked"); + }) + { + Version = "1", + }; + await CoreAgents.CreateAgent(session, new CreateAgentOptions { Executor = observer }).ActAsync("look at the page"); + var observes = fake.Operations.Skip(before).Where(operation => operation.Method == "observe").ToList(); + Assert.NotEmpty(observes); + foreach (var operation in observes) + { + Assert.True(operation.Timeout <= TimeSpan.FromMilliseconds(1_234)); + } + await session.CloseAsync(); } diff --git a/tests/E2E.Tests/Core/CreateAgentTests.cs b/tests/E2E.Tests/Core/CreateAgentTests.cs new file mode 100644 index 000000000..aebefe54e --- /dev/null +++ b/tests/E2E.Tests/Core/CreateAgentTests.cs @@ -0,0 +1,204 @@ +// Copyright 2026 TesterArmy. +// Modified by Dario Kondratiuk. +// SPDX-License-Identifier: Apache-2.0 + +using E2E.Engine; +using E2E.Tests.AgentAct; +using static E2E.Tests.AgentAct.ExecutorKit; + +namespace E2E.Tests.Core; + +/// Upstream's describe('createAgent') (tests/integration/core.test.ts): an agent over a core session, and the replay cache it keeps by cacheKey. +public sealed class CreateAgentTests : IAsyncLifetime +{ + private static readonly LaunchApp FakeApp = new() { Url = "http://127.0.0.1:4500" }; + + private readonly string _dir = Directory.CreateTempSubdirectory("e2e-core-agent-tests-").FullName; + + private readonly List _hosts = []; + + public Task InitializeAsync() => Task.CompletedTask; + + public async Task DisposeAsync() + { + foreach (var opened in _hosts) + { + try + { + await opened.CloseAsync(); + } + catch (AggregateException) + { + // Closing is checked by the tests that look at it. + } + } + + Directory.Delete(_dir, recursive: true); + } + + /// Upstream's shop(): a product page whose button leads to a cart with a coupon and a payment button. + private static DocumentWorld Shop() => new DocumentWorld().Map("/", page => + { + var paid = new DocumentElement { Role = "status", Name = "Paid", Hidden = true }; + var coupon = new DocumentElement { Role = "checkbox", Name = "Apply coupon SAVE10" }; + var pay = new DocumentElement { Role = "button", Name = "Pay now", OnTap = () => paid.Hidden = false }; + var cart = new DocumentElement { Role = "region", Name = "Cart", Hidden = true }; + cart.Children.Add(coupon); + cart.Children.Add(pay); + page.Roots.Add(new DocumentElement { Role = "button", Name = "Add to cart", OnTap = () => cart.Hidden = false }); + page.Roots.Add(new DocumentElement { Role = "paragraph", Text = "39,000", TestId = "sale-price" }); + page.Roots.Add(cart); + page.Roots.Add(paid); + }); + + /// Upstream's scripted(): a brain that adds to cart, checks the coupon, and pays. + private static (TestExecutor Brain, Func Calls) Scripted() + { + var calls = 0; + var brain = new TestExecutor("scripted", async (context, _) => + { + Interlocked.Increment(ref calls); + await Tap(context, "button \"Add to cart\""); + var observation = await context.ObserveAsync(); + await context.Actions.CheckAsync(TargetFor(observation.Text, "checkbox \"Apply coupon SAVE10\""), true); + await Tap(context, "button \"Pay now\""); + return Passed("paid"); + }) + { + Version = "1", + }; + return (brain, () => Volatile.Read(ref calls)); + } + + private static async Task Tap(StepExecutorContext context, string description) + { + var observation = await context.ObserveAsync(); + await context.Actions.TapAsync(TargetFor(observation.Text, description)); + } + + private static string[] Recordings(string cacheDir) => + Directory.Exists(cacheDir) ? [.. Directory.EnumerateFiles(cacheDir, "*.json", SearchOption.AllDirectories)] : []; + + private async Task HostOf(IEngine engine) + { + var opened = await Launch.LaunchAsync(engine, new LaunchOptions { App = FakeApp, Cwd = _dir, Env = new Dictionary() }); + _hosts.Add(opened); + return opened; + } + + [Fact] + public async Task Records_a_flow_under_its_cacheKey_and_replays_it_with_no_model() + { + var (brain, calls) = Scripted(); + var cacheDir = Path.Combine(_dir, "cache"); + var opened = await HostOf(new DocumentEngine(Shop())); + foreach (var round in new[] { 1, 2 }) + { + var session = await opened.NewSessionAsync(); + await session.App.OpenAsync("/"); + var agent = CoreAgents.CreateAgent(session, new CreateAgentOptions { Executor = brain, Cache = new AgentCacheOptions { Dir = cacheDir } }); + var result = await agent.ActAsync("checkout with the coupon", new CachingActOptions { CacheKey = "checkout" }); + Assert.True(await session.Screen.GetByRole("checkbox").IsCheckedAsync()); + // A check after the step confirms its recording. + await session.Screen.GetByRole("status", "Paid").WaitForAsync(); + Assert.Equal(round == 1 ? "missed" : "self-finalized", result.Cache?.Mode); + await session.CloseAsync(); + Assert.Equal(1, calls()); + } + + Assert.Single(Recordings(cacheDir)); + } + + [Fact] + public async Task Keeps_a_confirmed_recording_when_the_host_closes_the_session_and_after_a_failure_the_program_caught() + { + var (brain, _) = Scripted(); + var cacheDir = Path.Combine(_dir, "cache"); + var opened = await HostOf(new DocumentEngine(Shop())); + var session = await opened.NewSessionAsync(); + await session.App.OpenAsync("/"); + var agent = CoreAgents.CreateAgent(session, new CreateAgentOptions { Executor = brain, Cache = new AgentCacheOptions { Dir = cacheDir } }); + await agent.ActAsync("checkout with the coupon", new CachingActOptions { CacheKey = "checkout" }); + await session.Screen.GetByRole("button", "Pay now").WaitForAsync(); + var caught = await Assert.ThrowsAnyAsync(() => session.Screen.GetByRole("button", "Missing").WaitForAsync(new LocatorWaitForOptions { Timeout = TimeSpan.FromMilliseconds(100) })); + Assert.Equal("LOCATOR_NOT_FOUND", caught.Code); + await opened.CloseAsync(); + + Assert.Single(Recordings(cacheDir)); + } + + [Fact] + public async Task Drops_only_what_a_caught_agent_failure_found_unconfirmed_and_evicts_nothing_when_a_session_closes_before_its_check() + { + var (brain, calls) = Scripted(); + var flaky = new TestExecutor("scripted", (context, token) => context.Step.Instruction.StartsWith("find", StringComparison.Ordinal) + ? Task.FromResult(Failed("there is no sale")) + : brain.RunStepAsync(context, token)) + { + Version = "1", + }; + var cacheDir = Path.Combine(_dir, "cache"); + var opened = await HostOf(new DocumentEngine(Shop())); + CoreAgent AgentOn(Session session) => CoreAgents.CreateAgent(session, new CreateAgentOptions { Executor = flaky, Cache = new AgentCacheOptions { Dir = cacheDir } }); + async Task Round(Func steps) + { + var session = await opened.NewSessionAsync(); + await session.App.OpenAsync("/"); + await steps(AgentOn(session), session); + await session.CloseAsync(); + } + + async Task Fails(CoreAgent agent) => + Assert.Equal("ACTION_FAILED", (await Assert.ThrowsAsync(() => agent.ActAsync("find the sale", new CachingActOptions { CacheKey = "sale" }))).Code); + + // A recording made after a failure the program caught is confirmed by the check after it. + await Round(async (agent, session) => + { + await Fails(agent); + Assert.Equal("missed", (await agent.ActAsync("checkout with the coupon", new CachingActOptions { CacheKey = "checkout" })).Cache?.Mode); + await session.Screen.GetByRole("button", "Pay now").WaitForAsync(); + }); + Assert.Single(Recordings(cacheDir)); + // A session that closes between a replay and its check leaves the recording alone. + await Round(async (agent, _) => + Assert.Equal("self-finalized", (await agent.ActAsync("checkout with the coupon", new CachingActOptions { CacheKey = "checkout" })).Cache?.Mode)); + Assert.Single(Recordings(cacheDir)); + // So does a replay that a check confirms after a failure. + await Round(async (agent, session) => + { + await Fails(agent); + Assert.Equal("self-finalized", (await agent.ActAsync("checkout with the coupon", new CachingActOptions { CacheKey = "checkout" })).Cache?.Mode); + await session.Screen.GetByRole("button", "Pay now").WaitForAsync(); + }); + Assert.Single(Recordings(cacheDir)); + // A recording still unconfirmed when a step fails is dropped, whatever passes later. + await Round(async (agent, session) => + { + Assert.Equal("missed", (await agent.ActAsync("checkout with the coupon", new CachingActOptions { CacheKey = "again" })).Cache?.Mode); + await Fails(agent); + await session.Screen.GetByRole("button", "Pay now").WaitForAsync(); + }); + Assert.Single(Recordings(cacheDir)); + Assert.Equal(2, calls()); + } + + [Fact] + public async Task Requires_a_cacheKey_on_an_agent_with_a_cache_refuses_one_without_and_fails_a_call_with_no_model_as_MODEL_UNAVAILABLE() + { + var opened = await HostOf(new DocumentEngine(Shop())); + var session = await opened.NewSessionAsync(); + var caching = CoreAgents.CreateAgent(session, new CreateAgentOptions { Executor = Scripted().Brain, Cache = new AgentCacheOptions { Dir = Path.Combine(_dir, "cache") } }); + // An agent with a cache names its recording. + Assert.Equal("INVALID_ARGUMENT", (await Assert.ThrowsAsync(() => caching.ActAsync("checkout"))).Code); + var live = CoreAgents.CreateAgent(session, new CreateAgentOptions { Executor = Scripted().Brain }); + // CacheKey needs an agent with a cache. + Assert.Equal("INVALID_ARGUMENT", (await Assert.ThrowsAsync(() => live.ActAsync("checkout", new CachingActOptions { CacheKey = "checkout" }))).Code); + var noModel = await Assert.ThrowsAsync(() => CoreAgents.CreateAgent(session, new CreateAgentOptions()).ActAsync("anything")); + Assert.Equal("MODEL_UNAVAILABLE", noModel.Code); + Assert.Contains("pass one to CreateAgent", noModel.Message, StringComparison.Ordinal); + // The C# types hold both forms of act, so the runtime refuses the wrong one, as it does for options held in a variable upstream. + Assert.Equal("INVALID_ARGUMENT", (await Assert.ThrowsAsync(() => caching.ActAsync("checkout", new CachingActOptions()))).Code); + Assert.Equal("INVALID_ARGUMENT", Assert.Throws(() => CoreAgents.CreateAgent(null!, new CreateAgentOptions())).Code); + await session.CloseAsync(); + } +} From b08145530248b7c7ba9bb1473d344aab05181541 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 16 Nov 2026 17:45:40 +0000 Subject: [PATCH 2/3] Address review: thread-safe staging and claimer, report closer errors, guard CachingActOptions on a test agent Co-Authored-By: Claude Sonnet 5.5 Claude-Session: https://claude.ai/code/session_01Ls2e1zqvVudTEe6MWBzFXR --- COMPATIBILITY.md | 4 ++-- src/E2E/Agent/Act.cs | 8 +++++++- src/E2E/Agent/Agent.cs | 14 +++++++++++--- src/E2E/Cache/Context.cs | 9 +++++++-- src/E2E/Core/CoreAgent.cs | 11 ++++++++++- src/E2E/Core/Launch.cs | 5 +++++ src/E2E/Run/E2ESession.cs | 3 ++- tests/E2E.Tests/Core/CreateAgentTests.cs | 11 +++++++++-- 8 files changed, 53 insertions(+), 12 deletions(-) diff --git a/COMPATIBILITY.md b/COMPATIBILITY.md index ce5d3dc0a..d6637934d 100644 --- a/COMPATIBILITY.md +++ b/COMPATIBILITY.md @@ -172,8 +172,8 @@ Differences: - `RedactThrown` rebuilds an error that carries a secret value (type and code kept, causes redacted) where upstream edits the thrown error in place, which a .NET exception's read-only message does not allow. An error with no secret is thrown as it is. - `SessionState.ExportAsync` and `ImportAsync` are `Task`-returning like upstream's `async` methods, so a bad key rejects instead of throwing. The key is `SessionStateKeyOptions.Key`, a `byte[]` of 32 bytes; there is no unknown-option check (`plaintext: true` cannot be passed). - `app.linkBase` (`AppConfig.LinkBase`, `ResolvedApp.LinkBase`, `LaunchApp.LinkBase`, checked by `TargetApps.CheckLinkBase`, upstream's `checkLinkBase`) and the engine `openLink` hook (`IEngineSession.OpenLinkAsync`, `EngineSessionSpec.OpenLink`, `ISessionApp.OpenLinkAsync`, `session/openLink` in the manifest schema) are ported, with `App.OpenAsync(path)` joining and opening the link as upstream's `launchPinnedApp` does. `EngineCapabilities.OpenLink` is .NET-only: C# has no optional member to test, so an engine declares the hook the way it declares `Resize`. `TargetApps.JoinLink`, `TargetApps.RefusesLink`, and `Urls.IsRefusedLinkScheme` (upstream's `joinLink`, `refusesLink`, `isRefusedLinkScheme`) are shared with `DeviceLinks`. Not ported: the unknown-key message of a `defineEngine` session, which a typed `EngineSessionSpec` cannot receive (`engine.test.ts`). -- `CoreAgents.CreateAgent` returns one `CoreAgent` where upstream types `CachingAgent`, `CoreAgent`, and `UncheckedAgent` by the options (a TypeScript union C# overloads cannot express): `ActAsync` takes `CachingActOptions` (an `ActOptions` plus `CacheKey` and `Cache`, which can only be `false`) and refuses at runtime what upstream's types refuse at compile time (`INVALID_ARGUMENT`). `CreateAgentOptions` is an `AgentOptions` plus `Cache` (`AgentCacheOptions`: `Dir`, `Mode`, `Store`, `Strict`); `Mode` reuses `CacheMode` and `Off` is `INVALID_ARGUMENT`. `ActOptions` and `AgentOptions` are no longer sealed so these can derive from them. `CoreAgents` (.NET-only name) holds the function, which has no class in upstream; `CoreAgentBinding` (.NET-only) stands for the `AgentContext` override (`select`, `cache`) `createAgent` builds, since the port's `Agent` reads an `AttemptScope`. Upstream's `WorkerModels` (preflight of the model at the first call) has no counterpart: a model is called, and refused, as in a test. `newSession` takes no signal (see above), so there is no cancelled-session settlement (`settlement()` of `SessionInternals` always settles); the options `createAgent` rejects as unknown keys (`rejectUnknownKeys`) cannot be passed in C#. -- `FailureWatch`, `SessionInternals`, `AttemptSettlement`, `SurvivedFailure`, and `CacheContexts.SettleStagedTraces` (`Core/Internals.cs`, `Cache/Context.cs`) are upstream's `FailureWatch`, `SessionInternals`, `AttemptSettlement`, `SurvivedFailure`, and `settleStagedTraces`. `AgentCacheContext` now has `TestId`, `ClaimKey`, and `Staged` (upstream's `claimKey` and `staged`) for a context a program creates; a test's attempt still keeps them on `AttemptScope`, which builds its context per act (`AttemptScope.ActCache`). A program's settlement confirms an act by its step index against the last verified step, as upstream does; `E2ESession.Flush` takes the per-act `preserve` it needs. `AgentError.ModelUnreachableCodes` is upstream's `MODEL_UNREACHABLE_CODES`. A recording's key uses the project id `@e2e-dev/core` and the test id `core:`, as upstream's. +- `CoreAgents.CreateAgent` returns one `CoreAgent` where upstream types `CachingAgent`, `CoreAgent`, and `UncheckedAgent` by the options (a TypeScript union C# overloads cannot express): `ActAsync` takes `CachingActOptions` (an `ActOptions` plus `CacheKey` and `Cache`, which can only be `false`) and refuses at runtime what upstream's types refuse at compile time (`INVALID_ARGUMENT`). `CreateAgentOptions` is an `AgentOptions` plus `Cache` (`AgentCacheOptions`: `Dir`, `Mode`, `Store`, `Strict`); `Mode` reuses `CacheMode` and `Off` is `INVALID_ARGUMENT`. `ActOptions` and `AgentOptions` are no longer sealed so these can derive from them. `CoreAgents` (.NET-only name) holds the function, which has no class in upstream; `CoreAgentBinding` (.NET-only; its `Select` is upstream's `select`) stands for the `AgentContext` override (`select`, `cache`) `createAgent` builds, since the port's `Agent` reads an `AttemptScope`. Upstream's `WorkerModels` (preflight of the model at the first call) has no counterpart: a model is called, and refused, as in a test. `newSession` takes no signal (see above), so there is no cancelled-session settlement (`settlement()` of `SessionInternals` always settles); the options `createAgent` rejects as unknown keys (`rejectUnknownKeys`) cannot be passed in C#. +- `FailureWatch`, `SessionInternals`, `AttemptSettlement`, `SurvivedFailure`, and `CacheContexts.SettleStagedTraces` (`Core/Internals.cs`, `Cache/Context.cs`) are upstream's `FailureWatch`, `SessionInternals`, `AttemptSettlement`, `SurvivedFailure`, and `settleStagedTraces`. `AgentCacheContext` now has `TestId`, `ClaimKey`, and `Staged` (upstream's `claimKey` and `staged`) for a context a program creates; a test's attempt still keeps them on `AttemptScope`, which builds its context per act (`AttemptScope.ActCache`). A program's settlement confirms an act by its step index against the last verified step, as upstream does; `E2ESession.Flush` takes the per-act `preserve` it needs. `CacheContexts.KeyClaimer` and `ClaimedTraceKey` (.NET-only names) are upstream's `claimKey` closure of `createAgentCacheContext` and its result, extracted so a test's `AttemptScope` shares the one claim path (`AttemptScope.Claimer` and `IdentityOf` are its .NET-only helpers); `AgentError.ModelUnreachableCodes` is upstream's `MODEL_UNREACHABLE_CODES`. A recording's key uses the project id `@e2e-dev/core` and the test id `core:`, as upstream's. - Tests: `Core/CoreSessionStateTests` is `core-state.test.ts`; `Core/CoreTests` is the ported part of `core.test.ts` (partial, and renamed `Partial_...`: 'opens sessions one at a time' (no `gadget` fixture or key list), 'refuses what is not an engine' (no non-engine or unknown app key to pass), 'says a closed session is closed' (the redaction half only)); `AppLinkBase/` is `app-link-base.test.ts`; `Core/CoreTests` also holds 'opens a path as a link in the pinned app on a device target with a link base' (`Opens_a_path_as_a_link_...`); `SchemaFixtures/SessionStateV1SchemaTests`; `StepVerification/StepRecorderLastVerifiedStepIndexTests` has the two new `it` (`RedactThrown` is partial: the caught error is checked, not the original in place); `TraceCache/Trace1EntryTests` has the space-press `it`. `Core/CreateAgentTests` is `describe('createAgent')` of `core.test.ts` (all four `it`; 'requires a cacheKey' also checks the typed-options refusal at runtime, since C# has no `@ts-expect-error`). Not ported: `core-signals.test.ts` and the `engine.test.ts` fixture-name check (#1677, #1678). ## Not ported diff --git a/src/E2E/Agent/Act.cs b/src/E2E/Agent/Act.cs index 6b433c75b..9cef4b6e2 100644 --- a/src/E2E/Agent/Act.cs +++ b/src/E2E/Agent/Act.cs @@ -209,7 +209,13 @@ private async Task DispatchCoreAsync(StepSpec spec, ResolvedAgent age AppOrigin = _scope.AppOrigin, }); _scope.Acts.Add(session.Pending); - cache.Staged?.Add(session.Pending); + if (cache.Staged is { } staged) + { + lock (staged) + { + staged.Add(session.Pending); + } + } } // The baseline a recording's delta is read from, the screen a replay starts on, and the built-in agent's first look: diff --git a/src/E2E/Agent/Agent.cs b/src/E2E/Agent/Agent.cs index 91a1f8836..9d5508df1 100644 --- a/src/E2E/Agent/Agent.cs +++ b/src/E2E/Agent/Agent.cs @@ -84,8 +84,16 @@ internal Agent(AttemptScope scope, CoreAgentBinding? binding = null) // Upstream's `select` of the agent context: the agent a call runs as. A core agent is the one it was created with. private ResolvedAgent Select(string? name) => _binding is null ? _scope.Select(name) : _binding.Select(name); - public Task ActAsync(string instruction, ActOptions? options = null, CancellationToken cancellationToken = default) => - _scope.Track(RunActStepAsync(instruction, options, cancellationToken)); + public Task ActAsync(string instruction, ActOptions? options = null, CancellationToken cancellationToken = default) + { + // A test's agent has no cache of its own to name: a recording is keyed by the test, so CachingActOptions belongs to a core agent. + if (options is CachingActOptions) + { + throw new TestException("INVALID_ARGUMENT", "CachingActOptions (CacheKey, Cache) is for an agent created with CoreAgents.CreateAgent; pass ActOptions"); + } + + return _scope.Track(RunActStepAsync(instruction, options, cancellationToken)); + } public Task AssertAsync(string statement, AssertOptions? options = null, CancellationToken cancellationToken = default) { @@ -1753,7 +1761,7 @@ public string ClaimKey(string instruction, IReadOnlyDictionary? private Func?, CacheAgentIdentity, ClaimedTraceKey>? _claimKey; private Func?, CacheAgentIdentity, ClaimedTraceKey> Claimer() => - _claimKey ??= CacheContexts.KeyClaimer(CacheProjectId, TestTitle, new CacheTargetIdentity(TargetName, EnginePlatform, EngineName, BuildReport.EngineSpiVersion, AppIdentity)); + LazyInitializer.EnsureInitialized(ref _claimKey, () => CacheContexts.KeyClaimer(CacheProjectId, TestTitle, new CacheTargetIdentity(TargetName, EnginePlatform, EngineName, BuildReport.EngineSpiVersion, AppIdentity)))!; /// The cache context of one act of this attempt, or null when its cache is off: the store and mode the attempt resolved, and its key claims. internal AgentCacheContext? ActCache() => diff --git a/src/E2E/Cache/Context.cs b/src/E2E/Cache/Context.cs index e2ebb6a41..1ac42b0fa 100644 --- a/src/E2E/Cache/Context.cs +++ b/src/E2E/Cache/Context.cs @@ -125,8 +125,13 @@ public static void SettleStagedTraces(AgentCacheContext context, AttemptSettleme return; } - var acts = new List(staged); - staged.Clear(); + List acts; + lock (staged) + { + acts = [.. staged]; + staged.Clear(); + } + if (settlement is null || context.Mode != CacheMode.ReadWrite) { return; diff --git a/src/E2E/Core/CoreAgent.cs b/src/E2E/Core/CoreAgent.cs index b6db86084..a14afcb91 100644 --- a/src/E2E/Core/CoreAgent.cs +++ b/src/E2E/Core/CoreAgent.cs @@ -99,7 +99,16 @@ internal CoreAgent(AttemptScope scope, ResolvedAgent resolved, Func public async Task ActAsync(string instruction, CachingActOptions? options = null, CancellationToken cancellationToken = default) { - ActOptions? rest = options; + var rest = options is null ? null : new ActOptions + { + Params = options.Params, + Timeout = options.Timeout, + MaxSteps = options.MaxSteps, + MaxModelCalls = options.MaxModelCalls, + Agent = options.Agent, + OnAction = options.OnAction, + OnBeforeAction = options.OnBeforeAction, + }; var cacheKey = options?.CacheKey; var perCall = options?.Cache; if (_cacheFor is null) diff --git a/src/E2E/Core/Launch.cs b/src/E2E/Core/Launch.cs index 1d853618f..fef0eb9e3 100644 --- a/src/E2E/Core/Launch.cs +++ b/src/E2E/Core/Launch.cs @@ -166,6 +166,11 @@ private async Task CloseOnceAsync() { errors.AddRange(await _attempt.CloseAsync().ConfigureAwait(false)); } + catch (Exception cause) when (errors.Count > 0) + { + // What the closers found is reported with the failure of the attempt's own close. + errors.Add(cause); + } finally { _onClosed(); diff --git a/src/E2E/Run/E2ESession.cs b/src/E2E/Run/E2ESession.cs index eea62a46e..4134b772c 100644 --- a/src/E2E/Run/E2ESession.cs +++ b/src/E2E/Run/E2ESession.cs @@ -105,13 +105,14 @@ private E2ESession(AttemptScope scope, CancellationTokenSource timeout, AttemptB /// The attempt's step timeline as the report tells it. Upstream's steps(), which returns StepRecord[]. public IReadOnlyList Steps() => [.. StepRecords.Select(StepProgressWire.Public)]; - /// The attempt's step timeline, with the screen each step left when the attempt keeps a trace. /// The attempt's step timeline, which a program's agent reads the verified steps off (upstream's attempt.steps). internal StepRecorder Recorder => _scope.Steps; /// The attempt's scope, which a program's agent runs over (upstream's attempt.agentRuntime). internal AttemptScope Scope => _scope; + /// The attempt's step timeline, with the screen each step left when the attempt keeps a trace. + internal IReadOnlyList StepRecords => _scope.Steps.All(); /// The hook steps run in from now on; null for the test body (upstream's enter). A test framework's hook methods name theirs themselves. diff --git a/tests/E2E.Tests/Core/CreateAgentTests.cs b/tests/E2E.Tests/Core/CreateAgentTests.cs index aebefe54e..93786dae7 100644 --- a/tests/E2E.Tests/Core/CreateAgentTests.cs +++ b/tests/E2E.Tests/Core/CreateAgentTests.cs @@ -27,13 +27,20 @@ public async Task DisposeAsync() { await opened.CloseAsync(); } - catch (AggregateException) + catch (Exception) { // Closing is checked by the tests that look at it. } } - Directory.Delete(_dir, recursive: true); + try + { + Directory.Delete(_dir, recursive: true); + } + catch (IOException) + { + // A directory a process still holds is the system's to clean. + } } /// Upstream's shop(): a product page whose button leads to a cart with a coupon and a payment button. From 6208977c562289214b1ec614f4cb2e19f9e43057 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 16 Nov 2026 22:25:41 +0000 Subject: [PATCH 3/3] Address CodeRabbit: drop stale API pages, fix dead doc links, close the attempt when a closer is cancelled Co-Authored-By: Claude Sonnet 5.5 Claude-Session: https://claude.ai/code/session_01Ls2e1zqvVudTEe6MWBzFXR --- ...ngine.LocatorAction.PressSequentially.html | 13 --- docs-site/public/api/E2E.ModelUsage.html | 13 --- .../LocatorAction.PressSequentially.md | 78 --------------- .../src/content/docs/api/E2E.Engine/index.md | 1 - .../src/content/docs/api/E2E/ModelUsage.md | 97 ------------------- docs-site/src/content/docs/api/E2E/index.md | 1 - docs-site/src/content/docs/docs/core.mdx | 6 +- src/E2E/Core/Launch.cs | 3 +- 8 files changed, 5 insertions(+), 207 deletions(-) delete mode 100644 docs-site/public/api/E2E.Engine.LocatorAction.PressSequentially.html delete mode 100644 docs-site/public/api/E2E.ModelUsage.html delete mode 100644 docs-site/src/content/docs/api/E2E.Engine/LocatorAction.PressSequentially.md delete mode 100644 docs-site/src/content/docs/api/E2E/ModelUsage.md diff --git a/docs-site/public/api/E2E.Engine.LocatorAction.PressSequentially.html b/docs-site/public/api/E2E.Engine.LocatorAction.PressSequentially.html deleted file mode 100644 index fdc3b15de..000000000 --- a/docs-site/public/api/E2E.Engine.LocatorAction.PressSequentially.html +++ /dev/null @@ -1,13 +0,0 @@ - - - - - - -Redirecting… - - - -

Continue

- - diff --git a/docs-site/public/api/E2E.ModelUsage.html b/docs-site/public/api/E2E.ModelUsage.html deleted file mode 100644 index 381683ca8..000000000 --- a/docs-site/public/api/E2E.ModelUsage.html +++ /dev/null @@ -1,13 +0,0 @@ - - - - - - -Redirecting… - - - -

Continue

- - diff --git a/docs-site/src/content/docs/api/E2E.Engine/LocatorAction.PressSequentially.md b/docs-site/src/content/docs/api/E2E.Engine/LocatorAction.PressSequentially.md deleted file mode 100644 index 828196809..000000000 --- a/docs-site/src/content/docs/api/E2E.Engine/LocatorAction.PressSequentially.md +++ /dev/null @@ -1,78 +0,0 @@ ---- -title: "LocatorAction.PressSequentially" -description: "Focuses the node and types Text one character at a time." -editUrl: false -prev: false -next: false -sidebar: - label: "LocatorAction.PressSequentially" - hidden: true ---- - - - -Namespace: [E2E.Engine](/api/E2E.Engine/) - - -```csharp -public sealed record LocatorAction.PressSequentially : LocatorAction, IEquatable, IEquatable -``` - - -Focuses the node and types <code class="paramref">Text</code> one character at a time. - -**Inheritance:** [object](https://learn.microsoft.com/dotnet/api/system.object) → [LocatorAction](/api/E2E.Engine/LocatorAction/) - -**Implements:** [IEquatable<LocatorAction>](https://learn.microsoft.com/dotnet/api/system.iequatable-1), [IEquatable<LocatorAction.PressSequentially>](https://learn.microsoft.com/dotnet/api/system.iequatable-1) - -[View source](https://github.com/hardkoded/e2e-dotnet/blob/main/src/E2E/Engine/LocatorAction.cs) - -## Constructors - - - - -### PressSequentially(string, TimeSpan?) - - -```csharp -public PressSequentially(string Text, TimeSpan? Delay = null) -``` - - -Focuses the node and types <code class="paramref">Text</code> one character at a time. - -**Parameters** - -| Name | Type | Description | -| --- | --- | --- | -| `Text` | [string](https://learn.microsoft.com/dotnet/api/system.string) | | -| `Delay` | [TimeSpan?](https://learn.microsoft.com/dotnet/api/system.timespan) | | - -## Properties - - - - -### Delay - - -```csharp -public TimeSpan? Delay { get; init; } -``` - - -**Value:** [TimeSpan?](https://learn.microsoft.com/dotnet/api/system.timespan) - - - - -### Text - - -```csharp -public string Text { get; init; } -``` - - -**Value:** [string](https://learn.microsoft.com/dotnet/api/system.string) diff --git a/docs-site/src/content/docs/api/E2E.Engine/index.md b/docs-site/src/content/docs/api/E2E.Engine/index.md index 413a71b75..f6000322f 100644 --- a/docs-site/src/content/docs/api/E2E.Engine/index.md +++ b/docs-site/src/content/docs/api/E2E.Engine/index.md @@ -79,7 +79,6 @@ Public types in the `E2E.Engine` namespace. | [LocatorAction.Hover](/api/E2E.Engine/LocatorAction.Hover/) | Moves the pointer over the node. | | [LocatorAction.LongPress](/api/E2E.Engine/LocatorAction.LongPress/) | A press held on the node for Duration, or for the engine's own default hold when it is null. | | [LocatorAction.Press](/api/E2E.Engine/LocatorAction.Press/) | | -| [LocatorAction.PressSequentially](/api/E2E.Engine/LocatorAction.PressSequentially/) | Focuses the node and types Text one character at a time. | | [LocatorAction.ScrollIntoView](/api/E2E.Engine/LocatorAction.ScrollIntoView/) | Scrolls the node's nearest scrollable container until the node is in the viewport. | | [LocatorAction.SecondaryTap](/api/E2E.Engine/LocatorAction.SecondaryTap/) | A right click, or the platform's equivalent. Modifiers are held for the click, as for LocatorAction.Tap. | | [LocatorAction.Select](/api/E2E.Engine/LocatorAction.Select/) | Selects one option: the one whose value or label is Text, or the one Option names by its label, its value, or its index. Exactly one of the two is set. | diff --git a/docs-site/src/content/docs/api/E2E/ModelUsage.md b/docs-site/src/content/docs/api/E2E/ModelUsage.md deleted file mode 100644 index e78bad60b..000000000 --- a/docs-site/src/content/docs/api/E2E/ModelUsage.md +++ /dev/null @@ -1,97 +0,0 @@ ---- -title: "ModelUsage" -description: "ModelUsage in E2E." -editUrl: false -prev: false -next: false -sidebar: - label: "ModelUsage" - hidden: true ---- - - - -Namespace: [E2E](/api/E2E/) - - -```csharp -public sealed class ModelUsage -``` - - -**Inheritance:** [object](https://learn.microsoft.com/dotnet/api/system.object) - -[View source](https://github.com/hardkoded/e2e-dotnet/blob/main/src/E2E/Agent/Model.cs) - -## Properties - - - - -### CacheReadTokens - - -```csharp -public int? CacheReadTokens { get; init; } -``` - - -Input tokens the provider served from its prompt cache, when it reports the split. A part of [InputTokens](#E2E_ModelUsage_InputTokens). - -**Value:** [int?](https://learn.microsoft.com/dotnet/api/system.int32) - - - - -### CacheWriteTokens - - -```csharp -public int? CacheWriteTokens { get; init; } -``` - - -Input tokens the provider wrote to its prompt cache, when it reports the split. A part of [InputTokens](#E2E_ModelUsage_InputTokens). - -**Value:** [int?](https://learn.microsoft.com/dotnet/api/system.int32) - - - - -### EstimatedCostUsd - - -```csharp -public double? EstimatedCostUsd { get; init; } -``` - - -The billed cost of the call in US dollars, when the provider reports one. - -**Value:** [double?](https://learn.microsoft.com/dotnet/api/system.double) - - - - -### InputTokens - - -```csharp -public int InputTokens { get; init; } -``` - - -**Value:** [int](https://learn.microsoft.com/dotnet/api/system.int32) - - - - -### OutputTokens - - -```csharp -public int OutputTokens { get; init; } -``` - - -**Value:** [int](https://learn.microsoft.com/dotnet/api/system.int32) diff --git a/docs-site/src/content/docs/api/E2E/index.md b/docs-site/src/content/docs/api/E2E/index.md index bece6eeb3..03b1e2a5a 100644 --- a/docs-site/src/content/docs/api/E2E/index.md +++ b/docs-site/src/content/docs/api/E2E/index.md @@ -125,7 +125,6 @@ The core. Agent, Screen, Expect, the engines, model clients, config, and the rep | [ModelTool](/api/E2E/ModelTool/) | | | [ModelToolCall](/api/E2E/ModelToolCall/) | | | [ModelToolChoice](/api/E2E/ModelToolChoice/) | The AI SDK's ToolChoice: auto lets the model answer in prose or call any offered tool, required makes it call some tool, and a named tool makes it call that one. Each provider sends its own form of it (Anthropic's any, Bedrock's any, OpenAI's required, Gemini's ANY). | -| [ModelUsage](/api/E2E/ModelUsage/) | | | [ModelWarning](/api/E2E/ModelWarning/) | One of upstream's call warnings (CallWarning): the provider changed the request instead of failing it. A provider that sends auto in place of the forced tool choice it was asked for says so with Type unsupported and Feature toolChoice. | | [NewSessionOptions](/api/E2E/NewSessionOptions/) | What NewSessionAsync is given. Upstream's NewSessionOptions (core/launch.ts). | | [OpenAiCompatibleModel](/api/E2E/OpenAiCompatibleModel/) | Chat-completions client for OpenAI and any server that speaks the same tool-call JSON. Set BaseUrl to a local server such as http://127.0.0.1:11434/v1. The API key is optional for servers that do not check one. ModelProviders has presets for OpenRouter, the Vercel AI Gateway, SpaceXAI, and Ollama. | diff --git a/docs-site/src/content/docs/docs/core.mdx b/docs-site/src/content/docs/docs/core.mdx index de3aaf92f..1f118e1e7 100644 --- a/docs-site/src/content/docs/docs/core.mdx +++ b/docs-site/src/content/docs/docs/core.mdx @@ -45,7 +45,7 @@ Not ported yet: closing the host on SIGINT, SIGTERM, and SIGHUP. Issue [#1677](h ## Let the agent act -`CoreAgents.CreateAgent(session, options)` puts an agent over one session. Its `ActAsync`, `AssertAsync`, `WaitForAsync`, and `ExtractAsync` work as they do in a [test](/reference/agent). +`CoreAgents.CreateAgent(session, options)` puts an agent over one session. Its `ActAsync`, `AssertAsync`, `WaitForAsync`, and `ExtractAsync` work as they do in a test. ```csharp await using var shop = await Launch.LaunchAsync(new WebEngine(), new LaunchOptions { App = new LaunchApp { Url = "https://shop.example.com" } }); @@ -84,7 +84,7 @@ else record Order(int Total); ``` -`CreateAgentOptions` takes the options of an [`agents` entry](/reference/config#agents) (`AgentOptions`), such as `Model`, `Judge`, `MaxSteps`, and `Context`, plus `Cache`. There is no default model: an agent without one fails its first call with `MODEL_UNAVAILABLE`. An `ActAsync` step has its own timeout, 120000 ms: set it per call with `Timeout`. A call takes no `Agent` option: it is `INVALID_ARGUMENT`, so create another agent for another configuration. A session that did not come from `Host.NewSessionAsync` is `INVALID_ARGUMENT` too. +`CreateAgentOptions` takes the options of an `agents` entry (`AgentOptions`), such as `Model`, `Judge`, `MaxSteps`, and `Context`, plus `Cache`. There is no default model: an agent without one fails its first call with `MODEL_UNAVAILABLE`. An `ActAsync` step has its own timeout, 120000 ms: set it per call with `Timeout`. A call takes no `Agent` option: it is `INVALID_ARGUMENT`, so create another agent for another configuration. A session that did not come from `Host.NewSessionAsync` is `INVALID_ARGUMENT` too. ### Replay recorded steps @@ -92,7 +92,7 @@ With `Cache`, `ActAsync` records each flow and replays it on the next run with n A recording is keyed by the `CacheKey`, the instruction, its params, the app, and how many times the session made the same call. Pass a value that changes between runs as a param: a value the agent reads off the screen is decided by the model again on each run. -A recording is saved when the session closes, or its host does, and only when a check passed after the step, such as `Locator.WaitForAsync()`, `AssertAsync`, or `WaitForAsync`. When an agent step fails, the recordings not yet confirmed at that moment are dropped. A failure of your own code's locator call, such as a `WaitForAsync` that ran out, drops nothing. A step that fails only because no model answered (`MODEL_UNAVAILABLE`, `MODEL_PROVIDER_FAILED`) leaves the recordings as they are. Read [Replay cache](/cache) for how a replay checks the screen and hands off to the model. +A recording is saved when the session closes, or its host does, and only when a check passed after the step, such as `Locator.WaitForAsync()`, `AssertAsync`, or `WaitForAsync`. When an agent step fails, the recordings not yet confirmed at that moment are dropped. A failure of your own code's locator call, such as a `WaitForAsync` that ran out, drops nothing. A step that fails only because no model answered (`MODEL_UNAVAILABLE`, `MODEL_PROVIDER_FAILED`) leaves the recordings as they are. Read [Replay cache](/features/caching/) for how a replay checks the screen and hands off to the model. | Option | Default | Meaning | | --- | --- | --- | diff --git a/src/E2E/Core/Launch.cs b/src/E2E/Core/Launch.cs index 3a83b052f..755e5d626 100644 --- a/src/E2E/Core/Launch.cs +++ b/src/E2E/Core/Launch.cs @@ -159,8 +159,9 @@ private async Task CloseOnceAsync() { await closer().ConfigureAwait(false); } - catch (Exception cause) when (cause is not OperationCanceledException) + catch (Exception cause) { + // Whatever a closer throws, a cancellation included, is reported after the attempt is closed. errors.Add(cause); } }