From 3abe1905ab9e971912b2aae5399980e88d288a59 Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 15 Nov 2026 12:12:29 +0000 Subject: [PATCH 1/3] Agent tools: fill_secret takes name, as upstream's type_secret The node target is now one id (#1649), so the secret's name no longer collides with a target field. The MCP type_secret description lists the configured secrets. Co-Authored-By: Claude Sonnet 5.5 Claude-Session: https://claude.ai/code/session_01Ls2e21pM1pH12yG3orQ5KF --- docs-site/src/content/docs/coding-agents.mdx | 2 +- skills/e2e-dotnet/references/agent.md | 3 ++- skills/e2e-dotnet/references/mcp.md | 2 +- src/E2E.Cli/Mcp/Session.cs | 3 ++- src/E2E.Cli/Mcp/SessionCatalog.cs | 15 +++++++++------ src/E2E.Cli/Mcp/TestLines.cs | 2 +- src/E2E/Agent/ActTools.cs | 4 ++-- src/E2E/Agent/AgentTools.cs | 6 +++--- tests/E2E.Cli.Tests/McpServer/E2EMcpTests.cs | 2 +- .../McpTestLines/E2EMcpTestLinesTests.cs | 2 +- .../McpTestLines/SessionTestLinesTests.cs | 6 +++--- ...AnInconclusiveJudgmentAfterASecretFillTests.cs | 4 ++-- .../AgentAct/AgentActVerdictMappingTests.cs | 4 ++-- .../SecretFillPolicyUnderAHostileModelTests.cs | 6 +++--- tests/E2E.Tests/AgentActVerbs/GesturesSession.cs | 8 ++++++++ .../TheGrammarToolsHaveClosedSchemasTests.cs | 2 +- tests/E2E.Tests/Explore/E2eExploreTests.cs | 2 +- .../SecretsCutShortByObservationLimitsTests.cs | 2 +- 18 files changed, 44 insertions(+), 31 deletions(-) diff --git a/docs-site/src/content/docs/coding-agents.mdx b/docs-site/src/content/docs/coding-agents.mdx index 6360157ff..2694191fa 100644 --- a/docs-site/src/content/docs/coding-agents.mdx +++ b/docs-site/src/content/docs/coding-agents.mdx @@ -58,7 +58,7 @@ claude mcp add e2e -- e2e mcp The server's instructions and every `open_session` result tell the agent to read the `writing-tests` guide (`e2e://guide/writing-tests`, or `e2e guide writing-tests` in a shell) before it writes or edits a test. -`open_session` names the app URL (and the port a free-port `url` was assigned) and the app command it started: its executable, readiness URL, and the names of its `env` and `inheritEnv`. It prints no `args` and no `env` values. The config reads the MCP server's environment, not the agent's shell. The `screenshot` tool takes an optional `path` to save a `.png` file inside the project. `type_secret {target, secret}` is listed when the config declares `credentials` or `secrets`: it fills a credential's password (the secret `.password`) into a password field, or a `secrets` entry into any editable field, and the agent never sees the value. The opening text lists each credential's name, username, and password secret, and the test line names `Credentials.User(name).Password` or `Secrets.Get(name)`. What a test run through `open_session {test}` prints goes to the server's stderr with every secret value replaced by its `` marker, even when written in pieces; output printed while a config loads is held until its secrets are known, and withheld (with a warning line) if the load fails. +`open_session` names the app URL (and the port a free-port `url` was assigned) and the app command it started: its executable, readiness URL, and the names of its `env` and `inheritEnv`. It prints no `args` and no `env` values. The config reads the MCP server's environment, not the agent's shell. The `screenshot` tool takes an optional `path` to save a `.png` file inside the project. `type_secret {target, name}` is listed when the config declares `credentials` or `secrets`: it fills a credential's password (the secret `.password`) into a password field, or a `secrets` entry into any editable field, and the agent never sees the value. The opening text lists each credential's name, username, and password secret, and the test line names `Credentials.User(name).Password` or `Secrets.Get(name)`. What a test run through `open_session {test}` prints goes to the server's stderr with every secret value replaced by its `` marker, even when written in pieces; output printed while a config loads is held until its secrets are known, and withheld (with a warning line) if the load fails. Each node in `observe` starts its line with a node id (`#n42 button "Save"`), and an action takes that id as its `target`. `observe` shows the whole screen under a revision (`Current screen (revision b3, path /, 12 nodes):`). An action reports only what changed since the screen the agent last received: the lines that were `added`, `changed` (with what each read before), or `removed`. An id stays the same for as long as its element exists, so a node that is not listed as removed is still on screen under the id the agent has. When the screen changed mostly, the action sends it whole again. diff --git a/skills/e2e-dotnet/references/agent.md b/skills/e2e-dotnet/references/agent.md index 4ee715b32..5d6447bf1 100644 --- a/skills/e2e-dotnet/references/agent.md +++ b/skills/e2e-dotnet/references/agent.md @@ -291,7 +291,8 @@ On an engine that declares `EngineCapabilities.TapModifiers` (the web engine), extends a selection. The held keys are recorded and replayed from the cache. A project tool adds its own name to this list (see "Project tools"). `fill_secret` fills any editable field, a rich-text editor included, with a -generic secret; a password secret needs a password field. +generic secret; a password secret needs a password field. It takes `target` +(the node id) and `name` (the secret's name), as upstream's `type_secret`. ## AssertAsync, WaitForAsync, ExtractAsync: one question diff --git a/skills/e2e-dotnet/references/mcp.md b/skills/e2e-dotnet/references/mcp.md index 99c5608ff..430a3f664 100644 --- a/skills/e2e-dotnet/references/mcp.md +++ b/skills/e2e-dotnet/references/mcp.md @@ -138,7 +138,7 @@ config declares `credentials` or a `secrets` entry. | `tap`, `double_tap`, `long_press`, `right_click`, `hover`, `check`, `uncheck`, `clear` `{target}` | Act on one node. `long_press` is for a control with a long-press menu, `right_click` opens a context menu, `hover` shows what the pointer resting on a node reveals. On an engine that declares `TapModifiers` (the web engine), `tap`, `double_tap`, and `right_click` also take an optional `modifiers` list (`Shift`, `Control`, `Alt`, `Meta`, `ControlOrMeta`) to hold keys for the click, as a Shift-click extends a selection | | `drag {target, to}` | Drag one node and drop it on another, both by node id | | `type {target, text}` | Replace the text in a field. With a keyboard (the web engine), `target` is optional: without one the text is typed into whatever has focus, and `replace` clears the field first | -| `type_secret {target, secret}` | Fill a configured secret by name: a credential's password (`.password`) into a password field, a `secrets` entry into any editable field; you never see the value, and its test line names the credential or secret (`FillAsync(Credentials.User(name).Password)`, `FillAsync(Secrets.Get(name))`) | +| `type_secret {target, name}` | Fill a configured secret by name: a credential's password (`.password`) into a password field, a `secrets` entry into any editable field; you never see the value, and its test line names the credential or secret (`FillAsync(Credentials.User(name).Password)`, `FillAsync(Secrets.Get(name))`) | | `press {key, target?}` | Press a key on a node, or on whatever has focus | | `select {target, value}` | Choose an option | | `scroll {direction, times?, target?}`, `scroll_to {target?, text?, direction?}` | Move the viewport, or page to a node | diff --git a/src/E2E.Cli/Mcp/Session.cs b/src/E2E.Cli/Mcp/Session.cs index 44d1e7bba..01bb65029 100644 --- a/src/E2E.Cli/Mcp/Session.cs +++ b/src/E2E.Cli/Mcp/Session.cs @@ -434,7 +434,8 @@ [.. secrets.Select(secret => secret.Name)], session.Engine.CanDismissKeyboard, config.Agent.Tools, target.Platform, - message => Log("warning", message)); + message => Log("warning", message), + [.. secrets.Select(secret => new ExecutorSecret(secret.Name, secret.Purpose))]); // The app as the session resolved it: the declared URL on the port its command was started on. var app = TargetApps.ResolveTargetApp(target.Name, declared.App, config.ProjectRoot, Uri.TryCreate(target.App.Url, UriKind.Absolute, out var served) ? served.Port : null); diff --git a/src/E2E.Cli/Mcp/SessionCatalog.cs b/src/E2E.Cli/Mcp/SessionCatalog.cs index f4f752f3f..a739d4411 100644 --- a/src/E2E.Cli/Mcp/SessionCatalog.cs +++ b/src/E2E.Cli/Mcp/SessionCatalog.cs @@ -79,7 +79,7 @@ private SessionCatalog(IReadOnlyList> entries, /// (), which the point tools (tap_at and the other _at tools) follow. /// is : dismiss_keyboard is a tool only where it is true. /// - public static SessionCatalog Create(ISessionContext context, EngineCapabilities capabilities, IReadOnlyCollection secrets, string projectRoot, string attemptDir, Uri? appBase = null, SessionRecorder? recorder = null, string? testRan = null, TestLineFills? fills = null, IReadOnlyCollection? actionKinds = null, IReadOnlyCollection? pointerKinds = null, bool canDismissKeyboard = false, IReadOnlyDictionary? projectTools = null, string? platform = null, Action? warn = null) + public static SessionCatalog Create(ISessionContext context, EngineCapabilities capabilities, IReadOnlyCollection secrets, string projectRoot, string attemptDir, Uri? appBase = null, SessionRecorder? recorder = null, string? testRan = null, TestLineFills? fills = null, IReadOnlyCollection? actionKinds = null, IReadOnlyCollection? pointerKinds = null, bool canDismissKeyboard = false, IReadOnlyDictionary? projectTools = null, string? platform = null, Action? warn = null, IReadOnlyList? secretHandles = null) { ArgumentNullException.ThrowIfNull(context); ArgumentNullException.ThrowIfNull(secrets); @@ -229,7 +229,7 @@ public static SessionCatalog Create(ISessionContext context, EngineCapabilities if (secrets.Count > 0 && verbs.Contains(ExecutorVerb.TypeSecret)) { - Add("type_secret", TypeSecret(context, screen)); + Add("type_secret", TypeSecret(context, screen, secretHandles)); } Add("locate", Locate(context)); @@ -488,15 +488,18 @@ private static CatalogTool Verb(ISessionContext context, ScreenPresenter screen, Execute = async (args, token) => await tool.ExecuteAsync(args.ValueKind == JsonValueKind.Object ? JsonNode.Parse(args.GetRawText())!.AsObject() : throw new ConfigurationException("INVALID_ARGUMENT", "a project tool takes an object"), token).ConfigureAwait(false), }; - private static CatalogTool TypeSecret(ISessionContext context, ScreenPresenter screen) + private static CatalogTool TypeSecret(ISessionContext context, ScreenPresenter screen, IReadOnlyList? secretHandles) { return new CatalogTool { - Description = "Fill a configured secret into an editable field by its name. You never see the value: every output shows the secret as .", - InputSchema = Schema(Target, ("secret", Property("string", "The secret's name from the config"), true)), + Description = "Fill a configured secret into an editable field by its name. You never see the value: every output shows the secret as ." + + (secretHandles is { Count: > 0 } + ? " Available: " + string.Join(", ", secretHandles.Select(secret => "\"" + secret.Name + "\"" + (secret.Purpose is null ? "" : " (" + secret.Purpose + ")"))) + "." + : ""), + InputSchema = Schema(Target, ("name", Property("string", "The secret's name from the config"), true)), Execute = async (args, token) => { - var mapped = new JsonObject { ["target"] = args.GetProperty("target").GetString(), ["secret"] = args.GetProperty("secret").GetString() }; + var mapped = new JsonObject { ["target"] = args.GetProperty("target").GetString(), ["name"] = args.GetProperty("name").GetString() }; // Marked before the fill runs: a fill that fails after typing still leaves the secret on screen. context.SecretFilled(); var outcome = await RunAsync(context, screen, "type_secret", "fill_secret", args, token, mapped).ConfigureAwait(false); diff --git a/src/E2E.Cli/Mcp/TestLines.cs b/src/E2E.Cli/Mcp/TestLines.cs index 985e9adee..f5ff2ab93 100644 --- a/src/E2E.Cli/Mcp/TestLines.cs +++ b/src/E2E.Cli/Mcp/TestLines.cs @@ -287,7 +287,7 @@ private async Task BuildAsync(string name, JsonElement args, Cancellat case "type_secret": { - return SecretExpression(Field(args, "secret") ?? "") is { } secret + return SecretExpression(Field(args, "name") ?? "") is { } secret ? await OnTargetAsync(target, locator => locator + ".FillAsync(" + secret + ")", token).ConfigureAwait(false) : LinePlan.None("the secret is neither a credential password nor a config secret"); } diff --git a/src/E2E/Agent/ActTools.cs b/src/E2E/Agent/ActTools.cs index 97b6cc257..d4e4aff37 100644 --- a/src/E2E/Agent/ActTools.cs +++ b/src/E2E/Agent/ActTools.cs @@ -192,7 +192,7 @@ private async Task RunActionAsync(ModelToolCall call, IExecutorActi "right_click" => actions.SecondaryTapAsync(target, ClickOptionsOf(arguments), token), "hover" => actions.HoverAsync(target, token), "fill" => actions.TypeAsync(target, Args.String(arguments, "value") ?? "", token), - "fill_secret" => actions.TypeSecretAsync(target, Args.String(arguments, "secret") ?? "", token), + "fill_secret" => actions.TypeSecretAsync(target, Args.String(arguments, "name") ?? "", token), "select" => actions.SelectAsync(target, Args.String(arguments, "value") ?? "", token), "check" => actions.CheckAsync(target, true, token), "uncheck" => actions.CheckAsync(target, false, token), @@ -489,7 +489,7 @@ private static ActionLabel NodeLabel(string tool, JsonElement arguments, string { var key = Args.String(arguments, "key") ?? "Enter"; var value = Args.String(arguments, "value") ?? ""; - var secret = Args.String(arguments, "secret") ?? ""; + var secret = Args.String(arguments, "name") ?? ""; return tool switch { "tap" => new("Tapped " + reference + ".", "tap " + reference), diff --git a/src/E2E/Agent/AgentTools.cs b/src/E2E/Agent/AgentTools.cs index 146fe9019..937777329 100644 --- a/src/E2E/Agent/AgentTools.cs +++ b/src/E2E/Agent/AgentTools.cs @@ -258,15 +258,15 @@ private static readonly (string Tool, ExecutorVerb Bare, ExecutorVerb Node, stri /// /// Upstream's type_secret tool (agent/primitives.ts), which the port names fill_secret. Offered only when the step - /// declared secrets and the engine can fill: its description lists each as "name" (purpose). The secret's name is secret - /// where upstream's is name, which the target's own name field already takes. + /// declared secrets and the engine can fill: its description lists each as "name" (purpose). It takes target and the + /// secret's name, as upstream. /// private static ModelTool FillSecret(IReadOnlyList secrets) => Tool( "fill_secret", "Fill one declared secret into an input by its name; the plaintext never passes through you. A password fills only a password field; a generic-secret fills any editable input. Available: " + string.Join(", ", secrets.Select(secret => "\"" + secret.Name + "\"" + (secret.Purpose is null ? "" : " (" + secret.Purpose + ")"))) + ".", - WithValue(Target, "secret")); + WithValue(Target, "name")); private static readonly ModelTool LongPress = Tool( "long_press", diff --git a/tests/E2E.Cli.Tests/McpServer/E2EMcpTests.cs b/tests/E2E.Cli.Tests/McpServer/E2EMcpTests.cs index 4f66b2e3c..fe4115bac 100644 --- a/tests/E2E.Cli.Tests/McpServer/E2EMcpTests.cs +++ b/tests/E2E.Cli.Tests/McpServer/E2EMcpTests.cs @@ -171,7 +171,7 @@ public async Task Opens_a_session_with_its_catalog_then_locates_acts_fills_a_sec Assert.Contains("Tapped the point", pointed.Text, StringComparison.Ordinal); Assert.DoesNotContain("No screenshot has been taken", pointed.Text, StringComparison.Ordinal); - var filled = await _served.CallAsync("type_secret", new() { ["target"] = ServedCli.RefOf(after.Text, "textbox \"Password\""), ["secret"] = "adminPassword" }); + var filled = await _served.CallAsync("type_secret", new() { ["target"] = ServedCli.RefOf(after.Text, "textbox \"Password\""), ["name"] = "adminPassword" }); Assert.False(filled.IsError, filled.Text); Assert.Contains("Filled secret \"adminPassword\" into #", filled.Text, StringComparison.Ordinal); Assert.DoesNotContain(Served.AdminPassword, filled.Text, StringComparison.Ordinal); diff --git a/tests/E2E.Cli.Tests/McpTestLines/E2EMcpTestLinesTests.cs b/tests/E2E.Cli.Tests/McpTestLines/E2EMcpTestLinesTests.cs index 2105de9af..559996283 100644 --- a/tests/E2E.Cli.Tests/McpTestLines/E2EMcpTestLinesTests.cs +++ b/tests/E2E.Cli.Tests/McpTestLines/E2EMcpTestLinesTests.cs @@ -57,7 +57,7 @@ public async Task Answers_each_action_with_its_test_line_and_the_drafted_lines_p (string Tool, Dictionary Args, string Line)[] steps = [ ("type", new() { ["target"] = ServedCli.RefOf(screen, "textbox \"Username\""), ["text"] = Username }, "await Screen.GetByRole(\"textbox\", \"Username\").FillAsync(Credentials.User(\"admin\").Username);"), - ("type_secret", new() { ["target"] = ServedCli.RefOf(screen, "textbox \"Password\""), ["secret"] = "admin.password" }, "await Screen.GetByRole(\"textbox\", \"Password\").FillAsync(Credentials.User(\"admin\").Password);"), + ("type_secret", new() { ["target"] = ServedCli.RefOf(screen, "textbox \"Password\""), ["name"] = "admin.password" }, "await Screen.GetByRole(\"textbox\", \"Password\").FillAsync(Credentials.User(\"admin\").Password);"), ("select", new() { ["target"] = ServedCli.RefOf(screen, "combobox \"Plan\""), ["value"] = "Pro" }, "await Screen.GetByRole(\"combobox\", \"Plan\").SelectOptionAsync(\"Pro\");"), ("check", new() { ["target"] = ServedCli.RefOf(screen, "checkbox \"Accept terms\"") }, "await Screen.GetByRole(\"checkbox\", \"Accept terms\").CheckAsync();"), ( diff --git a/tests/E2E.Cli.Tests/McpTestLines/SessionTestLinesTests.cs b/tests/E2E.Cli.Tests/McpTestLines/SessionTestLinesTests.cs index 112e07b34..2757c88e2 100644 --- a/tests/E2E.Cli.Tests/McpTestLines/SessionTestLinesTests.cs +++ b/tests/E2E.Cli.Tests/McpTestLines/SessionTestLinesTests.cs @@ -200,10 +200,10 @@ public async Task Partial_Writes_credentials_and_secrets_by_name_never_by_value( NoteOf(await session.RunAsync("type", """{ "target": "n40", "text": "ada@example.test" }"""))); Assert.Equal( "Test line: await Screen.GetByRole(\"textbox\", \"Password\").FillAsync(Credentials.User(\"admin\").Password);", - NoteOf(await session.RunAsync("type_secret", """{ "target": "n41", "secret": "admin.password" }"""))); + NoteOf(await session.RunAsync("type_secret", """{ "target": "n41", "name": "admin.password" }"""))); Assert.Equal( "Test line: await Screen.GetByRole(\"textbox\", \"API key\").FillAsync(Secrets.Get(\"apiKey\"));", - NoteOf(await session.RunAsync("type_secret", """{ "target": "n42", "secret": "apiKey" }"""))); + NoteOf(await session.RunAsync("type_secret", """{ "target": "n42", "name": "apiKey" }"""))); Assert.DoesNotContain("sk-live-0000", session.Lines.Draft(), StringComparison.Ordinal); } @@ -283,7 +283,7 @@ public async Task Partial_Says_why_an_action_has_no_line_for_no_node_a_scroll_a_ NoteOf(await session.RunAsync("press", """{ "target": "n60", "key": "ArrowDown", "times": 3 }"""))); Assert.Matches(@"^No test line: a test action brings its node into view itself", NoteOf(await session.RunAsync("scroll", """{ "direction": "down" }"""))); Assert.Matches(@"^No test line: scrolling to a text has none", NoteOf(await session.RunAsync("scroll_to", """{ "text": "Row 40" }"""))); - Assert.Matches(@"^No test line: the secret is neither", NoteOf(await session.RunAsync("type_secret", """{ "target": "n60", "secret": "unknown" }"""))); + Assert.Matches(@"^No test line: the secret is neither", NoteOf(await session.RunAsync("type_secret", """{ "target": "n60", "name": "unknown" }"""))); Assert.Matches(@"^No test line: a locator tap takes no modifier keys", NoteOf(await session.RunAsync("tap", """{ "target": "n2", "modifiers": ["Shift"] }"""))); Assert.Matches(@"^No test line: a locator right click takes no modifier keys", NoteOf(await session.RunAsync("right_click", """{ "target": "n2", "modifiers": ["Shift"] }"""))); Assert.Equal("No test line: the drop target: node n404 is not on the newest screen.", NoteOf(await session.RunAsync("drag", """{ "target": "n2", "to": "n404" }"""))); diff --git a/tests/E2E.Tests/Agent/AnInconclusiveJudgmentAfterASecretFillTests.cs b/tests/E2E.Tests/Agent/AnInconclusiveJudgmentAfterASecretFillTests.cs index 86eaa0429..68e8a8c03 100644 --- a/tests/E2E.Tests/Agent/AnInconclusiveJudgmentAfterASecretFillTests.cs +++ b/tests/E2E.Tests/Agent/AnInconclusiveJudgmentAfterASecretFillTests.cs @@ -22,8 +22,8 @@ public async Task Carries_the_vision_hint_a_secret_fill_does_not_withhold_pixels // The act model fills the password through fill_secret; a screenshot can still reach a judge // afterwards. var calls = 0; - var actor = new ScriptedModel(_ => calls++ == 0 - ? ModelResponses.Call("fill_secret", new { role = "textbox", name = "Password", secret = "member-password" }) + var actor = new ScriptedModel(request => calls++ == 0 + ? ModelResponses.Call("fill_secret", new { target = E2E.Tests.AgentActVerbs.GesturesSession.RefFor(request, "textbox \"Password\""), name = "member-password" }) : ModelResponses.Done("passed", "filled the password")); var judge = new ScriptedModel(_ => ModelResponses.Done("failed", "no order total or line items are on this screen", "ASSERTION_INCONCLUSIVE")); await using var session = await E2ESession.StartAsync(new E2ESessionOptions diff --git a/tests/E2E.Tests/AgentAct/AgentActVerdictMappingTests.cs b/tests/E2E.Tests/AgentAct/AgentActVerdictMappingTests.cs index 8af125a6f..c0692150e 100644 --- a/tests/E2E.Tests/AgentAct/AgentActVerdictMappingTests.cs +++ b/tests/E2E.Tests/AgentAct/AgentActVerdictMappingTests.cs @@ -118,9 +118,9 @@ public async Task Fills_a_declared_secret_through_fill_secret_and_refuses_an_und })); var model = new ScriptedModel(request => request.Messages.Count(message => message.Role == "tool") switch { - 0 => ModelResponses.Call("fill_secret", new { role = "textbox", name = "Password", secret = "admin.password" }), + 0 => ModelResponses.Call("fill_secret", new { target = E2E.Tests.AgentActVerbs.GesturesSession.RefFor(request, "textbox \"Password\""), name = "admin.password" }), // A configured secret the step never declared is refused. - 1 => ModelResponses.Call("fill_secret", new { role = "textbox", name = "Password", secret = "stripe-key" }), + 1 => ModelResponses.Call("fill_secret", new { target = E2E.Tests.AgentActVerbs.GesturesSession.RefFor(request, "textbox \"Password\""), name = "stripe-key" }), _ => ModelResponses.Done("passed", "filled the declared secret only"), }); var result = await RunAsync( diff --git a/tests/E2E.Tests/AgentActStress/SecretFillPolicyUnderAHostileModelTests.cs b/tests/E2E.Tests/AgentActStress/SecretFillPolicyUnderAHostileModelTests.cs index df1aedc52..4424fa995 100644 --- a/tests/E2E.Tests/AgentActStress/SecretFillPolicyUnderAHostileModelTests.cs +++ b/tests/E2E.Tests/AgentActStress/SecretFillPolicyUnderAHostileModelTests.cs @@ -31,8 +31,8 @@ public async Task Fills_a_generic_secret_into_a_contenteditable_host_and_refuses toolResults.Add(LastToolResult(request)); return calls++ switch { - 0 => ModelResponses.Call("fill_secret", new { role = "textbox", name = "Notes", secret = "admin" }), - 1 => ModelResponses.Call("fill_secret", new { role = "textbox", name = "Notes", secret = "stripe-key" }), + 0 => ModelResponses.Call("fill_secret", new { target = E2E.Tests.AgentActVerbs.GesturesSession.RefFor(request, "textbox \"Notes\""), name = "admin" }), + 1 => ModelResponses.Call("fill_secret", new { target = E2E.Tests.AgentActVerbs.GesturesSession.RefFor(request, "textbox \"Notes\""), name = "stripe-key" }), _ => ModelResponses.Done("passed", "filled the API key"), }; }); @@ -62,7 +62,7 @@ public async Task Never_leaks_the_plaintext_into_prompts_or_tool_results() { using var site = await WebFixtureApp.StartAsync(); var model = new ScriptedModel(request => LastToolResult(request) == "" - ? ModelResponses.Call("fill_secret", new { role = "textbox", name = "Password", secret = "admin" }) + ? ModelResponses.Call("fill_secret", new { target = E2E.Tests.AgentActVerbs.GesturesSession.RefFor(request, "textbox \"Password\""), name = "admin" }) : ModelResponses.Done("passed", "filled the secret")); await using var session = await StartAsync(site.Url, model); await WebSemanticsTests.RunAsync(session, async () => diff --git a/tests/E2E.Tests/AgentActVerbs/GesturesSession.cs b/tests/E2E.Tests/AgentActVerbs/GesturesSession.cs index b2ade56d8..dd3c59773 100644 --- a/tests/E2E.Tests/AgentActVerbs/GesturesSession.cs +++ b/tests/E2E.Tests/AgentActVerbs/GesturesSession.cs @@ -363,6 +363,14 @@ await RunAsync(session, async () => return (error, session.StepRecords); } + /// The ref of the last listed node whose line matches , in every message of a model request. + public static string RefFor(ModelRequest request, string line) + { + var prompt = string.Join('\n', request.Messages.Select(message => message.Content)); + var matches = Regex.Matches(prompt, @"^\s*(?:added |changed )?#(e\d+) " + Regex.Escape(line) + @"(?: .*)?$", RegexOptions.Multiline); + return matches.Count > 0 ? matches[^1].Groups[1].Value : throw new InvalidOperationException("No line " + line + " in the screen:\n" + prompt); + } + /// The ref of the last listed node whose line matches . public static string RefFor(string prompt, string line) { diff --git a/tests/E2E.Tests/AgentGrammarTools/TheGrammarToolsHaveClosedSchemasTests.cs b/tests/E2E.Tests/AgentGrammarTools/TheGrammarToolsHaveClosedSchemasTests.cs index 644c1e498..7a939d60f 100644 --- a/tests/E2E.Tests/AgentGrammarTools/TheGrammarToolsHaveClosedSchemasTests.cs +++ b/tests/E2E.Tests/AgentGrammarTools/TheGrammarToolsHaveClosedSchemasTests.cs @@ -24,7 +24,7 @@ public sealed class TheGrammarToolsHaveClosedSchemasTests ["right_click"] = """{ "target": "n1" }""", ["hover"] = """{ "target": "n1" }""", ["fill"] = """{ "target": "n1", "value": "ada" }""", - ["fill_secret"] = """{ "target": "n1", "secret": "admin" }""", + ["fill_secret"] = """{ "target": "n1", "name": "admin" }""", ["press"] = """{ "target": "n1", "key": "Shift+ArrowLeft", "times": 8 }""", ["select"] = """{ "target": "n1", "value": "Pro" }""", ["check"] = """{ "target": "n1" }""", diff --git a/tests/E2E.Tests/Explore/E2eExploreTests.cs b/tests/E2E.Tests/Explore/E2eExploreTests.cs index 4db64c9d6..07066e97e 100644 --- a/tests/E2E.Tests/Explore/E2eExploreTests.cs +++ b/tests/E2E.Tests/Explore/E2eExploreTests.cs @@ -514,7 +514,7 @@ public async Task Lets_the_explorer_sign_in_with_a_configured_credential_through { if (call.Turn == 1) { - return ModelResponses.Call("fill_secret", new { role = "textbox", name = "Password", secret = "ada.password" }); + return ModelResponses.Call("fill_secret", new { target = E2E.Tests.AgentActVerbs.GesturesSession.RefFor(call.Request, "textbox \"Password\""), name = "ada.password" }); } fillResult = call.LastToolResult; diff --git a/tests/E2E.Tests/TruncatedSecrets/SecretsCutShortByObservationLimitsTests.cs b/tests/E2E.Tests/TruncatedSecrets/SecretsCutShortByObservationLimitsTests.cs index e0ae86229..d4c7f94b3 100644 --- a/tests/E2E.Tests/TruncatedSecrets/SecretsCutShortByObservationLimitsTests.cs +++ b/tests/E2E.Tests/TruncatedSecrets/SecretsCutShortByObservationLimitsTests.cs @@ -110,7 +110,7 @@ public async Task Keeps_the_cut_fragment_out_of_the_failure_screen_the_report_an return last.Role == "tool" ? ModelResponses.Done("passed", "filled") - : ModelResponses.Call("fill_secret", new { role = "textbox", name = "Source", secret = "member" }); + : ModelResponses.Call("fill_secret", new { target = E2E.Tests.AgentActVerbs.GesturesSession.RefFor(request, "textbox \"Source\""), name = "member" }); }); await using var session = await E2ESession.StartAsync(new E2ESessionOptions { From f36d601a92d38a50ce0ca958f1b39704f662ca53 Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 15 Nov 2026 12:14:54 +0000 Subject: [PATCH 2/3] Read the fill_secret name only in its own lead Co-Authored-By: Claude Sonnet 5.5 Claude-Session: https://claude.ai/code/session_01Ls2e21pM1pH12yG3orQ5KF --- src/E2E/Agent/ActTools.cs | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/src/E2E/Agent/ActTools.cs b/src/E2E/Agent/ActTools.cs index d4e4aff37..6aaf807ba 100644 --- a/src/E2E/Agent/ActTools.cs +++ b/src/E2E/Agent/ActTools.cs @@ -485,11 +485,15 @@ private async Task FailedAsync(ActionLabel label, E2EException erro // The done and attempt leads of a node tool, upstream's wording; `reference` stands where upstream's `#id` does. fill and fill_secret // keep the port's tool names in the attempt, where upstream's are type and type_secret. clear has no upstream tool. + private static ActionLabel SecretLabel(string secret, string reference) => new( + "Filled secret \"" + secret + "\" into " + reference + "; its value is masked in every observation.", + "fill_secret \"" + secret + "\" into " + reference, + false); + private static ActionLabel NodeLabel(string tool, JsonElement arguments, string reference) { var key = Args.String(arguments, "key") ?? "Enter"; var value = Args.String(arguments, "value") ?? ""; - var secret = Args.String(arguments, "name") ?? ""; return tool switch { "tap" => new("Tapped " + reference + ".", "tap " + reference), @@ -498,10 +502,7 @@ private static ActionLabel NodeLabel(string tool, JsonElement arguments, string "right_click" => new("Right-clicked " + reference + ".", "right_click " + reference), "hover" => new("Hovered over " + reference + ".", "hover " + reference), "fill" => new("Typed into " + reference + ".", "fill " + reference), - "fill_secret" => new( - "Filled secret \"" + secret + "\" into " + reference + "; its value is masked in every observation.", - "fill_secret \"" + secret + "\" into " + reference, - false), + "fill_secret" => SecretLabel(Args.String(arguments, "name") ?? "", reference), "press" => new("Pressed " + key + " on " + reference + ".", "press " + key + " on " + reference, !MovesFocusOnly(key)), "select" => new("Selected \"" + value + "\" in " + reference + ".", "select \"" + value + "\" in " + reference), "check" => new("Checked " + reference + ".", "check " + reference), From 03492b2439b7e95ca9a29f973892b1cb77be292b Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 15 Nov 2026 12:36:17 +0000 Subject: [PATCH 3/3] Give the replay look test's fill_secret a node id, since ScriptedModel reads name as the node Co-Authored-By: Claude Sonnet 5.5 Claude-Session: https://claude.ai/code/session_01Ls2e21pM1pH12yG3orQ5KF --- .../TraceReplay/ReplayThroughTheActTests.cs | 21 +++++++++++-------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/tests/E2E.Tests/TraceReplay/ReplayThroughTheActTests.cs b/tests/E2E.Tests/TraceReplay/ReplayThroughTheActTests.cs index 6341b6ffd..bc38867a3 100644 --- a/tests/E2E.Tests/TraceReplay/ReplayThroughTheActTests.cs +++ b/tests/E2E.Tests/TraceReplay/ReplayThroughTheActTests.cs @@ -28,13 +28,13 @@ public async Task Partial_Settles_each_look_as_far_as_the_previous_actions_polic var tap = ModelResponses.Tap("button", "Upgrade"); var looks = await RecordThenReplayAsync( TapWorld, - tap, - ModelResponses.Fill("textbox", "Email", "a@b.c"), - tap, - ModelResponses.Call("fill_secret", new { role = "textbox", name = "Password", secret = "password" }), - tap, - ModelResponses.Call("select", new { role = "combobox", name = "Plan", value = "Pro" }), - tap); + _ => tap, + _ => ModelResponses.Fill("textbox", "Email", "a@b.c"), + _ => tap, + request => ModelResponses.Call("fill_secret", new { target = E2E.Tests.AgentActVerbs.GesturesSession.RefFor(request, "textbox \"Password\""), name = "password" }), + _ => tap, + _ => ModelResponses.Call("select", new { role = "combobox", name = "Plan", value = "Pro" }), + _ => tap); // The start holds still and serves the first tap; so does the look after each tap or select. The look // after a typed fill waits for its value only. A secret fill is masked out of the tree, so the look after @@ -419,11 +419,14 @@ async Task Body(TestContext ctx) }); /// Records the steps live on a fresh cache, then replays them, and returns the replaying act's looks. - private static async Task> RecordThenReplayAsync(Func world, params ModelResponse[] steps) + private static Task> RecordThenReplayAsync(Func world, params ModelResponse[] steps) => + RecordThenReplayAsync(world, [.. steps.Select(step => (Func)(_ => step))]); + + private static async Task> RecordThenReplayAsync(Func world, params Func[] steps) { var directory = CoreTests.TempCache(); var next = 0; - var recorded = await ActAsync(world(), directory, new ScriptedModel(_ => next < steps.Length ? steps[next++] : ModelResponses.Done("passed", "Done."))); + var recorded = await ActAsync(world(), directory, new ScriptedModel(request => next < steps.Length ? steps[next++](request) : ModelResponses.Done("passed", "Done."))); Assert.Equal("missed", recorded.Mode); var replayed = await ActAsync(world(), directory, new ScriptedModel(_ => throw new InvalidOperationException("no model call expected")));