diff --git a/tests/E2E.Acp.Tests/Executor/AcpHarness.cs b/tests/E2E.Acp.Tests/Executor/AcpHarness.cs index e27cc517..e2c5a8ae 100644 --- a/tests/E2E.Acp.Tests/Executor/AcpHarness.cs +++ b/tests/E2E.Acp.Tests/Executor/AcpHarness.cs @@ -3,6 +3,9 @@ // SPDX-License-Identifier: Apache-2.0 using System.Diagnostics; +using System.Net; +using System.Net.Sockets; +using System.Text; using System.Text.Json; using System.Text.Json.Nodes; using E2E.Engine; @@ -56,6 +59,72 @@ internal sealed class TodoPage }); } +/// A local page for the web engine to open. Upstream's step context mocks the observation; the port opens a real page. +internal sealed class PixelSite : IDisposable +{ + private readonly HttpListener _listener = new(); + private readonly CancellationTokenSource _stop = new(); + + private PixelSite(string url) => Url = url; + + public string Url { get; } + + public static PixelSite Start(string html) + { + for (var attempt = 0; ; attempt++) + { + var probe = new TcpListener(IPAddress.Loopback, 0); + probe.Start(); + var port = ((IPEndPoint)probe.LocalEndpoint).Port; + probe.Stop(); + var site = new PixelSite("http://127.0.0.1:" + port.ToString(System.Globalization.CultureInfo.InvariantCulture) + "/"); + site._listener.Prefixes.Add(site.Url); + try + { + site._listener.Start(); + } + catch (HttpListenerException) when (attempt < 5) + { + site._listener.Close(); + continue; + } + + _ = site.ServeAsync(Encoding.UTF8.GetBytes(html)); + return site; + } + } + + public void Dispose() + { + _stop.Cancel(); + _listener.Close(); + _stop.Dispose(); + } + + private async Task ServeAsync(byte[] page) + { + while (!_stop.IsCancellationRequested) + { + try + { + var context = await _listener.GetContextAsync(); + context.Response.ContentType = "text/html; charset=utf-8"; + context.Response.ContentLength64 = page.Length; + await context.Response.OutputStream.WriteAsync(page); + context.Response.Close(); + } + catch (Exception) when (_stop.IsCancellationRequested || !_listener.IsListening) + { + return; + } + catch (Exception ex) when (ex is HttpListenerException or IOException or ObjectDisposedException) + { + // A client that hung up does not stop the site. + } + } + } +} + internal static class AcpHarness { /// The program that stands in for an agent. Its apphost lands beside the tests. @@ -112,14 +181,15 @@ public static async Task StartAsync( bool debug = false, string? projectRoot = null, IReadOnlyDictionary? agents = null, - IEngine? engine = null) + IEngine? engine = null, + string baseUrl = "https://todos.test") { var session = await E2ESession.StartAsync(new E2ESessionOptions { Engine = engine ?? new DocumentEngine(world), AgentExecutor = executor, Agents = agents, - BaseUrl = "https://todos.test", + BaseUrl = baseUrl, TestTitle = "acp", CacheEnabled = false, StepTimeout = TimeSpan.FromSeconds(60), diff --git a/tests/E2E.Acp.Tests/Executor/AcpTests.cs b/tests/E2E.Acp.Tests/Executor/AcpTests.cs index 258e809d..ce6bde82 100644 --- a/tests/E2E.Acp.Tests/Executor/AcpTests.cs +++ b/tests/E2E.Acp.Tests/Executor/AcpTests.cs @@ -10,7 +10,7 @@ namespace E2E.Acp.Tests.Executor; /// /// Ports describe('acp') of tester-army/e2e packages/acp/tests/executor.test.ts (0410fc9c). The scripted agent is the C# program -/// E2E.Acp.Tests.Agent, and the steps run through a real session over the in-memory document engine, since the grammar tools drive the +/// E2E.Acp.Tests.Agent, and the steps run through a real session over the in-memory document engine (the web engine where a test needs pixels), since the grammar tools drive the /// context of a step the runtime opened. The port's verdict tool is done (upstream's complete_step) and takes errorCode. /// [Collection(WorkingDirectoryCollection.Name)] @@ -593,12 +593,12 @@ public async Task Reports_a_model_the_agent_does_not_offer() } [Fact] - public async Task Partial_Lists_the_tools_once_and_tells_the_agent_which_ones_a_step_does_not_offer() + public async Task Lists_the_tools_once_and_tells_the_agent_which_ones_a_step_does_not_offer() { - // Partial: the document engine offers every step the same tools, so no step lacks a tool the session listed. - // fill_secret is not listed: the steps declare no secret. - var agent = Scripted(_directory, Turns([Pass], [Pass])); - await using var session = await StartAsync(agent.Executor, new TodoPage().World()); + // fill_secret is not listed: the steps declare no secret. The web engine takes the screenshot. + using var site = PixelSite.Start("t

Hello

"); + var agent = Scripted(_directory, Turns([Call("screenshot"), Pass], [Pass])); + await using var session = await StartAsync(agent.Executor, new TodoPage().World(), engine: new WebEngine(headless: true), baseUrl: site.Url); await session.Agent.ActAsync("add a todo"); await session.Agent.ActAsync("add a todo"); @@ -606,6 +606,9 @@ public async Task Partial_Lists_the_tools_once_and_tells_the_agent_which_ones_a_ var offered = Assert.Single(agent.Sessions())["session"]!["tools"]!.AsArray().Select(tool => tool!.GetValue()).ToList(); Assert.DoesNotContain("fill_secret", offered); Assert.DoesNotContain("Not offered in this step", agent.Prompts()[0], StringComparison.Ordinal); + // A screenshot reaches the agent as an MCP image. + var shot = agent.Log().First(entry => entry["call"]?.GetValue() == "screenshot")["result"]!["content"]!.AsArray(); + Assert.Contains(shot, part => part?["type"]?.GetValue() == "image" && part["mimeType"]?.GetValue() == "image/png"); } [Fact]