diff --git a/tests/E2E.Acp.Tests/Executor/AcpHarness.cs b/tests/E2E.Acp.Tests/Executor/AcpHarness.cs
index e27cc517..e2c5a8ae 100644
--- a/tests/E2E.Acp.Tests/Executor/AcpHarness.cs
+++ b/tests/E2E.Acp.Tests/Executor/AcpHarness.cs
@@ -3,6 +3,9 @@
// SPDX-License-Identifier: Apache-2.0
using System.Diagnostics;
+using System.Net;
+using System.Net.Sockets;
+using System.Text;
using System.Text.Json;
using System.Text.Json.Nodes;
using E2E.Engine;
@@ -56,6 +59,72 @@ internal sealed class TodoPage
});
}
+/// A local page for the web engine to open. Upstream's step context mocks the observation; the port opens a real page.
+internal sealed class PixelSite : IDisposable
+{
+ private readonly HttpListener _listener = new();
+ private readonly CancellationTokenSource _stop = new();
+
+ private PixelSite(string url) => Url = url;
+
+ public string Url { get; }
+
+ public static PixelSite Start(string html)
+ {
+ for (var attempt = 0; ; attempt++)
+ {
+ var probe = new TcpListener(IPAddress.Loopback, 0);
+ probe.Start();
+ var port = ((IPEndPoint)probe.LocalEndpoint).Port;
+ probe.Stop();
+ var site = new PixelSite("http://127.0.0.1:" + port.ToString(System.Globalization.CultureInfo.InvariantCulture) + "/");
+ site._listener.Prefixes.Add(site.Url);
+ try
+ {
+ site._listener.Start();
+ }
+ catch (HttpListenerException) when (attempt < 5)
+ {
+ site._listener.Close();
+ continue;
+ }
+
+ _ = site.ServeAsync(Encoding.UTF8.GetBytes(html));
+ return site;
+ }
+ }
+
+ public void Dispose()
+ {
+ _stop.Cancel();
+ _listener.Close();
+ _stop.Dispose();
+ }
+
+ private async Task ServeAsync(byte[] page)
+ {
+ while (!_stop.IsCancellationRequested)
+ {
+ try
+ {
+ var context = await _listener.GetContextAsync();
+ context.Response.ContentType = "text/html; charset=utf-8";
+ context.Response.ContentLength64 = page.Length;
+ await context.Response.OutputStream.WriteAsync(page);
+ context.Response.Close();
+ }
+ catch (Exception) when (_stop.IsCancellationRequested || !_listener.IsListening)
+ {
+ return;
+ }
+ catch (Exception ex) when (ex is HttpListenerException or IOException or ObjectDisposedException)
+ {
+ // A client that hung up does not stop the site.
+ }
+ }
+ }
+}
+
internal static class AcpHarness
{
/// The program that stands in for an agent. Its apphost lands beside the tests.
@@ -112,14 +181,15 @@ public static async Task StartAsync(
bool debug = false,
string? projectRoot = null,
IReadOnlyDictionary? agents = null,
- IEngine? engine = null)
+ IEngine? engine = null,
+ string baseUrl = "https://todos.test")
{
var session = await E2ESession.StartAsync(new E2ESessionOptions
{
Engine = engine ?? new DocumentEngine(world),
AgentExecutor = executor,
Agents = agents,
- BaseUrl = "https://todos.test",
+ BaseUrl = baseUrl,
TestTitle = "acp",
CacheEnabled = false,
StepTimeout = TimeSpan.FromSeconds(60),
diff --git a/tests/E2E.Acp.Tests/Executor/AcpTests.cs b/tests/E2E.Acp.Tests/Executor/AcpTests.cs
index 258e809d..ce6bde82 100644
--- a/tests/E2E.Acp.Tests/Executor/AcpTests.cs
+++ b/tests/E2E.Acp.Tests/Executor/AcpTests.cs
@@ -10,7 +10,7 @@ namespace E2E.Acp.Tests.Executor;
///
/// Ports describe('acp') of tester-army/e2e packages/acp/tests/executor.test.ts (0410fc9c). The scripted agent is the C# program
-/// E2E.Acp.Tests.Agent, and the steps run through a real session over the in-memory document engine, since the grammar tools drive the
+/// E2E.Acp.Tests.Agent, and the steps run through a real session over the in-memory document engine (the web engine where a test needs pixels), since the grammar tools drive the
/// context of a step the runtime opened. The port's verdict tool is done (upstream's complete_step) and takes errorCode.
///
[Collection(WorkingDirectoryCollection.Name)]
@@ -593,12 +593,12 @@ public async Task Reports_a_model_the_agent_does_not_offer()
}
[Fact]
- public async Task Partial_Lists_the_tools_once_and_tells_the_agent_which_ones_a_step_does_not_offer()
+ public async Task Lists_the_tools_once_and_tells_the_agent_which_ones_a_step_does_not_offer()
{
- // Partial: the document engine offers every step the same tools, so no step lacks a tool the session listed.
- // fill_secret is not listed: the steps declare no secret.
- var agent = Scripted(_directory, Turns([Pass], [Pass]));
- await using var session = await StartAsync(agent.Executor, new TodoPage().World());
+ // fill_secret is not listed: the steps declare no secret. The web engine takes the screenshot.
+ using var site = PixelSite.Start("tHello
");
+ var agent = Scripted(_directory, Turns([Call("screenshot"), Pass], [Pass]));
+ await using var session = await StartAsync(agent.Executor, new TodoPage().World(), engine: new WebEngine(headless: true), baseUrl: site.Url);
await session.Agent.ActAsync("add a todo");
await session.Agent.ActAsync("add a todo");
@@ -606,6 +606,9 @@ public async Task Partial_Lists_the_tools_once_and_tells_the_agent_which_ones_a_
var offered = Assert.Single(agent.Sessions())["session"]!["tools"]!.AsArray().Select(tool => tool!.GetValue()).ToList();
Assert.DoesNotContain("fill_secret", offered);
Assert.DoesNotContain("Not offered in this step", agent.Prompts()[0], StringComparison.Ordinal);
+ // A screenshot reaches the agent as an MCP image.
+ var shot = agent.Log().First(entry => entry["call"]?.GetValue() == "screenshot")["result"]!["content"]!.AsArray();
+ Assert.Contains(shot, part => part?["type"]?.GetValue() == "image" && part["mimeType"]?.GetValue() == "image/png");
}
[Fact]