Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
74 changes: 72 additions & 2 deletions tests/E2E.Acp.Tests/Executor/AcpHarness.cs
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,9 @@
// SPDX-License-Identifier: Apache-2.0

using System.Diagnostics;
using System.Net;
using System.Net.Sockets;
using System.Text;
using System.Text.Json;
using System.Text.Json.Nodes;
using E2E.Engine;
Expand Down Expand Up @@ -56,6 +59,72 @@ internal sealed class TodoPage
});
}

/// <summary>A local page for the web engine to open. Upstream's step context mocks the observation; the port opens a real page.</summary>
internal sealed class PixelSite : IDisposable
{
private readonly HttpListener _listener = new();
private readonly CancellationTokenSource _stop = new();

private PixelSite(string url) => Url = url;

public string Url { get; }

public static PixelSite Start(string html)
{
for (var attempt = 0; ; attempt++)
{
var probe = new TcpListener(IPAddress.Loopback, 0);
probe.Start();
var port = ((IPEndPoint)probe.LocalEndpoint).Port;
probe.Stop();
var site = new PixelSite("http://127.0.0.1:" + port.ToString(System.Globalization.CultureInfo.InvariantCulture) + "/");
site._listener.Prefixes.Add(site.Url);
try
{
site._listener.Start();
}
catch (HttpListenerException) when (attempt < 5)
{
site._listener.Close();
continue;
}

_ = site.ServeAsync(Encoding.UTF8.GetBytes(html));
return site;
}
}

public void Dispose()
{
_stop.Cancel();
_listener.Close();
_stop.Dispose();
}

private async Task ServeAsync(byte[] page)
{
while (!_stop.IsCancellationRequested)
{
try
{
var context = await _listener.GetContextAsync();
context.Response.ContentType = "text/html; charset=utf-8";
context.Response.ContentLength64 = page.Length;
await context.Response.OutputStream.WriteAsync(page);
context.Response.Close();
}
catch (Exception) when (_stop.IsCancellationRequested || !_listener.IsListening)
{
return;
}
catch (Exception ex) when (ex is HttpListenerException or IOException or ObjectDisposedException)
{
// A client that hung up does not stop the site.
}
}
}
}

internal static class AcpHarness
{
/// <summary>The program that stands in for an agent. Its apphost lands beside the tests.</summary>
Expand Down Expand Up @@ -112,14 +181,15 @@ public static async Task<E2ESession> StartAsync(
bool debug = false,
string? projectRoot = null,
IReadOnlyDictionary<string, AgentOptions>? agents = null,
IEngine? engine = null)
IEngine? engine = null,
string baseUrl = "https://todos.test")
{
var session = await E2ESession.StartAsync(new E2ESessionOptions
{
Engine = engine ?? new DocumentEngine(world),
AgentExecutor = executor,
Agents = agents,
BaseUrl = "https://todos.test",
BaseUrl = baseUrl,
TestTitle = "acp",
CacheEnabled = false,
StepTimeout = TimeSpan.FromSeconds(60),
Expand Down
15 changes: 9 additions & 6 deletions tests/E2E.Acp.Tests/Executor/AcpTests.cs
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@ namespace E2E.Acp.Tests.Executor;

/// <summary>
/// Ports <c>describe('acp')</c> of tester-army/e2e <c>packages/acp/tests/executor.test.ts</c> (0410fc9c). The scripted agent is the C# program
/// <c>E2E.Acp.Tests.Agent</c>, and the steps run through a real session over the in-memory document engine, since the grammar tools drive the
/// <c>E2E.Acp.Tests.Agent</c>, and the steps run through a real session over the in-memory document engine (the web engine where a test needs pixels), since the grammar tools drive the
/// context of a step the runtime opened. The port's verdict tool is <c>done</c> (upstream's <c>complete_step</c>) and takes <c>errorCode</c>.
/// </summary>
[Collection(WorkingDirectoryCollection.Name)]
Expand Down Expand Up @@ -593,19 +593,22 @@ public async Task Reports_a_model_the_agent_does_not_offer()
}

[Fact]
public async Task Partial_Lists_the_tools_once_and_tells_the_agent_which_ones_a_step_does_not_offer()
public async Task Lists_the_tools_once_and_tells_the_agent_which_ones_a_step_does_not_offer()
{
// Partial: the document engine offers every step the same tools, so no step lacks a tool the session listed.
// fill_secret is not listed: the steps declare no secret.
var agent = Scripted(_directory, Turns([Pass], [Pass]));
await using var session = await StartAsync(agent.Executor, new TodoPage().World());
// fill_secret is not listed: the steps declare no secret. The web engine takes the screenshot.
using var site = PixelSite.Start("<!doctype html><title>t</title><h1>Hello</h1>");
var agent = Scripted(_directory, Turns([Call("screenshot"), Pass], [Pass]));
await using var session = await StartAsync(agent.Executor, new TodoPage().World(), engine: new WebEngine(headless: true), baseUrl: site.Url);

await session.Agent.ActAsync("add a todo");
await session.Agent.ActAsync("add a todo");

var offered = Assert.Single(agent.Sessions())["session"]!["tools"]!.AsArray().Select(tool => tool!.GetValue<string>()).ToList();
Assert.DoesNotContain("fill_secret", offered);
Assert.DoesNotContain("Not offered in this step", agent.Prompts()[0], StringComparison.Ordinal);
// A screenshot reaches the agent as an MCP image.
var shot = agent.Log().First(entry => entry["call"]?.GetValue<string>() == "screenshot")["result"]!["content"]!.AsArray();
Assert.Contains(shot, part => part?["type"]?.GetValue<string>() == "image" && part["mimeType"]?.GetValue<string>() == "image/png");
}

[Fact]
Expand Down
Loading