Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 3 additions & 3 deletions COMPATIBILITY.md

Large diffs are not rendered by default.

4 changes: 2 additions & 2 deletions docs-site/src/content/docs/api/E2E/StepExecutorContext.md
Original file line number Diff line number Diff line change
Expand Up @@ -211,8 +211,8 @@ public virtual void AttachTranscript(string text)
```


Attaches the executor's model transcript to the step. It is written to the results directory as a `log` file
when the attempt keeps results, and dropped otherwise. Call it once, at the end.
Attaches the executor's model transcript to the step. It is kept as a `log` artifact of the step (`transcript-<step id>.txt`)
when the attempt keeps results, and dropped otherwise. The last call wins.

**Parameters**

Expand Down
9 changes: 6 additions & 3 deletions skills/e2e-dotnet/references/debugging.md
Original file line number Diff line number Diff line change
Expand Up @@ -393,9 +393,12 @@ endpoint that sends a `cost` in its usage, such as OpenRouter, reports a cost.
agent.act "increment the counter once" 2.3s 1.9s 180ms 95ms 2 3120/84 - -
```

Under debug, the transcript a custom executor passes to `AttachTranscript`
is also kept, redacted, as `transcript-<n>.txt` beside the attempt's other
results. Without debug it is dropped.
Under debug, the transcript of an agent step is also kept, redacted, as a
`log` artifact of the step: `transcript-<step id>.txt` in the attempt's
artifact directory, listed under the step in the report. The built-in executor
writes each turn as `--- turn <n> ---`, its `tool call: <name>(<arguments>)`
lines, the results, and the loop's `[loop]` notes; a custom executor passes
its own text to `AttachTranscript`. Without debug it is dropped.

## Model calls

Expand Down
14 changes: 9 additions & 5 deletions src/E2E/Agent/ActContext.cs
Original file line number Diff line number Diff line change
Expand Up @@ -164,21 +164,25 @@ public override Task<string> AttachScreenshotAsync(ExecutorPixels pixels, string
return Task.FromResult(id);
}

/// <summary>Keeps the transcript the executor attached, redacted like any log. Best effort: a transcript never fails the step.</summary>
/// <summary>Persists the executor transcript as a step-attributed <c>log</c> artifact, redacted like any log. Best effort: a transcript never fails the step. Upstream's <c>writeTranscript</c>.</summary>
public void WriteTranscript()
{
if (_transcript is null || _scope.Screenshots is not { } screenshots)
if (_transcript is null || _scope.Artifacts is not { } artifacts)
{
return;
}

var stepId = _scope.Steps.ArtifactStepId() ?? "act";
var name = "transcript-" + UnsafeInName().Replace(stepId, "-") + ".txt";
try
{
screenshots.KeepTranscript(_scope.Redactor.Redact(_transcript));
// Model prose and project-tool output are not model input, but they are a log: the redactor that guards the tree guards the file.
File.WriteAllText(Path.Combine(artifacts.Dir, name), _scope.Redactor.Redact(_transcript));
_scope.Steps.AttachArtifact(artifacts.Sink.Register(Reports.ArtifactKind.Log, name));
}
catch (Exception ex) when (ex is IOException or UnauthorizedAccessException)
catch (Exception ex) when (ex is not OperationCanceledException)
{
// The transcript is debug detail.
// The transcript is best-effort debug detail; never fail the step for it, as upstream.
}
}

Expand Down
4 changes: 2 additions & 2 deletions src/E2E/Agent/Executor.cs
Original file line number Diff line number Diff line change
Expand Up @@ -118,8 +118,8 @@ public virtual Task<ExecutorObservation> ObserveAsync(ExecutorObserveOptions? op
throw new NotSupportedException("This context does not observe.");

/// <summary>
/// Attaches the executor's model transcript to the step. It is written to the results directory as a <c>log</c> file
/// when the attempt keeps results, and dropped otherwise. Call it once, at the end.
/// Attaches the executor's model transcript to the step. It is kept as a <c>log</c> artifact of the step (<c>transcript-&lt;step id&gt;.txt</c>)
/// when the attempt keeps results, and dropped otherwise. The last call wins.
/// </summary>
public virtual void AttachTranscript(string text)
{
Expand Down
20 changes: 12 additions & 8 deletions src/E2E/Agent/ToolLoop.cs
Original file line number Diff line number Diff line change
Expand Up @@ -684,11 +684,17 @@ internal sealed class TurnLog(StepExecutorContext context)
// The loop's own notes on a reported turn, kept past the outcome clip.
private const int MaxReportedNotesChars = 1_000;

// Transcript ceiling per step; enough for every turn without unbounded logs.
private const int MaxTranscriptChars = 262_144;

private const string LoopNote = "[loop] ";

private readonly Redactor _redactor = context.Host?.Scope.Redactor ?? Redactor.None;
private readonly List<LoopTurn> _turns = [];

// Loop notes from before the first turn.
private readonly List<string> _preamble = [];

/// <summary>The number of the turn that runs next.</summary>
public int Next => _turns.Count + 1;

Expand All @@ -715,21 +721,16 @@ public void AddResult(LoopTurn turn, string tool, string content)
Attach();
}

/// <summary>A loop event between turns is written onto the turn it followed. Before the first turn there is none to write it on, and the port keeps no transcript to open with it.</summary>
/// <summary>A loop event between turns is written onto the turn it followed, so the report and the transcript both show it where it happened; before the first turn it opens the record.</summary>
public void Note(string text)
{
if (_turns.Count == 0)
{
return;
}

_turns[^1].Notes.Add(LoopNote + text);
(_turns.Count == 0 ? _preamble : _turns[^1].Notes).Add(LoopNote + text);
Attach();
}

private void Attach()
{
if (_turns.Count == 0)
if (_turns.Count == 0 && _preamble.Count == 0)
{
return;
}
Expand All @@ -738,6 +739,9 @@ private void Attach()
turn.Index,
[.. turn.Calls.Take(MaxReportedCalls).Select(call => Truncate(call, MaxReportedCallChars))],
ReportedOutcome(turn)))]);
var transcript = new List<string>(_preamble);
transcript.AddRange(_turns.Select(turn => string.Join('\n', [$"--- turn {turn.Index} ---", .. turn.Calls.Select(call => "tool call: " + call), string.Join('\n', turn.Results), .. turn.Notes])));
context.AttachTranscript(Truncate(string.Join('\n', transcript), MaxTranscriptChars));
}

// The tool results clipped, and the loop's notes after them, clipped apart, since they come last and one clip would drop them.
Expand Down
14 changes: 0 additions & 14 deletions src/E2E/Run/ScreenshotContext.cs
Original file line number Diff line number Diff line change
Expand Up @@ -19,8 +19,6 @@ internal sealed class ScreenshotContext

private int _evidence;

private int _transcripts;

/// <summary>The project root; stored screenshots are reported relative to it.</summary>
public required string ProjectRoot { get; init; }

Expand Down Expand Up @@ -77,18 +75,6 @@ public string KeepEvidence(string label, byte[] png)
return ScreenshotAssertion.Shown(this, file);
}

/// <summary>
/// Keeps the transcript an executor attached to its step under the attempt's results, at <c>transcript-&lt;n&gt;.txt</c>, and
/// returns its path as a failure names it: relative to the project root.
/// </summary>
public string KeepTranscript(string text)
{
var file = Path.Combine(ResultsDirectory, "transcript-" + Interlocked.Increment(ref _transcripts).ToString("000", System.Globalization.CultureInfo.InvariantCulture) + ".txt");
Directory.CreateDirectory(ResultsDirectory);
File.WriteAllText(file, text);
return ScreenshotAssertion.Shown(this, file);
}

/// <summary>The operating system as upstream names it: <c>darwin</c>, <c>linux</c>, or <c>win32</c>.</summary>
public static string OperatingSystemName() =>
RuntimeInformation.IsOSPlatform(OSPlatform.OSX) ? "darwin"
Expand Down
14 changes: 10 additions & 4 deletions tests/E2E.Tests/AgentAct/LoopGuardsAndTranscriptsTests.cs
Original file line number Diff line number Diff line change
Expand Up @@ -9,10 +9,8 @@ namespace E2E.Tests.AgentAct;

public sealed class LoopGuardsAndTranscriptsTests
{
// Upstream also persists the executor transcript as a log artifact under --debug and checks it.
// The port has no debug transcript, so only the verdict and the call count are ported.
[Fact]
public async Task Forces_a_verdict_when_the_model_repeats_itself()
public async Task Forces_a_verdict_when_the_model_repeats_itself_and_persists_the_transcript()
{
var model = new ScriptedModel(request =>
{
Expand All @@ -38,11 +36,19 @@ public async Task Forces_a_verdict_when_the_model_repeats_itself()
await ctx.Agent.ActAsync("keep poking the same thing forever");
},
world,
model);
model,
debug: true);

Assert.Contains("stuck repeating the same tap", Assert.IsType<AgentException>(attempt.Error).Message, StringComparison.Ordinal);
// Five identical calls trip the stop; the forced turn concludes. Without the guard this
// model would burn the whole 25-call budget.
Assert.True(model.CallCount <= 8, "model calls: " + model.CallCount);

// --debug persists the executor transcript as a step-attributed log artifact.
var log = Assert.IsType<E2E.Reports.ArtifactRecord>(attempt.Log);
Assert.Equal(E2E.Reports.ArtifactProducerKind.Step, log.Producer.Kind);
Assert.Equal(attempt.AgentStepId, log.Producer.StepId);
Assert.Contains("tool call: tap", attempt.LogText, StringComparison.Ordinal);
Assert.Contains("--- turn 1 ---", attempt.LogText, StringComparison.Ordinal);
}
}
18 changes: 15 additions & 3 deletions tests/E2E.Tests/CoreTests.cs
Original file line number Diff line number Diff line change
Expand Up @@ -906,7 +906,7 @@ internal static string TempCache()
return Path.Combine(Path.GetTempPath(), "e2e-tests", Guid.NewGuid().ToString("n"));
}

internal readonly record struct Attempt(Exception? Error, int ModelCalls, int Replayed, int Missed, int InputTokens = 0, int OutputTokens = 0, IReadOnlyList<StepRecord>? Steps = null);
internal readonly record struct Attempt(Exception? Error, int ModelCalls, int Replayed, int Missed, int InputTokens = 0, int OutputTokens = 0, IReadOnlyList<StepRecord>? Steps = null, E2E.Reports.ArtifactRecord? Log = null, string? LogText = null, string? AgentStepId = null);

internal static async Task<Attempt> RunAsync(
Func<TestContext, Task> body,
Expand All @@ -918,7 +918,8 @@ internal static async Task<Attempt> RunAsync(
IReadOnlyDictionary<string, AgentTool>? agentTools = null,
IStepExecutor? executor = null,
ReportJson.ReportDirectory? report = null,
Func<IEngine, IEngine>? wrapEngine = null)
Func<IEngine, IEngine>? wrapEngine = null,
bool debug = false)
{
IEngine engine = new DocumentEngine(world ?? BillingWorld.Create());
await using var session = await E2ESession.StartAsync(new E2ESessionOptions
Expand All @@ -937,6 +938,8 @@ internal static async Task<Attempt> RunAsync(
ReplayTimeout = actionTimeout ?? TimeSpan.FromMilliseconds(300),
TestTimeout = TimeSpan.FromSeconds(10),
StepTimeout = TimeSpan.FromSeconds(5),
Debug = debug,
DebugOutput = debug ? TextWriter.Null : null,
});
Exception? error = null;
try
Expand All @@ -949,7 +952,16 @@ internal static async Task<Attempt> RunAsync(
}

session.Complete(error);
return new Attempt(error, session.ModelCalls, session.Replayed, session.Missed, session.InputTokens, session.OutputTokens);
if (!debug)
{
return new Attempt(error, session.ModelCalls, session.Replayed, session.Missed, session.InputTokens, session.OutputTokens);
}

await session.DisposeAsync();
var log = session.Artifacts.FirstOrDefault(artifact => artifact.Kind == E2E.Reports.ArtifactKind.Log);
var logText = Directory.EnumerateFiles(session.ArtifactsDir, "transcript-*.txt").Select(File.ReadAllText).FirstOrDefault();
var agentStep = session.StepRecords.FirstOrDefault(step => step.Api == "agent.act");
return new Attempt(error, session.ModelCalls, session.Replayed, session.Missed, session.InputTokens, session.OutputTokens, session.StepRecords, log, logText, agentStep?.Id);
}

private sealed class StubHandler : HttpMessageHandler
Expand Down
Loading