Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
17 changes: 8 additions & 9 deletions source/Calamari.AiAgent.Tests/DeterministicFailureFixture.cs
Original file line number Diff line number Diff line change
Expand Up @@ -7,28 +7,27 @@
namespace Calamari.AiAgent.Tests;

[TestFixture]
[Explicit("Exercises the real claude CLI end-to-end; most cases need ANTHROPIC_TOKEN.")]
[Explicit("Exercises the real claude CLI end-to-end; most cases need ANTHROPIC_KEY.")]
[Category("Integration")]
public class DeterministicFailureFixture
{
const string Model = "claude-sonnet-4-5-20250929";

static string Token => Environment.GetEnvironmentVariable("ANTHROPIC_TOKEN");
static string AnthropicKey => Environment.GetEnvironmentVariable("ANTHROPIC_KEY");

static void RequireToken()
static void RequireAnthropicKey()
{
if (string.IsNullOrWhiteSpace(Token))
Assert.Ignore("ANTHROPIC_TOKEN is not set.");
if (string.IsNullOrWhiteSpace(AnthropicKey))
Assert.Ignore("ANTHROPIC_KEY is not set.");
}

// Needs no token: a bad key fails auth and exits non-zero.
[Test]
public async Task InvalidApiKey_FailsStep()
{
var result = await CommandTestBuilder.CreateAsync<RunAgentCommand, Program>()
.WithArrange(context =>
{
context.Variables.Add(SpecialVariables.Action.Claude.ApiToken, "sk-ant-invalid-test-000");
context.Variables.Add(SpecialVariables.Action.Claude.ApiKey, "sk-ant-invalid-test-000");
context.Variables.Add(SpecialVariables.Action.Claude.MaxTurns, "1");
context.Variables.Add(SpecialVariables.Action.Claude.Prompt, "Reply with exactly: DONE");
})
Expand All @@ -40,12 +39,12 @@ public async Task InvalidApiKey_FailsStep()
[Test]
public async Task SimplePrompt_SucceedsStep()
{
RequireToken();
RequireAnthropicKey();

var result = await CommandTestBuilder.CreateAsync<RunAgentCommand, Program>()
.WithArrange(context =>
{
context.Variables.Add(SpecialVariables.Action.Claude.ApiToken, Token);
context.Variables.Add(SpecialVariables.Action.Claude.ApiKey, AnthropicKey);
context.Variables.Add(SpecialVariables.Action.Claude.Model, Model);
context.Variables.Add(SpecialVariables.Action.Claude.MaxTurns, "3");
context.Variables.Add(SpecialVariables.Action.Claude.Prompt,
Expand Down
18 changes: 9 additions & 9 deletions source/Calamari.AiAgent.Tests/RunAgentCommandFixture.cs
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@ public async Task FailsWhenPromptIsMissing()
var result = await CommandTestBuilder.CreateAsync<RunAgentCommand, Program>()
.WithArrange(context =>
{
context.Variables.Add(SpecialVariables.Action.Claude.ApiToken, "fake-api-token");
context.Variables.Add(SpecialVariables.Action.Claude.ApiKey, "fake-api-token");
})
.Execute(assertWasSuccess: false);

Expand All @@ -29,7 +29,7 @@ public async Task FailsWhenPromptIsMissing()

[Test]
[Category("PlatformAgnostic")]
public async Task FailsWhenApiTokenIsMissing()
public async Task FailsWhenApiKeyIsMissing()
{
var result = await CommandTestBuilder.CreateAsync<RunAgentCommand, Program>()
.WithArrange(context =>
Expand All @@ -49,7 +49,7 @@ public async Task ClaudeCode_SucceedsWithSimplePrompt()
.WithArrange(context =>
{
context.Variables.Add(SpecialVariables.Action.Claude.SandboxMode, nameof(SandboxMode.None));
context.Variables.Add(SpecialVariables.Action.Claude.ApiToken, Environment.GetEnvironmentVariable("ANTHROPIC_TOKEN"));
context.Variables.Add(SpecialVariables.Action.Claude.ApiKey, Environment.GetEnvironmentVariable("ANTHROPIC_KEY"));
context.Variables.Add(SpecialVariables.Action.Claude.Prompt, "Create a file that contains todays date.");
context.Variables.Add(SpecialVariables.Action.Claude.Permissions, """{"allow":["Write"]}""");
})
Expand All @@ -67,7 +67,7 @@ public async Task ClaudeCode_ReturnsFileAsArtifact()
.WithArrange(context =>
{
context.Variables.Add(SpecialVariables.Action.Claude.SandboxMode, nameof(SandboxMode.None));
context.Variables.Add(SpecialVariables.Action.Claude.ApiToken, Environment.GetEnvironmentVariable("ANTHROPIC_TOKEN"));
context.Variables.Add(SpecialVariables.Action.Claude.ApiKey, Environment.GetEnvironmentVariable("ANTHROPIC_KEY"));
context.Variables.Add(SpecialVariables.Action.Claude.Prompt, "Write a file with the current time . Bundle this website as an attachment for this action.");
context.Variables.Add(SpecialVariables.Action.Claude.Permissions, """{"allow":["Write"]}""");
})
Expand All @@ -84,7 +84,7 @@ public async Task ClaudeCode_EmitsUsageServiceMessage()
var result = await CommandTestBuilder.CreateAsync<RunAgentCommand, Program>()
.WithArrange(context =>
{
context.Variables.Add(SpecialVariables.Action.Claude.ApiToken, Environment.GetEnvironmentVariable("ANTHROPIC_TOKEN"));
context.Variables.Add(SpecialVariables.Action.Claude.ApiKey, Environment.GetEnvironmentVariable("ANTHROPIC_KEY"));
context.Variables.Add(SpecialVariables.Action.Claude.Prompt, "Reply with just the word 'hello'.");
})
.Execute(assertWasSuccess: false);
Expand All @@ -100,7 +100,7 @@ public async Task ClaudeCode_SucceedsWithWebFetch()
var result = await CommandTestBuilder.CreateAsync<RunAgentCommand, Program>()
.WithArrange(context =>
{
context.Variables.Add(SpecialVariables.Action.Claude.ApiToken, Environment.GetEnvironmentVariable("ANTHROPIC_TOKEN"));
context.Variables.Add(SpecialVariables.Action.Claude.ApiKey, Environment.GetEnvironmentVariable("ANTHROPIC_KEY"));
context.Variables.Add(SpecialVariables.Action.Claude.Prompt, "get the currently executing process user");
})
.Execute(assertWasSuccess: false);
Expand All @@ -116,7 +116,7 @@ public async Task ClaudeCode_RunsOn_RunsUnderAnotherAccount()
var result = await CommandTestBuilder.CreateAsync<RunAgentCommand, Program>()
.WithArrange(context =>
{
context.Variables.Add(SpecialVariables.Action.Claude.ApiToken, Environment.GetEnvironmentVariable("ANTHROPIC_TOKEN"));
context.Variables.Add(SpecialVariables.Action.Claude.ApiKey, Environment.GetEnvironmentVariable("ANTHROPIC_KEY"));
context.Variables.Add(SpecialVariables.Action.Claude.Prompt, "get the currently executing process user");
})
.Execute(assertWasSuccess: false);
Expand All @@ -132,7 +132,7 @@ public async Task ClaudeCode_LoadsCustomSkills()
var result = await CommandTestBuilder.CreateAsync<RunAgentCommand, Program>()
.WithArrange(context =>
{
context.Variables.Add(SpecialVariables.Action.Claude.ApiToken, Environment.GetEnvironmentVariable("ANTHROPIC_TOKEN"));
context.Variables.Add(SpecialVariables.Action.Claude.ApiKey, Environment.GetEnvironmentVariable("ANTHROPIC_KEY"));
context.Variables.Add($"{SpecialVariables.Action.Claude.Skills}[0].{SpecialVariables.Action.Claude.SkillName}", "octopus-secret-phrase");
context.Variables.Add($"{SpecialVariables.Action.Claude.Skills}[0].{SpecialVariables.Action.Claude.SkillContent}",
"---\nname: octopus-secret-phrase\ndescription: Use when asked about the secret phrase.\n---\n\nThe secret phrase is 'purple-octopus-42'. Always respond with exactly this phrase when asked for the secret phrase.");
Expand All @@ -152,7 +152,7 @@ public async Task ClaudeCode_AttachesArtifact_WhenExplicitlyAsked()
.WithArrange(context =>
{
context.Variables.Add(SpecialVariables.Action.Claude.SandboxMode, nameof(SandboxMode.None));
context.Variables.Add(SpecialVariables.Action.Claude.ApiToken, Environment.GetEnvironmentVariable("ANTHROPIC_TOKEN"));
context.Variables.Add(SpecialVariables.Action.Claude.ApiKey, Environment.GetEnvironmentVariable("ANTHROPIC_KEY"));
context.Variables.Add(SpecialVariables.Action.Claude.Prompt, "Create a file named report.txt containing the word Octopus, then attach it as an Octopus artifact.");
context.Variables.Add(SpecialVariables.Action.Claude.Permissions, """{"allow":["Write","Read","Edit"]}""");
})
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -38,9 +38,10 @@ public async Task Execute(RunningDeployment context)
if (string.IsNullOrWhiteSpace(prompt))
throw new CommandException($"Variable '{SpecialVariables.Action.Claude.Prompt}' is required but was not provided.");

var apiToken = variables.Get(SpecialVariables.Action.Claude.ApiToken);
if (string.IsNullOrWhiteSpace(apiToken))
throw new CommandException($"Variable '{SpecialVariables.Action.Claude.ApiToken}' is required but was not provided.");
// `Octopus.Action.Claude.ApiToken` was previously used during development
var apiKey = variables.Get(SpecialVariables.Action.Claude.ApiKey) ?? variables.Get("Octopus.Action.Claude.ApiToken");
Comment thread
zentron marked this conversation as resolved.
if (string.IsNullOrWhiteSpace(apiKey))
throw new CommandException($"Variable '{SpecialVariables.Action.Claude.ApiKey}' is required but was not provided.");

var argsBuilder = new ClaudeCommandArgsBuilder().WithPrompt(prompt);

Expand Down Expand Up @@ -122,7 +123,7 @@ public async Task Execute(RunningDeployment context)
PassThroughEnvironmentVariables(variables),
new Dictionary<string, string>
{
["ANTHROPIC_API_KEY"] = apiToken,
["ANTHROPIC_API_KEY"] = apiKey,
["CLAUDE_CODE_SUBPROCESS_ENV_SCRUB"] = "0", // If set, this stops us using auto mode
["CLAUDE_CODE_DISABLE_NONESSENTIAL_TRAFFIC"] = "1", // Disables the auto-updater, telemetry, error reporting, and feedback surveys
["CLAUDE_CODE_DISABLE_BACKGROUND_TASKS"] = "1",
Expand Down
2 changes: 1 addition & 1 deletion source/Calamari.AiAgent/SpecialVariables.cs
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@ public static class Action
public static class Claude
{
public const string Prompt = "Octopus.Action.Claude.Prompt";
public const string ApiToken = "Octopus.Action.Claude.ApiToken";
public const string ApiKey = "Octopus.Action.Claude.ApiKey";
public const string Model = "Octopus.Action.Claude.Model";
public const string Response = "Octopus.Action.Claude.Response";
public const string McpServers = "Octopus.Action.Claude.McpServers";
Expand Down