MCP Broker spike

source/Calamari.AiAgent.Tests/ClaudeCodeBehaviour/ClaudeCommandArgsBuilderFixture.cs

@@ -21,10 +21,19 @@ public void Build_IncludesRequiredFlags()
         args.Should().Contain("--verbose");
         args.Should().Contain("--permission-mode dontAsk");
         args.Should().Contain("--no-session-persistence");
-        args.Should().Contain("--bare");
         args.Should().Contain("--strict-mcp-config");
     }
 
+    [Test]
+    public void Build_DoesNotUseBareMode()
+    {
+        // --bare (Agent SDK mode) defers HTTP MCP tools behind tool search, which would hide the
+        // Octopus MCP server now served over loopback HTTP by the credential broker (MD-2096).
+        var args = MinimalBuilder().Build();
+
+        args.Should().NotContain("--bare");
+    }
+
     [Test]
     public void Build_IncludesModel_WhenSet()
     {

source/Calamari.AiAgent.Tests/ClaudeCodeBehaviour/McpBrokerSmokeTest.cs

@@ -0,0 +1,68 @@
+#nullable enable
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Threading;
+using System.Threading.Tasks;
+using Calamari.AiAgent.ClaudeCodeBehaviour;
+using Calamari.Testing.Helpers;
+using FluentAssertions;
+using ModelContextProtocol.Client;
+using NUnit.Framework;
+
+namespace Calamari.AiAgent.Tests.ClaudeCodeBehaviour;
+
+/// <summary>
+/// Spike verification (MD-2096): proves the broker mechanism end-to-end without any Octopus token or
+/// Claude — it spawns a public stub MCP server as a child, re-exposes it over loopback HTTP, and an
+/// HTTP MCP client lists and calls tools through it. Explicit because it runs `npx` (needs Node and a
+/// one-time package fetch); not part of normal CI.
+/// </summary>
+[TestFixture]
+[Explicit("Spike smoke test: requires Node/npx and network to fetch @modelcontextprotocol/server-everything.")]
+[Category("Integration")]
+public class McpBrokerSmokeTest
+{
+    [Test]
+    public async Task Broker_ProxiesAStdioMcpServer_OverLoopbackHttp()
+    {
+        using var cts = new CancellationTokenSource(TimeSpan.FromMinutes(2));
+        var log = new InMemoryLog();
+
+        // The Octopus server is just one spec; here we broker a generic stub MCP server the same way.
+        var spec = new McpServerSpec
+        {
+            Name = "everything",
+            Command = "npx",
+            Args = new[] { "-y", "@modelcontextprotocol/server-everything" },
+        };
+
+        await using var broker = await McpBroker.StartAsync(new[] { spec }, log, cts.Token);
+
+        // The agent-facing endpoint is loopback only — the secret-free hop.
+        var endpoint = broker.Endpoints["everything"];
+        endpoint.Host.Should().Be("127.0.0.1");
+
+        // Connect as the agent would: an HTTP MCP client against the broker endpoint (no secret on this hop).
+        var clientTransport = new HttpClientTransport(new HttpClientTransportOptions
+        {
+            Name = "smoke-test-client",
+            Endpoint = endpoint,
+            TransportMode = HttpTransportMode.StreamableHttp,
+        });
+        await using var client = await McpClient.CreateAsync(clientTransport, cancellationToken: cts.Token);
+
+        // tools/list is forwarded from the upstream child through the broker.
+        var tools = await client.ListToolsAsync(cancellationToken: cts.Token);
+        tools.Should().Contain(t => t.Name == "echo", "the stub server's tools must be visible through the broker");
+
+        // tools/call round-trips: the broker forwards the call to the child and returns its result verbatim.
+        var result = await client.CallToolAsync(
+            "echo",
+            new Dictionary<string, object?> { ["message"] = "broker-works" },
+            cancellationToken: cts.Token);
+
+        var text = string.Concat(result.Content.OfType<ModelContextProtocol.Protocol.TextContentBlock>().Select(c => c.Text));
+        text.Should().Contain("broker-works");
+    }
+}

source/Calamari.AiAgent.Tests/ClaudeCodeBehaviour/McpWriterFixture.cs

@@ -1,5 +1,7 @@
+using System;
 using System.Collections.Generic;
 using System.IO;
+using System.Linq;
 using System.Text.Json;
 using Calamari.AiAgent.ClaudeCodeBehaviour;
 using Calamari.Common.Commands;
@@ -29,7 +31,7 @@ public void TearDown()
     }
 
     [Test]
-    public void SetupMcpConfig_WritesValidJson_WithServers()
+    public void BuildServerSpecs_IncludesCustomServers()
     {
         var vars = new CalamariVariables();
         var mcpJson = JsonSerializer.Serialize(new[]
@@ -44,165 +46,113 @@ public void SetupMcpConfig_WritesValidJson_WithServers()
         });
         vars.Set(SpecialVariables.Action.Claude.McpServers, mcpJson);
 
-        var configPath = new McpWriter(vars).SetupMcpConfig(workingDir);
+        var specs = new McpWriter(vars).BuildServerSpecs();
 
-        File.Exists(configPath).Should().BeTrue();
-
-        var json = File.ReadAllText(configPath);
-        var doc = JsonDocument.Parse(json);
-        doc.RootElement.TryGetProperty("mcpServers", out var mcpServers).Should().BeTrue();
-        mcpServers.TryGetProperty("github", out var github).Should().BeTrue();
-        github.GetProperty("command").GetString().Should().Be("npx");
+        var github = specs.Single(s => s.Name == "github");
+        github.Command.Should().Be("npx");
+        github.Args.Should().Equal("-y", "@modelcontextprotocol/server-github");
+        github.Env!["TOKEN"].Should().Be("abc123");
     }
 
     [Test]
-    public void SetupMcpConfig_WritesEmptyServers_WhenNoneProvided()
-    {
-        var configPath = new McpWriter(new CalamariVariables()).SetupMcpConfig(workingDir);
-
-        var json = File.ReadAllText(configPath);
-        var doc = JsonDocument.Parse(json);
-        doc.RootElement.TryGetProperty("mcpServers", out var mcpServers).Should().BeTrue();
-        mcpServers.EnumerateObject().Should().BeEmpty();
-    }
-
-    [Test]
-    public void GetAllowedTools_ReturnsMcpWildcardPerServer()
-    {
-        var vars = new CalamariVariables();
-        var mcpJson = JsonSerializer.Serialize(new[]
-        {
-            new { name = "github", command = "npx" },
-            new { name = "slack", command = "npx" },
-        });
-        vars.Set(SpecialVariables.Action.Claude.McpServers, mcpJson);
-
-        var tools = new McpWriter(vars).GetAllowedTools();
-
-        tools.Should().Contain("mcp__github__*");
-        tools.Should().Contain("mcp__slack__*");
-    }
-
-    [Test]
-    public void GetAllowedTools_ReturnsEmpty_WhenNoServersConfigured()
-    {
-        var tools = new McpWriter(new CalamariVariables()).GetAllowedTools();
-
-        tools.Should().BeEmpty();
-    }
-
-    [Test]
-    public void SetupMcpConfig_AddsOctopusMcpServer_WhenTokenAndUrlProvided()
+    public void BuildServerSpecs_IncludesOctopusServer_WhenTokenAndUrlProvided()
     {
         var vars = new CalamariVariables();
         vars.Set(SpecialVariables.Action.Claude.OctopusToken, "API-TESTKEY");
         vars.Set(SpecialVariables.Web.ServerUri, "https://octopus.example.com");
 
-        var configPath = new McpWriter(vars).SetupMcpConfig(workingDir);
+        var octopus = new McpWriter(vars).BuildServerSpecs().Single(s => s.Name == "octopus");
 
-        var json = File.ReadAllText(configPath);
-        var doc = JsonDocument.Parse(json);
-        var mcpServers = doc.RootElement.GetProperty("mcpServers");
-        mcpServers.TryGetProperty("octopus", out var octopus).Should().BeTrue();
-        octopus.GetProperty("command").GetString().Should().Be("npx");
-
-        var env = octopus.GetProperty("env");
-        env.GetProperty("OCTOPUS_SERVER_URL").GetString().Should().Be("https://octopus.example.com");
-        env.GetProperty("OCTOPUS_API_KEY").GetString().Should().Be("API-TESTKEY");
+        octopus.Command.Should().Be("npx");
+        octopus.Args.Should().Contain("@octopusdeploy/mcp-server");
+        // The secret rides in the spec so the broker can hand it to the child process — never to disk.
+        octopus.Env!["OCTOPUS_SERVER_URL"].Should().Be("https://octopus.example.com");
+        octopus.Env!["OCTOPUS_API_KEY"].Should().Be("API-TESTKEY");
     }
 
     [Test]
-    public void SetupMcpConfig_SkipsOctopusMcpServer_WhenTokenMissing()
+    public void BuildServerSpecs_OmitsOctopusServer_WhenTokenMissing()
     {
         var vars = new CalamariVariables();
         vars.Set(SpecialVariables.Web.ServerUri, "https://octopus.example.com");
 
-        var configPath = new McpWriter(vars).SetupMcpConfig(workingDir);
+        var specs = new McpWriter(vars).BuildServerSpecs();
 
-        var json = File.ReadAllText(configPath);
-        var doc = JsonDocument.Parse(json);
-        doc.RootElement.GetProperty("mcpServers").TryGetProperty("octopus", out _).Should().BeFalse();
+        specs.Should().NotContain(s => s.Name == "octopus");
     }
 
     [Test]
-    public void SetupMcpConfig_ThrowsOnInvalidMcpJson()
+    public void BuildServerSpecs_ThrowsOnInvalidMcpJson()
     {
         var vars = new CalamariVariables();
         vars.Set(SpecialVariables.Action.Claude.McpServers, "not valid json {{{");
 
-        var act = () => new McpWriter(vars).SetupMcpConfig(workingDir);
+        var act = () => new McpWriter(vars).BuildServerSpecs();
 
         act.Should().Throw<CommandException>().WithMessage("*Failed to parse*");
     }
 
     [Test]
-    public void SetupMcpConfig_ThrowsWhenServerMissingName()
+    public void BuildServerSpecs_ThrowsWhenServerNameBlank()
     {
         var vars = new CalamariVariables();
-        var mcpJson = JsonSerializer.Serialize(new[] { new { command = "npx" } });
-        vars.Set(SpecialVariables.Action.Claude.McpServers, mcpJson);
+        vars.Set(SpecialVariables.Action.Claude.McpServers, JsonSerializer.Serialize(new[] { new { name = "", command = "npx" } }));
 
-        var act = () => new McpWriter(vars).SetupMcpConfig(workingDir);
+        var act = () => new McpWriter(vars).BuildServerSpecs();
 
         act.Should().Throw<CommandException>().WithMessage("*must have a name*");
     }
 
     [Test]
-    public void SetupMcpConfig_ThrowsWhenServerMissingCommand()
+    public void BuildServerSpecs_ThrowsWhenServerCommandBlank()
     {
         var vars = new CalamariVariables();
-        var mcpJson = JsonSerializer.Serialize(new[] { new { name = "my-server" } });
-        vars.Set(SpecialVariables.Action.Claude.McpServers, mcpJson);
+        vars.Set(SpecialVariables.Action.Claude.McpServers, JsonSerializer.Serialize(new[] { new { name = "my-server", command = "" } }));
 
-        var act = () => new McpWriter(vars).SetupMcpConfig(workingDir);
+        var act = () => new McpWriter(vars).BuildServerSpecs();
 
         act.Should().Throw<CommandException>().WithMessage("*must have a command*");
     }
 
     [Test]
-    public void SetupMcpConfig_InjectsPathEnvVar_WhenNotProvidedByUser()
+    public void GetAllowedTools_ReturnsMcpWildcardPerServer()
     {
         var vars = new CalamariVariables();
-        var mcpJson = JsonSerializer.Serialize(new[]
+        vars.Set(SpecialVariables.Action.Claude.McpServers, JsonSerializer.Serialize(new[]
         {
-            new { name = "test-server", command = "node" },
-        });
-        vars.Set(SpecialVariables.Action.Claude.McpServers, mcpJson);
+            new { name = "github", command = "npx" },
+            new { name = "slack", command = "npx" },
+        }));
 
-        var configPath = new McpWriter(vars).SetupMcpConfig(workingDir);
+        var writer = new McpWriter(vars);
+        var tools = writer.GetAllowedTools(writer.BuildServerSpecs());
 
-        var json = File.ReadAllText(configPath);
-        var doc = JsonDocument.Parse(json);
-        var env = doc.RootElement
-            .GetProperty("mcpServers")
-            .GetProperty("test-server")
-            .GetProperty("env");
-        env.TryGetProperty("PATH", out _).Should().BeTrue();
+        tools.Should().Contain("mcp__github__*");
+        tools.Should().Contain("mcp__slack__*");
     }
 
     [Test]
-    public void SetupMcpConfig_PreservesUserProvidedPathEnvVar()
+    public void WriteConfig_WritesSecretFreeHttpEntries()
     {
-        var vars = new CalamariVariables();
-        var mcpJson = JsonSerializer.Serialize(new[]
+        // mcp-config.json only references the broker's loopback endpoints — never a command, env, or token.
+        var endpoints = new Dictionary<string, Uri>
         {
-            new
-            {
-                name = "test-server",
-                command = "node",
-                env = new Dictionary<string, string> { ["PATH"] = "/custom/path" },
-            },
-        });
-        vars.Set(SpecialVariables.Action.Claude.McpServers, mcpJson);
+            ["octopus"] = new Uri("http://127.0.0.1:54321/"),
+            ["github"] = new Uri("http://127.0.0.1:54322/"),
+        };
 
-        var configPath = new McpWriter(vars).SetupMcpConfig(workingDir);
+        var configPath = new McpWriter(new CalamariVariables()).WriteConfig(workingDir, endpoints);
 
         var json = File.ReadAllText(configPath);
-        var doc = JsonDocument.Parse(json);
-        var env = doc.RootElement
-            .GetProperty("mcpServers")
-            .GetProperty("test-server")
-            .GetProperty("env");
-        env.GetProperty("PATH").GetString().Should().Be("/custom/path");
+        var mcpServers = JsonDocument.Parse(json).RootElement.GetProperty("mcpServers");
+
+        var octopus = mcpServers.GetProperty("octopus");
+        octopus.GetProperty("type").GetString().Should().Be("http");
+        octopus.GetProperty("url").GetString().Should().Be("http://127.0.0.1:54321/");
+        octopus.TryGetProperty("command", out _).Should().BeFalse();
+        octopus.TryGetProperty("env", out _).Should().BeFalse();
+        mcpServers.GetProperty("github").GetProperty("url").GetString().Should().Be("http://127.0.0.1:54322/");
+
+        json.Should().NotContain("OCTOPUS_API_KEY");
     }
 }

source/Calamari.AiAgent/Calamari.AiAgent.csproj

@@ -15,6 +15,12 @@
         <ProjectReference Include="..\Calamari.Common\Calamari.Common.csproj" />
     </ItemGroup>
 
+    <ItemGroup>
+        <!-- MD-2096 credential-broker spike: in-process MCP client (upstream) + loopback HTTP MCP server
+             (downstream). The AspNetCore package transitively brings ModelContextProtocol(.Core). -->
+        <PackageReference Include="ModelContextProtocol.AspNetCore" Version="1.4.0" />
+    </ItemGroup>
+
     <ItemGroup>
         <InternalsVisibleTo Include="Calamari.AiAgent.Tests" />
     </ItemGroup>

source/Calamari.AiAgent/ClaudeCodeBehaviour/ClaudeCodeCliRunner.cs

@@ -3,7 +3,6 @@
 using System.Diagnostics;
 using System.IO;
 using System.Text;
-using System.Text.Json.Serialization;
 using System.Threading;
 using System.Threading.Tasks;
 using Calamari.Common.Plumbing.Logging;
@@ -124,27 +123,3 @@ public record UserSkill
     public required string Name { get; init; }
     public required string Content { get; init; }
 }
-
-public record McpServerConfig
-{
-    [JsonPropertyName("type")]
-    public string Type { get; init; } = "stdio";
-
-    [JsonPropertyName("command")]
-    public required string Command { get; init; }
-
-    [JsonPropertyName("args")]
-    public IReadOnlyList<string>? Args { get; init; }
-
-    [JsonPropertyName("env")]
-    public IReadOnlyDictionary<string, string>? Env { get; init; }
-}
-
-public record McpServerEntry
-{
-    public string? Name { get; init; }
-    public string? Type { get; init; }
-    public string? Command { get; init; }
-    public IReadOnlyList<string>? Args { get; init; }
-    public IReadOnlyDictionary<string, string>? Env { get; init; }
-}

source/Calamari.AiAgent/ClaudeCodeBehaviour/ClaudeCommandArgsBuilder.cs

@@ -102,7 +102,9 @@ public string Build()
             args.Append(EscapeArg(model));
         }
 
-        args.Append(" --bare");
+        // --bare (Agent SDK mode) is intentionally NOT used: under --bare, tool search defaults on and
+        // defers HTTP MCP tools, so the broker's loopback HTTP Octopus server would not load under
+        // --allowedTools. Standard headless mode loads HTTP MCP tools normally (MD-2096 broker spike).
         args.Append(" --strict-mcp-config");
         args.Append(" --output-format stream-json");
         args.Append(" --verbose");