Claude eval sessions run in a container seeded from managed settings

app/build.gradle.kts

@@ -372,6 +372,18 @@ tasks.named<Javadoc>("javadoc") {
     (options as StandardJavadocDocletOptions).addStringOption("Xdoclint:none", "-quiet")
 }
 
+// The container image used for Claude eval sessions (see
+// ClaudeEvalContainer). Built on demand -- `./gradlew claudeEvalImage` --
+// against whatever `docker` context is active (Colima on macOS, a local or
+// remote daemon on Linux). Not wired into the regular build: it is a host
+// setup step, not a per-checkout artifact, and it needs a running daemon.
+tasks.register<Exec>("claudeEvalImage") {
+    group = "build"
+    description = "Builds the drydock-claude-eval Docker image for Claude eval sessions."
+    workingDir = file("src/main/docker/claude-eval")
+    commandLine("docker", "build", "-t", "drydock-claude-eval:latest", ".")
+}
+
 // Central Portal Publisher API transport + signing for the custom `drydock`
 // publication created above. Credentials come from Gradle properties / env
 // (ORG_GRADLE_PROJECT_mavenCentralUsername/Password) and the in-memory GPG key

app/src/main/docker/claude-eval/Dockerfile

@@ -0,0 +1,32 @@
+# The container image used for Claude eval sessions (see
+# ClaudeEvalContainer). Runs Claude Code's native Linux build inside a
+# container so a seeded local settings file can carry the eval-specific
+# x-target-account header, which the host's managed settings (locked to
+# the highest precedence scope) forbid us from adding.
+#
+# Build on demand:  ./gradlew claudeEvalImage
+# (uses whatever `docker` context is active -- Colima on macOS, a local or
+# remote daemon on Linux).
+
+FROM debian:bookworm-slim
+
+# git: Claude's own Bash tool calls `git`; the worktree's .git points back
+#       into the main repo object store, both mounted from the host.
+# curl + ca-certificates: the native installer downloads over HTTPS.
+RUN apt-get update && apt-get install -y --no-install-recommends \
+        ca-certificates curl git \
+    && rm -rf /var/lib/apt/lists/*
+
+# Claude Code native build. The install script places the launcher at
+# ~/.local/bin/claude and the versioned binary under ~/.local/share/claude.
+# Runs as root in this image, so HOME=/root.
+RUN curl -fsSL https://claude.ai/install.sh | bash
+
+ENV PATH="/root/.local/bin:${PATH}"
+
+# The inner `claude ...` command arrives as an entrypoint script mounted from
+# the host (ClaudeEvalContainer.wrap writes it into the config-dir mount).
+# This image's ENTRYPOINT is `sh`, so `docker run <img> <script>` becomes
+# `sh <script>` -- the launcher MUST NOT prepend an extra `sh`, or it would
+# be `sh sh <script>` and fail with "cannot open sh".
+ENTRYPOINT ["sh"]

app/src/main/java/app/drydock/agent/api/AgentRegistry.java

@@ -4,6 +4,7 @@
 
 import java.lang.System.Logger;
 import java.lang.System.Logger.Level;
+import java.time.Instant;
 import java.util.ArrayList;
 import java.util.EnumMap;
 import java.util.List;
@@ -141,6 +142,15 @@ public boolean evalAvailable(AgentKind kind) {
         return provider(kind).map(AgentProvider::evalAvailable).orElse(false);
     }
 
+    /**
+     * When the eval session's auth token expires, for display only. Delegates
+     * to the provider; safe to read on the FX thread (it reads an in-memory
+     * snapshot, no I/O).
+     */
+    public Optional<Instant> evalTokenExpiry(AgentKind kind, String sessionKey) {
+        return provider(kind).flatMap(p -> p.evalTokenExpiry(sessionKey));
+    }
+
     /**
      * Whether {@code kind}'s provider reports remote-session support, per
      * {@link app.drydock.agent.spi.AgentProvider#supportsRemote()}. Cached

app/src/main/java/app/drydock/agent/api/EvalTokenResolver.java

@@ -0,0 +1,38 @@
+package app.drydock.agent.api;
+
+import java.time.Instant;
+import java.util.Optional;
+
+/**
+ * Resolves an auth token for the eval account, plus its expiry when
+ * knowable. The token charges a session's model traffic to the eval
+ * account instead of the user's.
+ *
+ * <p>This is an SPI rather than a hardcoded command so the resolution
+ * mechanism is swappable: the default implementation runs a host-side
+ * credential helper ({@code ddtool}), but once Drydock grows plugin
+ * support a provider-specific plugin can supply this so the eval
+ * integration is not bound to a single credential source.</p>
+ *
+ * <p>Implementations may block (process spawn, network) and MUST be called
+ * off the JavaFX application thread. {@link #resolveToken()} is repeatable:
+ * a long-running eval session may outlive the first token's TTL, so callers
+ * re-invoke this to refresh (e.g. on resume).</p>
+ */
+public interface EvalTokenResolver {
+
+    /** A resolved auth token and, when the credential carries it, the instant it expires. */
+    record ResolvedToken(String token, Optional<Instant> expiry) {
+        public ResolvedToken {
+            java.util.Objects.requireNonNull(token, "token");
+            java.util.Objects.requireNonNull(expiry, "expiry");
+        }
+    }
+
+    /**
+     * Resolves a fresh token. Empty when the credential is unavailable
+     * (helper missing, refused, malformed): the caller fails the launch
+     * loudly rather than silently shipping an unauthenticated session.
+     */
+    Optional<ResolvedToken> resolveToken();
+}

app/src/main/java/app/drydock/agent/providers/claude/ClaudeAgentProvider.java

@@ -16,13 +16,16 @@
 import app.drydock.agent.spi.AgentProvider;
 import app.drydock.agent.providers.claude.internal.ClaudeCapabilities;
 import app.drydock.agent.providers.claude.internal.ClaudeCapabilityService;
-import app.drydock.agent.providers.claude.internal.ClaudeEvalProxy;
+import app.drydock.agent.providers.claude.internal.ClaudeEvalContainer;
+import app.drydock.agent.providers.claude.internal.ClaudeEvalContainer.EvalSetup;
 import app.drydock.agent.providers.claude.internal.ClaudeExecutableLocator;
 import app.drydock.agent.providers.claude.internal.ClaudeHookInstaller;
 import app.drydock.agent.providers.claude.internal.ConversationCatalog;
 import app.drydock.process.SshCommandBuilder;
 
+import java.io.IOException;
 import java.nio.file.Path;
+import java.time.Instant;
 import java.util.Optional;
 
 /**
@@ -40,18 +43,27 @@ public final class ClaudeAgentProvider implements AgentProvider {
     private ClaudeCapabilityService capabilityService;
     private ClaudeConversationSource conversationSource;
     private ClaudeActivityReporter activityReporter;
-    private final ClaudeEvalProxy evalProxy = new ClaudeEvalProxy();
+    /** The eval container; a test stub from the constructor, or the real one built at {@link #init}. */
+    private ClaudeEvalContainer evalContainer;
+    /** The activity state-word dir, needed to mount it into the eval container. Set at {@link #init}. */
+    private Path activityDirectory;
     /** Set once by the background probe at {@link #init}; read by {@link #evalAvailable()} on the FX thread. */
     private volatile boolean evalAvailable;
 
     /** Public no-arg constructor required by {@link java.util.ServiceLoader}. */
     public ClaudeAgentProvider() {
-        this(new ClaudeExecutableLocator());
+        this(new ClaudeExecutableLocator(), null);
     }
 
     /** For tests: inject a locator (e.g. a nonexistent path to force conservative caps). */
     public ClaudeAgentProvider(ClaudeExecutableLocator locator) {
+        this(locator, null);
+    }
+
+    /** For tests: inject the eval container (e.g. a stub that reports unavailable). */
+    public ClaudeAgentProvider(ClaudeExecutableLocator locator, ClaudeEvalContainer evalContainer) {
         this.locator = locator;
+        this.evalContainer = evalContainer;
     }
 
     @Override
@@ -69,11 +81,11 @@ public void init(AgentContext ctx) {
         this.capabilityService = new ClaudeCapabilityService(locator, ctx.backgroundExecutor());
         this.conversationSource = new ClaudeConversationSource(new ConversationCatalog());
         this.activityReporter = new ClaudeActivityReporter(new ClaudeHookInstaller(ctx.stateDirectory()));
-        // Probe omlx_proxy off the FX thread; the result is cached so the
-        // UI's evalAvailable() read never blocks. omlx_proxy may start later
-        // than drydock -- re-probe on the next launch via markEvalSession's
-        // own probe is not done; restart drydock if the proxy comes up after.
-        ctx.backgroundExecutor().execute(() -> evalAvailable = evalProxy.probe());
+        this.activityDirectory = ctx.activityDirectory();
+        if (evalContainer == null) {
+            evalContainer = new ClaudeEvalContainer(ctx.stateDirectory());
+        }
+        ctx.backgroundExecutor().execute(() -> evalAvailable = evalContainer.probe());
     }
 
     @Override
@@ -125,6 +137,12 @@ public LaunchPlan buildCreateCommand(CreateContext c) {
         }
         command.append(activitySettingsFlag(caps));
         command.append(mcpConfigFlag(caps, c.mcp().flatMap(McpAccess::credentialFile)));
+        if (c.evalMode()) {
+            return LaunchPlan.of(
+                    wrapEval(command.toString(), c.sessionId(), c.workingDirectory(),
+                            c.mcp().flatMap(McpAccess::credentialFile)),
+                    sessionIdUsed);
+        }
         return LaunchPlan.of(command.toString(), sessionIdUsed);
     }
 
@@ -141,13 +159,24 @@ public LaunchPlan buildResumeCommand(ResumeContext r) {
         }
         ClaudeCapabilities caps = detectCaps();
         String suffix = activitySettingsFlag(caps) + mcpConfigFlag(caps, r.mcp().flatMap(McpAccess::credentialFile));
+        String inner;
         if (r.agentSessionId().isPresent()) {
-            return LaunchPlan.of(ENV_CLEANUP_PREFIX + "claude --resume " + AgentCommands.shellQuote(r.agentSessionId().get()) + suffix, false);
+            inner = ENV_CLEANUP_PREFIX + "claude --resume " + AgentCommands.shellQuote(r.agentSessionId().get()) + suffix;
+        } else if (r.agentSessionName().isPresent()) {
+            inner = ENV_CLEANUP_PREFIX + "claude --resume " + AgentCommands.shellQuote(r.agentSessionName().get()) + suffix;
+        } else {
+            inner = ENV_CLEANUP_PREFIX + "claude --resume" + suffix;
         }
-        if (r.agentSessionName().isPresent()) {
-            return LaunchPlan.of(ENV_CLEANUP_PREFIX + "claude --resume " + AgentCommands.shellQuote(r.agentSessionName().get()) + suffix, false);
+        if (r.evalMode()) {
+            // Resume key is the agent session id; for PRESET it equals the
+            // --session-id drydock minted at create, so the seeded config
+            // dir (keyed by it) persists across resumes and mark() refreshes
+            // the token.
+            String key = r.agentSessionId().orElse("");
+            return LaunchPlan.of(wrapEval(inner, key, r.workingDirectory(),
+                    r.mcp().flatMap(McpAccess::credentialFile)), false);
         }
-        return LaunchPlan.of(ENV_CLEANUP_PREFIX + "claude --resume" + suffix, false);
+        return LaunchPlan.of(inner, false);
     }
 
     @Override
@@ -177,12 +206,17 @@ public boolean evalAvailable() {
 
     @Override
     public void markEvalSession(String sessionKey) {
-        evalProxy.mark(sessionKey);
+        evalContainer.mark(sessionKey);
     }
 
     @Override
     public void unmarkEvalSession(String sessionKey) {
-        evalProxy.unmark(sessionKey);
+        evalContainer.unmark(sessionKey);
+    }
+
+    @Override
+    public Optional<Instant> evalTokenExpiry(String sessionKey) {
+        return evalContainer.setupFor(sessionKey).map(EvalSetup::tokenExpiry);
     }
 
     /** Uncached, like the pre-seam code: every launch/resume re-probes. Runs on the caller's (background) thread. */
@@ -195,6 +229,28 @@ private ClaudeCapabilities detectCaps() {
         }
     }
 
+    /**
+     * Wraps the bare {@code claude ...} command in a {@code docker run} that
+     * runs it inside the eval container. Throws if the container setup is
+     * missing (ddtool or docker unavailable): an eval session that cannot
+     * be containerized fails loudly rather than silently shipping an
+     * unauthenticated host launch.
+     */
+    private String wrapEval(String innerCommand, String sessionKey, Path worktree, Optional<Path> mcpConfig) {
+        EvalSetup setup = evalContainer.setupFor(sessionKey).orElseThrow(() -> new IllegalStateException(
+                "Eval session " + sessionKey + " has no container setup; ddtool or docker unavailable"));
+        Optional<Path> settingsFile = activityReporter.settingsFile();
+        if (settingsFile.isEmpty() || activityDirectory == null) {
+            throw new IllegalStateException("Eval container needs the activity hook dirs, which are not installed");
+        }
+        try {
+            return evalContainer.wrap(setup, innerCommand, worktree, mcpConfig,
+                    settingsFile.get().getParent(), activityDirectory);
+        } catch (IOException e) {
+            throw new IllegalStateException("Could not write eval entrypoint: " + e.getMessage(), e);
+        }
+    }
+
     private String activitySettingsFlag(ClaudeCapabilities caps) {
         Optional<Path> settings = activityReporter.settingsFile();
         if (!caps.supportsSettings() || settings.isEmpty()) {