diff --git a/.env.example b/.env.example index 28cd555b..2fec1167 100644 --- a/.env.example +++ b/.env.example @@ -3,7 +3,9 @@ # You only need one LLM provider key to run Opfor. # .env is gitignored — never commit your real keys. -# ── LLM providers ──────────────────────────────────────────────────────────── +# ── LLM providers — the attacker LLM for `opfor run` ───────────────────────── +# Any one of these is enough. NOTE: `opfor hunt` does not use these — its agents +# run on Claude only. See "Autonomous hunt" below. # Groq (fast, free tier available — good default for getting started) # GROQ_API_KEY= @@ -22,6 +24,42 @@ # put the API key here if the endpoint requires one. # OPFOR_API_KEY= +# ── Autonomous hunt (`opfor hunt`) — the attacker agents ───────────────────── +# The commander/operator/scout agents run on the Claude Agent SDK, so they are +# Claude-only regardless of which provider key you set above. Your TARGET can +# still be any model or agent. +# +# Credentials are resolved in this order — the first match wins: +# +# 1. ANTHROPIC_API_KEY pay-per-token API key +# 2. ANTHROPIC_BASE_URL + ANTHROPIC_AUTH_TOKEN gateway (LiteLLM, proxy, …) +# 3. CLAUDE_CODE_OAUTH_TOKEN from `claude setup-token` +# 4. ~/.claude/.credentials.json from `claude login` (Pro/Max) +# +# ANTHROPIC_API_KEY= + +# Gateway / self-hosted proxy — SET BOTH OR NEITHER. +# ANTHROPIC_AUTH_TOKEN on its own is ignored (it is indistinguishable from a token +# inherited from a parent Claude Code session), and the run silently falls through +# to option 3 or 4 — e.g. billing your personal subscription instead of the gateway. +# ANTHROPIC_BASE_URL=https://your-gateway.example.com +# ANTHROPIC_AUTH_TOKEN= + +# Subscription token, if you are not using an API key or gateway. +# CLAUDE_CODE_OAUTH_TOKEN= + +# Optional — pin the `haiku` / `sonnet` / `opus` aliases to specific snapshots. +# Applies to --model / --operator-model / --scout-model and the --ui model pickers. +# ANTHROPIC_DEFAULT_HAIKU_MODEL= +# ANTHROPIC_DEFAULT_SONNET_MODEL= +# ANTHROPIC_DEFAULT_OPUS_MODEL= + +# ── Target authentication (the system under test) ──────────────────────────── +# Separate from everything above — this is the credential Opfor sends TO your +# target, not one it uses itself. Reference the variable NAME via +# `opfor hunt --target-key-env`, `apiKeyEnv` in a config, or the --ui setup form. +# TARGET_API_KEY= + # ── Telemetry enrichment (optional) ────────────────────────────────────────── # Langfuse — pulls production traces to ground attack prompts diff --git a/docs/cli.md b/docs/cli.md index 818f4c72..60f982ad 100644 --- a/docs/cli.md +++ b/docs/cli.md @@ -153,6 +153,8 @@ The CLI loads `.env` from the current working directory automatically. Add `.env opfor run --config .opfor/configs/opfor-config-....json --env .env.prod ``` +> This key is for `opfor run`'s attacker LLM only. `opfor hunt` uses a separate, Claude-only credential — see [hunt.md § Authentication](hunt.md#authentication). + Telemetry credentials (Langfuse, Netra) also come from env vars — see [Trace-aware testing](#trace-aware-testing-agent-only). > Add `.opfor/` to `.gitignore` — it contains configs and reports with embedded target metadata. diff --git a/docs/hunt.md b/docs/hunt.md index 8fccd4d3..ed2d1a6b 100644 --- a/docs/hunt.md +++ b/docs/hunt.md @@ -150,13 +150,33 @@ Credentials are resolved in order: Options 2 and 3 require the [Claude Code CLI](https://docs.claude.com/claude-code) (`npm install -g @anthropic-ai/claude-code`). -**Gateway / self-hosted proxy** — set both together (a token without a base URL is ignored): +Note this is Claude-only, and independent of the provider key `opfor run` uses for its attacker LLM. Your target can still be any model or agent. + +**Gateway / self-hosted proxy** — set both together: ```bash ANTHROPIC_BASE_URL=https://your-gateway.example.com ANTHROPIC_AUTH_TOKEN=... ``` +> `ANTHROPIC_AUTH_TOKEN` on its own is **ignored**. A bare token is indistinguishable from one inherited from a parent Claude Code session, so it is stripped before the agents start — and the run silently falls through to the next credential in the list, which may mean billing your personal subscription instead of the gateway. `opfor hunt` warns about this at startup and on the `--ui` setup form. + +The credential actually in use is printed at startup (`Authenticating via: …`) and shown on the `--ui` setup form. + +**Skipping `.env` entirely** — the `--ui` setup form can also take an API key or gateway pair directly, if nothing is detected in the environment (or you'd rather not touch one at all). It's applied for that run only and never written to disk. + +### Pinning model snapshots + +`--model`, `--operator-model`, and `--scout-model` take the aliases `haiku` / `sonnet` / `opus`. To pin those aliases to specific snapshots — for a gateway that only exposes certain ids, or to freeze behaviour across runs — set: + +```bash +ANTHROPIC_DEFAULT_HAIKU_MODEL=claude-haiku-4-5-20251001 +ANTHROPIC_DEFAULT_SONNET_MODEL=claude-sonnet-4-6 +ANTHROPIC_DEFAULT_OPUS_MODEL=claude-opus-4-8 +``` + +Unset, each alias falls back to a built-in default. Full model ids can also be passed directly to the `--*-model` flags, bypassing aliases entirely. + ## Vulnerability Classes `bias` · `harmful` · `accuracy` · `disclosure` · `injection` · `excessive-agency` · `brand-conduct` · `access-control` · `mcp-usage` diff --git a/runners/cli/src/commands/hunt.ts b/runners/cli/src/commands/hunt.ts index 362556aa..9f9ea2f1 100644 --- a/runners/cli/src/commands/hunt.ts +++ b/runners/cli/src/commands/hunt.ts @@ -1,8 +1,7 @@ import type { Command } from "commander"; import path from "node:path"; import { readFile } from "node:fs/promises"; -import { createWriteStream, existsSync, mkdirSync, type WriteStream } from "node:fs"; -import { homedir } from "node:os"; +import { createWriteStream, mkdirSync, type WriteStream } from "node:fs"; import { consola } from "consola"; import type { HuntOptions, @@ -17,6 +16,7 @@ import { } from "@keyvaluesystems/agent-opfor-core/autonomous/report/writeReport.js"; import { startUiServer } from "../ui/server.js"; import { mergeReporters } from "../ui/bridge.js"; +import { resolveBrainAuth, noBrainAuthMessage } from "../lib/brainAuth.js"; /** Short HH:MM:SS timestamp for live log lines. */ function clock(): string { @@ -107,35 +107,6 @@ function mapAgentTargetToAutonomous(t: ReturnType): Tar }; } -const NO_BRAIN_AUTH_MESSAGE = - "No Claude credentials found. Set ANTHROPIC_API_KEY, or run `claude login` / `claude setup-token` to use a Claude subscription."; - -/** - * Resolve which credential the Claude Agent SDK will authenticate with, for a - * user-facing log line — or null if none is configured. - * - * The SDK resolves credentials itself (first match wins): ANTHROPIC_API_KEY → - * CLAUDE_CODE_OAUTH_TOKEN → a stored `~/.claude/.credentials.json` from a Claude - * subscription login (`claude setup-token` / `claude login`). This is a courtesy - * pre-check so we can emit an actionable message instead of a cryptic SDK error; - * it must therefore recognize the subscription path, not just env vars. - */ -function resolveBrainAuth(): string | null { - if (process.env.ANTHROPIC_API_KEY?.trim()) return "ANTHROPIC_API_KEY"; - // ANTHROPIC_AUTH_TOKEN only counts alongside ANTHROPIC_BASE_URL: buildChildEnv() - // strips a bare token (it's treated as an inherited session token), so counting - // it here without a gateway URL would pass the gate then lose the credential. - if (process.env.ANTHROPIC_AUTH_TOKEN?.trim() && process.env.ANTHROPIC_BASE_URL?.trim()) { - return `gateway (${process.env.ANTHROPIC_BASE_URL})`; - } - if (process.env.CLAUDE_CODE_OAUTH_TOKEN?.trim()) return "CLAUDE_CODE_OAUTH_TOKEN"; - // Claude subscription: credentials stored on disk by `claude setup-token` / `claude login`. - if (existsSync(path.join(homedir(), ".claude", ".credentials.json"))) { - return "Claude subscription (~/.claude/.credentials.json)"; - } - return null; -} - function intOr(value: string | undefined, fallback: number): number { const n = parseInt(value ?? "", 10); return Number.isFinite(n) && n > 0 ? n : fallback; @@ -237,13 +208,19 @@ export function registerHuntCommand(program: Command): void { // --target-config, e.g. a local-script target), launch the setup wizard. // Otherwise --ui means the live dashboard for the already-configured target. if (opts.ui && !opts.endpoint && !opts.targetConfig) { + // Unlike the direct-run path below, the setup form can accept a brain-auth + // override (API key or gateway pair) for this run only — so a missing + // credential here is not fatal; the browser still opens and the form + // requires an override before it lets you start. const brainAuth = resolveBrainAuth(); - if (!brainAuth) { - consola.error(NO_BRAIN_AUTH_MESSAGE); - process.exitCode = 1; - return; + if (brainAuth) { + consola.info(`Authenticating via: ${brainAuth.method}`); + if (brainAuth.warning) consola.warn(brainAuth.warning); + } else { + consola.warn( + "No Claude credential detected — provide one on the setup page before starting." + ); } - consola.info(`Authenticating via: ${brainAuth}`); const uiPort = intOr(opts.uiPort, 3847); @@ -283,6 +260,7 @@ export function registerHuntCommand(program: Command): void { }, setupMode: true, initialConfig, + brainAuth: brainAuth ?? undefined, openBrowser: true, onLog: (line) => { process.stdout.write(line + "\n"); @@ -324,11 +302,12 @@ export function registerHuntCommand(program: Command): void { const brainAuth = resolveBrainAuth(); if (!brainAuth) { - consola.error(NO_BRAIN_AUTH_MESSAGE); + consola.error(noBrainAuthMessage()); process.exitCode = 1; return; } - consola.info(`Authenticating via: ${brainAuth}`); + consola.info(`Authenticating via: ${brainAuth.method}`); + if (brainAuth.warning) consola.warn(brainAuth.warning); // Check endpoint is provided when not using setup UI (the endpoint may // instead come from --target-config). diff --git a/runners/cli/src/lib/brainAuth.ts b/runners/cli/src/lib/brainAuth.ts new file mode 100644 index 00000000..ba77882f --- /dev/null +++ b/runners/cli/src/lib/brainAuth.ts @@ -0,0 +1,83 @@ +// Which credential the Claude Agent SDK will authenticate the commander/operator/ +// scout agents with. Shared by the CLI's startup precheck (hunt.ts) and the setup +// server's /api/brain-auth + override handling (ui/server.ts) — kept in its own +// module so neither has to import the other. + +import { existsSync } from "node:fs"; +import { homedir } from "node:os"; +import path from "node:path"; + +/** + * Human-readable credential source, e.g. "ANTHROPIC_API_KEY". Never a secret value. + * `warning` is set when a configured credential was silently ignored (see below). + */ +export interface BrainAuthInfo { + method: string; + warning?: string; +} + +const NO_BRAIN_AUTH_MESSAGE = + "No Claude credentials found. Set ANTHROPIC_API_KEY, or run `claude login` / `claude setup-token` to use a Claude subscription."; + +const ORPHAN_GATEWAY_TOKEN_WARNING = + "ANTHROPIC_AUTH_TOKEN is set but ANTHROPIC_BASE_URL is not — the token is ignored and the run " + + "falls back to the next credential. Set both together to route through a gateway."; + +/** True when ANTHROPIC_AUTH_TOKEN is set but its required pair, ANTHROPIC_BASE_URL, is not. */ +function hasOrphanedGatewayToken(): boolean { + return Boolean( + process.env.ANTHROPIC_AUTH_TOKEN?.trim() && !process.env.ANTHROPIC_BASE_URL?.trim() + ); +} + +/** + * Resolve which credential the Claude Agent SDK will authenticate with, for a + * user-facing log line — or null if none is configured. + * + * The SDK resolves credentials itself (first match wins): ANTHROPIC_API_KEY → + * CLAUDE_CODE_OAUTH_TOKEN → a stored `~/.claude/.credentials.json` from a Claude + * subscription login (`claude setup-token` / `claude login`). This is a courtesy + * pre-check so we can emit an actionable message instead of a cryptic SDK error; + * it must therefore recognize the subscription path, not just env vars. + */ +export function resolveBrainAuth(): BrainAuthInfo | null { + // A gateway token without its base URL is stripped by buildChildEnv(), so the run + // silently proceeds on a *different* credential — e.g. billing a personal Claude + // subscription instead of the intended gateway. Surface that rather than let it pass. + const warning = hasOrphanedGatewayToken() ? ORPHAN_GATEWAY_TOKEN_WARNING : undefined; + + if (process.env.ANTHROPIC_API_KEY?.trim()) return { method: "ANTHROPIC_API_KEY", warning }; + // ANTHROPIC_AUTH_TOKEN only counts alongside ANTHROPIC_BASE_URL: buildChildEnv() + // strips a bare token (it's treated as an inherited session token), so counting + // it here without a gateway URL would pass the gate then lose the credential. + if (process.env.ANTHROPIC_AUTH_TOKEN?.trim() && process.env.ANTHROPIC_BASE_URL?.trim()) { + // Never interpolate the actual URL: it may carry userinfo or a signed query + // string, and this label is rendered in the setup UI, not just the terminal. + return { method: "gateway (ANTHROPIC_BASE_URL)" }; + } + if (process.env.CLAUDE_CODE_OAUTH_TOKEN?.trim()) { + return { method: "CLAUDE_CODE_OAUTH_TOKEN", warning }; + } + // Claude subscription: credentials stored on disk by `claude setup-token` / `claude login`. + if (existsSync(path.join(homedir(), ".claude", ".credentials.json"))) { + return { method: "Claude subscription (~/.claude/.credentials.json)", warning }; + } + return null; +} + +/** + * The error printed when resolveBrainAuth() finds nothing. Special-cased for the + * orphaned-gateway-token footgun — otherwise a user who DID set ANTHROPIC_AUTH_TOKEN + * sees "no credentials found" with no hint that what they configured was silently + * discarded for missing its required ANTHROPIC_BASE_URL pair. + */ +export function noBrainAuthMessage(): string { + if (hasOrphanedGatewayToken()) { + return ( + "ANTHROPIC_AUTH_TOKEN is set but ANTHROPIC_BASE_URL is not, so it was ignored, and no " + + "other Claude credential was found. Set ANTHROPIC_BASE_URL alongside it, or set " + + "ANTHROPIC_API_KEY, or run `claude login` / `claude setup-token`." + ); + } + return NO_BRAIN_AUTH_MESSAGE; +} diff --git a/runners/cli/src/ui/bridge.ts b/runners/cli/src/ui/bridge.ts index c66c1a01..a9524b97 100644 --- a/runners/cli/src/ui/bridge.ts +++ b/runners/cli/src/ui/bridge.ts @@ -64,6 +64,18 @@ export class UiBridge implements ProgressReporter { this.clients.delete(client); } + /** End every open SSE response so the HTTP server can finish closing. */ + closeAllClients(): void { + for (const client of this.clients) { + try { + client.close(); + } catch { + // Already torn down by the peer — nothing to do. + } + } + this.clients.clear(); + } + snapshot(): UiRunState { if (this.overrideState) return this.overrideState; if (!this.runLog) { diff --git a/runners/cli/src/ui/server.ts b/runners/cli/src/ui/server.ts index a81f2e18..9c58af3b 100644 --- a/runners/cli/src/ui/server.ts +++ b/runners/cli/src/ui/server.ts @@ -16,6 +16,31 @@ import type { SessionConfig } from "@keyvaluesystems/agent-opfor-core/execute/ty import type { RunEvent } from "@keyvaluesystems/agent-opfor-core/autonomous/state/observe.js"; import { UiBridge, type SseClient } from "./bridge.js"; import type { SnapshotMeta } from "./snapshot.js"; +import { resolveBrainAuth, noBrainAuthMessage, type BrainAuthInfo } from "../lib/brainAuth.js"; + +/** + * An explicit choice to run on a credential the form collected instead of what the + * environment resolves to. Applied to `process.env` for this run only — never + * written to disk. See the /api/start handler for validation and application. + */ +interface BrainAuthOverride { + mode: "apiKey" | "gateway"; + apiKey?: string; + baseUrl?: string; + authToken?: string; +} + +/** + * The setup form's POST body. Every field arrives as a string except `headers` + * (a name→value map), the two booleans, and `brainAuthOverride`, which the form + * sends natively. + */ +interface SetupPayload extends Record { + headers?: Record; + sequential?: boolean; + verify?: boolean; + brainAuthOverride?: BrainAuthOverride; +} // Build the session config from the setup form's flat fields (see SetupPage.tsx). // A set-cookie receive must echo via the Cookie header regardless of the form's Send @@ -63,6 +88,8 @@ export interface UiServerOptions { openBrowser?: boolean; setupMode?: boolean; initialConfig?: InitialConfig; + /** Resolved by the CLI before the server starts; displayed on the setup form. */ + brainAuth?: BrainAuthInfo; /** Called for each log line - use to stream to terminal */ onLog?: (line: string) => void; /** Called when the run completes or fails - use to exit the process */ @@ -115,6 +142,8 @@ export async function startUiServer(options: UiServerOptions): Promise(); bridge.setMeta(options.meta); const staticDir = resolveStaticDir(); @@ -124,6 +153,18 @@ export async function startUiServer(options: UiServerOptions): Promise { + const name = typeof req.query.name === "string" ? req.query.name.trim() : ""; + if (!name) { + res.status(400).json({ error: "name is required" }); + return; + } + const value = process.env[name]; + res.json({ set: typeof value === "string" && value.length > 0 }); + }); + app.get("/api/state", (_req, res) => { res.json(bridge.snapshot()); }); @@ -133,6 +174,13 @@ export async function startUiServer(options: UiServerOptions): Promise { + res.json(options.brainAuth ?? {}); + }); + app.get("/api/lines", (_req, res) => { res.json({ lines: bridge.getRecentLines() }); }); @@ -168,7 +216,11 @@ export async function startUiServer(options: UiServerOptions): Promise { res.write(": keepalive\n\n"); }, 15000); - req.on("close", () => clearInterval(heartbeat)); + heartbeats.add(heartbeat); + req.on("close", () => { + clearInterval(heartbeat); + heartbeats.delete(heartbeat); + }); }); // Setup mode: start a run from the UI @@ -181,7 +233,9 @@ export async function startUiServer(options: UiServerOptions): Promise; + const body = req.body as SetupPayload; + // Everything except `headers` and the two booleans arrives as a string. + const config = body as unknown as Record; if (!config.endpoint) { res.status(400).json({ error: "Endpoint URL is required" }); @@ -192,23 +246,61 @@ export async function startUiServer(options: UiServerOptions): Promise = {}; + for (const [name, value] of Object.entries(body.headers ?? {})) { + if (name.trim() && typeof value === "string") headers[name.trim()] = value; + } + const target: TargetConfig = { name: targetName, endpoint: config.endpoint, apiKey, - headers: {}, + headers, mode, session, model: config.model || undefined, + promptPath: config.promptPath?.trim() || undefined, + responsePath: config.responsePath?.trim() || undefined, }; const intOr = (val: string | undefined, fallback: number): number => { @@ -217,6 +309,13 @@ export async function startUiServer(options: UiServerOptions): Promise { + if (!val?.trim()) return undefined; + const n = parseInt(val, 10); + return Number.isNaN(n) ? undefined : n; + }; + // Resolve model aliases to full model IDs from env vars if available const resolveModel = (alias: string | undefined, fallback: string): string => { const a = alias || fallback; @@ -241,14 +340,16 @@ export async function startUiServer(options: UiServerOptions): Promise { server.close((err) => (err ? reject(err) : resolve())); + server.closeAllConnections(); }); }, }; diff --git a/runners/cli/tests/knowledge.test.ts b/runners/cli/tests/knowledge.test.ts index 35a1a73a..5f99b509 100644 --- a/runners/cli/tests/knowledge.test.ts +++ b/runners/cli/tests/knowledge.test.ts @@ -11,7 +11,10 @@ test("loadKnowledge loads the bundled seed libraries", async () => { // Vuln-class ids are the evaluator *category* ids (evaluators/agent//README.md), // not individual evaluator ids — "injection", not "prompt-injection". const injection = kb.vulnClasses.find((v) => v.id === "injection"); - assert.ok(injection, "injection vuln-class present"); + assert.ok( + injection, + `injection vuln-class present (got: ${kb.vulnClasses.map((v) => v.id).join(", ")})` + ); assert.ok(injection!.failRubric.length > 0, "fail rubric parsed"); assert.ok(injection!.passRubric.length > 0, "pass rubric parsed"); assert.equal(injection!.severity, "critical"); diff --git a/runners/cli/tests/resolveBrainAuth.test.ts b/runners/cli/tests/resolveBrainAuth.test.ts new file mode 100644 index 00000000..ee881451 --- /dev/null +++ b/runners/cli/tests/resolveBrainAuth.test.ts @@ -0,0 +1,95 @@ +import { test } from "node:test"; +import assert from "node:assert/strict"; +import { existsSync } from "node:fs"; +import { homedir } from "node:os"; +import path from "node:path"; +import { resolveBrainAuth, noBrainAuthMessage } from "../src/lib/brainAuth.js"; + +/** + * Regression coverage for the orphaned-gateway-token footgun: ANTHROPIC_AUTH_TOKEN + * set without its required ANTHROPIC_BASE_URL pair is silently discarded by + * buildChildEnv() in core, and the run falls through to whatever credential is + * next — which can mean billing a personal Claude subscription instead of the + * intended gateway. resolveBrainAuth()/noBrainAuthMessage() exist to surface that + * instead of letting it pass unnoticed. + */ + +const BRAIN_AUTH_VARS = [ + "ANTHROPIC_API_KEY", + "ANTHROPIC_BASE_URL", + "ANTHROPIC_AUTH_TOKEN", + "CLAUDE_CODE_OAUTH_TOKEN", +] as const; + +type BrainEnv = Partial>; + +function withBrainEnv(vars: BrainEnv, fn: () => void): void { + const saved = new Map(); + for (const key of BRAIN_AUTH_VARS) saved.set(key, process.env[key]); + try { + for (const key of BRAIN_AUTH_VARS) delete process.env[key]; + for (const [key, value] of Object.entries(vars)) process.env[key] = value; + fn(); + } finally { + for (const [key, value] of saved) { + if (value === undefined) delete process.env[key]; + else process.env[key] = value; + } + } +} + +const hasClaudeSubscription = existsSync(path.join(homedir(), ".claude", ".credentials.json")); + +test("ANTHROPIC_API_KEY resolves first, but still flags an orphaned gateway token", () => { + withBrainEnv({ ANTHROPIC_API_KEY: "sk-ant-test", ANTHROPIC_AUTH_TOKEN: "orphaned" }, () => { + const result = resolveBrainAuth(); + assert.ok(result); + assert.equal(result!.method, "ANTHROPIC_API_KEY"); + assert.ok( + result!.warning, + "a leftover orphaned token is a real misconfiguration even when a working key resolves" + ); + }); +}); + +test("the gateway pair resolves cleanly with no warning", () => { + withBrainEnv( + { ANTHROPIC_BASE_URL: "https://gateway.example.com", ANTHROPIC_AUTH_TOKEN: "tok" }, + () => { + const result = resolveBrainAuth(); + assert.ok(result); + assert.equal(result!.method, "gateway (ANTHROPIC_BASE_URL)"); + assert.equal(result!.warning, undefined); + } + ); +}); + +test("a bare ANTHROPIC_AUTH_TOKEN is not treated as a gateway credential", () => { + withBrainEnv({ ANTHROPIC_AUTH_TOKEN: "orphaned-token" }, () => { + const result = resolveBrainAuth(); + if (hasClaudeSubscription) { + // Falls through to the subscription tier on this machine — still flagged. + assert.ok(result); + assert.ok(result!.warning); + } else { + assert.equal(result, null); + } + }); +}); + +// noBrainAuthMessage() never touches the filesystem — unlike resolveBrainAuth(), its +// behavior is deterministic on every machine, real Claude login or not. +test("noBrainAuthMessage explains the orphaned-token case regardless of any fallback credential", () => { + withBrainEnv({ ANTHROPIC_AUTH_TOKEN: "orphaned-token" }, () => { + // The bug this guards against: without the special case, this says "no credentials + // found" even though the user configured one — just not correctly. + assert.match(noBrainAuthMessage(), /ANTHROPIC_BASE_URL is not/); + }); +}); + +test("noBrainAuthMessage falls back to the generic message when nothing is configured", () => { + withBrainEnv({}, () => { + assert.doesNotMatch(noBrainAuthMessage(), /ANTHROPIC_BASE_URL is not/); + assert.match(noBrainAuthMessage(), /No Claude credentials found/); + }); +}); diff --git a/runners/cli/tests/uiBrainAuthOverride.test.ts b/runners/cli/tests/uiBrainAuthOverride.test.ts new file mode 100644 index 00000000..c5e5e6d1 --- /dev/null +++ b/runners/cli/tests/uiBrainAuthOverride.test.ts @@ -0,0 +1,65 @@ +import { test } from "node:test"; +import assert from "node:assert/strict"; +import { startUiServer } from "../src/ui/server.js"; + +/** + * /api/start's brainAuthOverride lets the setup form supply a credential for this + * run only, applied to process.env before the assessment starts (see server.ts). + * These tests cover only the validation paths, which 400 before any assessment + * — and therefore any outbound network call — begins. A test asserting the + * override actually starts a run would need to either mock runAssessmentInProcess + * or make a real network/API call, so that path is left to code review + the + * resolveBrainAuth()/noBrainAuthMessage() unit tests, which cover the same logic + * this handler calls. + */ + +async function postStart(port: number, body: unknown): Promise { + return fetch(`http://127.0.0.1:${port}/api/start`, { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify(body), + }); +} + +test("brainAuthOverride mode 'apiKey' with no key is rejected before a run starts", async () => { + const handle = await startUiServer({ + port: 0, + meta: {}, + setupMode: true, + openBrowser: false, + }); + try { + const res = await postStart(handle.port, { + endpoint: "https://example.com/chat", + objective: "test", + brainAuthOverride: { mode: "apiKey" }, + }); + assert.equal(res.status, 400); + const data = await res.json(); + assert.match(data.error, /API key is required/); + } finally { + await handle.close(); + } +}); + +test("brainAuthOverride mode 'gateway' missing a field is rejected before a run starts", async () => { + const handle = await startUiServer({ + port: 0, + meta: {}, + setupMode: true, + openBrowser: false, + }); + try { + const res = await postStart(handle.port, { + endpoint: "https://example.com/chat", + objective: "test", + // baseUrl only — authToken is missing, which must be rejected same as the reverse. + brainAuthOverride: { mode: "gateway", baseUrl: "https://gateway.example.com" }, + }); + assert.equal(res.status, 400); + const data = await res.json(); + assert.match(data.error, /Gateway base URL and auth token are both required/); + } finally { + await handle.close(); + } +}); diff --git a/runners/cli/tests/uiEnvCheck.test.ts b/runners/cli/tests/uiEnvCheck.test.ts new file mode 100644 index 00000000..e276d54c --- /dev/null +++ b/runners/cli/tests/uiEnvCheck.test.ts @@ -0,0 +1,47 @@ +import { test } from "node:test"; +import assert from "node:assert/strict"; +import { startUiServer } from "../src/ui/server.js"; + +/** + * The setup form uses /api/env-check to tell the user a bearer-token env var is + * missing before the run starts, rather than surfacing it as a 401 mid-hunt. + * It reports existence only — the value must never leave the process. + */ +test("/api/env-check reports existence without leaking the value", async () => { + process.env.OPFOR_TEST_TOKEN_PRESENT = "super-secret-value"; + delete process.env.OPFOR_TEST_TOKEN_ABSENT; + + const handle = await startUiServer({ + port: 0, + meta: { objective: "test", targetName: "test" }, + openBrowser: false, + }); + const base = `http://127.0.0.1:${handle.port}/api/env-check`; + + try { + const present = await fetch(`${base}?name=OPFOR_TEST_TOKEN_PRESENT`); + const presentBody = await present.text(); + assert.equal(present.status, 200); + assert.deepEqual(JSON.parse(presentBody), { set: true }); + assert.ok( + !presentBody.includes("super-secret-value"), + "response must not echo the env var value" + ); + + const absent = await fetch(`${base}?name=OPFOR_TEST_TOKEN_ABSENT`); + assert.equal(absent.status, 200); + assert.deepEqual(await absent.json(), { set: false }); + + // An env var set to the empty string is not usable as a credential. + process.env.OPFOR_TEST_TOKEN_BLANK = ""; + const blank = await fetch(`${base}?name=OPFOR_TEST_TOKEN_BLANK`); + assert.deepEqual(await blank.json(), { set: false }); + + const missingName = await fetch(base); + assert.equal(missingName.status, 400); + } finally { + delete process.env.OPFOR_TEST_TOKEN_PRESENT; + delete process.env.OPFOR_TEST_TOKEN_BLANK; + await handle.close(); + } +}); diff --git a/runners/cli/tests/uiServerShutdown.test.ts b/runners/cli/tests/uiServerShutdown.test.ts new file mode 100644 index 00000000..c63aa0fd --- /dev/null +++ b/runners/cli/tests/uiServerShutdown.test.ts @@ -0,0 +1,56 @@ +import { test } from "node:test"; +import assert from "node:assert/strict"; +import { startUiServer } from "../src/ui/server.js"; + +/** + * Regression: `opfor hunt --ui` used to ignore Ctrl+C until the dashboard tab was + * reloaded or closed. `server.close()` waits for every open connection to end, and + * the /api/events SSE response never ends on its own — so the close callback could + * not fire while a browser was watching. Shutdown now tears the streams down first. + */ +test("close() resolves while a dashboard SSE client is still connected", async () => { + let handle: Awaited> | undefined; + let closed = false; + let timer: NodeJS.Timeout | undefined; + const ac = new AbortController(); + + try { + handle = await startUiServer({ + port: 0, // let the OS pick, so parallel runs never collide + meta: { objective: "test", targetName: "test" }, + openBrowser: false, + }); + + const res = await fetch(`http://127.0.0.1:${handle.port}/api/events`, { + signal: ac.signal, + headers: { accept: "text/event-stream" }, + }); + assert.equal(res.status, 200); + + // Read the first payload so the client is registered server-side before we close. + const reader = res.body!.getReader(); + + try { + await reader.read(); + + const hangGuard = new Promise((_, reject) => { + timer = setTimeout( + () => + reject(new Error("close() did not resolve with a live SSE client — shutdown hangs")), + 5000 + ); + }); + + await Promise.race([handle.close(), hangGuard]); + closed = true; + } finally { + await reader.cancel().catch(() => {}); + } + } finally { + clearTimeout(timer); + ac.abort(); + // Only reached if an earlier step threw before the close() above ran, so + // the listening server doesn't leak past this test. + if (!closed) await handle?.close().catch(() => {}); + } +}); diff --git a/runners/cli/ui/index.html b/runners/cli/ui/index.html index ba2cc9aa..85dfc9f2 100644 --- a/runners/cli/ui/index.html +++ b/runners/cli/ui/index.html @@ -3,7 +3,10 @@ - Opfor Hunt — Live Run + + + + Agent OPFOR — Live Hunt
diff --git a/runners/cli/ui/package-lock.json b/runners/cli/ui/package-lock.json index fe2469d1..fbbb0d5c 100644 --- a/runners/cli/ui/package-lock.json +++ b/runners/cli/ui/package-lock.json @@ -8,6 +8,8 @@ "name": "@keyvaluesystems/agent-opfor-autonomous-ui", "version": "0.10.1", "dependencies": { + "@fontsource-variable/inter": "^5.3.0", + "@fontsource-variable/jetbrains-mono": "^5.3.0", "react": "^19.1.0", "react-dom": "^19.1.0" }, @@ -743,6 +745,24 @@ "node": ">=18" } }, + "node_modules/@fontsource-variable/inter": { + "version": "5.3.0", + "resolved": "https://registry.npmjs.org/@fontsource-variable/inter/-/inter-5.3.0.tgz", + "integrity": "sha512-OupL48va4JNofb97w6NYeF9S7W/kHNKM0Er8Dem5nqi4jeOLrVJDoE8tZEpnMJmtkvNbB1EIPPwHcdkF6b1oUA==", + "license": "OFL-1.1", + "funding": { + "url": "https://github.com/sponsors/ayuhito" + } + }, + "node_modules/@fontsource-variable/jetbrains-mono": { + "version": "5.3.0", + "resolved": "https://registry.npmjs.org/@fontsource-variable/jetbrains-mono/-/jetbrains-mono-5.3.0.tgz", + "integrity": "sha512-F32xpS2NsGYoQi2ADSkKTgpJj7ozajsGgDJ8woTnqjmIB+dxDIqImjl4pXZVEExu8UFZ2ndhmX18EBS/hdz3Lw==", + "license": "OFL-1.1", + "funding": { + "url": "https://github.com/sponsors/ayuhito" + } + }, "node_modules/@jridgewell/gen-mapping": { "version": "0.3.13", "resolved": "https://registry.npmjs.org/@jridgewell/gen-mapping/-/gen-mapping-0.3.13.tgz", diff --git a/runners/cli/ui/package.json b/runners/cli/ui/package.json index 749ecc48..1487f951 100644 --- a/runners/cli/ui/package.json +++ b/runners/cli/ui/package.json @@ -9,6 +9,8 @@ "preview": "vite preview" }, "dependencies": { + "@fontsource-variable/inter": "^5.3.0", + "@fontsource-variable/jetbrains-mono": "^5.3.0", "react": "^19.1.0", "react-dom": "^19.1.0" }, diff --git a/runners/cli/ui/public/favicon.svg b/runners/cli/ui/public/favicon.svg new file mode 100644 index 00000000..bdddb743 --- /dev/null +++ b/runners/cli/ui/public/favicon.svg @@ -0,0 +1,66 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/runners/cli/ui/src/assets/opfor-wordmark.svg b/runners/cli/ui/src/assets/opfor-wordmark.svg new file mode 100644 index 00000000..24b5ed13 --- /dev/null +++ b/runners/cli/ui/src/assets/opfor-wordmark.svg @@ -0,0 +1,68 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/runners/cli/ui/src/components/ConversationView.tsx b/runners/cli/ui/src/components/ConversationView.tsx index e47f8542..7027cee0 100644 --- a/runners/cli/ui/src/components/ConversationView.tsx +++ b/runners/cli/ui/src/components/ConversationView.tsx @@ -21,7 +21,6 @@ function TurnCard({
{turnNumber} - {isFail && }
@@ -63,7 +62,9 @@ export function ConversationView({ thread, findings }: Props) { if (!thread) { return (
-
💬
+

Select a thread to view the conversation

); diff --git a/runners/cli/ui/src/components/FindingsPanel.tsx b/runners/cli/ui/src/components/FindingsPanel.tsx index ac61b309..3ac859c0 100644 --- a/runners/cli/ui/src/components/FindingsPanel.tsx +++ b/runners/cli/ui/src/components/FindingsPanel.tsx @@ -22,7 +22,9 @@ export function FindingsPanel({ findings, selectedThreadId, onSelectThread }: Pr
{sorted.length === 0 ? (
-
🛡️
+

No vulnerabilities found yet

) : ( diff --git a/runners/cli/ui/src/components/SetupPage.tsx b/runners/cli/ui/src/components/SetupPage.tsx index b71e5f3a..3ae5c89c 100644 --- a/runners/cli/ui/src/components/SetupPage.tsx +++ b/runners/cli/ui/src/components/SetupPage.tsx @@ -1,4 +1,5 @@ -import { useState, useEffect } from "react"; +import { useState, useEffect, useRef } from "react"; +import wordmark from "../assets/opfor-wordmark.svg"; interface Props { onStart: () => void; @@ -9,7 +10,11 @@ interface Config { model: string; targetName: string; objective: string; + // The TARGET's bearer token, read from this env var. Opfor's own commander/operator + // models authenticate separately via ANTHROPIC_API_KEY et al. apiKeyEnv: string; + promptPath: string; + responsePath: string; // Session handling: "stateless" (replay history), "client" (we send the id), // "server" (target returns its own id). send/receive location is body|header. sessionMode: string; @@ -24,6 +29,21 @@ interface Config { maxTurns: string; maxThreadTurns: string; budgetUsd: string; + maxTotalThreads: string; + maxForksPerThread: string; + maxDepth: string; + maxLeadsPerWave: string; + maxReconProbes: string; + maxTotalSends: string; + verifierModel: string; + sequential: boolean; + verify: boolean; +} + +interface HeaderRow { + id: number; + name: string; + value: string; } const defaultConfig: Config = { @@ -31,7 +51,9 @@ const defaultConfig: Config = { model: "", targetName: "", objective: "Probe for jailbreaks, system-prompt leakage, and safety bypasses.", - apiKeyEnv: "TARGET_API_KEY", + apiKeyEnv: "", + promptPath: "", + responsePath: "", sessionMode: "stateless", sessionSendIn: "body", sessionSendName: "session_id", @@ -44,10 +66,45 @@ const defaultConfig: Config = { maxTurns: "50", maxThreadTurns: "8", budgetUsd: "2", + maxTotalThreads: "40", + maxForksPerThread: "4", + maxDepth: "3", + maxLeadsPerWave: "4", + maxReconProbes: "8", + maxTotalSends: "", + verifierModel: "", + sequential: false, + verify: false, }; +type EnvStatus = "idle" | "checking" | "set" | "missing"; + +/** Which credential the commander/operator/scout agents run on. Never a secret value. */ +interface BrainAuth { + method?: string; + warning?: string; +} + +/** + * "detected" runs on whatever resolveBrainAuth() found in the environment (or blocks + * start if that's nothing). The other two are a one-run override, applied to + * process.env server-side and never written to disk — for someone who launched + * --ui specifically to avoid touching a terminal or .env file at all. + */ +type BrainAuthMode = "detected" | "apiKey" | "gateway"; + export function SetupPage({ onStart }: Props) { const [config, setConfig] = useState(defaultConfig); + const [headers, setHeaders] = useState([]); + // A ref, not state: row ids only need to be unique React keys, and reading a + // counter out of state here would hand every add in the same tick the same id. + const headerIdRef = useRef(1); + const [envStatus, setEnvStatus] = useState("idle"); + const [brainAuth, setBrainAuth] = useState({}); + const [brainAuthMode, setBrainAuthMode] = useState("detected"); + const [brainAuthApiKey, setBrainAuthApiKey] = useState(""); + const [brainAuthBaseUrl, setBrainAuthBaseUrl] = useState(""); + const [brainAuthAuthToken, setBrainAuthAuthToken] = useState(""); const [loading, setLoading] = useState(true); const [running, setRunning] = useState(false); const [error, setError] = useState(null); @@ -68,10 +125,64 @@ export function SetupPage({ onStart }: Props) { .finally(() => setLoading(false)); }, []); - const updateConfig = (key: keyof Config, value: string) => { + // Resolved by the CLI before this page was served. Defaults to "detected" only + // when something was actually found — otherwise the override fields are already + // open, since --ui exists so this doesn't need a terminal or .env edit at all. + useEffect(() => { + fetch("/api/brain-auth") + .then((res) => res.json()) + .then((data: BrainAuth) => { + setBrainAuth(data ?? {}); + setBrainAuthMode(data?.method ? "detected" : "apiKey"); + }) + .catch(() => setBrainAuthMode("apiKey")); + }, []); + + // Tell the user whether the named env var actually resolves, rather than letting + // them discover a typo as a 401 twenty seconds into a run. + useEffect(() => { + const name = config.apiKeyEnv.trim(); + if (!name) { + setEnvStatus("idle"); + return; + } + setEnvStatus("checking"); + // The debounce alone can't stop a request already in flight from resolving + // after a newer one and overwriting the badge with a stale result. + let stale = false; + const timer = setTimeout(() => { + fetch(`/api/env-check?name=${encodeURIComponent(name)}`) + .then((res) => res.json()) + .then((data: { set?: boolean }) => { + if (!stale) setEnvStatus(data.set ? "set" : "missing"); + }) + .catch(() => { + if (!stale) setEnvStatus("idle"); + }); + }, 350); + return () => { + stale = true; + clearTimeout(timer); + }; + }, [config.apiKeyEnv]); + + const updateConfig = (key: K, value: Config[K]) => { setConfig((prev) => ({ ...prev, [key]: value })); }; + const addHeader = () => { + const id = headerIdRef.current++; + setHeaders((prev) => [...prev, { id, name: "", value: "" }]); + }; + + const updateHeader = (id: number, field: "name" | "value", value: string) => { + setHeaders((prev) => prev.map((h) => (h.id === id ? { ...h, [field]: value } : h))); + }; + + const removeHeader = (id: number) => { + setHeaders((prev) => prev.filter((h) => h.id !== id)); + }; + const handleStart = async () => { setError(null); @@ -84,13 +195,38 @@ export function SetupPage({ onStart }: Props) { return; } + let brainAuthOverride: Record | undefined; + if (brainAuthMode === "apiKey") { + if (!brainAuthApiKey.trim()) { + setError("Provide an API key, or switch to Detected/Gateway"); + return; + } + brainAuthOverride = { mode: "apiKey", apiKey: brainAuthApiKey.trim() }; + } else if (brainAuthMode === "gateway") { + if (!brainAuthBaseUrl.trim() || !brainAuthAuthToken.trim()) { + setError("Gateway needs both a base URL and an auth token"); + return; + } + brainAuthOverride = { + mode: "gateway", + baseUrl: brainAuthBaseUrl.trim(), + authToken: brainAuthAuthToken.trim(), + }; + } + + const headerMap: Record = {}; + for (const h of headers) { + const name = h.name.trim(); + if (name) headerMap[name] = h.value; + } + setRunning(true); try { const res = await fetch("/api/start", { method: "POST", headers: { "Content-Type": "application/json" }, - body: JSON.stringify(config), + body: JSON.stringify({ ...config, headers: headerMap, brainAuthOverride }), }); if (!res.ok) { @@ -108,8 +244,9 @@ export function SetupPage({ onStart }: Props) { if (loading) { return (
+ ); @@ -117,42 +254,144 @@ export function SetupPage({ onStart }: Props) { return (
+