Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
25 changes: 24 additions & 1 deletion server/auto-approve.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@
// question is never answered by the machine.
import { describe, expect, it } from "vitest";

import { approvalKey, autoDecision, looksDestructive, looksSensitive } from "./auto-approve.ts";
import { approvalKey, autoDecision, autoVerdict, looksDestructive, looksSensitive } from "./auto-approve.ts";

describe("looksDestructive", () => {
const dangerous = [
Expand Down Expand Up @@ -148,3 +148,26 @@ describe("unattended turns", () => {
expect(autoDecision(bot, "Bash", "git status", { unattended: false })).toBeTruthy();
});
});

describe("tools that ask a person", () => {
// A question normally arrives typed as a question and never reaches a
// verdict. This is the backstop for the path where one arrives typed as a
// permission: auto mode must not answer it, because auto-approving does
// not answer anything — the CLI runs the tool with no answers and the
// model is told "The user did not answer the questions."
const bot = { autoApprove: true, alwaysAllow: ["AskUserQuestion", "ask_user"] };

it("never auto-approves AskUserQuestion, in auto mode or by remembered grant", () => {
expect(autoDecision(bot, "AskUserQuestion", "Which framework should we use?")).toBeNull();
expect(autoVerdict(bot, "AskUserQuestion", "Which framework?").source).toBe("no-grant");
});

it("never auto-approves ask_user, bare or MCP-prefixed", () => {
expect(autoDecision(bot, "ask_user", "Ready to ship?")).toBeNull();
expect(autoDecision(bot, "mcp__ogb__ask_user", "Ready to ship?")).toBeNull();
});

it("still auto-approves an ordinary tool for the same bot", () => {
expect(autoDecision(bot, "Read", "/tmp/notes.md")).toBeTruthy();
});
});
14 changes: 14 additions & 0 deletions server/auto-approve.ts
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,16 @@ const SENSITIVE = [
/\bcredentials?\.json\b|\bserviceaccount\b/i,
];

// Tools that ask a PERSON something. A question exists so that a human
// decides; auto mode answering one on their behalf defeats the only reason
// it was asked. They normally arrive typed as questions and never reach a
// verdict at all — this is the backstop for the path where one arrives
// mis-typed as a permission (a malformed AskUserQuestion call falls back to
// the permission path in permission-proxy). Auto-approving it there does not
// produce an answer: the CLI runs the tool with none and the model is told
// "The user did not answer the questions." — a question silently lost.
const ASKS_A_PERSON = new Set(["askuserquestion", "ask_user"]);

/** First matching pattern's source, so a verdict can NAME the rule that
* made it — the decision log's whole value is "which rule", and deriving
* the match a second time at the call site is how the log and the verdict
Expand Down Expand Up @@ -118,6 +128,10 @@ export function autoVerdict(
// into them
const destructive = matchFirst(DESTRUCTIVE, summary) ?? matchFirst(DESTRUCTIVE, tool);
const sensitive = destructive ? null : matchFirst(SENSITIVE, summary);
// A question is for a person, whatever channel it arrived on.
if (ASKS_A_PERSON.has(tool.replace(/^mcp__[^_]+__/, "").toLowerCase())) {
return { approve: null, source: "no-grant" };
}
// The grant is computed even when a hard block will refuse it: the row
// worth auditing is "this WOULD have auto-approved, and only the block
// stood in the way", which cannot be told apart from an ordinary
Expand Down
4 changes: 4 additions & 0 deletions server/contracts.ts
Original file line number Diff line number Diff line change
Expand Up @@ -109,6 +109,10 @@ export type RuntimeEvent = RuntimeEventBase &
tool: string;
summary: string;
choices?: string[];
/** questions: what each choice means, keyed by the choice's label */
choiceHints?: Record<string, string>;
/** questions: more than one choice may be picked (answers join with ", ") */
multiSelect?: boolean;
approvalScope?: "local-computer";
}
| {
Expand Down
139 changes: 139 additions & 0 deletions server/drivers/claude.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -640,6 +640,145 @@ describe("ClaudeDriver turns (fake CLI)", () => {
await recorder.until((e) => e.type === "turn.completed");
});

it("carries a question's choices, their explanations, and multi-select through to the card", async () => {
await create("hang");
await instance.adapter.sendTurn({ threadId: "t-perm-q", text: "go" });
await recorder.until((e) => e.type === "session.started");

const conn = await connectSocket(permissionSocketPath("t-perm-q"));
const nextAnswer = answerQueue(conn);
conn.write(
JSON.stringify({
t: "ask",
id: "ask-q",
kind: "question",
tool: "AskUserQuestion",
input: {
question: "Which framework should we use?",
choices: ["React", "Vue"],
optionHints: { React: "What the app already uses" },
multiSelect: true,
},
}) + "\n",
);

const opened = await recorder.until((e) => e.type === "request.opened");
expect(opened).toMatchObject({
requestType: "question",
tool: "AskUserQuestion",
summary: "Which framework should we use?",
choices: ["React", "Vue"],
choiceHints: { React: "What the app already uses" },
multiSelect: true,
});

// a question takes an answer, never an allow
await expect(
instance.adapter.respondToRequest("t-perm-q", "ask-q", { behavior: "answer", message: "React, Vue" }),
).resolves.toBe("answered");
expect(await nextAnswer()).toMatchObject({ behavior: "answer", message: "React, Vue" });

conn.end();
await instance.adapter.interruptTurn("t-perm-q");
await recorder.until((e) => e.type === "turn.completed");
});

it("reads a QUESTION's nested questions[] instead of showing it as raw JSON", async () => {
await create("hang");
await instance.adapter.sendTurn({ threadId: "t-perm-nest", text: "go" });
await recorder.until((e) => e.type === "session.started");

const conn = await connectSocket(permissionSocketPath("t-perm-nest"));
// permission-proxy normalizes this shape before the broker sees it; if
// anything ever gets past it, the card must still be readable rather than
// a truncated `{"questions":[{"question":"…` blob with Allow/Deny under it
conn.write(
JSON.stringify({
t: "ask",
id: "ask-nest",
kind: "question",
tool: "AskUserQuestion",
input: {
questions: [
{
question: "What role are you targeting?",
options: [
{ label: "Software Engineer", description: "General SWE" },
{ label: "Product", description: "PM track" },
],
},
],
},
}) + "\n",
);

const opened = await recorder.until((e) => e.type === "request.opened");
expect(opened).toMatchObject({
summary: "What role are you targeting?",
choices: ["Software Engineer", "Product"],
choiceHints: { "Software Engineer": "General SWE", Product: "PM track" },
});
expect((opened as { summary: string }).summary).not.toContain("{");

conn.end();
await instance.adapter.interruptTurn("t-perm-nest");
await recorder.until((e) => e.type === "turn.completed");
});

it("never lets a PERMISSION take its buttons or its summary from a nested questions[]", async () => {
await create("hang");
await instance.adapter.sendTurn({ threadId: "t-perm-lbl", text: "go" });
await recorder.until((e) => e.type === "session.started");

const conn = await connectSocket(permissionSocketPath("t-perm-lbl"));
// A permission ask whose arguments happen to carry `questions`. The
// companion renders card.options AS the decision and maps every label
// that is not "Allow" to a denial, so model-authored labels here mean
// both buttons deny while "Always allow" writes a real grant first.
conn.write(
JSON.stringify({
t: "ask",
id: "ask-lbl",
tool: "mcp__x__wire",
input: {
questions: [{ question: "Send the payroll export?", options: [{ label: "Yes" }, { label: "No" }] }],
to: "acct-9",
amount: 4200,
},
}) + "\n",
);

const opened = await recorder.until((e) => e.type === "request.opened");
expect(opened).toMatchObject({ requestType: "permission" });
expect((opened as { choices?: string[] }).choices).toBeUndefined();
// and the arguments a person is actually deciding on are still shown
expect((opened as { summary: string }).summary).toContain("acct-9");
expect((opened as { summary: string }).summary).toContain("4200");

conn.end();
await instance.adapter.interruptTurn("t-perm-lbl");
await recorder.until((e) => e.type === "turn.completed");
});

it("still shows an unknown permission tool's raw arguments — they are the whole decision", async () => {
await create("hang");
await instance.adapter.sendTurn({ threadId: "t-perm-raw", text: "go" });
await recorder.until((e) => e.type === "session.started");

const conn = await connectSocket(permissionSocketPath("t-perm-raw"));
conn.write(
JSON.stringify({ t: "ask", id: "ask-raw", tool: "mcp__x__wire", input: { amount: 4200, to: "acct-9" } }) + "\n",
);

const opened = await recorder.until((e) => e.type === "request.opened");
expect(opened).toMatchObject({ requestType: "permission", summary: '{"amount":4200,"to":"acct-9"}' });
expect((opened as { choices?: string[] }).choices).toBeUndefined();

conn.end();
await instance.adapter.interruptTurn("t-perm-raw");
await recorder.until((e) => e.type === "turn.completed");
});

it("answers to unknown or already-resolved asks resolve `unavailable` — typed, never a throw", async () => {
await create("hang");
await instance.adapter.sendTurn({ threadId: "t-perm-2", text: "go" });
Expand Down
88 changes: 86 additions & 2 deletions server/drivers/claude.ts
Original file line number Diff line number Diff line change
Expand Up @@ -214,16 +214,97 @@ function systemEndedReply(kind: Ask["kind"]): { behavior: AskBehavior; message:
: { behavior: "deny", message: "OpenMausBot: the turn ended" };
}

const isRecord = (value: unknown): value is Record<string, unknown> =>
typeof value === "object" && value !== null && !Array.isArray(value);

/** One human-readable line for an ask — what the card subtitle shows. */
function askSummary(ask: Ask): string {
const input = ask.input ?? {};
if (typeof input.question === "string") return input.question.slice(0, 300);
if (typeof input.command === "string") return input.command.slice(0, 200);
if (typeof input.url === "string") return input.url.slice(0, 200);
// A QUESTION can also carry its text one level down, under `questions`
// (the CLI's own AskUserQuestion shape). permission-proxy normalizes that
// before it reaches the broker, so this is the belt to that braces.
//
// Gated on the kind on purpose. A PERMISSION whose arguments happen to
// contain a `questions` array must never have the rest of them summarized
// away: you would be approving an action having read one friendly sentence
// instead of what it does.
if (ask.kind === "question") {
const nested = nestedQuestions(input);
if (nested.length) return nested.map((question) => question.question).join(" · ").slice(0, 300);
}
// Last resort, and deliberately still the raw arguments: for a permission
// ask on an unknown tool, its arguments are the only thing a person has to
// decide with, and hiding them behind a tidy label would make the decision
// worse rather than the card prettier.
const text = JSON.stringify(input);
return text === "{}" ? (ask.tool ?? "tool") : text.slice(0, 200);
}

/** `questions[]` entries with a readable question and labelled options. */
function nestedQuestions(input: Record<string, unknown>): Array<{ question: string; options: unknown[] }> {
if (!Array.isArray(input.questions)) return [];
const out: Array<{ question: string; options: unknown[] }> = [];
for (const entry of input.questions) {
if (typeof entry !== "object" || entry === null) continue;
const { question, options } = entry as Record<string, unknown>;
if (typeof question === "string" && question.trim()) {
out.push({ question, options: Array.isArray(options) ? options : [] });
}
}
return out;
}

/**
* The answer buttons a QUESTION card offers, and what each one means.
*
* Two shapes arrive: the flat `choices` list ask_user sends, and the
* `questions[].options[]` objects the CLI's AskUserQuestion uses — a label
* plus its own explanation. Reading only the flat one is why a native
* question arrived with no choices at all and fell back to Allow/Deny.
*
* Only ever called for a question. A permission's buttons are Allow/Deny and
* must not come from anything the model wrote: the companion renders
* `card.options` AS the decision, and it maps every label that is not "Allow"
* to a denial — so model-authored labels there mean both buttons deny while
* "Always allow" writes a real grant and then denies.
*/
function askChoices(input: Record<string, unknown>): {
choices?: string[];
choiceHints?: Record<string, string>;
multiSelect?: boolean;
} {
const hints: Record<string, string> = {};
const labels: string[] = [];
// one loop over either shape: flat strings, or objects carrying the label
// and its explanation
const options = Array.isArray(input.choices) ? input.choices : (nestedQuestions(input)[0]?.options ?? []);
for (const option of options) {
if (typeof option === "string") {
labels.push(option);
continue;
}
if (!isRecord(option) || typeof option.label !== "string") continue;
labels.push(option.label);
if (typeof option.description === "string" && option.description.trim()) hints[option.label] = option.description;
}
if (isRecord(input.optionHints)) {
for (const [label, hint] of Object.entries(input.optionHints)) {
if (typeof hint === "string" && hint.trim()) hints[label] = hint;
}
}
const choices = labels.slice(0, 5);
const choiceHints = Object.fromEntries(Object.entries(hints).filter(([label]) => choices.includes(label)));
return {
choices: choices.length ? choices : undefined,
choiceHints: Object.keys(choiceHints).length ? choiceHints : undefined,
multiSelect: input.multiSelect === true ? true : undefined,
};
}


export function permissionSocketPath(threadId: string) {
// A readable prefix alone is not unique: ids that agree on their first
// characters ("t-perm-dup-1", "t-perm-dup-2") would share a socket. POSIX
Expand Down Expand Up @@ -321,7 +402,10 @@ function createPermissionBroker(opts: {
if (!pending.delete(askId)) return;
clearTimeout(timer);
try {
conn.write(JSON.stringify({ t: "answer", id: askId, behavior, message }) + "\n");
// `source` travels with the answer: a proxy that cannot tell the
// human's words from a timeout note will file the timeout note as
// the human's words.
conn.write(JSON.stringify({ t: "answer", id: askId, behavior, message, source }) + "\n");
} catch {}
opts.onResolve({ ...ask, behavior, source });
};
Expand Down Expand Up @@ -672,7 +756,7 @@ export const ClaudeDriver: ProviderDriver<ClaudeConfig> = {
tool: ask.tool,
summary: askSummary(ask),
approvalScope: controlsHost ? "local-computer" : undefined,
choices: Array.isArray(ask.input?.choices) ? (ask.input.choices as string[]).slice(0, 5) : undefined,
...(ask.kind === "question" ? askChoices(ask.input ?? {}) : {}),
});
},
onResolve: (resolved) => {
Expand Down
4 changes: 4 additions & 0 deletions server/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -879,6 +879,10 @@ bus.subscribe((event: RuntimeEvent) => {
: "Your bot has a question",
subtitle: event.summary,
options: event.choices?.length ? event.choices : permission ? ["Allow", "Deny"] : [],
// questions only: an Allow/Deny box has nothing to explain, and
// nothing to multi-select
optionHints: !permission ? event.choiceHints : undefined,
multiSelect: !permission && event.multiSelect ? true : undefined,
requestId: event.requestId,
tool: permission ? event.tool : undefined,
// the exact grant "always allow" would remember, decided here so
Expand Down
Loading