Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
29 changes: 26 additions & 3 deletions src/lib/__tests__/wizard-ask-bridge.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -48,7 +48,8 @@ describe('createWizardAskBridge', () => {
resolveAnswers({ goal: 'Help users find the export button' });

await expect(requestPromise).resolves.toEqual({
goal: 'Help users find the export button',
answers: { goal: 'Help users find the export button' },
timedOut: false,
});
});

Expand Down Expand Up @@ -167,6 +168,25 @@ describe('createWizardAskBridge', () => {
});
});

it('reports a user-dismissed ask as cancelled but not timed out', async () => {
// The two arrive identically in `answers`, so `timedOut` is the only thing
// that tells the tool facades a decline from an unattended terminal.
const bridge = createWizardAskBridge({
getSource: () => 'product-tours',
showQuestion: () => Promise.resolve({ host: CANCELLED_SENTINEL }),
timeoutMs: 60_000,
});

await expect(
bridge.request({
questions: [{ id: 'host', prompt: 'Host?', kind: 'text' }],
}),
).resolves.toEqual({
answers: { host: CANCELLED_SENTINEL },
timedOut: false,
});
});

describe('isFullyCancelled', () => {
// Gates the per-run cap refund in wizard-tools: a fully cancelled ask must
// not burn a wizard_ask slot, while any real answer must still count.
Expand Down Expand Up @@ -214,8 +234,11 @@ describe('createWizardAskBridge', () => {
vi.advanceTimersByTime(1000);

await expect(promise).resolves.toEqual({
goal: CANCELLED_SENTINEL,
audience: CANCELLED_SENTINEL,
answers: {
goal: CANCELLED_SENTINEL,
audience: CANCELLED_SENTINEL,
},
timedOut: true,
});

// Without this, the host's pending-question state survives the
Expand Down
59 changes: 59 additions & 0 deletions src/lib/__tests__/wizard-tools.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,9 @@ import * as path from 'path';
import { zipSync } from 'fflate';
import {
ASK_BATCH_THRESHOLD,
ASK_CANCELLED_NOTE,
ASK_SUBJECT_UNSPECIFIED,
ASK_TIMED_OUT_NOTE,
DEFAULT_ASK_MAX_QUESTIONS,
WIZARD_ASK_SUBJECT_DESCRIPTION,
WIZARD_ASK_TOOL_DESCRIPTION,
Expand All @@ -16,6 +18,7 @@ import {
createAskAccounting,
downloadSkill,
ensureGitignoreCoverage,
describeAskCancellation,
evaluateAskCap,
fetchSkillMenu,
mergeEnvValues,
Expand Down Expand Up @@ -924,13 +927,69 @@ describe('wizard_ask shared descriptions', () => {
);
});

it('points the agent at the cancellation envelope rather than the answer values', () => {
expect(WIZARD_ASK_TOOL_DESCRIPTION).toMatch(/`cancelled` object/);
expect(WIZARD_ASK_TOOL_DESCRIPTION).toMatch(
/instead of inspecting the answer values/,
);
});

it('explains what a subject is and what omitting it costs', () => {
expect(WIZARD_ASK_SUBJECT_DESCRIPTION).toMatch(/Postgres/);
expect(WIZARD_ASK_SUBJECT_DESCRIPTION).toMatch(/consecutive calls/i);
expect(WIZARD_ASK_SUBJECT_DESCRIPTION).toMatch(/Omit it/);
});
});

describe('describeAskCancellation', () => {
const CANCELLED = '__cancelled__';

it('is undefined when every question was answered', () => {
expect(
describeAskCancellation(
{ host: 'db.example.com', ssl: ['require'] },
false,
),
).toBeUndefined();
});

it('names the uncollected questions and reads a dismissal as a decline', () => {
expect(
describeAskCancellation({ host: CANCELLED, password: CANCELLED }, false),
).toEqual({
reason: 'user-cancelled',
questionIds: ['host', 'password'],
note: ASK_CANCELLED_NOTE,
});
});

it('separates a timed-out prompt from a dismissed one', () => {
expect(describeAskCancellation({ host: CANCELLED }, true)).toEqual({
reason: 'timed-out',
questionIds: ['host'],
note: ASK_TIMED_OUT_NOTE,
});
});

it('reports a partly answered ask, and never counts a vaulted answer as cancelled', () => {
expect(
describeAskCancellation(
{
host: 'db.example.com',
password: { secretRef: 'secret:abc' },
tunnel: CANCELLED,
},
false,
),
).toMatchObject({ reason: 'user-cancelled', questionIds: ['tunnel'] });
});

it('tells a dismissal to fall back and a timeout to stop asking', () => {
expect(ASK_CANCELLED_NOTE).toMatch(/do not re-ask/i);
expect(ASK_TIMED_OUT_NOTE).toMatch(/stop asking/i);
});
});

describe('extractZipArchive', () => {
let dest: string;

Expand Down
61 changes: 57 additions & 4 deletions src/lib/agent/runner/harness/pi/__tests__/tools.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,8 @@ import { evaluateToolCall } from '../security';
import { allowedPiCodingTools, allowedOrchestratorTools } from '../task';
import {
ASK_BATCH_THRESHOLD,
ASK_CANCELLED_NOTE,
ASK_TIMED_OUT_NOTE,
WIZARD_ASK_SENSITIVE_DESCRIPTION,
WIZARD_ASK_SUBJECT_DESCRIPTION,
WIZARD_ASK_TOOL_DESCRIPTION,
Expand All @@ -27,8 +29,9 @@ const SECRET = 'phx_live_zendesk_token_123';
const makeTools = (
answers: Record<string, string | string[]>,
maxQuestions?: number,
timedOut = false,
) => {
const request = vi.fn().mockResolvedValue(answers);
const request = vi.fn().mockResolvedValue({ answers, timedOut });
const workingDirectory = mkdtempSync(join(tmpdir(), 'pi-tools-vault-'));
const tools = createWizardPiTools({
workingDirectory,
Expand Down Expand Up @@ -92,6 +95,54 @@ describe('pi wizard_ask — sensitive answers are vaulted', () => {
expect(answers.token).toBe(CANCELLED_SENTINEL);
});

it('names the cancellation explicitly instead of leaving the sentinel to be read', async () => {
// The agent's only signal used to be the sentinel string inside `answers`,
// which says neither "this was not collected" nor who ended the prompt.
const { wizardAsk } = makeTools({
host: CANCELLED_SENTINEL,
password: CANCELLED_SENTINEL,
});
const result = await call(wizardAsk, {
questions: [
{ id: 'host', prompt: 'Host', kind: 'text' },
{ id: 'password', prompt: 'Password', kind: 'text', sensitive: true },
],
subject: 'Postgres',
});
const { cancelled } = JSON.parse(textOf(result)) as {
cancelled: { reason: string; questionIds: string[]; note: string };
};
expect(cancelled.reason).toBe('user-cancelled');
expect(cancelled.questionIds).toEqual(['host', 'password']);
expect(cancelled.note).toBe(ASK_CANCELLED_NOTE);
});

it('distinguishes a timed-out prompt from a dismissed one', async () => {
// A timeout means nobody is reading the terminal, so every later prompt in
// the run costs another full timeout before it fails the same way.
const { wizardAsk } = makeTools(
{ host: CANCELLED_SENTINEL },
undefined,
true,
);
const result = await call(wizardAsk, {
questions: [{ id: 'host', prompt: 'Host', kind: 'text' }],
});
const { cancelled } = JSON.parse(textOf(result)) as {
cancelled: { reason: string; note: string };
};
expect(cancelled.reason).toBe('timed-out');
expect(cancelled.note).toBe(ASK_TIMED_OUT_NOTE);
});

it('carries no cancellation envelope when every question was answered', async () => {
const { wizardAsk } = makeTools({ host: 'db.example.com' });
const result = await call(wizardAsk, {
questions: [{ id: 'host', prompt: 'Host', kind: 'text' }],
});
expect(JSON.parse(textOf(result))).not.toHaveProperty('cancelled');
});

it('still rejects sensitive=true on non-text kinds', async () => {
const { wizardAsk, request } = makeTools({});
const result = await call(wizardAsk, {
Expand Down Expand Up @@ -400,9 +451,11 @@ describe('pi task wiring — wizard_ask pauses Write/Edit', () => {
let release!: (answers: Record<string, string>) => void;
const request = vi.fn(
() =>
new Promise<Record<string, string>>((resolve) => {
release = resolve;
}),
new Promise<{ answers: Record<string, string>; timedOut: boolean }>(
(resolve) => {
release = (answers) => resolve({ answers, timedOut: false });
},
),
);
const [wizardAsk] = createWizardPiTools({
workingDirectory: mkdtempSync(join(tmpdir(), 'pi-ask-pause-')),
Expand Down
18 changes: 15 additions & 3 deletions src/lib/agent/runner/harness/pi/tools.ts
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,7 @@ import {
WIZARD_TOOL_NAMES,
checkEnvKeys as checkEnvKeysCore,
createAskAccounting,
describeAskCancellation,
fetchSkillMenu,
installSkillById,
mergeEnvValues,
Expand Down Expand Up @@ -376,7 +377,7 @@ export function createWizardPiTools(ctx: PiToolsContext): ToolDefinition[] {
// mutate files while it's waiting on the user's answer.
onAskPendingChange?.(true);
try {
const answers = await askBridge.request({
const { answers, timedOut } = await askBridge.request({
questions: args.questions,
subject: normaliseAskSubject(args.subject),
});
Expand All @@ -388,12 +389,23 @@ export function createWizardPiTools(ctx: PiToolsContext): ToolDefinition[] {
answers,
secretVault,
);
// State an uncollected field as an outcome rather than leaving the
// agent to recognise a sentinel answer value (same as the MCP facade).
const cancelled = describeAskCancellation(sanitised, timedOut);
logToFile(
`[pi] wizard_ask: resolved ${
Object.keys(answers).length
} answer(s) for ${args.questions.length} question(s)`,
} answer(s) for ${args.questions.length} question(s)${
cancelled ? `, cancelled: ${cancelled.reason}` : ''
}`,
);
return text(
JSON.stringify(
{ answers: sanitised, ...(cancelled ? { cancelled } : {}) },
null,
2,
),
);
return text(JSON.stringify({ answers: sanitised }, null, 2));
} catch (err) {
askAccounting.refund(args.subject);
const message = err instanceof Error ? err.message : String(err);
Expand Down
30 changes: 22 additions & 8 deletions src/lib/wizard-ask-bridge.ts
Original file line number Diff line number Diff line change
Expand Up @@ -34,13 +34,25 @@ export interface WizardAskRequest {
subject?: string;
}

/**
* One ask's outcome.
*
* `answers` holds one answer per question id (string for `single`/`text`,
* string[] for `multi`); cancelled fields come back as the literal
* `"__cancelled__"`. `timedOut` records that the per-question timeout, rather
* than the user, ended the request — the one fact only the bridge holds, and
* the difference between "the user said no to this" and "nobody is at the
* terminal any more". Both arrive as {@link CANCELLED_SENTINEL} answers, so
* without it the two are indistinguishable to the tool facades and to the agent.
*/
export interface AskResponse {
answers: AskAnswers;
timedOut: boolean;
}

export interface WizardAskBridge {
/**
* Open the WizardAsk overlay and resolve with the user's answers.
* One answer per question id (string for `single`/`text`, string[] for
* `multi`). Cancelled fields come back as the literal `"__cancelled__"`.
*/
request(req: WizardAskRequest): Promise<AskAnswers>;
/** Open the WizardAsk overlay and resolve with the user's answers. */
request(req: WizardAskRequest): Promise<AskResponse>;
}

export interface WizardAskBridgeOptions {
Expand Down Expand Up @@ -107,13 +119,15 @@ export function createWizardAskBridge(

const startedAt = Date.now();
let timer: ReturnType<typeof setTimeout> | undefined;
let timedOut = false;

// Race the user against the timeout. Whichever fires first wins. On
// timeout we also cancel the host's overlay: resolving our side alone
// would leave the host's pending-question state set, and the next
// wizard_ask would be rejected as a duplicate request.
const timeoutPromise = new Promise<AskAnswers>((resolve) => {
timer = setTimeout(() => {
timedOut = true;
opts.cancelQuestion?.();
resolve(buildCancelledAnswers(questions));
}, timeoutMs);
Expand All @@ -132,7 +146,7 @@ export function createWizardAskBridge(
subject,
question_count: questions.length,
duration_ms: durationMs,
timed_out: durationMs >= timeoutMs,
timed_out: timedOut,
});
} else {
analytics.wizardCapture('wizard_ask answered', {
Expand All @@ -143,7 +157,7 @@ export function createWizardAskBridge(
});
}

return answers;
return { answers, timedOut };
} finally {
if (timer) clearTimeout(timer);
}
Expand Down
15 changes: 12 additions & 3 deletions src/lib/wizard-tools/mcp.ts
Original file line number Diff line number Diff line change
Expand Up @@ -44,6 +44,7 @@ import {
downloadSkill,
ensureGitignoreCoverage,
createAskAccounting,
describeAskCancellation,
fetchSkillMenu,
checkEnvKeys as checkEnvKeysCore,
mergeEnvValues,
Expand Down Expand Up @@ -738,7 +739,7 @@ export async function createWizardToolsServer(options: WizardToolsOptions) {
askAccounting.record(args.subject);

try {
const answers = await askBridge.request({
const { answers, timedOut } = await askBridge.request({
questions: args.questions,
subject: normaliseAskSubject(args.subject),
});
Expand All @@ -759,16 +760,24 @@ export async function createWizardToolsServer(options: WizardToolsOptions) {
secretVault,
);

// State an uncollected field as an outcome rather than leaving the
// agent to recognise a sentinel answer value (same as the pi facade).
const cancelled = describeAskCancellation(sanitised, timedOut);

logToFile(
`wizard_ask: resolved ${Object.keys(answers).length} answer(s) for ${
args.questions.length
} question(s)`,
} question(s)${cancelled ? `, cancelled: ${cancelled.reason}` : ''}`,
);
return {
content: [
{
type: 'text' as const,
text: JSON.stringify({ answers: sanitised }, null, 2),
text: JSON.stringify(
{ answers: sanitised, ...(cancelled ? { cancelled } : {}) },
null,
2,
),
},
],
};
Expand Down
Loading
Loading