Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
84 changes: 80 additions & 4 deletions src/lib/__tests__/wizard-ask-bridge.test.ts
Original file line number Diff line number Diff line change
@@ -1,10 +1,16 @@
import {
CANCELLED_SENTINEL,
TIMED_OUT_SENTINEL,
createWizardAskBridge,
isFullyCancelled,
isFullyTimedOut,
} from '@lib/wizard-ask-bridge';
import { analytics } from '@utils/analytics';
import type { AskAnswers, PendingQuestion } from '@lib/wizard-session';
import type {
AskAnswers,
AskQuestion,
PendingQuestion,
} from '@lib/wizard-session';

vi.mock('../../utils/analytics', () => ({
analytics: {
Expand Down Expand Up @@ -189,10 +195,31 @@ describe('createWizardAskBridge', () => {
it('is false for an empty answer map', () => {
expect(isFullyCancelled({})).toBe(false);
});

it('is true for a timed-out ask, so a timeout also refunds the slot', () => {
expect(
isFullyCancelled({ a: TIMED_OUT_SENTINEL, b: TIMED_OUT_SENTINEL }),
).toBe(true);
});
});

describe('isFullyTimedOut', () => {
// Gates the timeout guidance the facades return: it must fire for a
// timeout and never for a dismissal, which needs the opposite advice.
it('is true only when every field is the timed-out sentinel', () => {
expect(
isFullyTimedOut({ a: TIMED_OUT_SENTINEL, b: TIMED_OUT_SENTINEL }),
).toBe(true);
expect(
isFullyTimedOut({ a: CANCELLED_SENTINEL, b: CANCELLED_SENTINEL }),
).toBe(false);
expect(isFullyTimedOut({ a: TIMED_OUT_SENTINEL, b: 'real' })).toBe(false);
expect(isFullyTimedOut({})).toBe(false);
});
});

describe('timeout', () => {
it('resolves every field with the cancelled sentinel and dismisses the host overlay when the user does not answer in time', async () => {
it('resolves every field with the timed-out sentinel and dismisses the host overlay when the user does not answer in time', async () => {
vi.useFakeTimers();
try {
// showQuestion intentionally never resolves — the timeout has to win.
Expand All @@ -213,9 +240,12 @@ describe('createWizardAskBridge', () => {

vi.advanceTimersByTime(1000);

// A timeout must not look like a dismissal. The agent reads the
// dismissal sentinel as "the user declined" and unwinds its work, and
// the user who walked off to run a build is still coming back.
await expect(promise).resolves.toEqual({
goal: CANCELLED_SENTINEL,
audience: CANCELLED_SENTINEL,
goal: TIMED_OUT_SENTINEL,
audience: TIMED_OUT_SENTINEL,
});

// Without this, the host's pending-question state survives the
Expand All @@ -232,6 +262,52 @@ describe('createWizardAskBridge', () => {
}
});

// The regression this guards: `cancelQuestion` is not a no-op on the real
// TUI path. `WizardStore.cancelPendingQuestion()` fills every field with
// the dismissal sentinel and resolves the pending `showQuestion` promise
// synchronously, so cancelling before resolving let the dismissal settle
// first and win the `Promise.race` — the agent got "__cancelled__" and
// none of the timeout guidance, on exactly the path this all exists for.
it('answers with the timeout sentinel even when cancelQuestion settles the host promise', async () => {
vi.useFakeTimers();
try {
const questions: AskQuestion[] = [
{ id: 'goal', prompt: 'Goal?', kind: 'text' },
{ id: 'audience', prompt: 'Who?', kind: 'text' },
];
let resolveHost!: (answers: AskAnswers) => void;

const bridge = createWizardAskBridge({
getSource: () => 'product-tours',
showQuestion: () =>
new Promise<AskAnswers>((r) => {
resolveHost = r;
}),
cancelQuestion: () => {
const cancelled: AskAnswers = {};
for (const q of questions) cancelled[q.id] = CANCELLED_SENTINEL;
resolveHost(cancelled);
},
timeoutMs: 1000,
});

const promise = bridge.request({ questions });
vi.advanceTimersByTime(1000);

await expect(promise).resolves.toEqual({
goal: TIMED_OUT_SENTINEL,
audience: TIMED_OUT_SENTINEL,
});

const cancelledCall = wizardCaptureMock.mock.calls.find(
([name]) => name === 'wizard_ask cancelled',
);
expect(cancelledCall?.[1]).toMatchObject({ timed_out: true });
} finally {
vi.useRealTimers();
}
});

it('does not dismiss the overlay when the user answers before the timeout', async () => {
vi.useFakeTimers();
try {
Expand Down
19 changes: 17 additions & 2 deletions src/lib/__tests__/wizard-tools.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ import { zipSync } from 'fflate';
import {
ASK_BATCH_THRESHOLD,
ASK_SUBJECT_UNSPECIFIED,
ASK_TIMED_OUT_NOTE,
DEFAULT_ASK_MAX_QUESTIONS,
WIZARD_ASK_SUBJECT_DESCRIPTION,
WIZARD_ASK_TOOL_DESCRIPTION,
Expand Down Expand Up @@ -918,12 +919,26 @@ describe('wizard_ask shared descriptions', () => {
expect(WIZARD_ASK_TOOL_DESCRIPTION).toMatch(/never blocked/i);
});

it('keeps the cancellation promise the warehouse skill relies on', () => {
it('keeps the free-cancellation promise the warehouse skill relies on', () => {
expect(WIZARD_ASK_TOOL_DESCRIPTION).toMatch(
/cancelled or timed-out response does NOT count/,
/Neither a cancelled nor a timed-out response counts/,
);
});

it('separates a dismissal from a timeout, and forbids reverting on a timeout', () => {
expect(WIZARD_ASK_TOOL_DESCRIPTION).toMatch(/__cancelled__/);
expect(WIZARD_ASK_TOOL_DESCRIPTION).toMatch(/__timed_out__/);
expect(WIZARD_ASK_TOOL_DESCRIPTION).toMatch(/ask the same question again/i);
expect(WIZARD_ASK_TOOL_DESCRIPTION).toMatch(/never undo or revert/i);
});

it('tells the agent on a timeout to keep waiting and leave its changes in place', () => {
expect(ASK_TIMED_OUT_NOTE).toMatch(/not a decline/i);
expect(ASK_TIMED_OUT_NOTE).toMatch(/Do NOT undo, revert, or delete/);
expect(ASK_TIMED_OUT_NOTE).toMatch(/Ask the same question again/);
expect(ASK_TIMED_OUT_NOTE).toMatch(/costs nothing/);
});

it('explains what a subject is and what omitting it costs', () => {
expect(WIZARD_ASK_SUBJECT_DESCRIPTION).toMatch(/Postgres/);
expect(WIZARD_ASK_SUBJECT_DESCRIPTION).toMatch(/consecutive calls/i);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@ When a skill provides a numbered or bulleted list of questions, translate the en
- For \`single\` and \`multi\`, extract the alternatives from the prose into \`options\` as \`{ label, value }\` pairs. Use the human phrase as \`label\` and a lowercase-hyphenated form as \`value\` (e.g., \`label: "Vanilla JS"\`, \`value: "vanilla-js"\`).
- Use a kebab-case slug of the question label as \`id\` (e.g., "Tech stack" → \`tech-stack\`, "Show frequency" → \`show-frequency\`).
- Do not invent fields the schema does not define (no \`source\`, \`category\`, \`priority\`, etc.) — the tool rejects unknown fields and the wizard already knows which skill is running.
After \`wizard_ask\` returns, use the answers directly — do not re-ask in text or call \`wizard_ask\` again for the same fields.
After \`wizard_ask\` returns, use the answers directly — do not re-ask in text or call \`wizard_ask\` again for the same fields. The one exception is a fully timed-out return, where every answer is \`__timed_out__\`: nothing was answered, so ask the same questions again to keep waiting, and follow the guidance that result carries.
Use the Task tools to plan and track the whole run so the user always sees where you are. Create the task list once you understand the work — after you load and skim the skill workflow, not before — in a single tool call, in the order you will run them, with one task per stage covering the whole run through to instrumenting events, creating the dashboard, and writing the setup report. Give each an imperative subject AND an \`activeForm\` (the present-continuous label the panel shows while it runs, e.g. subject "Install SDK" / activeForm "Installing SDK"). Keep the list current: add a task the moment you discover work it is missing.
Try to keep exactly ONE task \`in_progress\`. \`TaskUpdate\` it to \`in_progress\` right before you start that stage, and to \`completed\` the instant you finish it — one at a time, never batched at the end. Only mark \`completed\` when the work is genuinely done; if the build fails, a step is partial, or you hit a blocker, keep it \`in_progress\` and add a task for the fix.
After you complete a task, take the next one in order (lowest id first — earlier stages set up later ones), mark it \`in_progress\`, and continue. Driving the list in order top to bottom is how you finish every stage.
Expand Down Expand Up @@ -51,7 +51,7 @@ When a skill provides a numbered or bulleted list of questions, translate the en
- For \`single\` and \`multi\`, extract the alternatives from the prose into \`options\` as \`{ label, value }\` pairs. Use the human phrase as \`label\` and a lowercase-hyphenated form as \`value\` (e.g., \`label: "Vanilla JS"\`, \`value: "vanilla-js"\`).
- Use a kebab-case slug of the question label as \`id\` (e.g., "Tech stack" → \`tech-stack\`, "Show frequency" → \`show-frequency\`).
- Do not invent fields the schema does not define (no \`source\`, \`category\`, \`priority\`, etc.) — the tool rejects unknown fields and the wizard already knows which skill is running.
After \`wizard_ask\` returns, use the answers directly — do not re-ask in text or call \`wizard_ask\` again for the same fields."
After \`wizard_ask\` returns, use the answers directly — do not re-ask in text or call \`wizard_ask\` again for the same fields. The one exception is a fully timed-out return, where every answer is \`__timed_out__\`: nothing was answered, so ask the same questions again to keep waiting, and follow the guidance that result carries."
`;

exports[`commandments by axis > 'pi' + 'linear' > matches the published prompt 1`] = `
Expand All @@ -76,7 +76,7 @@ When a skill provides a numbered or bulleted list of questions, translate the en
- For \`single\` and \`multi\`, extract the alternatives from the prose into \`options\` as \`{ label, value }\` pairs. Use the human phrase as \`label\` and a lowercase-hyphenated form as \`value\` (e.g., \`label: "Vanilla JS"\`, \`value: "vanilla-js"\`).
- Use a kebab-case slug of the question label as \`id\` (e.g., "Tech stack" → \`tech-stack\`, "Show frequency" → \`show-frequency\`).
- Do not invent fields the schema does not define (no \`source\`, \`category\`, \`priority\`, etc.) — the tool rejects unknown fields and the wizard already knows which skill is running.
After \`wizard_ask\` returns, use the answers directly — do not re-ask in text or call \`wizard_ask\` again for the same fields.
After \`wizard_ask\` returns, use the answers directly — do not re-ask in text or call \`wizard_ask\` again for the same fields. The one exception is a fully timed-out return, where every answer is \`__timed_out__\`: nothing was answered, so ask the same questions again to keep waiting, and follow the guidance that result carries.
Use the Task tools to plan and track the whole run so the user always sees where you are. Create the task list once you understand the work — after you load and skim the skill workflow, not before — in a single tool call, in the order you will run them, with one task per stage covering the whole run through to instrumenting events, creating the dashboard, and writing the setup report. Give each an imperative subject AND an \`activeForm\` (the present-continuous label the panel shows while it runs, e.g. subject "Install SDK" / activeForm "Installing SDK"). Keep the list current: add a task the moment you discover work it is missing.
Try to keep exactly ONE task \`in_progress\`. \`TaskUpdate\` it to \`in_progress\` right before you start that stage, and to \`completed\` the instant you finish it — one at a time, never batched at the end. Only mark \`completed\` when the work is genuinely done; if the build fails, a step is partial, or you hit a blocker, keep it \`in_progress\` and add a task for the fix.
After you complete a task, take the next one in order (lowest id first — earlier stages set up later ones), mark it \`in_progress\`, and continue. Driving the list in order top to bottom is how you finish every stage.
Expand Down Expand Up @@ -129,7 +129,7 @@ When a skill provides a numbered or bulleted list of questions, translate the en
- For \`single\` and \`multi\`, extract the alternatives from the prose into \`options\` as \`{ label, value }\` pairs. Use the human phrase as \`label\` and a lowercase-hyphenated form as \`value\` (e.g., \`label: "Vanilla JS"\`, \`value: "vanilla-js"\`).
- Use a kebab-case slug of the question label as \`id\` (e.g., "Tech stack" → \`tech-stack\`, "Show frequency" → \`show-frequency\`).
- Do not invent fields the schema does not define (no \`source\`, \`category\`, \`priority\`, etc.) — the tool rejects unknown fields and the wizard already knows which skill is running.
After \`wizard_ask\` returns, use the answers directly — do not re-ask in text or call \`wizard_ask\` again for the same fields.
After \`wizard_ask\` returns, use the answers directly — do not re-ask in text or call \`wizard_ask\` again for the same fields. The one exception is a fully timed-out return, where every answer is \`__timed_out__\`: nothing was answered, so ask the same questions again to keep waiting, and follow the guidance that result carries.

## This runtime
Below are important guidance on the harness constraints you are bound to. Follow them as commandments.
Expand Down Expand Up @@ -172,7 +172,7 @@ When a skill provides a numbered or bulleted list of questions, translate the en
- For \`single\` and \`multi\`, extract the alternatives from the prose into \`options\` as \`{ label, value }\` pairs. Use the human phrase as \`label\` and a lowercase-hyphenated form as \`value\` (e.g., \`label: "Vanilla JS"\`, \`value: "vanilla-js"\`).
- Use a kebab-case slug of the question label as \`id\` (e.g., "Tech stack" → \`tech-stack\`, "Show frequency" → \`show-frequency\`).
- Do not invent fields the schema does not define (no \`source\`, \`category\`, \`priority\`, etc.) — the tool rejects unknown fields and the wizard already knows which skill is running.
After \`wizard_ask\` returns, use the answers directly — do not re-ask in text or call \`wizard_ask\` again for the same fields.
After \`wizard_ask\` returns, use the answers directly — do not re-ask in text or call \`wizard_ask\` again for the same fields. The one exception is a fully timed-out return, where every answer is \`__timed_out__\`: nothing was answered, so ask the same questions again to keep waiting, and follow the guidance that result carries.
ALWAYS surface a custom-scout proposal in step 6b: bring the user your one or two strongest candidate scouts even when the built-in troop looks sufficient. The proposal ask leads with a "None — keep the built-in troop" option, so declining costs the user one keystroke — but a proposal you silently skip is coverage they never got to see or judge. Where the skill says to skip the ask when the gap analysis finds no candidate, do NOT skip: pick your best candidates anyway and let the user decide.
Rank candidates at the discriminator level, not the category level. "Covered" only means an enabled scout would actually FIRE for that failure mode: a conversion-rate watcher does not catch entry volume collapsing; a Stripe-transaction watcher does not catch a lead form going silent. A surface whose failure mode has no firing condition among the enabled scouts is your strongest candidate.
Be honest in the option descriptions: if a candidate overlaps something an enabled scout partially watches, say so in its description rather than dropping the candidate. The user chooses with full information; you do not gatekeep on their behalf.
Expand Down
9 changes: 9 additions & 0 deletions src/lib/agent/__tests__/commandments.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -133,6 +133,15 @@ describe('commandments by axis', () => {
it('tells the agent to use answers directly without re-asking', () => {
expect(text).toMatch(/do not re-ask/i);
});

// The no-re-ask rule and the timeout guidance in `WIZARD_ASK_TOOL_DESCRIPTION`
// both reach the agent in one context. Without this carve-out the commandment
// — assembled first, in every run — forbids the retry that is the whole point
// of answering a timeout with its own sentinel.
it('exempts a fully timed-out return from the no-re-ask rule', () => {
expect(text).toMatch(/`__timed_out__`/);
expect(text).toMatch(/ask the same questions again to keep waiting/i);
});
});
});

Expand Down
2 changes: 1 addition & 1 deletion src/lib/agent/commandments.ts
Original file line number Diff line number Diff line change
Expand Up @@ -45,6 +45,6 @@ export const WIZARD_COMMANDMENTS = [
' - For `single` and `multi`, extract the alternatives from the prose into `options` as `{ label, value }` pairs. Use the human phrase as `label` and a lowercase-hyphenated form as `value` (e.g., `label: "Vanilla JS"`, `value: "vanilla-js"`).',
' - Use a kebab-case slug of the question label as `id` (e.g., "Tech stack" → `tech-stack`, "Show frequency" → `show-frequency`).',
' - Do not invent fields the schema does not define (no `source`, `category`, `priority`, etc.) — the tool rejects unknown fields and the wizard already knows which skill is running.',
'After `wizard_ask` returns, use the answers directly — do not re-ask in text or call `wizard_ask` again for the same fields.',
'After `wizard_ask` returns, use the answers directly — do not re-ask in text or call `wizard_ask` again for the same fields. The one exception is a fully timed-out return, where every answer is `__timed_out__`: nothing was answered, so ask the same questions again to keep waiting, and follow the guidance that result carries.',
].join('\n'),
];
Loading
Loading