From 3cfd0098e4b7430b48a87e124fbd7d8879366d88 Mon Sep 17 00:00:00 2001 From: Evanfeenstra Date: Tue, 15 Sep 2026 12:05:16 -0700 Subject: [PATCH 1/2] propose_code_change: take the diff from the swarm's worktree capture, refuse incomplete runs Preview runs were coming back with narration instead of a diff while the edit had landed in the swarm's worktree (stakwork/stakgraph#1692 has the root cause: a gateway tool-version rewrite made the AI SDK's tool loop exit silently, and the swarm reported success). The swarm now reads the diff from the ephemeral worktree itself and returns it as `diff`, with `diff_error` when it looked and found a reason there is none, and `incomplete` when the run never reached a proper termination. - Prefer the swarm's `diff` over anything in the model's text. It is ground truth and includes new files; the model's pasted diff was a transcription that `git diff HEAD` would have left new files out of. - Refuse `incomplete` runs outright: the text is narration and the worktree may hold a half-applied change, so no proposal card. - Believe `diff_error` (no_changes, change_too_large, secrets_detected) over a diff the model pasted, with a specific message for each. - Keep the regex-on-text extraction as the fallback for swarms that predate the capture, and make that fallback stage first (`git add -A && git diff --cached`) so it too includes new files. - Surface the swarm's error text when the run fails instead of a generic "timed out or returned an error". Tests cover each source and refusal, the error passthrough, and the prompt change. Backward compatible with older swarms. --- .../unit/lib/ai/codeChangeTools.test.ts | 119 +++++++++++- src/lib/ai/codeChangeTools.ts | 182 ++++++++++-------- 2 files changed, 215 insertions(+), 86 deletions(-) diff --git a/src/__tests__/unit/lib/ai/codeChangeTools.test.ts b/src/__tests__/unit/lib/ai/codeChangeTools.test.ts index 45aa621ebf..725eddf2d9 100644 --- a/src/__tests__/unit/lib/ai/codeChangeTools.test.ts +++ b/src/__tests__/unit/lib/ai/codeChangeTools.test.ts @@ -124,9 +124,7 @@ beforeEach(() => { json: async () => ({ default_branch: "master" }), }), ); - vi.mocked(db.repository.findFirst).mockImplementation((async (a: { - where: { repositoryUrl: string }; - }) => + vi.mocked(db.repository.findFirst).mockImplementation((async (a: { where: { repositoryUrl: string } }) => a.where.repositoryUrl === TARGET_REPO || a.where.repositoryUrl === OTHER_REPO ? { id: "repo-1", name: "hive", repositoryUrl: a.where.repositoryUrl } : null) as never); @@ -142,9 +140,7 @@ describe("propose_code_change — repository selection", () => { expect(out.error).toBeUndefined(); expect(out.kind).toBe("codeChange"); - expect((out.payload as { repositoryUrl: string }).repositoryUrl).toBe( - TARGET_REPO, - ); + expect((out.payload as { repositoryUrl: string }).repositoryUrl).toBe(TARGET_REPO); }); it("forwards exactly one explicit repo_url to the swarm", async () => { @@ -229,3 +225,114 @@ describe("propose_code_change — authorization is unchanged", () => { expect(mockRepoAgent).not.toHaveBeenCalled(); }); }); + +// ── Diff source ───────────────────────────────────────────────────────────── +// +// Newer swarms read the diff from the ephemeral worktree after the run and +// return it as `diff`, with `diff_error` when there is a reason there is +// none and `incomplete` when the run never reached a proper termination. +// Older swarms return only the model's text. The tool must prefer the +// worktree, believe the swarm's reasons, and refuse unfinished runs. + +const WORKTREE_DIFF = [ + "diff --git a/src/lib/ai/canvasTools.ts b/src/lib/ai/canvasTools.ts", + "index 1111111..2222222 100644", + "--- a/src/lib/ai/canvasTools.ts", + "+++ b/src/lib/ai/canvasTools.ts", + "@@ -1,3 +1,4 @@", + " function compactNode(n: CanvasNode) {", + "+ // captured from the worktree", + " const out = {};", + " }", + "diff --git a/src/tests/new.test.ts b/src/tests/new.test.ts", + "new file mode 100644", + "index 0000000..3333333", + "--- /dev/null", + "+++ b/src/tests/new.test.ts", + "@@ -0,0 +1 @@", + "+export const x = 1;", + "", +].join("\n"); + +describe("propose_code_change — diff source", () => { + beforeEach(() => mockWorkspace(2)); + + it("prefers the swarm's worktree diff over anything in the model's text", async () => { + mockRepoAgent.mockResolvedValue({ + content: "Done. Here is what I changed:\n" + DIFF, + diff: WORKTREE_DIFF + "\n\n", + diff_files: 2, + }); + + const out = await run(); + + expect(out.error).toBeUndefined(); + expect(out.kind).toBe("codeChange"); + const payload = out.payload as { diff: string; filesChanged: number }; + expect(payload.diff).toBe(WORKTREE_DIFF.trimEnd()); + expect(payload.filesChanged).toBe(2); + expect(payload.diff).not.toContain("bg-blue-500"); + }); + + it("falls back to the model's text when the swarm sends no diff field", async () => { + mockRepoAgent.mockResolvedValue({ content: "Applied.\n" + DIFF }); + + const out = await run(); + + expect(out.error).toBeUndefined(); + expect((out.payload as { diff: string }).diff).toBe(DIFF.trimEnd()); + }); + + it("refuses a run the swarm flags as incomplete, even if the text holds a diff", async () => { + mockRepoAgent.mockResolvedValue({ + content: DIFF, + incomplete: { reason: "stall" }, + }); + + const out = await run(); + + expect(out.kind).toBeUndefined(); + expect(out.error).toContain("stopped before finishing"); + expect(out.error).toContain("stall"); + }); + + it("believes the swarm's no_changes over a diff pasted by the model", async () => { + mockRepoAgent.mockResolvedValue({ + content: DIFF, + diff_error: { failure: "no_changes", error: "No changes in the worktree after the run" }, + }); + + const out = await run(); + + expect(out.kind).toBeUndefined(); + expect(out.error).toContain("without changing any files"); + }); + + it("maps the swarm's secret scan to the credentials refusal", async () => { + mockRepoAgent.mockResolvedValue({ + content: "", + diff_error: { failure: "secrets_detected", error: "Secret scan found 1 finding(s)" }, + }); + + const out = await run(); + + expect(out.error).toContain("known credentials"); + }); + + it("surfaces the swarm's error text when the run fails", async () => { + mockRepoAgent.mockRejectedValue(new Error("Run ended on a tool call that could not be resolved: code_execution.")); + + const out = await run(); + + expect(out.error).toContain("could not be resolved: code_execution"); + }); + + it("tells the agent to stage before diffing so new files are not lost", async () => { + await run(); + + const params = mockRepoAgent.mock.calls[0][2]; + expect(params.prompt).toContain("git add -A && git diff --cached"); + expect(params.prompt).not.toContain("git diff HEAD"); + expect(params.ephemeral).toBe(true); + }); +}); diff --git a/src/lib/ai/codeChangeTools.ts b/src/lib/ai/codeChangeTools.ts index 25255ac812..d2613e6369 100644 --- a/src/lib/ai/codeChangeTools.ts +++ b/src/lib/ai/codeChangeTools.ts @@ -34,10 +34,7 @@ import crypto from "crypto"; import { tool, type ToolSet } from "ai"; import { z } from "zod"; import { db } from "@/lib/db"; -import { - repoAgent, - REPO_AGENT_CANCELLED_MARKER, -} from "@/lib/ai/askTools"; +import { repoAgent, REPO_AGENT_CANCELLED_MARKER } from "@/lib/ai/askTools"; import { parseUnifiedDiff, enforceDiffCaps, @@ -58,15 +55,51 @@ function sha256Hex(text: string): string { return crypto.createHash("sha256").update(text, "utf8").digest("hex"); } +/** + * Fields of the swarm's terminal result this tool reads. `content` is the + * model's final text (every swarm). `diff`, `diff_error` and `incomplete` + * come from swarms that read the preview worktree themselves after the run + * (stakgraph `captureWorktreeDiff`) and flag runs that never finished; older + * swarms omit them, and the model's text is then the only source. + */ +interface SwarmPreviewResult { + content?: unknown; + diff?: string; + diff_files?: number; + diff_error?: { failure: string; error: string }; + incomplete?: { reason: string }; +} + +function describeSwarmDiffError(e: { failure: string; error: string }): string { + switch (e.failure) { + case "no_changes": + return ( + "The agent finished without changing any files in the repository, " + + "so there is nothing to propose. Try a more specific prompt." + ); + case "change_too_large": + return ( + `The change is too large for a preview (${e.error}). ` + + "Use `propose_feature` — the feature pipeline handles large changes." + ); + case "secrets_detected": + return ( + "The change contains patterns matching known credentials. " + + "Review it manually and ensure no secrets are included before proposing." + ); + default: + return ( + `The swarm could not read the change from the preview worktree (${e.error}). ` + + "Try again, or use `propose_feature`." + ); + } +} + /** * Fetch the GitHub default branch for a repo via a plain authenticated GET. * Falls back to "main" on any error so a PAT scope issue doesn't block preview. */ -async function fetchDefaultBranch( - owner: string, - repo: string, - pat: string, -): Promise { +async function fetchDefaultBranch(owner: string, repo: string, pat: string): Promise { try { const url = `https://api.github.com/repos/${encodeURIComponent(owner)}/${encodeURIComponent(repo)}`; const res = await fetch(url, { @@ -102,9 +135,7 @@ export function buildCodeChangeTools(ctx: CapabilityContext): ToolSet { "repository, when you cannot tell which repository it belongs in, or " + "for database migrations, schema changes, and large multi-file refactors.", inputSchema: z.object({ - workspaceSlug: z - .string() - .describe("The workspace slug that owns the target repository."), + workspaceSlug: z.string().describe("The workspace slug that owns the target repository."), repositoryUrl: z .string() .url() @@ -119,15 +150,8 @@ export function buildCodeChangeTools(ctx: CapabilityContext): ToolSet { .string() .min(1) .max(256) - .describe( - "Short PR title (no leading dash). " + - "The [Jamie] prefix is added automatically.", - ), - body: z - .string() - .max(65536) - .default("") - .describe("PR body / description (markdown)."), + .describe("Short PR title (no leading dash). " + "The [Jamie] prefix is added automatically."), + body: z.string().max(65536).default("").describe("PR body / description (markdown)."), prompt: z .string() .min(1) @@ -166,22 +190,15 @@ export function buildCodeChangeTools(ctx: CapabilityContext): ToolSet { // before any credentials or swarm keys are accessed. if (workspace.members.length === 0) { return { - error: - "You do not have access to workspace '" + workspaceSlug + "'.", + error: "You do not have access to workspace '" + workspaceSlug + "'.", }; } // Org-membership check: ctx.orgId is the SourceControlOrg.id the // caller already validated. Refuse cross-org requests. - if ( - ctx.orgId && - workspace.sourceControlOrg && - workspace.sourceControlOrg.id !== ctx.orgId - ) { + if (ctx.orgId && workspace.sourceControlOrg && workspace.sourceControlOrg.id !== ctx.orgId) { return { - error: - "Repository does not belong to the active org. " + - "Use `propose_feature` for cross-org changes.", + error: "Repository does not belong to the active org. " + "Use `propose_feature` for cross-org changes.", }; } @@ -222,10 +239,7 @@ export function buildCodeChangeTools(ctx: CapabilityContext): ToolSet { let swarmApiKey: string; try { - swarmApiKey = EncryptionService.getInstance().decryptField( - "swarmApiKey", - workspace.swarm.swarmApiKey, - ); + swarmApiKey = EncryptionService.getInstance().decryptField("swarmApiKey", workspace.swarm.swarmApiKey); } catch { return { error: "Failed to decrypt swarm credentials for this workspace.", @@ -239,13 +253,8 @@ export function buildCodeChangeTools(ctx: CapabilityContext): ToolSet { : `https://${swarmUrlObj.hostname}:3355`; // ── 3. GitHub PAT ────────────────────────────────────────────── - const { getGithubUsernameAndPAT } = await import( - "@/lib/auth/nextauth" - ); - const githubProfile = await getGithubUsernameAndPAT( - ctx.userId, - workspaceSlug, - ); + const { getGithubUsernameAndPAT } = await import("@/lib/auth/nextauth"); + const githubProfile = await getGithubUsernameAndPAT(ctx.userId, workspaceSlug); const pat = githubProfile?.token ?? ""; // ── 4. Fetch live baseBranchDisplay from GitHub ──────────────── @@ -274,11 +283,8 @@ export function buildCodeChangeTools(ctx: CapabilityContext): ToolSet { let activeRequestId: string | undefined; const hooks = convId ? await (async () => { - const { - setActiveRun, - isAbortRequestedForRun, - notifyRunActive, - } = await import("@/services/canvas-active-runs-hooks"); + const { setActiveRun, isAbortRequestedForRun, notifyRunActive } = + await import("@/services/canvas-active-runs-hooks"); return { onRequestId: async (requestId: string) => { activeRequestId = requestId; @@ -293,8 +299,7 @@ export function buildCodeChangeTools(ctx: CapabilityContext): ToolSet { ); await notifyRunActive(convId, true); }, - isAbortRequested: async () => - isAbortRequestedForRun(convId, activeRequestId ?? ""), + isAbortRequested: async () => isAbortRequestedForRun(convId, activeRequestId ?? ""), }; })() : undefined; @@ -302,18 +307,21 @@ export function buildCodeChangeTools(ctx: CapabilityContext): ToolSet { // ── 7. READ-ONLY repo_agent call ─────────────────────────────── // `toolsConfig.create_pr` is INTENTIONALLY ABSENT — structural // guarantee that no PR is opened during this preview run. + // The swarm reads the diff from the preview worktree itself when the + // run finishes (`diff` on the result), so the diff the model pastes + // is a fallback for swarms that predate that. It stages first: + // `git diff HEAD` alone omits new files. const agentPrompt = `${prompt}\n\n` + `Repository: ${repositoryUrl}\n` + `IMPORTANT: This is a READ-ONLY preview run. Do NOT push any branch ` + - `or open a pull request. Apply the change locally using apply_patch ` + - `or git apply, then output the full unified diff by running: git diff HEAD\n` + - `Output ONLY the raw unified diff (starting with "--- " / "+++ ") ` + - `with no additional commentary after it.`; - - let rawResult: - | Record - | typeof REPO_AGENT_CANCELLED_MARKER; + `or open a pull request. Apply the change directly in the working tree ` + + `(bash, or the file editor). When every edit is done, run: ` + + `git add -A && git diff --cached\n` + + `and output that unified diff (starting with "--- " / "+++ ") as your ` + + `final answer, with no commentary after it.`; + + let rawResult: Record | typeof REPO_AGENT_CANCELLED_MARKER; try { rawResult = await repoAgent( swarmUrl, @@ -331,47 +339,61 @@ export function buildCodeChangeTools(ctx: CapabilityContext): ToolSet { bifrost, hooks, ); - } catch { + } catch (err) { + const detail = err instanceof Error && err.message ? ` (${err.message})` : ""; return { error: - "The code-change preview timed out or the swarm returned an error. " + + `The code-change preview failed${detail}. ` + "For large or multi-file changes, use `propose_feature` so the " + "work runs as a background task instead.", }; } finally { if (convId && activeRequestId) { - const { clearActiveRun, notifyRunActive } = await import( - "@/services/canvas-active-runs-hooks" - ); - const { wasLast } = await clearActiveRun( - convId, - activeRequestId, - ).catch(() => ({ wasLast: true })); + const { clearActiveRun, notifyRunActive } = await import("@/services/canvas-active-runs-hooks"); + const { wasLast } = await clearActiveRun(convId, activeRequestId).catch(() => ({ wasLast: true })); if (wasLast) await notifyRunActive(convId, false).catch(() => {}); } } if (rawResult === REPO_AGENT_CANCELLED_MARKER) { + return { + error: "Code-change preview was cancelled. " + "For large changes, use `propose_feature` instead.", + }; + } + + // ── 8. Take the diff ─────────────────────────────────────────── + const swarm = rawResult as unknown as SwarmPreviewResult; + + // The run ended without a proper termination (stall, truncation, + // stream error). Whatever text came back is narration, and the + // worktree may hold a half-applied change. Not a proposal. + if (swarm.incomplete) { return { error: - "Code-change preview was cancelled. " + - "For large changes, use `propose_feature` instead.", + `The agent stopped before finishing the change (${swarm.incomplete.reason}). ` + + "No proposal was created. Try again with a more specific prompt, " + + "or use `propose_feature` for larger changes.", }; } - // ── 8. Extract unified diff from swarm output ────────────────── - const agentOutput = - typeof rawResult.content === "string" - ? rawResult.content - : JSON.stringify(rawResult); + // A swarm that read the worktree and found a reason there is no diff + // is believed over anything the model pasted into its answer. + if (swarm.diff_error) { + return { error: describeSwarmDiffError(swarm.diff_error) }; + } - // Grab the first unified-diff block from the output. - const diffMatch = agentOutput.match( - /(---[ \t][^\n]+\n\+\+\+[ \t][^\n]+[\s\S]*)/, - ); - const rawDiff = diffMatch - ? diffMatch[1].trimEnd() - : agentOutput.trim(); + // Newer swarms return the worktree's own diff — ground truth, new + // files included. Older swarms return only the model's text, so the + // regex extraction stays as the fallback. + let rawDiff: string; + if (typeof swarm.diff === "string" && swarm.diff.trim()) { + rawDiff = swarm.diff.trimEnd(); + } else { + const agentOutput = typeof swarm.content === "string" ? swarm.content : JSON.stringify(rawResult); + // Grab the first unified-diff block from the output. + const diffMatch = agentOutput.match(/(---[ \t][^\n]+\n\+\+\+[ \t][^\n]+[\s\S]*)/); + rawDiff = diffMatch ? diffMatch[1].trimEnd() : agentOutput.trim(); + } // ── 9. diffHygiene validation ────────────────────────────────── const parseResult = parseUnifiedDiff(rawDiff); From d5fed2a85c734328fb6c05fdb06ea2b0c884de28 Mon Sep 17 00:00:00 2001 From: Evanfeenstra Date: Tue, 15 Sep 2026 12:11:09 -0700 Subject: [PATCH 2/2] Read the swarm's preview diff from `preview`, mirroring `pr` The swarm reports the ephemeral worktree diff as one `preview` field with the create_pr result's ok / failure / error shape, instead of three top-level fields. --- .../unit/lib/ai/codeChangeTools.test.ts | 21 +++++++++++------ src/lib/ai/codeChangeTools.ts | 23 ++++++++++--------- 2 files changed, 26 insertions(+), 18 deletions(-) diff --git a/src/__tests__/unit/lib/ai/codeChangeTools.test.ts b/src/__tests__/unit/lib/ai/codeChangeTools.test.ts index 725eddf2d9..a135349e09 100644 --- a/src/__tests__/unit/lib/ai/codeChangeTools.test.ts +++ b/src/__tests__/unit/lib/ai/codeChangeTools.test.ts @@ -229,8 +229,8 @@ describe("propose_code_change — authorization is unchanged", () => { // ── Diff source ───────────────────────────────────────────────────────────── // // Newer swarms read the diff from the ephemeral worktree after the run and -// return it as `diff`, with `diff_error` when there is a reason there is -// none and `incomplete` when the run never reached a proper termination. +// return it as `preview` (same `ok` / `failure` / `error` shape as `pr`), +// plus `incomplete` when the run never reached a proper termination. // Older swarms return only the model's text. The tool must prefer the // worktree, believe the swarm's reasons, and refuse unfinished runs. @@ -260,8 +260,7 @@ describe("propose_code_change — diff source", () => { it("prefers the swarm's worktree diff over anything in the model's text", async () => { mockRepoAgent.mockResolvedValue({ content: "Done. Here is what I changed:\n" + DIFF, - diff: WORKTREE_DIFF + "\n\n", - diff_files: 2, + preview: { ok: true, diff: WORKTREE_DIFF + "\n\n", filesChanged: 2 }, }); const out = await run(); @@ -274,7 +273,7 @@ describe("propose_code_change — diff source", () => { expect(payload.diff).not.toContain("bg-blue-500"); }); - it("falls back to the model's text when the swarm sends no diff field", async () => { + it("falls back to the model's text when the swarm sends no preview", async () => { mockRepoAgent.mockResolvedValue({ content: "Applied.\n" + DIFF }); const out = await run(); @@ -299,7 +298,11 @@ describe("propose_code_change — diff source", () => { it("believes the swarm's no_changes over a diff pasted by the model", async () => { mockRepoAgent.mockResolvedValue({ content: DIFF, - diff_error: { failure: "no_changes", error: "No changes in the worktree after the run" }, + preview: { + ok: false, + failure: "no_changes", + error: "No changes in the worktree after the run", + }, }); const out = await run(); @@ -311,7 +314,11 @@ describe("propose_code_change — diff source", () => { it("maps the swarm's secret scan to the credentials refusal", async () => { mockRepoAgent.mockResolvedValue({ content: "", - diff_error: { failure: "secrets_detected", error: "Secret scan found 1 finding(s)" }, + preview: { + ok: false, + failure: "secrets_detected", + error: "Secret scan found 1 finding(s)", + }, }); const out = await run(); diff --git a/src/lib/ai/codeChangeTools.ts b/src/lib/ai/codeChangeTools.ts index d2613e6369..adaf04d23d 100644 --- a/src/lib/ai/codeChangeTools.ts +++ b/src/lib/ai/codeChangeTools.ts @@ -57,16 +57,17 @@ function sha256Hex(text: string): string { /** * Fields of the swarm's terminal result this tool reads. `content` is the - * model's final text (every swarm). `diff`, `diff_error` and `incomplete` - * come from swarms that read the preview worktree themselves after the run - * (stakgraph `captureWorktreeDiff`) and flag runs that never finished; older - * swarms omit them, and the model's text is then the only source. + * model's final text (every swarm). `preview` and `incomplete` come from + * swarms that read the preview worktree themselves after the run (stakgraph + * `captureWorktreeDiff`) and flag runs that never finished; older swarms + * omit them, and the model's text is then the only source. + * + * `preview` is the ephemeral-run counterpart of the `pr` result on create_pr + * runs, with the same `ok` / `failure` / `error` shape. */ interface SwarmPreviewResult { content?: unknown; - diff?: string; - diff_files?: number; - diff_error?: { failure: string; error: string }; + preview?: { ok: true; diff: string; filesChanged: number } | { ok: false; failure: string; error: string }; incomplete?: { reason: string }; } @@ -378,16 +379,16 @@ export function buildCodeChangeTools(ctx: CapabilityContext): ToolSet { // A swarm that read the worktree and found a reason there is no diff // is believed over anything the model pasted into its answer. - if (swarm.diff_error) { - return { error: describeSwarmDiffError(swarm.diff_error) }; + if (swarm.preview && !swarm.preview.ok) { + return { error: describeSwarmDiffError(swarm.preview) }; } // Newer swarms return the worktree's own diff — ground truth, new // files included. Older swarms return only the model's text, so the // regex extraction stays as the fallback. let rawDiff: string; - if (typeof swarm.diff === "string" && swarm.diff.trim()) { - rawDiff = swarm.diff.trimEnd(); + if (swarm.preview?.ok && swarm.preview.diff.trim()) { + rawDiff = swarm.preview.diff.trimEnd(); } else { const agentOutput = typeof swarm.content === "string" ? swarm.content : JSON.stringify(rawResult); // Grab the first unified-diff block from the output.