Skip to content

Commit 5c46bd3

Browse files
V48 Gate 3 (implementation-only): stitch repairs carry the validation error; the schema shape hint renders every field
Root-caused from the raw LLM I/O sidecar of the two deposit runs that died on 'StitchUntilComplete exceeded maximum stitch attempts (5)' (runs ea150f3e/02b93ba8, 2026-07-01): every option the model produced omitted 'confidence' — the structured-output shape hint truncated nested objects to their first six fields, so the model never saw confidence/patch/ needinessSignal in the schema hint, and the stitch instruction ('Continue and complete the previous output') never said what failed validation, so five stitch cycles reproduced the same omission until the step threw. - inferField now renders EVERY nested field of a ZodObject (no slice(0,6) ellipsis hiding required fields). - factoryStitchUntilComplete captures the last schema-validation error and prompts the repair with it, instead of a truncation-only 'continue'. Regression-pinned in stitch-self-repair-and-schema-shape.test.ts against the deposit candidateSetSchema shape. agent-generics 23 tests green; tsc clean; asset-pack 227 green. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
1 parent 2b0c821 commit 5c46bd3

2 files changed

Lines changed: 79 additions & 9 deletions

File tree

Lines changed: 62 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,62 @@
1+
// @ts-nocheck
2+
import { z } from 'zod';
3+
import { Execution } from '@bitcode/execution-generics';
4+
import { factoryStitchUntilComplete, factoryStructuredOutput } from '../substeps/factories';
5+
6+
describe('StitchUntilComplete self-repair instruction', () => {
7+
it('tells the model exactly what failed schema validation instead of only "continue"', async () => {
8+
const schema = z.object({ value: z.string(), confidence: z.number() });
9+
const stitchInputs: any[] = [];
10+
const gen = async (input: any) => {
11+
stitchInputs.push(input);
12+
return { value: 'x', confidence: 0.9 };
13+
};
14+
const stitch = factoryStitchUntilComplete([gen], schema);
15+
const result = await stitch({ value: 'x' }, new Execution('root'));
16+
17+
expect(stitchInputs).toHaveLength(1);
18+
expect(stitchInputs[0].instruction).toContain('failed schema validation');
19+
expect(stitchInputs[0].instruction).toContain('confidence');
20+
expect(result.finalOutput).toEqual({ value: 'x', confidence: 0.9 });
21+
});
22+
});
23+
24+
describe('structured output schema shape rendering', () => {
25+
it('renders every nested field — required fields beyond the sixth must reach the model', async () => {
26+
// Mirrors the deposit candidateSetSchema shape whose 7th..9th fields
27+
// (confidence, patch, needinessSignal) were hidden by the truncated hint.
28+
const candidate = z.object({
29+
kind: z.string(),
30+
title: z.string(),
31+
summary: z.string(),
32+
coveredSourcePaths: z.array(z.string()),
33+
measurements: z.record(z.string(), z.number()),
34+
measurementRationale: z.string(),
35+
confidence: z.number(),
36+
patch: z.object({ fileChanges: z.array(z.object({ path: z.string(), op: z.enum(['create', 'modify', 'delete']) })), patchSummary: z.string() }),
37+
needinessSignal: z.object({ demand: z.number(), saturation: z.number(), rationale: z.string() }).optional(),
38+
});
39+
const schema = z.object({ options: z.array(candidate) });
40+
41+
let capturedUserPrompt = '';
42+
const exec = new Execution('root') as any;
43+
exec.llms = {
44+
getDefaultLLM: () => async (llmInput: any) => {
45+
const user = (llmInput.messages || []).find((m: any) => m.role === 'user');
46+
capturedUserPrompt = user?.content ?? '';
47+
return {
48+
content: JSON.stringify({ options: [] }),
49+
usage: { totalTokens: 5 },
50+
metadata: { provider: 'test', model: 'test-model' },
51+
};
52+
},
53+
};
54+
55+
const structured = factoryStructuredOutput(schema);
56+
await structured({ context: 'synthesize' }, exec);
57+
58+
for (const field of ['kind', 'title', 'summary', 'coveredSourcePaths', 'measurements', 'measurementRationale', 'confidence', 'patch', 'needinessSignal']) {
59+
expect(capturedUserPrompt).toContain(`"${field}"`);
60+
}
61+
});
62+
});

packages/agent-generics/src/substeps/factories.ts

Lines changed: 17 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -561,6 +561,10 @@ export function factoryStitchUntilComplete<T>(
561561
let currentResult = input;
562562
let stitchCount = 0;
563563
const maxStitches = 5; // Prevent infinite loops
564+
// The most recent schema-validation failure. A stitch prompted only with
565+
// "continue" cannot repair a schema gap (e.g. a field the model never
566+
// emits) — the model must be told exactly what failed validation.
567+
let lastValidationError: string | undefined;
564568

565569
// Check if output appears truncated (measure the structured output if present)
566570
const checkTruncation = (candidate: any): boolean => {
@@ -583,8 +587,9 @@ export function factoryStitchUntilComplete<T>(
583587
: currentResult;
584588
outputSchema.parse(candidate);
585589
break; // Valid complete output; no stitching required
586-
} catch {
590+
} catch (e) {
587591
// Fall through to truncation/stitching logic
592+
lastValidationError = e instanceof Error ? e.message : String(e);
588593
}
589594
}
590595

@@ -599,11 +604,8 @@ export function factoryStitchUntilComplete<T>(
599604
break; // Valid complete output
600605
} catch (e) {
601606
// Output incomplete, needs stitching
602-
failsafeExec.store(
603-
'validation',
604-
'error',
605-
e instanceof Error ? e.message : String(e)
606-
);
607+
lastValidationError = e instanceof Error ? e.message : String(e);
608+
failsafeExec.store('validation', 'error', lastValidationError);
607609
}
608610
} else {
609611
break; // No schema to validate against
@@ -618,7 +620,10 @@ export function factoryStitchUntilComplete<T>(
618620
const stitchInput = {
619621
context: buildStitchContext(input),
620622
partialOutput: minimalPartial,
621-
instruction: 'Continue and complete the previous output'
623+
instruction: lastValidationError
624+
? `The previous output failed schema validation: ${lastValidationError.slice(0, 600)}. ` +
625+
'Return the full corrected JSON object with every required field present and within its constraints.'
626+
: 'Continue and complete the previous output'
622627
} as any;
623628

624629
for (let i = 0; i < generationSubSteps.length; i++) {
@@ -830,8 +835,11 @@ function inferField(v: z.ZodTypeAny): string {
830835
case z.ZodFirstPartyTypeKind.ZodObject: {
831836
const shape = getZodObjectShape(v);
832837
if (!shape) return '{ ... }';
833-
const keys = Object.entries(shape).slice(0, 6);
834-
return `{ ${keys.map(([key, value]) => `"${key}": ${inferField(value)}`).join(', ')}${Object.keys(shape).length > keys.length ? ', ...' : ''} }`;
838+
// Render EVERY field: a truncated shape hides required fields from the
839+
// model, which then systematically omits them and no amount of
840+
// stitching/retrying can converge on a schema-valid output.
841+
const keys = Object.entries(shape);
842+
return `{ ${keys.map(([key, value]) => `"${key}": ${inferField(value)}`).join(', ')} }`;
835843
}
836844
case z.ZodFirstPartyTypeKind.ZodRecord: return '{ [key: string]: any }';
837845
case z.ZodFirstPartyTypeKind.ZodEnum: {

0 commit comments

Comments
 (0)