From 527aa7ede109090f8ab8afe1e13de4d7be258088 Mon Sep 17 00:00:00 2001 From: Matt Date: Tue, 30 Jun 2026 14:51:59 +0200 Subject: [PATCH 1/7] feat(workflow-executor): add 3-way execution mode to the load related record step (PRD-148) Manual presents the narrowed list with no AI; AI-assisted suggests a record; Full AI selects + loads and auto-skips on no source record, empty list, or AI judging none relevant. selectedRecordRef is optional now (no-source skip); the wire serializer guards it. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../load-related-record-step-executor.ts | 177 ++++++++---- .../src/http/step-serializer.ts | 6 +- .../src/types/step-execution-data.ts | 4 +- .../src/types/validated/step-definition.ts | 8 +- .../load-related-record-step-executor.test.ts | 265 ++++++++++++++++-- .../test/http/step-serializer.test.ts | 13 + 6 files changed, 396 insertions(+), 77 deletions(-) diff --git a/packages/workflow-executor/src/executors/load-related-record-step-executor.ts b/packages/workflow-executor/src/executors/load-related-record-step-executor.ts index 170f91b347..60ea709e01 100644 --- a/packages/workflow-executor/src/executors/load-related-record-step-executor.ts +++ b/packages/workflow-executor/src/executors/load-related-record-step-executor.ts @@ -21,6 +21,7 @@ import { NoRelationshipFieldsError, RelatedRecordNotFoundError, RelationNotFoundError, + SourceRecordMissingError, StepStateError, } from '../errors'; import RecordStepExecutor from './record-step-executor'; @@ -103,13 +104,18 @@ export default class LoadRelatedRecordStepExecutor extends RecordStepExecutor { - if (!execution.pendingData) { + if (!execution.pendingData || !execution.selectedRecordRef) { throw new StepStateError(`Step at index ${this.context.stepIndex} has no pending data`); } + // Switching the relation in Manual mode must still not pre-select a record. + const suggestViaAi = this.context.stepDefinition.executionType !== StepExecutionMode.Manual; const schema = await this.getCollectionSchema(execution.selectedRecordRef.collectionName); const target = await this.buildTarget(schema, fieldName, execution.selectedRecordRef); - const { availableRecordIds, suggestedRecord } = await this.collectCandidateIds(target); + const { availableRecordIds, suggestedRecord } = await this.collectCandidateIds( + target, + suggestViaAi, + ); await this.context.runStore.saveStepExecution(this.context.runId, { ...execution, @@ -127,23 +133,26 @@ export default class LoadRelatedRecordStepExecutor extends RecordStepExecutor { const { stepDefinition: step } = this.context; - const target = await this.resolveTarget(); + // Manual mode never invokes AI — neither to pick the relation nor to suggest a record. + const useAi = step.executionType !== StepExecutionMode.Manual; - // Branch B -- fully automated execution + // Branch B -- Full AI: AI selects and the record is loaded with no user input (may auto-skip). if (step.executionType === StepExecutionMode.FullyAutomated) { - return this.resolveAndLoadAutomatic(target); + return this.resolveAndLoadAutomatic(useAi); } - // Branch C -- pre-fetch candidates, await user confirmation + // Branches C & D -- pre-fetch candidates, await user confirmation. AI-assisted pre-selects a + // record (suggestedRecord); Manual presents the narrowed list with no AI pick. + const target = await this.resolveTarget(useAi); const sourceSchema = await this.getCollectionSchema(target.selectedRecordRef.collectionName); - return this.saveAndAwaitInput(target, sourceSchema); + return this.saveAndAwaitInput(target, sourceSchema, useAi); } // Picks the (record, relation) pair to follow. Unlike a separate record-then-relation choice, // this lets the AI decide by what each relation LEADS TO — so "load the dvd" follows // store→dvds rather than latching onto a previously-loaded dvd whose collection just matches. - private async resolveTarget(): Promise { + private async resolveTarget(useAi: boolean): Promise { const { preRecordedArgs } = this.context.stepDefinition; const sourceRecords = @@ -168,8 +177,10 @@ export default class LoadRelatedRecordStepExecutor extends RecordStepExecutor 1 && useAi ? await this.selectRelationToFollow(eligible) : eligible[0]; return this.targetFromCandidate(chosen); } @@ -265,15 +276,20 @@ export default class LoadRelatedRecordStepExecutor extends RecordStepExecutor { const { selectedRecordRef, name, displayName } = target; - const { availableRecordIds, suggestedRecord } = await this.collectCandidateIds(target); + const { availableRecordIds, suggestedRecord } = await this.collectCandidateIds( + target, + suggestViaAi, + ); const availableFields: RelationRef[] = sourceSchema.fields .filter(isFollowableRelation) @@ -294,21 +310,28 @@ export default class LoadRelatedRecordStepExecutor extends RecordStepExecutor { if (target.relationType === 'BelongsTo' || target.relationType === 'HasOne') { const candidate = await this.fetchXToOneCandidate(target); + // The lone xToOne record pre-fills in every mode — it's the only option, not an AI pick. return candidate ? { availableRecordIds: [candidate], suggestedRecord: candidate } : { availableRecordIds: [] }; } + // Rank (AI-suggest) only when AI-assisted. allowNone is Full-AI-only, never on the await path — + // here the human is the one who decides "none relevant" via the checkbox. const { relatedData, bestIndex, relatedSchema } = await this.selectBestFromRelatedData( target, 50, + { rank: suggestViaAi, allowNone: false }, ); if (relatedData.length === 0) { @@ -325,7 +348,8 @@ export default class LoadRelatedRecordStepExecutor extends RecordStepExecutor= 0 ? toCandidate(relatedData[bestIndex]) : undefined, }; } @@ -338,14 +362,32 @@ export default class LoadRelatedRecordStepExecutor extends RecordStepExecutor { + /** + * Branch B: Full AI. xToOne loads the linked record; HasMany ranks candidates via AI; + * BelongsToMany takes the first. Auto-skips (persists `skipped` + success) when there is no source + * record, no candidate, or the AI judges none relevant — the run then advances with nothing loaded. + */ + private async resolveAndLoadAutomatic(useAi: boolean): Promise { + let target: RelationTarget; + + try { + target = await this.resolveTarget(useAi); + } catch (error) { + // No source record to load from → Full AI auto-continues. (Manual/AI-assisted let this error + // surface so the front can offer "continue without" — PRD-550.) + if (error instanceof SourceRecordMissingError) return this.persistSkip(); + throw error; + } + const record = await this.fetchRecordForRelation(target); + // Empty candidate list or AI judged none relevant → skip and move on. + if (record === null) return this.persistSkip(target.selectedRecordRef); + return this.persistAndReturn(record, target, undefined); } - private async fetchRecordForRelation(target: RelationTarget): Promise { + private async fetchRecordForRelation(target: RelationTarget): Promise { if (target.relationType === 'BelongsTo' || target.relationType === 'HasOne') { return this.fetchXToOneRecordRef(target); } @@ -357,12 +399,11 @@ export default class LoadRelatedRecordStepExecutor extends RecordStepExecutor { + // Returns null (→ Full AI skip) when the relation has no linked record. + private async fetchXToOneRecordRef(target: RelationTarget): Promise { const candidate = await this.fetchXToOneCandidate(target); - if (!candidate) { - throw new RelatedRecordNotFoundError(target.selectedRecordRef.collectionName, target.name); - } + if (!candidate) return null; return { collectionName: target.relatedCollectionName, @@ -401,7 +442,7 @@ export default class LoadRelatedRecordStepExecutor extends RecordStepExecutor { const { selectedRecordRef, pendingData, userConfirmation } = execution; - if (!pendingData) { + if (!pendingData || !selectedRecordRef) { throw new StepStateError(`Step at index ${this.context.stepIndex} has no pending data`); } @@ -448,6 +489,7 @@ export default class LoadRelatedRecordStepExecutor extends RecordStepExecutor, limit: number, + opts: { rank: boolean; allowNone: boolean } = { rank: true, allowNone: false }, ): Promise<{ relatedData: RecordData[]; bestIndex: number; @@ -457,13 +499,20 @@ export default class LoadRelatedRecordStepExecutor extends RecordStepExecutor { - const { relatedData, bestIndex } = await this.selectBestFromRelatedData(target, 50); + /** + * HasMany + Full AI: fetch top 50, then AI selects the best record. Returns null (→ skip) when + * there is no candidate or the AI judges none of them relevant. + */ + private async selectBestRelatedRecord(target: RelationTarget): Promise { + const { relatedData, bestIndex } = await this.selectBestFromRelatedData(target, 50, { + rank: true, + allowNone: true, + }); - if (relatedData.length === 0) { - throw new RelatedRecordNotFoundError(target.selectedRecordRef.collectionName, target.name); - } + if (relatedData.length === 0 || bestIndex < 0) return null; return this.toRecordRef(relatedData[bestIndex]); } - private async fetchFirstCandidate(target: RelationTarget): Promise { - const candidates = await this.fetchCandidates(target, 1); - - return candidates[0]; - } - - private async fetchCandidates( - target: Pick, - limit: number, - ): Promise { - const { selectedRecordRef, name } = target; + // BelongsToMany + Full AI: take the first candidate. Returns null (→ skip) when there is none. + private async fetchFirstCandidate(target: RelationTarget): Promise { const relatedSchema = await this.getCollectionSchema(target.relatedCollectionName); - const relatedData = await this.fetchRelatedData(target, relatedSchema, limit); + const relatedData = await this.fetchRelatedData(target, relatedSchema, 1); - if (relatedData.length === 0) { - throw new RelatedRecordNotFoundError(selectedRecordRef.collectionName, name); - } - - return relatedData.map(r => this.toRecordRef(r)); + return relatedData.length > 0 ? this.toRecordRef(relatedData[0]) : null; } private async fetchRelatedData( @@ -544,6 +585,20 @@ export default class LoadRelatedRecordStepExecutor extends RecordStepExecutor { + await this.context.runStore.saveStepExecution(this.context.runId, { + type: 'load-related-record', + stepIndex: this.context.stepIndex, + selectedRecordRef, + executionResult: { skipped: true }, + }); + + return this.buildOutcomeResult({ status: 'success' }); + } + private relationOptionLabel(candidate: RelationCandidate): string { const { record, schema, field } = candidate; @@ -650,11 +705,15 @@ export default class LoadRelatedRecordStepExecutor extends RecordStepExecutor nonRelationFields.find(f => f.displayName === dn)?.fieldName ?? dn); } - /** AI call 2 for HasMany: selects the best record by index from the candidate list. */ + /** + * AI call 2 for HasMany: selects the best record by index from the candidate list. When + * `allowNone` (Full AI), the AI may return -1 to signal that none of the candidates is relevant. + */ private async selectBestRecordIndex( candidates: RecordData[], fieldNames: string[], prompt: string | undefined, + allowNone = false, ): Promise { const filteredCandidates = candidates.map((c, i) => { const entries = Object.entries(c.values).filter( @@ -690,6 +749,7 @@ export default class LoadRelatedRecordStepExecutor extends RecordStepExecutor maxIndex) { + // NOTE: The Zod schema's .min().max() shapes the tool prompt only — it is NOT validated against + // the AI response. This guard is the sole runtime enforcement. -1 (none relevant) is accepted + // only when allowNone. + if (!Number.isInteger(recordIndex) || recordIndex < minIndex || recordIndex > maxIndex) { throw new InvalidAIResponseError( - `AI selected record index ${recordIndex} which is out of range (0-${maxIndex}) or not an integer`, + `AI selected record index ${recordIndex} which is out of range (${minIndex}-${maxIndex}) or not an integer`, ); } diff --git a/packages/workflow-executor/src/http/step-serializer.ts b/packages/workflow-executor/src/http/step-serializer.ts index 7a8a7de619..f5c057c8fb 100644 --- a/packages/workflow-executor/src/http/step-serializer.ts +++ b/packages/workflow-executor/src/http/step-serializer.ts @@ -15,9 +15,13 @@ export default function serializeStepForWire(step: StepExecutionData): unknown { return { ...step, selectedRecordRef: serializeRecordRef(step.selectedRecordRef) }; case 'load-related-record': { + // selectedRecordRef is absent on a Full AI no-source auto-skip — omit it rather than + // dereferencing undefined (which would throw and 503 the run-fetch). const result: Record = { ...step, - selectedRecordRef: serializeRecordRef(step.selectedRecordRef), + ...(step.selectedRecordRef && { + selectedRecordRef: serializeRecordRef(step.selectedRecordRef), + }), }; if (step.pendingData) { diff --git a/packages/workflow-executor/src/types/step-execution-data.ts b/packages/workflow-executor/src/types/step-execution-data.ts index 0585c237d6..59448737ee 100644 --- a/packages/workflow-executor/src/types/step-execution-data.ts +++ b/packages/workflow-executor/src/types/step-execution-data.ts @@ -183,7 +183,9 @@ export interface LoadRelatedRecordStepExecutionData WithUserConfirmation { type: 'load-related-record'; pendingData?: LoadRelatedRecordPendingData; - selectedRecordRef: RecordRef; + // Absent only for the Full AI auto-skip when there is no source record to load from + // (nothing was resolved). Always set on the await/load paths. + selectedRecordRef?: RecordRef; executionParams?: RelationRef; executionResult?: { relation: RelationRef; record: RecordRef } | { skipped: true }; } diff --git a/packages/workflow-executor/src/types/validated/step-definition.ts b/packages/workflow-executor/src/types/validated/step-definition.ts index 6ac1a42c46..089f9856da 100644 --- a/packages/workflow-executor/src/types/validated/step-definition.ts +++ b/packages/workflow-executor/src/types/validated/step-definition.ts @@ -98,10 +98,12 @@ export type TriggerActionStepDefinition = z.infer { // fetchCandidates throws RelatedRecordNotFoundError when the agent returns an // empty list. Same user-facing message as the other empty-result paths. - it('returns error when getRelatedData returns an empty array', async () => { + it('auto-skips when getRelatedData returns an empty array (Branch B)', async () => { const belongsToManySchema = makeCollectionSchema({ fields: [ { @@ -1025,11 +1025,213 @@ describe('LoadRelatedRecordStepExecutor', () => { const result = await executor.execute(); - expect(result.stepOutcome.status).toBe('error'); - expect(result.stepOutcome.error).toBe( - 'The related record could not be found. It may have been deleted.', + // Full AI: no candidate to load → skip so the run can advance (no error). + expect(result.stepOutcome.status).toBe('success'); + expect(runStore.saveStepExecution).toHaveBeenCalledWith( + 'run-1', + expect.objectContaining({ + type: 'load-related-record', + executionResult: { skipped: true }, + }), + ); + }); + }); + + describe('executionType=Manual: no AI pre-selection (PRD-148)', () => { + const customersWithAddress = makeCollectionSchema({ + fields: [ + { + fieldName: 'address', + displayName: 'Address', + isRelationship: true, + relationType: 'HasMany', + relatedCollectionName: 'addresses', + }, + ], + }); + const addressSchema = makeCollectionSchema({ + collectionName: 'addresses', + collectionDisplayName: 'Addresses', + fields: [{ fieldName: 'city', displayName: 'City', isRelationship: false }], + }); + + it('lists the narrowed candidates with no suggestion and never invokes the AI model', async () => { + const relatedData: RecordData[] = [ + { collectionName: 'addresses', recordId: [1], values: { city: 'Paris' } }, + { collectionName: 'addresses', recordId: [2], values: { city: 'Lyon' } }, + ]; + const agentPort = makeMockAgentPort(relatedData); + const bindTools = jest.fn(); + const model = { bindTools } as unknown as ExecutionContext['model']; + const runStore = makeMockRunStore(); + const context = makeContext({ + model, + agentPort, + runStore, + workflowPort: makeMockWorkflowPort({ + customers: customersWithAddress, + addresses: addressSchema, + }), + stepDefinition: makeStep({ + executionType: StepExecutionMode.Manual, + preRecordedArgs: { relationName: 'address' }, + }), + }); + + const result = await new LoadRelatedRecordStepExecutor(context).execute(); + + expect(result.stepOutcome.status).toBe('awaiting-input'); + // Manual mode performs zero AI calls — no relation chooser, no field/record ranking. + expect(bindTools).not.toHaveBeenCalled(); + + const saved = (runStore.saveStepExecution as jest.Mock).mock.calls[0][1]; + expect(saved.pendingData.availableRecordIds).toEqual([cand([1]), cand([2])]); + expect(saved.pendingData.suggestedRecord).toBeUndefined(); + }); + + it('still pre-fills the single xToOne record in Manual (the only option, not an AI pick)', async () => { + // BelongsTo 'order' → one linked record; pre-filled even in Manual, with no AI call. + const agentPort = makeMockAgentPort(); // default: 1 related order #99 via getSingleRelatedData + const bindTools = jest.fn(); + const model = { bindTools } as unknown as ExecutionContext['model']; + const runStore = makeMockRunStore(); + const context = makeContext({ + model, + agentPort, + runStore, + stepDefinition: makeStep({ + executionType: StepExecutionMode.Manual, + preRecordedArgs: { relationName: 'order' }, + }), + }); + + const result = await new LoadRelatedRecordStepExecutor(context).execute(); + + expect(result.stepOutcome.status).toBe('awaiting-input'); + expect(bindTools).not.toHaveBeenCalled(); + + const saved = (runStore.saveStepExecution as jest.Mock).mock.calls[0][1]; + expect(saved.pendingData.availableRecordIds).toEqual([cand(['99'])]); + expect(saved.pendingData.suggestedRecord).toEqual(cand(['99'])); + }); + + it('switching relation (refreshCandidatesForField) in Manual makes no AI suggestion', async () => { + const execution = makePendingExecution({ + pendingData: { + availableFields: [ + { name: 'order', displayName: 'Order' }, + { name: 'address', displayName: 'Address' }, + ], + suggestedField: { name: 'order', displayName: 'Order' }, + availableRecordIds: [cand(['99'])], + suggestedRecord: cand(['99']), + }, + }); + const agentPort = makeMockAgentPort([ + { collectionName: 'addresses', recordId: [1], values: { city: 'Paris' } }, + { collectionName: 'addresses', recordId: [2], values: { city: 'Lyon' } }, + ]); + const addressesSchema = makeCollectionSchema({ + collectionName: 'addresses', + collectionDisplayName: 'Addresses', + fields: [{ fieldName: 'city', displayName: 'City', isRelationship: false }], + }); + const bindTools = jest.fn(); + const model = { bindTools } as unknown as ExecutionContext['model']; + const runStore = makeMockRunStore({ + getStepExecutions: jest.fn().mockResolvedValue([execution]), + }); + const context = makeContext({ + model, + agentPort, + runStore, + workflowPort: makeMockWorkflowPort({ + customers: makeCollectionSchema(), + addresses: addressesSchema, + }), + incomingPendingData: { fieldName: 'address' }, + stepDefinition: makeStep({ executionType: StepExecutionMode.Manual }), + }); + + const result = await new LoadRelatedRecordStepExecutor(context).execute(); + + expect(result.stepOutcome.status).toBe('awaiting-input'); + // Manual: switching to a multi-record relation must neither call the AI nor pre-select. + expect(bindTools).not.toHaveBeenCalled(); + + const finalSave = (runStore.saveStepExecution as jest.Mock).mock.calls.at(-1)?.[1]; + expect(finalSave.pendingData.availableRecordIds).toEqual([cand([1]), cand([2])]); + expect(finalSave.pendingData.suggestedRecord).toBeUndefined(); + }); + }); + + describe('executionType=FullyAutomated: AI judges none relevant → skip (PRD-148)', () => { + it('skips (no record loaded) when select-record-by-content returns -1', async () => { + const customersWithAddress = makeCollectionSchema({ + fields: [ + { + fieldName: 'address', + displayName: 'Address', + isRelationship: true, + relationType: 'HasMany', + relatedCollectionName: 'addresses', + }, + ], + }); + const addressSchema = makeCollectionSchema({ + collectionName: 'addresses', + collectionDisplayName: 'Addresses', + fields: [{ fieldName: 'city', displayName: 'City', isRelationship: false }], + }); + const relatedData: RecordData[] = [ + { collectionName: 'addresses', recordId: [1], values: { city: 'Paris' } }, + { collectionName: 'addresses', recordId: [2], values: { city: 'Lyon' } }, + ]; + const agentPort = makeMockAgentPort(relatedData); + const invoke = jest + .fn() + .mockResolvedValueOnce({ + tool_calls: [{ name: 'select-fields', args: { fieldNames: ['City'] }, id: 'c2' }], + }) + .mockResolvedValueOnce({ + tool_calls: [ + { + name: 'select-record-by-content', + args: { recordIndex: -1, reasoning: 'None of the candidates is relevant' }, + id: 'c3', + }, + ], + }); + const bindTools = jest.fn().mockReturnValue({ invoke }); + const model = { bindTools } as unknown as ExecutionContext['model']; + const runStore = makeMockRunStore(); + const context = makeContext({ + model, + agentPort, + runStore, + workflowPort: makeMockWorkflowPort({ + customers: customersWithAddress, + addresses: addressSchema, + }), + stepDefinition: makeStep({ + executionType: StepExecutionMode.FullyAutomated, + preRecordedArgs: { relationName: 'address' }, + }), + }); + + const result = await new LoadRelatedRecordStepExecutor(context).execute(); + + expect(result.stepOutcome.status).toBe('success'); + // Two AI calls run (field + record selection); the record call returns -1 → skip. + expect(bindTools).toHaveBeenCalledTimes(2); + expect(bindTools.mock.calls[1][0][0].name).toBe('select-record-by-content'); + expect(runStore.saveStepExecution).toHaveBeenCalledWith( + 'run-1', + expect.objectContaining({ + type: 'load-related-record', + executionResult: { skipped: true }, + }), ); - expect(runStore.saveStepExecution).not.toHaveBeenCalled(); }); }); @@ -2524,7 +2726,7 @@ describe('LoadRelatedRecordStepExecutor', () => { }); describe('RelatedRecordNotFoundError', () => { - it('returns error when BelongsTo getRelatedData returns empty array (Branch B)', async () => { + it('auto-skips when BelongsTo getRelatedData returns empty array (Branch B)', async () => { const agentPort = makeMockAgentPort([]); const mockModel = makeMockModel({ relation: relationOption({ @@ -2545,14 +2747,18 @@ describe('LoadRelatedRecordStepExecutor', () => { const result = await executor.execute(); - expect(result.stepOutcome.status).toBe('error'); - expect(result.stepOutcome.error).toBe( - 'The related record could not be found. It may have been deleted.', + // Full AI: linked record absent → skip (no error). + expect(result.stepOutcome.status).toBe('success'); + expect(runStore.saveStepExecution).toHaveBeenCalledWith( + 'run-1', + expect.objectContaining({ + type: 'load-related-record', + executionResult: { skipped: true }, + }), ); - expect(runStore.saveStepExecution).not.toHaveBeenCalled(); }); - it('returns error when HasMany getRelatedData returns empty array (Branch B)', async () => { + it('auto-skips when HasMany getRelatedData returns empty array (Branch B)', async () => { const hasManySchema = makeCollectionSchema({ fields: [ { @@ -2578,11 +2784,15 @@ describe('LoadRelatedRecordStepExecutor', () => { const result = await executor.execute(); - expect(result.stepOutcome.status).toBe('error'); - expect(result.stepOutcome.error).toBe( - 'The related record could not be found. It may have been deleted.', + // Full AI: empty candidate list → skip (no error). + expect(result.stepOutcome.status).toBe('success'); + expect(runStore.saveStepExecution).toHaveBeenCalledWith( + 'run-1', + expect.objectContaining({ + type: 'load-related-record', + executionResult: { skipped: true }, + }), ); - expect(runStore.saveStepExecution).not.toHaveBeenCalled(); }); }); @@ -3610,7 +3820,7 @@ describe('LoadRelatedRecordStepExecutor', () => { expect(result.stepOutcome.error).toBe('The pre-configured step parameters are invalid'); }); - it('surfaces a distinct "no source record" message when the source step loaded nothing', async () => { + it('auto-skips (Full AI) when the source step loaded nothing', async () => { // The source step exists in the live path but its run-store execution has no record. const runStore = makeMockRunStore({ getStepExecutions: jest.fn().mockResolvedValue([]) }); const context = makeContext({ @@ -3624,6 +3834,29 @@ describe('LoadRelatedRecordStepExecutor', () => { const result = await new LoadRelatedRecordStepExecutor(context).execute(); + // Full AI: no source record to load from → skip and let the run advance. + expect(result.stepOutcome.status).toBe('success'); + expect(runStore.saveStepExecution).toHaveBeenCalledWith( + 'run-1', + expect.objectContaining({ executionResult: { skipped: true } }), + ); + }); + + it('surfaces a "no source record" error in await modes when the source step loaded nothing', async () => { + // Only Full AI auto-skips; Manual / AI-assisted surface the error so the front can offer + // "continue without" (PRD-550). + const runStore = makeMockRunStore({ getStepExecutions: jest.fn().mockResolvedValue([]) }); + const context = makeContext({ + runStore, + previousSteps: [makeLoadRelatedPreviousStep(1)], + stepDefinition: makeStep({ + executionType: StepExecutionMode.AutomatedWithConfirmation, + preRecordedArgs: { selectedRecordStepId: 'load-1', relationName: 'customer' }, + }), + }); + + const result = await new LoadRelatedRecordStepExecutor(context).execute(); + expect(result.stepOutcome.status).toBe('error'); expect(result.stepOutcome.error).toContain("didn't load any record"); }); diff --git a/packages/workflow-executor/test/http/step-serializer.test.ts b/packages/workflow-executor/test/http/step-serializer.test.ts index 1ff16a3c6c..3dc5cc8021 100644 --- a/packages/workflow-executor/test/http/step-serializer.test.ts +++ b/packages/workflow-executor/test/http/step-serializer.test.ts @@ -108,6 +108,19 @@ describe('serializeStepForWire', () => { expect(result.executionResult).toEqual({ skipped: true }); }); + + it('does not throw when selectedRecordRef is absent (Full AI no-source skip)', () => { + const step: LoadRelatedRecordStepExecutionData = { + type: 'load-related-record', + stepIndex: 2, + executionResult: { skipped: true }, + }; + + const result = serializeStepForWire(step) as Record; + + expect(result.executionResult).toEqual({ skipped: true }); + expect('selectedRecordRef' in result).toBe(false); + }); }); it('returns non-record steps unchanged', () => { From ea0b5f83320aeb73c58f445a0f9c35a771144700 Mon Sep 17 00:00:00 2001 From: Matt Date: Tue, 30 Jun 2026 15:06:07 +0200 Subject: [PATCH 2/7] docs(workflow-executor): clarify the Manual source-record default in resolveTarget (PRD-148) Co-Authored-By: Claude Opus 4.8 (1M context) --- .../src/executors/load-related-record-step-executor.ts | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/packages/workflow-executor/src/executors/load-related-record-step-executor.ts b/packages/workflow-executor/src/executors/load-related-record-step-executor.ts index 60ea709e01..be91d73e94 100644 --- a/packages/workflow-executor/src/executors/load-related-record-step-executor.ts +++ b/packages/workflow-executor/src/executors/load-related-record-step-executor.ts @@ -177,8 +177,9 @@ export default class LoadRelatedRecordStepExecutor extends RecordStepExecutor 1 && useAi ? await this.selectRelationToFollow(eligible) : eligible[0]; From 15793400ce3ce894a736194576386012cb9a165b Mon Sep 17 00:00:00 2001 From: Matt Date: Tue, 30 Jun 2026 15:13:58 +0200 Subject: [PATCH 3/7] fix(workflow-executor): don't allow none-relevant skip on a truncated candidate list (PRD-148) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When the HasMany candidate list is truncated to fit the AI prompt budget, -1 (none relevant) is no longer offered — the AI picks from what it saw, so a match in the unseen tail is not silently skipped. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../executors/load-related-record-step-executor.ts | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/packages/workflow-executor/src/executors/load-related-record-step-executor.ts b/packages/workflow-executor/src/executors/load-related-record-step-executor.ts index be91d73e94..9b0e3b40e5 100644 --- a/packages/workflow-executor/src/executors/load-related-record-step-executor.ts +++ b/packages/workflow-executor/src/executors/load-related-record-step-executor.ts @@ -741,7 +741,9 @@ export default class LoadRelatedRecordStepExecutor extends RecordStepExecutor maxIndex) { throw new InvalidAIResponseError( `AI selected record index ${recordIndex} which is out of range (${minIndex}-${maxIndex}) or not an integer`, From 4f8b3981705cbf2015530c79ceca2c29731f7c46 Mon Sep 17 00:00:00 2001 From: Matt Date: Tue, 30 Jun 2026 15:39:24 +0200 Subject: [PATCH 4/7] style(workflow-executor): trim comments to the non-obvious why only (PRD-148) Co-Authored-By: Claude Opus 4.8 (1M context) --- .../load-related-record-step-executor.ts | 39 +++---------------- .../src/http/step-serializer.ts | 3 +- .../src/types/step-execution-data.ts | 3 +- .../src/types/validated/step-definition.ts | 4 +- 4 files changed, 8 insertions(+), 41 deletions(-) diff --git a/packages/workflow-executor/src/executors/load-related-record-step-executor.ts b/packages/workflow-executor/src/executors/load-related-record-step-executor.ts index 9b0e3b40e5..63ea394f17 100644 --- a/packages/workflow-executor/src/executors/load-related-record-step-executor.ts +++ b/packages/workflow-executor/src/executors/load-related-record-step-executor.ts @@ -108,7 +108,6 @@ export default class LoadRelatedRecordStepExecutor extends RecordStepExecutor { const { stepDefinition: step } = this.context; - // Manual mode never invokes AI — neither to pick the relation nor to suggest a record. const useAi = step.executionType !== StepExecutionMode.Manual; - // Branch B -- Full AI: AI selects and the record is loaded with no user input (may auto-skip). if (step.executionType === StepExecutionMode.FullyAutomated) { return this.resolveAndLoadAutomatic(useAi); } - // Branches C & D -- pre-fetch candidates, await user confirmation. AI-assisted pre-selects a - // record (suggestedRecord); Manual presents the narrowed list with no AI pick. const target = await this.resolveTarget(useAi); const sourceSchema = await this.getCollectionSchema(target.selectedRecordRef.collectionName); @@ -277,9 +272,7 @@ export default class LoadRelatedRecordStepExecutor extends RecordStepExecutor= 0 ? toCandidate(relatedData[bestIndex]) : undefined, }; } @@ -363,11 +352,6 @@ export default class LoadRelatedRecordStepExecutor extends RecordStepExecutor { let target: RelationTarget; @@ -382,7 +366,6 @@ export default class LoadRelatedRecordStepExecutor extends RecordStepExecutor { const { relatedData, bestIndex } = await this.selectBestFromRelatedData(target, 50, { rank: true, @@ -543,7 +520,6 @@ export default class LoadRelatedRecordStepExecutor extends RecordStepExecutor { const relatedSchema = await this.getCollectionSchema(target.relatedCollectionName); const relatedData = await this.fetchRelatedData(target, relatedSchema, 1); @@ -586,9 +562,8 @@ export default class LoadRelatedRecordStepExecutor extends RecordStepExecutor { await this.context.runStore.saveStepExecution(this.context.runId, { type: 'load-related-record', @@ -706,10 +681,6 @@ export default class LoadRelatedRecordStepExecutor extends RecordStepExecutor nonRelationFields.find(f => f.displayName === dn)?.fieldName ?? dn); } - /** - * AI call 2 for HasMany: selects the best record by index from the candidate list. When - * `allowNone` (Full AI), the AI may return -1 to signal that none of the candidates is relevant. - */ private async selectBestRecordIndex( candidates: RecordData[], fieldNames: string[], diff --git a/packages/workflow-executor/src/http/step-serializer.ts b/packages/workflow-executor/src/http/step-serializer.ts index f5c057c8fb..198fcff7e3 100644 --- a/packages/workflow-executor/src/http/step-serializer.ts +++ b/packages/workflow-executor/src/http/step-serializer.ts @@ -15,8 +15,7 @@ export default function serializeStepForWire(step: StepExecutionData): unknown { return { ...step, selectedRecordRef: serializeRecordRef(step.selectedRecordRef) }; case 'load-related-record': { - // selectedRecordRef is absent on a Full AI no-source auto-skip — omit it rather than - // dereferencing undefined (which would throw and 503 the run-fetch). + // Omit selectedRecordRef when absent (Full AI no-source skip) — serializing undefined throws. const result: Record = { ...step, ...(step.selectedRecordRef && { diff --git a/packages/workflow-executor/src/types/step-execution-data.ts b/packages/workflow-executor/src/types/step-execution-data.ts index 59448737ee..626c435e28 100644 --- a/packages/workflow-executor/src/types/step-execution-data.ts +++ b/packages/workflow-executor/src/types/step-execution-data.ts @@ -183,8 +183,7 @@ export interface LoadRelatedRecordStepExecutionData WithUserConfirmation { type: 'load-related-record'; pendingData?: LoadRelatedRecordPendingData; - // Absent only for the Full AI auto-skip when there is no source record to load from - // (nothing was resolved). Always set on the await/load paths. + // Absent only on the Full AI no-source auto-skip; always set on the await/load paths. selectedRecordRef?: RecordRef; executionParams?: RelationRef; executionResult?: { relation: RelationRef; record: RecordRef } | { skipped: true }; diff --git a/packages/workflow-executor/src/types/validated/step-definition.ts b/packages/workflow-executor/src/types/validated/step-definition.ts index 089f9856da..a2c6cda898 100644 --- a/packages/workflow-executor/src/types/validated/step-definition.ts +++ b/packages/workflow-executor/src/types/validated/step-definition.ts @@ -98,9 +98,7 @@ export type TriggerActionStepDefinition = z.infer Date: Wed, 1 Jul 2026 17:36:37 +0200 Subject: [PATCH 5/7] feat(workflow-executor): degrade Full AI to a confirmation and let the AI decline (PRD-148) Full AI no longer silently skips when it can't load a relevant record. With nothing relevant it degrades to an AI-assisted confirmation (a human decides, "No X to load" pre-checked); no source record now surfaces an error like the await modes instead of skipping. AI-assisted may decline too (allowNone), and the record-selection prompt tells the AI to return -1 on an impossible request rather than forcing a weak match. pendingData carries a suggestNoRecord flag so the front distinguishes an active "nothing relevant" from a plain no-suggestion (Manual). Co-Authored-By: Claude Opus 4.8 (1M context) --- .../load-related-record-step-executor.ts | 130 ++++------ .../src/http/step-serializer.ts | 2 +- .../src/types/step-execution-data.ts | 5 +- .../load-related-record-step-executor.test.ts | 223 +++++++++++++----- 4 files changed, 222 insertions(+), 138 deletions(-) diff --git a/packages/workflow-executor/src/executors/load-related-record-step-executor.ts b/packages/workflow-executor/src/executors/load-related-record-step-executor.ts index 63ea394f17..6f7b9bc124 100644 --- a/packages/workflow-executor/src/executors/load-related-record-step-executor.ts +++ b/packages/workflow-executor/src/executors/load-related-record-step-executor.ts @@ -21,7 +21,6 @@ import { NoRelationshipFieldsError, RelatedRecordNotFoundError, RelationNotFoundError, - SourceRecordMissingError, StepStateError, } from '../errors'; import RecordStepExecutor from './record-step-executor'; @@ -43,6 +42,10 @@ Choose the fields that are most useful for determining which record best matches const SELECT_RECORD_SYSTEM_PROMPT = `You are an AI agent selecting the most relevant related record from a list of candidates. Choose the record that best matches the user request based on the provided field values.`; +// Only appended when -1 is a valid answer. The base prompt tells the AI to always pick, so without +// this it forces a weak match on an impossible request instead of declining. +const SELECT_RECORD_NONE_ALLOWED_PROMPT = `If the request states a specific requirement that NO candidate satisfies, return -1 instead of forcing an arbitrary or loosely-related match. Do NOT return -1 merely because a match is imperfect — only when no candidate genuinely fits the request.`; + // Bound only what is sent to the AI in selectBestRecordIndex — the full candidate list is // always returned to the front via availableRecordIds. These cap the prompt size, not the data. const MAX_RELEVANT_FIELDS = 6; @@ -135,7 +138,7 @@ export default class LoadRelatedRecordStepExecutor extends RecordStepExecutor { - const { selectedRecordRef, name, displayName } = target; - const { availableRecordIds, suggestedRecord } = await this.collectCandidateIds( target, suggestViaAi, ); + return this.persistAwaitInput(target, sourceSchema, { + availableRecordIds, + suggestedRecord, + // An AI pass that yields no record is a deliberate "nothing relevant" → pre-check "No X to load". + // A Manual pass with no suggestion just means the user picks, so no pre-check. + suggestNoRecord: suggestViaAi && !suggestedRecord, + }); + } + + private async persistAwaitInput( + target: RelationTarget, + sourceSchema: CollectionSchema, + pending: { + availableRecordIds: LoadRelatedRecordCandidate[]; + suggestedRecord?: LoadRelatedRecordCandidate; + suggestNoRecord: boolean; + }, + ): Promise { + const { selectedRecordRef, name, displayName } = target; + const availableFields: RelationRef[] = sourceSchema.fields .filter(isFollowableRelation) .map(f => ({ name: f.fieldName, displayName: f.displayName })); @@ -295,8 +316,9 @@ export default class LoadRelatedRecordStepExecutor extends RecordStepExecutor { - let target: RelationTarget; - - try { - target = await this.resolveTarget(useAi); - } catch (error) { - // No source record to load from → Full AI auto-continues. (Manual/AI-assisted let this error - // surface so the front can offer "continue without" — PRD-550.) - if (error instanceof SourceRecordMissingError) return this.persistSkip(); - throw error; - } + private async resolveAndLoadAutomatic(): Promise { + // No source record throws (like Manual/AI-assisted) → the front offers "continue without" (PRD-550). + const target = await this.resolveTarget(true); + const { availableRecordIds, suggestedRecord } = await this.collectCandidateIds(target, true); - const record = await this.fetchRecordForRelation(target); + // Full AI auto-loads a confident pick and advances. With nothing relevant to load it degrades to + // an AI-assisted confirmation (a human decides, "No X to load" pre-checked) instead of skipping — + // mirrors the Trigger Action automated→confirmation fallback. + if (!suggestedRecord) { + const sourceSchema = await this.getCollectionSchema(target.selectedRecordRef.collectionName); - if (record === null) return this.persistSkip(target.selectedRecordRef); - - return this.persistAndReturn(record, target, undefined); - } - - private async fetchRecordForRelation(target: RelationTarget): Promise { - if (target.relationType === 'BelongsTo' || target.relationType === 'HasOne') { - return this.fetchXToOneRecordRef(target); - } - - if (target.relationType === 'HasMany') { - return this.selectBestRelatedRecord(target); + return this.persistAwaitInput(target, sourceSchema, { + availableRecordIds, + suggestNoRecord: true, + }); } - return this.fetchFirstCandidate(target); - } - - // Returns null (→ Full AI skip) when the relation has no linked record. - private async fetchXToOneRecordRef(target: RelationTarget): Promise { - const candidate = await this.fetchXToOneCandidate(target); - - if (!candidate) return null; - - return { + const record: RecordRef = { collectionName: target.relatedCollectionName, - recordId: candidate.recordId, + recordId: suggestedRecord.recordId, stepIndex: this.context.stepIndex, }; + + return this.persistAndReturn(record, target, undefined); } private async fetchXToOneCandidate( @@ -509,24 +515,6 @@ export default class LoadRelatedRecordStepExecutor extends RecordStepExecutor { - const { relatedData, bestIndex } = await this.selectBestFromRelatedData(target, 50, { - rank: true, - allowNone: true, - }); - - if (relatedData.length === 0 || bestIndex < 0) return null; - - return this.toRecordRef(relatedData[bestIndex]); - } - - private async fetchFirstCandidate(target: RelationTarget): Promise { - const relatedSchema = await this.getCollectionSchema(target.relatedCollectionName); - const relatedData = await this.fetchRelatedData(target, relatedSchema, 1); - - return relatedData.length > 0 ? this.toRecordRef(relatedData[0]) : null; - } - private async fetchRelatedData( target: Pick, relatedSchema: CollectionSchema, @@ -562,19 +550,6 @@ export default class LoadRelatedRecordStepExecutor extends RecordStepExecutor { - await this.context.runStore.saveStepExecution(this.context.runId, { - type: 'load-related-record', - stepIndex: this.context.stepIndex, - selectedRecordRef, - executionResult: { skipped: true }, - }); - - return this.buildOutcomeResult({ status: 'success' }); - } - private relationOptionLabel(candidate: RelationCandidate): string { const { record, schema, field } = candidate; @@ -749,6 +724,7 @@ export default class LoadRelatedRecordStepExecutor extends RecordStepExecutor = { ...step, ...(step.selectedRecordRef && { diff --git a/packages/workflow-executor/src/types/step-execution-data.ts b/packages/workflow-executor/src/types/step-execution-data.ts index 626c435e28..537a0ae9a3 100644 --- a/packages/workflow-executor/src/types/step-execution-data.ts +++ b/packages/workflow-executor/src/types/step-execution-data.ts @@ -176,6 +176,9 @@ export interface LoadRelatedRecordPendingData { availableRecordIds: LoadRelatedRecordCandidate[]; // Absent when the relation has no linked record(s): the list is empty and there's nothing to suggest. suggestedRecord?: LoadRelatedRecordCandidate; + // The AI actively judged no candidate relevant (incl. Full AI degrading to confirmation) → the front + // pre-checks "No X to load". Distinct from a plain absent suggestedRecord (Manual: the user picks). + suggestNoRecord?: boolean; } export interface LoadRelatedRecordStepExecutionData @@ -183,7 +186,7 @@ export interface LoadRelatedRecordStepExecutionData WithUserConfirmation { type: 'load-related-record'; pendingData?: LoadRelatedRecordPendingData; - // Absent only on the Full AI no-source auto-skip; always set on the await/load paths. + // Set on every await/load path (and preserved through the user-initiated "continue without" skip). selectedRecordRef?: RecordRef; executionParams?: RelationRef; executionResult?: { relation: RelationRef; record: RecordRef } | { skipped: true }; diff --git a/packages/workflow-executor/test/executors/load-related-record-step-executor.test.ts b/packages/workflow-executor/test/executors/load-related-record-step-executor.test.ts index dd6c8d82c1..4f7b5145c0 100644 --- a/packages/workflow-executor/test/executors/load-related-record-step-executor.test.ts +++ b/packages/workflow-executor/test/executors/load-related-record-step-executor.test.ts @@ -975,13 +975,14 @@ describe('LoadRelatedRecordStepExecutor', () => { const result = await executor.execute(); expect(result.stepOutcome.status).toBe('success'); - // To-many path: /relationships call with limit: 1, no parent-record projection. + // To-many path: /relationships call with the candidate-list limit (a lone record short-circuits + // ranking, so it loads without an AI call). expect(agentPort.getRelatedData).toHaveBeenCalledWith( expect.objectContaining({ collection: 'customers', id: [42], relation: 'tags', - limit: 1, + limit: 50, relatedSchema: expect.objectContaining({ collectionName: 'tags' }), }), expect.objectContaining({ id: 1 }), @@ -997,9 +998,9 @@ describe('LoadRelatedRecordStepExecutor', () => { ); }); - // fetchCandidates throws RelatedRecordNotFoundError when the agent returns an - // empty list. Same user-facing message as the other empty-result paths. - it('auto-skips when getRelatedData returns an empty array (Branch B)', async () => { + // Empty list → nothing to load: Full AI degrades to an AI-assisted confirmation ("No X to load" + // pre-checked) instead of skipping, so a human decides. + it('falls back to awaiting-input (No X to load) when getRelatedData returns an empty array (Branch B)', async () => { const belongsToManySchema = makeCollectionSchema({ fields: [ { @@ -1025,15 +1026,12 @@ describe('LoadRelatedRecordStepExecutor', () => { const result = await executor.execute(); - // Full AI: no candidate to load → skip so the run can advance (no error). - expect(result.stepOutcome.status).toBe('success'); - expect(runStore.saveStepExecution).toHaveBeenCalledWith( - 'run-1', - expect.objectContaining({ - type: 'load-related-record', - executionResult: { skipped: true }, - }), - ); + // Full AI with no candidate → hands off to the user with "No X to load" pre-checked. + expect(result.stepOutcome.status).toBe('awaiting-input'); + const saved = (runStore.saveStepExecution as jest.Mock).mock.calls.at(-1)?.[1]; + expect(saved.pendingData.availableRecordIds).toEqual([]); + expect(saved.pendingData.suggestedRecord).toBeUndefined(); + expect(saved.pendingData.suggestNoRecord).toBe(true); }); }); @@ -1165,8 +1163,135 @@ describe('LoadRelatedRecordStepExecutor', () => { }); }); - describe('executionType=FullyAutomated: AI judges none relevant → skip (PRD-148)', () => { - it('skips (no record loaded) when select-record-by-content returns -1', async () => { + describe('executionType=AutomatedWithConfirmation: AI may decline (PRD-148)', () => { + it('pre-checks "No X to load" (suggestNoRecord) when the AI judges no candidate relevant', async () => { + const customersWithAddress = makeCollectionSchema({ + fields: [ + { + fieldName: 'address', + displayName: 'Address', + isRelationship: true, + relationType: 'HasMany', + relatedCollectionName: 'addresses', + }, + ], + }); + const addressSchema = makeCollectionSchema({ + collectionName: 'addresses', + collectionDisplayName: 'Addresses', + fields: [{ fieldName: 'city', displayName: 'City', isRelationship: false }], + }); + const relatedData: RecordData[] = [ + { collectionName: 'addresses', recordId: [1], values: { city: 'Paris' } }, + { collectionName: 'addresses', recordId: [2], values: { city: 'Lyon' } }, + ]; + const agentPort = makeMockAgentPort(relatedData); + const invoke = jest + .fn() + .mockResolvedValueOnce({ + tool_calls: [{ name: 'select-fields', args: { fieldNames: ['City'] }, id: 'c2' }], + }) + .mockResolvedValueOnce({ + tool_calls: [ + { + name: 'select-record-by-content', + args: { recordIndex: -1, reasoning: 'None of the candidates is relevant' }, + id: 'c3', + }, + ], + }); + const bindTools = jest.fn().mockReturnValue({ invoke }); + const model = { bindTools } as unknown as ExecutionContext['model']; + const runStore = makeMockRunStore(); + const context = makeContext({ + model, + agentPort, + runStore, + workflowPort: makeMockWorkflowPort({ + customers: customersWithAddress, + addresses: addressSchema, + }), + stepDefinition: makeStep({ + executionType: StepExecutionMode.AutomatedWithConfirmation, + preRecordedArgs: { relationName: 'address' }, + }), + }); + + const result = await new LoadRelatedRecordStepExecutor(context).execute(); + + // AI-assisted no longer forces a pick: the AI may return -1, and the user reviews with + // "No X to load" pre-checked (rather than a potentially misleading forced suggestion). + expect(result.stepOutcome.status).toBe('awaiting-input'); + const saved = (runStore.saveStepExecution as jest.Mock).mock.calls.at(-1)?.[1]; + expect(saved.pendingData.availableRecordIds).toEqual([cand([1]), cand([2])]); + expect(saved.pendingData.suggestedRecord).toBeUndefined(); + expect(saved.pendingData.suggestNoRecord).toBe(true); + }); + + it('tells the AI it may return -1 in the record-selection prompt (Bug 2)', async () => { + const customersWithAddress = makeCollectionSchema({ + fields: [ + { + fieldName: 'address', + displayName: 'Address', + isRelationship: true, + relationType: 'HasMany', + relatedCollectionName: 'addresses', + }, + ], + }); + const addressSchema = makeCollectionSchema({ + collectionName: 'addresses', + collectionDisplayName: 'Addresses', + fields: [{ fieldName: 'city', displayName: 'City', isRelationship: false }], + }); + const relatedData: RecordData[] = [ + { collectionName: 'addresses', recordId: [1], values: { city: 'Paris' } }, + { collectionName: 'addresses', recordId: [2], values: { city: 'Lyon' } }, + ]; + const agentPort = makeMockAgentPort(relatedData); + const invoke = jest + .fn() + .mockResolvedValueOnce({ + tool_calls: [{ name: 'select-fields', args: { fieldNames: ['City'] }, id: 'c2' }], + }) + .mockResolvedValueOnce({ + tool_calls: [ + { + name: 'select-record-by-content', + args: { recordIndex: 0, reasoning: 'match' }, + id: 'c3', + }, + ], + }); + const bindTools = jest.fn().mockReturnValue({ invoke }); + const model = { bindTools } as unknown as ExecutionContext['model']; + const context = makeContext({ + model, + agentPort, + runStore: makeMockRunStore(), + workflowPort: makeMockWorkflowPort({ + customers: customersWithAddress, + addresses: addressSchema, + }), + stepDefinition: makeStep({ + executionType: StepExecutionMode.AutomatedWithConfirmation, + preRecordedArgs: { relationName: 'address' }, + }), + }); + + await new LoadRelatedRecordStepExecutor(context).execute(); + + // 2nd AI call = record selection. Without the decline guidance the base prompt forces a pick, + // so an impossible request would return a weak match instead of -1 (Bug 2). + const recordSelectionMessages = invoke.mock.calls[1][0] as Array<{ content: unknown }>; + const content = recordSelectionMessages.map(m => String(m.content)).join('\n'); + expect(content).toContain('-1'); + }); + }); + + describe('executionType=FullyAutomated: AI judges none relevant → confirmation (PRD-148)', () => { + it('falls back to awaiting-input (No X to load) when select-record-by-content returns -1', async () => { const customersWithAddress = makeCollectionSchema({ fields: [ { @@ -1221,17 +1346,15 @@ describe('LoadRelatedRecordStepExecutor', () => { const result = await new LoadRelatedRecordStepExecutor(context).execute(); - expect(result.stepOutcome.status).toBe('success'); - // Two AI calls run (field + record selection); the record call returns -1 → skip. + expect(result.stepOutcome.status).toBe('awaiting-input'); + // Two AI calls run (field + record selection); the record call returns -1 → hand off to the + // user with the candidates listed and "No X to load" pre-checked. expect(bindTools).toHaveBeenCalledTimes(2); expect(bindTools.mock.calls[1][0][0].name).toBe('select-record-by-content'); - expect(runStore.saveStepExecution).toHaveBeenCalledWith( - 'run-1', - expect.objectContaining({ - type: 'load-related-record', - executionResult: { skipped: true }, - }), - ); + const saved = (runStore.saveStepExecution as jest.Mock).mock.calls.at(-1)?.[1]; + expect(saved.pendingData.availableRecordIds).toEqual([cand([1]), cand([2])]); + expect(saved.pendingData.suggestedRecord).toBeUndefined(); + expect(saved.pendingData.suggestNoRecord).toBe(true); }); }); @@ -2725,8 +2848,8 @@ describe('LoadRelatedRecordStepExecutor', () => { }); }); - describe('RelatedRecordNotFoundError', () => { - it('auto-skips when BelongsTo getRelatedData returns empty array (Branch B)', async () => { + describe('empty candidate list → awaiting-input (No X to load)', () => { + it('falls back to awaiting-input when BelongsTo has no linked record (Branch B)', async () => { const agentPort = makeMockAgentPort([]); const mockModel = makeMockModel({ relation: relationOption({ @@ -2747,18 +2870,14 @@ describe('LoadRelatedRecordStepExecutor', () => { const result = await executor.execute(); - // Full AI: linked record absent → skip (no error). - expect(result.stepOutcome.status).toBe('success'); - expect(runStore.saveStepExecution).toHaveBeenCalledWith( - 'run-1', - expect.objectContaining({ - type: 'load-related-record', - executionResult: { skipped: true }, - }), - ); + // Full AI: linked record absent → hand off to the user with "No X to load" pre-checked. + expect(result.stepOutcome.status).toBe('awaiting-input'); + const saved = (runStore.saveStepExecution as jest.Mock).mock.calls.at(-1)?.[1]; + expect(saved.pendingData.availableRecordIds).toEqual([]); + expect(saved.pendingData.suggestNoRecord).toBe(true); }); - it('auto-skips when HasMany getRelatedData returns empty array (Branch B)', async () => { + it('falls back to awaiting-input when HasMany returns an empty array (Branch B)', async () => { const hasManySchema = makeCollectionSchema({ fields: [ { @@ -2784,15 +2903,11 @@ describe('LoadRelatedRecordStepExecutor', () => { const result = await executor.execute(); - // Full AI: empty candidate list → skip (no error). - expect(result.stepOutcome.status).toBe('success'); - expect(runStore.saveStepExecution).toHaveBeenCalledWith( - 'run-1', - expect.objectContaining({ - type: 'load-related-record', - executionResult: { skipped: true }, - }), - ); + // Full AI: empty candidate list → hand off to the user with "No X to load" pre-checked. + expect(result.stepOutcome.status).toBe('awaiting-input'); + const saved = (runStore.saveStepExecution as jest.Mock).mock.calls.at(-1)?.[1]; + expect(saved.pendingData.availableRecordIds).toEqual([]); + expect(saved.pendingData.suggestNoRecord).toBe(true); }); }); @@ -3820,8 +3935,10 @@ describe('LoadRelatedRecordStepExecutor', () => { expect(result.stepOutcome.error).toBe('The pre-configured step parameters are invalid'); }); - it('auto-skips (Full AI) when the source step loaded nothing', async () => { - // The source step exists in the live path but its run-store execution has no record. + it('errors (Full AI) when the source step loaded nothing — no longer auto-skips', async () => { + // The source step exists in the live path but its run-store execution has no record. Full AI + // now surfaces the error like the await modes (no silent skip); the front offers "continue + // without" (PRD-550). const runStore = makeMockRunStore({ getStepExecutions: jest.fn().mockResolvedValue([]) }); const context = makeContext({ runStore, @@ -3834,16 +3951,12 @@ describe('LoadRelatedRecordStepExecutor', () => { const result = await new LoadRelatedRecordStepExecutor(context).execute(); - // Full AI: no source record to load from → skip and let the run advance. - expect(result.stepOutcome.status).toBe('success'); - expect(runStore.saveStepExecution).toHaveBeenCalledWith( - 'run-1', - expect.objectContaining({ executionResult: { skipped: true } }), - ); + expect(result.stepOutcome.status).toBe('error'); + expect(result.stepOutcome.error).toContain("didn't load any record"); }); it('surfaces a "no source record" error in await modes when the source step loaded nothing', async () => { - // Only Full AI auto-skips; Manual / AI-assisted surface the error so the front can offer + // Same as Full AI now — every mode surfaces the error so the front can offer // "continue without" (PRD-550). const runStore = makeMockRunStore({ getStepExecutions: jest.fn().mockResolvedValue([]) }); const context = makeContext({ From 2b2251d03ea345bd1a0ea8243bcaae82b91ae35d Mon Sep 17 00:00:00 2001 From: Brian Fox Date: Thu, 2 Jul 2026 15:33:38 +0000 Subject: [PATCH 6/7] test(workflow-executor): cover load-related executionType schema parsing MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Pin LoadRelatedRecordStepDefinitionSchema behaviour: each valid mode parses to its own value (incl. 'manual' → 'manual'), a missing mode defaults to AutomatedWithConfirmation, and an invalid mode is rejected rather than silently coerced (no `.catch`, so a `manual` typo can't flip AI prefill back on). Co-Authored-By: Claude Opus 4.8 --- .../test/types/step-definition.test.ts | 43 +++++++++++++++++++ 1 file changed, 43 insertions(+) create mode 100644 packages/workflow-executor/test/types/step-definition.test.ts diff --git a/packages/workflow-executor/test/types/step-definition.test.ts b/packages/workflow-executor/test/types/step-definition.test.ts new file mode 100644 index 0000000000..04fdd1c6aa --- /dev/null +++ b/packages/workflow-executor/test/types/step-definition.test.ts @@ -0,0 +1,43 @@ +import { + LoadRelatedRecordStepDefinitionSchema, + StepExecutionMode, + StepType, +} from '../../src/types/validated/step-definition'; + +describe('LoadRelatedRecordStepDefinitionSchema executionType', () => { + const base = { type: StepType.LoadRelatedRecord as const }; + + it('parses each valid execution mode to its own value', () => { + expect( + LoadRelatedRecordStepDefinitionSchema.parse({ ...base, executionType: 'manual' }) + .executionType, + ).toBe(StepExecutionMode.Manual); + expect( + LoadRelatedRecordStepDefinitionSchema.parse({ + ...base, + executionType: 'automated-with-confirmation', + }).executionType, + ).toBe(StepExecutionMode.AutomatedWithConfirmation); + expect( + LoadRelatedRecordStepDefinitionSchema.parse({ ...base, executionType: 'fully-automated' }) + .executionType, + ).toBe(StepExecutionMode.FullyAutomated); + }); + + it('defaults a missing executionType to AutomatedWithConfirmation', () => { + expect(LoadRelatedRecordStepDefinitionSchema.parse(base).executionType).toBe( + StepExecutionMode.AutomatedWithConfirmation, + ); + }); + + // No `.catch` on the enum: an invalid value must be rejected, not silently coerced to + // AutomatedWithConfirmation (which would turn AI prefill back on for a `manual` typo). + it('rejects an invalid executionType instead of coercing it', () => { + const result = LoadRelatedRecordStepDefinitionSchema.safeParse({ + ...base, + executionType: 'not-a-mode', + }); + + expect(result.success).toBe(false); + }); +}); From f90b8681c628345305435b4868bd8d5aec26692c Mon Sep 17 00:00:00 2001 From: Brian Fox Date: Thu, 2 Jul 2026 15:33:51 +0000 Subject: [PATCH 7/7] feat(workflow-executor): degrade Full AI on low-confidence match and AI failure MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Load Related Record 3-way execution mode hardening. Bundled because the changes are interwoven in the executor and its test: - feat: Full AI now degrades to an AI-assisted confirmation when the AI returns a best guess but cannot confidently single one out among several viable candidates. selectBestRecordIndex gains a `confident` flag (distinct from the existing -1 "none relevant" sentinel, whose semantics are unchanged); an ambiguous positive pick pre-selects the best guess as suggestedRecord for a human to confirm instead of auto-loading. Confident picks still auto-load; a missing flag defaults to confident (backward compatible). - fix: AI failure/timeout in any AI mode (relation pick, field/record ranking, or a Full AI auto-load) now degrades to the deterministic Manual path — deterministic candidate list presented for the user to pick — instead of erroring the run or auto-skipping. AI calls are tagged via an internal marker so genuine config/data errors still surface. - fix: refreshCandidatesForField rebuilds pendingData from scratch on a field switch (keeping only the immutable relation list) so no stale suggestion state (suggestedRecord / suggestNoRecord) lingers. - style: drop ticket IDs from comments and test titles. - test: pin the truncated-list none-guard (noneAllowed = allowNone && !truncated — the AI is not offered -1 when the list was truncated). - tighten selectBestFromRelatedData options so allowNone only applies when rank is true. Co-Authored-By: Claude Opus 4.8 --- .../load-related-record-step-executor.ts | 196 +++++++--- .../load-related-record-step-executor.test.ts | 350 ++++++++++++++++-- 2 files changed, 459 insertions(+), 87 deletions(-) diff --git a/packages/workflow-executor/src/executors/load-related-record-step-executor.ts b/packages/workflow-executor/src/executors/load-related-record-step-executor.ts index 6f7b9bc124..c011f4e67d 100644 --- a/packages/workflow-executor/src/executors/load-related-record-step-executor.ts +++ b/packages/workflow-executor/src/executors/load-related-record-step-executor.ts @@ -1,3 +1,4 @@ +/* eslint-disable max-classes-per-file */ import type { StepExecutionResult } from '../types/execution-context'; import type { LoadRelatedRecordCandidate, @@ -22,6 +23,7 @@ import { RelatedRecordNotFoundError, RelationNotFoundError, StepStateError, + extractErrorMessage, } from '../errors'; import RecordStepExecutor from './record-step-executor'; import { StepExecutionMode } from '../types/validated/step-definition'; @@ -80,6 +82,19 @@ function isFollowableRelation(field: FieldSchema): boolean { return field.isRelationship && Boolean(field.relatedCollectionName || field.polymorphicTypeField); } +// Internal marker: wraps any failure raised while calling the AI (timeout, provider outage, +// missing/malformed tool call, out-of-range choice) so the step handlers can degrade to the +// deterministic Manual path instead of erroring the run. Never leaves this module. +class AiAssistUnavailableError extends Error { + readonly reason: unknown; + + constructor(reason: unknown) { + super('AI assistance unavailable'); + this.name = 'AiAssistUnavailableError'; + this.reason = reason; + } +} + export default class LoadRelatedRecordStepExecutor extends RecordStepExecutor { protected async doExecute(): Promise { // Branch A -- Re-entry after pending execution found in RunStore @@ -111,22 +126,37 @@ export default class LoadRelatedRecordStepExecutor extends RecordStepExecutor { + const target = await this.resolveTarget(false); const sourceSchema = await this.getCollectionSchema(target.selectedRecordRef.collectionName); - return this.saveAndAwaitInput(target, sourceSchema, useAi); + return this.saveAndAwaitInput(target, sourceSchema, false); } // Picks the (record, relation) pair to follow. Unlike a separate record-then-relation choice, @@ -179,7 +238,9 @@ export default class LoadRelatedRecordStepExecutor extends RecordStepExecutor 1 && useAi ? await this.selectRelationToFollow(eligible) : eligible[0]; + eligible.length > 1 && useAi + ? await this.withAiAssist(() => this.selectRelationToFollow(eligible)) + : eligible[0]; return this.targetFromCandidate(chosen); } @@ -332,25 +393,29 @@ export default class LoadRelatedRecordStepExecutor extends RecordStepExecutor { if (target.relationType === 'BelongsTo' || target.relationType === 'HasOne') { const candidate = await this.fetchXToOneCandidate(target); return candidate - ? { availableRecordIds: [candidate], suggestedRecord: candidate } - : { availableRecordIds: [] }; + ? { availableRecordIds: [candidate], suggestedRecord: candidate, ambiguous: false } + : { availableRecordIds: [], ambiguous: false }; } - const { relatedData, bestIndex, relatedSchema } = await this.selectBestFromRelatedData( - target, - 50, - // allowNone: the AI may judge no candidate relevant (→ "No X to load"); only meaningful when - // ranking (Manual passes rank=false and never calls the AI). - { rank: suggestViaAi, allowNone: true }, - ); + const { relatedData, bestIndex, confident, relatedSchema } = + await this.selectBestFromRelatedData( + target, + 50, + // allowNone: the AI may judge no candidate relevant (→ "No X to load"); only meaningful when + // ranking (Manual passes rank=false and never calls the AI). + suggestViaAi ? { rank: true, allowNone: true } : { rank: false }, + ); if (relatedData.length === 0) { - return { availableRecordIds: [] }; + return { availableRecordIds: [], ambiguous: false }; } const referenceField = relatedSchema.referenceField ?? null; @@ -364,6 +429,8 @@ export default class LoadRelatedRecordStepExecutor extends RecordStepExecutor= 0 ? toCandidate(relatedData[bestIndex]) : undefined, + // -1 (none relevant) is not ambiguity; only a low-confidence positive pick is. + ambiguous: bestIndex >= 0 && !confident, }; } @@ -377,19 +444,26 @@ export default class LoadRelatedRecordStepExecutor extends RecordStepExecutor { - // No source record throws (like Manual/AI-assisted) → the front offers "continue without" (PRD-550). + // No source record throws (like Manual/AI-assisted) → the front offers "continue without". const target = await this.resolveTarget(true); - const { availableRecordIds, suggestedRecord } = await this.collectCandidateIds(target, true); + const { availableRecordIds, suggestedRecord, ambiguous } = await this.collectCandidateIds( + target, + true, + ); - // Full AI auto-loads a confident pick and advances. With nothing relevant to load it degrades to - // an AI-assisted confirmation (a human decides, "No X to load" pre-checked) instead of skipping — - // mirrors the Trigger Action automated→confirmation fallback. - if (!suggestedRecord) { + // Full AI auto-loads a confident pick and advances. It degrades to an AI-assisted confirmation + // (a human decides) instead of auto-loading when either: + // - nothing is relevant to load → "No X to load" pre-checked (suggestNoRecord); or + // - the AI cannot confidently single out one of several viable candidates → its best guess is + // pre-selected for the human to confirm (no "No X to load"). + // Mirrors the Trigger Action automated→confirmation fallback. + if (!suggestedRecord || ambiguous) { const sourceSchema = await this.getCollectionSchema(target.selectedRecordRef.collectionName); return this.persistAwaitInput(target, sourceSchema, { availableRecordIds, - suggestNoRecord: true, + suggestedRecord, + suggestNoRecord: !suggestedRecord, }); } @@ -479,10 +553,15 @@ export default class LoadRelatedRecordStepExecutor extends RecordStepExecutor, limit: number, - opts: { rank: boolean; allowNone: boolean } = { rank: true, allowNone: false }, + // allowNone is meaningful only when ranking, so it's absent from the rank:false variant. + opts: { rank: true; allowNone: boolean } | { rank: false } = { rank: true, allowNone: false }, ): Promise<{ relatedData: RecordData[]; bestIndex: number; + // Whether the suggested record is a confident pick. The deterministic/short-circuit paths + // (single candidate, no ranking) are confident by construction; the ranked path reflects the + // AI's own confidence flag. + confident: boolean; suggestedFields: string[]; relatedSchema: CollectionSchema; }> { @@ -492,27 +571,40 @@ export default class LoadRelatedRecordStepExecutor extends RecordStepExecutor + this.selectRelevantFields(relatedSchema, this.context.stepDefinition.prompt), ); - const bestIndex = await this.selectBestRecordIndex( - relatedData, - suggestedFields, - this.context.stepDefinition.prompt, - opts.allowNone, + const { index: bestIndex, confident } = await this.withAiAssist(() => + this.selectBestRecordIndex( + relatedData, + suggestedFields, + this.context.stepDefinition.prompt, + opts.allowNone, + ), ); - return { relatedData, bestIndex, suggestedFields, relatedSchema }; + return { relatedData, bestIndex, confident, suggestedFields, relatedSchema }; + } + + // Tags any failure raised inside an AI call so the step handlers can degrade to the deterministic + // Manual path. Passing through an already-tagged error avoids double-wrapping when calls nest. + private async withAiAssist(call: () => Promise): Promise { + try { + return await call(); + } catch (error) { + if (error instanceof AiAssistUnavailableError) throw error; + throw new AiAssistUnavailableError(error); + } } private async fetchRelatedData( @@ -661,7 +753,7 @@ export default class LoadRelatedRecordStepExecutor extends RecordStepExecutor { + ): Promise<{ index: number; confident: boolean }> { const filteredCandidates = candidates.map((c, i) => { const entries = Object.entries(c.values).filter( ([k]) => fieldNames.length === 0 || fieldNames.includes(k), @@ -716,6 +808,12 @@ export default class LoadRelatedRecordStepExecutor extends RecordStepExecutor( - messages, - tool, - ); + const { recordIndex, confident: rawConfident } = await this.invokeWithTool<{ + recordIndex: number; + confident?: boolean; + reasoning: string; + }>(messages, tool); // NOTE: The Zod schema's .min().max() shapes the tool prompt only — it is NOT validated against // the AI response. This guard is the sole runtime enforcement. -1 (none relevant) is accepted @@ -743,6 +842,11 @@ export default class LoadRelatedRecordStepExecutor extends RecordStepExecutor { expect(bindTools).not.toHaveBeenCalled(); }); - it('returns error outcome when AI selects an out-of-range record index', async () => { + it('degrades to manual selection when the AI returns an out-of-range record index', async () => { const hasManySchema = makeCollectionSchema({ fields: [ { @@ -835,14 +835,16 @@ describe('LoadRelatedRecordStepExecutor', () => { const result = await executor.execute(); - expect(result.stepOutcome.status).toBe('error'); - expect(result.stepOutcome.error).toBe( - "The AI made an unexpected choice. Try rephrasing the step's prompt.", - ); - expect(runStore.saveStepExecution).not.toHaveBeenCalled(); + // An out-of-range index is an AI failure → degrade to the Manual path (deterministic list, + // user picks) rather than erroring the run or auto-loading a guess. + expect(result.stepOutcome.status).toBe('awaiting-input'); + const saved = (runStore.saveStepExecution as jest.Mock).mock.calls.at(-1)?.[1]; + expect(saved.pendingData.availableRecordIds).toEqual([cand([1]), cand([2])]); + expect(saved.pendingData.suggestedRecord).toBeUndefined(); + expect(saved.pendingData.suggestNoRecord).toBeUndefined(); }); - it('returns error when AI returns empty fieldNames violating the min:1 constraint', async () => { + it('degrades to manual selection when the AI returns empty fieldNames (min:1 violation)', async () => { const hasManySchema = makeCollectionSchema({ fields: [ { @@ -884,11 +886,12 @@ describe('LoadRelatedRecordStepExecutor', () => { const result = await executor.execute(); - expect(result.stepOutcome.status).toBe('error'); - expect(result.stepOutcome.error).toBe( - "The AI made an unexpected choice. Try rephrasing the step's prompt.", - ); - expect(runStore.saveStepExecution).not.toHaveBeenCalled(); + // An invalid field-selection response is an AI failure → degrade to the Manual path. + expect(result.stepOutcome.status).toBe('awaiting-input'); + const saved = (runStore.saveStepExecution as jest.Mock).mock.calls.at(-1)?.[1]; + expect(saved.pendingData.availableRecordIds).toEqual([cand([1]), cand([2])]); + expect(saved.pendingData.suggestedRecord).toBeUndefined(); + expect(saved.pendingData.suggestNoRecord).toBeUndefined(); }); }); @@ -1035,7 +1038,7 @@ describe('LoadRelatedRecordStepExecutor', () => { }); }); - describe('executionType=Manual: no AI pre-selection (PRD-148)', () => { + describe('executionType=Manual: no AI pre-selection', () => { const customersWithAddress = makeCollectionSchema({ fields: [ { @@ -1163,7 +1166,7 @@ describe('LoadRelatedRecordStepExecutor', () => { }); }); - describe('executionType=AutomatedWithConfirmation: AI may decline (PRD-148)', () => { + describe('executionType=AutomatedWithConfirmation: AI may decline', () => { it('pre-checks "No X to load" (suggestNoRecord) when the AI judges no candidate relevant', async () => { const customersWithAddress = makeCollectionSchema({ fields: [ @@ -1290,7 +1293,7 @@ describe('LoadRelatedRecordStepExecutor', () => { }); }); - describe('executionType=FullyAutomated: AI judges none relevant → confirmation (PRD-148)', () => { + describe('executionType=FullyAutomated: AI judges none relevant → confirmation', () => { it('falls back to awaiting-input (No X to load) when select-record-by-content returns -1', async () => { const customersWithAddress = makeCollectionSchema({ fields: [ @@ -1358,6 +1361,201 @@ describe('LoadRelatedRecordStepExecutor', () => { }); }); + describe('executionType=FullyAutomated: low-confidence match → confirmation', () => { + const customersWithAddress = makeCollectionSchema({ + fields: [ + { + fieldName: 'address', + displayName: 'Address', + isRelationship: true, + relationType: 'HasMany', + relatedCollectionName: 'addresses', + }, + ], + }); + const addressSchema = makeCollectionSchema({ + collectionName: 'addresses', + collectionDisplayName: 'Addresses', + fields: [{ fieldName: 'city', displayName: 'City', isRelationship: false }], + }); + const relatedData: RecordData[] = [ + { collectionName: 'addresses', recordId: [1], values: { city: 'Paris' } }, + { collectionName: 'addresses', recordId: [2], values: { city: 'Lyon' } }, + ]; + + it('degrades to awaiting-input with the best guess pre-selected when the AI is not confident', async () => { + const agentPort = makeMockAgentPort(relatedData); + const invoke = jest + .fn() + .mockResolvedValueOnce({ + tool_calls: [{ name: 'select-fields', args: { fieldNames: ['City'] }, id: 'c2' }], + }) + .mockResolvedValueOnce({ + tool_calls: [ + { + name: 'select-record-by-content', + args: { + recordIndex: 0, + confident: false, + reasoning: 'Paris and Lyon both plausible', + }, + id: 'c3', + }, + ], + }); + const bindTools = jest.fn().mockReturnValue({ invoke }); + const model = { bindTools } as unknown as ExecutionContext['model']; + const runStore = makeMockRunStore(); + const context = makeContext({ + model, + agentPort, + runStore, + workflowPort: makeMockWorkflowPort({ + customers: customersWithAddress, + addresses: addressSchema, + }), + stepDefinition: makeStep({ + executionType: StepExecutionMode.FullyAutomated, + preRecordedArgs: { relationName: 'address' }, + }), + }); + + const result = await new LoadRelatedRecordStepExecutor(context).execute(); + + // Full AI can't confidently single one out → hand off to a human with the AI's best guess + // (index 0 → recordId [1]) pre-selected, rather than auto-loading an arbitrary pick. + expect(result.stepOutcome.status).toBe('awaiting-input'); + const saved = (runStore.saveStepExecution as jest.Mock).mock.calls.at(-1)?.[1]; + expect(saved.pendingData.availableRecordIds).toEqual([cand([1]), cand([2])]); + expect(saved.pendingData.suggestedRecord).toEqual(cand([1])); + // Not a "none relevant" outcome — the "No X to load" box must stay unchecked. + expect(saved.pendingData.suggestNoRecord).toBeUndefined(); + // Nothing was auto-loaded. + expect(saved.executionResult).toBeUndefined(); + }); + + it('auto-loads (unchanged) when the AI is confident about its pick', async () => { + const agentPort = makeMockAgentPort(relatedData); + const invoke = jest + .fn() + .mockResolvedValueOnce({ + tool_calls: [{ name: 'select-fields', args: { fieldNames: ['City'] }, id: 'c2' }], + }) + .mockResolvedValueOnce({ + tool_calls: [ + { + name: 'select-record-by-content', + args: { recordIndex: 1, confident: true, reasoning: 'Lyon clearly matches' }, + id: 'c3', + }, + ], + }); + const bindTools = jest.fn().mockReturnValue({ invoke }); + const model = { bindTools } as unknown as ExecutionContext['model']; + const runStore = makeMockRunStore(); + const context = makeContext({ + model, + agentPort, + runStore, + workflowPort: makeMockWorkflowPort({ + customers: customersWithAddress, + addresses: addressSchema, + }), + stepDefinition: makeStep({ + executionType: StepExecutionMode.FullyAutomated, + preRecordedArgs: { relationName: 'address' }, + }), + }); + + const result = await new LoadRelatedRecordStepExecutor(context).execute(); + + expect(result.stepOutcome.status).toBe('success'); + const saved = (runStore.saveStepExecution as jest.Mock).mock.calls.at(-1)?.[1]; + expect(saved.executionResult).toEqual( + expect.objectContaining({ + record: expect.objectContaining({ collectionName: 'addresses', recordId: [2] }), + }), + ); + }); + }); + + describe('AI failure/timeout degrades to Manual (does not error the run)', () => { + it('Full AI: an AI invoke that throws degrades to awaiting-input with the candidate list', async () => { + const customersWithAddress = makeCollectionSchema({ + fields: [ + { + fieldName: 'address', + displayName: 'Address', + isRelationship: true, + relationType: 'HasMany', + relatedCollectionName: 'addresses', + }, + ], + }); + const addressSchema = makeCollectionSchema({ + collectionName: 'addresses', + collectionDisplayName: 'Addresses', + fields: [{ fieldName: 'city', displayName: 'City', isRelationship: false }], + }); + const relatedData: RecordData[] = [ + { collectionName: 'addresses', recordId: [1], values: { city: 'Paris' } }, + { collectionName: 'addresses', recordId: [2], values: { city: 'Lyon' } }, + ]; + const agentPort = makeMockAgentPort(relatedData); + // AI provider outage/timeout: the model invoke rejects. + const invoke = jest.fn().mockRejectedValue(new Error('AI provider timed out')); + const bindTools = jest.fn().mockReturnValue({ invoke }); + const model = { bindTools } as unknown as ExecutionContext['model']; + const runStore = makeMockRunStore(); + const context = makeContext({ + model, + agentPort, + runStore, + workflowPort: makeMockWorkflowPort({ + customers: customersWithAddress, + addresses: addressSchema, + }), + stepDefinition: makeStep({ + executionType: StepExecutionMode.FullyAutomated, + preRecordedArgs: { relationName: 'address' }, + }), + }); + + const result = await new LoadRelatedRecordStepExecutor(context).execute(); + + // Deterministic list presented for the user to pick — no error, no auto-skip, no suggestion. + expect(result.stepOutcome.status).toBe('awaiting-input'); + const saved = (runStore.saveStepExecution as jest.Mock).mock.calls.at(-1)?.[1]; + expect(saved.pendingData.availableRecordIds).toEqual([cand([1]), cand([2])]); + expect(saved.pendingData.suggestedRecord).toBeUndefined(); + expect(saved.pendingData.suggestNoRecord).toBeUndefined(); + expect(saved.executionResult).toBeUndefined(); + }); + + it('AI-assisted: an AI invoke that throws degrades to awaiting-input (no error)', async () => { + const agentPort = makeMockAgentPort(); + const invoke = jest.fn().mockRejectedValue(new Error('AI provider unavailable')); + const bindTools = jest.fn().mockReturnValue({ invoke }); + const model = { bindTools } as unknown as ExecutionContext['model']; + const runStore = makeMockRunStore(); + // Default 2-relation schema → relation selection calls the AI (which throws) → degrade. + const context = makeContext({ + model, + agentPort, + runStore, + stepDefinition: makeStep({ executionType: StepExecutionMode.AutomatedWithConfirmation }), + }); + + const result = await new LoadRelatedRecordStepExecutor(context).execute(); + + expect(result.stepOutcome.status).toBe('awaiting-input'); + const saved = (runStore.saveStepExecution as jest.Mock).mock.calls.at(-1)?.[1]; + // Degrade picks the first eligible relation (Order) and pre-fills its single linked record. + expect(saved.pendingData.suggestedField).toEqual({ name: 'order', displayName: 'Order' }); + expect(saved.pendingData.suggestedRecord).toEqual(cand(['99'])); + }); + }); + describe('operation activity log', () => { it('logs listRelatedData against the source record and its collection, not the trigger', async () => { const runStore = makeMockRunStore(); @@ -2212,6 +2410,46 @@ describe('LoadRelatedRecordStepExecutor', () => { ); }); + it('clears a stale suggestNoRecord flag when switching to a relation that yields a suggestion', async () => { + // Previous field ended in "No X to load" (suggestNoRecord true, no suggestion). Switching to a + // relation that resolves a record must drop the stale flag and surface the new suggestion — + // pendingData is rebuilt for the new field, not spread over the old one. + const execution = makePendingExecution({ + pendingData: { + availableFields: [ + { name: 'order', displayName: 'Order' }, + { name: 'address', displayName: 'Address' }, + ], + suggestedField: { name: 'address', displayName: 'Address' }, + availableRecordIds: [cand([1]), cand([2])], + suggestedRecord: undefined, + suggestNoRecord: true, + }, + }); + const agentPort = makeMockAgentPort(); // default: order #99 via getSingleRelatedData + const runStore = makeMockRunStore({ + getStepExecutions: jest.fn().mockResolvedValue([execution]), + }); + const mockModel = makeMockModel({}); + const context = makeContext({ + model: mockModel.model, + agentPort, + runStore, + workflowPort: makeMockWorkflowPort({ customers: makeCollectionSchema() }), + incomingPendingData: { fieldName: 'order' }, + stepDefinition: makeStep({ executionType: StepExecutionMode.AutomatedWithConfirmation }), + }); + + const result = await new LoadRelatedRecordStepExecutor(context).execute(); + + expect(result.stepOutcome.status).toBe('awaiting-input'); + const finalSave = (runStore.saveStepExecution as jest.Mock).mock.calls.at(-1)?.[1]; + expect(finalSave.pendingData.suggestedField).toEqual({ name: 'order', displayName: 'Order' }); + expect(finalSave.pendingData.suggestedRecord).toEqual(cand(['99'])); + // The stale "No X to load" flag from the previous field must not linger. + expect(finalSave.pendingData.suggestNoRecord).toBeUndefined(); + }); + it('reruns xToOne candidate lookup when previewing a BelongsTo relation', async () => { // Same setup but switching to Order (BelongsTo). Verifies the xToOne path is // used inside refreshCandidatesForField — no AI calls, single candidate from @@ -2672,6 +2910,38 @@ describe('LoadRelatedRecordStepExecutor', () => { expect(saved.pendingData.availableRecordIds).toHaveLength(50); }); + // Safety core (noneAllowed = allowNone && !truncated): when the candidate list is truncated the + // AI must NOT be offered the -1 "none relevant" option, so it can't decline based on a partial + // view — a match could be hiding in the unseen tail. + it('withholds the -1 (none) option from the AI when the candidate list is truncated', async () => { + const big = 'y'.repeat(80); + const values = Object.fromEntries(Array.from({ length: 6 }, (_, i) => [`f${i}`, big])); + const relatedData: RecordData[] = Array.from({ length: 50 }, (_, i) => ({ + collectionName: 'addresses', + recordId: [i + 1], + values, + })); + const { invoke, model } = buildModel(Array.from({ length: 6 }, (_, i) => `F${i}`)); + const context = makeContext({ + model, + logger: jest.fn(), + agentPort: makeMockAgentPort(relatedData), + runStore: makeMockRunStore(), + workflowPort: makeMockWorkflowPort({ + customers: makeCollectionSchema(), + addresses: wideSchema(6), + }), + // AI-assisted so allowNone is true at the call site — the truncation guard is what suppresses + // the -1 option, not the mode. + stepDefinition: makeStep({ executionType: StepExecutionMode.AutomatedWithConfirmation }), + }); + + await new LoadRelatedRecordStepExecutor(context).execute(); + + // Neither the decline guidance nor the "-1" index appears in the record-selection prompt. + expect(selectRecordPrompt(invoke)).not.toContain('-1'); + }); + // The AI sees only the budgeted prefix, but its index maps back into the FULL list — so a // non-zero pick must resolve to the correct full-list record (cap the prompt, not the data). it('maps a non-zero AI index back into the full candidate list after truncation', async () => { @@ -2962,9 +3232,10 @@ describe('LoadRelatedRecordStepExecutor', () => { }); describe('select-relation-to-follow failure', () => { - // With >=2 candidates the AI must echo back one of the offered labels. A label that - // matches no option (here a fabricated relation) is rejected as an invalid AI response. - it('returns error when AI selects a relation label not among the offered options', async () => { + // With >=2 candidates the AI must echo back one of the offered labels. A label that matches no + // option (here a fabricated relation) is an invalid AI response → the step degrades to the + // deterministic Manual path instead of erroring the run. + it('degrades to manual selection when AI selects a relation label not among the offered options', async () => { const agentPort = makeMockAgentPort(); // Default schema has two relations (Order, Address) → select-relation-to-follow IS invoked. const mockModel = makeMockModel({ @@ -2986,16 +3257,19 @@ describe('LoadRelatedRecordStepExecutor', () => { const result = await executor.execute(); - expect(result.stepOutcome.status).toBe('error'); - expect(result.stepOutcome.error).toBe( - "The AI made an unexpected choice. Try rephrasing the step's prompt.", - ); - expect(agentPort.getRelatedData).not.toHaveBeenCalled(); + // Degrade picks the first eligible relation (Order, BelongsTo) deterministically and presents + // its single linked record for confirmation — no error, no auto-load off a bad relation pick. + expect(result.stepOutcome.status).toBe('awaiting-input'); + const saved = (runStore.saveStepExecution as jest.Mock).mock.calls.at(-1)?.[1]; + expect(saved.pendingData.suggestedField).toEqual({ name: 'order', displayName: 'Order' }); + expect(saved.pendingData.suggestedRecord).toEqual(cand(['99'])); }); }); describe('AI malformed/missing tool call', () => { - it('returns error on malformed tool call', async () => { + // A malformed or missing tool call is an AI failure. In an AI mode it degrades to the Manual + // path (first eligible relation + its candidate) rather than erroring the run. + it('degrades to manual selection on a malformed tool call', async () => { const invoke = jest.fn().mockResolvedValue({ tool_calls: [], invalid_tool_calls: [ @@ -3013,16 +3287,13 @@ describe('LoadRelatedRecordStepExecutor', () => { const result = await executor.execute(); expect(result.stepOutcome.type).toBe('record'); - expect(result.stepOutcome.stepId).toBe('load-1'); - expect(result.stepOutcome.stepIndex).toBe(0); - expect(result.stepOutcome.status).toBe('error'); - expect(result.stepOutcome.error).toBe( - "The AI returned an unexpected response. Try rephrasing the step's prompt.", - ); - expect(runStore.saveStepExecution).not.toHaveBeenCalled(); + expect(result.stepOutcome.status).toBe('awaiting-input'); + const saved = (runStore.saveStepExecution as jest.Mock).mock.calls.at(-1)?.[1]; + expect(saved.pendingData.suggestedField).toEqual({ name: 'order', displayName: 'Order' }); + expect(saved.pendingData.suggestedRecord).toEqual(cand(['99'])); }); - it('returns error when AI returns no tool call', async () => { + it('degrades to manual selection when AI returns no tool call', async () => { const invoke = jest.fn().mockResolvedValue({ tool_calls: [] }); const bindTools = jest.fn().mockReturnValue({ invoke }); const runStore = makeMockRunStore(); @@ -3035,13 +3306,10 @@ describe('LoadRelatedRecordStepExecutor', () => { const result = await executor.execute(); expect(result.stepOutcome.type).toBe('record'); - expect(result.stepOutcome.stepId).toBe('load-1'); - expect(result.stepOutcome.stepIndex).toBe(0); - expect(result.stepOutcome.status).toBe('error'); - expect(result.stepOutcome.error).toBe( - "The AI couldn't decide what to do. Try rephrasing the step's prompt.", - ); - expect(runStore.saveStepExecution).not.toHaveBeenCalled(); + expect(result.stepOutcome.status).toBe('awaiting-input'); + const saved = (runStore.saveStepExecution as jest.Mock).mock.calls.at(-1)?.[1]; + expect(saved.pendingData.suggestedField).toEqual({ name: 'order', displayName: 'Order' }); + expect(saved.pendingData.suggestedRecord).toEqual(cand(['99'])); }); }); @@ -3938,7 +4206,7 @@ describe('LoadRelatedRecordStepExecutor', () => { it('errors (Full AI) when the source step loaded nothing — no longer auto-skips', async () => { // The source step exists in the live path but its run-store execution has no record. Full AI // now surfaces the error like the await modes (no silent skip); the front offers "continue - // without" (PRD-550). + // without". const runStore = makeMockRunStore({ getStepExecutions: jest.fn().mockResolvedValue([]) }); const context = makeContext({ runStore, @@ -3957,7 +4225,7 @@ describe('LoadRelatedRecordStepExecutor', () => { it('surfaces a "no source record" error in await modes when the source step loaded nothing', async () => { // Same as Full AI now — every mode surfaces the error so the front can offer - // "continue without" (PRD-550). + // "continue without". const runStore = makeMockRunStore({ getStepExecutions: jest.fn().mockResolvedValue([]) }); const context = makeContext({ runStore,