Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
80 changes: 79 additions & 1 deletion packages/api/src/agents/__tests__/initialize.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -137,6 +137,7 @@ jest.mock('../../middleware/modelBoundContent', () => {
import { initializeAgent } from '../initialize';
import { primeResources } from '../resources';
import { isFatalAgentInitializationError } from '../errors';
import * as agentTools from '../tools';

const realUtils = jest.requireActual<typeof import('~/utils')>('~/utils');

Expand Down Expand Up @@ -5166,7 +5167,7 @@ describe('initializeAgent tool-routed text fallback', () => {
csv: pdf(),
requestFiles: [],
llmDeliveryPolicy: 'automatic',
csvOverride: false,
csvOverride: true,
queuedForSearch: true,
extraQueued: [
{
Expand Down Expand Up @@ -5439,6 +5440,83 @@ describe('initializeAgent tool-routed text fallback', () => {
expect(off.result.deliveryRouting.overBudgetFileIds?.size ?? 0).toBe(0);
});

describe('Run Code spreadsheets and the File Search queue', () => {
const codeAndSearch = {
provider: EModelEndpoint.anthropic,
...bothTools,
csvOverride: false,
codeQueued: true,
queuedForSearch: true,
};
const sheet = () => sandboxCsvFile();
const automatic = { llmDeliveryPolicy: 'automatic' as const };

it('keeps a Run Code spreadsheet off the File Search queue', async () => {
const { result, loadTools } = await initializeWith({
...codeAndSearch,
...automatic,
csv: sheet(),
});
expect(ids(result.provisionState?.codeEnvFiles)).toEqual(['csv']);
expect(ids(result.provisionState?.vectorDBFiles)).toEqual([]);
expect(loadTools.mock.calls[0][0].tool_resources?.file_search?.files).toBeUndefined();
expect(result.tool_resources?.file_search?.files).toBeUndefined();
});

it('still queues a document for File Search', async () => {
const { result } = await initializeWith({
...codeAndSearch,
...automatic,
csv: sheet(),
requestFiles: [sheet(), pdf()],
maxContextTokens: 100_000,
});
expect(ids(result.provisionState?.vectorDBFiles)).toEqual(['pdf-file']);
expect(ids(result.provisionState?.codeEnvFiles)?.sort()).toEqual(['csv', 'pdf-file']);
expect(ids(result.tool_resources?.file_search?.files)).toEqual(['pdf-file']);
});

it('keeps an agent-scoped spreadsheet on the queue', async () => {
const { result } = await initializeWith({
...codeAndSearch,
...automatic,
csv: sheet(),
agentScoped: true,
});
expect(ids(result.provisionState?.vectorDBFiles)).toEqual(['csv']);
});

it('leaves the queue alone when classic', async () => {
const { result } = await initializeWith({
...codeAndSearch,
csv: sandboxCsvFile({ text: 'a,b' }),
});
expect(ids(result.provisionState?.vectorDBFiles)).toEqual(['csv']);
});

it('puts the spreadsheet back when Run Code does not load', async () => {
/* The loader would drop Run Code: register no code tools for this turn. */
const spy = jest
.spyOn(agentTools, 'registerCodeExecutionTools')
.mockImplementation(
({ toolDefinitions }) => ({ toolDefinitions, toolNames: [] }) as never,
);
let result: Awaited<ReturnType<typeof initializeWith>>['result'];
try {
({ result } = await initializeWith({
...codeAndSearch,
...automatic,
csv: sandboxCsvFile({ text: 'a,b' }),
loadedToolDefinitions: [],
}));
} finally {
spy.mockRestore();
}
expect(result.fileConsumers?.executeCode).toBe(false);
expect(ids(result.provisionState?.vectorDBFiles)).toEqual(['csv']);
});
});

it('names a queued sandbox file for the model', async () => {
const run = (extra: Partial<Parameters<typeof initializeWith>[0]>) =>
initializeWith({
Expand Down
26 changes: 26 additions & 0 deletions packages/api/src/agents/initialize.ts
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@ import {
resolveSandboxFilename,
resolveTurnLLMDeliveryPath,
hasInferredLLMDeliveryPath,
isRunCodeSpreadsheet,
} from 'librechat-data-provider';
import type {
AgentToolResources,
Expand Down Expand Up @@ -1922,6 +1923,23 @@ export async function initializeAgent(
},
});

/* A spreadsheet automatic reading sends to Run Code is Run Code's alone: queued for File
* Search too, a programmatic tool call would embed every row before the code could run. */
let runCodeSpreadsheets: TFile[] = [];
if (automaticDelivery && provisionState != null) {
runCodeSpreadsheets = provisionState.vectorDBFiles.filter(
(file) =>
!provisionState.agentScopedFileIds.has(file.file_id) &&
isRunCodeSpreadsheet(deliveryRouting, file, fileConsumers),
);
if (runCodeSpreadsheets.length > 0) {
const removed = new Set(runCodeSpreadsheets.map((file) => file.file_id));
provisionState.vectorDBFiles = provisionState.vectorDBFiles.filter(
(file) => !removed.has(file.file_id),
);
}
}

/* Automatic reading moves files the model would otherwise see; classic keeps the note as
* it was, including under the legacy chooser. */
const searchToolResources = automaticDelivery
Expand Down Expand Up @@ -2527,6 +2545,14 @@ export async function initializeAgent(
fileConsumers.executeCode !== finalFileConsumers.executeCode ||
fileConsumers.fileSearch !== finalFileConsumers.fileSearch;
Object.assign(fileConsumers, finalFileConsumers);
/* Run Code did not load after all: a File Search route can still serve the spreadsheets.
* Known ceiling: they are not added to the File Search note, as tools are already loaded. */
if (runCodeSpreadsheets.length > 0 && !fileConsumers.executeCode && provisionState != null) {
const queued = new Set(provisionState.vectorDBFiles.map((file) => file.file_id));
provisionState.vectorDBFiles.push(
...runCodeSpreadsheets.filter((file) => !queued.has(file.file_id)),
);
}
if (consumersChanged && automaticDelivery && contextWindowKnown) {
deliveryRouting.overBudgetFileIds = planInlineAttachmentBudget(budgetCandidates, budgetParams);
}
Expand Down
60 changes: 60 additions & 0 deletions packages/data-provider/src/resolve-llm-delivery-path.spec.ts
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@ import {
getCustomEndpointProvider,
resolveTurnLLMDeliveryPath as resolveStoredTurnPath,
hasInferredLLMDeliveryPath,
isRunCodeSpreadsheet,
resolveDefaultLLMDeliveryPath,
resolveUploadLLMDeliveryPath,
SYSTEM_LLM_DELIVERY_DEFAULTS,
Expand Down Expand Up @@ -1266,3 +1267,62 @@ describe('resolveTurnLLMDeliveryPath with llmDeliveryPolicy automatic', () => {
).toBe('text');
});
});

describe('isRunCodeSpreadsheet', () => {
const auto: EndpointFileConfig = { llmDeliveryPolicy: 'automatic' };
const routing: Partial<TurnDeliveryRouting> = { endpoint: 'anthropic', endpointConfig: auto };
const both: TurnFileConsumers = { executeCode: true, fileSearch: true };
const file = (extra: Record<string, unknown> = {}): TurnDeliveryFile =>
({
file_id: 'c',
type: 'text/csv',
llmDeliveryPath: 'text',
metadata: { destinationChosen: false },
...extra,
}) as TurnDeliveryFile;

it.each(['text/csv', 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet'])(
'is true for an inferred %s',
(type) => {
expect(isRunCodeSpreadsheet(routing, file({ type }), both)).toBe(true);
},
);

it.each([
['classic', { endpoint: 'anthropic', endpointConfig: {} }, file(), both],
['no Run Code', routing, file(), { executeCode: false, fileSearch: true }],
['unknown consumers', routing, file(), undefined],
['a chosen destination', routing, file({ metadata: { destinationChosen: true } }), both],
['an agent file', routing, file({ context: 'agents' }), both],
[
'an admin override',
{
...routing,
endpointConfig: {
...auto,
defaultLLMDeliveryPath: { overrides: { 'text/csv': 'text' } },
},
},
file(),
both,
],
[
'a Word document',
routing,
file({ type: 'application/vnd.openxmlformats-officedocument.wordprocessingml.document' }),
both,
],
['an image', routing, file({ type: 'image/png' }), both],
[
'the legacy chooser',
{ ...routing, endpointConfig: { ...auto, legacyFileUploadUX: true } },
file(),
both,
],
] as Array<[string, Partial<TurnDeliveryRouting>, TurnDeliveryFile, TurnFileConsumers?]>)(
'is false for %s',
(_name, r, f, c) => {
expect(isRunCodeSpreadsheet(r, f, c)).toBe(false);
},
);
});
23 changes: 22 additions & 1 deletion packages/data-provider/src/resolve-llm-delivery-path.ts
Original file line number Diff line number Diff line change
Expand Up @@ -489,6 +489,27 @@ export function isAutomaticLLMDelivery(
);
}

/** Whether automatic reading sends this attachment to Run Code because it is a spreadsheet:
* an inferred route, not the agent's own file, no admin override for the type, on a turn that
* runs Run Code. Such a file is Run Code's alone, so File Search neither queues nor lists it. */
export function isRunCodeSpreadsheet(
routing: Partial<TurnDeliveryRouting> | undefined,
file: TurnDeliveryFile,
consumers?: TurnFileConsumers,
): boolean {
if (routing == null || !hasInferredLLMDeliveryPath(file)) {
return false;
}
const mimeType = file.metadata?.routingMimeType ?? file.type ?? '';
return (
isAutomaticLLMDelivery(routing.endpointConfig) &&
consumers?.executeCode === true &&
file.context !== FileContext.agents &&
isSpreadsheetMimeType(mimeType) &&
!hasLLMDeliveryOverride(mimeType, routing)
);
}

/**
* Delivery path for one attachment on one agent's turn.
*
Expand Down Expand Up @@ -531,7 +552,7 @@ export function resolveTurnLLMDeliveryPath(
file.context !== FileContext.agents &&
!isMediaMimeType(mimeType) &&
!hasLLMDeliveryOverride(mimeType, routing) &&
((consumers.executeCode && isSpreadsheetMimeType(mimeType)) ||
(isRunCodeSpreadsheet(routing, file, consumers) ||
(file.file_id != null &&
routing.overBudgetFileIds?.has(file.file_id) === true &&
hasTurnFileConsumer(mimeType, consumers)))
Expand Down
Loading