From 4942a6f86aee1b47f1a0c4bd13267ea4bb63f0d3 Mon Sep 17 00:00:00 2001 From: Chris Coutinho Date: Mon, 27 Jul 2026 22:09:50 +0200 Subject: [PATCH 1/2] fix(multimodal): send documents in the shape Mistral expects buildDocumentContent() only ever emitted OpenAI's content part, so every document attachment sent to Mistral came back as an HTTP 422 with a confusing "Input should be a valid string". Mistral takes a flat document_url string rather than a nested file object; its schema has no variant matching ours, so the discriminated union falls through to parsing the content as a plain string and reports that instead. Pick the envelope from the configured service URL. The payload itself is unchanged: every backend that supports documents accepts them inline as a base64 data URI, so no separate upload API is involved either way. Signed-off-by: Chris Coutinho --- lib/AppInfo/Application.php | 2 ++ lib/Service/OpenAiFileService.php | 25 +++++++++++++++++++++++-- 2 files changed, 25 insertions(+), 2 deletions(-) diff --git a/lib/AppInfo/Application.php b/lib/AppInfo/Application.php index 27be6312..3c029b7c 100644 --- a/lib/AppInfo/Application.php +++ b/lib/AppInfo/Application.php @@ -39,6 +39,8 @@ class Application extends App implements IBootstrap { public const APP_ID = 'integration_openai'; public const OPENAI_API_BASE_URL = 'https://api.openai.com/v1'; + // Mistral speaks the OpenAI chat completion API but wants its own shape for document attachments + public const MISTRAL_API_BASE_URL_PREFIX = 'https://api.mistral.ai'; public const OPENAI_DEFAULT_REQUEST_TIMEOUT = 60 * 4; public const USER_AGENT = 'Nextcloud OpenAI/LocalAI integration'; diff --git a/lib/Service/OpenAiFileService.php b/lib/Service/OpenAiFileService.php index d1977bca..a581ae88 100644 --- a/lib/Service/OpenAiFileService.php +++ b/lib/Service/OpenAiFileService.php @@ -228,7 +228,11 @@ private function buildVideoContent(File $file, string $fileType): array { } /** - * @return list + * Backends disagree on how a document is attached to a chat completion, but + * they all take the document inline as a base64 data URI, so no separate + * upload API is needed - only the envelope differs. + * + * @return list> */ private function buildDocumentContent(File $file, string $fileType): array { if (!$this->openAiSettingsService->getMultimodalDocumentEnabled()) { @@ -239,11 +243,23 @@ private function buildDocumentContent(File $file, string $fileType): array { $this->l10n->t('Document attachments are unsupported.'), ); } + $dataUri = 'data:' . $fileType . ';base64,' . base64_encode(stream_get_contents($file->fopen('rb'))); + + if ($this->isUsingMistral()) { + // Mistral takes a flat string, not an object. Sending OpenAI's shape gets + // rejected with a confusing HTTP 422 "Input should be a valid string". + return [[ + 'type' => 'document_url', + 'document_url' => $dataUri, + 'document_name' => $file->getName(), + ]]; + } + return [[ 'type' => 'file', 'file' => [ 'filename' => $file->getName(), - 'file_data' => 'data:' . $fileType . ';base64,' . base64_encode(stream_get_contents($file->fopen('rb'))), + 'file_data' => $dataUri, ], ]]; } @@ -271,4 +287,9 @@ private function isUsingOpenAi(): bool { $serviceUrl = $this->openAiSettingsService->getServiceUrl(); return $serviceUrl === '' || $serviceUrl === Application::OPENAI_API_BASE_URL; } + + private function isUsingMistral(): bool { + $serviceUrl = strtolower($this->openAiSettingsService->getServiceUrl()); + return str_starts_with($serviceUrl, Application::MISTRAL_API_BASE_URL_PREFIX); + } } From 04f3bbbe49c51a509f55c70f56d2c6e6d5c16a80 Mon Sep 17 00:00:00 2001 From: Chris Coutinho Date: Mon, 27 Jul 2026 22:09:50 +0200 Subject: [PATCH 2/2] feat(summary): summarize attached files instead of extracted text The summary provider could only ever work on a string, so callers had to extract the text of a document before asking for a summary. That loses whatever the extractor cannot read, which for PDFs is often everything. Advertise an optional input_attachments slot and, when files are given, inline them in a single chat completion instead. Chunking is skipped entirely on that path: the point is that we never read the file, so there is no text to split. The slot is only advertised when document support is enabled and the chat completion endpoint is usable, since attachments cannot be carried over the legacy completions endpoint. Advertising it unconditionally would tell callers attachments work and then fail every request. An empty prompt with no usable attachment is now rejected rather than summarized, so a dropped attachment surfaces as an error instead of a summary of nothing. Signed-off-by: Chris Coutinho --- lib/TaskProcessing/SummaryProvider.php | 156 +++++++++++++++--- tests/unit/Providers/OpenAiProviderTest.php | 172 ++++++++++++++++++++ 2 files changed, 309 insertions(+), 19 deletions(-) diff --git a/lib/TaskProcessing/SummaryProvider.php b/lib/TaskProcessing/SummaryProvider.php index ab9dce24..2905b2e7 100644 --- a/lib/TaskProcessing/SummaryProvider.php +++ b/lib/TaskProcessing/SummaryProvider.php @@ -13,6 +13,7 @@ use OCA\OpenAi\Service\ChunkService; use OCA\OpenAi\Service\OpenAiAPIService; use OCA\OpenAi\Service\OpenAiSettingsService; +use OCP\Files\File; use OCP\IL10N; use OCP\TaskProcessing\EShapeType; use OCP\TaskProcessing\Exception\ProcessingException; @@ -24,6 +25,8 @@ class SummaryProvider implements ISynchronousProvider { + private const MAX_INPUT_ATTACHMENTS = 10; + public function __construct( private OpenAiAPIService $openAiAPIService, private OpenAiSettingsService $openAiSettingsService, @@ -58,7 +61,7 @@ public function getInputShapeDefaults(): array { } public function getOptionalInputShape(): array { - return [ + $shape = [ 'format' => new ShapeDescriptor( $this->l->t('Format'), $this->l->t('The format of the summary'), @@ -80,6 +83,27 @@ public function getOptionalInputShape(): array { EShapeType::Enum ), ]; + + // Only advertise attachments when we can actually turn them into a document part, + // otherwise callers would send us files that always fail at request time. + if ($this->canSummarizeAttachments()) { + $shape['input_attachments'] = new ShapeDescriptor( + $this->l->t('Input attachments'), + $this->l->t('Files to summarize, sent to the language model as-is instead of extracting their text first.'), + EShapeType::ListOfFiles + ); + } + + return $shape; + } + + /** + * Attachments are inlined in a chat completion request, so both document + * support and the chat endpoint are required. + */ + private function canSummarizeAttachments(): bool { + return $this->openAiSettingsService->getMultimodalDocumentEnabled() + && ($this->openAiAPIService->isUsingOpenAi() || $this->openAiSettingsService->getChatEndpointEnabled()); } public function getOptionalInputShapeEnumValues(): array { @@ -139,6 +163,28 @@ public function process(?string $userId, array $input, callable $reportProgress) $model = $input['model']; } + // core resolves file slots into File objects before handing us the input + /** @var list $files */ + $files = []; + if (isset($input['input_attachments']) && is_array($input['input_attachments'])) { + foreach ($input['input_attachments'] as $attachment) { + if ($attachment instanceof File) { + $files[] = $attachment; + } + } + } + if ($files !== []) { + $result = $this->summarizeAttachments($userId, $files, $input, $model, $maxTokens, $reportProgress); + $this->openAiAPIService->updateExpTextProcessingTime(time() - $startTime); + return $result; + } + + if ($prompt === '') { + // Callers leave the prompt empty when they attach a file. Getting here means the + // attachment never arrived, so fail loudly instead of summarizing nothing. + throw new ProcessingException('Nothing to summarize: no text and no readable attachment was provided'); + } + $prompts = $this->chunkService->chunkSplitPrompt($prompt); $newNumChunks = count($prompts); $progress = 0.0; @@ -153,24 +199,7 @@ public function process(?string $userId, array $input, callable $reportProgress) try { $completions = []; - $summarySystemPrompt = 'You are a helpful assistant that summarizes text in the same language as the text. ' - . 'You should only return the summary without any additional information. '; - if (isset($input['format'])) { - if ($input['format'] === 'paragraph') { - $summarySystemPrompt .= 'Return the summary as a paragraph. '; - } elseif ($input['format'] === 'bullet_points') { - $summarySystemPrompt .= 'Return the summary as a list of bullet points. '; - } elseif ($input['format'] === 'sentence') { - $summarySystemPrompt .= 'Return the summary as a single sentence. Do not include more than one sentence. '; - } - } - if (isset($input['complexity'])) { - if ($input['complexity'] === 'complex') { - $summarySystemPrompt .= 'Use complex language and vocabulary appropriate for an expert in the subject. '; - } elseif ($input['complexity'] === 'simple') { - $summarySystemPrompt .= 'Use simple language and vocabulary appropriate for a 5 year old. '; - } - } + $summarySystemPrompt = $this->buildSummarySystemPrompt($input); if ($this->openAiAPIService->isUsingOpenAi() || $this->openAiSettingsService->getChatEndpointEnabled()) { foreach ($prompts as $p) { @@ -223,4 +252,93 @@ public function process(?string $userId, array $input, callable $reportProgress) return ['output' => $summary]; } + /** + * @param array $input + */ + private function buildSummarySystemPrompt(array $input): string { + $summarySystemPrompt = 'You are a helpful assistant that summarizes text in the same language as the text. ' + . 'You should only return the summary without any additional information. '; + if (isset($input['format'])) { + if ($input['format'] === 'paragraph') { + $summarySystemPrompt .= 'Return the summary as a paragraph. '; + } elseif ($input['format'] === 'bullet_points') { + $summarySystemPrompt .= 'Return the summary as a list of bullet points. '; + } elseif ($input['format'] === 'sentence') { + $summarySystemPrompt .= 'Return the summary as a single sentence. Do not include more than one sentence. '; + } + } + if (isset($input['complexity'])) { + if ($input['complexity'] === 'complex') { + $summarySystemPrompt .= 'Use complex language and vocabulary appropriate for an expert in the subject. '; + } elseif ($input['complexity'] === 'simple') { + $summarySystemPrompt .= 'Use simple language and vocabulary appropriate for a 5 year old. '; + } + } + return $summarySystemPrompt; + } + + /** + * Summarize attached files by handing them to the model directly. + * + * Nothing is chunked here: the whole point is that we never read the file + * ourselves, so we have no text to split. The document is inlined in the + * request, which means this only works over the chat completion endpoint. + * + * @param list $files + * @param array $input + * @return array{output: string} + * @throws ProcessingException + * @throws UserFacingProcessingException + */ + private function summarizeAttachments( + ?string $userId, + array $files, + array $input, + string $model, + ?int $maxTokens, + callable $reportProgress, + ): array { + if (!$this->openAiAPIService->isUsingOpenAi() && !$this->openAiSettingsService->getChatEndpointEnabled()) { + throw new UserFacingProcessingException( + 'Summarizing attachments requires the chat completion endpoint', + 0, + null, + $this->l->t('Summarizing files requires a service that supports the chat completion endpoint.'), + ); + } + if (count($files) > self::MAX_INPUT_ATTACHMENTS) { + throw new UserFacingProcessingException( + 'Too many files. Max is ' . self::MAX_INPUT_ATTACHMENTS, + 0, + null, + $this->l->t('Too many files given. A maximum of %d files is allowed.', [self::MAX_INPUT_ATTACHMENTS]), + ); + } + + $userPrompt = isset($input['input']) && is_string($input['input']) && $input['input'] !== '' + ? $input['input'] + : null; + + if (!$reportProgress(0.0)) { + throw new ProcessingException('OpenAI/LocalAI task cancelled'); + } + + try { + $completion = $this->openAiAPIService->createChatCompletion( + $userId, $model, $userPrompt, $this->buildSummarySystemPrompt($input), null, 1, $maxTokens, + null, null, null, $files, + ); + } catch (UserFacingProcessingException $e) { + throw $e; + } catch (\Throwable $e) { + throw new ProcessingException('OpenAI/LocalAI request failed: ' . $e->getMessage()); + } + + $summary = array_pop($completion['messages']); + if (!is_string($summary) || $summary === '') { + throw new ProcessingException('No result in OpenAI/LocalAI response.'); + } + return ['output' => $summary]; + } + } diff --git a/tests/unit/Providers/OpenAiProviderTest.php b/tests/unit/Providers/OpenAiProviderTest.php index d739eb65..402ba5c7 100644 --- a/tests/unit/Providers/OpenAiProviderTest.php +++ b/tests/unit/Providers/OpenAiProviderTest.php @@ -1216,4 +1216,176 @@ public function testMultimodalChatWithToolsProvider(): void { $this->quotaUsageMapper->deleteUserQuotaUsages(self::TEST_USER1); } + private function mockPdfFile(string $content, string $name = 'report.pdf'): \OCP\Files\File&MockObject { + $stream = fopen('php://temp', 'r+'); + if ($stream === false) { + throw new \RuntimeException('Could not open temp stream'); + } + fwrite($stream, $content); + rewind($stream); + + $file = $this->createMock(\OCP\Files\File::class); + $file->method('isReadable')->willReturn(true); + $file->method('getSize')->willReturn(strlen($content)); + $file->method('getMimeType')->willReturn('application/pdf'); + $file->method('getName')->willReturn($name); + $file->method('fopen')->with('rb')->willReturn($stream); + return $file; + } + + private const SUMMARY_SYSTEM_PROMPT = 'You are a helpful assistant that summarizes text in the same language as the text. ' + . 'You should only return the summary without any additional information. '; + + private const CHAT_COMPLETION_RESPONSE = '{ + "id": "chatcmpl-123", + "object": "chat.completion", + "created": 1677652288, + "model": "gpt-4.1-mini", + "choices": [ + { + "index": 0, + "message": { + "role": "assistant", + "content": "This is a test response." + }, + "finish_reason": "stop" + } + ], + "usage": { + "prompt_tokens": 9, + "completion_tokens": 12, + "total_tokens": 21 + } + }'; + + private function buildSummaryProvider(): SummaryProvider { + return new SummaryProvider( + $this->openAiApiService, + $this->openAiSettingsService, + $this->createMock(\OCP\IL10N::class), + $this->chunkService, + self::TEST_USER1, + ); + } + + private function mockChatCompletionResponse(): \OCP\Http\Client\IResponse&MockObject { + $iResponse = $this->createMock(\OCP\Http\Client\IResponse::class); + $iResponse->method('getBody')->willReturn(self::CHAT_COMPLETION_RESPONSE); + $iResponse->method('getStatusCode')->willReturn(200); + $iResponse->method('getHeader')->with('Content-Type')->willReturn('application/json'); + return $iResponse; + } + + /** + * A PDF is inlined as an OpenAI "file" content part, and no text is chunked: + * the whole point is that we never read the document ourselves. + */ + public function testSummaryProviderWithPdfAttachment(): void { + $provider = $this->buildSummaryProvider(); + $pdfContent = 'fake-pdf-bytes'; + $file = $this->mockPdfFile($pdfContent); + + $url = self::OPENAI_API_BASE . 'chat/completions'; + $options = ['timeout' => Application::OPENAI_DEFAULT_REQUEST_TIMEOUT, 'headers' => ['User-Agent' => Application::USER_AGENT, 'Authorization' => self::AUTHORIZATION_HEADER, 'Content-Type' => 'application/json']]; + $options['body'] = json_encode([ + 'model' => Application::DEFAULT_COMPLETION_MODEL_ID, + 'messages' => [ + ['role' => 'system', 'content' => self::SUMMARY_SYSTEM_PROMPT], + [ + 'role' => 'user', + 'content' => [ + [ + 'type' => 'file', + 'file' => [ + 'filename' => 'report.pdf', + 'file_data' => 'data:application/pdf;base64,' . base64_encode($pdfContent), + ], + ], + ], + ], + ], + 'n' => 1, + 'stream' => false, + 'max_completion_tokens' => Application::DEFAULT_MAX_NUM_OF_TOKENS, + 'user' => self::TEST_USER1, + ]); + + $this->iClient->expects($this->once())->method('post')->with($url, $options) + ->willReturn($this->mockChatCompletionResponse()); + + $result = $provider->process(self::TEST_USER1, [ + // the caller leaves the text input empty, the file is the whole request + 'input' => '', + 'input_attachments' => [$file], + ], fn () => true); + + $this->assertEquals('This is a test response.', $result['output']); + $this->quotaUsageMapper->deleteUserQuotaUsages(self::TEST_USER1); + } + + /** + * Mistral speaks the OpenAI chat completion API but rejects the OpenAI "file" + * part with a 422, so documents have to go out as a flat document_url string. + */ + public function testSummaryProviderWithPdfAttachmentOnMistral(): void { + $previousServiceUrl = $this->openAiSettingsService->getServiceUrl(); + $this->openAiSettingsService->setServiceUrl(Application::MISTRAL_API_BASE_URL_PREFIX . '/v1'); + + try { + $provider = $this->buildSummaryProvider(); + $pdfContent = 'fake-pdf-bytes'; + $file = $this->mockPdfFile($pdfContent); + + $url = Application::MISTRAL_API_BASE_URL_PREFIX . '/v1/chat/completions'; + $options = ['timeout' => Application::OPENAI_DEFAULT_REQUEST_TIMEOUT, 'headers' => ['User-Agent' => Application::USER_AGENT, 'Authorization' => self::AUTHORIZATION_HEADER, 'Content-Type' => 'application/json']]; + $options['body'] = json_encode([ + 'model' => Application::DEFAULT_COMPLETION_MODEL_ID, + 'messages' => [ + ['role' => 'system', 'content' => self::SUMMARY_SYSTEM_PROMPT], + [ + 'role' => 'user', + 'content' => [ + [ + 'type' => 'document_url', + 'document_url' => 'data:application/pdf;base64,' . base64_encode($pdfContent), + 'document_name' => 'report.pdf', + ], + ], + ], + ], + 'n' => 1, + 'stream' => false, + 'max_completion_tokens' => Application::DEFAULT_MAX_NUM_OF_TOKENS, + ]); + + $this->iClient->expects($this->once())->method('post')->with($url, $options) + ->willReturn($this->mockChatCompletionResponse()); + + $result = $provider->process(self::TEST_USER1, [ + 'input' => '', + 'input_attachments' => [$file], + ], fn () => true); + + $this->assertEquals('This is a test response.', $result['output']); + $this->quotaUsageMapper->deleteUserQuotaUsages(self::TEST_USER1); + } finally { + $this->openAiSettingsService->setServiceUrl($previousServiceUrl); + } + } + + public function testSummaryProviderAdvertisesAttachmentsOnlyWhenDocumentsAreEnabled(): void { + $provider = $this->buildSummaryProvider(); + $previous = $this->openAiSettingsService->getMultimodalDocumentEnabled(); + + try { + $this->openAiSettingsService->setMultimodalDocumentEnabled(true); + $this->assertArrayHasKey('input_attachments', $provider->getOptionalInputShape()); + + $this->openAiSettingsService->setMultimodalDocumentEnabled(false); + $this->assertArrayNotHasKey('input_attachments', $provider->getOptionalInputShape()); + } finally { + $this->openAiSettingsService->setMultimodalDocumentEnabled($previous); + } + } + }