diff --git a/lib/AppInfo/Application.php b/lib/AppInfo/Application.php index 27be6312..3c029b7c 100644 --- a/lib/AppInfo/Application.php +++ b/lib/AppInfo/Application.php @@ -39,6 +39,8 @@ class Application extends App implements IBootstrap { public const APP_ID = 'integration_openai'; public const OPENAI_API_BASE_URL = 'https://api.openai.com/v1'; + // Mistral speaks the OpenAI chat completion API but wants its own shape for document attachments + public const MISTRAL_API_BASE_URL_PREFIX = 'https://api.mistral.ai'; public const OPENAI_DEFAULT_REQUEST_TIMEOUT = 60 * 4; public const USER_AGENT = 'Nextcloud OpenAI/LocalAI integration'; diff --git a/lib/Service/OpenAiFileService.php b/lib/Service/OpenAiFileService.php index d1977bca..a581ae88 100644 --- a/lib/Service/OpenAiFileService.php +++ b/lib/Service/OpenAiFileService.php @@ -228,7 +228,11 @@ private function buildVideoContent(File $file, string $fileType): array { } /** - * @return list + * Backends disagree on how a document is attached to a chat completion, but + * they all take the document inline as a base64 data URI, so no separate + * upload API is needed - only the envelope differs. + * + * @return list> */ private function buildDocumentContent(File $file, string $fileType): array { if (!$this->openAiSettingsService->getMultimodalDocumentEnabled()) { @@ -239,11 +243,23 @@ private function buildDocumentContent(File $file, string $fileType): array { $this->l10n->t('Document attachments are unsupported.'), ); } + $dataUri = 'data:' . $fileType . ';base64,' . base64_encode(stream_get_contents($file->fopen('rb'))); + + if ($this->isUsingMistral()) { + // Mistral takes a flat string, not an object. Sending OpenAI's shape gets + // rejected with a confusing HTTP 422 "Input should be a valid string". + return [[ + 'type' => 'document_url', + 'document_url' => $dataUri, + 'document_name' => $file->getName(), + ]]; + } + return [[ 'type' => 'file', 'file' => [ 'filename' => $file->getName(), - 'file_data' => 'data:' . $fileType . ';base64,' . base64_encode(stream_get_contents($file->fopen('rb'))), + 'file_data' => $dataUri, ], ]]; } @@ -271,4 +287,9 @@ private function isUsingOpenAi(): bool { $serviceUrl = $this->openAiSettingsService->getServiceUrl(); return $serviceUrl === '' || $serviceUrl === Application::OPENAI_API_BASE_URL; } + + private function isUsingMistral(): bool { + $serviceUrl = strtolower($this->openAiSettingsService->getServiceUrl()); + return str_starts_with($serviceUrl, Application::MISTRAL_API_BASE_URL_PREFIX); + } } diff --git a/lib/TaskProcessing/SummaryProvider.php b/lib/TaskProcessing/SummaryProvider.php index ab9dce24..2905b2e7 100644 --- a/lib/TaskProcessing/SummaryProvider.php +++ b/lib/TaskProcessing/SummaryProvider.php @@ -13,6 +13,7 @@ use OCA\OpenAi\Service\ChunkService; use OCA\OpenAi\Service\OpenAiAPIService; use OCA\OpenAi\Service\OpenAiSettingsService; +use OCP\Files\File; use OCP\IL10N; use OCP\TaskProcessing\EShapeType; use OCP\TaskProcessing\Exception\ProcessingException; @@ -24,6 +25,8 @@ class SummaryProvider implements ISynchronousProvider { + private const MAX_INPUT_ATTACHMENTS = 10; + public function __construct( private OpenAiAPIService $openAiAPIService, private OpenAiSettingsService $openAiSettingsService, @@ -58,7 +61,7 @@ public function getInputShapeDefaults(): array { } public function getOptionalInputShape(): array { - return [ + $shape = [ 'format' => new ShapeDescriptor( $this->l->t('Format'), $this->l->t('The format of the summary'), @@ -80,6 +83,27 @@ public function getOptionalInputShape(): array { EShapeType::Enum ), ]; + + // Only advertise attachments when we can actually turn them into a document part, + // otherwise callers would send us files that always fail at request time. + if ($this->canSummarizeAttachments()) { + $shape['input_attachments'] = new ShapeDescriptor( + $this->l->t('Input attachments'), + $this->l->t('Files to summarize, sent to the language model as-is instead of extracting their text first.'), + EShapeType::ListOfFiles + ); + } + + return $shape; + } + + /** + * Attachments are inlined in a chat completion request, so both document + * support and the chat endpoint are required. + */ + private function canSummarizeAttachments(): bool { + return $this->openAiSettingsService->getMultimodalDocumentEnabled() + && ($this->openAiAPIService->isUsingOpenAi() || $this->openAiSettingsService->getChatEndpointEnabled()); } public function getOptionalInputShapeEnumValues(): array { @@ -139,6 +163,28 @@ public function process(?string $userId, array $input, callable $reportProgress) $model = $input['model']; } + // core resolves file slots into File objects before handing us the input + /** @var list $files */ + $files = []; + if (isset($input['input_attachments']) && is_array($input['input_attachments'])) { + foreach ($input['input_attachments'] as $attachment) { + if ($attachment instanceof File) { + $files[] = $attachment; + } + } + } + if ($files !== []) { + $result = $this->summarizeAttachments($userId, $files, $input, $model, $maxTokens, $reportProgress); + $this->openAiAPIService->updateExpTextProcessingTime(time() - $startTime); + return $result; + } + + if ($prompt === '') { + // Callers leave the prompt empty when they attach a file. Getting here means the + // attachment never arrived, so fail loudly instead of summarizing nothing. + throw new ProcessingException('Nothing to summarize: no text and no readable attachment was provided'); + } + $prompts = $this->chunkService->chunkSplitPrompt($prompt); $newNumChunks = count($prompts); $progress = 0.0; @@ -153,24 +199,7 @@ public function process(?string $userId, array $input, callable $reportProgress) try { $completions = []; - $summarySystemPrompt = 'You are a helpful assistant that summarizes text in the same language as the text. ' - . 'You should only return the summary without any additional information. '; - if (isset($input['format'])) { - if ($input['format'] === 'paragraph') { - $summarySystemPrompt .= 'Return the summary as a paragraph. '; - } elseif ($input['format'] === 'bullet_points') { - $summarySystemPrompt .= 'Return the summary as a list of bullet points. '; - } elseif ($input['format'] === 'sentence') { - $summarySystemPrompt .= 'Return the summary as a single sentence. Do not include more than one sentence. '; - } - } - if (isset($input['complexity'])) { - if ($input['complexity'] === 'complex') { - $summarySystemPrompt .= 'Use complex language and vocabulary appropriate for an expert in the subject. '; - } elseif ($input['complexity'] === 'simple') { - $summarySystemPrompt .= 'Use simple language and vocabulary appropriate for a 5 year old. '; - } - } + $summarySystemPrompt = $this->buildSummarySystemPrompt($input); if ($this->openAiAPIService->isUsingOpenAi() || $this->openAiSettingsService->getChatEndpointEnabled()) { foreach ($prompts as $p) { @@ -223,4 +252,93 @@ public function process(?string $userId, array $input, callable $reportProgress) return ['output' => $summary]; } + /** + * @param array $input + */ + private function buildSummarySystemPrompt(array $input): string { + $summarySystemPrompt = 'You are a helpful assistant that summarizes text in the same language as the text. ' + . 'You should only return the summary without any additional information. '; + if (isset($input['format'])) { + if ($input['format'] === 'paragraph') { + $summarySystemPrompt .= 'Return the summary as a paragraph. '; + } elseif ($input['format'] === 'bullet_points') { + $summarySystemPrompt .= 'Return the summary as a list of bullet points. '; + } elseif ($input['format'] === 'sentence') { + $summarySystemPrompt .= 'Return the summary as a single sentence. Do not include more than one sentence. '; + } + } + if (isset($input['complexity'])) { + if ($input['complexity'] === 'complex') { + $summarySystemPrompt .= 'Use complex language and vocabulary appropriate for an expert in the subject. '; + } elseif ($input['complexity'] === 'simple') { + $summarySystemPrompt .= 'Use simple language and vocabulary appropriate for a 5 year old. '; + } + } + return $summarySystemPrompt; + } + + /** + * Summarize attached files by handing them to the model directly. + * + * Nothing is chunked here: the whole point is that we never read the file + * ourselves, so we have no text to split. The document is inlined in the + * request, which means this only works over the chat completion endpoint. + * + * @param list $files + * @param array $input + * @return array{output: string} + * @throws ProcessingException + * @throws UserFacingProcessingException + */ + private function summarizeAttachments( + ?string $userId, + array $files, + array $input, + string $model, + ?int $maxTokens, + callable $reportProgress, + ): array { + if (!$this->openAiAPIService->isUsingOpenAi() && !$this->openAiSettingsService->getChatEndpointEnabled()) { + throw new UserFacingProcessingException( + 'Summarizing attachments requires the chat completion endpoint', + 0, + null, + $this->l->t('Summarizing files requires a service that supports the chat completion endpoint.'), + ); + } + if (count($files) > self::MAX_INPUT_ATTACHMENTS) { + throw new UserFacingProcessingException( + 'Too many files. Max is ' . self::MAX_INPUT_ATTACHMENTS, + 0, + null, + $this->l->t('Too many files given. A maximum of %d files is allowed.', [self::MAX_INPUT_ATTACHMENTS]), + ); + } + + $userPrompt = isset($input['input']) && is_string($input['input']) && $input['input'] !== '' + ? $input['input'] + : null; + + if (!$reportProgress(0.0)) { + throw new ProcessingException('OpenAI/LocalAI task cancelled'); + } + + try { + $completion = $this->openAiAPIService->createChatCompletion( + $userId, $model, $userPrompt, $this->buildSummarySystemPrompt($input), null, 1, $maxTokens, + null, null, null, $files, + ); + } catch (UserFacingProcessingException $e) { + throw $e; + } catch (\Throwable $e) { + throw new ProcessingException('OpenAI/LocalAI request failed: ' . $e->getMessage()); + } + + $summary = array_pop($completion['messages']); + if (!is_string($summary) || $summary === '') { + throw new ProcessingException('No result in OpenAI/LocalAI response.'); + } + return ['output' => $summary]; + } + } diff --git a/tests/unit/Providers/OpenAiProviderTest.php b/tests/unit/Providers/OpenAiProviderTest.php index d739eb65..402ba5c7 100644 --- a/tests/unit/Providers/OpenAiProviderTest.php +++ b/tests/unit/Providers/OpenAiProviderTest.php @@ -1216,4 +1216,176 @@ public function testMultimodalChatWithToolsProvider(): void { $this->quotaUsageMapper->deleteUserQuotaUsages(self::TEST_USER1); } + private function mockPdfFile(string $content, string $name = 'report.pdf'): \OCP\Files\File&MockObject { + $stream = fopen('php://temp', 'r+'); + if ($stream === false) { + throw new \RuntimeException('Could not open temp stream'); + } + fwrite($stream, $content); + rewind($stream); + + $file = $this->createMock(\OCP\Files\File::class); + $file->method('isReadable')->willReturn(true); + $file->method('getSize')->willReturn(strlen($content)); + $file->method('getMimeType')->willReturn('application/pdf'); + $file->method('getName')->willReturn($name); + $file->method('fopen')->with('rb')->willReturn($stream); + return $file; + } + + private const SUMMARY_SYSTEM_PROMPT = 'You are a helpful assistant that summarizes text in the same language as the text. ' + . 'You should only return the summary without any additional information. '; + + private const CHAT_COMPLETION_RESPONSE = '{ + "id": "chatcmpl-123", + "object": "chat.completion", + "created": 1677652288, + "model": "gpt-4.1-mini", + "choices": [ + { + "index": 0, + "message": { + "role": "assistant", + "content": "This is a test response." + }, + "finish_reason": "stop" + } + ], + "usage": { + "prompt_tokens": 9, + "completion_tokens": 12, + "total_tokens": 21 + } + }'; + + private function buildSummaryProvider(): SummaryProvider { + return new SummaryProvider( + $this->openAiApiService, + $this->openAiSettingsService, + $this->createMock(\OCP\IL10N::class), + $this->chunkService, + self::TEST_USER1, + ); + } + + private function mockChatCompletionResponse(): \OCP\Http\Client\IResponse&MockObject { + $iResponse = $this->createMock(\OCP\Http\Client\IResponse::class); + $iResponse->method('getBody')->willReturn(self::CHAT_COMPLETION_RESPONSE); + $iResponse->method('getStatusCode')->willReturn(200); + $iResponse->method('getHeader')->with('Content-Type')->willReturn('application/json'); + return $iResponse; + } + + /** + * A PDF is inlined as an OpenAI "file" content part, and no text is chunked: + * the whole point is that we never read the document ourselves. + */ + public function testSummaryProviderWithPdfAttachment(): void { + $provider = $this->buildSummaryProvider(); + $pdfContent = 'fake-pdf-bytes'; + $file = $this->mockPdfFile($pdfContent); + + $url = self::OPENAI_API_BASE . 'chat/completions'; + $options = ['timeout' => Application::OPENAI_DEFAULT_REQUEST_TIMEOUT, 'headers' => ['User-Agent' => Application::USER_AGENT, 'Authorization' => self::AUTHORIZATION_HEADER, 'Content-Type' => 'application/json']]; + $options['body'] = json_encode([ + 'model' => Application::DEFAULT_COMPLETION_MODEL_ID, + 'messages' => [ + ['role' => 'system', 'content' => self::SUMMARY_SYSTEM_PROMPT], + [ + 'role' => 'user', + 'content' => [ + [ + 'type' => 'file', + 'file' => [ + 'filename' => 'report.pdf', + 'file_data' => 'data:application/pdf;base64,' . base64_encode($pdfContent), + ], + ], + ], + ], + ], + 'n' => 1, + 'stream' => false, + 'max_completion_tokens' => Application::DEFAULT_MAX_NUM_OF_TOKENS, + 'user' => self::TEST_USER1, + ]); + + $this->iClient->expects($this->once())->method('post')->with($url, $options) + ->willReturn($this->mockChatCompletionResponse()); + + $result = $provider->process(self::TEST_USER1, [ + // the caller leaves the text input empty, the file is the whole request + 'input' => '', + 'input_attachments' => [$file], + ], fn () => true); + + $this->assertEquals('This is a test response.', $result['output']); + $this->quotaUsageMapper->deleteUserQuotaUsages(self::TEST_USER1); + } + + /** + * Mistral speaks the OpenAI chat completion API but rejects the OpenAI "file" + * part with a 422, so documents have to go out as a flat document_url string. + */ + public function testSummaryProviderWithPdfAttachmentOnMistral(): void { + $previousServiceUrl = $this->openAiSettingsService->getServiceUrl(); + $this->openAiSettingsService->setServiceUrl(Application::MISTRAL_API_BASE_URL_PREFIX . '/v1'); + + try { + $provider = $this->buildSummaryProvider(); + $pdfContent = 'fake-pdf-bytes'; + $file = $this->mockPdfFile($pdfContent); + + $url = Application::MISTRAL_API_BASE_URL_PREFIX . '/v1/chat/completions'; + $options = ['timeout' => Application::OPENAI_DEFAULT_REQUEST_TIMEOUT, 'headers' => ['User-Agent' => Application::USER_AGENT, 'Authorization' => self::AUTHORIZATION_HEADER, 'Content-Type' => 'application/json']]; + $options['body'] = json_encode([ + 'model' => Application::DEFAULT_COMPLETION_MODEL_ID, + 'messages' => [ + ['role' => 'system', 'content' => self::SUMMARY_SYSTEM_PROMPT], + [ + 'role' => 'user', + 'content' => [ + [ + 'type' => 'document_url', + 'document_url' => 'data:application/pdf;base64,' . base64_encode($pdfContent), + 'document_name' => 'report.pdf', + ], + ], + ], + ], + 'n' => 1, + 'stream' => false, + 'max_completion_tokens' => Application::DEFAULT_MAX_NUM_OF_TOKENS, + ]); + + $this->iClient->expects($this->once())->method('post')->with($url, $options) + ->willReturn($this->mockChatCompletionResponse()); + + $result = $provider->process(self::TEST_USER1, [ + 'input' => '', + 'input_attachments' => [$file], + ], fn () => true); + + $this->assertEquals('This is a test response.', $result['output']); + $this->quotaUsageMapper->deleteUserQuotaUsages(self::TEST_USER1); + } finally { + $this->openAiSettingsService->setServiceUrl($previousServiceUrl); + } + } + + public function testSummaryProviderAdvertisesAttachmentsOnlyWhenDocumentsAreEnabled(): void { + $provider = $this->buildSummaryProvider(); + $previous = $this->openAiSettingsService->getMultimodalDocumentEnabled(); + + try { + $this->openAiSettingsService->setMultimodalDocumentEnabled(true); + $this->assertArrayHasKey('input_attachments', $provider->getOptionalInputShape()); + + $this->openAiSettingsService->setMultimodalDocumentEnabled(false); + $this->assertArrayNotHasKey('input_attachments', $provider->getOptionalInputShape()); + } finally { + $this->openAiSettingsService->setMultimodalDocumentEnabled($previous); + } + } + }