diff --git a/lib/AppInfo/Application.php b/lib/AppInfo/Application.php index 27be6312..a1a87397 100644 --- a/lib/AppInfo/Application.php +++ b/lib/AppInfo/Application.php @@ -19,6 +19,8 @@ use OCA\OpenAi\TaskProcessing\ContextWriteProvider; use OCA\OpenAi\TaskProcessing\EmojiProvider; use OCA\OpenAi\TaskProcessing\HeadlineProvider; +use OCA\OpenAi\TaskProcessing\ImageToImageProvider; +use OCA\OpenAi\TaskProcessing\ImageToImageTaskType; use OCA\OpenAi\TaskProcessing\ReformulateProvider; use OCA\OpenAi\TaskProcessing\SummaryProvider; use OCA\OpenAi\TaskProcessing\TextToImageImprovedPromptProvider; @@ -157,6 +159,10 @@ public function register(IRegistrationContext $context): void { if ($this->appConfig->getValueString(Application::APP_ID, 't2i_provider_enabled', '1') === '1') { $context->registerTaskProcessingProvider(TextToImageProvider::class); $context->registerTaskProcessingProvider(TextToImageImprovedPromptProvider::class); + if (!class_exists('OCP\\TaskProcessing\\TaskTypes\\ImageToImage')) { + $context->registerTaskProcessingTaskType(ImageToImageTaskType::class); + } + $context->registerTaskProcessingProvider(ImageToImageProvider::class); } // only register audio chat stuff if we're using OpenAI or stt+llm+tts are enabled diff --git a/lib/Service/OpenAiAPIService.php b/lib/Service/OpenAiAPIService.php index ac853808..739b17b4 100644 --- a/lib/Service/OpenAiAPIService.php +++ b/lib/Service/OpenAiAPIService.php @@ -113,6 +113,26 @@ public function isUsingOpenRouter(?string $serviceType = null): bool { return str_starts_with(strtolower($serviceUrl), 'https://openrouter.ai'); } + /** + * @param ?string $serviceType + * @return bool + */ + public function isUsingIonos(?string $serviceType = null): bool { + $serviceUrl = ''; + if ($serviceType === Application::SERVICE_TYPE_IMAGE) { + $serviceUrl = $this->openAiSettingsService->getImageServiceUrl(); + } elseif ($serviceType === Application::SERVICE_TYPE_STT) { + $serviceUrl = $this->openAiSettingsService->getSttServiceUrl(); + } elseif ($serviceType === Application::SERVICE_TYPE_TTS) { + $serviceUrl = $this->openAiSettingsService->getTtsServiceUrl(); + } + if ($serviceUrl === '') { + $serviceUrl = $this->openAiSettingsService->getServiceUrl(); + } + // Return true if the service URL references IONOS + return (bool)preg_match('#^https://([a-zA-Z0-9-]+\.)+ionos\.com#', strtolower($serviceUrl)); + } + /** * @param ?string $serviceType * @@ -1073,6 +1093,193 @@ public function requestImageCreation( return $apiResponse; } + /** + * @param string|null $userId + * @param string $prompt + * @param list $images + * @param string $model + * @param string $size + * @return array + * @throws Exception + * @throws UserFacingProcessingException + */ + public function requestImageEdit( + ?string $userId, + string $prompt, + array $images, + string $model, + string $size = Application::DEFAULT_DEFAULT_IMAGE_SIZE, + ): array { + + if ($this->isQuotaExceeded($userId, Application::QUOTA_TYPE_IMAGE)) { + throw new Exception($this->l10n->t('Image generation quota exceeded'), Http::STATUS_TOO_MANY_REQUESTS); + } + + $model = $model === Application::DEFAULT_MODEL_ID ? Application::DEFAULT_IMAGE_MODEL_ID : $model; + + if ($this->isUsingOpenAi(Application::SERVICE_TYPE_IMAGE)) { + $apiResponse = $this->requestOpenAiImageEdit($userId, $prompt, $images, $model, $size); + } elseif ($this->isUsingOpenRouter(Application::SERVICE_TYPE_IMAGE)) { + $apiResponse = $this->requestOpenRouterImageEdit($userId, $prompt, $images, $model, $size); + } elseif ($this->isUsingIonos(Application::SERVICE_TYPE_IMAGE)) { + $apiResponse = $this->requestIonosImageEdit($userId, $prompt, $images, $model, $size); + } else { + $apiResponse = $this->requestLocalAiImageEdit($userId, $prompt, $images, $model, $size); + } + + if (!isset($apiResponse['data']) || !is_array($apiResponse['data'])) { + $this->logger->warning('OpenAI image edit error', ['api_response' => $apiResponse]); + throw new Exception($this->l10n->t('Unknown image generation error'), Http::STATUS_INTERNAL_SERVER_ERROR); + } + + try { + $this->createQuotaUsage($userId ?? '', Application::QUOTA_TYPE_IMAGE, 1); + } catch (DBException $e) { + $this->logger->warning('Could not create quota usage for user: ' . $userId . ' and quota type: ' . Application::QUOTA_TYPE_IMAGE . '. Error: ' . $e->getMessage(), ['app' => Application::APP_ID]); + } + + return $apiResponse; + } + + /** + * @param list $images + * @return array + * @throws Exception + */ + private function requestOpenAiImageEdit( + ?string $userId, + string $prompt, + array $images, + string $model, + string $size, + ): array { + $params = [ + 'prompt' => $prompt, + 'size' => $size, + 'n' => 1, + 'model' => $model, + ]; + foreach ($images as $index => $image) { + $mimeType = $image['mimeType']; + $extension = match ($mimeType) { + 'image/jpeg' => 'jpg', + 'image/webp' => 'webp', + 'image/gif' => 'gif', + default => 'png', + }; + $name = 'image_' . ($index); + $params[$name] = [ + 'name' => 'image[]', + 'contents' => $image['content'], + 'filename' => $name . '.' . $extension, + 'headers' => [ + 'Content-Type' => $mimeType, + ], + ]; + } + + return $this->request($userId, 'images/edits', $params, 'POST', 'multipart/form-data', serviceType: Application::SERVICE_TYPE_IMAGE); + } + + /** + * @param list $images + * @return array + * @throws Exception + * @throws UserFacingProcessingException + */ + private function requestIonosImageEdit( + ?string $userId, + string $prompt, + array $images, + string $model, + string $size, + ): array { + if (count($images) > 1) { + throw new UserFacingProcessingException( + 'IONOS image editing supports only one input image', + 0, + null, + $this->l10n->t('Only one input image is supported.'), + ); + } + + $image = $images[0]; + $params = [ + 'prompt' => $prompt, + 'size' => $size, + 'n' => 1, + 'model' => $model, + 'url' => 'data:' . $image['mimeType'] . ';base64,' . base64_encode($image['content']), + ]; + + return $this->request($userId, 'images/edits', $params, 'POST', 'multipart/form-data', serviceType: Application::SERVICE_TYPE_IMAGE); + } + + /** + * OpenRouter image edit path using the unified /images API with input_references. + * + * @param list $images + * @return array + * @throws Exception + */ + private function requestOpenRouterImageEdit( + ?string $userId, + string $prompt, + array $images, + string $model, + string $size, + ): array { + $inputReferences = []; + foreach ($images as $image) { + $inputReferences[] = [ + 'type' => 'image_url', + 'image_url' => [ + 'url' => 'data:' . $image['mimeType'] . ';base64,' . base64_encode($image['content']), + ], + ]; + } + + $params = [ + 'prompt' => $prompt, + 'size' => $size, + 'n' => 1, + 'model' => $model, + 'input_references' => $inputReferences, + ]; + + return $this->request($userId, 'images', $params, 'POST', serviceType: Application::SERVICE_TYPE_IMAGE); + } + + /** + * LocalAI and other OpenAI-compatible image edit path via /images/generations. + * + * @param list $images + * @return array + * @throws Exception + */ + private function requestLocalAiImageEdit( + ?string $userId, + string $prompt, + array $images, + string $model, + string $size, + ): array { + $refImages = []; + foreach ($images as $image) { + $refImages[] = base64_encode($image['content']); + } + + $params = [ + 'prompt' => $prompt, + 'size' => $size, + 'n' => 1, + 'model' => $model, + 'ref_images' => $refImages, + ]; + + return $this->request($userId, 'images/generations', $params, 'POST', serviceType: Application::SERVICE_TYPE_IMAGE); + } + /** * @param string|null $userId * @return array @@ -1283,12 +1490,16 @@ public function request( if ($contentType === 'multipart/form-data') { $multipart = []; foreach ($params as $key => $value) { - $part = [ - 'name' => $key, - 'contents' => $value, - ]; - if ($key === 'file') { - $part['filename'] = 'file.mp3'; + if (is_array($value) && array_key_exists('contents', $value)) { + $part = $value; + } else { + $part = [ + 'name' => $key, + 'contents' => $value, + ]; + if ($key === 'file') { + $part['filename'] = 'file.mp3'; + } } $multipart[] = $part; } diff --git a/lib/TaskProcessing/ImageToImageProvider.php b/lib/TaskProcessing/ImageToImageProvider.php new file mode 100644 index 00000000..bb6a5930 --- /dev/null +++ b/lib/TaskProcessing/ImageToImageProvider.php @@ -0,0 +1,239 @@ +openAiAPIService->getServiceName(Application::SERVICE_TYPE_IMAGE); + } + + public function getTaskTypeId(): string { + if (class_exists('OCP\\TaskProcessing\\TaskTypes\\ImageToImage')) { + return \OCP\TaskProcessing\TaskTypes\ImageToImage::ID; + } + return ImageToImageTaskType::ID; + } + + public function getExpectedRuntime(): int { + return $this->openAiAPIService->getExpImgProcessingTime(); + } + + public function getInputShapeEnumValues(): array { + return []; + } + + public function getInputShapeDefaults(): array { + return []; + } + + public function getOptionalInputShape(): array { + $defaultImageSize = $this->appConfig->getValueString(Application::APP_ID, 'default_image_size', lazy: true) ?: Application::DEFAULT_DEFAULT_IMAGE_SIZE; + return [ + 'size' => new ShapeDescriptor( + $this->l->t('Size'), + $this->l->t('Optional. The size of the generated images. Must be in 256x256 format. Default is %s', [$defaultImageSize]), + EShapeType::Text + ), + 'model' => new ShapeDescriptor( + $this->l->t('Model'), + $this->l->t('The model used to generate the images'), + EShapeType::Enum + ), + ]; + } + + public function getOptionalInputShapeEnumValues(): array { + return [ + 'model' => $this->openAiAPIService->getModelEnumValues($this->userId, serviceType: Application::SERVICE_TYPE_IMAGE), + ]; + } + + public function getOptionalInputShapeDefaults(): array { + $adminModel = $this->openAiAPIService->isUsingOpenAi(Application::SERVICE_TYPE_IMAGE) + ? ($this->appConfig->getValueString(Application::APP_ID, 'default_image_model_id', Application::DEFAULT_MODEL_ID, lazy: true) ?: Application::DEFAULT_MODEL_ID) + : $this->appConfig->getValueString(Application::APP_ID, 'default_image_model_id', lazy: true); + return [ + 'model' => $adminModel, + ]; + } + + public function getOutputShapeEnumValues(): array { + return []; + } + + public function getOptionalOutputShape(): array { + return []; + } + + public function getOptionalOutputShapeEnumValues(): array { + return []; + } + + public function process( + ?string $userId, + array $input, + callable $reportProgress, + SynchronousProviderOptions $options = new SynchronousProviderOptions(), + ): array { + $startTime = time(); + $includeWatermark = $options->getIncludeWatermarks(); + + if (!isset($input['input']) || !is_array($input['input']) || $input['input'] === []) { + throw new ProcessingException('Invalid input files'); + } + + if (!isset($input['prompt']) || !is_string($input['prompt'])) { + throw new ProcessingException('Invalid prompt'); + } + $prompt = $input['prompt']; + + $images = []; + + if (count($input['input']) > 16) { + throw new UserFacingProcessingException( + 'Too many input images. Max is 16', + 0, + null, + $this->l->t('Cannot use more than 16 input images.'), + ); + } + + foreach ($input['input'] as $inputFile) { + if (!$inputFile instanceof File || !$inputFile->isReadable()) { + throw new ProcessingException('Invalid input file'); + } + if ($inputFile->getSize() > self::MAX_FILE_SIZE_BYTES) { + throw new UserFacingProcessingException( + 'Filesize of input file too large. Max is 25MB', + 0, + null, + $this->l->t('The size of the input file is too large. A maximum of 25MB is allowed.'), + ); + } + + $mimeType = $inputFile->getMimeType(); + if (!in_array($mimeType, self::VALID_IMAGE_MIME_TYPES, true)) { + throw new UserFacingProcessingException( + 'Invalid input file type for OpenAI ' . $mimeType, + 0, + null, + $this->l->t('Invalid input file type "%1$s".', [$mimeType]), + ); + } + + $images[] = [ + 'content' => $inputFile->getContent(), + 'mimeType' => $mimeType, + ]; + } + + $size = $this->appConfig->getValueString(Application::APP_ID, 'default_image_size', lazy: true) ?: Application::DEFAULT_DEFAULT_IMAGE_SIZE; + if (isset($input['size']) && is_string($input['size']) && preg_match('/^\d+x\d+$/', $input['size'])) { + $size = trim($input['size']); + } + [$x, $y] = explode('x', $size, 2); + if ((int)$x > 4096 || (int)$y > 4096) { + throw new UserFacingProcessingException('size is out of bounds', userFacingMessage: $this->l->t('Cannot generate images larger than 4096x4096')); + } + + if (isset($input['model']) && is_string($input['model'])) { + $model = $input['model']; + } else { + $model = $this->appConfig->getValueString(Application::APP_ID, 'default_image_model_id', Application::DEFAULT_MODEL_ID, lazy: true) ?: Application::DEFAULT_MODEL_ID; + } + + try { + $apiResponse = $this->openAiAPIService->requestImageEdit( + $userId, + $prompt, + $images, + $model, + $size, + ); + $b64s = array_map(static function (array $result) { + return $result['b64_json'] ?? null; + }, $apiResponse['data']); + $b64s = array_values(array_filter($b64s, static function (?string $b64) { + return $b64 !== null; + })); + + $urls = array_map(static function (array $result) { + return $result['url'] ?? null; + }, $apiResponse['data']); + $urls = array_values(array_filter($urls, static function (?string $url) { + return $url !== null; + })); + + if (empty($urls) && empty($b64s)) { + $this->logger->warning('OpenAI/LocalAI\'s image to image generation failed: no image returned'); + throw new ProcessingException('OpenAI/LocalAI\'s image to image generation failed: no image returned'); + } + + $image = null; + if (!empty($urls)) { + $client = $this->clientService->newClient(); + $requestOptions = $this->openAiAPIService->getImageRequestOptions($userId); + $imageResponse = $client->get($urls[0], $requestOptions); + $image = $imageResponse->getBody(); + } else { + $image = base64_decode($b64s[0]); + } + + $image = $includeWatermark ? $this->watermarkingService->markImage($image) : $image; + $endTime = time(); + $this->openAiAPIService->updateExpImgProcessingTime($endTime - $startTime); + return ['output' => $image]; + } catch (UserFacingProcessingException $e) { + throw $e; + } catch (\Throwable $e) { + $this->logger->warning('OpenAI/LocalAI\'s image to image generation failed with: ' . $e->getMessage(), ['exception' => $e]); + throw new ProcessingException('OpenAI/LocalAI\'s image to image generation failed with: ' . $e->getMessage()); + } + } +} diff --git a/lib/TaskProcessing/ImageToImageTaskType.php b/lib/TaskProcessing/ImageToImageTaskType.php new file mode 100644 index 00000000..c19a8651 --- /dev/null +++ b/lib/TaskProcessing/ImageToImageTaskType.php @@ -0,0 +1,77 @@ +l->t('Edit image'); + } + + /** + * @inheritDoc + */ + public function getDescription(): string { + return $this->l->t('Edit an image based on a text description of the changes'); + } + + /** + * @return string + */ + public function getId(): string { + return self::ID; + } + + /** + * @return ShapeDescriptor[] + */ + public function getInputShape(): array { + return [ + 'input' => new ShapeDescriptor( + $this->l->t('Input images'), + $this->l->t('The images to edit'), + EShapeType::ListOfImages + ), + 'prompt' => new ShapeDescriptor( + $this->l->t('Prompt'), + $this->l->t('Describe the changes you want to make to the image'), + EShapeType::Text + ), + ]; + } + + /** + * @return ShapeDescriptor[] + */ + public function getOutputShape(): array { + return [ + 'output' => new ShapeDescriptor( + $this->l->t('Output image'), + $this->l->t('The edited image'), + EShapeType::Image + ), + ]; + } +} diff --git a/tests/unit/Providers/OpenAiProviderTest.php b/tests/unit/Providers/OpenAiProviderTest.php index d739eb65..65946337 100644 --- a/tests/unit/Providers/OpenAiProviderTest.php +++ b/tests/unit/Providers/OpenAiProviderTest.php @@ -26,6 +26,7 @@ use OCA\OpenAi\TaskProcessing\ChangeToneProvider; use OCA\OpenAi\TaskProcessing\EmojiProvider; use OCA\OpenAi\TaskProcessing\HeadlineProvider; +use OCA\OpenAi\TaskProcessing\ImageToImageProvider; use OCA\OpenAi\TaskProcessing\MultimodalChatWithToolsProvider; use OCA\OpenAi\TaskProcessing\ProofreadProvider; use OCA\OpenAi\TaskProcessing\ReformatParagraphsProvider; @@ -937,6 +938,78 @@ public function testTextToImageProvider(): void { $this->quotaUsageMapper->deleteUserQuotaUsages(self::TEST_USER1); } + public function testImageToImageProvider(): void { + $imageToImageProvider = new ImageToImageProvider( + $this->openAiApiService, + $this->createMock(\OCP\IL10N::class), + $this->createMock(\Psr\Log\LoggerInterface::class), + \OCP\Server::get(IClientService::class), + \OCP\Server::get(IAppConfig::class), + self::TEST_USER1, + \OCP\Server::get(WatermarkingService::class), + ); + + $inputImage = file_get_contents(__DIR__ . '/../../res/trees.jpg'); + if (!$inputImage) { + throw new \RuntimeException('Could not read test resource `trees.jpg`'); + } + + $file = $this->createMock(\OCP\Files\File::class); + $file->method('isReadable')->willReturn(true); + $file->method('getContent')->willReturn($inputImage); + $file->method('getSize')->willReturn(strlen($inputImage)); + $file->method('getMimeType')->willReturn('image/jpeg'); + + $prompt = 'Make the sky blue'; + $response = json_encode([ + 'data' => [ + [ + 'b64_json' => base64_encode($inputImage), + ] + ] + ]); + + $url = self::OPENAI_API_BASE . 'images/edits'; + $options = [ + 'timeout' => Application::OPENAI_DEFAULT_REQUEST_TIMEOUT, + 'headers' => [ + 'User-Agent' => Application::USER_AGENT, + 'Authorization' => self::AUTHORIZATION_HEADER, + ], + 'multipart' => [ + ['name' => 'prompt', 'contents' => $prompt], + ['name' => 'size', 'contents' => '1024x1024'], + ['name' => 'n', 'contents' => 1], + ['name' => 'model', 'contents' => Application::DEFAULT_IMAGE_MODEL_ID], + [ + 'name' => 'image[]', + 'contents' => $inputImage, + 'filename' => 'image_0.jpg', + 'headers' => ['Content-Type' => 'image/jpeg'], + ], + ], + ]; + + $iResponse = $this->createMock(\OCP\Http\Client\IResponse::class); + $iResponse->method('getHeader')->with('Content-Type')->willReturn('application/json'); + $iResponse->method('getBody')->willReturn($response); + $iResponse->method('getStatusCode')->willReturn(200); + + $this->iClient->expects($this->once())->method('post')->with($url, $options)->willReturn($iResponse); + + $result = $imageToImageProvider->process( + self::TEST_USER1, + ['input' => [$file], 'prompt' => $prompt], + fn () => true, + ); + $this->assertArrayHasKey('output', $result); + $this->assertEquals($inputImage, $result['output']); + + $usage = $this->quotaUsageMapper->getQuotaUnitsOfUser(self::TEST_USER1, Application::QUOTA_TYPE_IMAGE); + $this->assertEquals(1, $usage); + $this->quotaUsageMapper->deleteUserQuotaUsages(self::TEST_USER1); + } + public function testReformatParagraphsProvider(): void { if (!class_exists(TextToTextReformatParagraphs::class)) { $this->markTestSkipped('TextToTextReformatParagraphs task type is not available in this Nextcloud version.'); diff --git a/tests/unit/Service/ServiceOverrideTest.php b/tests/unit/Service/ServiceOverrideTest.php index a7d8cfa7..a0d19b55 100644 --- a/tests/unit/Service/ServiceOverrideTest.php +++ b/tests/unit/Service/ServiceOverrideTest.php @@ -22,6 +22,7 @@ use OCA\OpenAi\Service\StreamingService; use OCA\OpenAi\Service\WatermarkingService; use OCA\OpenAi\TaskProcessing\AudioToTextProvider; +use OCA\OpenAi\TaskProcessing\ImageToImageProvider; use OCA\OpenAi\TaskProcessing\TextToImageProvider; use OCA\OpenAi\TaskProcessing\TextToSpeechProvider; use OCP\Http\Client\IClient; @@ -218,6 +219,148 @@ public function testTextToImageProvider(): void { $TextToImageProvider->process(self::TEST_USER1, ['input' => $inputText, 'numberOfImages' => 1], fn () => null); } + public function testImageToImageProvider(): void { + $this->openAiSettingsService->setImageServiceUrl(self::OVERRIDE_IMAGE_BASE); + $this->openAiSettingsService->setAdminImageApiKey(self::APIKEY_IMAGE); + $this->openAiSettingsService->setImageRequestTimeout(self::REQUEST_TIMEOUT_IMAGE); + + $imageToImageProvider = new ImageToImageProvider( + $this->openAiApiService, + $this->createMock(\OCP\IL10N::class), + $this->createMock(\Psr\Log\LoggerInterface::class), + \OCP\Server::get(IClientService::class), + \OCP\Server::get(IAppConfig::class), + self::TEST_USER1, + \OCP\Server::get(WatermarkingService::class), + ); + + $inputImage = file_get_contents(__DIR__ . '/../../res/trees.jpg'); + if (!$inputImage) { + throw new \RuntimeException('Could not read test resource `trees.jpg`'); + } + + $file = $this->createMock(\OCP\Files\File::class); + $file->method('isReadable')->willReturn(true); + $file->method('getContent')->willReturn($inputImage); + $file->method('getSize')->willReturn(strlen($inputImage)); + $file->method('getMimeType')->willReturn('image/jpeg'); + + $prompt = 'Make the sky blue'; + $response = json_encode([ + 'data' => [ + [ + 'b64_json' => base64_encode($inputImage), + ] + ] + ]); + + $url = self::OVERRIDE_IMAGE_BASE . 'images/generations'; + $options = [ + 'timeout' => self::REQUEST_TIMEOUT_IMAGE, + 'headers' => [ + 'User-Agent' => Application::USER_AGENT, + 'Authorization' => 'Bearer ' . self::APIKEY_IMAGE, + 'Content-Type' => 'application/json', + ], + 'nextcloud' => ['allow_local_address' => true], + 'body' => json_encode([ + 'prompt' => $prompt, + 'size' => '1024x1024', + 'n' => 1, + 'model' => Application::DEFAULT_IMAGE_MODEL_ID, + 'ref_images' => [base64_encode($inputImage)], + ]), + ]; + + $iResponse = $this->createMock(\OCP\Http\Client\IResponse::class); + $iResponse->method('getHeader')->with('Content-Type')->willReturn('application/json'); + $iResponse->method('getBody')->willReturn($response); + $iResponse->method('getStatusCode')->willReturn(200); + + $this->iClient->expects($this->once())->method('post')->with($url, $options)->willReturn($iResponse); + + $imageToImageProvider->process( + self::TEST_USER1, + ['input' => [$file], 'prompt' => $prompt], + fn () => null, + ); + } + + public function testImageToImageProviderOpenRouter(): void { + $openRouterBase = 'https://openrouter.ai/api/v1/'; + $this->openAiSettingsService->setImageServiceUrl($openRouterBase); + $this->openAiSettingsService->setAdminImageApiKey(self::APIKEY_IMAGE); + $this->openAiSettingsService->setImageRequestTimeout(self::REQUEST_TIMEOUT_IMAGE); + + $imageToImageProvider = new ImageToImageProvider( + $this->openAiApiService, + $this->createMock(\OCP\IL10N::class), + $this->createMock(\Psr\Log\LoggerInterface::class), + \OCP\Server::get(IClientService::class), + \OCP\Server::get(IAppConfig::class), + self::TEST_USER1, + \OCP\Server::get(WatermarkingService::class), + ); + + $inputImage = file_get_contents(__DIR__ . '/../../res/trees.jpg'); + if (!$inputImage) { + throw new \RuntimeException('Could not read test resource `trees.jpg`'); + } + + $file = $this->createMock(\OCP\Files\File::class); + $file->method('isReadable')->willReturn(true); + $file->method('getContent')->willReturn($inputImage); + $file->method('getSize')->willReturn(strlen($inputImage)); + $file->method('getMimeType')->willReturn('image/jpeg'); + + $prompt = 'Make the sky blue'; + $response = json_encode([ + 'data' => [ + [ + 'b64_json' => base64_encode($inputImage), + ] + ] + ]); + + $url = $openRouterBase . 'images'; + $options = [ + 'timeout' => self::REQUEST_TIMEOUT_IMAGE, + 'headers' => [ + 'User-Agent' => Application::USER_AGENT, + 'Authorization' => 'Bearer ' . self::APIKEY_IMAGE, + 'Content-Type' => 'application/json', + ], + 'nextcloud' => ['allow_local_address' => true], + 'body' => json_encode([ + 'prompt' => $prompt, + 'size' => '1024x1024', + 'n' => 1, + 'model' => Application::DEFAULT_IMAGE_MODEL_ID, + 'input_references' => [ + [ + 'type' => 'image_url', + 'image_url' => [ + 'url' => 'data:image/jpeg;base64,' . base64_encode($inputImage), + ], + ], + ], + ]), + ]; + + $iResponse = $this->createMock(\OCP\Http\Client\IResponse::class); + $iResponse->method('getHeader')->with('Content-Type')->willReturn('application/json'); + $iResponse->method('getBody')->willReturn($response); + $iResponse->method('getStatusCode')->willReturn(200); + + $this->iClient->expects($this->once())->method('post')->with($url, $options)->willReturn($iResponse); + + $imageToImageProvider->process( + self::TEST_USER1, + ['input' => [$file], 'prompt' => $prompt], + fn () => null, + ); + } + public function testAudioToTextProvider(): void { $this->openAiSettingsService->setSttServiceUrl(self::OVERRIDE_TRANSCRIPTION_BASE); $this->openAiSettingsService->setAdminSttApiKey(self::APIKEY_TRANSCRIPTION);