diff --git a/astrbot/core/config/default.py b/astrbot/core/config/default.py index 37eb480f39..7bc35a450e 100644 --- a/astrbot/core/config/default.py +++ b/astrbot/core/config/default.py @@ -2218,6 +2218,36 @@ "labels": ["文本", "图像", "音频", "工具使用"], "render_type": "checkbox", "hint": "模型支持的模态及能力。", + "default": ["text", "image", "audio", "tool_use"], + }, + "image_formats": { + "description": "图片格式支持", + "type": "list", + "items": {"type": "string"}, + "options": [ + "jpeg", + "png", + "webp", + "gif", + "bmp", + "heic", + "heif", + "*", + ], + "labels": [ + "JPEG", + "PNG", + "WebP", + "GIF", + "BMP", + "HEIC", + "HEIF", + "不限制", + ], + "render_type": "checkbox", + "hint": "发送给该提供商前允许的图片格式，不兼容的格式会自动转换。勾选「不限制」时其他选项不生效。", + "default": [], + "condition": {"modalities": "image"}, }, "custom_headers": { "description": "自定义请求头", diff --git a/astrbot/core/provider/entities.py b/astrbot/core/provider/entities.py index 2fab40ca78..12342e4931 100644 --- a/astrbot/core/provider/entities.py +++ b/astrbot/core/provider/entities.py @@ -23,7 +23,7 @@ from astrbot.core.agent.tool import ToolSet from astrbot.core.db.po import Conversation from astrbot.core.message.message_event_result import MessageChain -from astrbot.core.utils.media_utils import MediaResolver +from astrbot.core.utils.media_utils import MediaResolver, resolve_image_ref_to_images class ProviderType(enum.Enum): @@ -208,19 +208,17 @@ async def assemble_context(self) -> dict: # 3. 图片内容 if self.image_urls: for image_url in self.image_urls: - image_data = await MediaResolver( - image_url, - media_type="image", - ).to_base64_data() - if not image_data: + image_datas = await resolve_image_ref_to_images(image_url) + if not image_datas: logger.warning("图片预处理结果为空，将忽略。") continue - content_blocks.append( - { - "type": "image_url", - "image_url": {"url": image_data.to_data_url()}, - }, - ) + for image_data in image_datas: + content_blocks.append( + { + "type": "image_url", + "image_url": {"url": image_data.to_data_url()}, + }, + ) # 4. 音频内容 if self.audio_urls: diff --git a/astrbot/core/provider/provider.py b/astrbot/core/provider/provider.py index 891bfdea9e..ea034a5038 100644 --- a/astrbot/core/provider/provider.py +++ b/astrbot/core/provider/provider.py @@ -2,8 +2,9 @@ import asyncio import os from collections.abc import AsyncGenerator -from typing import Literal, TypeAlias, Union +from typing import ClassVar, Literal, TypeAlias, Union +from astrbot import logger from astrbot.core.agent.message import ContentPart, Message, is_checkpoint_message from astrbot.core.agent.tool import ToolSet from astrbot.core.provider.entities import ( @@ -14,6 +15,14 @@ ) from astrbot.core.provider.register import provider_cls_map from astrbot.core.utils.astrbot_path import get_astrbot_path +from astrbot.core.utils.media_utils import ( + IMAGE_COMPRESS_DEFAULT_MAX_SIZE, + IMAGE_SHORT_MIME_TYPES, + VENDOR_IMAGE_FORMATS, +) + +DEFAULT_FALLBACK_IMAGE_FORMATS = frozenset({"image/jpeg", "image/png"}) +"""Conservative image formats assumed for providers without a declared set.""" Providers: TypeAlias = Union[ "Provider", @@ -66,6 +75,14 @@ async def test(self) -> None: class Provider(AbstractProvider): """Chat Provider""" + supported_image_formats: ClassVar[frozenset[str] | None] = None + """Image MIME types the provider officially accepts. + + ``None`` means undeclared and falls back to DEFAULT_FALLBACK_IMAGE_FORMATS. + Aggregator subclasses (e.g. OpenRouter) must set this back to ``None`` + explicitly so they do not inherit an official vendor's format set. + """ + def __init__( self, provider_config: dict, @@ -74,6 +91,63 @@ def __init__( super().__init__(provider_config) self.provider_settings = provider_settings + def resolve_allowed_image_formats(self) -> frozenset[str] | None: + """Resolve the image MIME types allowed for this provider instance. + + Priority: ``provider_config["image_formats"]`` (per-instance override, + short names like ``jpeg`` or MIME types, ``*`` disables restriction) > + the vendor-level set keyed by ``provider_config["provider"]`` (brand of + the provider source, for branded sources running on generic adapters) > + the class-level ``supported_image_formats`` > the conservative + DEFAULT_FALLBACK_IMAGE_FORMATS (jpeg/png). + Returns: + The allowed MIME types, or ``None`` when unrestricted. + """ + configured = self.provider_config.get("image_formats") + if configured: + normalized = { + str(value).strip().lower() for value in configured if str(value).strip() + } + if "*" in normalized: + return None + mapped = { + IMAGE_SHORT_MIME_TYPES.get(value, value) + for value in normalized + if value.startswith("image/") or value in IMAGE_SHORT_MIME_TYPES + } + if mapped: + return frozenset(mapped) + logger.warning( + "Provider %s: image_formats %s contains no valid entries; " + "falling back to the default format set.", + self.provider_config.get("id"), + sorted(normalized), + ) + brand = self.provider_config.get("provider") + if isinstance(brand, str): + vendor_formats = VENDOR_IMAGE_FORMATS.get(brand.strip().lower()) + if vendor_formats is not None: + return vendor_formats + if self.supported_image_formats is not None: + return self.supported_image_formats + return DEFAULT_FALLBACK_IMAGE_FORMATS + + def get_animated_montage_max_size(self) -> int: + """Longest-edge cap for the animated-image montage, in pixels. + + Reuses ``provider_settings.image_compress_options.max_size`` so still + images and montages share one size control. + + Returns: + The configured cap, or the compress default when unset or invalid. + """ + options = self.provider_settings.get("image_compress_options") + raw = options.get("max_size") if isinstance(options, dict) else None + try: + return max(int(raw), 1) + except (TypeError, ValueError): + return IMAGE_COMPRESS_DEFAULT_MAX_SIZE + @abc.abstractmethod def get_current_key(self) -> str: raise NotImplementedError diff --git a/astrbot/core/provider/sources/anthropic_source.py b/astrbot/core/provider/sources/anthropic_source.py index c861ded6ba..54ad84de51 100644 --- a/astrbot/core/provider/sources/anthropic_source.py +++ b/astrbot/core/provider/sources/anthropic_source.py @@ -17,8 +17,10 @@ from astrbot.core.provider.entities import LLMResponse, TokenUsage from astrbot.core.provider.func_tool_manager import ToolSet from astrbot.core.utils.media_utils import ( + VENDOR_IMAGE_FORMATS, describe_media_ref, - resolve_media_ref_to_base64_data, + detect_image_mime_type, + resolve_image_ref_to_images, ) from astrbot.core.utils.network_utils import ( create_proxy_client, @@ -37,6 +39,9 @@ class ProviderAnthropic(Provider): _PROMPT_CACHE_CONTROL = {"type": "ephemeral"} + supported_image_formats = VENDOR_IMAGE_FORMATS["anthropic"] + """Formats accepted by the official Anthropic vision API.""" + @staticmethod def _ensure_usable_response( llm_response: LLMResponse, @@ -273,19 +278,32 @@ def _prepare_payload(self, messages: list[dict]): _, base64_data = url.split(",", 1) # Detect actual image format from binary data image_bytes = base64.b64decode(base64_data) - media_type = self._detect_image_mime_type( - image_bytes + media_type = detect_image_mime_type( + image_bytes, + default_mime_type=None, ) - converted_content.append( - { - "type": "image", - "source": { - "type": "base64", - "media_type": media_type, - "data": base64_data, - }, - } + allowed_formats = ( + self.resolve_allowed_image_formats() ) + if media_type and ( + allowed_formats is None + or media_type in allowed_formats + ): + converted_content.append( + { + "type": "image", + "source": { + "type": "base64", + "media_type": media_type, + "data": base64_data, + }, + } + ) + else: + logger.warning( + "Skipping context image with unsupported or undetectable format: %s...", + url[:50], + ) except ValueError: logger.warning( f"Failed to parse image data URI: {url[:50]}..." @@ -899,17 +917,15 @@ async def text_chat_stream( ): yield llm_response - def _detect_image_mime_type(self, data: bytes) -> str: - """根据图片二进制数据的 magic bytes 检测 MIME 类型""" - if data[:8] == b"\x89PNG\r\n\x1a\n": - return "image/png" - if data[:2] == b"\xff\xd8": - return "image/jpeg" - if data[:6] in (b"GIF87a", b"GIF89a"): - return "image/gif" - if data[:4] == b"RIFF" and data[8:12] == b"WEBP": - return "image/webp" - return "image/jpeg" + async def _image_ref_to_images(self, image_ref: str, *, strict: bool = False): + """Resolve an image ref with this provider's format adaptation applied.""" + montage_max_size = self.get_animated_montage_max_size() + return await resolve_image_ref_to_images( + image_ref, + allowed_mime_types=self.resolve_allowed_image_formats(), + montage_max_size=montage_max_size, + strict=strict, + ) async def assemble_context( self, @@ -920,23 +936,23 @@ async def assemble_context( ): """组装上下文，支持文本和图片""" - async def resolve_image_url(image_url: str) -> dict | None: - image_data = await resolve_media_ref_to_base64_data( - image_url, - media_type="image", - ) - if not image_data: + async def resolve_image_url(image_url: str) -> list[dict]: + image_datas = await self._image_ref_to_images(image_url) + if not image_datas: logger.warning("图片预处理结果为空，将忽略。") - return None - - return { - "type": "image", - "source": { - "type": "base64", - "media_type": image_data.mime_type, - "data": image_data.base64_data, - }, - } + return [] + + return [ + { + "type": "image", + "source": { + "type": "base64", + "media_type": image_data.mime_type, + "data": image_data.base64_data, + }, + } + for image_data in image_datas + ] content = [] @@ -958,9 +974,7 @@ async def resolve_image_url(image_url: str) -> dict | None: if isinstance(block, TextPart): content.append({"type": "text", "text": block.text}) elif isinstance(block, ImageURLPart): - image_dict = await resolve_image_url(block.image_url.url) - if image_dict: - content.append(image_dict) + content.extend(await resolve_image_url(block.image_url.url)) elif isinstance(block, AudioURLPart): content.append({"type": "text", "text": "[Audio]"}) else: @@ -969,9 +983,7 @@ async def resolve_image_url(image_url: str) -> dict | None: # 3. 图片内容 if image_urls: for image_url in image_urls: - image_dict = await resolve_image_url(image_url) - if image_dict: - content.append(image_dict) + content.extend(await resolve_image_url(image_url)) if audio_urls: for _audio_path in audio_urls: content.append({"type": "text", "text": "[Audio]"}) @@ -992,15 +1004,12 @@ async def resolve_image_url(image_url: str) -> dict | None: async def encode_image_bs64(self, image_url: str) -> tuple[str, str]: """将图片转换为 base64，同时检测实际 MIME 类型""" - image_data = await resolve_media_ref_to_base64_data( - image_url, - media_type="image", - strict=True, - ) - if image_data is None: + image_datas = await self._image_ref_to_images(image_url, strict=True) + if not image_datas: raise RuntimeError( f"Failed to encode image data: {describe_media_ref(image_url)}" ) + image_data = image_datas[0] return image_data.to_data_url(), image_data.mime_type def get_current_key(self) -> str: diff --git a/astrbot/core/provider/sources/gemini_source.py b/astrbot/core/provider/sources/gemini_source.py index d8ec6dbffc..0e697ab879 100644 --- a/astrbot/core/provider/sources/gemini_source.py +++ b/astrbot/core/provider/sources/gemini_source.py @@ -20,7 +20,9 @@ from astrbot.core.provider.entities import LLMResponse, TokenUsage from astrbot.core.provider.func_tool_manager import ToolSet from astrbot.core.utils.media_utils import ( + VENDOR_IMAGE_FORMATS, describe_media_ref, + resolve_image_ref_to_images, resolve_media_ref_to_base64_data, ) from astrbot.core.utils.network_utils import is_connection_error, log_connection_failure @@ -44,6 +46,9 @@ def filter(self, record): "Google Gemini Chat Completion 提供商适配器", ) class ProviderGoogleGenAI(Provider): + supported_image_formats = VENDOR_IMAGE_FORMATS["google"] + """Formats accepted by the official Gemini vision API.""" + CATEGORY_MAPPING = { "harassment": types.HarmCategory.HARM_CATEGORY_HARASSMENT, "hate_speech": types.HarmCategory.HARM_CATEGORY_HATE_SPEECH, @@ -995,18 +1000,23 @@ async def assemble_context( ): """组装上下文。""" - async def resolve_image_part(image_url: str) -> dict | None: - image_data = await resolve_media_ref_to_base64_data( + async def resolve_image_part(image_url: str) -> list[dict]: + montage_max_size = self.get_animated_montage_max_size() + image_datas = await resolve_image_ref_to_images( image_url, - media_type="image", + allowed_mime_types=self.resolve_allowed_image_formats(), + montage_max_size=montage_max_size, ) - if not image_data: + if not image_datas: logger.warning("Image preprocessing returned no data; ignoring it.") - return None - return { - "type": "image_url", - "image_url": {"url": image_data.to_data_url()}, - } + return [] + return [ + { + "type": "image_url", + "image_url": {"url": image_data.to_data_url()}, + } + for image_data in image_datas + ] async def resolve_audio_part(audio_path: str) -> dict | None: try: @@ -1050,9 +1060,8 @@ async def resolve_audio_part(audio_path: str) -> dict | None: if isinstance(part, TextPart): content_blocks.append({"type": "text", "text": part.text}) elif isinstance(part, ImageURLPart): - image_part = await resolve_image_part(part.image_url.url) - if image_part: - content_blocks.append(image_part) + image_parts = await resolve_image_part(part.image_url.url) + content_blocks.extend(image_parts) elif isinstance(part, AudioURLPart): audio_part = await resolve_audio_part(part.audio_url.url) if audio_part: @@ -1065,9 +1074,8 @@ async def resolve_audio_part(audio_path: str) -> dict | None: # 3. 图片内容 if image_urls: for image_url in image_urls: - image_part = await resolve_image_part(image_url) - if image_part: - content_blocks.append(image_part) + image_parts = await resolve_image_part(image_url) + content_blocks.extend(image_parts) if audio_urls: for audio_path in audio_urls: @@ -1091,16 +1099,18 @@ async def resolve_audio_part(audio_path: str) -> dict | None: async def encode_image_bs64(self, image_url: str) -> str: """将图片转换为 base64""" - image_data = await resolve_media_ref_to_base64_data( + montage_max_size = self.get_animated_montage_max_size() + image_datas = await resolve_image_ref_to_images( image_url, - media_type="image", + allowed_mime_types=self.resolve_allowed_image_formats(), + montage_max_size=montage_max_size, strict=True, ) - if image_data is None: + if not image_datas: raise RuntimeError( f"Failed to encode image data: {describe_media_ref(image_url)}" ) - return image_data.to_data_url() + return image_datas[0].to_data_url() async def _close_httpx_client(self, client: httpx.AsyncClient | None) -> None: """Safely close an httpx.AsyncClient, swallowing errors for idempotency.""" diff --git a/astrbot/core/provider/sources/groq_source.py b/astrbot/core/provider/sources/groq_source.py index af4029f67c..ff10699f6b 100644 --- a/astrbot/core/provider/sources/groq_source.py +++ b/astrbot/core/provider/sources/groq_source.py @@ -1,3 +1,5 @@ +from astrbot.core.utils.media_utils import VENDOR_IMAGE_FORMATS + from ..register import register_provider_adapter from .openai_source import ProviderOpenAIOfficial @@ -6,6 +8,8 @@ "groq_chat_completion", "Groq Chat Completion Provider Adapter" ) class ProviderGroq(ProviderOpenAIOfficial): + supported_image_formats = VENDOR_IMAGE_FORMATS["groq"] + def __init__( self, provider_config: dict, diff --git a/astrbot/core/provider/sources/kimi_code_source.py b/astrbot/core/provider/sources/kimi_code_source.py index 02c200271f..f072b97c1f 100644 --- a/astrbot/core/provider/sources/kimi_code_source.py +++ b/astrbot/core/provider/sources/kimi_code_source.py @@ -1,3 +1,5 @@ +from astrbot.core.utils.media_utils import VENDOR_IMAGE_FORMATS + from ..register import register_provider_adapter from .anthropic_source import ProviderAnthropic @@ -11,6 +13,8 @@ "Kimi Code Provider Adapter", ) class ProviderKimiCode(ProviderAnthropic): + supported_image_formats = VENDOR_IMAGE_FORMATS["kimi-code"] + def __init__( self, provider_config: dict, diff --git a/astrbot/core/provider/sources/minimax_token_plan_source.py b/astrbot/core/provider/sources/minimax_token_plan_source.py index 8d86c77b73..3e8a05ba78 100644 --- a/astrbot/core/provider/sources/minimax_token_plan_source.py +++ b/astrbot/core/provider/sources/minimax_token_plan_source.py @@ -2,6 +2,7 @@ from astrbot import logger from astrbot.core.provider.sources.anthropic_source import ProviderAnthropic +from astrbot.core.utils.media_utils import VENDOR_IMAGE_FORMATS from ..register import register_provider_adapter @@ -11,6 +12,7 @@ "MiniMax Token Plan Provider Adapter", ) class ProviderMiniMaxTokenPlan(ProviderAnthropic): + supported_image_formats = VENDOR_IMAGE_FORMATS["minimax-token-plan"] """MiniMax Token Plan provider. The model list is fetched dynamically from the MiniMax API's /v1/models diff --git a/astrbot/core/provider/sources/oai_aihubmix_source.py b/astrbot/core/provider/sources/oai_aihubmix_source.py index ca8ad59596..05404743b7 100644 --- a/astrbot/core/provider/sources/oai_aihubmix_source.py +++ b/astrbot/core/provider/sources/oai_aihubmix_source.py @@ -6,6 +6,9 @@ "aihubmix_chat_completion", "AIHubMix Chat Completion Provider Adapter" ) class ProviderAIHubMix(ProviderOpenAIOfficial): + # Aggregator gateway: do not inherit the official OpenAI format set. + supported_image_formats = None + def __init__( self, provider_config: dict, diff --git a/astrbot/core/provider/sources/openai_source.py b/astrbot/core/provider/sources/openai_source.py index f7870b7137..d4b49ba708 100644 --- a/astrbot/core/provider/sources/openai_source.py +++ b/astrbot/core/provider/sources/openai_source.py @@ -30,7 +30,10 @@ from astrbot.core.message.message_event_result import MessageChain from astrbot.core.provider.entities import LLMResponse, TokenUsage, ToolCallsResult from astrbot.core.utils.media_utils import ( + VENDOR_IMAGE_FORMATS, + ResolvedMediaData, describe_media_ref, + resolve_image_ref_to_images, resolve_media_ref_to_base64_data, ) from astrbot.core.utils.network_utils import ( @@ -51,6 +54,9 @@ class ProviderOpenAIOfficial(Provider): _ERROR_TEXT_CANDIDATE_MAX_CHARS = 4096 + supported_image_formats = VENDOR_IMAGE_FORMATS["openai"] + """Formats accepted by the official OpenAI vision API (non-animated GIF).""" + @classmethod def _truncate_error_text_candidate(cls, text: str) -> str: if len(text) <= cls._ERROR_TEXT_CANDIDATE_MAX_CHARS: @@ -176,37 +182,52 @@ def _is_invalid_attachment_error(self, error: Exception) -> bool: return True return False - async def _image_ref_to_data_url( + async def _image_ref_to_images( self, image_ref: str, *, mode: Literal["safe", "strict"] = "safe", - ) -> str | None: - image_data = await resolve_media_ref_to_base64_data( + ) -> list[ResolvedMediaData]: + """Resolve an image ref with this provider's format adaptation applied.""" + montage_max_size = self.get_animated_montage_max_size() + return await resolve_image_ref_to_images( image_ref, - media_type="image", + allowed_mime_types=self.resolve_allowed_image_formats(), + montage_max_size=montage_max_size, strict=mode == "strict", ) - return image_data.to_data_url() if image_data else None - async def _resolve_image_part( + async def _image_ref_to_data_url( + self, + image_ref: str, + *, + mode: Literal["safe", "strict"] = "safe", + ) -> str | None: + images = await self._image_ref_to_images(image_ref, mode=mode) + return images[0].to_data_url() if images else None + + async def _resolve_image_parts( self, image_url: str, *, image_detail: str | None = None, - ) -> dict | None: - image_data = await self._image_ref_to_data_url(image_url, mode="safe") - if not image_data: + ) -> list[dict]: + images = await self._image_ref_to_images(image_url, mode="safe") + if not images: logger.warning("图片预处理结果为空，将忽略。") - return None - image_payload = {"url": image_data} - - if image_detail: - image_payload["detail"] = image_detail - return { - "type": "image_url", - "image_url": image_payload, - } + return [] + parts = [] + for image_data in images: + image_payload = {"url": image_data.to_data_url()} + if image_detail: + image_payload["detail"] = image_detail + parts.append( + { + "type": "image_url", + "image_url": image_payload, + } + ) + return parts def _extract_image_part_info(self, part: dict) -> tuple[str | None, str | None]: if not isinstance(part, dict) or part.get("type") != "image_url": @@ -266,17 +287,17 @@ async def _resolve_audio_part(self, audio_ref: str) -> dict | None: }, } - async def _transform_content_part(self, part: dict) -> dict: + async def _transform_content_part(self, part: dict) -> list[dict]: if not isinstance(part, dict): - return part + return [part] if part.get("type") == "image_url": url, image_detail = self._extract_image_part_info(part) if not url: - return part + return [part] try: - resolved_part = await self._resolve_image_part( + resolved_parts = await self._resolve_image_parts( url, image_detail=image_detail ) except Exception as exc: @@ -285,25 +306,27 @@ async def _transform_content_part(self, part: dict) -> dict: url, exc, ) - return part + return [part] - return resolved_part or part + return resolved_parts or [part] if part.get("type") == "audio_url": audio_ref = self._extract_audio_part_info(part) if not audio_ref: - return part + return [part] resolved_part = await self._resolve_audio_part(audio_ref) - return resolved_part or part + return [resolved_part] if resolved_part else [part] - return part + return [part] async def _materialize_message_image_parts(self, message: dict) -> dict: content = message.get("content") if not isinstance(content, list): return {**message} - new_content = [await self._transform_content_part(part) for part in content] + new_content: list[dict] = [] + for part in content: + new_content.extend(await self._transform_content_part(part)) return {**message, "content": new_content} async def _materialize_context_image_parts( @@ -1394,11 +1417,10 @@ async def assemble_context( if isinstance(part, TextPart): content_blocks.append({"type": "text", "text": part.text}) elif isinstance(part, ImageURLPart): - image_part = await self._resolve_image_part( + image_parts = await self._resolve_image_parts( part.image_url.url, ) - if image_part: - content_blocks.append(image_part) + content_blocks.extend(image_parts) elif isinstance(part, AudioURLPart): audio_part = await self._resolve_audio_part(part.audio_url.url) if audio_part: @@ -1409,9 +1431,8 @@ async def assemble_context( # 3. 图片内容 if image_urls: for image_url in image_urls: - image_part = await self._resolve_image_part(image_url) - if image_part: - content_blocks.append(image_part) + image_parts = await self._resolve_image_parts(image_url) + content_blocks.extend(image_parts) if audio_urls: for audio_path in audio_urls: diff --git a/astrbot/core/provider/sources/openrouter_source.py b/astrbot/core/provider/sources/openrouter_source.py index a308ad309d..bcac162aa4 100644 --- a/astrbot/core/provider/sources/openrouter_source.py +++ b/astrbot/core/provider/sources/openrouter_source.py @@ -6,6 +6,10 @@ "openrouter_chat_completion", "OpenRouter Chat Completion Provider Adapter" ) class ProviderOpenRouter(ProviderOpenAIOfficial): + # Aggregator gateway: the backend model varies, so do not inherit the + # official OpenAI format set; use the conservative fallback instead. + supported_image_formats = None + def __init__( self, provider_config: dict, diff --git a/astrbot/core/provider/sources/ssycloud_source.py b/astrbot/core/provider/sources/ssycloud_source.py index 04a2ff677b..837d7376b7 100644 --- a/astrbot/core/provider/sources/ssycloud_source.py +++ b/astrbot/core/provider/sources/ssycloud_source.py @@ -12,6 +12,9 @@ class ProviderSSYCloud(ProviderOpenAIOfficial): """SSYCloud provider using its OpenAI-compatible Chat Completions API.""" + # Aggregator gateway: do not inherit the official OpenAI format set. + supported_image_formats = None + def __init__(self, provider_config: dict, provider_settings: dict) -> None: """Initialize the SSYCloud client with provider defaults. diff --git a/astrbot/core/provider/sources/xai_source.py b/astrbot/core/provider/sources/xai_source.py index b7b432b49a..a9403a90a9 100644 --- a/astrbot/core/provider/sources/xai_source.py +++ b/astrbot/core/provider/sources/xai_source.py @@ -1,3 +1,5 @@ +from astrbot.core.utils.media_utils import VENDOR_IMAGE_FORMATS + from ..register import register_provider_adapter from .openai_source import ProviderOpenAIOfficial @@ -6,6 +8,9 @@ "xai_chat_completion", "xAI Chat Completion Provider Adapter" ) class ProviderXAI(ProviderOpenAIOfficial): + supported_image_formats = VENDOR_IMAGE_FORMATS["xai"] + """The official xAI vision API accepts only JPEG and PNG.""" + def __init__( self, provider_config: dict, diff --git a/astrbot/core/provider/sources/xiaomi_source.py b/astrbot/core/provider/sources/xiaomi_source.py index 1f14175f41..485920b51c 100644 --- a/astrbot/core/provider/sources/xiaomi_source.py +++ b/astrbot/core/provider/sources/xiaomi_source.py @@ -1,5 +1,6 @@ from astrbot import logger from astrbot.core.provider.sources.openai_source import ProviderOpenAIOfficial +from astrbot.core.utils.media_utils import VENDOR_IMAGE_FORMATS from ..register import register_provider_adapter @@ -16,6 +17,7 @@ "xiaomi_chat_completion", "Xiaomi API 提供商适配器 (OpenAI 兼容)" ) class ProviderXiaomi(ProviderOpenAIOfficial): + supported_image_formats = VENDOR_IMAGE_FORMATS["xiaomi"] """Xiaomi provider using OpenAI-compatible API. Supports both standard API and multimodal capabilities. diff --git a/astrbot/core/provider/sources/xiaomi_token_plan_source.py b/astrbot/core/provider/sources/xiaomi_token_plan_source.py index 70a2cf0afc..02b5a20a5b 100644 --- a/astrbot/core/provider/sources/xiaomi_token_plan_source.py +++ b/astrbot/core/provider/sources/xiaomi_token_plan_source.py @@ -1,5 +1,6 @@ from astrbot import logger from astrbot.core.provider.sources.anthropic_source import ProviderAnthropic +from astrbot.core.utils.media_utils import VENDOR_IMAGE_FORMATS from ..register import register_provider_adapter @@ -14,6 +15,7 @@ @register_provider_adapter("xiaomi_token_plan", "Xiaomi Token Plan 提供商适配器") class ProviderXiaomiTokenPlan(ProviderAnthropic): + supported_image_formats = VENDOR_IMAGE_FORMATS["xiaomi-token-plan"] """Xiaomi Token Plan provider. The Token Plan API uses Anthropic-compatible endpoint with Bearer token auth. diff --git a/astrbot/core/provider/sources/zhipu_source.py b/astrbot/core/provider/sources/zhipu_source.py index ed4bc0bf89..3149fe83f2 100644 --- a/astrbot/core/provider/sources/zhipu_source.py +++ b/astrbot/core/provider/sources/zhipu_source.py @@ -2,12 +2,17 @@ # It is no longer specifically adapted to Zhipu's models. To ensure compatibility, this +from astrbot.core.utils.media_utils import VENDOR_IMAGE_FORMATS + from ..register import register_provider_adapter from .openai_source import ProviderOpenAIOfficial @register_provider_adapter("zhipu_chat_completion", "智谱 Chat Completion 提供商适配器") class ProviderZhipu(ProviderOpenAIOfficial): + supported_image_formats = VENDOR_IMAGE_FORMATS["zhipu"] + """GLM vision officially guarantees JPEG/PNG; keep it conservative.""" + def __init__( self, provider_config: dict, diff --git a/astrbot/core/utils/media_utils.py b/astrbot/core/utils/media_utils.py index 6cd58b92ec..606cede419 100644 --- a/astrbot/core/utils/media_utils.py +++ b/astrbot/core/utils/media_utils.py @@ -7,12 +7,14 @@ import asyncio import base64 import binascii +import hashlib import io import mimetypes import os import shutil import subprocess -from collections.abc import AsyncIterator +import tempfile +from collections.abc import AsyncIterator, Collection from contextlib import asynccontextmanager from dataclasses import dataclass, field from pathlib import Path @@ -70,6 +72,101 @@ "AVIF": "image/avif", } +IMAGE_SHORT_MIME_TYPES = { + "jpeg": "image/jpeg", + "jpg": "image/jpeg", + "png": "image/png", + "gif": "image/gif", + "webp": "image/webp", + "bmp": "image/bmp", + "tif": "image/tiff", + "tiff": "image/tiff", + "avif": "image/avif", + "heic": "image/heic", + "heif": "image/heif", +} +"""User-facing short image format names mapped to MIME types.""" + +IMAGE_MIME_SHORT_NAMES = { + "image/jpeg": "jpeg", + "image/png": "png", + "image/webp": "webp", + "image/gif": "gif", + "image/bmp": "bmp", + "image/tiff": "tiff", + "image/avif": "avif", + "image/heic": "heic", + "image/heif": "heif", +} +"""Canonical short display name for each known image MIME type.""" + +_MINIMAX_IMAGE_FORMATS = frozenset( + {"image/jpeg", "image/png", "image/webp", "image/gif"} +) +_XIAOMI_IMAGE_FORMATS = frozenset( + {"image/jpeg", "image/png", "image/webp", "image/gif", "image/bmp"} +) +_MOONSHOT_IMAGE_FORMATS = frozenset( + { + "image/jpeg", + "image/png", + "image/webp", + "image/gif", + "image/bmp", + "image/heic", + "image/heif", + } +) + +VENDOR_IMAGE_FORMATS = { + "openai": frozenset({"image/jpeg", "image/png", "image/webp", "image/gif"}), + "azure": frozenset({"image/jpeg", "image/png", "image/webp", "image/gif"}), + "xai": frozenset({"image/jpeg", "image/png"}), + "deepseek": frozenset({"image/jpeg", "image/png", "image/gif", "image/webp"}), + "anthropic": frozenset({"image/jpeg", "image/png", "image/gif", "image/webp"}), + "google": frozenset( + {"image/jpeg", "image/png", "image/webp", "image/heic", "image/heif"} + ), + "zhipu": frozenset({"image/jpeg", "image/png"}), + # NVIDIA NIM VLM docs list JPG/JPEG/PNG for most models (a few also take GIF). + "nvidia": frozenset({"image/jpeg", "image/png"}), + # Groq docs only demonstrate JPEG; PNG is the other reliably referenced type. + "groq": frozenset({"image/jpeg", "image/png"}), + "moonshot": _MOONSHOT_IMAGE_FORMATS, + "kimi-code": _MOONSHOT_IMAGE_FORMATS, + "minimax": _MINIMAX_IMAGE_FORMATS, + "minimax-token-plan": _MINIMAX_IMAGE_FORMATS, + "xiaomi": _XIAOMI_IMAGE_FORMATS, + "xiaomi-token-plan": _XIAOMI_IMAGE_FORMATS, +} +"""Image MIME types officially documented by each vendor (brand key from provider +source templates). Absent brand = no vendor-level opinion; the adapter class +declaration then governs.""" + +ANIMATED_MONTAGE_GRID = 3 +"""Animated images become a grid x grid frame montage (contact sheet).""" + +ANIMATED_MONTAGE_FRAME_COUNT = ANIMATED_MONTAGE_GRID * ANIMATED_MONTAGE_GRID + +CONVERT_CACHE_DIR_NAME = "media_convert_cache" +"""Cache directory (under the AstrBot temp dir) for converted images and frames.""" + +_MIME_PIL_FORMAT = { + "image/jpeg": "JPEG", + "image/png": "PNG", + "image/webp": "WEBP", + "image/gif": "GIF", + "image/bmp": "BMP", +} + +_MIME_FILE_SUFFIX = { + "image/jpeg": ".jpg", + "image/png": ".png", + "image/webp": ".webp", + "image/gif": ".gif", + "image/bmp": ".bmp", +} + AUDIO_FORMAT_MIME_TYPES = { "aac": "audio/aac", "amr": "audio/amr", @@ -894,7 +991,259 @@ async def resolve_image_ref_to_base64_data( assembly can skip bad image refs without failing the whole request. """ - return await MediaResolver( + images = await resolve_image_ref_to_images( + image_ref, + strict=strict, + default_mime_type=default_mime_type, + ) + return images[0] if images else None + + +def _image_convert_cache_dir() -> Path: + cache_dir = Path(get_astrbot_temp_path()) / CONVERT_CACHE_DIR_NAME + cache_dir.mkdir(parents=True, exist_ok=True) + return cache_dir + + +def _image_convert_cache_key(source_bytes: bytes, params: str) -> str: + """Build a content-addressed cache key for a converted image or frame set.""" + source_digest = hashlib.sha256(source_bytes).hexdigest()[:32] + params_digest = hashlib.sha256(params.encode()).hexdigest()[:8] + return f"{source_digest}_{params_digest}" + + +def _image_has_alpha(image: PILImage.Image) -> bool: + return image.mode in {"RGBA", "LA"} or ( + image.mode == "P" and "transparency" in image.info + ) + + +def _inspect_image(image_bytes: bytes) -> tuple[bool, int]: + """Inspect decodable image bytes. + + Args: + image_bytes: Encoded image bytes. + + Returns: + Tuple of ``(has_alpha, frame_count)``. + + Raises: + Exception: Raised by Pillow when the bytes are not a decodable image. + """ + with PILImage.open(io.BytesIO(image_bytes)) as image: + return _image_has_alpha(image), getattr(image, "n_frames", 1) + + +def _pick_target_image_mime_type( + has_alpha: bool, + allowed_mime_types: Collection[str] | None, +) -> str: + """Pick the best Pillow-savable target MIME type within the allowed set. + + Alpha images prefer PNG to preserve transparency; opaque images prefer JPEG. + """ + preferred = ["image/png"] if has_alpha else ["image/jpeg"] + preferred += ["image/jpeg", "image/png", "image/webp"] + for mime_type in preferred: + if allowed_mime_types is None or mime_type in allowed_mime_types: + return mime_type + for mime_type in allowed_mime_types or (): + if mime_type in _MIME_PIL_FORMAT: + return mime_type + return "image/png" + + +def _save_current_image_frame( + image: PILImage.Image, + target_mime_type: str, + output_path: Path, +) -> None: + """Save the currently selected frame of an opened image.""" + working: PILImage.Image | None = None + try: + frame = image + if target_mime_type == "image/jpeg" and image.mode != "RGB": + working = image.convert("RGB") + frame = working + elif ( + target_mime_type == "image/png" + and image.mode == "P" + and "transparency" in image.info + ): + working = image.convert("RGBA") + frame = working + save_kwargs: dict[str, int] = {} + if target_mime_type == "image/jpeg": + save_kwargs = { + "quality": IMAGE_COMPRESS_DEFAULT_QUALITY, + "subsampling": 0, + } + frame.save(output_path, _MIME_PIL_FORMAT[target_mime_type], **save_kwargs) + finally: + if working is not None: + working.close() + + +def _save_image_frame_atomic( + image: PILImage.Image, + target_mime_type: str, + output_path: Path, +) -> None: + """Save via a unique temp file then atomically replace the cache entry. + + Concurrent readers never observe a partially written cache file. + """ + fd, tmp_name = tempfile.mkstemp(dir=output_path.parent, suffix=".tmp") + os.close(fd) + tmp_path = Path(tmp_name) + try: + _save_current_image_frame(image, target_mime_type, tmp_path) + os.replace(tmp_path, output_path) + finally: + tmp_path.unlink(missing_ok=True) + + +def _convert_image_bytes_sync( + source_bytes: bytes, + target_mime_type: str, +) -> Path: + """Convert image bytes to the target format, cached under the temp dir. + + Args: + source_bytes: Encoded source image bytes. + target_mime_type: Target MIME type; must be Pillow-savable. + + Returns: + Path of the converted (or previously cached) image. + """ + cache_key = _image_convert_cache_key(source_bytes, f"convert|{target_mime_type}") + output_path = _image_convert_cache_dir() / ( + cache_key + _MIME_FILE_SUFFIX[target_mime_type] + ) + if output_path.exists(): + return output_path + with PILImage.open(io.BytesIO(source_bytes)) as image: + _save_image_frame_atomic(image, target_mime_type, output_path) + return output_path + + +def _even_frame_indices(total_frames: int, max_frames: int) -> list[int]: + """Pick up to ``max_frames`` frame indices evenly spaced over the animation.""" + count = min(max_frames, total_frames) + if count <= 1: + return [0] + return sorted({round(i * (total_frames - 1) / (count - 1)) for i in range(count)}) + + +def _extract_animation_montage_sync( + source_bytes: bytes, + target_mime_type: str, + max_size: int, +) -> tuple[Path, bool]: + """Tile evenly spaced frames of an animated image into one grid montage. + + Frame sampling includes the first and last frames; grid cells beyond the + available frames stay blank (white). The montage is flattened onto a white + background and its longest edge is capped at ``max_size`` (never upscaled). + + Args: + source_bytes: Encoded animated image bytes. + target_mime_type: MIME type the montage is saved as. + max_size: Longest edge of the montage in pixels. + + Returns: + Tuple of the montage path and whether extraction actually ran + (``False`` when the cached montage was served). + """ + suffix = _MIME_FILE_SUFFIX[target_mime_type] + cache_key = _image_convert_cache_key( + source_bytes, f"montage|{target_mime_type}|s={max_size}" + ) + output_path = _image_convert_cache_dir() / (cache_key + suffix) + if output_path.exists(): + return output_path, False + with PILImage.open(io.BytesIO(source_bytes)) as image: + total_frames = getattr(image, "n_frames", 1) + frame_indices = _even_frame_indices(total_frames, ANIMATED_MONTAGE_FRAME_COUNT) + # Floor the per-cell scale so the montage never exceeds max_size. + longest_edge = max(image.size) * ANIMATED_MONTAGE_GRID + scale = min(1.0, max(max_size, 1) / longest_edge) + cell_size = ( + max(1, int(image.size[0] * scale)), + max(1, int(image.size[1] * scale)), + ) + canvas = PILImage.new( + "RGB", + ( + cell_size[0] * ANIMATED_MONTAGE_GRID, + cell_size[1] * ANIMATED_MONTAGE_GRID, + ), + (255, 255, 255), + ) + try: + for out_index, frame_index in enumerate(frame_indices): + image.seek(frame_index) + frame = image.convert("RGBA") + if frame.size != cell_size: + frame = frame.resize(cell_size, PILImage.Resampling.LANCZOS) + # Paste with the alpha band as mask so transparency shows white. + canvas.paste( + frame, + ( + (out_index % ANIMATED_MONTAGE_GRID) * cell_size[0], + (out_index // ANIMATED_MONTAGE_GRID) * cell_size[1], + ), + frame, + ) + frame.close() + _save_image_frame_atomic(canvas, target_mime_type, output_path) + finally: + canvas.close() + return output_path, True + + +async def _path_to_resolved_media_data(path: Path, mime_type: str) -> ResolvedMediaData: + data = await asyncio.to_thread(path.read_bytes) + return ResolvedMediaData( + base64_data=base64.b64encode(data).decode("utf-8"), + mime_type=mime_type, + ) + + +async def resolve_image_ref_to_images( + image_ref: MediaRefStr, + *, + allowed_mime_types: Collection[str] | None = None, + montage_max_size: int = IMAGE_COMPRESS_DEFAULT_MAX_SIZE, + strict: bool = False, + default_mime_type: str | None = "image/jpeg", +) -> list[ResolvedMediaData]: + """Resolve an image reference into provider-ready images. + + Applies per-provider format adaptation: still images whose detected MIME type + is not in ``allowed_mime_types`` are converted via Pillow (cached under the + AstrBot temp directory, so a source is only converted once until the cache is + cleaned), and animated images are tiled into one frame-grid montage. Single + frame animated images fall through to the still-image path. Compatible + images are returned untouched without any conversion or cache write. + + Args: + image_ref: Image reference in any form accepted by ``MediaResolver``. + allowed_mime_types: MIME types accepted by the target provider. ``None`` + or a collection containing ``"*"`` disables adaptation. + montage_max_size: Longest edge in pixels of the animated-image montage. + strict: Raise on invalid or undecodable images instead of skipping them. + default_mime_type: Fallback MIME type for legacy base64 payloads. + + Returns: + List of resolved images; empty when the reference is invalid and + ``strict`` is False. + + Raises: + ValueError: Raised in strict mode when the image is invalid or cannot be + decoded by Pillow. + """ + media_data = await MediaResolver( image_ref, media_type="image", default_suffix=".bin", @@ -902,6 +1251,62 @@ async def resolve_image_ref_to_base64_data( strict=strict, default_mime_type=default_mime_type, ) + if media_data is None: + return [] + + image_bytes = media_data.to_bytes() + unrestricted = allowed_mime_types is None or "*" in allowed_mime_types + try: + has_alpha, frame_count = await asyncio.to_thread(_inspect_image, image_bytes) + except Exception as exc: + # Pillow cannot decode some provider-supported formats (e.g. HEIC without + # pillow-heif); pass them through when the provider accepts them. + if unrestricted or ( + media_data.mime_type and media_data.mime_type in allowed_mime_types + ): + return [media_data] + if strict: + raise ValueError( + f"Invalid image file: {describe_media_ref(image_ref)}" + ) from exc + logger.warning( + "Image %s cannot be decoded and will be skipped.", + describe_media_ref(image_ref), + ) + return [] + + if frame_count > 1: + # Montage cells are flattened onto white, so alpha no longer matters + # when picking the target format. + target_mime_type = _pick_target_image_mime_type( + False, + None if unrestricted else allowed_mime_types, + ) + montage_path, extracted = await asyncio.to_thread( + _extract_animation_montage_sync, + image_bytes, + target_mime_type, + max(int(montage_max_size), 1), + ) + if extracted: + logger.info( + "Animated image %s tiled into a %dx%d frame montage for the provider.", + describe_media_ref(image_ref), + ANIMATED_MONTAGE_GRID, + ANIMATED_MONTAGE_GRID, + ) + return [await _path_to_resolved_media_data(montage_path, target_mime_type)] + + if unrestricted or media_data.mime_type in allowed_mime_types: + return [media_data] + + target_mime_type = _pick_target_image_mime_type(has_alpha, allowed_mime_types) + converted_path = await asyncio.to_thread( + _convert_image_bytes_sync, + image_bytes, + target_mime_type, + ) + return [await _path_to_resolved_media_data(converted_path, target_mime_type)] async def resolve_audio_ref_to_base64_data( diff --git a/astrbot/dashboard/services/config_service.py b/astrbot/dashboard/services/config_service.py index 21e5d60357..d4f18f0aeb 100644 --- a/astrbot/dashboard/services/config_service.py +++ b/astrbot/dashboard/services/config_service.py @@ -24,6 +24,7 @@ from astrbot.core.core_lifecycle import AstrBotCoreLifecycle from astrbot.core.db import BaseDatabase from astrbot.core.platform.register import platform_cls_map, platform_registry +from astrbot.core.provider.entities import ProviderType from astrbot.core.provider.register import provider_registry from astrbot.core.star.star import star_registry from astrbot.core.utils.astrbot_path import get_astrbot_plugin_data_path @@ -1404,7 +1405,12 @@ def get_provider_schema(self) -> dict: for provider in self.config.get("provider", []) if provider.get("provider_type") != "agent_runner" ] + from astrbot.core.provider.register import provider_cls_map from astrbot.core.utils.llm_metadata import LLM_METADATAS + from astrbot.core.utils.media_utils import ( + IMAGE_MIME_SHORT_NAMES, + VENDOR_IMAGE_FORMATS, + ) model_metadata = {} for provider in providers: @@ -1412,11 +1418,52 @@ def get_provider_schema(self) -> dict: model_id = provider.get("model") if isinstance(model_id, str) and model_id in LLM_METADATAS: model_metadata[model_id] = LLM_METADATAS[model_id] + # Officially declared image format sets per chat provider type, + # so the dashboard can display and pre-fill them. Adapters are imported + # lazily, so import every chat template type before reading the classes. + for tmpl in provider_default_tmpl.values(): + if ( + not isinstance(tmpl, dict) + or tmpl.get("provider_type") != "chat_completion" + ): + continue + adapter_type = tmpl.get("type") + if not adapter_type or adapter_type in provider_cls_map: + continue + try: + self.provider_manager.dynamic_import_provider(adapter_type) + except Exception as exc: + logger.debug( + f"Skip image format introspection for {adapter_type}: {exc}" + ) + provider_type_image_formats = {} + for provider in provider_registry: + if provider.provider_type != ProviderType.CHAT_COMPLETION: + continue + declared = getattr(provider.cls_type, "supported_image_formats", None) + if declared is None: + provider_type_image_formats[provider.type] = None + else: + provider_type_image_formats[provider.type] = [ + short + for mime, short in IMAGE_MIME_SHORT_NAMES.items() + if mime in declared + ] + provider_brand_image_formats = { + brand: [ + short + for mime, short in IMAGE_MIME_SHORT_NAMES.items() + if mime in formats + ] + for brand, formats in VENDOR_IMAGE_FORMATS.items() + } return { "config_schema": config_schema, "providers": providers, "provider_sources": self.config.get("provider_sources", []), "model_metadata": model_metadata, + "provider_type_image_formats": provider_type_image_formats, + "provider_brand_image_formats": provider_brand_image_formats, } def list_provider_sources(self) -> dict: diff --git a/dashboard/src/api/v1.ts b/dashboard/src/api/v1.ts index 709d4060a7..0824d107f4 100644 --- a/dashboard/src/api/v1.ts +++ b/dashboard/src/api/v1.ts @@ -79,6 +79,8 @@ export interface ProviderSchemaData { providers?: OpenConfig[]; provider_sources?: OpenConfig[]; model_metadata?: Record; + provider_type_image_formats?: Record; + provider_brand_image_formats?: Record; } export interface ProviderListData { diff --git a/dashboard/src/components/provider/ProviderChatCompletionPanel.vue b/dashboard/src/components/provider/ProviderChatCompletionPanel.vue index 4991aef426..3d9be7efe1 100644 --- a/dashboard/src/components/provider/ProviderChatCompletionPanel.vue +++ b/dashboard/src/components/provider/ProviderChatCompletionPanel.vue @@ -222,6 +222,8 @@ const { testingProviders, isSourceModified, configSchema, + providerTypeImageFormats, + providerBrandImageFormats, providerSourceSchema, manualModelId, modelSearch, @@ -281,6 +283,8 @@ const { } = useProviderModelConfigDialog({ selectedProviderSource, configSchema, + providerTypeImageFormats, + providerBrandImageFormats, buildModelProviderConfig, modelAlreadyConfigured, loadConfig, diff --git a/dashboard/src/components/shared/AstrBotConfig.vue b/dashboard/src/components/shared/AstrBotConfig.vue index 77a9d7092d..ed0eeeb0c4 100644 --- a/dashboard/src/components/shared/AstrBotConfig.vue +++ b/dashboard/src/components/shared/AstrBotConfig.vue @@ -158,7 +158,12 @@ function shouldShowItem(itemMeta, itemKey) { } for (const [conditionKey, expectedValue] of Object.entries(itemMeta.condition)) { const actualValue = getValueBySelector(props.iterable, conditionKey) - if (actualValue !== expectedValue) { + // List-valued fields (e.g. modalities) match when they contain the expected value + if (Array.isArray(actualValue) && !Array.isArray(expectedValue)) { + if (!actualValue.includes(expectedValue)) { + return false + } + } else if (actualValue !== expectedValue) { return false } } diff --git a/dashboard/src/components/shared/AstrBotConfigV4.vue b/dashboard/src/components/shared/AstrBotConfigV4.vue index 379cd52969..21d2b28906 100644 --- a/dashboard/src/components/shared/AstrBotConfigV4.vue +++ b/dashboard/src/components/shared/AstrBotConfigV4.vue @@ -154,7 +154,12 @@ function shouldShowItem(itemMeta, itemKey) { if (itemMeta?.condition) { for (const [conditionKey, expectedValue] of Object.entries(itemMeta.condition)) { const actualValue = getValueBySelector(props.iterable, conditionKey) - if (actualValue !== expectedValue) { + // List-valued fields (e.g. modalities) match when they contain the expected value + if (Array.isArray(actualValue) && !Array.isArray(expectedValue)) { + if (!actualValue.includes(expectedValue)) { + return false + } + } else if (actualValue !== expectedValue) { return false } } diff --git a/dashboard/src/composables/useProviderModelConfigDialog.ts b/dashboard/src/composables/useProviderModelConfigDialog.ts index c733c08308..9c38955729 100644 --- a/dashboard/src/composables/useProviderModelConfigDialog.ts +++ b/dashboard/src/composables/useProviderModelConfigDialog.ts @@ -4,6 +4,8 @@ import { providerApi } from '@/api/v1' interface UseProviderModelConfigDialogOptions { selectedProviderSource: Ref configSchema: Ref> + providerTypeImageFormats: Ref> + providerBrandImageFormats: Ref> buildModelProviderConfig: (modelId: string) => any modelAlreadyConfigured: (modelId: string) => boolean loadConfig: () => Promise | void @@ -15,6 +17,8 @@ export function useProviderModelConfigDialog(options: UseProviderModelConfigDial const { selectedProviderSource, configSchema, + providerTypeImageFormats, + providerBrandImageFormats, buildModelProviderConfig, modelAlreadyConfigured, loadConfig, @@ -59,6 +63,28 @@ export function useProviderModelConfigDialog(options: UseProviderModelConfigDial function openProviderEdit(provider: any) { const editableProvider = JSON.parse(JSON.stringify(provider)) + // Materialize schema-declared defaults for keys missing from stored configs, + // so fields introduced after the provider was created become editable. + const items = configSchema.value?.provider?.items + if (items) { + for (const [key, item] of Object.entries(items)) { + if (editableProvider[key] === undefined && item && typeof item === 'object' && 'default' in item) { + editableProvider[key] = JSON.parse(JSON.stringify(item.default)) + } + } + } + // Backfill the source's officially declared image format set (vendor brand + // first, then adapter type) so the effective default is visible and editable; + // undeclared aggregators fall back to jpeg/png. + if (Array.isArray(editableProvider.image_formats) && editableProvider.image_formats.length === 0) { + const source = selectedProviderSource.value + const declared = source + ? (providerBrandImageFormats.value[source.provider] ?? providerTypeImageFormats.value[source.type]) + : undefined + if (declared !== undefined) { + editableProvider.image_formats = [...(declared ?? ['jpeg', 'png'])] + } + } if (editableProvider.provider_source_id) { delete editableProvider.reasoning } diff --git a/dashboard/src/composables/useProviderSources.ts b/dashboard/src/composables/useProviderSources.ts index 4e6fcbf13c..2a648ebaa7 100644 --- a/dashboard/src/composables/useProviderSources.ts +++ b/dashboard/src/composables/useProviderSources.ts @@ -71,6 +71,9 @@ export function useProviderSources(options: UseProviderSourcesOptions) { const isSourceModified = ref(false) const configSchema = ref>({}) const providerTemplates = ref>({}) + // Officially declared image format sets per provider type, from the backend schema API. + const providerTypeImageFormats = ref>({}) + const providerBrandImageFormats = ref>({}) const manualModelId = ref('') const modelSearch = ref('') @@ -623,7 +626,8 @@ export function useProviderSources(options: UseProviderSourcesOptions) { model: modelName, modalities, custom_extra_body: {}, - max_context_tokens: max_context_tokens + max_context_tokens: max_context_tokens, + image_formats: [...(providerBrandImageFormats.value[selectedProviderSource.value.provider] ?? providerTypeImageFormats.value[selectedProviderSource.value.type] ?? ['jpeg', 'png'])] } } @@ -726,6 +730,8 @@ export function useProviderSources(options: UseProviderSourcesOptions) { providerTemplates.value = configSchema.value.provider.config_template } providerSources.value = response.data.data.provider_sources || [] + providerTypeImageFormats.value = response.data.data.provider_type_image_formats || {} + providerBrandImageFormats.value = response.data.data.provider_brand_image_formats || {} modelMetadata.value = (response.data.data.model_metadata || {}) as Record providers.value = response.data.data.providers || [] } @@ -764,6 +770,8 @@ export function useProviderSources(options: UseProviderSourcesOptions) { isSourceModified, configSchema, providerTemplates, + providerTypeImageFormats, + providerBrandImageFormats, manualModelId, modelSearch, diff --git a/dashboard/src/i18n/locales/en-US/features/config-metadata.json b/dashboard/src/i18n/locales/en-US/features/config-metadata.json index c1410e658d..da66a57261 100644 --- a/dashboard/src/i18n/locales/en-US/features/config-metadata.json +++ b/dashboard/src/i18n/locales/en-US/features/config-metadata.json @@ -1404,6 +1404,20 @@ "Tool use" ] }, + "image_formats": { + "description": "Supported image formats", + "hint": "Image formats allowed when sending to this provider; incompatible formats are converted automatically. Selecting \"Unrestricted\" overrides all other options.", + "labels": [ + "JPEG", + "PNG", + "WebP", + "GIF", + "BMP", + "HEIC", + "HEIF", + "Unrestricted" + ] + }, "custom_headers": { "description": "Custom request headers", "hint": "Key/value pairs added here are merged into the OpenAI SDK default_headers for custom HTTP headers. Values must be strings." diff --git a/dashboard/src/i18n/locales/ru-RU/features/config-metadata.json b/dashboard/src/i18n/locales/ru-RU/features/config-metadata.json index 7c3b9ae407..e65712b2b7 100644 --- a/dashboard/src/i18n/locales/ru-RU/features/config-metadata.json +++ b/dashboard/src/i18n/locales/ru-RU/features/config-metadata.json @@ -1393,6 +1393,20 @@ "Инструменты" ] }, + "image_formats": { + "description": "Поддерживаемые форматы изображений", + "hint": "Форматы изображений, разрешённые при отправке этому провайдеру; несовместимые форматы конвертируются автоматически. При выборе «Без ограничений» остальные варианты не действуют.", + "labels": [ + "JPEG", + "PNG", + "WebP", + "GIF", + "BMP", + "HEIC", + "HEIF", + "Без ограничений" + ] + }, "custom_headers": { "description": "Заголовки запроса", "hint": "Пары ключ/значение будут добавлены в заголовки запроса (default_headers). Значения должны быть строками." diff --git a/dashboard/src/i18n/locales/zh-CN/features/config-metadata.json b/dashboard/src/i18n/locales/zh-CN/features/config-metadata.json index e630d556cc..e4fe8462d5 100644 --- a/dashboard/src/i18n/locales/zh-CN/features/config-metadata.json +++ b/dashboard/src/i18n/locales/zh-CN/features/config-metadata.json @@ -1427,6 +1427,20 @@ "工具使用" ] }, + "image_formats": { + "description": "图片格式支持", + "hint": "发送给该提供商前允许的图片格式，不兼容的格式会自动转换。勾选「不限制」时其他选项不生效。", + "labels": [ + "JPEG", + "PNG", + "WebP", + "GIF", + "BMP", + "HEIC", + "HEIF", + "不限制" + ] + }, "custom_headers": { "description": "自定义请求头", "hint": "此处添加的键值对将被合并到 OpenAI SDK 的 default_headers 中，用于自定义 HTTP 请求头。值必须为字符串。" diff --git a/dashboard/src/views/ProviderPage.vue b/dashboard/src/views/ProviderPage.vue index 804e7aaec9..4d792cbc0a 100644 --- a/dashboard/src/views/ProviderPage.vue +++ b/dashboard/src/views/ProviderPage.vue @@ -336,6 +336,8 @@ const { testingProviders, isSourceModified, configSchema, + providerTypeImageFormats, + providerBrandImageFormats, providerSourceSchema, manualModelId, modelSearch, @@ -403,6 +405,8 @@ const { } = useProviderModelConfigDialog({ selectedProviderSource, configSchema, + providerTypeImageFormats, + providerBrandImageFormats, buildModelProviderConfig, modelAlreadyConfigured, loadConfig, diff --git a/tests/test_media_utils.py b/tests/test_media_utils.py index efe5e65f02..d8a533196b 100644 --- a/tests/test_media_utils.py +++ b/tests/test_media_utils.py @@ -871,3 +871,299 @@ async def test_wav_to_tencent_silk_skips_resample_for_supported_rate( assert len(fake.calls) == 1 assert fake.calls[0]["sample_rate"] == 24000 + + +def _make_image_bytes( + fmt: str = "PNG", + mode: str = "RGB", + size: tuple[int, int] = (8, 8), + color=(255, 0, 0, 255), +) -> bytes: + from PIL import Image as PILImage + + image = PILImage.new(mode, size, color) + buffer = BytesIO() + image.save(buffer, fmt) + return buffer.getvalue() + + +def _make_animated_gif_bytes( + colors: list[tuple[int, int, int]], + size: tuple[int, int] = (8, 8), +) -> bytes: + from PIL import Image as PILImage + + frames = [PILImage.new("RGB", size, color) for color in colors] + buffer = BytesIO() + frames[0].save( + buffer, + "GIF", + save_all=True, + append_images=frames[1:], + duration=50, + loop=0, + ) + return buffer.getvalue() + + +def _image_data_uri(mime_type: str, payload: bytes) -> str: + return f"data:{mime_type};base64,{base64.b64encode(payload).decode()}" + + +@pytest.mark.asyncio +async def test_resolve_images_compatible_passes_through_without_cache( + tmp_path, monkeypatch +): + monkeypatch.setattr(media_utils, "get_astrbot_temp_path", lambda: str(tmp_path)) + png_bytes = _make_image_bytes("PNG") + + images = await media_utils.resolve_image_ref_to_images( + _image_data_uri("image/png", png_bytes), + allowed_mime_types={"image/png"}, + ) + + assert len(images) == 1 + assert images[0].mime_type == "image/png" + assert images[0].to_bytes() == png_bytes + # Compatible images never touch the conversion cache. + assert not (tmp_path / media_utils.CONVERT_CACHE_DIR_NAME).exists() + + +@pytest.mark.asyncio +async def test_resolve_images_converts_bmp_to_jpeg_and_caches(tmp_path, monkeypatch): + monkeypatch.setattr(media_utils, "get_astrbot_temp_path", lambda: str(tmp_path)) + from PIL import Image as PILImage + + bmp_bytes = _make_image_bytes("BMP") + + images = await media_utils.resolve_image_ref_to_images( + _image_data_uri("image/bmp", bmp_bytes), + allowed_mime_types={"image/jpeg", "image/png"}, + ) + + assert len(images) == 1 + assert images[0].mime_type == "image/jpeg" + with PILImage.open(BytesIO(images[0].to_bytes())) as converted: + assert converted.format == "JPEG" + cache_dir = tmp_path / media_utils.CONVERT_CACHE_DIR_NAME + assert len(list(cache_dir.glob("*.jpg"))) == 1 + + +@pytest.mark.asyncio +async def test_resolve_images_alpha_source_prefers_png(tmp_path, monkeypatch): + monkeypatch.setattr(media_utils, "get_astrbot_temp_path", lambda: str(tmp_path)) + from PIL import Image as PILImage + + webp_bytes = _make_image_bytes("WEBP", mode="RGBA", color=(10, 20, 30, 128)) + + images = await media_utils.resolve_image_ref_to_images( + _image_data_uri("image/webp", webp_bytes), + allowed_mime_types={"image/jpeg", "image/png"}, + ) + + assert len(images) == 1 + assert images[0].mime_type == "image/png" + with PILImage.open(BytesIO(images[0].to_bytes())) as converted: + assert converted.format == "PNG" + assert converted.mode == "RGBA" + assert converted.getpixel((0, 0))[3] == 128 + + +@pytest.mark.asyncio +async def test_resolve_images_animated_gif_becomes_montage(tmp_path, monkeypatch): + monkeypatch.setattr(media_utils, "get_astrbot_temp_path", lambda: str(tmp_path)) + from PIL import Image as PILImage + + colors = [(255, 0, 0), (0, 255, 0), (0, 0, 255)] + gif_bytes = _make_animated_gif_bytes(colors) + + images = await media_utils.resolve_image_ref_to_images( + _image_data_uri("image/gif", gif_bytes), + allowed_mime_types={"image/png"}, + ) + + assert len(images) == 1 + assert images[0].mime_type == "image/png" + with PILImage.open(BytesIO(images[0].to_bytes())) as montage: + # 8px frames in a 3x3 grid: 3 colored cells, the rest padded white. + assert montage.size == (24, 24) + rgb = montage.convert("RGB") + assert rgb.getpixel((4, 4)) == colors[0] + assert rgb.getpixel((12, 4)) == colors[1] + assert rgb.getpixel((20, 4)) == colors[2] + for x, y in [(4, 12), (12, 12), (20, 12), (4, 20), (12, 20), (20, 20)]: + assert rgb.getpixel((x, y)) == (255, 255, 255) + + +@pytest.mark.asyncio +async def test_resolve_images_montage_keeps_first_and_last_frame(tmp_path, monkeypatch): + monkeypatch.setattr(media_utils, "get_astrbot_temp_path", lambda: str(tmp_path)) + from PIL import Image as PILImage + + colors = [(i * 12 % 256, 0, 0) for i in range(20)] + gif_bytes = _make_animated_gif_bytes(colors) + + images = await media_utils.resolve_image_ref_to_images( + _image_data_uri("image/gif", gif_bytes), + allowed_mime_types={"image/png"}, + ) + + assert len(images) == 1 + with PILImage.open(BytesIO(images[0].to_bytes())) as montage: + # 20 frames sampled to 9 cells; cell 0 and cell 8 are the endpoints. + rgb = montage.convert("RGB") + assert rgb.getpixel((4, 4)) == colors[0] + assert rgb.getpixel((20, 20)) == colors[-1] + + +@pytest.mark.asyncio +async def test_resolve_images_montage_respects_max_size(tmp_path, monkeypatch): + monkeypatch.setattr(media_utils, "get_astrbot_temp_path", lambda: str(tmp_path)) + from PIL import Image as PILImage + + gif_bytes = _make_animated_gif_bytes( + [(255, 0, 0), (0, 255, 0), (0, 0, 255)], size=(20, 20) + ) + + images = await media_utils.resolve_image_ref_to_images( + _image_data_uri("image/gif", gif_bytes), + allowed_mime_types={"image/png"}, + montage_max_size=30, + ) + + with PILImage.open(BytesIO(images[0].to_bytes())) as montage: + # Native 60x60 canvas scaled down to the 30px cap. + assert montage.size == (30, 30) + + +@pytest.mark.asyncio +async def test_resolve_images_montage_cache_hit_skips_reencode(tmp_path, monkeypatch): + monkeypatch.setattr(media_utils, "get_astrbot_temp_path", lambda: str(tmp_path)) + gif_bytes = _make_animated_gif_bytes([(255, 0, 0), (0, 255, 0), (0, 0, 255)]) + image_ref = _image_data_uri("image/gif", gif_bytes) + + save_calls = 0 + real_save = media_utils._save_current_image_frame + + def counting_save(*args, **kwargs): + nonlocal save_calls + save_calls += 1 + return real_save(*args, **kwargs) + + monkeypatch.setattr(media_utils, "_save_current_image_frame", counting_save) + + first = await media_utils.resolve_image_ref_to_images( + image_ref, allowed_mime_types={"image/png"} + ) + assert save_calls == 1 + + second = await media_utils.resolve_image_ref_to_images( + image_ref, allowed_mime_types={"image/png"} + ) + assert save_calls == 1 # cache hit: no re-encode + assert second[0].base64_data == first[0].base64_data + + +@pytest.mark.asyncio +async def test_resolve_images_single_frame_gif_treated_as_still(tmp_path, monkeypatch): + monkeypatch.setattr(media_utils, "get_astrbot_temp_path", lambda: str(tmp_path)) + from PIL import Image as PILImage + + gif_bytes = _make_animated_gif_bytes([(255, 0, 0)]) + + images = await media_utils.resolve_image_ref_to_images( + _image_data_uri("image/gif", gif_bytes), + allowed_mime_types={"image/jpeg", "image/png"}, + ) + + # A one-frame GIF is a still image: plain conversion, no montage canvas. + assert len(images) == 1 + assert images[0].mime_type == "image/jpeg" + with PILImage.open(BytesIO(images[0].to_bytes())) as still: + assert still.size == (8, 8) + + +@pytest.mark.asyncio +async def test_resolve_images_cache_hit_skips_reencode(tmp_path, monkeypatch): + monkeypatch.setattr(media_utils, "get_astrbot_temp_path", lambda: str(tmp_path)) + bmp_bytes = _make_image_bytes("BMP") + image_ref = _image_data_uri("image/bmp", bmp_bytes) + + save_calls = 0 + real_save = media_utils._save_current_image_frame + + def counting_save(*args, **kwargs): + nonlocal save_calls + save_calls += 1 + return real_save(*args, **kwargs) + + monkeypatch.setattr(media_utils, "_save_current_image_frame", counting_save) + + first = await media_utils.resolve_image_ref_to_images( + image_ref, allowed_mime_types={"image/jpeg"} + ) + assert save_calls == 1 + + second = await media_utils.resolve_image_ref_to_images( + image_ref, allowed_mime_types={"image/jpeg"} + ) + assert save_calls == 1 # cache hit: no re-encode + assert second[0].base64_data == first[0].base64_data + + # After the cache is cleaned, conversion runs again. + cache_dir = tmp_path / media_utils.CONVERT_CACHE_DIR_NAME + for cached_file in cache_dir.iterdir(): + cached_file.unlink() + await media_utils.resolve_image_ref_to_images( + image_ref, allowed_mime_types={"image/jpeg"} + ) + assert save_calls == 2 + + +@pytest.mark.asyncio +async def test_resolve_images_unrestricted_passes_any_format(tmp_path, monkeypatch): + monkeypatch.setattr(media_utils, "get_astrbot_temp_path", lambda: str(tmp_path)) + bmp_bytes = _make_image_bytes("BMP") + image_ref = _image_data_uri("image/bmp", bmp_bytes) + + for allowed in (None, {"*"}): + images = await media_utils.resolve_image_ref_to_images( + image_ref, allowed_mime_types=allowed + ) + assert len(images) == 1 + assert images[0].mime_type == "image/bmp" + assert images[0].to_bytes() == bmp_bytes + + +@pytest.mark.asyncio +async def test_resolve_images_undecodable_skipped_or_passed(tmp_path, monkeypatch): + monkeypatch.setattr(media_utils, "get_astrbot_temp_path", lambda: str(tmp_path)) + svg_ref = _image_data_uri( + "image/svg+xml", b'' + ) + + # Restricted providers skip undecodable images instead of mislabeling them. + assert ( + await media_utils.resolve_image_ref_to_images( + svg_ref, allowed_mime_types={"image/jpeg", "image/png"} + ) + == [] + ) + # Unrestricted resolution keeps the legacy pass-through behavior. + images = await media_utils.resolve_image_ref_to_images(svg_ref) + assert len(images) == 1 + assert images[0].mime_type == "image/svg+xml" + + +@pytest.mark.asyncio +async def test_resolve_image_ref_to_base64_data_delegates(tmp_path, monkeypatch): + monkeypatch.setattr(media_utils, "get_astrbot_temp_path", lambda: str(tmp_path)) + png_bytes = _make_image_bytes("PNG") + + resolved = await media_utils.resolve_image_ref_to_base64_data( + _image_data_uri("image/png", png_bytes) + ) + + assert resolved is not None + assert resolved.mime_type == "image/png" + assert resolved.to_bytes() == png_bytes diff --git a/tests/test_openai_source.py b/tests/test_openai_source.py index 911b76131f..74eeb209ac 100644 --- a/tests/test_openai_source.py +++ b/tests/test_openai_source.py @@ -718,21 +718,21 @@ async def test_prepare_chat_payload_materializes_context_http_image_urls(monkeyp provider = _make_provider() try: - async def fake_resolve_media_ref_to_base64_data( - media_ref: str, + async def fake_resolve_image_ref_to_images( + image_ref: str, *, - media_type: str, + allowed_mime_types=None, + montage_max_size: int = 1280, strict: bool = False, - ) -> ResolvedMediaData: - assert media_ref == "https://example.com/quoted.png" - assert media_type == "image" + ) -> list[ResolvedMediaData]: + assert image_ref == "https://example.com/quoted.png" assert strict is False - return ResolvedMediaData(base64_data="abcd", mime_type="image/png") + return [ResolvedMediaData(base64_data="abcd", mime_type="image/png")] monkeypatch.setattr( openai_source_module, - "resolve_media_ref_to_base64_data", - fake_resolve_media_ref_to_base64_data, + "resolve_image_ref_to_images", + fake_resolve_image_ref_to_images, ) contexts = [ @@ -935,7 +935,7 @@ async def test_resolve_image_part_rejects_invalid_local_file(tmp_path): invalid_file = tmp_path / "not-image.txt" invalid_file.write_text("not an image") - assert await provider._resolve_image_part(str(invalid_file)) is None + assert await provider._resolve_image_parts(str(invalid_file)) == [] finally: await provider.terminate() @@ -947,7 +947,7 @@ async def test_resolve_image_part_rejects_invalid_file_uri(tmp_path): invalid_file = tmp_path / "not-image.txt" invalid_file.write_text("not an image") - assert await provider._resolve_image_part(invalid_file.as_uri()) is None + assert await provider._resolve_image_parts(invalid_file.as_uri()) == [] finally: await provider.terminate() @@ -994,15 +994,17 @@ async def test_materialize_context_image_parts_returns_new_messages(monkeypatch) async def fake_resolve(image_url: str, *, image_detail: str | None = None): assert image_url == "https://example.com/quoted.png" assert image_detail == "high" - return { - "type": "image_url", - "image_url": { - "url": "data:image/png;base64,abcd", - "detail": "high", - }, - } + return [ + { + "type": "image_url", + "image_url": { + "url": "data:image/png;base64,abcd", + "detail": "high", + }, + } + ] - monkeypatch.setattr(provider, "_resolve_image_part", fake_resolve) + monkeypatch.setattr(provider, "_resolve_image_parts", fake_resolve) materialized = await provider._materialize_context_image_parts(context_query) @@ -1077,10 +1079,12 @@ async def test_encode_image_bs64_supports_file_uri(tmp_path): async def test_resolve_image_part_supports_base64_scheme(): provider = _make_provider() try: - assert await provider._resolve_image_part("base64://abcd") == { - "type": "image_url", - "image_url": {"url": "data:image/jpeg;base64,abcd"}, - } + assert await provider._resolve_image_parts("base64://abcd") == [ + { + "type": "image_url", + "image_url": {"url": "data:image/jpeg;base64,abcd"}, + } + ] finally: await provider.terminate() @@ -1096,12 +1100,14 @@ async def test_resolve_image_part_preserves_base64_png_mime_type(): ) image_base64 = base64.b64encode(image_buffer.getvalue()).decode("ascii") - image_part = await provider._resolve_image_part(f"base64://{image_base64}") + image_parts = await provider._resolve_image_parts(f"base64://{image_base64}") - assert image_part == { - "type": "image_url", - "image_url": {"url": f"data:image/png;base64,{image_base64}"}, - } + assert image_parts == [ + { + "type": "image_url", + "image_url": {"url": f"data:image/png;base64,{image_base64}"}, + } + ] finally: await provider.terminate() diff --git a/tests/unit/test_provider_image_formats.py b/tests/unit/test_provider_image_formats.py new file mode 100644 index 0000000000..189f340c44 --- /dev/null +++ b/tests/unit/test_provider_image_formats.py @@ -0,0 +1,388 @@ +import base64 +from io import BytesIO + +import pytest +from PIL import Image as PILImage + +import astrbot.core.utils.media_utils as media_utils +from astrbot.core.provider.provider import DEFAULT_FALLBACK_IMAGE_FORMATS, Provider +from astrbot.core.provider.sources.oai_aihubmix_source import ProviderAIHubMix +from astrbot.core.provider.sources.openai_source import ProviderOpenAIOfficial +from astrbot.core.provider.sources.openrouter_source import ProviderOpenRouter +from astrbot.core.provider.sources.ssycloud_source import ProviderSSYCloud +from astrbot.core.provider.sources.xai_source import ProviderXAI + + +class _DummyProvider(Provider): + supported_image_formats = frozenset({"image/jpeg", "image/png", "image/webp"}) + + def get_current_key(self) -> str: + return "" + + def set_key(self, key: str) -> None: + pass + + async def get_models(self) -> list[str]: + return [] + + async def text_chat(self, **kwargs): + raise NotImplementedError + + +class _UndeclaredProvider(_DummyProvider): + supported_image_formats = None + + +def _make_dummy( + provider_config: dict | None = None, + provider_settings: dict | None = None, +) -> _DummyProvider: + return _DummyProvider(provider_config or {}, provider_settings or {}) + + +def test_config_override_wins_over_class_default(): + provider = _make_dummy({"image_formats": ["jpeg", "png"]}) + assert provider.resolve_allowed_image_formats() == frozenset( + {"image/jpeg", "image/png"} + ) + + +def test_config_star_disables_restriction(): + provider = _make_dummy({"image_formats": ["*"]}) + assert provider.resolve_allowed_image_formats() is None + + +def test_config_accepts_mime_types_and_is_case_insensitive(): + provider = _make_dummy({"image_formats": ["image/webp", " JPEG "]}) + assert provider.resolve_allowed_image_formats() == frozenset( + {"image/webp", "image/jpeg"} + ) + + +def test_class_default_used_when_not_configured(): + provider = _make_dummy() + assert provider.resolve_allowed_image_formats() == frozenset( + {"image/jpeg", "image/png", "image/webp"} + ) + + +def test_fallback_used_when_class_undeclared(): + provider = _UndeclaredProvider({}, {}) + assert provider.resolve_allowed_image_formats() == DEFAULT_FALLBACK_IMAGE_FORMATS + + +def test_unknown_config_entries_fall_back_to_class_default(): + provider = _make_dummy({"image_formats": ["not-a-format"]}) + assert provider.resolve_allowed_image_formats() == frozenset( + {"image/jpeg", "image/png", "image/webp"} + ) + + +def test_unknown_config_entries_log_warning(caplog): + provider = _make_dummy({"image_formats": ["not-a-format"]}) + with caplog.at_level("WARNING"): + provider.resolve_allowed_image_formats() + assert "no valid entries" in caplog.text + assert "not-a-format" in caplog.text + + +def test_aggregators_do_not_inherit_openai_format_set(): + assert ProviderOpenRouter.supported_image_formats is None + assert ProviderAIHubMix.supported_image_formats is None + assert ProviderSSYCloud.supported_image_formats is None + assert ProviderOpenAIOfficial.supported_image_formats == frozenset( + {"image/png", "image/jpeg", "image/webp", "image/gif"} + ) + assert ProviderXAI.supported_image_formats == frozenset({"image/jpeg", "image/png"}) + + +def test_animated_montage_max_size_defaults_to_compress_default(): + provider = _make_dummy() + assert provider.get_animated_montage_max_size() == ( + media_utils.IMAGE_COMPRESS_DEFAULT_MAX_SIZE + ) + + +def test_animated_montage_max_size_reuses_compress_options(): + provider = _make_dummy( + provider_settings={"image_compress_options": {"max_size": 640}}, + ) + assert provider.get_animated_montage_max_size() == 640 + + # Invalid values fall back instead of breaking montage generation. + provider = _make_dummy( + provider_settings={"image_compress_options": {"max_size": "big"}}, + ) + assert provider.get_animated_montage_max_size() == ( + media_utils.IMAGE_COMPRESS_DEFAULT_MAX_SIZE + ) + provider = _make_dummy( + provider_settings={"image_compress_options": {"max_size": 0}}, + ) + assert provider.get_animated_montage_max_size() == 1 + + +def _animated_gif_data_uri(colors: list[tuple[int, int, int]]) -> str: + frames = [PILImage.new("RGB", (8, 8), color) for color in colors] + buffer = BytesIO() + frames[0].save( + buffer, + "GIF", + save_all=True, + append_images=frames[1:], + duration=50, + loop=0, + ) + return f"data:image/gif;base64,{base64.b64encode(buffer.getvalue()).decode()}" + + +@pytest.mark.asyncio +async def test_xai_animated_gif_becomes_montage(tmp_path, monkeypatch): + monkeypatch.setattr(media_utils, "get_astrbot_temp_path", lambda: str(tmp_path)) + provider = ProviderXAI( + { + "id": "test-xai", + "type": "xai_chat_completion", + "model": "grok-4", + "key": ["test-key"], + }, + {}, + ) + try: + gif_ref = _animated_gif_data_uri([(255, 0, 0), (0, 255, 0), (0, 0, 255)]) + message = await provider.assemble_context("look", image_urls=[gif_ref]) + + content = message["content"] + image_blocks = [block for block in content if block["type"] == "image_url"] + assert len(image_blocks) == 1 + url = image_blocks[0]["image_url"]["url"] + # xAI only accepts JPEG/PNG; the montage is flattened, so JPEG wins. + assert url.startswith("data:image/jpeg;base64,") + finally: + await provider.terminate() + + +@pytest.mark.asyncio +async def test_openai_animated_gif_becomes_single_montage_block(tmp_path, monkeypatch): + monkeypatch.setattr(media_utils, "get_astrbot_temp_path", lambda: str(tmp_path)) + provider = ProviderOpenAIOfficial( + { + "id": "test-openai", + "type": "openai_chat_completion", + "model": "gpt-4o-mini", + "key": ["test-key"], + # Stale per-provider strategy keys from earlier builds stay inert. + "animated_image_strategy": "multi_frame", + "animated_image_max_frames": 3, + }, + {}, + ) + try: + gif_ref = _animated_gif_data_uri( + [(255, 0, 0), (0, 255, 0), (0, 0, 255), (255, 255, 0), (0, 255, 255)] + ) + message = await provider.assemble_context("look", image_urls=[gif_ref]) + + image_blocks = [ + block for block in message["content"] if block["type"] == "image_url" + ] + assert len(image_blocks) == 1 + assert image_blocks[0]["image_url"]["url"].startswith("data:image/") + finally: + await provider.terminate() + + +@pytest.mark.asyncio +async def test_openai_montage_respects_compress_max_size(tmp_path, monkeypatch): + monkeypatch.setattr(media_utils, "get_astrbot_temp_path", lambda: str(tmp_path)) + provider = ProviderOpenAIOfficial( + { + "id": "test-openai", + "type": "openai_chat_completion", + "model": "gpt-4o-mini", + "key": ["test-key"], + # PNG keeps pixels lossless for exact montage assertions. + "image_formats": ["png"], + }, + {"image_compress_options": {"max_size": 24}}, + ) + try: + gif_ref = _animated_gif_data_uri( + [(255, 0, 0), (0, 255, 0), (0, 0, 255), (255, 255, 0)] + ) + message = await provider.assemble_context("look", image_urls=[gif_ref]) + + image_blocks = [ + block for block in message["content"] if block["type"] == "image_url" + ] + assert len(image_blocks) == 1 + url = image_blocks[0]["image_url"]["url"] + assert url.startswith("data:image/png;base64,") + _header, encoded = url.split(",", 1) + with PILImage.open(BytesIO(base64.b64decode(encoded))) as montage: + # 8px frames x 3 columns = 24px, already within the 24px cap. + assert montage.size == (24, 24) + rgb = montage.convert("RGB") + assert rgb.getpixel((4, 4)) == (255, 0, 0) + assert rgb.getpixel((12, 4)) == (0, 255, 0) + assert rgb.getpixel((20, 4)) == (0, 0, 255) + # The 4th frame fills cell 3; no blank cells for a 4-frame GIF. + assert rgb.getpixel((4, 12)) == (255, 255, 0) + finally: + await provider.terminate() + + +@pytest.mark.asyncio +async def test_anthropic_context_skips_unsupported_format(): + from astrbot.core.provider.sources.anthropic_source import ProviderAnthropic + + provider = ProviderAnthropic( + { + "id": "test-anthropic", + "type": "anthropic_chat_completion", + "model": "claude-sonnet-4-5", + "key": ["test-key"], + }, + {}, + ) + try: + # A BMP data URI must not be mislabeled as image/jpeg in the payload. + bmp_buffer = BytesIO() + PILImage.new("RGB", (4, 4), (1, 2, 3)).save(bmp_buffer, "BMP") + bmp_data_uri = ( + f"data:image/bmp;base64,{base64.b64encode(bmp_buffer.getvalue()).decode()}" + ) + _system_prompt, messages = provider._prepare_payload( + [ + { + "role": "user", + "content": [ + {"type": "text", "text": "look"}, + {"type": "image_url", "image_url": {"url": bmp_data_uri}}, + ], + } + ] + ) + + content = messages[0]["content"] + image_blocks = [block for block in content if block.get("type") == "image"] + assert image_blocks == [] + finally: + await provider.terminate() + + +def _make_openai(provider_config: dict) -> ProviderOpenAIOfficial: + config = { + "id": "test-openai", + "type": "openai_chat_completion", + "model": "deepseek-v4-flash-vision-exp", + "key": ["test-key"], + **provider_config, + } + return ProviderOpenAIOfficial(config, {}) + + +@pytest.mark.asyncio +async def test_vendor_brand_overrides_adapter_declared_formats(): + provider = _make_openai({"provider": "xai", "type": "openai_responses"}) + try: + assert provider.resolve_allowed_image_formats() == frozenset( + {"image/jpeg", "image/png"} + ) + finally: + await provider.terminate() + + +@pytest.mark.asyncio +async def test_unknown_brand_keeps_adapter_declared_formats(): + provider = _make_openai({"provider": "some-unknown-vendor"}) + try: + assert provider.resolve_allowed_image_formats() == frozenset( + {"image/png", "image/jpeg", "image/webp", "image/gif"} + ) + finally: + await provider.terminate() + + +@pytest.mark.asyncio +async def test_config_override_wins_over_vendor_brand(): + provider = _make_openai({"provider": "xai", "image_formats": ["webp"]}) + try: + assert provider.resolve_allowed_image_formats() == frozenset({"image/webp"}) + finally: + await provider.terminate() + + +def test_vendor_map_matches_adapter_declarations(): + from astrbot.core.provider.sources.anthropic_source import ProviderAnthropic + from astrbot.core.provider.sources.gemini_source import ProviderGoogleGenAI + from astrbot.core.provider.sources.openai_source import ProviderOpenAIOfficial + from astrbot.core.provider.sources.xai_source import ProviderXAI + from astrbot.core.provider.sources.zhipu_source import ProviderZhipu + from astrbot.core.utils.media_utils import VENDOR_IMAGE_FORMATS + + assert ( + ProviderOpenAIOfficial.supported_image_formats is VENDOR_IMAGE_FORMATS["openai"] + ) + assert ProviderXAI.supported_image_formats is VENDOR_IMAGE_FORMATS["xai"] + assert ProviderZhipu.supported_image_formats is VENDOR_IMAGE_FORMATS["zhipu"] + assert ProviderGoogleGenAI.supported_image_formats is VENDOR_IMAGE_FORMATS["google"] + assert ( + ProviderAnthropic.supported_image_formats is VENDOR_IMAGE_FORMATS["anthropic"] + ) + + +def test_vendor_map_matches_vendor_adapter_declarations(): + from astrbot.core.provider.sources.groq_source import ProviderGroq + from astrbot.core.provider.sources.kimi_code_source import ProviderKimiCode + from astrbot.core.provider.sources.minimax_token_plan_source import ( + ProviderMiniMaxTokenPlan, + ) + from astrbot.core.provider.sources.xiaomi_source import ProviderXiaomi + from astrbot.core.provider.sources.xiaomi_token_plan_source import ( + ProviderXiaomiTokenPlan, + ) + from astrbot.core.utils.media_utils import VENDOR_IMAGE_FORMATS + + assert ProviderGroq.supported_image_formats is VENDOR_IMAGE_FORMATS["groq"] + assert ProviderXiaomi.supported_image_formats is VENDOR_IMAGE_FORMATS["xiaomi"] + assert ProviderKimiCode.supported_image_formats is VENDOR_IMAGE_FORMATS["kimi-code"] + assert ( + ProviderMiniMaxTokenPlan.supported_image_formats + is VENDOR_IMAGE_FORMATS["minimax-token-plan"] + ) + assert ( + ProviderXiaomiTokenPlan.supported_image_formats + is VENDOR_IMAGE_FORMATS["xiaomi-token-plan"] + ) + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("brand", "expected"), + [ + ("deepseek", {"image/jpeg", "image/png", "image/webp", "image/gif"}), + ( + "moonshot", + { + "image/jpeg", + "image/png", + "image/webp", + "image/gif", + "image/bmp", + "image/heic", + "image/heif", + }, + ), + ("minimax", {"image/jpeg", "image/png", "image/webp", "image/gif"}), + ("xiaomi", {"image/jpeg", "image/png", "image/webp", "image/gif", "image/bmp"}), + ("groq", {"image/jpeg", "image/png"}), + ("nvidia", {"image/jpeg", "image/png"}), + ], +) +async def test_vendor_brand_format_sets(brand, expected): + provider = _make_openai({"provider": brand}) + try: + assert provider.resolve_allowed_image_formats() == frozenset(expected) + finally: + await provider.terminate()