From 229b3f5d42de16d3cb47bc8bbb89824d205413f9 Mon Sep 17 00:00:00 2001 From: piexian <64474352+piexian@users.noreply.github.com> Date: Sat, 15 Aug 2026 13:52:39 +0800 Subject: [PATCH 1/8] feat: adapt outbound images to each provider's supported formats, with animated-image strategies and local conversion cache --- astrbot/core/config/default.py | 29 ++ astrbot/core/provider/entities.py | 22 +- astrbot/core/provider/provider.py | 77 +++- .../core/provider/sources/anthropic_source.py | 113 +++--- .../core/provider/sources/gemini_source.py | 51 ++- .../provider/sources/oai_aihubmix_source.py | 3 + .../core/provider/sources/openai_source.py | 83 +++-- .../provider/sources/openrouter_source.py | 4 + .../core/provider/sources/ssycloud_source.py | 3 + astrbot/core/provider/sources/xai_source.py | 3 + astrbot/core/provider/sources/zhipu_source.py | 3 + astrbot/core/utils/media_utils.py | 330 +++++++++++++++++- tests/test_media_utils.py | 243 +++++++++++++ tests/test_openai_source.py | 63 ++-- tests/unit/test_provider_image_formats.py | 216 ++++++++++++ 15 files changed, 1102 insertions(+), 141 deletions(-) create mode 100644 tests/unit/test_provider_image_formats.py diff --git a/astrbot/core/config/default.py b/astrbot/core/config/default.py index b1bb8b3114..de472dbc03 100644 --- a/astrbot/core/config/default.py +++ b/astrbot/core/config/default.py @@ -2130,6 +2130,35 @@ "render_type": "checkbox", "hint": "模型支持的模态及能力。", }, + "image_formats": { + "description": "图片格式支持", + "type": "list", + "items": {"type": "string"}, + "options": ["jpeg", "png", "webp", "gif", "bmp", "heic", "*"], + "labels": [ + "JPEG", + "PNG", + "WebP", + "GIF", + "BMP", + "HEIC", + "不限制", + ], + "render_type": "checkbox", + "hint": "发送给该提供商前允许的图片格式,不兼容的格式会自动转换。留空使用内置默认值(官方 API 按其文档,未知第三方默认仅 jpeg/png),选 * 表示不限制。", + }, + "animated_image_strategy": { + "description": "动图处理策略", + "type": "string", + "options": ["first_frame", "multi_frame"], + "labels": ["仅首帧", "多帧抽取"], + "hint": "GIF 等动图发送给模型时的处理方式:仅取首帧(省 token),或按时长均匀抽帧后作为多张图片发送。", + }, + "animated_image_max_frames": { + "description": "动图最大抽帧数", + "type": "int", + "hint": "多帧抽取策略下最多发送的帧数(1-16),默认 4。帧数越多 token 消耗越大。", + }, "custom_headers": { "description": "自定义请求头", "type": "dict", diff --git a/astrbot/core/provider/entities.py b/astrbot/core/provider/entities.py index 2fab40ca78..12342e4931 100644 --- a/astrbot/core/provider/entities.py +++ b/astrbot/core/provider/entities.py @@ -23,7 +23,7 @@ from astrbot.core.agent.tool import ToolSet from astrbot.core.db.po import Conversation from astrbot.core.message.message_event_result import MessageChain -from astrbot.core.utils.media_utils import MediaResolver +from astrbot.core.utils.media_utils import MediaResolver, resolve_image_ref_to_images class ProviderType(enum.Enum): @@ -208,19 +208,17 @@ async def assemble_context(self) -> dict: # 3. 图片内容 if self.image_urls: for image_url in self.image_urls: - image_data = await MediaResolver( - image_url, - media_type="image", - ).to_base64_data() - if not image_data: + image_datas = await resolve_image_ref_to_images(image_url) + if not image_datas: logger.warning("图片预处理结果为空,将忽略。") continue - content_blocks.append( - { - "type": "image_url", - "image_url": {"url": image_data.to_data_url()}, - }, - ) + for image_data in image_datas: + content_blocks.append( + { + "type": "image_url", + "image_url": {"url": image_data.to_data_url()}, + }, + ) # 4. 音频内容 if self.audio_urls: diff --git a/astrbot/core/provider/provider.py b/astrbot/core/provider/provider.py index 891bfdea9e..1acc27b1c1 100644 --- a/astrbot/core/provider/provider.py +++ b/astrbot/core/provider/provider.py @@ -2,7 +2,7 @@ import asyncio import os from collections.abc import AsyncGenerator -from typing import Literal, TypeAlias, Union +from typing import ClassVar, Literal, TypeAlias, Union from astrbot.core.agent.message import ContentPart, Message, is_checkpoint_message from astrbot.core.agent.tool import ToolSet @@ -14,6 +14,16 @@ ) from astrbot.core.provider.register import provider_cls_map from astrbot.core.utils.astrbot_path import get_astrbot_path +from astrbot.core.utils.media_utils import ( + ANIMATED_DEFAULT_MAX_FRAMES, + ANIMATED_MAX_FRAMES_LIMIT, + ANIMATED_STRATEGY_FIRST_FRAME, + ANIMATED_STRATEGY_MULTI_FRAME, + IMAGE_SHORT_MIME_TYPES, +) + +DEFAULT_FALLBACK_IMAGE_FORMATS = frozenset({"image/jpeg", "image/png"}) +"""Conservative image formats assumed for providers without a declared set.""" Providers: TypeAlias = Union[ "Provider", @@ -66,6 +76,14 @@ async def test(self) -> None: class Provider(AbstractProvider): """Chat Provider""" + supported_image_formats: ClassVar[frozenset[str] | None] = None + """Image MIME types the provider officially accepts. + + ``None`` means undeclared and falls back to DEFAULT_FALLBACK_IMAGE_FORMATS. + Aggregator subclasses (e.g. OpenRouter) must set this back to ``None`` + explicitly so they do not inherit an official vendor's format set. + """ + def __init__( self, provider_config: dict, @@ -74,6 +92,63 @@ def __init__( super().__init__(provider_config) self.provider_settings = provider_settings + def resolve_allowed_image_formats(self) -> frozenset[str] | None: + """Resolve the image MIME types allowed for this provider instance. + + Priority: ``provider_config["image_formats"]`` (per-instance override, + short names like ``jpeg`` or MIME types, ``*`` disables restriction) > + the class-level ``supported_image_formats`` > the conservative + DEFAULT_FALLBACK_IMAGE_FORMATS (jpeg/png). + + Returns: + The allowed MIME types, or ``None`` when unrestricted. + """ + configured = self.provider_config.get("image_formats") + if configured: + normalized = { + str(value).strip().lower() for value in configured if str(value).strip() + } + if "*" in normalized: + return None + mapped = { + IMAGE_SHORT_MIME_TYPES.get(value, value) + for value in normalized + if value.startswith("image/") or value in IMAGE_SHORT_MIME_TYPES + } + if mapped: + return frozenset(mapped) + if self.supported_image_formats is not None: + return self.supported_image_formats + return DEFAULT_FALLBACK_IMAGE_FORMATS + + def get_animated_image_strategy(self) -> tuple[str, int]: + """Read the animated image handling strategy from the provider config. + + Returns: + Tuple of ``(strategy, max_frames)`` where strategy is + ``first_frame`` or ``multi_frame`` and max_frames is clamped to + ``[1, 16]``. + """ + strategy = str( + self.provider_config.get("animated_image_strategy") + or ANIMATED_STRATEGY_FIRST_FRAME + ) + if strategy not in ( + ANIMATED_STRATEGY_FIRST_FRAME, + ANIMATED_STRATEGY_MULTI_FRAME, + ): + strategy = ANIMATED_STRATEGY_FIRST_FRAME + raw_max_frames = self.provider_config.get("animated_image_max_frames") + try: + max_frames = ( + ANIMATED_DEFAULT_MAX_FRAMES + if raw_max_frames is None + else int(raw_max_frames) + ) + except (TypeError, ValueError): + max_frames = ANIMATED_DEFAULT_MAX_FRAMES + return strategy, min(max(max_frames, 1), ANIMATED_MAX_FRAMES_LIMIT) + @abc.abstractmethod def get_current_key(self) -> str: raise NotImplementedError diff --git a/astrbot/core/provider/sources/anthropic_source.py b/astrbot/core/provider/sources/anthropic_source.py index 27cc459622..111b204918 100644 --- a/astrbot/core/provider/sources/anthropic_source.py +++ b/astrbot/core/provider/sources/anthropic_source.py @@ -18,7 +18,8 @@ from astrbot.core.provider.func_tool_manager import ToolSet from astrbot.core.utils.media_utils import ( describe_media_ref, - resolve_media_ref_to_base64_data, + detect_image_mime_type, + resolve_image_ref_to_images, ) from astrbot.core.utils.network_utils import ( create_proxy_client, @@ -37,6 +38,11 @@ class ProviderAnthropic(Provider): _PROMPT_CACHE_CONTROL = {"type": "ephemeral"} + supported_image_formats = frozenset( + {"image/jpeg", "image/png", "image/gif", "image/webp"} + ) + """Formats accepted by the official Anthropic vision API.""" + @staticmethod def _ensure_usable_response( llm_response: LLMResponse, @@ -264,19 +270,32 @@ def _prepare_payload(self, messages: list[dict]): _, base64_data = url.split(",", 1) # Detect actual image format from binary data image_bytes = base64.b64decode(base64_data) - media_type = self._detect_image_mime_type( - image_bytes + media_type = detect_image_mime_type( + image_bytes, + default_mime_type=None, ) - converted_content.append( - { - "type": "image", - "source": { - "type": "base64", - "media_type": media_type, - "data": base64_data, - }, - } + allowed_formats = ( + self.resolve_allowed_image_formats() ) + if media_type and ( + allowed_formats is None + or media_type in allowed_formats + ): + converted_content.append( + { + "type": "image", + "source": { + "type": "base64", + "media_type": media_type, + "data": base64_data, + }, + } + ) + else: + logger.warning( + "Skipping context image with unsupported or undetectable format: %s...", + url[:50], + ) except ValueError: logger.warning( f"Failed to parse image data URI: {url[:50]}..." @@ -884,17 +903,16 @@ async def text_chat_stream( ): yield llm_response - def _detect_image_mime_type(self, data: bytes) -> str: - """根据图片二进制数据的 magic bytes 检测 MIME 类型""" - if data[:8] == b"\x89PNG\r\n\x1a\n": - return "image/png" - if data[:2] == b"\xff\xd8": - return "image/jpeg" - if data[:6] in (b"GIF87a", b"GIF89a"): - return "image/gif" - if data[:4] == b"RIFF" and data[8:12] == b"WEBP": - return "image/webp" - return "image/jpeg" + async def _image_ref_to_images(self, image_ref: str, *, strict: bool = False): + """Resolve an image ref with this provider's format adaptation applied.""" + strategy, max_frames = self.get_animated_image_strategy() + return await resolve_image_ref_to_images( + image_ref, + allowed_mime_types=self.resolve_allowed_image_formats(), + animated_strategy=strategy, + animated_max_frames=max_frames, + strict=strict, + ) async def assemble_context( self, @@ -905,23 +923,23 @@ async def assemble_context( ): """组装上下文,支持文本和图片""" - async def resolve_image_url(image_url: str) -> dict | None: - image_data = await resolve_media_ref_to_base64_data( - image_url, - media_type="image", - ) - if not image_data: + async def resolve_image_url(image_url: str) -> list[dict]: + image_datas = await self._image_ref_to_images(image_url) + if not image_datas: logger.warning("图片预处理结果为空,将忽略。") - return None - - return { - "type": "image", - "source": { - "type": "base64", - "media_type": image_data.mime_type, - "data": image_data.base64_data, - }, - } + return [] + + return [ + { + "type": "image", + "source": { + "type": "base64", + "media_type": image_data.mime_type, + "data": image_data.base64_data, + }, + } + for image_data in image_datas + ] content = [] @@ -943,9 +961,7 @@ async def resolve_image_url(image_url: str) -> dict | None: if isinstance(block, TextPart): content.append({"type": "text", "text": block.text}) elif isinstance(block, ImageURLPart): - image_dict = await resolve_image_url(block.image_url.url) - if image_dict: - content.append(image_dict) + content.extend(await resolve_image_url(block.image_url.url)) elif isinstance(block, AudioURLPart): content.append({"type": "text", "text": "[Audio]"}) else: @@ -954,9 +970,7 @@ async def resolve_image_url(image_url: str) -> dict | None: # 3. 图片内容 if image_urls: for image_url in image_urls: - image_dict = await resolve_image_url(image_url) - if image_dict: - content.append(image_dict) + content.extend(await resolve_image_url(image_url)) if audio_urls: for _audio_path in audio_urls: content.append({"type": "text", "text": "[Audio]"}) @@ -977,15 +991,12 @@ async def resolve_image_url(image_url: str) -> dict | None: async def encode_image_bs64(self, image_url: str) -> tuple[str, str]: """将图片转换为 base64,同时检测实际 MIME 类型""" - image_data = await resolve_media_ref_to_base64_data( - image_url, - media_type="image", - strict=True, - ) - if image_data is None: + image_datas = await self._image_ref_to_images(image_url, strict=True) + if not image_datas: raise RuntimeError( f"Failed to encode image data: {describe_media_ref(image_url)}" ) + image_data = image_datas[0] return image_data.to_data_url(), image_data.mime_type def get_current_key(self) -> str: diff --git a/astrbot/core/provider/sources/gemini_source.py b/astrbot/core/provider/sources/gemini_source.py index abf7bb7cf8..5bf52090b1 100644 --- a/astrbot/core/provider/sources/gemini_source.py +++ b/astrbot/core/provider/sources/gemini_source.py @@ -21,6 +21,7 @@ from astrbot.core.provider.func_tool_manager import ToolSet from astrbot.core.utils.media_utils import ( describe_media_ref, + resolve_image_ref_to_images, resolve_media_ref_to_base64_data, ) from astrbot.core.utils.network_utils import is_connection_error, log_connection_failure @@ -44,6 +45,11 @@ def filter(self, record): "Google Gemini Chat Completion 提供商适配器", ) class ProviderGoogleGenAI(Provider): + supported_image_formats = frozenset( + {"image/png", "image/jpeg", "image/webp", "image/heic", "image/heif"} + ) + """Formats accepted by the official Gemini vision API.""" + CATEGORY_MAPPING = { "harassment": types.HarmCategory.HARM_CATEGORY_HARASSMENT, "hate_speech": types.HarmCategory.HARM_CATEGORY_HATE_SPEECH, @@ -982,18 +988,24 @@ async def assemble_context( ): """组装上下文。""" - async def resolve_image_part(image_url: str) -> dict | None: - image_data = await resolve_media_ref_to_base64_data( + async def resolve_image_part(image_url: str) -> list[dict]: + strategy, max_frames = self.get_animated_image_strategy() + image_datas = await resolve_image_ref_to_images( image_url, - media_type="image", + allowed_mime_types=self.resolve_allowed_image_formats(), + animated_strategy=strategy, + animated_max_frames=max_frames, ) - if not image_data: + if not image_datas: logger.warning("Image preprocessing returned no data; ignoring it.") - return None - return { - "type": "image_url", - "image_url": {"url": image_data.to_data_url()}, - } + return [] + return [ + { + "type": "image_url", + "image_url": {"url": image_data.to_data_url()}, + } + for image_data in image_datas + ] async def resolve_audio_part(audio_path: str) -> dict | None: try: @@ -1037,9 +1049,8 @@ async def resolve_audio_part(audio_path: str) -> dict | None: if isinstance(part, TextPart): content_blocks.append({"type": "text", "text": part.text}) elif isinstance(part, ImageURLPart): - image_part = await resolve_image_part(part.image_url.url) - if image_part: - content_blocks.append(image_part) + image_parts = await resolve_image_part(part.image_url.url) + content_blocks.extend(image_parts) elif isinstance(part, AudioURLPart): audio_part = await resolve_audio_part(part.audio_url.url) if audio_part: @@ -1052,9 +1063,8 @@ async def resolve_audio_part(audio_path: str) -> dict | None: # 3. 图片内容 if image_urls: for image_url in image_urls: - image_part = await resolve_image_part(image_url) - if image_part: - content_blocks.append(image_part) + image_parts = await resolve_image_part(image_url) + content_blocks.extend(image_parts) if audio_urls: for audio_path in audio_urls: @@ -1078,16 +1088,19 @@ async def resolve_audio_part(audio_path: str) -> dict | None: async def encode_image_bs64(self, image_url: str) -> str: """将图片转换为 base64""" - image_data = await resolve_media_ref_to_base64_data( + strategy, max_frames = self.get_animated_image_strategy() + image_datas = await resolve_image_ref_to_images( image_url, - media_type="image", + allowed_mime_types=self.resolve_allowed_image_formats(), + animated_strategy=strategy, + animated_max_frames=max_frames, strict=True, ) - if image_data is None: + if not image_datas: raise RuntimeError( f"Failed to encode image data: {describe_media_ref(image_url)}" ) - return image_data.to_data_url() + return image_datas[0].to_data_url() async def _close_httpx_client(self, client: httpx.AsyncClient | None) -> None: """Safely close an httpx.AsyncClient, swallowing errors for idempotency.""" diff --git a/astrbot/core/provider/sources/oai_aihubmix_source.py b/astrbot/core/provider/sources/oai_aihubmix_source.py index ca8ad59596..05404743b7 100644 --- a/astrbot/core/provider/sources/oai_aihubmix_source.py +++ b/astrbot/core/provider/sources/oai_aihubmix_source.py @@ -6,6 +6,9 @@ "aihubmix_chat_completion", "AIHubMix Chat Completion Provider Adapter" ) class ProviderAIHubMix(ProviderOpenAIOfficial): + # Aggregator gateway: do not inherit the official OpenAI format set. + supported_image_formats = None + def __init__( self, provider_config: dict, diff --git a/astrbot/core/provider/sources/openai_source.py b/astrbot/core/provider/sources/openai_source.py index f7870b7137..d99ff2ab5f 100644 --- a/astrbot/core/provider/sources/openai_source.py +++ b/astrbot/core/provider/sources/openai_source.py @@ -30,7 +30,9 @@ from astrbot.core.message.message_event_result import MessageChain from astrbot.core.provider.entities import LLMResponse, TokenUsage, ToolCallsResult from astrbot.core.utils.media_utils import ( + ResolvedMediaData, describe_media_ref, + resolve_image_ref_to_images, resolve_media_ref_to_base64_data, ) from astrbot.core.utils.network_utils import ( @@ -51,6 +53,11 @@ class ProviderOpenAIOfficial(Provider): _ERROR_TEXT_CANDIDATE_MAX_CHARS = 4096 + supported_image_formats = frozenset( + {"image/png", "image/jpeg", "image/webp", "image/gif"} + ) + """Formats accepted by the official OpenAI vision API (non-animated GIF).""" + @classmethod def _truncate_error_text_candidate(cls, text: str) -> str: if len(text) <= cls._ERROR_TEXT_CANDIDATE_MAX_CHARS: @@ -176,37 +183,53 @@ def _is_invalid_attachment_error(self, error: Exception) -> bool: return True return False - async def _image_ref_to_data_url( + async def _image_ref_to_images( self, image_ref: str, *, mode: Literal["safe", "strict"] = "safe", - ) -> str | None: - image_data = await resolve_media_ref_to_base64_data( + ) -> list[ResolvedMediaData]: + """Resolve an image ref with this provider's format adaptation applied.""" + strategy, max_frames = self.get_animated_image_strategy() + return await resolve_image_ref_to_images( image_ref, - media_type="image", + allowed_mime_types=self.resolve_allowed_image_formats(), + animated_strategy=strategy, + animated_max_frames=max_frames, strict=mode == "strict", ) - return image_data.to_data_url() if image_data else None - async def _resolve_image_part( + async def _image_ref_to_data_url( + self, + image_ref: str, + *, + mode: Literal["safe", "strict"] = "safe", + ) -> str | None: + images = await self._image_ref_to_images(image_ref, mode=mode) + return images[0].to_data_url() if images else None + + async def _resolve_image_parts( self, image_url: str, *, image_detail: str | None = None, - ) -> dict | None: - image_data = await self._image_ref_to_data_url(image_url, mode="safe") - if not image_data: + ) -> list[dict]: + images = await self._image_ref_to_images(image_url, mode="safe") + if not images: logger.warning("图片预处理结果为空,将忽略。") - return None - image_payload = {"url": image_data} - - if image_detail: - image_payload["detail"] = image_detail - return { - "type": "image_url", - "image_url": image_payload, - } + return [] + parts = [] + for image_data in images: + image_payload = {"url": image_data.to_data_url()} + if image_detail: + image_payload["detail"] = image_detail + parts.append( + { + "type": "image_url", + "image_url": image_payload, + } + ) + return parts def _extract_image_part_info(self, part: dict) -> tuple[str | None, str | None]: if not isinstance(part, dict) or part.get("type") != "image_url": @@ -266,7 +289,7 @@ async def _resolve_audio_part(self, audio_ref: str) -> dict | None: }, } - async def _transform_content_part(self, part: dict) -> dict: + async def _transform_content_part(self, part: dict) -> dict | list[dict]: if not isinstance(part, dict): return part @@ -276,7 +299,7 @@ async def _transform_content_part(self, part: dict) -> dict: return part try: - resolved_part = await self._resolve_image_part( + resolved_parts = await self._resolve_image_parts( url, image_detail=image_detail ) except Exception as exc: @@ -287,7 +310,7 @@ async def _transform_content_part(self, part: dict) -> dict: ) return part - return resolved_part or part + return resolved_parts or part if part.get("type") == "audio_url": audio_ref = self._extract_audio_part_info(part) @@ -303,7 +326,13 @@ async def _materialize_message_image_parts(self, message: dict) -> dict: if not isinstance(content, list): return {**message} - new_content = [await self._transform_content_part(part) for part in content] + new_content: list[dict] = [] + for part in content: + transformed = await self._transform_content_part(part) + if isinstance(transformed, list): + new_content.extend(transformed) + else: + new_content.append(transformed) return {**message, "content": new_content} async def _materialize_context_image_parts( @@ -1394,11 +1423,10 @@ async def assemble_context( if isinstance(part, TextPart): content_blocks.append({"type": "text", "text": part.text}) elif isinstance(part, ImageURLPart): - image_part = await self._resolve_image_part( + image_parts = await self._resolve_image_parts( part.image_url.url, ) - if image_part: - content_blocks.append(image_part) + content_blocks.extend(image_parts) elif isinstance(part, AudioURLPart): audio_part = await self._resolve_audio_part(part.audio_url.url) if audio_part: @@ -1409,9 +1437,8 @@ async def assemble_context( # 3. 图片内容 if image_urls: for image_url in image_urls: - image_part = await self._resolve_image_part(image_url) - if image_part: - content_blocks.append(image_part) + image_parts = await self._resolve_image_parts(image_url) + content_blocks.extend(image_parts) if audio_urls: for audio_path in audio_urls: diff --git a/astrbot/core/provider/sources/openrouter_source.py b/astrbot/core/provider/sources/openrouter_source.py index a308ad309d..bcac162aa4 100644 --- a/astrbot/core/provider/sources/openrouter_source.py +++ b/astrbot/core/provider/sources/openrouter_source.py @@ -6,6 +6,10 @@ "openrouter_chat_completion", "OpenRouter Chat Completion Provider Adapter" ) class ProviderOpenRouter(ProviderOpenAIOfficial): + # Aggregator gateway: the backend model varies, so do not inherit the + # official OpenAI format set; use the conservative fallback instead. + supported_image_formats = None + def __init__( self, provider_config: dict, diff --git a/astrbot/core/provider/sources/ssycloud_source.py b/astrbot/core/provider/sources/ssycloud_source.py index 04a2ff677b..837d7376b7 100644 --- a/astrbot/core/provider/sources/ssycloud_source.py +++ b/astrbot/core/provider/sources/ssycloud_source.py @@ -12,6 +12,9 @@ class ProviderSSYCloud(ProviderOpenAIOfficial): """SSYCloud provider using its OpenAI-compatible Chat Completions API.""" + # Aggregator gateway: do not inherit the official OpenAI format set. + supported_image_formats = None + def __init__(self, provider_config: dict, provider_settings: dict) -> None: """Initialize the SSYCloud client with provider defaults. diff --git a/astrbot/core/provider/sources/xai_source.py b/astrbot/core/provider/sources/xai_source.py index b7b432b49a..0e8ca2c48c 100644 --- a/astrbot/core/provider/sources/xai_source.py +++ b/astrbot/core/provider/sources/xai_source.py @@ -6,6 +6,9 @@ "xai_chat_completion", "xAI Chat Completion Provider Adapter" ) class ProviderXAI(ProviderOpenAIOfficial): + supported_image_formats = frozenset({"image/jpeg", "image/png"}) + """The official xAI vision API accepts only JPEG and PNG.""" + def __init__( self, provider_config: dict, diff --git a/astrbot/core/provider/sources/zhipu_source.py b/astrbot/core/provider/sources/zhipu_source.py index ed4bc0bf89..70e9d22c64 100644 --- a/astrbot/core/provider/sources/zhipu_source.py +++ b/astrbot/core/provider/sources/zhipu_source.py @@ -8,6 +8,9 @@ @register_provider_adapter("zhipu_chat_completion", "智谱 Chat Completion 提供商适配器") class ProviderZhipu(ProviderOpenAIOfficial): + supported_image_formats = frozenset({"image/jpeg", "image/png"}) + """GLM vision officially guarantees JPEG/PNG; keep it conservative.""" + def __init__( self, provider_config: dict, diff --git a/astrbot/core/utils/media_utils.py b/astrbot/core/utils/media_utils.py index 6cd58b92ec..8d0e8ae9e3 100644 --- a/astrbot/core/utils/media_utils.py +++ b/astrbot/core/utils/media_utils.py @@ -7,12 +7,13 @@ import asyncio import base64 import binascii +import hashlib import io import mimetypes import os import shutil import subprocess -from collections.abc import AsyncIterator +from collections.abc import AsyncIterator, Collection from contextlib import asynccontextmanager from dataclasses import dataclass, field from pathlib import Path @@ -70,6 +71,49 @@ "AVIF": "image/avif", } +IMAGE_SHORT_MIME_TYPES = { + "jpeg": "image/jpeg", + "jpg": "image/jpeg", + "png": "image/png", + "gif": "image/gif", + "webp": "image/webp", + "bmp": "image/bmp", + "tif": "image/tiff", + "tiff": "image/tiff", + "avif": "image/avif", + "heic": "image/heic", + "heif": "image/heif", +} +"""User-facing short image format names mapped to MIME types.""" + +ANIMATED_STRATEGY_FIRST_FRAME = "first_frame" +"""Keep only the first frame of an animated image.""" + +ANIMATED_STRATEGY_MULTI_FRAME = "multi_frame" +"""Extract multiple evenly spaced frames from an animated image.""" + +ANIMATED_DEFAULT_MAX_FRAMES = 4 +ANIMATED_MAX_FRAMES_LIMIT = 16 + +CONVERT_CACHE_DIR_NAME = "media_convert_cache" +"""Cache directory (under the AstrBot temp dir) for converted images and frames.""" + +_MIME_PIL_FORMAT = { + "image/jpeg": "JPEG", + "image/png": "PNG", + "image/webp": "WEBP", + "image/gif": "GIF", + "image/bmp": "BMP", +} + +_MIME_FILE_SUFFIX = { + "image/jpeg": ".jpg", + "image/png": ".png", + "image/webp": ".webp", + "image/gif": ".gif", + "image/bmp": ".bmp", +} + AUDIO_FORMAT_MIME_TYPES = { "aac": "audio/aac", "amr": "audio/amr", @@ -894,7 +938,221 @@ async def resolve_image_ref_to_base64_data( assembly can skip bad image refs without failing the whole request. """ - return await MediaResolver( + images = await resolve_image_ref_to_images( + image_ref, + strict=strict, + default_mime_type=default_mime_type, + ) + return images[0] if images else None + + +def _image_convert_cache_dir() -> Path: + cache_dir = Path(get_astrbot_temp_path()) / CONVERT_CACHE_DIR_NAME + cache_dir.mkdir(parents=True, exist_ok=True) + return cache_dir + + +def _image_convert_cache_key(source_bytes: bytes, params: str) -> str: + """Build a content-addressed cache key for a converted image or frame set.""" + source_digest = hashlib.sha256(source_bytes).hexdigest()[:32] + params_digest = hashlib.sha256(params.encode()).hexdigest()[:8] + return f"{source_digest}_{params_digest}" + + +def _image_has_alpha(image: PILImage.Image) -> bool: + return image.mode in {"RGBA", "LA"} or ( + image.mode == "P" and "transparency" in image.info + ) + + +def _inspect_image(image_bytes: bytes) -> tuple[bool, int]: + """Inspect decodable image bytes. + + Args: + image_bytes: Encoded image bytes. + + Returns: + Tuple of ``(has_alpha, frame_count)``. + + Raises: + Exception: Raised by Pillow when the bytes are not a decodable image. + """ + with PILImage.open(io.BytesIO(image_bytes)) as image: + return _image_has_alpha(image), getattr(image, "n_frames", 1) + + +def _pick_target_image_mime_type( + has_alpha: bool, + allowed_mime_types: Collection[str] | None, +) -> str: + """Pick the best Pillow-savable target MIME type within the allowed set. + + Alpha images prefer PNG to preserve transparency; opaque images prefer JPEG. + """ + preferred = ["image/png"] if has_alpha else ["image/jpeg"] + preferred += ["image/jpeg", "image/png", "image/webp"] + for mime_type in preferred: + if allowed_mime_types is None or mime_type in allowed_mime_types: + return mime_type + for mime_type in allowed_mime_types or (): + if mime_type in _MIME_PIL_FORMAT: + return mime_type + return "image/png" + + +def _save_current_image_frame( + image: PILImage.Image, + target_mime_type: str, + output_path: Path, +) -> None: + """Save the currently selected frame of an opened image.""" + working: PILImage.Image | None = None + try: + frame = image + if target_mime_type == "image/jpeg" and image.mode != "RGB": + working = image.convert("RGB") + frame = working + elif ( + target_mime_type == "image/png" + and image.mode == "P" + and "transparency" in image.info + ): + working = image.convert("RGBA") + frame = working + save_kwargs: dict[str, int] = {} + if target_mime_type == "image/jpeg": + save_kwargs = { + "quality": IMAGE_COMPRESS_DEFAULT_QUALITY, + "subsampling": 0, + } + frame.save(output_path, _MIME_PIL_FORMAT[target_mime_type], **save_kwargs) + finally: + if working is not None: + working.close() + + +def _convert_image_bytes_sync( + source_bytes: bytes, + target_mime_type: str, + *, + frame_index: int | None = None, +) -> Path: + """Convert image bytes to the target format, cached under the temp dir. + + Args: + source_bytes: Encoded source image bytes. + target_mime_type: Target MIME type; must be Pillow-savable. + frame_index: Frame to extract first, for animated sources. + + Returns: + Path of the converted (or previously cached) image. + """ + cache_key = _image_convert_cache_key( + source_bytes, f"convert|{target_mime_type}|frame={frame_index}" + ) + output_path = _image_convert_cache_dir() / ( + cache_key + _MIME_FILE_SUFFIX[target_mime_type] + ) + if output_path.exists(): + return output_path + with PILImage.open(io.BytesIO(source_bytes)) as image: + if frame_index is not None: + image.seek(frame_index) + _save_current_image_frame(image, target_mime_type, output_path) + return output_path + + +def _even_frame_indices(total_frames: int, max_frames: int) -> list[int]: + """Pick up to ``max_frames`` frame indices evenly spaced over the animation.""" + count = min(max_frames, total_frames) + if count <= 1: + return [0] + return sorted({round(i * (total_frames - 1) / (count - 1)) for i in range(count)}) + + +def _extract_animation_frames_sync( + source_bytes: bytes, + target_mime_type: str, + max_frames: int, +) -> list[Path]: + """Extract evenly spaced frames from an animated image, with caching. + + Args: + source_bytes: Encoded animated image bytes. + target_mime_type: MIME type each extracted frame is saved as. + max_frames: Maximum number of frames to extract. + + Returns: + Paths of the extracted frame images in playback order. + """ + suffix = _MIME_FILE_SUFFIX[target_mime_type] + cache_key = _image_convert_cache_key( + source_bytes, f"frames|{target_mime_type}|n={max_frames}" + ) + cache_dir = _image_convert_cache_dir() + cached = sorted(cache_dir.glob(f"{cache_key}_f*{suffix}")) + if cached: + return cached + frame_paths: list[Path] = [] + with PILImage.open(io.BytesIO(source_bytes)) as image: + total_frames = getattr(image, "n_frames", 1) + for out_index, frame_index in enumerate( + _even_frame_indices(total_frames, max_frames) + ): + frame_path = cache_dir / f"{cache_key}_f{out_index}{suffix}" + image.seek(frame_index) + _save_current_image_frame(image, target_mime_type, frame_path) + frame_paths.append(frame_path) + return frame_paths + + +async def _path_to_resolved_media_data(path: Path, mime_type: str) -> ResolvedMediaData: + data = await asyncio.to_thread(path.read_bytes) + return ResolvedMediaData( + base64_data=base64.b64encode(data).decode("utf-8"), + mime_type=mime_type, + ) + + +async def resolve_image_ref_to_images( + image_ref: MediaRefStr, + *, + allowed_mime_types: Collection[str] | None = None, + animated_strategy: str = ANIMATED_STRATEGY_FIRST_FRAME, + animated_max_frames: int = ANIMATED_DEFAULT_MAX_FRAMES, + strict: bool = False, + default_mime_type: str | None = "image/jpeg", +) -> list[ResolvedMediaData]: + """Resolve an image reference into provider-ready images. + + Applies per-provider format adaptation: still images whose detected MIME type + is not in ``allowed_mime_types`` are converted via Pillow (cached under the + AstrBot temp directory, so a source is only converted once until the cache is + cleaned), and animated images are reduced to still frames according to + ``animated_strategy``. Compatible images are returned untouched without any + conversion or cache write. + + Args: + image_ref: Image reference in any form accepted by ``MediaResolver``. + allowed_mime_types: MIME types accepted by the target provider. ``None`` + or a collection containing ``"*"`` disables adaptation. + animated_strategy: ``first_frame`` keeps only the first frame; + ``multi_frame`` extracts up to ``animated_max_frames`` evenly spaced + frames as separate images. + animated_max_frames: Maximum frames for ``multi_frame``, clamped to + ``[1, 16]``. + strict: Raise on invalid or undecodable images instead of skipping them. + default_mime_type: Fallback MIME type for legacy base64 payloads. + + Returns: + List of resolved images; empty when the reference is invalid and + ``strict`` is False. + + Raises: + ValueError: Raised in strict mode when the image is invalid or cannot be + decoded by Pillow. + """ + media_data = await MediaResolver( image_ref, media_type="image", default_suffix=".bin", @@ -902,6 +1160,74 @@ async def resolve_image_ref_to_base64_data( strict=strict, default_mime_type=default_mime_type, ) + if media_data is None: + return [] + + image_bytes = media_data.to_bytes() + unrestricted = allowed_mime_types is None or "*" in allowed_mime_types + try: + has_alpha, frame_count = await asyncio.to_thread(_inspect_image, image_bytes) + except Exception as exc: + # Pillow cannot decode some provider-supported formats (e.g. HEIC without + # pillow-heif); pass them through when the provider accepts them. + if unrestricted or ( + media_data.mime_type and media_data.mime_type in allowed_mime_types + ): + return [media_data] + if strict: + raise ValueError( + f"Invalid image file: {describe_media_ref(image_ref)}" + ) from exc + logger.warning( + "Image %s cannot be decoded and will be skipped.", + describe_media_ref(image_ref), + ) + return [] + + if frame_count > 1 and ( + not unrestricted or animated_strategy == ANIMATED_STRATEGY_MULTI_FRAME + ): + max_frames = min(max(int(animated_max_frames), 1), ANIMATED_MAX_FRAMES_LIMIT) + target_mime_type = _pick_target_image_mime_type( + has_alpha, + None if unrestricted else allowed_mime_types, + ) + if animated_strategy == ANIMATED_STRATEGY_MULTI_FRAME: + frame_paths = await asyncio.to_thread( + _extract_animation_frames_sync, + image_bytes, + target_mime_type, + max_frames, + ) + logger.info( + "Animated image %s extracted into %d frame(s) for the provider.", + describe_media_ref(image_ref), + len(frame_paths), + ) + else: + frame_paths = [ + await asyncio.to_thread( + _convert_image_bytes_sync, + image_bytes, + target_mime_type, + frame_index=0, + ) + ] + return [ + await _path_to_resolved_media_data(path, target_mime_type) + for path in frame_paths + ] + + if unrestricted or media_data.mime_type in allowed_mime_types: + return [media_data] + + target_mime_type = _pick_target_image_mime_type(has_alpha, allowed_mime_types) + converted_path = await asyncio.to_thread( + _convert_image_bytes_sync, + image_bytes, + target_mime_type, + ) + return [await _path_to_resolved_media_data(converted_path, target_mime_type)] async def resolve_audio_ref_to_base64_data( diff --git a/tests/test_media_utils.py b/tests/test_media_utils.py index efe5e65f02..9ffba1a96c 100644 --- a/tests/test_media_utils.py +++ b/tests/test_media_utils.py @@ -871,3 +871,246 @@ async def test_wav_to_tencent_silk_skips_resample_for_supported_rate( assert len(fake.calls) == 1 assert fake.calls[0]["sample_rate"] == 24000 + + +def _make_image_bytes( + fmt: str = "PNG", + mode: str = "RGB", + size: tuple[int, int] = (8, 8), + color=(255, 0, 0, 255), +) -> bytes: + from PIL import Image as PILImage + + image = PILImage.new(mode, size, color) + buffer = BytesIO() + image.save(buffer, fmt) + return buffer.getvalue() + + +def _make_animated_gif_bytes(colors: list[tuple[int, int, int]]) -> bytes: + from PIL import Image as PILImage + + frames = [PILImage.new("RGB", (8, 8), color) for color in colors] + buffer = BytesIO() + frames[0].save( + buffer, + "GIF", + save_all=True, + append_images=frames[1:], + duration=50, + loop=0, + ) + return buffer.getvalue() + + +def _image_data_uri(mime_type: str, payload: bytes) -> str: + return f"data:{mime_type};base64,{base64.b64encode(payload).decode()}" + + +@pytest.mark.asyncio +async def test_resolve_images_compatible_passes_through_without_cache( + tmp_path, monkeypatch +): + monkeypatch.setattr(media_utils, "get_astrbot_temp_path", lambda: str(tmp_path)) + png_bytes = _make_image_bytes("PNG") + + images = await media_utils.resolve_image_ref_to_images( + _image_data_uri("image/png", png_bytes), + allowed_mime_types={"image/png"}, + ) + + assert len(images) == 1 + assert images[0].mime_type == "image/png" + assert images[0].to_bytes() == png_bytes + # Compatible images never touch the conversion cache. + assert not (tmp_path / media_utils.CONVERT_CACHE_DIR_NAME).exists() + + +@pytest.mark.asyncio +async def test_resolve_images_converts_bmp_to_jpeg_and_caches(tmp_path, monkeypatch): + monkeypatch.setattr(media_utils, "get_astrbot_temp_path", lambda: str(tmp_path)) + from PIL import Image as PILImage + + bmp_bytes = _make_image_bytes("BMP") + + images = await media_utils.resolve_image_ref_to_images( + _image_data_uri("image/bmp", bmp_bytes), + allowed_mime_types={"image/jpeg", "image/png"}, + ) + + assert len(images) == 1 + assert images[0].mime_type == "image/jpeg" + with PILImage.open(BytesIO(images[0].to_bytes())) as converted: + assert converted.format == "JPEG" + cache_dir = tmp_path / media_utils.CONVERT_CACHE_DIR_NAME + assert len(list(cache_dir.glob("*.jpg"))) == 1 + + +@pytest.mark.asyncio +async def test_resolve_images_alpha_source_prefers_png(tmp_path, monkeypatch): + monkeypatch.setattr(media_utils, "get_astrbot_temp_path", lambda: str(tmp_path)) + from PIL import Image as PILImage + + webp_bytes = _make_image_bytes("WEBP", mode="RGBA", color=(10, 20, 30, 128)) + + images = await media_utils.resolve_image_ref_to_images( + _image_data_uri("image/webp", webp_bytes), + allowed_mime_types={"image/jpeg", "image/png"}, + ) + + assert len(images) == 1 + assert images[0].mime_type == "image/png" + with PILImage.open(BytesIO(images[0].to_bytes())) as converted: + assert converted.format == "PNG" + assert converted.mode == "RGBA" + assert converted.getpixel((0, 0))[3] == 128 + + +@pytest.mark.asyncio +async def test_resolve_images_animated_first_frame(tmp_path, monkeypatch): + monkeypatch.setattr(media_utils, "get_astrbot_temp_path", lambda: str(tmp_path)) + gif_bytes = _make_animated_gif_bytes( + [(255, 0, 0), (0, 255, 0), (0, 0, 255)], + ) + + images = await media_utils.resolve_image_ref_to_images( + _image_data_uri("image/gif", gif_bytes), + allowed_mime_types={"image/jpeg", "image/png"}, + animated_strategy=media_utils.ANIMATED_STRATEGY_FIRST_FRAME, + ) + + assert len(images) == 1 + assert images[0].mime_type == "image/jpeg" + + +@pytest.mark.asyncio +async def test_resolve_images_multi_frame_evenly_spaced(tmp_path, monkeypatch): + monkeypatch.setattr(media_utils, "get_astrbot_temp_path", lambda: str(tmp_path)) + from PIL import Image as PILImage + + colors = [ + (255, 0, 0), + (0, 255, 0), + (0, 0, 255), + (255, 255, 0), + (0, 255, 255), + (255, 0, 255), + ] + gif_bytes = _make_animated_gif_bytes(colors) + + images = await media_utils.resolve_image_ref_to_images( + _image_data_uri("image/gif", gif_bytes), + allowed_mime_types={"image/png"}, + animated_strategy=media_utils.ANIMATED_STRATEGY_MULTI_FRAME, + animated_max_frames=4, + ) + + # 6 frames, 4 picks -> indices 0, 2, 3, 5 (evenly spaced). + expected_indices = [0, 2, 3, 5] + assert len(images) == len(expected_indices) + for image_data, frame_index in zip(images, expected_indices, strict=True): + assert image_data.mime_type == "image/png" + with PILImage.open(BytesIO(image_data.to_bytes())) as frame: + assert frame.convert("RGB").getpixel((0, 0)) == colors[frame_index] + + +@pytest.mark.asyncio +async def test_resolve_images_multi_frame_clamps_to_limit(tmp_path, monkeypatch): + monkeypatch.setattr(media_utils, "get_astrbot_temp_path", lambda: str(tmp_path)) + colors = [(i * 12 % 256, 0, 0) for i in range(20)] + gif_bytes = _make_animated_gif_bytes(colors) + + images = await media_utils.resolve_image_ref_to_images( + _image_data_uri("image/gif", gif_bytes), + allowed_mime_types={"image/png"}, + animated_strategy=media_utils.ANIMATED_STRATEGY_MULTI_FRAME, + animated_max_frames=100, + ) + + assert len(images) == media_utils.ANIMATED_MAX_FRAMES_LIMIT + + +@pytest.mark.asyncio +async def test_resolve_images_cache_hit_skips_reencode(tmp_path, monkeypatch): + monkeypatch.setattr(media_utils, "get_astrbot_temp_path", lambda: str(tmp_path)) + bmp_bytes = _make_image_bytes("BMP") + image_ref = _image_data_uri("image/bmp", bmp_bytes) + + save_calls = 0 + real_save = media_utils._save_current_image_frame + + def counting_save(*args, **kwargs): + nonlocal save_calls + save_calls += 1 + return real_save(*args, **kwargs) + + monkeypatch.setattr(media_utils, "_save_current_image_frame", counting_save) + + first = await media_utils.resolve_image_ref_to_images( + image_ref, allowed_mime_types={"image/jpeg"} + ) + assert save_calls == 1 + + second = await media_utils.resolve_image_ref_to_images( + image_ref, allowed_mime_types={"image/jpeg"} + ) + assert save_calls == 1 # cache hit: no re-encode + assert second[0].base64_data == first[0].base64_data + + # After the cache is cleaned, conversion runs again. + cache_dir = tmp_path / media_utils.CONVERT_CACHE_DIR_NAME + for cached_file in cache_dir.iterdir(): + cached_file.unlink() + await media_utils.resolve_image_ref_to_images( + image_ref, allowed_mime_types={"image/jpeg"} + ) + assert save_calls == 2 + + +@pytest.mark.asyncio +async def test_resolve_images_unrestricted_passes_any_format(tmp_path, monkeypatch): + monkeypatch.setattr(media_utils, "get_astrbot_temp_path", lambda: str(tmp_path)) + bmp_bytes = _make_image_bytes("BMP") + image_ref = _image_data_uri("image/bmp", bmp_bytes) + + for allowed in (None, {"*"}): + images = await media_utils.resolve_image_ref_to_images( + image_ref, allowed_mime_types=allowed + ) + assert len(images) == 1 + assert images[0].mime_type == "image/bmp" + assert images[0].to_bytes() == bmp_bytes + + +@pytest.mark.asyncio +async def test_resolve_images_undecodable_skipped_or_passed(tmp_path, monkeypatch): + monkeypatch.setattr(media_utils, "get_astrbot_temp_path", lambda: str(tmp_path)) + svg_ref = _image_data_uri( + "image/svg+xml", b'' + ) + + # Restricted providers skip undecodable images instead of mislabeling them. + assert ( + await media_utils.resolve_image_ref_to_images( + svg_ref, allowed_mime_types={"image/jpeg", "image/png"} + ) + == [] + ) + # Unrestricted resolution keeps the legacy pass-through behavior. + images = await media_utils.resolve_image_ref_to_images(svg_ref) + assert len(images) == 1 + assert images[0].mime_type == "image/svg+xml" + + +@pytest.mark.asyncio +async def test_resolve_image_ref_to_base64_data_delegates(tmp_path, monkeypatch): + monkeypatch.setattr(media_utils, "get_astrbot_temp_path", lambda: str(tmp_path)) + png_bytes = _make_image_bytes("PNG") + + resolved = await media_utils.resolve_image_ref_to_base64_data( + _image_data_uri("image/png", png_bytes) + ) + + assert resolved is not None + assert resolved.mime_type == "image/png" + assert resolved.to_bytes() == png_bytes diff --git a/tests/test_openai_source.py b/tests/test_openai_source.py index 911b76131f..1c7a677e3a 100644 --- a/tests/test_openai_source.py +++ b/tests/test_openai_source.py @@ -718,21 +718,22 @@ async def test_prepare_chat_payload_materializes_context_http_image_urls(monkeyp provider = _make_provider() try: - async def fake_resolve_media_ref_to_base64_data( - media_ref: str, + async def fake_resolve_image_ref_to_images( + image_ref: str, *, - media_type: str, + allowed_mime_types=None, + animated_strategy: str = "first_frame", + animated_max_frames: int = 4, strict: bool = False, - ) -> ResolvedMediaData: - assert media_ref == "https://example.com/quoted.png" - assert media_type == "image" + ) -> list[ResolvedMediaData]: + assert image_ref == "https://example.com/quoted.png" assert strict is False - return ResolvedMediaData(base64_data="abcd", mime_type="image/png") + return [ResolvedMediaData(base64_data="abcd", mime_type="image/png")] monkeypatch.setattr( openai_source_module, - "resolve_media_ref_to_base64_data", - fake_resolve_media_ref_to_base64_data, + "resolve_image_ref_to_images", + fake_resolve_image_ref_to_images, ) contexts = [ @@ -935,7 +936,7 @@ async def test_resolve_image_part_rejects_invalid_local_file(tmp_path): invalid_file = tmp_path / "not-image.txt" invalid_file.write_text("not an image") - assert await provider._resolve_image_part(str(invalid_file)) is None + assert await provider._resolve_image_parts(str(invalid_file)) == [] finally: await provider.terminate() @@ -947,7 +948,7 @@ async def test_resolve_image_part_rejects_invalid_file_uri(tmp_path): invalid_file = tmp_path / "not-image.txt" invalid_file.write_text("not an image") - assert await provider._resolve_image_part(invalid_file.as_uri()) is None + assert await provider._resolve_image_parts(invalid_file.as_uri()) == [] finally: await provider.terminate() @@ -994,15 +995,17 @@ async def test_materialize_context_image_parts_returns_new_messages(monkeypatch) async def fake_resolve(image_url: str, *, image_detail: str | None = None): assert image_url == "https://example.com/quoted.png" assert image_detail == "high" - return { - "type": "image_url", - "image_url": { - "url": "data:image/png;base64,abcd", - "detail": "high", - }, - } + return [ + { + "type": "image_url", + "image_url": { + "url": "data:image/png;base64,abcd", + "detail": "high", + }, + } + ] - monkeypatch.setattr(provider, "_resolve_image_part", fake_resolve) + monkeypatch.setattr(provider, "_resolve_image_parts", fake_resolve) materialized = await provider._materialize_context_image_parts(context_query) @@ -1077,10 +1080,12 @@ async def test_encode_image_bs64_supports_file_uri(tmp_path): async def test_resolve_image_part_supports_base64_scheme(): provider = _make_provider() try: - assert await provider._resolve_image_part("base64://abcd") == { - "type": "image_url", - "image_url": {"url": "data:image/jpeg;base64,abcd"}, - } + assert await provider._resolve_image_parts("base64://abcd") == [ + { + "type": "image_url", + "image_url": {"url": "data:image/jpeg;base64,abcd"}, + } + ] finally: await provider.terminate() @@ -1096,12 +1101,14 @@ async def test_resolve_image_part_preserves_base64_png_mime_type(): ) image_base64 = base64.b64encode(image_buffer.getvalue()).decode("ascii") - image_part = await provider._resolve_image_part(f"base64://{image_base64}") + image_parts = await provider._resolve_image_parts(f"base64://{image_base64}") - assert image_part == { - "type": "image_url", - "image_url": {"url": f"data:image/png;base64,{image_base64}"}, - } + assert image_parts == [ + { + "type": "image_url", + "image_url": {"url": f"data:image/png;base64,{image_base64}"}, + } + ] finally: await provider.terminate() diff --git a/tests/unit/test_provider_image_formats.py b/tests/unit/test_provider_image_formats.py new file mode 100644 index 0000000000..0246d922ad --- /dev/null +++ b/tests/unit/test_provider_image_formats.py @@ -0,0 +1,216 @@ +import base64 +from io import BytesIO + +import pytest +from PIL import Image as PILImage + +import astrbot.core.utils.media_utils as media_utils +from astrbot.core.provider.provider import DEFAULT_FALLBACK_IMAGE_FORMATS, Provider +from astrbot.core.provider.sources.oai_aihubmix_source import ProviderAIHubMix +from astrbot.core.provider.sources.openai_source import ProviderOpenAIOfficial +from astrbot.core.provider.sources.openrouter_source import ProviderOpenRouter +from astrbot.core.provider.sources.ssycloud_source import ProviderSSYCloud +from astrbot.core.provider.sources.xai_source import ProviderXAI + + +class _DummyProvider(Provider): + supported_image_formats = frozenset({"image/jpeg", "image/png", "image/webp"}) + + def get_current_key(self) -> str: + return "" + + def set_key(self, key: str) -> None: + pass + + async def get_models(self) -> list[str]: + return [] + + async def text_chat(self, **kwargs): + raise NotImplementedError + + +class _UndeclaredProvider(_DummyProvider): + supported_image_formats = None + + +def _make_dummy(provider_config: dict | None = None) -> _DummyProvider: + return _DummyProvider(provider_config or {}, {}) + + +def test_config_override_wins_over_class_default(): + provider = _make_dummy({"image_formats": ["jpeg", "png"]}) + assert provider.resolve_allowed_image_formats() == frozenset( + {"image/jpeg", "image/png"} + ) + + +def test_config_star_disables_restriction(): + provider = _make_dummy({"image_formats": ["*"]}) + assert provider.resolve_allowed_image_formats() is None + + +def test_config_accepts_mime_types_and_is_case_insensitive(): + provider = _make_dummy({"image_formats": ["image/webp", " JPEG "]}) + assert provider.resolve_allowed_image_formats() == frozenset( + {"image/webp", "image/jpeg"} + ) + + +def test_class_default_used_when_not_configured(): + provider = _make_dummy() + assert provider.resolve_allowed_image_formats() == frozenset( + {"image/jpeg", "image/png", "image/webp"} + ) + + +def test_fallback_used_when_class_undeclared(): + provider = _UndeclaredProvider({}, {}) + assert provider.resolve_allowed_image_formats() == DEFAULT_FALLBACK_IMAGE_FORMATS + + +def test_unknown_config_entries_fall_back_to_class_default(): + provider = _make_dummy({"image_formats": ["not-a-format"]}) + assert provider.resolve_allowed_image_formats() == frozenset( + {"image/jpeg", "image/png", "image/webp"} + ) + + +def test_aggregators_do_not_inherit_openai_format_set(): + assert ProviderOpenRouter.supported_image_formats is None + assert ProviderAIHubMix.supported_image_formats is None + assert ProviderSSYCloud.supported_image_formats is None + assert ProviderOpenAIOfficial.supported_image_formats == frozenset( + {"image/png", "image/jpeg", "image/webp", "image/gif"} + ) + assert ProviderXAI.supported_image_formats == frozenset( + {"image/jpeg", "image/png"} + ) + + +def test_animated_strategy_defaults(): + provider = _make_dummy() + assert provider.get_animated_image_strategy() == ("first_frame", 4) + + +def test_animated_strategy_from_config_and_clamped(): + provider = _make_dummy( + {"animated_image_strategy": "multi_frame", "animated_image_max_frames": 100} + ) + assert provider.get_animated_image_strategy() == ("multi_frame", 16) + + provider = _make_dummy({"animated_image_max_frames": 0}) + assert provider.get_animated_image_strategy() == ("first_frame", 1) + + provider = _make_dummy({"animated_image_strategy": "bogus"}) + assert provider.get_animated_image_strategy() == ("first_frame", 4) + + +def _animated_gif_data_uri(colors: list[tuple[int, int, int]]) -> str: + frames = [PILImage.new("RGB", (8, 8), color) for color in colors] + buffer = BytesIO() + frames[0].save( + buffer, + "GIF", + save_all=True, + append_images=frames[1:], + duration=50, + loop=0, + ) + return f"data:image/gif;base64,{base64.b64encode(buffer.getvalue()).decode()}" + + +@pytest.mark.asyncio +async def test_xai_animated_gif_reduced_to_still(tmp_path, monkeypatch): + monkeypatch.setattr(media_utils, "get_astrbot_temp_path", lambda: str(tmp_path)) + provider = ProviderXAI( + { + "id": "test-xai", + "type": "xai_chat_completion", + "model": "grok-4", + "key": ["test-key"], + }, + {}, + ) + try: + gif_ref = _animated_gif_data_uri([(255, 0, 0), (0, 255, 0), (0, 0, 255)]) + message = await provider.assemble_context("look", image_urls=[gif_ref]) + + content = message["content"] + image_blocks = [block for block in content if block["type"] == "image_url"] + assert len(image_blocks) == 1 + url = image_blocks[0]["image_url"]["url"] + # xAI only accepts JPEG/PNG; the animated GIF must become a still image. + assert url.startswith(("data:image/jpeg;base64,", "data:image/png;base64,")) + finally: + await provider.terminate() + + +@pytest.mark.asyncio +async def test_openai_multi_frame_strategy_expands_image_blocks( + tmp_path, monkeypatch +): + monkeypatch.setattr(media_utils, "get_astrbot_temp_path", lambda: str(tmp_path)) + provider = ProviderOpenAIOfficial( + { + "id": "test-openai", + "type": "openai_chat_completion", + "model": "gpt-4o-mini", + "key": ["test-key"], + "animated_image_strategy": "multi_frame", + "animated_image_max_frames": 3, + }, + {}, + ) + try: + gif_ref = _animated_gif_data_uri( + [(255, 0, 0), (0, 255, 0), (0, 0, 255), (255, 255, 0), (0, 255, 255)] + ) + message = await provider.assemble_context("look", image_urls=[gif_ref]) + + image_blocks = [ + block for block in message["content"] if block["type"] == "image_url" + ] + assert len(image_blocks) == 3 + for block in image_blocks: + assert block["image_url"]["url"].startswith("data:image/") + finally: + await provider.terminate() + + +@pytest.mark.asyncio +async def test_anthropic_context_skips_unsupported_format(): + from astrbot.core.provider.sources.anthropic_source import ProviderAnthropic + + provider = ProviderAnthropic( + { + "id": "test-anthropic", + "type": "anthropic_chat_completion", + "model": "claude-sonnet-4-5", + "key": ["test-key"], + }, + {}, + ) + try: + # A BMP data URI must not be mislabeled as image/jpeg in the payload. + bmp_buffer = BytesIO() + PILImage.new("RGB", (4, 4), (1, 2, 3)).save(bmp_buffer, "BMP") + bmp_data_uri = ( + f"data:image/bmp;base64,{base64.b64encode(bmp_buffer.getvalue()).decode()}" + ) + _system_prompt, messages = provider._prepare_payload( + [ + { + "role": "user", + "content": [ + {"type": "text", "text": "look"}, + {"type": "image_url", "image_url": {"url": bmp_data_uri}}, + ], + } + ] + ) + + content = messages[0]["content"] + image_blocks = [block for block in content if block.get("type") == "image"] + assert image_blocks == [] + finally: + await provider.terminate() From f8911ebab8a724e0183ec9c1449c53c1247fbc4d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=B0=95=E6=B0=99?= Date: Sat, 15 Aug 2026 16:20:31 +0800 Subject: [PATCH 2/8] fix: make image conversion cache atomic and warn on invalid image_formats config - Publish single-image conversions via temp file + atomic rename so concurrent readers never see truncated cache entries - Extract animation frames into a staging directory and publish with one atomic directory rename; a crash mid-extraction no longer poisons the frame cache with a partial set - Log a warning when image_formats contains no valid entries instead of silently falling back to the default format set --- astrbot/core/provider/provider.py | 7 +++ astrbot/core/utils/media_utils.py | 64 +++++++++++++++++------ tests/test_media_utils.py | 25 +++++++++ tests/unit/test_provider_image_formats.py | 16 +++--- 4 files changed, 91 insertions(+), 21 deletions(-) diff --git a/astrbot/core/provider/provider.py b/astrbot/core/provider/provider.py index 1acc27b1c1..89546c6ec2 100644 --- a/astrbot/core/provider/provider.py +++ b/astrbot/core/provider/provider.py @@ -4,6 +4,7 @@ from collections.abc import AsyncGenerator from typing import ClassVar, Literal, TypeAlias, Union +from astrbot import logger from astrbot.core.agent.message import ContentPart, Message, is_checkpoint_message from astrbot.core.agent.tool import ToolSet from astrbot.core.provider.entities import ( @@ -117,6 +118,12 @@ def resolve_allowed_image_formats(self) -> frozenset[str] | None: } if mapped: return frozenset(mapped) + logger.warning( + "Provider %s: image_formats %s contains no valid entries; " + "falling back to the default format set.", + self.provider_config.get("id"), + sorted(normalized), + ) if self.supported_image_formats is not None: return self.supported_image_formats return DEFAULT_FALLBACK_IMAGE_FORMATS diff --git a/astrbot/core/utils/media_utils.py b/astrbot/core/utils/media_utils.py index 8d0e8ae9e3..1e9f812e16 100644 --- a/astrbot/core/utils/media_utils.py +++ b/astrbot/core/utils/media_utils.py @@ -13,6 +13,7 @@ import os import shutil import subprocess +import tempfile from collections.abc import AsyncIterator, Collection from contextlib import asynccontextmanager from dataclasses import dataclass, field @@ -1031,6 +1032,25 @@ def _save_current_image_frame( working.close() +def _save_image_frame_atomic( + image: PILImage.Image, + target_mime_type: str, + output_path: Path, +) -> None: + """Save via a unique temp file then atomically replace the cache entry. + + Concurrent readers never observe a partially written cache file. + """ + fd, tmp_name = tempfile.mkstemp(dir=output_path.parent, suffix=".tmp") + os.close(fd) + tmp_path = Path(tmp_name) + try: + _save_current_image_frame(image, target_mime_type, tmp_path) + os.replace(tmp_path, output_path) + finally: + tmp_path.unlink(missing_ok=True) + + def _convert_image_bytes_sync( source_bytes: bytes, target_mime_type: str, @@ -1058,7 +1078,7 @@ def _convert_image_bytes_sync( with PILImage.open(io.BytesIO(source_bytes)) as image: if frame_index is not None: image.seek(frame_index) - _save_current_image_frame(image, target_mime_type, output_path) + _save_image_frame_atomic(image, target_mime_type, output_path) return output_path @@ -1090,20 +1110,34 @@ def _extract_animation_frames_sync( source_bytes, f"frames|{target_mime_type}|n={max_frames}" ) cache_dir = _image_convert_cache_dir() - cached = sorted(cache_dir.glob(f"{cache_key}_f*{suffix}")) - if cached: - return cached - frame_paths: list[Path] = [] - with PILImage.open(io.BytesIO(source_bytes)) as image: - total_frames = getattr(image, "n_frames", 1) - for out_index, frame_index in enumerate( - _even_frame_indices(total_frames, max_frames) - ): - frame_path = cache_dir / f"{cache_key}_f{out_index}{suffix}" - image.seek(frame_index) - _save_current_image_frame(image, target_mime_type, frame_path) - frame_paths.append(frame_path) - return frame_paths + frames_dir = cache_dir / f"{cache_key}_frames" + if frames_dir.is_dir(): + cached = sorted(frames_dir.glob(f"*{suffix}")) + if cached: + return cached + # Extract into a staging dir and publish it with one atomic rename, so a + # crash mid-extraction never leaves a partial frame set behind. + staging_dir = Path(tempfile.mkdtemp(dir=cache_dir, prefix=f".{cache_key}_")) + try: + with PILImage.open(io.BytesIO(source_bytes)) as image: + total_frames = getattr(image, "n_frames", 1) + for out_index, frame_index in enumerate( + _even_frame_indices(total_frames, max_frames) + ): + frame_path = staging_dir / f"f{out_index}{suffix}" + image.seek(frame_index) + _save_current_image_frame(image, target_mime_type, frame_path) + try: + os.replace(staging_dir, frames_dir) + except OSError: + # Lost a concurrent publish race; use the winner's complete set. + shutil.rmtree(staging_dir, ignore_errors=True) + if not frames_dir.is_dir(): + raise + except BaseException: + shutil.rmtree(staging_dir, ignore_errors=True) + raise + return sorted(frames_dir.glob(f"*{suffix}")) async def _path_to_resolved_media_data(path: Path, mime_type: str) -> ResolvedMediaData: diff --git a/tests/test_media_utils.py b/tests/test_media_utils.py index 9ffba1a96c..70b26fee96 100644 --- a/tests/test_media_utils.py +++ b/tests/test_media_utils.py @@ -1030,6 +1030,31 @@ async def test_resolve_images_multi_frame_clamps_to_limit(tmp_path, monkeypatch) assert len(images) == media_utils.ANIMATED_MAX_FRAMES_LIMIT +@pytest.mark.asyncio +async def test_resolve_images_ignores_stale_staging_dir(tmp_path, monkeypatch): + monkeypatch.setattr(media_utils, "get_astrbot_temp_path", lambda: str(tmp_path)) + gif_bytes = _make_animated_gif_bytes([(255, 0, 0), (0, 255, 0), (0, 0, 255)]) + + # Simulate a crash mid-extraction: a staging dir with a partial frame set. + cache_dir = tmp_path / media_utils.CONVERT_CACHE_DIR_NAME + cache_dir.mkdir(parents=True) + staging = cache_dir / ".stale_staging" + staging.mkdir() + (staging / "f0.png").write_bytes(b"partial") + + images = await media_utils.resolve_image_ref_to_images( + _image_data_uri("image/gif", gif_bytes), + allowed_mime_types={"image/png"}, + animated_strategy=media_utils.ANIMATED_STRATEGY_MULTI_FRAME, + animated_max_frames=3, + ) + + assert len(images) == 3 + # The published frame set lives in a dedicated dir, not the staging dir. + assert all(image_data.to_bytes() != b"partial" for image_data in images) + assert list(cache_dir.glob("*_frames")) + + @pytest.mark.asyncio async def test_resolve_images_cache_hit_skips_reencode(tmp_path, monkeypatch): monkeypatch.setattr(media_utils, "get_astrbot_temp_path", lambda: str(tmp_path)) diff --git a/tests/unit/test_provider_image_formats.py b/tests/unit/test_provider_image_formats.py index 0246d922ad..b9b37e992f 100644 --- a/tests/unit/test_provider_image_formats.py +++ b/tests/unit/test_provider_image_formats.py @@ -75,6 +75,14 @@ def test_unknown_config_entries_fall_back_to_class_default(): ) +def test_unknown_config_entries_log_warning(caplog): + provider = _make_dummy({"image_formats": ["not-a-format"]}) + with caplog.at_level("WARNING"): + provider.resolve_allowed_image_formats() + assert "no valid entries" in caplog.text + assert "not-a-format" in caplog.text + + def test_aggregators_do_not_inherit_openai_format_set(): assert ProviderOpenRouter.supported_image_formats is None assert ProviderAIHubMix.supported_image_formats is None @@ -82,9 +90,7 @@ def test_aggregators_do_not_inherit_openai_format_set(): assert ProviderOpenAIOfficial.supported_image_formats == frozenset( {"image/png", "image/jpeg", "image/webp", "image/gif"} ) - assert ProviderXAI.supported_image_formats == frozenset( - {"image/jpeg", "image/png"} - ) + assert ProviderXAI.supported_image_formats == frozenset({"image/jpeg", "image/png"}) def test_animated_strategy_defaults(): @@ -146,9 +152,7 @@ async def test_xai_animated_gif_reduced_to_still(tmp_path, monkeypatch): @pytest.mark.asyncio -async def test_openai_multi_frame_strategy_expands_image_blocks( - tmp_path, monkeypatch -): +async def test_openai_multi_frame_strategy_expands_image_blocks(tmp_path, monkeypatch): monkeypatch.setattr(media_utils, "get_astrbot_temp_path", lambda: str(tmp_path)) provider = ProviderOpenAIOfficial( { From 90f5a91756d770c652f10438e47d368fcf375a99 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=B0=95=E6=B0=99?= Date: Sat, 15 Aug 2026 18:18:08 +0800 Subject: [PATCH 3/8] feat: surface provider image format options in the dashboard - Declare defaults on the new provider schema fields (image_formats, animated_image_strategy, animated_image_max_frames) and on modalities - Materialize schema-declared defaults into the edit dialog when a stored provider config lacks the key, so existing providers can edit newly introduced options without rewriting their saved configuration - Materialize the new keys when adding a model to a provider source - Support list-contains semantics for metadata conditions and gate the image options on the image modality; gate the max-frames input on the multi-frame strategy - Clarify the image_formats hint: selecting the unrestricted option overrides every other selection - Add zh-CN, en-US, and ru-RU translations for the new fields --- astrbot/core/config/default.py | 12 +++++++++- .../src/components/shared/AstrBotConfig.vue | 7 +++++- .../src/components/shared/AstrBotConfigV4.vue | 7 +++++- .../useProviderModelConfigDialog.ts | 10 +++++++++ .../src/composables/useProviderSources.ts | 5 ++++- .../en-US/features/config-metadata.json | 22 +++++++++++++++++++ .../ru-RU/features/config-metadata.json | 22 +++++++++++++++++++ .../zh-CN/features/config-metadata.json | 22 +++++++++++++++++++ 8 files changed, 103 insertions(+), 4 deletions(-) diff --git a/astrbot/core/config/default.py b/astrbot/core/config/default.py index de472dbc03..3040110ef1 100644 --- a/astrbot/core/config/default.py +++ b/astrbot/core/config/default.py @@ -2129,6 +2129,7 @@ "labels": ["文本", "图像", "音频", "工具使用"], "render_type": "checkbox", "hint": "模型支持的模态及能力。", + "default": ["text", "image", "audio", "tool_use"], }, "image_formats": { "description": "图片格式支持", @@ -2145,7 +2146,9 @@ "不限制", ], "render_type": "checkbox", - "hint": "发送给该提供商前允许的图片格式,不兼容的格式会自动转换。留空使用内置默认值(官方 API 按其文档,未知第三方默认仅 jpeg/png),选 * 表示不限制。", + "hint": "发送给该提供商前允许的图片格式,不兼容的格式会自动转换。留空使用内置默认值(官方 API 按其文档,未知第三方默认仅 jpeg/png)。勾选「不限制」时其他选项不生效。", + "default": [], + "condition": {"modalities": "image"}, }, "animated_image_strategy": { "description": "动图处理策略", @@ -2153,11 +2156,18 @@ "options": ["first_frame", "multi_frame"], "labels": ["仅首帧", "多帧抽取"], "hint": "GIF 等动图发送给模型时的处理方式:仅取首帧(省 token),或按时长均匀抽帧后作为多张图片发送。", + "default": "first_frame", + "condition": {"modalities": "image"}, }, "animated_image_max_frames": { "description": "动图最大抽帧数", "type": "int", "hint": "多帧抽取策略下最多发送的帧数(1-16),默认 4。帧数越多 token 消耗越大。", + "default": 4, + "condition": { + "modalities": "image", + "animated_image_strategy": "multi_frame", + }, }, "custom_headers": { "description": "自定义请求头", diff --git a/dashboard/src/components/shared/AstrBotConfig.vue b/dashboard/src/components/shared/AstrBotConfig.vue index 9702b300f0..69fe5e1437 100644 --- a/dashboard/src/components/shared/AstrBotConfig.vue +++ b/dashboard/src/components/shared/AstrBotConfig.vue @@ -158,7 +158,12 @@ function shouldShowItem(itemMeta, itemKey) { } for (const [conditionKey, expectedValue] of Object.entries(itemMeta.condition)) { const actualValue = getValueBySelector(props.iterable, conditionKey) - if (actualValue !== expectedValue) { + // List-valued fields (e.g. modalities) match when they contain the expected value + if (Array.isArray(actualValue) && !Array.isArray(expectedValue)) { + if (!actualValue.includes(expectedValue)) { + return false + } + } else if (actualValue !== expectedValue) { return false } } diff --git a/dashboard/src/components/shared/AstrBotConfigV4.vue b/dashboard/src/components/shared/AstrBotConfigV4.vue index a6b8c81705..7ad60ca7d2 100644 --- a/dashboard/src/components/shared/AstrBotConfigV4.vue +++ b/dashboard/src/components/shared/AstrBotConfigV4.vue @@ -154,7 +154,12 @@ function shouldShowItem(itemMeta, itemKey) { if (itemMeta?.condition) { for (const [conditionKey, expectedValue] of Object.entries(itemMeta.condition)) { const actualValue = getValueBySelector(props.iterable, conditionKey) - if (actualValue !== expectedValue) { + // List-valued fields (e.g. modalities) match when they contain the expected value + if (Array.isArray(actualValue) && !Array.isArray(expectedValue)) { + if (!actualValue.includes(expectedValue)) { + return false + } + } else if (actualValue !== expectedValue) { return false } } diff --git a/dashboard/src/composables/useProviderModelConfigDialog.ts b/dashboard/src/composables/useProviderModelConfigDialog.ts index c2c49d35cc..e01c822a89 100644 --- a/dashboard/src/composables/useProviderModelConfigDialog.ts +++ b/dashboard/src/composables/useProviderModelConfigDialog.ts @@ -59,6 +59,16 @@ export function useProviderModelConfigDialog(options: UseProviderModelConfigDial function openProviderEdit(provider: any) { const editableProvider = JSON.parse(JSON.stringify(provider)) + // Materialize schema-declared defaults for keys missing from stored configs, + // so fields introduced after the provider was created become editable. + const items = configSchema.value?.provider?.items + if (items) { + for (const [key, item] of Object.entries(items)) { + if (editableProvider[key] === undefined && item && typeof item === 'object' && 'default' in item) { + editableProvider[key] = JSON.parse(JSON.stringify(item.default)) + } + } + } providerEditData.value = editableProvider providerEditOriginalId.value = provider.id providerEditMode.value = 'edit' diff --git a/dashboard/src/composables/useProviderSources.ts b/dashboard/src/composables/useProviderSources.ts index 150ace9c9c..3670a7b554 100644 --- a/dashboard/src/composables/useProviderSources.ts +++ b/dashboard/src/composables/useProviderSources.ts @@ -615,7 +615,10 @@ export function useProviderSources(options: UseProviderSourcesOptions) { modalities, custom_extra_body: {}, max_context_tokens: max_context_tokens, - reasoning: supportsReasoning(metadata) + reasoning: supportsReasoning(metadata), + image_formats: [], + animated_image_strategy: 'first_frame', + animated_image_max_frames: 4 } } diff --git a/dashboard/src/i18n/locales/en-US/features/config-metadata.json b/dashboard/src/i18n/locales/en-US/features/config-metadata.json index fdd430c63e..44ab74de27 100644 --- a/dashboard/src/i18n/locales/en-US/features/config-metadata.json +++ b/dashboard/src/i18n/locales/en-US/features/config-metadata.json @@ -1279,6 +1279,28 @@ "Tool use" ] }, + "image_formats": { + "description": "Supported image formats", + "hint": "Image formats allowed when sending to this provider; incompatible formats are converted automatically. Leave empty to use built-in defaults (official APIs follow their documentation; unknown third parties default to jpeg/png). Selecting \"Unrestricted\" overrides all other options.", + "labels": [ + "JPEG", + "PNG", + "WebP", + "GIF", + "BMP", + "HEIC", + "Unrestricted" + ] + }, + "animated_image_strategy": { + "description": "Animated image strategy", + "hint": "How animated images such as GIFs are sent to the model: keep only the first frame (saves tokens), or extract evenly spaced frames and send them as multiple images.", + "labels": ["First frame only", "Multi-frame extraction"] + }, + "animated_image_max_frames": { + "description": "Max animation frames", + "hint": "Maximum number of frames sent under the multi-frame strategy (1-16), default 4. More frames consume more tokens." + }, "custom_headers": { "description": "Custom request headers", "hint": "Key/value pairs added here are merged into the OpenAI SDK default_headers for custom HTTP headers. Values must be strings." diff --git a/dashboard/src/i18n/locales/ru-RU/features/config-metadata.json b/dashboard/src/i18n/locales/ru-RU/features/config-metadata.json index 3cd0104ca1..4028f11f65 100644 --- a/dashboard/src/i18n/locales/ru-RU/features/config-metadata.json +++ b/dashboard/src/i18n/locales/ru-RU/features/config-metadata.json @@ -1280,6 +1280,28 @@ "Инструменты" ] }, + "image_formats": { + "description": "Поддерживаемые форматы изображений", + "hint": "Форматы изображений, разрешённые при отправке этому провайдеру; несовместимые форматы конвертируются автоматически. Оставьте пустым для встроенных значений по умолчанию (официальные API — согласно документации, неизвестные сторонние — только jpeg/png). При выборе «Без ограничений» остальные варианты не действуют.", + "labels": [ + "JPEG", + "PNG", + "WebP", + "GIF", + "BMP", + "HEIC", + "Без ограничений" + ] + }, + "animated_image_strategy": { + "description": "Стратегия для анимированных изображений", + "hint": "Как анимированные изображения (например, GIF) отправляются в модель: только первый кадр (экономит токены) или равномерно извлечённые кадры в виде нескольких изображений.", + "labels": ["Только первый кадр", "Извлечение нескольких кадров"] + }, + "animated_image_max_frames": { + "description": "Максимум кадров анимации", + "hint": "Максимальное число кадров, отправляемых при стратегии извлечения нескольких кадров (1-16), по умолчанию 4. Больше кадров — больше расход токенов." + }, "custom_headers": { "description": "Заголовки запроса", "hint": "Пары ключ/значение будут добавлены в заголовки запроса (default_headers). Значения должны быть строками." diff --git a/dashboard/src/i18n/locales/zh-CN/features/config-metadata.json b/dashboard/src/i18n/locales/zh-CN/features/config-metadata.json index e25cb8e0fb..be3ea0bf47 100644 --- a/dashboard/src/i18n/locales/zh-CN/features/config-metadata.json +++ b/dashboard/src/i18n/locales/zh-CN/features/config-metadata.json @@ -1281,6 +1281,28 @@ "工具使用" ] }, + "image_formats": { + "description": "图片格式支持", + "hint": "发送给该提供商前允许的图片格式,不兼容的格式会自动转换。留空使用内置默认值(官方 API 按其文档,未知第三方默认仅 jpeg/png)。勾选「不限制」时其他选项不生效。", + "labels": [ + "JPEG", + "PNG", + "WebP", + "GIF", + "BMP", + "HEIC", + "不限制" + ] + }, + "animated_image_strategy": { + "description": "动图处理策略", + "hint": "GIF 等动图发送给模型时的处理方式:仅取首帧(省 token),或按时长均匀抽帧后作为多张图片发送。", + "labels": ["仅首帧", "多帧抽取"] + }, + "animated_image_max_frames": { + "description": "动图最大抽帧数", + "hint": "多帧抽取策略下最多发送的帧数(1-16),默认 4。帧数越多 token 消耗越大。" + }, "custom_headers": { "description": "自定义请求头", "hint": "此处添加的键值对将被合并到 OpenAI SDK 的 default_headers 中,用于自定义 HTTP 请求头。值必须为字符串。" From 382b67e7ccee29e4871a6a481f2017d01b73b8e7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=B0=95=E6=B0=99?= Date: Sat, 15 Aug 2026 18:18:17 +0800 Subject: [PATCH 4/8] fix: log animated frame extraction only when frames are written The tool loop re-assembles the payload on every iteration, so the info log fired on each cached read and looked like repeated extraction work. Log only when the frame set is actually extracted and published. --- astrbot/core/utils/media_utils.py | 24 ++++++++++++++---------- 1 file changed, 14 insertions(+), 10 deletions(-) diff --git a/astrbot/core/utils/media_utils.py b/astrbot/core/utils/media_utils.py index 1e9f812e16..b2b5082a48 100644 --- a/astrbot/core/utils/media_utils.py +++ b/astrbot/core/utils/media_utils.py @@ -1094,7 +1094,7 @@ def _extract_animation_frames_sync( source_bytes: bytes, target_mime_type: str, max_frames: int, -) -> list[Path]: +) -> tuple[list[Path], bool]: """Extract evenly spaced frames from an animated image, with caching. Args: @@ -1103,7 +1103,8 @@ def _extract_animation_frames_sync( max_frames: Maximum number of frames to extract. Returns: - Paths of the extracted frame images in playback order. + Tuple of frame paths in playback order and whether extraction actually + ran (``False`` when the cached frame set was served). """ suffix = _MIME_FILE_SUFFIX[target_mime_type] cache_key = _image_convert_cache_key( @@ -1114,10 +1115,11 @@ def _extract_animation_frames_sync( if frames_dir.is_dir(): cached = sorted(frames_dir.glob(f"*{suffix}")) if cached: - return cached + return cached, False # Extract into a staging dir and publish it with one atomic rename, so a # crash mid-extraction never leaves a partial frame set behind. staging_dir = Path(tempfile.mkdtemp(dir=cache_dir, prefix=f".{cache_key}_")) + published = True try: with PILImage.open(io.BytesIO(source_bytes)) as image: total_frames = getattr(image, "n_frames", 1) @@ -1134,10 +1136,11 @@ def _extract_animation_frames_sync( shutil.rmtree(staging_dir, ignore_errors=True) if not frames_dir.is_dir(): raise + published = False except BaseException: shutil.rmtree(staging_dir, ignore_errors=True) raise - return sorted(frames_dir.glob(f"*{suffix}")) + return sorted(frames_dir.glob(f"*{suffix}")), published async def _path_to_resolved_media_data(path: Path, mime_type: str) -> ResolvedMediaData: @@ -1227,17 +1230,18 @@ async def resolve_image_ref_to_images( None if unrestricted else allowed_mime_types, ) if animated_strategy == ANIMATED_STRATEGY_MULTI_FRAME: - frame_paths = await asyncio.to_thread( + frame_paths, extracted = await asyncio.to_thread( _extract_animation_frames_sync, image_bytes, target_mime_type, max_frames, ) - logger.info( - "Animated image %s extracted into %d frame(s) for the provider.", - describe_media_ref(image_ref), - len(frame_paths), - ) + if extracted: + logger.info( + "Animated image %s extracted into %d frame(s) for the provider.", + describe_media_ref(image_ref), + len(frame_paths), + ) else: frame_paths = [ await asyncio.to_thread( From 67504d4b93fce9f216cce74f6267e1eeacc3f726 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=B0=95=E6=B0=99?= Date: Sat, 15 Aug 2026 18:18:17 +0800 Subject: [PATCH 5/8] refactor: return content part lists uniformly from part transforms _transform_content_part now always returns list[dict], so _materialize_message_image_parts can extend unconditionally instead of branching on the dual dict | list[dict] return shape. --- .../core/provider/sources/openai_source.py | 22 ++++++++----------- 1 file changed, 9 insertions(+), 13 deletions(-) diff --git a/astrbot/core/provider/sources/openai_source.py b/astrbot/core/provider/sources/openai_source.py index d99ff2ab5f..f8b3fd8022 100644 --- a/astrbot/core/provider/sources/openai_source.py +++ b/astrbot/core/provider/sources/openai_source.py @@ -289,14 +289,14 @@ async def _resolve_audio_part(self, audio_ref: str) -> dict | None: }, } - async def _transform_content_part(self, part: dict) -> dict | list[dict]: + async def _transform_content_part(self, part: dict) -> list[dict]: if not isinstance(part, dict): - return part + return [part] if part.get("type") == "image_url": url, image_detail = self._extract_image_part_info(part) if not url: - return part + return [part] try: resolved_parts = await self._resolve_image_parts( @@ -308,18 +308,18 @@ async def _transform_content_part(self, part: dict) -> dict | list[dict]: url, exc, ) - return part + return [part] - return resolved_parts or part + return resolved_parts or [part] if part.get("type") == "audio_url": audio_ref = self._extract_audio_part_info(part) if not audio_ref: - return part + return [part] resolved_part = await self._resolve_audio_part(audio_ref) - return resolved_part or part + return [resolved_part] if resolved_part else [part] - return part + return [part] async def _materialize_message_image_parts(self, message: dict) -> dict: content = message.get("content") @@ -328,11 +328,7 @@ async def _materialize_message_image_parts(self, message: dict) -> dict: new_content: list[dict] = [] for part in content: - transformed = await self._transform_content_part(part) - if isinstance(transformed, list): - new_content.extend(transformed) - else: - new_content.append(transformed) + new_content.extend(await self._transform_content_part(part)) return {**message, "content": new_content} async def _materialize_context_image_parts( From 8de9646c15ba6592c8464c8c630529a635a475aa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=B0=95=E6=B0=99?= Date: Mon, 31 Aug 2026 01:47:01 +0800 Subject: [PATCH 6/8] feat: make provider image format declarations vendor-aware and editable in dashboard - Add VENDOR_IMAGE_FORMATS (14 brands) as the single source of truth, audited against official vendor docs: MiniMax/Xiaomi widened to their documented sets, Groq/NVIDIA narrowed to jpeg/png, Moonshot/Kimi declared with bmp/heic/heif - Resolve allowed formats by vendor brand (provider_config["provider"]) before falling back to the adapter class declaration, so generic OpenAI-compatible sources for xAI/Zhipu/Groq/etc. get their real sets - Expose provider_type_image_formats and provider_brand_image_formats from the llm schema endpoint and backfill the image_formats checkboxes in the model config dialog so the effective set is visible and editable - Point vendor adapter class declarations at the vendor map to keep a single source of truth --- astrbot/core/config/default.py | 14 ++- astrbot/core/provider/provider.py | 9 +- .../core/provider/sources/anthropic_source.py | 5 +- .../core/provider/sources/gemini_source.py | 5 +- astrbot/core/provider/sources/groq_source.py | 4 + .../core/provider/sources/kimi_code_source.py | 4 + .../sources/minimax_token_plan_source.py | 2 + .../core/provider/sources/openai_source.py | 5 +- astrbot/core/provider/sources/xai_source.py | 4 +- .../core/provider/sources/xiaomi_source.py | 2 + .../sources/xiaomi_token_plan_source.py | 2 + astrbot/core/provider/sources/zhipu_source.py | 4 +- astrbot/core/utils/media_utils.py | 56 +++++++++ astrbot/dashboard/services/config_service.py | 47 +++++++ dashboard/src/api/v1.ts | 2 + .../provider/ProviderChatCompletionPanel.vue | 4 + .../useProviderModelConfigDialog.ts | 16 +++ .../src/composables/useProviderSources.ts | 9 +- .../en-US/features/config-metadata.json | 3 +- .../ru-RU/features/config-metadata.json | 3 +- .../zh-CN/features/config-metadata.json | 3 +- dashboard/src/views/ProviderPage.vue | 4 + tests/unit/test_provider_image_formats.py | 117 ++++++++++++++++++ 23 files changed, 306 insertions(+), 18 deletions(-) diff --git a/astrbot/core/config/default.py b/astrbot/core/config/default.py index c41f6c65d0..ea64859de2 100644 --- a/astrbot/core/config/default.py +++ b/astrbot/core/config/default.py @@ -2135,7 +2135,16 @@ "description": "图片格式支持", "type": "list", "items": {"type": "string"}, - "options": ["jpeg", "png", "webp", "gif", "bmp", "heic", "*"], + "options": [ + "jpeg", + "png", + "webp", + "gif", + "bmp", + "heic", + "heif", + "*", + ], "labels": [ "JPEG", "PNG", @@ -2143,10 +2152,11 @@ "GIF", "BMP", "HEIC", + "HEIF", "不限制", ], "render_type": "checkbox", - "hint": "发送给该提供商前允许的图片格式,不兼容的格式会自动转换。留空使用内置默认值(官方 API 按其文档,未知第三方默认仅 jpeg/png)。勾选「不限制」时其他选项不生效。", + "hint": "发送给该提供商前允许的图片格式,不兼容的格式会自动转换。勾选「不限制」时其他选项不生效。", "default": [], "condition": {"modalities": "image"}, }, diff --git a/astrbot/core/provider/provider.py b/astrbot/core/provider/provider.py index 89546c6ec2..01b0404fb9 100644 --- a/astrbot/core/provider/provider.py +++ b/astrbot/core/provider/provider.py @@ -21,6 +21,7 @@ ANIMATED_STRATEGY_FIRST_FRAME, ANIMATED_STRATEGY_MULTI_FRAME, IMAGE_SHORT_MIME_TYPES, + VENDOR_IMAGE_FORMATS, ) DEFAULT_FALLBACK_IMAGE_FORMATS = frozenset({"image/jpeg", "image/png"}) @@ -98,9 +99,10 @@ def resolve_allowed_image_formats(self) -> frozenset[str] | None: Priority: ``provider_config["image_formats"]`` (per-instance override, short names like ``jpeg`` or MIME types, ``*`` disables restriction) > + the vendor-level set keyed by ``provider_config["provider"]`` (brand of + the provider source, for branded sources running on generic adapters) > the class-level ``supported_image_formats`` > the conservative DEFAULT_FALLBACK_IMAGE_FORMATS (jpeg/png). - Returns: The allowed MIME types, or ``None`` when unrestricted. """ @@ -124,6 +126,11 @@ def resolve_allowed_image_formats(self) -> frozenset[str] | None: self.provider_config.get("id"), sorted(normalized), ) + brand = self.provider_config.get("provider") + if isinstance(brand, str): + vendor_formats = VENDOR_IMAGE_FORMATS.get(brand.strip().lower()) + if vendor_formats is not None: + return vendor_formats if self.supported_image_formats is not None: return self.supported_image_formats return DEFAULT_FALLBACK_IMAGE_FORMATS diff --git a/astrbot/core/provider/sources/anthropic_source.py b/astrbot/core/provider/sources/anthropic_source.py index 111b204918..3cdb4d065f 100644 --- a/astrbot/core/provider/sources/anthropic_source.py +++ b/astrbot/core/provider/sources/anthropic_source.py @@ -17,6 +17,7 @@ from astrbot.core.provider.entities import LLMResponse, TokenUsage from astrbot.core.provider.func_tool_manager import ToolSet from astrbot.core.utils.media_utils import ( + VENDOR_IMAGE_FORMATS, describe_media_ref, detect_image_mime_type, resolve_image_ref_to_images, @@ -38,9 +39,7 @@ class ProviderAnthropic(Provider): _PROMPT_CACHE_CONTROL = {"type": "ephemeral"} - supported_image_formats = frozenset( - {"image/jpeg", "image/png", "image/gif", "image/webp"} - ) + supported_image_formats = VENDOR_IMAGE_FORMATS["anthropic"] """Formats accepted by the official Anthropic vision API.""" @staticmethod diff --git a/astrbot/core/provider/sources/gemini_source.py b/astrbot/core/provider/sources/gemini_source.py index 5bf52090b1..355fe55e69 100644 --- a/astrbot/core/provider/sources/gemini_source.py +++ b/astrbot/core/provider/sources/gemini_source.py @@ -20,6 +20,7 @@ from astrbot.core.provider.entities import LLMResponse, TokenUsage from astrbot.core.provider.func_tool_manager import ToolSet from astrbot.core.utils.media_utils import ( + VENDOR_IMAGE_FORMATS, describe_media_ref, resolve_image_ref_to_images, resolve_media_ref_to_base64_data, @@ -45,9 +46,7 @@ def filter(self, record): "Google Gemini Chat Completion 提供商适配器", ) class ProviderGoogleGenAI(Provider): - supported_image_formats = frozenset( - {"image/png", "image/jpeg", "image/webp", "image/heic", "image/heif"} - ) + supported_image_formats = VENDOR_IMAGE_FORMATS["google"] """Formats accepted by the official Gemini vision API.""" CATEGORY_MAPPING = { diff --git a/astrbot/core/provider/sources/groq_source.py b/astrbot/core/provider/sources/groq_source.py index af4029f67c..ff10699f6b 100644 --- a/astrbot/core/provider/sources/groq_source.py +++ b/astrbot/core/provider/sources/groq_source.py @@ -1,3 +1,5 @@ +from astrbot.core.utils.media_utils import VENDOR_IMAGE_FORMATS + from ..register import register_provider_adapter from .openai_source import ProviderOpenAIOfficial @@ -6,6 +8,8 @@ "groq_chat_completion", "Groq Chat Completion Provider Adapter" ) class ProviderGroq(ProviderOpenAIOfficial): + supported_image_formats = VENDOR_IMAGE_FORMATS["groq"] + def __init__( self, provider_config: dict, diff --git a/astrbot/core/provider/sources/kimi_code_source.py b/astrbot/core/provider/sources/kimi_code_source.py index 02c200271f..f072b97c1f 100644 --- a/astrbot/core/provider/sources/kimi_code_source.py +++ b/astrbot/core/provider/sources/kimi_code_source.py @@ -1,3 +1,5 @@ +from astrbot.core.utils.media_utils import VENDOR_IMAGE_FORMATS + from ..register import register_provider_adapter from .anthropic_source import ProviderAnthropic @@ -11,6 +13,8 @@ "Kimi Code Provider Adapter", ) class ProviderKimiCode(ProviderAnthropic): + supported_image_formats = VENDOR_IMAGE_FORMATS["kimi-code"] + def __init__( self, provider_config: dict, diff --git a/astrbot/core/provider/sources/minimax_token_plan_source.py b/astrbot/core/provider/sources/minimax_token_plan_source.py index 8d86c77b73..3e8a05ba78 100644 --- a/astrbot/core/provider/sources/minimax_token_plan_source.py +++ b/astrbot/core/provider/sources/minimax_token_plan_source.py @@ -2,6 +2,7 @@ from astrbot import logger from astrbot.core.provider.sources.anthropic_source import ProviderAnthropic +from astrbot.core.utils.media_utils import VENDOR_IMAGE_FORMATS from ..register import register_provider_adapter @@ -11,6 +12,7 @@ "MiniMax Token Plan Provider Adapter", ) class ProviderMiniMaxTokenPlan(ProviderAnthropic): + supported_image_formats = VENDOR_IMAGE_FORMATS["minimax-token-plan"] """MiniMax Token Plan provider. The model list is fetched dynamically from the MiniMax API's /v1/models diff --git a/astrbot/core/provider/sources/openai_source.py b/astrbot/core/provider/sources/openai_source.py index f8b3fd8022..5715c3d87c 100644 --- a/astrbot/core/provider/sources/openai_source.py +++ b/astrbot/core/provider/sources/openai_source.py @@ -30,6 +30,7 @@ from astrbot.core.message.message_event_result import MessageChain from astrbot.core.provider.entities import LLMResponse, TokenUsage, ToolCallsResult from astrbot.core.utils.media_utils import ( + VENDOR_IMAGE_FORMATS, ResolvedMediaData, describe_media_ref, resolve_image_ref_to_images, @@ -53,9 +54,7 @@ class ProviderOpenAIOfficial(Provider): _ERROR_TEXT_CANDIDATE_MAX_CHARS = 4096 - supported_image_formats = frozenset( - {"image/png", "image/jpeg", "image/webp", "image/gif"} - ) + supported_image_formats = VENDOR_IMAGE_FORMATS["openai"] """Formats accepted by the official OpenAI vision API (non-animated GIF).""" @classmethod diff --git a/astrbot/core/provider/sources/xai_source.py b/astrbot/core/provider/sources/xai_source.py index 0e8ca2c48c..a9403a90a9 100644 --- a/astrbot/core/provider/sources/xai_source.py +++ b/astrbot/core/provider/sources/xai_source.py @@ -1,3 +1,5 @@ +from astrbot.core.utils.media_utils import VENDOR_IMAGE_FORMATS + from ..register import register_provider_adapter from .openai_source import ProviderOpenAIOfficial @@ -6,7 +8,7 @@ "xai_chat_completion", "xAI Chat Completion Provider Adapter" ) class ProviderXAI(ProviderOpenAIOfficial): - supported_image_formats = frozenset({"image/jpeg", "image/png"}) + supported_image_formats = VENDOR_IMAGE_FORMATS["xai"] """The official xAI vision API accepts only JPEG and PNG.""" def __init__( diff --git a/astrbot/core/provider/sources/xiaomi_source.py b/astrbot/core/provider/sources/xiaomi_source.py index 1f14175f41..485920b51c 100644 --- a/astrbot/core/provider/sources/xiaomi_source.py +++ b/astrbot/core/provider/sources/xiaomi_source.py @@ -1,5 +1,6 @@ from astrbot import logger from astrbot.core.provider.sources.openai_source import ProviderOpenAIOfficial +from astrbot.core.utils.media_utils import VENDOR_IMAGE_FORMATS from ..register import register_provider_adapter @@ -16,6 +17,7 @@ "xiaomi_chat_completion", "Xiaomi API 提供商适配器 (OpenAI 兼容)" ) class ProviderXiaomi(ProviderOpenAIOfficial): + supported_image_formats = VENDOR_IMAGE_FORMATS["xiaomi"] """Xiaomi provider using OpenAI-compatible API. Supports both standard API and multimodal capabilities. diff --git a/astrbot/core/provider/sources/xiaomi_token_plan_source.py b/astrbot/core/provider/sources/xiaomi_token_plan_source.py index 70a2cf0afc..02b5a20a5b 100644 --- a/astrbot/core/provider/sources/xiaomi_token_plan_source.py +++ b/astrbot/core/provider/sources/xiaomi_token_plan_source.py @@ -1,5 +1,6 @@ from astrbot import logger from astrbot.core.provider.sources.anthropic_source import ProviderAnthropic +from astrbot.core.utils.media_utils import VENDOR_IMAGE_FORMATS from ..register import register_provider_adapter @@ -14,6 +15,7 @@ @register_provider_adapter("xiaomi_token_plan", "Xiaomi Token Plan 提供商适配器") class ProviderXiaomiTokenPlan(ProviderAnthropic): + supported_image_formats = VENDOR_IMAGE_FORMATS["xiaomi-token-plan"] """Xiaomi Token Plan provider. The Token Plan API uses Anthropic-compatible endpoint with Bearer token auth. diff --git a/astrbot/core/provider/sources/zhipu_source.py b/astrbot/core/provider/sources/zhipu_source.py index 70e9d22c64..3149fe83f2 100644 --- a/astrbot/core/provider/sources/zhipu_source.py +++ b/astrbot/core/provider/sources/zhipu_source.py @@ -2,13 +2,15 @@ # It is no longer specifically adapted to Zhipu's models. To ensure compatibility, this +from astrbot.core.utils.media_utils import VENDOR_IMAGE_FORMATS + from ..register import register_provider_adapter from .openai_source import ProviderOpenAIOfficial @register_provider_adapter("zhipu_chat_completion", "智谱 Chat Completion 提供商适配器") class ProviderZhipu(ProviderOpenAIOfficial): - supported_image_formats = frozenset({"image/jpeg", "image/png"}) + supported_image_formats = VENDOR_IMAGE_FORMATS["zhipu"] """GLM vision officially guarantees JPEG/PNG; keep it conservative.""" def __init__( diff --git a/astrbot/core/utils/media_utils.py b/astrbot/core/utils/media_utils.py index b2b5082a48..15d5920bdc 100644 --- a/astrbot/core/utils/media_utils.py +++ b/astrbot/core/utils/media_utils.py @@ -87,6 +87,62 @@ } """User-facing short image format names mapped to MIME types.""" +IMAGE_MIME_SHORT_NAMES = { + "image/jpeg": "jpeg", + "image/png": "png", + "image/webp": "webp", + "image/gif": "gif", + "image/bmp": "bmp", + "image/tiff": "tiff", + "image/avif": "avif", + "image/heic": "heic", + "image/heif": "heif", +} +"""Canonical short display name for each known image MIME type.""" + +_MINIMAX_IMAGE_FORMATS = frozenset( + {"image/jpeg", "image/png", "image/webp", "image/gif"} +) +_XIAOMI_IMAGE_FORMATS = frozenset( + {"image/jpeg", "image/png", "image/webp", "image/gif", "image/bmp"} +) +_MOONSHOT_IMAGE_FORMATS = frozenset( + { + "image/jpeg", + "image/png", + "image/webp", + "image/gif", + "image/bmp", + "image/heic", + "image/heif", + } +) + +VENDOR_IMAGE_FORMATS = { + "openai": frozenset({"image/jpeg", "image/png", "image/webp", "image/gif"}), + "azure": frozenset({"image/jpeg", "image/png", "image/webp", "image/gif"}), + "xai": frozenset({"image/jpeg", "image/png"}), + "deepseek": frozenset({"image/jpeg", "image/png", "image/gif", "image/webp"}), + "anthropic": frozenset({"image/jpeg", "image/png", "image/gif", "image/webp"}), + "google": frozenset( + {"image/jpeg", "image/png", "image/webp", "image/heic", "image/heif"} + ), + "zhipu": frozenset({"image/jpeg", "image/png"}), + # NVIDIA NIM VLM docs list JPG/JPEG/PNG for most models (a few also take GIF). + "nvidia": frozenset({"image/jpeg", "image/png"}), + # Groq docs only demonstrate JPEG; PNG is the other reliably referenced type. + "groq": frozenset({"image/jpeg", "image/png"}), + "moonshot": _MOONSHOT_IMAGE_FORMATS, + "kimi-code": _MOONSHOT_IMAGE_FORMATS, + "minimax": _MINIMAX_IMAGE_FORMATS, + "minimax-token-plan": _MINIMAX_IMAGE_FORMATS, + "xiaomi": _XIAOMI_IMAGE_FORMATS, + "xiaomi-token-plan": _XIAOMI_IMAGE_FORMATS, +} +"""Image MIME types officially documented by each vendor (brand key from provider +source templates). Absent brand = no vendor-level opinion; the adapter class +declaration then governs.""" + ANIMATED_STRATEGY_FIRST_FRAME = "first_frame" """Keep only the first frame of an animated image.""" diff --git a/astrbot/dashboard/services/config_service.py b/astrbot/dashboard/services/config_service.py index 2b87573107..d2cac0c7be 100644 --- a/astrbot/dashboard/services/config_service.py +++ b/astrbot/dashboard/services/config_service.py @@ -23,6 +23,7 @@ from astrbot.core.core_lifecycle import AstrBotCoreLifecycle from astrbot.core.db import BaseDatabase from astrbot.core.platform.register import platform_cls_map, platform_registry +from astrbot.core.provider.entities import ProviderType from astrbot.core.provider.register import provider_registry from astrbot.core.star.star import star_registry from astrbot.core.utils.astrbot_path import get_astrbot_plugin_data_path @@ -1392,7 +1393,12 @@ def get_provider_schema(self) -> dict: if provider.default_config_tmpl: provider_default_tmpl[provider.type] = provider.default_config_tmpl providers = copy.deepcopy(self.config.get("provider", [])) + from astrbot.core.provider.register import provider_cls_map from astrbot.core.utils.llm_metadata import LLM_METADATAS + from astrbot.core.utils.media_utils import ( + IMAGE_MIME_SHORT_NAMES, + VENDOR_IMAGE_FORMATS, + ) model_metadata = {} for provider in providers: @@ -1400,11 +1406,52 @@ def get_provider_schema(self) -> dict: model_id = provider.get("model") if isinstance(model_id, str) and model_id in LLM_METADATAS: model_metadata[model_id] = LLM_METADATAS[model_id] + # Officially declared image format sets per chat provider type, + # so the dashboard can display and pre-fill them. Adapters are imported + # lazily, so import every chat template type before reading the classes. + for tmpl in provider_default_tmpl.values(): + if ( + not isinstance(tmpl, dict) + or tmpl.get("provider_type") != "chat_completion" + ): + continue + adapter_type = tmpl.get("type") + if not adapter_type or adapter_type in provider_cls_map: + continue + try: + self.provider_manager.dynamic_import_provider(adapter_type) + except Exception as exc: + logger.debug( + f"Skip image format introspection for {adapter_type}: {exc}" + ) + provider_type_image_formats = {} + for provider in provider_registry: + if provider.provider_type != ProviderType.CHAT_COMPLETION: + continue + declared = getattr(provider.cls_type, "supported_image_formats", None) + if declared is None: + provider_type_image_formats[provider.type] = None + else: + provider_type_image_formats[provider.type] = [ + short + for mime, short in IMAGE_MIME_SHORT_NAMES.items() + if mime in declared + ] + provider_brand_image_formats = { + brand: [ + short + for mime, short in IMAGE_MIME_SHORT_NAMES.items() + if mime in formats + ] + for brand, formats in VENDOR_IMAGE_FORMATS.items() + } return { "config_schema": config_schema, "providers": providers, "provider_sources": self.config.get("provider_sources", []), "model_metadata": model_metadata, + "provider_type_image_formats": provider_type_image_formats, + "provider_brand_image_formats": provider_brand_image_formats, } def list_provider_sources(self) -> dict: diff --git a/dashboard/src/api/v1.ts b/dashboard/src/api/v1.ts index f136e08c20..a5f247b1dc 100644 --- a/dashboard/src/api/v1.ts +++ b/dashboard/src/api/v1.ts @@ -79,6 +79,8 @@ export interface ProviderSchemaData { providers?: OpenConfig[]; provider_sources?: OpenConfig[]; model_metadata?: Record; + provider_type_image_formats?: Record; + provider_brand_image_formats?: Record; } export interface ProviderListData { diff --git a/dashboard/src/components/provider/ProviderChatCompletionPanel.vue b/dashboard/src/components/provider/ProviderChatCompletionPanel.vue index 5e5bb28911..399851bec1 100644 --- a/dashboard/src/components/provider/ProviderChatCompletionPanel.vue +++ b/dashboard/src/components/provider/ProviderChatCompletionPanel.vue @@ -221,6 +221,8 @@ const { testingProviders, isSourceModified, configSchema, + providerTypeImageFormats, + providerBrandImageFormats, providerSourceSchema, manualModelId, modelSearch, @@ -279,6 +281,8 @@ const { } = useProviderModelConfigDialog({ selectedProviderSource, configSchema, + providerTypeImageFormats, + providerBrandImageFormats, buildModelProviderConfig, modelAlreadyConfigured, loadConfig, diff --git a/dashboard/src/composables/useProviderModelConfigDialog.ts b/dashboard/src/composables/useProviderModelConfigDialog.ts index 4d3109481f..9c38955729 100644 --- a/dashboard/src/composables/useProviderModelConfigDialog.ts +++ b/dashboard/src/composables/useProviderModelConfigDialog.ts @@ -4,6 +4,8 @@ import { providerApi } from '@/api/v1' interface UseProviderModelConfigDialogOptions { selectedProviderSource: Ref configSchema: Ref> + providerTypeImageFormats: Ref> + providerBrandImageFormats: Ref> buildModelProviderConfig: (modelId: string) => any modelAlreadyConfigured: (modelId: string) => boolean loadConfig: () => Promise | void @@ -15,6 +17,8 @@ export function useProviderModelConfigDialog(options: UseProviderModelConfigDial const { selectedProviderSource, configSchema, + providerTypeImageFormats, + providerBrandImageFormats, buildModelProviderConfig, modelAlreadyConfigured, loadConfig, @@ -69,6 +73,18 @@ export function useProviderModelConfigDialog(options: UseProviderModelConfigDial } } } + // Backfill the source's officially declared image format set (vendor brand + // first, then adapter type) so the effective default is visible and editable; + // undeclared aggregators fall back to jpeg/png. + if (Array.isArray(editableProvider.image_formats) && editableProvider.image_formats.length === 0) { + const source = selectedProviderSource.value + const declared = source + ? (providerBrandImageFormats.value[source.provider] ?? providerTypeImageFormats.value[source.type]) + : undefined + if (declared !== undefined) { + editableProvider.image_formats = [...(declared ?? ['jpeg', 'png'])] + } + } if (editableProvider.provider_source_id) { delete editableProvider.reasoning } diff --git a/dashboard/src/composables/useProviderSources.ts b/dashboard/src/composables/useProviderSources.ts index 31c7b53d58..8fc41d1e70 100644 --- a/dashboard/src/composables/useProviderSources.ts +++ b/dashboard/src/composables/useProviderSources.ts @@ -63,6 +63,9 @@ export function useProviderSources(options: UseProviderSourcesOptions) { const isSourceModified = ref(false) const configSchema = ref>({}) const providerTemplates = ref>({}) + // Officially declared image format sets per provider type, from the backend schema API. + const providerTypeImageFormats = ref>({}) + const providerBrandImageFormats = ref>({}) const manualModelId = ref('') const modelSearch = ref('') @@ -614,7 +617,7 @@ export function useProviderSources(options: UseProviderSourcesOptions) { modalities, custom_extra_body: {}, max_context_tokens: max_context_tokens, - image_formats: [], + image_formats: [...(providerBrandImageFormats.value[selectedProviderSource.value.provider] ?? providerTypeImageFormats.value[selectedProviderSource.value.type] ?? ['jpeg', 'png'])], animated_image_strategy: 'first_frame', animated_image_max_frames: 4 } @@ -716,6 +719,8 @@ export function useProviderSources(options: UseProviderSourcesOptions) { providerTemplates.value = configSchema.value.provider.config_template } providerSources.value = response.data.data.provider_sources || [] + providerTypeImageFormats.value = response.data.data.provider_type_image_formats || {} + providerBrandImageFormats.value = response.data.data.provider_brand_image_formats || {} modelMetadata.value = (response.data.data.model_metadata || {}) as Record providers.value = response.data.data.providers || [] } @@ -751,6 +756,8 @@ export function useProviderSources(options: UseProviderSourcesOptions) { isSourceModified, configSchema, providerTemplates, + providerTypeImageFormats, + providerBrandImageFormats, manualModelId, modelSearch, diff --git a/dashboard/src/i18n/locales/en-US/features/config-metadata.json b/dashboard/src/i18n/locales/en-US/features/config-metadata.json index 68ccda628b..a06f9f4464 100644 --- a/dashboard/src/i18n/locales/en-US/features/config-metadata.json +++ b/dashboard/src/i18n/locales/en-US/features/config-metadata.json @@ -1281,7 +1281,7 @@ }, "image_formats": { "description": "Supported image formats", - "hint": "Image formats allowed when sending to this provider; incompatible formats are converted automatically. Leave empty to use built-in defaults (official APIs follow their documentation; unknown third parties default to jpeg/png). Selecting \"Unrestricted\" overrides all other options.", + "hint": "Image formats allowed when sending to this provider; incompatible formats are converted automatically. Selecting \"Unrestricted\" overrides all other options.", "labels": [ "JPEG", "PNG", @@ -1289,6 +1289,7 @@ "GIF", "BMP", "HEIC", + "HEIF", "Unrestricted" ] }, diff --git a/dashboard/src/i18n/locales/ru-RU/features/config-metadata.json b/dashboard/src/i18n/locales/ru-RU/features/config-metadata.json index 6a43926ab3..df073eb941 100644 --- a/dashboard/src/i18n/locales/ru-RU/features/config-metadata.json +++ b/dashboard/src/i18n/locales/ru-RU/features/config-metadata.json @@ -1282,7 +1282,7 @@ }, "image_formats": { "description": "Поддерживаемые форматы изображений", - "hint": "Форматы изображений, разрешённые при отправке этому провайдеру; несовместимые форматы конвертируются автоматически. Оставьте пустым для встроенных значений по умолчанию (официальные API — согласно документации, неизвестные сторонние — только jpeg/png). При выборе «Без ограничений» остальные варианты не действуют.", + "hint": "Форматы изображений, разрешённые при отправке этому провайдеру; несовместимые форматы конвертируются автоматически. При выборе «Без ограничений» остальные варианты не действуют.", "labels": [ "JPEG", "PNG", @@ -1290,6 +1290,7 @@ "GIF", "BMP", "HEIC", + "HEIF", "Без ограничений" ] }, diff --git a/dashboard/src/i18n/locales/zh-CN/features/config-metadata.json b/dashboard/src/i18n/locales/zh-CN/features/config-metadata.json index 1421ff728e..d9871a343f 100644 --- a/dashboard/src/i18n/locales/zh-CN/features/config-metadata.json +++ b/dashboard/src/i18n/locales/zh-CN/features/config-metadata.json @@ -1283,7 +1283,7 @@ }, "image_formats": { "description": "图片格式支持", - "hint": "发送给该提供商前允许的图片格式,不兼容的格式会自动转换。留空使用内置默认值(官方 API 按其文档,未知第三方默认仅 jpeg/png)。勾选「不限制」时其他选项不生效。", + "hint": "发送给该提供商前允许的图片格式,不兼容的格式会自动转换。勾选「不限制」时其他选项不生效。", "labels": [ "JPEG", "PNG", @@ -1291,6 +1291,7 @@ "GIF", "BMP", "HEIC", + "HEIF", "不限制" ] }, diff --git a/dashboard/src/views/ProviderPage.vue b/dashboard/src/views/ProviderPage.vue index c5238bd644..ad6070469a 100644 --- a/dashboard/src/views/ProviderPage.vue +++ b/dashboard/src/views/ProviderPage.vue @@ -382,6 +382,8 @@ const { testingProviders, isSourceModified, configSchema, + providerTypeImageFormats, + providerBrandImageFormats, providerSourceSchema, manualModelId, modelSearch, @@ -440,6 +442,8 @@ const { } = useProviderModelConfigDialog({ selectedProviderSource, configSchema, + providerTypeImageFormats, + providerBrandImageFormats, buildModelProviderConfig, modelAlreadyConfigured, loadConfig, diff --git a/tests/unit/test_provider_image_formats.py b/tests/unit/test_provider_image_formats.py index b9b37e992f..23122eb998 100644 --- a/tests/unit/test_provider_image_formats.py +++ b/tests/unit/test_provider_image_formats.py @@ -218,3 +218,120 @@ async def test_anthropic_context_skips_unsupported_format(): assert image_blocks == [] finally: await provider.terminate() + + +def _make_openai(provider_config: dict) -> ProviderOpenAIOfficial: + config = { + "id": "test-openai", + "type": "openai_chat_completion", + "model": "deepseek-v4-flash-vision-exp", + "key": ["test-key"], + **provider_config, + } + return ProviderOpenAIOfficial(config, {}) + + +@pytest.mark.asyncio +async def test_vendor_brand_overrides_adapter_declared_formats(): + provider = _make_openai({"provider": "xai", "type": "openai_responses"}) + try: + assert provider.resolve_allowed_image_formats() == frozenset( + {"image/jpeg", "image/png"} + ) + finally: + await provider.terminate() + + +@pytest.mark.asyncio +async def test_unknown_brand_keeps_adapter_declared_formats(): + provider = _make_openai({"provider": "some-unknown-vendor"}) + try: + assert provider.resolve_allowed_image_formats() == frozenset( + {"image/png", "image/jpeg", "image/webp", "image/gif"} + ) + finally: + await provider.terminate() + + +@pytest.mark.asyncio +async def test_config_override_wins_over_vendor_brand(): + provider = _make_openai({"provider": "xai", "image_formats": ["webp"]}) + try: + assert provider.resolve_allowed_image_formats() == frozenset({"image/webp"}) + finally: + await provider.terminate() + + +def test_vendor_map_matches_adapter_declarations(): + from astrbot.core.provider.sources.anthropic_source import ProviderAnthropic + from astrbot.core.provider.sources.gemini_source import ProviderGoogleGenAI + from astrbot.core.provider.sources.openai_source import ProviderOpenAIOfficial + from astrbot.core.provider.sources.xai_source import ProviderXAI + from astrbot.core.provider.sources.zhipu_source import ProviderZhipu + from astrbot.core.utils.media_utils import VENDOR_IMAGE_FORMATS + + assert ( + ProviderOpenAIOfficial.supported_image_formats is VENDOR_IMAGE_FORMATS["openai"] + ) + assert ProviderXAI.supported_image_formats is VENDOR_IMAGE_FORMATS["xai"] + assert ProviderZhipu.supported_image_formats is VENDOR_IMAGE_FORMATS["zhipu"] + assert ProviderGoogleGenAI.supported_image_formats is VENDOR_IMAGE_FORMATS["google"] + assert ( + ProviderAnthropic.supported_image_formats is VENDOR_IMAGE_FORMATS["anthropic"] + ) + + +def test_vendor_map_matches_vendor_adapter_declarations(): + from astrbot.core.provider.sources.groq_source import ProviderGroq + from astrbot.core.provider.sources.kimi_code_source import ProviderKimiCode + from astrbot.core.provider.sources.minimax_token_plan_source import ( + ProviderMiniMaxTokenPlan, + ) + from astrbot.core.provider.sources.xiaomi_source import ProviderXiaomi + from astrbot.core.provider.sources.xiaomi_token_plan_source import ( + ProviderXiaomiTokenPlan, + ) + from astrbot.core.utils.media_utils import VENDOR_IMAGE_FORMATS + + assert ProviderGroq.supported_image_formats is VENDOR_IMAGE_FORMATS["groq"] + assert ProviderXiaomi.supported_image_formats is VENDOR_IMAGE_FORMATS["xiaomi"] + assert ProviderKimiCode.supported_image_formats is VENDOR_IMAGE_FORMATS["kimi-code"] + assert ( + ProviderMiniMaxTokenPlan.supported_image_formats + is VENDOR_IMAGE_FORMATS["minimax-token-plan"] + ) + assert ( + ProviderXiaomiTokenPlan.supported_image_formats + is VENDOR_IMAGE_FORMATS["xiaomi-token-plan"] + ) + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("brand", "expected"), + [ + ("deepseek", {"image/jpeg", "image/png", "image/webp", "image/gif"}), + ( + "moonshot", + { + "image/jpeg", + "image/png", + "image/webp", + "image/gif", + "image/bmp", + "image/heic", + "image/heif", + }, + ), + ("minimax", {"image/jpeg", "image/png", "image/webp", "image/gif"}), + ("xiaomi", {"image/jpeg", "image/png", "image/webp", "image/gif", "image/bmp"}), + ("groq", {"image/jpeg", "image/png"}), + ("nvidia", {"image/jpeg", "image/png"}), + ], +) +async def test_vendor_brand_format_sets(brand, expected): + provider = _make_openai({"provider": brand}) + try: + assert provider.resolve_allowed_image_formats() == frozenset(expected) + finally: + await provider.terminate() From 80ae1b508584be74f3920581a839d703f0b38c25 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=B0=95=E6=B0=99?= Date: Wed, 2 Sep 2026 03:06:40 +0800 Subject: [PATCH 7/8] feat: replace animated image strategies with a fixed 3x3 frame montage - Animated images are always tiled into a single 3x3 contact sheet: first and last frames guaranteed, missing cells padded white, flattened for JPEG, one cached output per source - Montage size reuses provider_settings.image_compress_options.max_size (shared with still-image compression) instead of new config keys - Remove first_frame/multi_frame strategies, their per-provider config schema, and the dashboard option entries; single-frame GIFs keep the still-image conversion path --- astrbot/core/config/default.py | 19 -- astrbot/core/provider/provider.py | 37 ++-- .../core/provider/sources/anthropic_source.py | 5 +- .../core/provider/sources/gemini_source.py | 10 +- .../core/provider/sources/openai_source.py | 5 +- astrbot/core/utils/media_utils.py | 173 ++++++++---------- .../en-US/features/config-metadata.json | 9 - .../ru-RU/features/config-metadata.json | 9 - .../zh-CN/features/config-metadata.json | 9 - tests/test_media_utils.py | 122 +++++++----- tests/test_openai_source.py | 3 +- tests/unit/test_provider_image_formats.py | 89 +++++++-- 12 files changed, 244 insertions(+), 246 deletions(-) diff --git a/astrbot/core/config/default.py b/astrbot/core/config/default.py index 4c62c84d19..7bc35a450e 100644 --- a/astrbot/core/config/default.py +++ b/astrbot/core/config/default.py @@ -2249,25 +2249,6 @@ "default": [], "condition": {"modalities": "image"}, }, - "animated_image_strategy": { - "description": "动图处理策略", - "type": "string", - "options": ["first_frame", "multi_frame"], - "labels": ["仅首帧", "多帧抽取"], - "hint": "GIF 等动图发送给模型时的处理方式:仅取首帧(省 token),或按时长均匀抽帧后作为多张图片发送。", - "default": "first_frame", - "condition": {"modalities": "image"}, - }, - "animated_image_max_frames": { - "description": "动图最大抽帧数", - "type": "int", - "hint": "多帧抽取策略下最多发送的帧数(1-16),默认 4。帧数越多 token 消耗越大。", - "default": 4, - "condition": { - "modalities": "image", - "animated_image_strategy": "multi_frame", - }, - }, "custom_headers": { "description": "自定义请求头", "type": "dict", diff --git a/astrbot/core/provider/provider.py b/astrbot/core/provider/provider.py index 01b0404fb9..ea034a5038 100644 --- a/astrbot/core/provider/provider.py +++ b/astrbot/core/provider/provider.py @@ -16,10 +16,7 @@ from astrbot.core.provider.register import provider_cls_map from astrbot.core.utils.astrbot_path import get_astrbot_path from astrbot.core.utils.media_utils import ( - ANIMATED_DEFAULT_MAX_FRAMES, - ANIMATED_MAX_FRAMES_LIMIT, - ANIMATED_STRATEGY_FIRST_FRAME, - ANIMATED_STRATEGY_MULTI_FRAME, + IMAGE_COMPRESS_DEFAULT_MAX_SIZE, IMAGE_SHORT_MIME_TYPES, VENDOR_IMAGE_FORMATS, ) @@ -135,33 +132,21 @@ def resolve_allowed_image_formats(self) -> frozenset[str] | None: return self.supported_image_formats return DEFAULT_FALLBACK_IMAGE_FORMATS - def get_animated_image_strategy(self) -> tuple[str, int]: - """Read the animated image handling strategy from the provider config. + def get_animated_montage_max_size(self) -> int: + """Longest-edge cap for the animated-image montage, in pixels. + + Reuses ``provider_settings.image_compress_options.max_size`` so still + images and montages share one size control. Returns: - Tuple of ``(strategy, max_frames)`` where strategy is - ``first_frame`` or ``multi_frame`` and max_frames is clamped to - ``[1, 16]``. + The configured cap, or the compress default when unset or invalid. """ - strategy = str( - self.provider_config.get("animated_image_strategy") - or ANIMATED_STRATEGY_FIRST_FRAME - ) - if strategy not in ( - ANIMATED_STRATEGY_FIRST_FRAME, - ANIMATED_STRATEGY_MULTI_FRAME, - ): - strategy = ANIMATED_STRATEGY_FIRST_FRAME - raw_max_frames = self.provider_config.get("animated_image_max_frames") + options = self.provider_settings.get("image_compress_options") + raw = options.get("max_size") if isinstance(options, dict) else None try: - max_frames = ( - ANIMATED_DEFAULT_MAX_FRAMES - if raw_max_frames is None - else int(raw_max_frames) - ) + return max(int(raw), 1) except (TypeError, ValueError): - max_frames = ANIMATED_DEFAULT_MAX_FRAMES - return strategy, min(max(max_frames, 1), ANIMATED_MAX_FRAMES_LIMIT) + return IMAGE_COMPRESS_DEFAULT_MAX_SIZE @abc.abstractmethod def get_current_key(self) -> str: diff --git a/astrbot/core/provider/sources/anthropic_source.py b/astrbot/core/provider/sources/anthropic_source.py index 6c5f508167..54ad84de51 100644 --- a/astrbot/core/provider/sources/anthropic_source.py +++ b/astrbot/core/provider/sources/anthropic_source.py @@ -919,12 +919,11 @@ async def text_chat_stream( async def _image_ref_to_images(self, image_ref: str, *, strict: bool = False): """Resolve an image ref with this provider's format adaptation applied.""" - strategy, max_frames = self.get_animated_image_strategy() + montage_max_size = self.get_animated_montage_max_size() return await resolve_image_ref_to_images( image_ref, allowed_mime_types=self.resolve_allowed_image_formats(), - animated_strategy=strategy, - animated_max_frames=max_frames, + montage_max_size=montage_max_size, strict=strict, ) diff --git a/astrbot/core/provider/sources/gemini_source.py b/astrbot/core/provider/sources/gemini_source.py index 582a636128..0e697ab879 100644 --- a/astrbot/core/provider/sources/gemini_source.py +++ b/astrbot/core/provider/sources/gemini_source.py @@ -1001,12 +1001,11 @@ async def assemble_context( """组装上下文。""" async def resolve_image_part(image_url: str) -> list[dict]: - strategy, max_frames = self.get_animated_image_strategy() + montage_max_size = self.get_animated_montage_max_size() image_datas = await resolve_image_ref_to_images( image_url, allowed_mime_types=self.resolve_allowed_image_formats(), - animated_strategy=strategy, - animated_max_frames=max_frames, + montage_max_size=montage_max_size, ) if not image_datas: logger.warning("Image preprocessing returned no data; ignoring it.") @@ -1100,12 +1099,11 @@ async def resolve_audio_part(audio_path: str) -> dict | None: async def encode_image_bs64(self, image_url: str) -> str: """将图片转换为 base64""" - strategy, max_frames = self.get_animated_image_strategy() + montage_max_size = self.get_animated_montage_max_size() image_datas = await resolve_image_ref_to_images( image_url, allowed_mime_types=self.resolve_allowed_image_formats(), - animated_strategy=strategy, - animated_max_frames=max_frames, + montage_max_size=montage_max_size, strict=True, ) if not image_datas: diff --git a/astrbot/core/provider/sources/openai_source.py b/astrbot/core/provider/sources/openai_source.py index 5715c3d87c..d4b49ba708 100644 --- a/astrbot/core/provider/sources/openai_source.py +++ b/astrbot/core/provider/sources/openai_source.py @@ -189,12 +189,11 @@ async def _image_ref_to_images( mode: Literal["safe", "strict"] = "safe", ) -> list[ResolvedMediaData]: """Resolve an image ref with this provider's format adaptation applied.""" - strategy, max_frames = self.get_animated_image_strategy() + montage_max_size = self.get_animated_montage_max_size() return await resolve_image_ref_to_images( image_ref, allowed_mime_types=self.resolve_allowed_image_formats(), - animated_strategy=strategy, - animated_max_frames=max_frames, + montage_max_size=montage_max_size, strict=mode == "strict", ) diff --git a/astrbot/core/utils/media_utils.py b/astrbot/core/utils/media_utils.py index 15d5920bdc..606cede419 100644 --- a/astrbot/core/utils/media_utils.py +++ b/astrbot/core/utils/media_utils.py @@ -143,14 +143,10 @@ source templates). Absent brand = no vendor-level opinion; the adapter class declaration then governs.""" -ANIMATED_STRATEGY_FIRST_FRAME = "first_frame" -"""Keep only the first frame of an animated image.""" +ANIMATED_MONTAGE_GRID = 3 +"""Animated images become a grid x grid frame montage (contact sheet).""" -ANIMATED_STRATEGY_MULTI_FRAME = "multi_frame" -"""Extract multiple evenly spaced frames from an animated image.""" - -ANIMATED_DEFAULT_MAX_FRAMES = 4 -ANIMATED_MAX_FRAMES_LIMIT = 16 +ANIMATED_MONTAGE_FRAME_COUNT = ANIMATED_MONTAGE_GRID * ANIMATED_MONTAGE_GRID CONVERT_CACHE_DIR_NAME = "media_convert_cache" """Cache directory (under the AstrBot temp dir) for converted images and frames.""" @@ -1110,30 +1106,23 @@ def _save_image_frame_atomic( def _convert_image_bytes_sync( source_bytes: bytes, target_mime_type: str, - *, - frame_index: int | None = None, ) -> Path: """Convert image bytes to the target format, cached under the temp dir. Args: source_bytes: Encoded source image bytes. target_mime_type: Target MIME type; must be Pillow-savable. - frame_index: Frame to extract first, for animated sources. Returns: Path of the converted (or previously cached) image. """ - cache_key = _image_convert_cache_key( - source_bytes, f"convert|{target_mime_type}|frame={frame_index}" - ) + cache_key = _image_convert_cache_key(source_bytes, f"convert|{target_mime_type}") output_path = _image_convert_cache_dir() / ( cache_key + _MIME_FILE_SUFFIX[target_mime_type] ) if output_path.exists(): return output_path with PILImage.open(io.BytesIO(source_bytes)) as image: - if frame_index is not None: - image.seek(frame_index) _save_image_frame_atomic(image, target_mime_type, output_path) return output_path @@ -1146,57 +1135,71 @@ def _even_frame_indices(total_frames: int, max_frames: int) -> list[int]: return sorted({round(i * (total_frames - 1) / (count - 1)) for i in range(count)}) -def _extract_animation_frames_sync( +def _extract_animation_montage_sync( source_bytes: bytes, target_mime_type: str, - max_frames: int, -) -> tuple[list[Path], bool]: - """Extract evenly spaced frames from an animated image, with caching. + max_size: int, +) -> tuple[Path, bool]: + """Tile evenly spaced frames of an animated image into one grid montage. + + Frame sampling includes the first and last frames; grid cells beyond the + available frames stay blank (white). The montage is flattened onto a white + background and its longest edge is capped at ``max_size`` (never upscaled). Args: source_bytes: Encoded animated image bytes. - target_mime_type: MIME type each extracted frame is saved as. - max_frames: Maximum number of frames to extract. + target_mime_type: MIME type the montage is saved as. + max_size: Longest edge of the montage in pixels. Returns: - Tuple of frame paths in playback order and whether extraction actually - ran (``False`` when the cached frame set was served). + Tuple of the montage path and whether extraction actually ran + (``False`` when the cached montage was served). """ suffix = _MIME_FILE_SUFFIX[target_mime_type] cache_key = _image_convert_cache_key( - source_bytes, f"frames|{target_mime_type}|n={max_frames}" + source_bytes, f"montage|{target_mime_type}|s={max_size}" ) - cache_dir = _image_convert_cache_dir() - frames_dir = cache_dir / f"{cache_key}_frames" - if frames_dir.is_dir(): - cached = sorted(frames_dir.glob(f"*{suffix}")) - if cached: - return cached, False - # Extract into a staging dir and publish it with one atomic rename, so a - # crash mid-extraction never leaves a partial frame set behind. - staging_dir = Path(tempfile.mkdtemp(dir=cache_dir, prefix=f".{cache_key}_")) - published = True - try: - with PILImage.open(io.BytesIO(source_bytes)) as image: - total_frames = getattr(image, "n_frames", 1) - for out_index, frame_index in enumerate( - _even_frame_indices(total_frames, max_frames) - ): - frame_path = staging_dir / f"f{out_index}{suffix}" - image.seek(frame_index) - _save_current_image_frame(image, target_mime_type, frame_path) + output_path = _image_convert_cache_dir() / (cache_key + suffix) + if output_path.exists(): + return output_path, False + with PILImage.open(io.BytesIO(source_bytes)) as image: + total_frames = getattr(image, "n_frames", 1) + frame_indices = _even_frame_indices(total_frames, ANIMATED_MONTAGE_FRAME_COUNT) + # Floor the per-cell scale so the montage never exceeds max_size. + longest_edge = max(image.size) * ANIMATED_MONTAGE_GRID + scale = min(1.0, max(max_size, 1) / longest_edge) + cell_size = ( + max(1, int(image.size[0] * scale)), + max(1, int(image.size[1] * scale)), + ) + canvas = PILImage.new( + "RGB", + ( + cell_size[0] * ANIMATED_MONTAGE_GRID, + cell_size[1] * ANIMATED_MONTAGE_GRID, + ), + (255, 255, 255), + ) try: - os.replace(staging_dir, frames_dir) - except OSError: - # Lost a concurrent publish race; use the winner's complete set. - shutil.rmtree(staging_dir, ignore_errors=True) - if not frames_dir.is_dir(): - raise - published = False - except BaseException: - shutil.rmtree(staging_dir, ignore_errors=True) - raise - return sorted(frames_dir.glob(f"*{suffix}")), published + for out_index, frame_index in enumerate(frame_indices): + image.seek(frame_index) + frame = image.convert("RGBA") + if frame.size != cell_size: + frame = frame.resize(cell_size, PILImage.Resampling.LANCZOS) + # Paste with the alpha band as mask so transparency shows white. + canvas.paste( + frame, + ( + (out_index % ANIMATED_MONTAGE_GRID) * cell_size[0], + (out_index // ANIMATED_MONTAGE_GRID) * cell_size[1], + ), + frame, + ) + frame.close() + _save_image_frame_atomic(canvas, target_mime_type, output_path) + finally: + canvas.close() + return output_path, True async def _path_to_resolved_media_data(path: Path, mime_type: str) -> ResolvedMediaData: @@ -1211,8 +1214,7 @@ async def resolve_image_ref_to_images( image_ref: MediaRefStr, *, allowed_mime_types: Collection[str] | None = None, - animated_strategy: str = ANIMATED_STRATEGY_FIRST_FRAME, - animated_max_frames: int = ANIMATED_DEFAULT_MAX_FRAMES, + montage_max_size: int = IMAGE_COMPRESS_DEFAULT_MAX_SIZE, strict: bool = False, default_mime_type: str | None = "image/jpeg", ) -> list[ResolvedMediaData]: @@ -1221,19 +1223,15 @@ async def resolve_image_ref_to_images( Applies per-provider format adaptation: still images whose detected MIME type is not in ``allowed_mime_types`` are converted via Pillow (cached under the AstrBot temp directory, so a source is only converted once until the cache is - cleaned), and animated images are reduced to still frames according to - ``animated_strategy``. Compatible images are returned untouched without any - conversion or cache write. + cleaned), and animated images are tiled into one frame-grid montage. Single + frame animated images fall through to the still-image path. Compatible + images are returned untouched without any conversion or cache write. Args: image_ref: Image reference in any form accepted by ``MediaResolver``. allowed_mime_types: MIME types accepted by the target provider. ``None`` or a collection containing ``"*"`` disables adaptation. - animated_strategy: ``first_frame`` keeps only the first frame; - ``multi_frame`` extracts up to ``animated_max_frames`` evenly spaced - frames as separate images. - animated_max_frames: Maximum frames for ``multi_frame``, clamped to - ``[1, 16]``. + montage_max_size: Longest edge in pixels of the animated-image montage. strict: Raise on invalid or undecodable images instead of skipping them. default_mime_type: Fallback MIME type for legacy base64 payloads. @@ -1277,40 +1275,27 @@ async def resolve_image_ref_to_images( ) return [] - if frame_count > 1 and ( - not unrestricted or animated_strategy == ANIMATED_STRATEGY_MULTI_FRAME - ): - max_frames = min(max(int(animated_max_frames), 1), ANIMATED_MAX_FRAMES_LIMIT) + if frame_count > 1: + # Montage cells are flattened onto white, so alpha no longer matters + # when picking the target format. target_mime_type = _pick_target_image_mime_type( - has_alpha, + False, None if unrestricted else allowed_mime_types, ) - if animated_strategy == ANIMATED_STRATEGY_MULTI_FRAME: - frame_paths, extracted = await asyncio.to_thread( - _extract_animation_frames_sync, - image_bytes, - target_mime_type, - max_frames, + montage_path, extracted = await asyncio.to_thread( + _extract_animation_montage_sync, + image_bytes, + target_mime_type, + max(int(montage_max_size), 1), + ) + if extracted: + logger.info( + "Animated image %s tiled into a %dx%d frame montage for the provider.", + describe_media_ref(image_ref), + ANIMATED_MONTAGE_GRID, + ANIMATED_MONTAGE_GRID, ) - if extracted: - logger.info( - "Animated image %s extracted into %d frame(s) for the provider.", - describe_media_ref(image_ref), - len(frame_paths), - ) - else: - frame_paths = [ - await asyncio.to_thread( - _convert_image_bytes_sync, - image_bytes, - target_mime_type, - frame_index=0, - ) - ] - return [ - await _path_to_resolved_media_data(path, target_mime_type) - for path in frame_paths - ] + return [await _path_to_resolved_media_data(montage_path, target_mime_type)] if unrestricted or media_data.mime_type in allowed_mime_types: return [media_data] diff --git a/dashboard/src/i18n/locales/en-US/features/config-metadata.json b/dashboard/src/i18n/locales/en-US/features/config-metadata.json index 09309d0aeb..da66a57261 100644 --- a/dashboard/src/i18n/locales/en-US/features/config-metadata.json +++ b/dashboard/src/i18n/locales/en-US/features/config-metadata.json @@ -1418,15 +1418,6 @@ "Unrestricted" ] }, - "animated_image_strategy": { - "description": "Animated image strategy", - "hint": "How animated images such as GIFs are sent to the model: keep only the first frame (saves tokens), or extract evenly spaced frames and send them as multiple images.", - "labels": ["First frame only", "Multi-frame extraction"] - }, - "animated_image_max_frames": { - "description": "Max animation frames", - "hint": "Maximum number of frames sent under the multi-frame strategy (1-16), default 4. More frames consume more tokens." - }, "custom_headers": { "description": "Custom request headers", "hint": "Key/value pairs added here are merged into the OpenAI SDK default_headers for custom HTTP headers. Values must be strings." diff --git a/dashboard/src/i18n/locales/ru-RU/features/config-metadata.json b/dashboard/src/i18n/locales/ru-RU/features/config-metadata.json index 4450e35a15..e65712b2b7 100644 --- a/dashboard/src/i18n/locales/ru-RU/features/config-metadata.json +++ b/dashboard/src/i18n/locales/ru-RU/features/config-metadata.json @@ -1407,15 +1407,6 @@ "Без ограничений" ] }, - "animated_image_strategy": { - "description": "Стратегия для анимированных изображений", - "hint": "Как анимированные изображения (например, GIF) отправляются в модель: только первый кадр (экономит токены) или равномерно извлечённые кадры в виде нескольких изображений.", - "labels": ["Только первый кадр", "Извлечение нескольких кадров"] - }, - "animated_image_max_frames": { - "description": "Максимум кадров анимации", - "hint": "Максимальное число кадров, отправляемых при стратегии извлечения нескольких кадров (1-16), по умолчанию 4. Больше кадров — больше расход токенов." - }, "custom_headers": { "description": "Заголовки запроса", "hint": "Пары ключ/значение будут добавлены в заголовки запроса (default_headers). Значения должны быть строками." diff --git a/dashboard/src/i18n/locales/zh-CN/features/config-metadata.json b/dashboard/src/i18n/locales/zh-CN/features/config-metadata.json index de2634fe00..e4fe8462d5 100644 --- a/dashboard/src/i18n/locales/zh-CN/features/config-metadata.json +++ b/dashboard/src/i18n/locales/zh-CN/features/config-metadata.json @@ -1441,15 +1441,6 @@ "不限制" ] }, - "animated_image_strategy": { - "description": "动图处理策略", - "hint": "GIF 等动图发送给模型时的处理方式:仅取首帧(省 token),或按时长均匀抽帧后作为多张图片发送。", - "labels": ["仅首帧", "多帧抽取"] - }, - "animated_image_max_frames": { - "description": "动图最大抽帧数", - "hint": "多帧抽取策略下最多发送的帧数(1-16),默认 4。帧数越多 token 消耗越大。" - }, "custom_headers": { "description": "自定义请求头", "hint": "此处添加的键值对将被合并到 OpenAI SDK 的 default_headers 中,用于自定义 HTTP 请求头。值必须为字符串。" diff --git a/tests/test_media_utils.py b/tests/test_media_utils.py index 70b26fee96..d8a533196b 100644 --- a/tests/test_media_utils.py +++ b/tests/test_media_utils.py @@ -887,10 +887,13 @@ def _make_image_bytes( return buffer.getvalue() -def _make_animated_gif_bytes(colors: list[tuple[int, int, int]]) -> bytes: +def _make_animated_gif_bytes( + colors: list[tuple[int, int, int]], + size: tuple[int, int] = (8, 8), +) -> bytes: from PIL import Image as PILImage - frames = [PILImage.new("RGB", (8, 8), color) for color in colors] + frames = [PILImage.new("RGB", size, color) for color in colors] buffer = BytesIO() frames[0].save( buffer, @@ -967,92 +970,117 @@ async def test_resolve_images_alpha_source_prefers_png(tmp_path, monkeypatch): @pytest.mark.asyncio -async def test_resolve_images_animated_first_frame(tmp_path, monkeypatch): +async def test_resolve_images_animated_gif_becomes_montage(tmp_path, monkeypatch): monkeypatch.setattr(media_utils, "get_astrbot_temp_path", lambda: str(tmp_path)) - gif_bytes = _make_animated_gif_bytes( - [(255, 0, 0), (0, 255, 0), (0, 0, 255)], - ) + from PIL import Image as PILImage + + colors = [(255, 0, 0), (0, 255, 0), (0, 0, 255)] + gif_bytes = _make_animated_gif_bytes(colors) images = await media_utils.resolve_image_ref_to_images( _image_data_uri("image/gif", gif_bytes), - allowed_mime_types={"image/jpeg", "image/png"}, - animated_strategy=media_utils.ANIMATED_STRATEGY_FIRST_FRAME, + allowed_mime_types={"image/png"}, ) assert len(images) == 1 - assert images[0].mime_type == "image/jpeg" + assert images[0].mime_type == "image/png" + with PILImage.open(BytesIO(images[0].to_bytes())) as montage: + # 8px frames in a 3x3 grid: 3 colored cells, the rest padded white. + assert montage.size == (24, 24) + rgb = montage.convert("RGB") + assert rgb.getpixel((4, 4)) == colors[0] + assert rgb.getpixel((12, 4)) == colors[1] + assert rgb.getpixel((20, 4)) == colors[2] + for x, y in [(4, 12), (12, 12), (20, 12), (4, 20), (12, 20), (20, 20)]: + assert rgb.getpixel((x, y)) == (255, 255, 255) @pytest.mark.asyncio -async def test_resolve_images_multi_frame_evenly_spaced(tmp_path, monkeypatch): +async def test_resolve_images_montage_keeps_first_and_last_frame(tmp_path, monkeypatch): monkeypatch.setattr(media_utils, "get_astrbot_temp_path", lambda: str(tmp_path)) from PIL import Image as PILImage - colors = [ - (255, 0, 0), - (0, 255, 0), - (0, 0, 255), - (255, 255, 0), - (0, 255, 255), - (255, 0, 255), - ] + colors = [(i * 12 % 256, 0, 0) for i in range(20)] gif_bytes = _make_animated_gif_bytes(colors) images = await media_utils.resolve_image_ref_to_images( _image_data_uri("image/gif", gif_bytes), allowed_mime_types={"image/png"}, - animated_strategy=media_utils.ANIMATED_STRATEGY_MULTI_FRAME, - animated_max_frames=4, ) - # 6 frames, 4 picks -> indices 0, 2, 3, 5 (evenly spaced). - expected_indices = [0, 2, 3, 5] - assert len(images) == len(expected_indices) - for image_data, frame_index in zip(images, expected_indices, strict=True): - assert image_data.mime_type == "image/png" - with PILImage.open(BytesIO(image_data.to_bytes())) as frame: - assert frame.convert("RGB").getpixel((0, 0)) == colors[frame_index] + assert len(images) == 1 + with PILImage.open(BytesIO(images[0].to_bytes())) as montage: + # 20 frames sampled to 9 cells; cell 0 and cell 8 are the endpoints. + rgb = montage.convert("RGB") + assert rgb.getpixel((4, 4)) == colors[0] + assert rgb.getpixel((20, 20)) == colors[-1] @pytest.mark.asyncio -async def test_resolve_images_multi_frame_clamps_to_limit(tmp_path, monkeypatch): +async def test_resolve_images_montage_respects_max_size(tmp_path, monkeypatch): monkeypatch.setattr(media_utils, "get_astrbot_temp_path", lambda: str(tmp_path)) - colors = [(i * 12 % 256, 0, 0) for i in range(20)] - gif_bytes = _make_animated_gif_bytes(colors) + from PIL import Image as PILImage + + gif_bytes = _make_animated_gif_bytes( + [(255, 0, 0), (0, 255, 0), (0, 0, 255)], size=(20, 20) + ) images = await media_utils.resolve_image_ref_to_images( _image_data_uri("image/gif", gif_bytes), allowed_mime_types={"image/png"}, - animated_strategy=media_utils.ANIMATED_STRATEGY_MULTI_FRAME, - animated_max_frames=100, + montage_max_size=30, ) - assert len(images) == media_utils.ANIMATED_MAX_FRAMES_LIMIT + with PILImage.open(BytesIO(images[0].to_bytes())) as montage: + # Native 60x60 canvas scaled down to the 30px cap. + assert montage.size == (30, 30) @pytest.mark.asyncio -async def test_resolve_images_ignores_stale_staging_dir(tmp_path, monkeypatch): +async def test_resolve_images_montage_cache_hit_skips_reencode(tmp_path, monkeypatch): monkeypatch.setattr(media_utils, "get_astrbot_temp_path", lambda: str(tmp_path)) gif_bytes = _make_animated_gif_bytes([(255, 0, 0), (0, 255, 0), (0, 0, 255)]) + image_ref = _image_data_uri("image/gif", gif_bytes) - # Simulate a crash mid-extraction: a staging dir with a partial frame set. - cache_dir = tmp_path / media_utils.CONVERT_CACHE_DIR_NAME - cache_dir.mkdir(parents=True) - staging = cache_dir / ".stale_staging" - staging.mkdir() - (staging / "f0.png").write_bytes(b"partial") + save_calls = 0 + real_save = media_utils._save_current_image_frame + + def counting_save(*args, **kwargs): + nonlocal save_calls + save_calls += 1 + return real_save(*args, **kwargs) + + monkeypatch.setattr(media_utils, "_save_current_image_frame", counting_save) + + first = await media_utils.resolve_image_ref_to_images( + image_ref, allowed_mime_types={"image/png"} + ) + assert save_calls == 1 + + second = await media_utils.resolve_image_ref_to_images( + image_ref, allowed_mime_types={"image/png"} + ) + assert save_calls == 1 # cache hit: no re-encode + assert second[0].base64_data == first[0].base64_data + + +@pytest.mark.asyncio +async def test_resolve_images_single_frame_gif_treated_as_still(tmp_path, monkeypatch): + monkeypatch.setattr(media_utils, "get_astrbot_temp_path", lambda: str(tmp_path)) + from PIL import Image as PILImage + + gif_bytes = _make_animated_gif_bytes([(255, 0, 0)]) images = await media_utils.resolve_image_ref_to_images( _image_data_uri("image/gif", gif_bytes), - allowed_mime_types={"image/png"}, - animated_strategy=media_utils.ANIMATED_STRATEGY_MULTI_FRAME, - animated_max_frames=3, + allowed_mime_types={"image/jpeg", "image/png"}, ) - assert len(images) == 3 - # The published frame set lives in a dedicated dir, not the staging dir. - assert all(image_data.to_bytes() != b"partial" for image_data in images) - assert list(cache_dir.glob("*_frames")) + # A one-frame GIF is a still image: plain conversion, no montage canvas. + assert len(images) == 1 + assert images[0].mime_type == "image/jpeg" + with PILImage.open(BytesIO(images[0].to_bytes())) as still: + assert still.size == (8, 8) @pytest.mark.asyncio diff --git a/tests/test_openai_source.py b/tests/test_openai_source.py index 1c7a677e3a..74eeb209ac 100644 --- a/tests/test_openai_source.py +++ b/tests/test_openai_source.py @@ -722,8 +722,7 @@ async def fake_resolve_image_ref_to_images( image_ref: str, *, allowed_mime_types=None, - animated_strategy: str = "first_frame", - animated_max_frames: int = 4, + montage_max_size: int = 1280, strict: bool = False, ) -> list[ResolvedMediaData]: assert image_ref == "https://example.com/quoted.png" diff --git a/tests/unit/test_provider_image_formats.py b/tests/unit/test_provider_image_formats.py index 23122eb998..189f340c44 100644 --- a/tests/unit/test_provider_image_formats.py +++ b/tests/unit/test_provider_image_formats.py @@ -33,8 +33,11 @@ class _UndeclaredProvider(_DummyProvider): supported_image_formats = None -def _make_dummy(provider_config: dict | None = None) -> _DummyProvider: - return _DummyProvider(provider_config or {}, {}) +def _make_dummy( + provider_config: dict | None = None, + provider_settings: dict | None = None, +) -> _DummyProvider: + return _DummyProvider(provider_config or {}, provider_settings or {}) def test_config_override_wins_over_class_default(): @@ -93,22 +96,30 @@ def test_aggregators_do_not_inherit_openai_format_set(): assert ProviderXAI.supported_image_formats == frozenset({"image/jpeg", "image/png"}) -def test_animated_strategy_defaults(): +def test_animated_montage_max_size_defaults_to_compress_default(): provider = _make_dummy() - assert provider.get_animated_image_strategy() == ("first_frame", 4) + assert provider.get_animated_montage_max_size() == ( + media_utils.IMAGE_COMPRESS_DEFAULT_MAX_SIZE + ) -def test_animated_strategy_from_config_and_clamped(): +def test_animated_montage_max_size_reuses_compress_options(): provider = _make_dummy( - {"animated_image_strategy": "multi_frame", "animated_image_max_frames": 100} + provider_settings={"image_compress_options": {"max_size": 640}}, ) - assert provider.get_animated_image_strategy() == ("multi_frame", 16) - - provider = _make_dummy({"animated_image_max_frames": 0}) - assert provider.get_animated_image_strategy() == ("first_frame", 1) + assert provider.get_animated_montage_max_size() == 640 - provider = _make_dummy({"animated_image_strategy": "bogus"}) - assert provider.get_animated_image_strategy() == ("first_frame", 4) + # Invalid values fall back instead of breaking montage generation. + provider = _make_dummy( + provider_settings={"image_compress_options": {"max_size": "big"}}, + ) + assert provider.get_animated_montage_max_size() == ( + media_utils.IMAGE_COMPRESS_DEFAULT_MAX_SIZE + ) + provider = _make_dummy( + provider_settings={"image_compress_options": {"max_size": 0}}, + ) + assert provider.get_animated_montage_max_size() == 1 def _animated_gif_data_uri(colors: list[tuple[int, int, int]]) -> str: @@ -126,7 +137,7 @@ def _animated_gif_data_uri(colors: list[tuple[int, int, int]]) -> str: @pytest.mark.asyncio -async def test_xai_animated_gif_reduced_to_still(tmp_path, monkeypatch): +async def test_xai_animated_gif_becomes_montage(tmp_path, monkeypatch): monkeypatch.setattr(media_utils, "get_astrbot_temp_path", lambda: str(tmp_path)) provider = ProviderXAI( { @@ -145,14 +156,14 @@ async def test_xai_animated_gif_reduced_to_still(tmp_path, monkeypatch): image_blocks = [block for block in content if block["type"] == "image_url"] assert len(image_blocks) == 1 url = image_blocks[0]["image_url"]["url"] - # xAI only accepts JPEG/PNG; the animated GIF must become a still image. - assert url.startswith(("data:image/jpeg;base64,", "data:image/png;base64,")) + # xAI only accepts JPEG/PNG; the montage is flattened, so JPEG wins. + assert url.startswith("data:image/jpeg;base64,") finally: await provider.terminate() @pytest.mark.asyncio -async def test_openai_multi_frame_strategy_expands_image_blocks(tmp_path, monkeypatch): +async def test_openai_animated_gif_becomes_single_montage_block(tmp_path, monkeypatch): monkeypatch.setattr(media_utils, "get_astrbot_temp_path", lambda: str(tmp_path)) provider = ProviderOpenAIOfficial( { @@ -160,6 +171,7 @@ async def test_openai_multi_frame_strategy_expands_image_blocks(tmp_path, monkey "type": "openai_chat_completion", "model": "gpt-4o-mini", "key": ["test-key"], + # Stale per-provider strategy keys from earlier builds stay inert. "animated_image_strategy": "multi_frame", "animated_image_max_frames": 3, }, @@ -174,9 +186,48 @@ async def test_openai_multi_frame_strategy_expands_image_blocks(tmp_path, monkey image_blocks = [ block for block in message["content"] if block["type"] == "image_url" ] - assert len(image_blocks) == 3 - for block in image_blocks: - assert block["image_url"]["url"].startswith("data:image/") + assert len(image_blocks) == 1 + assert image_blocks[0]["image_url"]["url"].startswith("data:image/") + finally: + await provider.terminate() + + +@pytest.mark.asyncio +async def test_openai_montage_respects_compress_max_size(tmp_path, monkeypatch): + monkeypatch.setattr(media_utils, "get_astrbot_temp_path", lambda: str(tmp_path)) + provider = ProviderOpenAIOfficial( + { + "id": "test-openai", + "type": "openai_chat_completion", + "model": "gpt-4o-mini", + "key": ["test-key"], + # PNG keeps pixels lossless for exact montage assertions. + "image_formats": ["png"], + }, + {"image_compress_options": {"max_size": 24}}, + ) + try: + gif_ref = _animated_gif_data_uri( + [(255, 0, 0), (0, 255, 0), (0, 0, 255), (255, 255, 0)] + ) + message = await provider.assemble_context("look", image_urls=[gif_ref]) + + image_blocks = [ + block for block in message["content"] if block["type"] == "image_url" + ] + assert len(image_blocks) == 1 + url = image_blocks[0]["image_url"]["url"] + assert url.startswith("data:image/png;base64,") + _header, encoded = url.split(",", 1) + with PILImage.open(BytesIO(base64.b64decode(encoded))) as montage: + # 8px frames x 3 columns = 24px, already within the 24px cap. + assert montage.size == (24, 24) + rgb = montage.convert("RGB") + assert rgb.getpixel((4, 4)) == (255, 0, 0) + assert rgb.getpixel((12, 4)) == (0, 255, 0) + assert rgb.getpixel((20, 4)) == (0, 0, 255) + # The 4th frame fills cell 3; no blank cells for a 4-frame GIF. + assert rgb.getpixel((4, 12)) == (255, 255, 0) finally: await provider.terminate() From 2aec78272cb87a2b20278e087c2354d516be7a70 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=B0=95=E6=B0=99?= Date: Thu, 3 Sep 2026 02:09:17 +0800 Subject: [PATCH 8/8] fix(dashboard): drop removed animated image keys from new-model defaults --- dashboard/src/composables/useProviderSources.ts | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/dashboard/src/composables/useProviderSources.ts b/dashboard/src/composables/useProviderSources.ts index ca812a5cc8..2a648ebaa7 100644 --- a/dashboard/src/composables/useProviderSources.ts +++ b/dashboard/src/composables/useProviderSources.ts @@ -627,9 +627,7 @@ export function useProviderSources(options: UseProviderSourcesOptions) { modalities, custom_extra_body: {}, max_context_tokens: max_context_tokens, - image_formats: [...(providerBrandImageFormats.value[selectedProviderSource.value.provider] ?? providerTypeImageFormats.value[selectedProviderSource.value.type] ?? ['jpeg', 'png'])], - animated_image_strategy: 'first_frame', - animated_image_max_frames: 4 + image_formats: [...(providerBrandImageFormats.value[selectedProviderSource.value.provider] ?? providerTypeImageFormats.value[selectedProviderSource.value.type] ?? ['jpeg', 'png'])] } }