feat:Adapt outgoing images to the formats supported by the provider, with support for animation strategies and local conversion caching.

astrbot/core/config/default.py

@@ -2218,6 +2218,36 @@
                         "labels": ["文本", "图像", "音频", "工具使用"],
                         "render_type": "checkbox",
                         "hint": "模型支持的模态及能力。",
+                        "default": ["text", "image", "audio", "tool_use"],
+                    },
+                    "image_formats": {
+                        "description": "图片格式支持",
+                        "type": "list",
+                        "items": {"type": "string"},
+                        "options": [
+                            "jpeg",
+                            "png",
+                            "webp",
+                            "gif",
+                            "bmp",
+                            "heic",
+                            "heif",
+                            "*",
+                        ],
+                        "labels": [
+                            "JPEG",
+                            "PNG",
+                            "WebP",
+                            "GIF",
+                            "BMP",
+                            "HEIC",
+                            "HEIF",
+                            "不限制",
+                        ],
+                        "render_type": "checkbox",
+                        "hint": "发送给该提供商前允许的图片格式，不兼容的格式会自动转换。勾选「不限制」时其他选项不生效。",
+                        "default": [],
+                        "condition": {"modalities": "image"},
                     },
                     "custom_headers": {
                         "description": "自定义请求头",

astrbot/core/provider/entities.py

@@ -23,7 +23,7 @@
 from astrbot.core.agent.tool import ToolSet
 from astrbot.core.db.po import Conversation
 from astrbot.core.message.message_event_result import MessageChain
-from astrbot.core.utils.media_utils import MediaResolver
+from astrbot.core.utils.media_utils import MediaResolver, resolve_image_ref_to_images
 
 
 class ProviderType(enum.Enum):
@@ -208,19 +208,17 @@ async def assemble_context(self) -> dict:
         # 3. 图片内容
         if self.image_urls:
             for image_url in self.image_urls:
-                image_data = await MediaResolver(
-                    image_url,
-                    media_type="image",
-                ).to_base64_data()
-                if not image_data:
+                image_datas = await resolve_image_ref_to_images(image_url)
+                if not image_datas:
                     logger.warning("图片预处理结果为空，将忽略。")
                     continue
-                content_blocks.append(
-                    {
-                        "type": "image_url",
-                        "image_url": {"url": image_data.to_data_url()},
-                    },
-                )
+                for image_data in image_datas:
+                    content_blocks.append(
+                        {
+                            "type": "image_url",
+                            "image_url": {"url": image_data.to_data_url()},
+                        },
+                    )
 
         # 4. 音频内容
         if self.audio_urls:

astrbot/core/provider/provider.py

@@ -2,8 +2,9 @@
 import asyncio
 import os
 from collections.abc import AsyncGenerator
-from typing import Literal, TypeAlias, Union
+from typing import ClassVar, Literal, TypeAlias, Union
 
+from astrbot import logger
 from astrbot.core.agent.message import ContentPart, Message, is_checkpoint_message
 from astrbot.core.agent.tool import ToolSet
 from astrbot.core.provider.entities import (
@@ -14,6 +15,14 @@
 )
 from astrbot.core.provider.register import provider_cls_map
 from astrbot.core.utils.astrbot_path import get_astrbot_path
+from astrbot.core.utils.media_utils import (
+    IMAGE_COMPRESS_DEFAULT_MAX_SIZE,
+    IMAGE_SHORT_MIME_TYPES,
+    VENDOR_IMAGE_FORMATS,
+)
+
+DEFAULT_FALLBACK_IMAGE_FORMATS = frozenset({"image/jpeg", "image/png"})
+"""Conservative image formats assumed for providers without a declared set."""
 
 Providers: TypeAlias = Union[
     "Provider",
@@ -66,6 +75,14 @@ async def test(self) -> None:
 class Provider(AbstractProvider):
     """Chat Provider"""
 
+    supported_image_formats: ClassVar[frozenset[str] | None] = None
+    """Image MIME types the provider officially accepts.
+
+    ``None`` means undeclared and falls back to DEFAULT_FALLBACK_IMAGE_FORMATS.
+    Aggregator subclasses (e.g. OpenRouter) must set this back to ``None``
+    explicitly so they do not inherit an official vendor's format set.
+    """
+
     def __init__(
         self,
         provider_config: dict,
@@ -74,6 +91,63 @@ def __init__(
         super().__init__(provider_config)
         self.provider_settings = provider_settings
 
+    def resolve_allowed_image_formats(self) -> frozenset[str] | None:
+        """Resolve the image MIME types allowed for this provider instance.
+
+        Priority: ``provider_config["image_formats"]`` (per-instance override,
+        short names like ``jpeg`` or MIME types, ``*`` disables restriction) >
+        the vendor-level set keyed by ``provider_config["provider"]`` (brand of
+        the provider source, for branded sources running on generic adapters) >
+        the class-level ``supported_image_formats`` > the conservative
+        DEFAULT_FALLBACK_IMAGE_FORMATS (jpeg/png).
+        Returns:
+            The allowed MIME types, or ``None`` when unrestricted.
+        """
+        configured = self.provider_config.get("image_formats")
+        if configured:
+            normalized = {
+                str(value).strip().lower() for value in configured if str(value).strip()
+            }
+            if "*" in normalized:
+                return None
+            mapped = {
+                IMAGE_SHORT_MIME_TYPES.get(value, value)
+                for value in normalized
+                if value.startswith("image/") or value in IMAGE_SHORT_MIME_TYPES
+            }
+            if mapped:
+                return frozenset(mapped)
+            logger.warning(
+                "Provider %s: image_formats %s contains no valid entries; "
+                "falling back to the default format set.",
+                self.provider_config.get("id"),
+                sorted(normalized),
+            )
+        brand = self.provider_config.get("provider")
+        if isinstance(brand, str):
+            vendor_formats = VENDOR_IMAGE_FORMATS.get(brand.strip().lower())
+            if vendor_formats is not None:
+                return vendor_formats
+        if self.supported_image_formats is not None:
+            return self.supported_image_formats
+        return DEFAULT_FALLBACK_IMAGE_FORMATS
+
+    def get_animated_montage_max_size(self) -> int:
+        """Longest-edge cap for the animated-image montage, in pixels.
+
+        Reuses ``provider_settings.image_compress_options.max_size`` so still
+        images and montages share one size control.
+
+        Returns:
+            The configured cap, or the compress default when unset or invalid.
+        """
+        options = self.provider_settings.get("image_compress_options")
+        raw = options.get("max_size") if isinstance(options, dict) else None
+        try:
+            return max(int(raw), 1)
+        except (TypeError, ValueError):
+            return IMAGE_COMPRESS_DEFAULT_MAX_SIZE
+
     @abc.abstractmethod
     def get_current_key(self) -> str:
         raise NotImplementedError

astrbot/core/provider/sources/anthropic_source.py

@@ -17,8 +17,10 @@
 from astrbot.core.provider.entities import LLMResponse, TokenUsage
 from astrbot.core.provider.func_tool_manager import ToolSet
 from astrbot.core.utils.media_utils import (
+    VENDOR_IMAGE_FORMATS,
     describe_media_ref,
-    resolve_media_ref_to_base64_data,
+    detect_image_mime_type,
+    resolve_image_ref_to_images,
 )
 from astrbot.core.utils.network_utils import (
     create_proxy_client,
@@ -37,6 +39,9 @@
 class ProviderAnthropic(Provider):
     _PROMPT_CACHE_CONTROL = {"type": "ephemeral"}
 
+    supported_image_formats = VENDOR_IMAGE_FORMATS["anthropic"]
+    """Formats accepted by the official Anthropic vision API."""
+
     @staticmethod
     def _ensure_usable_response(
         llm_response: LLMResponse,
@@ -273,19 +278,32 @@ def _prepare_payload(self, messages: list[dict]):
                                     _, base64_data = url.split(",", 1)
                                     # Detect actual image format from binary data
                                     image_bytes = base64.b64decode(base64_data)
-                                    media_type = self._detect_image_mime_type(
-                                        image_bytes
+                                    media_type = detect_image_mime_type(
+                                        image_bytes,
+                                        default_mime_type=None,
                                     )
-                                    converted_content.append(
-                                        {
-                                            "type": "image",
-                                            "source": {
-                                                "type": "base64",
-                                                "media_type": media_type,
-                                                "data": base64_data,
-                                            },
-                                        }
+                                    allowed_formats = (
+                                        self.resolve_allowed_image_formats()
                                     )
+                                    if media_type and (
+                                        allowed_formats is None
+                                        or media_type in allowed_formats
+                                    ):
+                                        converted_content.append(
+                                            {
+                                                "type": "image",
+                                                "source": {
+                                                    "type": "base64",
+                                                    "media_type": media_type,
+                                                    "data": base64_data,
+                                                },
+                                            }
+                                        )
+                                    else:
+                                        logger.warning(
+                                            "Skipping context image with unsupported or undetectable format: %s...",
+                                            url[:50],
+                                        )
                                 except ValueError:
                                     logger.warning(
                                         f"Failed to parse image data URI: {url[:50]}..."
@@ -899,17 +917,15 @@ async def text_chat_stream(
         ):
             yield llm_response
 
-    def _detect_image_mime_type(self, data: bytes) -> str:
-        """根据图片二进制数据的 magic bytes 检测 MIME 类型"""
-        if data[:8] == b"\x89PNG\r\n\x1a\n":
-            return "image/png"
-        if data[:2] == b"\xff\xd8":
-            return "image/jpeg"
-        if data[:6] in (b"GIF87a", b"GIF89a"):
-            return "image/gif"
-        if data[:4] == b"RIFF" and data[8:12] == b"WEBP":
-            return "image/webp"
-        return "image/jpeg"
+    async def _image_ref_to_images(self, image_ref: str, *, strict: bool = False):
+        """Resolve an image ref with this provider's format adaptation applied."""
+        montage_max_size = self.get_animated_montage_max_size()
+        return await resolve_image_ref_to_images(
+            image_ref,
+            allowed_mime_types=self.resolve_allowed_image_formats(),
+            montage_max_size=montage_max_size,
+            strict=strict,
+        )
 
     async def assemble_context(
         self,
@@ -920,23 +936,23 @@ async def assemble_context(
     ):
         """组装上下文，支持文本和图片"""
 
-        async def resolve_image_url(image_url: str) -> dict | None:
-            image_data = await resolve_media_ref_to_base64_data(
-                image_url,
-                media_type="image",
-            )
-            if not image_data:
+        async def resolve_image_url(image_url: str) -> list[dict]:
+            image_datas = await self._image_ref_to_images(image_url)
+            if not image_datas:
                 logger.warning("图片预处理结果为空，将忽略。")
-                return None
-
-            return {
-                "type": "image",
-                "source": {
-                    "type": "base64",
-                    "media_type": image_data.mime_type,
-                    "data": image_data.base64_data,
-                },
-            }
+                return []
+
+            return [
+                {
+                    "type": "image",
+                    "source": {
+                        "type": "base64",
+                        "media_type": image_data.mime_type,
+                        "data": image_data.base64_data,
+                    },
+                }
+                for image_data in image_datas
+            ]
 
         content = []
 
@@ -958,9 +974,7 @@ async def resolve_image_url(image_url: str) -> dict | None:
                 if isinstance(block, TextPart):
                     content.append({"type": "text", "text": block.text})
                 elif isinstance(block, ImageURLPart):
-                    image_dict = await resolve_image_url(block.image_url.url)
-                    if image_dict:
-                        content.append(image_dict)
+                    content.extend(await resolve_image_url(block.image_url.url))
                 elif isinstance(block, AudioURLPart):
                     content.append({"type": "text", "text": "[Audio]"})
                 else:
@@ -969,9 +983,7 @@ async def resolve_image_url(image_url: str) -> dict | None:
         # 3. 图片内容
         if image_urls:
             for image_url in image_urls:
-                image_dict = await resolve_image_url(image_url)
-                if image_dict:
-                    content.append(image_dict)
+                content.extend(await resolve_image_url(image_url))
         if audio_urls:
             for _audio_path in audio_urls:
                 content.append({"type": "text", "text": "[Audio]"})
@@ -992,15 +1004,12 @@ async def resolve_image_url(image_url: str) -> dict | None:
 
     async def encode_image_bs64(self, image_url: str) -> tuple[str, str]:
         """将图片转换为 base64，同时检测实际 MIME 类型"""
-        image_data = await resolve_media_ref_to_base64_data(
-            image_url,
-            media_type="image",
-            strict=True,
-        )
-        if image_data is None:
+        image_datas = await self._image_ref_to_images(image_url, strict=True)
+        if not image_datas:
             raise RuntimeError(
                 f"Failed to encode image data: {describe_media_ref(image_url)}"
             )
+        image_data = image_datas[0]
         return image_data.to_data_url(), image_data.mime_type
 
     def get_current_key(self) -> str: