diff --git a/astrbot/core/knowledge_base/retrieval/manager.py b/astrbot/core/knowledge_base/retrieval/manager.py index 5b414b171e..a49f626ef5 100644 --- a/astrbot/core/knowledge_base/retrieval/manager.py +++ b/astrbot/core/knowledge_base/retrieval/manager.py @@ -4,6 +4,7 @@ """ import time +import unicodedata from dataclasses import dataclass from typing import TYPE_CHECKING @@ -34,6 +35,28 @@ class RetrievalResult: metadata: dict +def _strip_invisible_chars(text: str) -> str: + """Remove control and format characters from a retrieval query. + + Control (Cc, except tab/newline/CR) and format (Cf) characters are + invisible payloads that some embedding providers reject with HTTP 400 + (e.g. SiliconFlow code 20015), typically originating from sticker / + image-caption pipelines. + + Args: + text: Raw query text. + + Returns: + The query with invisible characters removed. + + """ + return "".join( + ch + for ch in text + if ch not in "\t\n\r" and unicodedata.category(ch) not in ("Cc", "Cf") + ) + + class RetrievalManager: """检索管理器 @@ -87,6 +110,16 @@ async def retrieve( List[RetrievalResult]: 检索结果列表 """ + # Remove invisible characters and skip retrieval when nothing + # remains, so invalid queries never reach the embedding provider. + query = _strip_invisible_chars(query).strip() + if not query: + logger.debug( + "Knowledge base retrieval skipped: query is empty after " + "removing invisible characters.", + ) + return [] + if not kb_ids: return [] @@ -229,7 +262,8 @@ async def _dense_retrieve( all_results.extend(vec_results) except Exception as e: logger.error( - f"知识库 {kb_id} 稠密检索失败: {type(e).__name__}: {e}", + f"知识库 {kb_id} 稠密检索失败: {type(e).__name__}: {e} " + f"(query={query!r}, query_length={len(query)})", exc_info=True, ) # skip the faulty KB and continue diff --git a/tests/test_retrieval_query_sanitize.py b/tests/test_retrieval_query_sanitize.py new file mode 100644 index 0000000000..aeebac56c2 --- /dev/null +++ b/tests/test_retrieval_query_sanitize.py @@ -0,0 +1,63 @@ +from __future__ import annotations + +from unittest.mock import AsyncMock, MagicMock + +import pytest + +# Import astrbot.api first: the kb retrieval import chain +# (manager -> kb_helper -> provider.manager -> persona_mgr -> astrbot.api) +# only resolves when astrbot.api is fully initialized beforehand. +import astrbot.api # noqa: F401 +from astrbot.core.knowledge_base.retrieval.manager import RetrievalManager + + +def _make_manager() -> RetrievalManager: + manager = RetrievalManager( + sparse_retriever=MagicMock(), + rank_fusion=MagicMock(), + kb_db=MagicMock(), + ) + manager.rank_fusion.fuse = AsyncMock(return_value=[]) + manager.kb_db.get_documents_with_metadata_batch = AsyncMock(return_value={}) + return manager + + +@pytest.mark.asyncio +async def test_invisible_only_query_skips_retrieval() -> None: + manager = _make_manager() + manager._dense_retrieve = AsyncMock(return_value=[]) + manager.sparse_retriever.retrieve = AsyncMock(return_value=[]) + + results = await manager.retrieve( + query="\u200b\ufeff\x00", + kb_ids=["kb-1"], + kb_id_helper_map={}, + ) + + assert results == [] + manager._dense_retrieve.assert_not_awaited() + manager.sparse_retriever.retrieve.assert_not_awaited() + + +@pytest.mark.asyncio +async def test_invisible_chars_are_stripped_before_retrieval() -> None: + manager = _make_manager() + manager._dense_retrieve = AsyncMock(return_value=[]) + manager.sparse_retriever.retrieve = AsyncMock(return_value=[]) + manager.rank_fusion.fuse = AsyncMock(return_value=[]) + + kb_helper = MagicMock() + kb_helper.kb.top_k_dense = 50 + kb_helper.kb.top_k_sparse = 50 + kb_helper.kb.top_m_final = 5 + kb_helper.vec_db.rerank_provider = None + + results = await manager.retrieve( + query="\u200bOni\u200f 恶鬼 ", + kb_ids=["kb-1"], + kb_id_helper_map={"kb-1": kb_helper}, + ) + + assert results == [] + sent_query = manager._dense_retrieve.await_args.kwargs["query"] + assert sent_query == "Oni 恶鬼"