From 410de791e12f743fc40c9c67005798c4d634dcfe Mon Sep 17 00:00:00 2001 From: lxfight <1686540385@qq.com> Date: Wed, 2 Sep 2026 14:26:46 +0800 Subject: [PATCH 1/2] fix: sanitize KB retrieval query before dense embedding Some sticker/image-caption pipelines leave invisible control/format characters in the query, which certain embedding providers reject with HTTP 400 (e.g. SiliconFlow code 20015), surfacing as recurring dense retrieval failures. Strip control (Cc) and format (Cf) characters at the RetrievalManager entry and skip retrieval entirely when nothing remains. Also log repr(query) and its length when dense retrieval fails to make such cases diagnosable. --- .../core/knowledge_base/retrieval/manager.py | 24 ++++++- tests/test_retrieval_query_sanitize.py | 63 +++++++++++++++++++ 2 files changed, 86 insertions(+), 1 deletion(-) create mode 100644 tests/test_retrieval_query_sanitize.py diff --git a/astrbot/core/knowledge_base/retrieval/manager.py b/astrbot/core/knowledge_base/retrieval/manager.py index 5b414b171e..db879c05d7 100644 --- a/astrbot/core/knowledge_base/retrieval/manager.py +++ b/astrbot/core/knowledge_base/retrieval/manager.py @@ -3,6 +3,7 @@ 协调稠密检索、稀疏检索和 Rerank,提供统一的检索接口 """ +import re import time from dataclasses import dataclass from typing import TYPE_CHECKING @@ -34,6 +35,16 @@ class RetrievalResult: metadata: dict +# Control (Cc, except tab/newline/CR) and format (Cf) characters are +# invisible payloads that some embedding providers reject with HTTP 400 +# (e.g. SiliconFlow code 20015), typically originating from sticker / +# image-caption pipelines. They are stripped before retrieval. +_INVISIBLE_CHARS_PATTERN = re.compile( + "[\x00-\x08\x0b\x0c\x0e-\x1f\x7f-\x9f" + "\u00ad\u061c\u180e\u200b-\u200f\u202a-\u202e\u2060-\u206f\ufeff\ufff9-\ufffb]" +) + + class RetrievalManager: """检索管理器 @@ -87,6 +98,16 @@ async def retrieve( List[RetrievalResult]: 检索结果列表 """ + # Remove invisible characters and skip retrieval when nothing + # remains, so invalid queries never reach the embedding provider. + query = _INVISIBLE_CHARS_PATTERN.sub("", query).strip() + if not query: + logger.debug( + "Knowledge base retrieval skipped: query is empty after " + "removing invisible characters.", + ) + return [] + if not kb_ids: return [] @@ -229,7 +250,8 @@ async def _dense_retrieve( all_results.extend(vec_results) except Exception as e: logger.error( - f"知识库 {kb_id} 稠密检索失败: {type(e).__name__}: {e}", + f"知识库 {kb_id} 稠密检索失败: {type(e).__name__}: {e} " + f"(query={query!r}, query_length={len(query)})", exc_info=True, ) # skip the faulty KB and continue diff --git a/tests/test_retrieval_query_sanitize.py b/tests/test_retrieval_query_sanitize.py new file mode 100644 index 0000000000..aeebac56c2 --- /dev/null +++ b/tests/test_retrieval_query_sanitize.py @@ -0,0 +1,63 @@ +from __future__ import annotations + +from unittest.mock import AsyncMock, MagicMock + +import pytest + +# Import astrbot.api first: the kb retrieval import chain +# (manager -> kb_helper -> provider.manager -> persona_mgr -> astrbot.api) +# only resolves when astrbot.api is fully initialized beforehand. +import astrbot.api # noqa: F401 +from astrbot.core.knowledge_base.retrieval.manager import RetrievalManager + + +def _make_manager() -> RetrievalManager: + manager = RetrievalManager( + sparse_retriever=MagicMock(), + rank_fusion=MagicMock(), + kb_db=MagicMock(), + ) + manager.rank_fusion.fuse = AsyncMock(return_value=[]) + manager.kb_db.get_documents_with_metadata_batch = AsyncMock(return_value={}) + return manager + + +@pytest.mark.asyncio +async def test_invisible_only_query_skips_retrieval() -> None: + manager = _make_manager() + manager._dense_retrieve = AsyncMock(return_value=[]) + manager.sparse_retriever.retrieve = AsyncMock(return_value=[]) + + results = await manager.retrieve( + query="\u200b\ufeff\x00", + kb_ids=["kb-1"], + kb_id_helper_map={}, + ) + + assert results == [] + manager._dense_retrieve.assert_not_awaited() + manager.sparse_retriever.retrieve.assert_not_awaited() + + +@pytest.mark.asyncio +async def test_invisible_chars_are_stripped_before_retrieval() -> None: + manager = _make_manager() + manager._dense_retrieve = AsyncMock(return_value=[]) + manager.sparse_retriever.retrieve = AsyncMock(return_value=[]) + manager.rank_fusion.fuse = AsyncMock(return_value=[]) + + kb_helper = MagicMock() + kb_helper.kb.top_k_dense = 50 + kb_helper.kb.top_k_sparse = 50 + kb_helper.kb.top_m_final = 5 + kb_helper.vec_db.rerank_provider = None + + results = await manager.retrieve( + query="\u200bOni\u200f 恶鬼 ", + kb_ids=["kb-1"], + kb_id_helper_map={"kb-1": kb_helper}, + ) + + assert results == [] + sent_query = manager._dense_retrieve.await_args.kwargs["query"] + assert sent_query == "Oni 恶鬼" From cb55b17d0c3b2471d0793daefe2140a5e7214b8c Mon Sep 17 00:00:00 2001 From: lxfight <1686540385@qq.com> Date: Wed, 2 Sep 2026 14:32:55 +0800 Subject: [PATCH 2/2] refactor: replace invisible-char regex with unicodedata categories CodeQL flagged the explicit control/format character ranges as an overly permissive regular expression range. Classify characters via unicodedata.category() instead, which is clearer and avoids the suspicious-range pattern entirely. --- .../core/knowledge_base/retrieval/manager.py | 32 +++++++++++++------ 1 file changed, 22 insertions(+), 10 deletions(-) diff --git a/astrbot/core/knowledge_base/retrieval/manager.py b/astrbot/core/knowledge_base/retrieval/manager.py index db879c05d7..a49f626ef5 100644 --- a/astrbot/core/knowledge_base/retrieval/manager.py +++ b/astrbot/core/knowledge_base/retrieval/manager.py @@ -3,8 +3,8 @@ 协调稠密检索、稀疏检索和 Rerank,提供统一的检索接口 """ -import re import time +import unicodedata from dataclasses import dataclass from typing import TYPE_CHECKING @@ -35,14 +35,26 @@ class RetrievalResult: metadata: dict -# Control (Cc, except tab/newline/CR) and format (Cf) characters are -# invisible payloads that some embedding providers reject with HTTP 400 -# (e.g. SiliconFlow code 20015), typically originating from sticker / -# image-caption pipelines. They are stripped before retrieval. -_INVISIBLE_CHARS_PATTERN = re.compile( - "[\x00-\x08\x0b\x0c\x0e-\x1f\x7f-\x9f" - "\u00ad\u061c\u180e\u200b-\u200f\u202a-\u202e\u2060-\u206f\ufeff\ufff9-\ufffb]" -) +def _strip_invisible_chars(text: str) -> str: + """Remove control and format characters from a retrieval query. + + Control (Cc, except tab/newline/CR) and format (Cf) characters are + invisible payloads that some embedding providers reject with HTTP 400 + (e.g. SiliconFlow code 20015), typically originating from sticker / + image-caption pipelines. + + Args: + text: Raw query text. + + Returns: + The query with invisible characters removed. + + """ + return "".join( + ch + for ch in text + if ch not in "\t\n\r" and unicodedata.category(ch) not in ("Cc", "Cf") + ) class RetrievalManager: @@ -100,7 +112,7 @@ async def retrieve( """ # Remove invisible characters and skip retrieval when nothing # remains, so invalid queries never reach the embedding provider. - query = _INVISIBLE_CHARS_PATTERN.sub("", query).strip() + query = _strip_invisible_chars(query).strip() if not query: logger.debug( "Knowledge base retrieval skipped: query is empty after "