From a6f8ebc607d9dc69c7b9f6ab699734d4c069a83d Mon Sep 17 00:00:00 2001 From: zuizhuai Date: Tue, 10 Nov 2026 18:22:50 +0800 Subject: [PATCH] fix(system2): reuse shared language detection for tech-heavy Chinese memories _detect_language used a standalone full-text zh_ratio >= 0.7 scan that dilutes the Chinese denominator with paths, timestamps, commands and English terms. For developer-facing Chinese memories the ratio drops well below 0.7 (e.g. zh_chars=2110 / total=6644 -> 0.318), so the materials were classified "en" and the L6 schemas were emitted in English, making them nearly unretrievable by Chinese queries (see #243). Reuse utils.lang_detect.detect_language, which already handles code-heavy Chinese via a leading-instruction window plus a 0.3 CJK threshold and langdetect fallback, and is already covered by tests. Normalize its result back to the zh/en contract the callers expect. Add regression tests for tech-heavy Chinese, plain English, and empty materials. --- hy_memory/pipelines/system2_agent.py | 29 +++++++++--------- tests/test_lang_detect.py | 45 ++++++++++++++++++++++++++++ 2 files changed, 60 insertions(+), 14 deletions(-) diff --git a/hy_memory/pipelines/system2_agent.py b/hy_memory/pipelines/system2_agent.py index 11eaad93..7b4c785a 100644 --- a/hy_memory/pipelines/system2_agent.py +++ b/hy_memory/pipelines/system2_agent.py @@ -31,6 +31,7 @@ from ..core.embed_service import EmbedService from ..data.vector_store_base import VectorStoreBase from ..data.graph_store_base import GraphStoreBase +from ..utils.lang_detect import detect_language as _detect_language_shared from .system2_tools import System2ToolExecutor from .specific_supplement import ( @@ -2310,11 +2311,12 @@ def _parse_operations_json(text: str) -> Optional[List[Dict[str, Any]]]: def _detect_language(materials: Dict[str, Any]) -> str: """ 从 materials 的 fact content 检测语言。 - - 扫描所有聚类中的 fact content,计算中文字符占比。 - 中文比例 >= 70% → "zh",否则 → "en"。 + + 复用 ``utils.lang_detect.detect_language`` 的分层策略(leading 指令窗口 + + 0.3 CJK 阈值 + langdetect 兜底),而不是旧的全文 ``zh_ratio >= 0.7`` + 扫描。旧算法对技术/开发类中文记忆结构性误判:路径、时间戳、命令、英文 + 术语等 ASCII 内容会稀释分母,把真实的中文事实判成英文(见 #243)。 """ - import re all_text = "" for cluster in materials.get("clusters", []): for f in cluster.get("facts", []): @@ -2325,18 +2327,17 @@ def _detect_language(materials: Dict[str, Any]) -> str: if not all_text.strip(): return "en" - # 统计中文字符(CJK Unified Ideographs) - zh_chars = len(re.findall(r'[\u4e00-\u9fff]', all_text)) - # 统计所有非空白字符 - total_chars = len(re.findall(r'\S', all_text)) - - if total_chars == 0: + detected = _detect_language_shared(all_text) + if not detected: return "en" + # 归一化到调用方期望的 zh/en 两值语义;其余语言码原样透传给 + # _s2_language_code 处理。 + if detected == "zh": + return "zh" + if detected.startswith("zh"): + return "zh" + return "en" - zh_ratio = zh_chars / total_chars - lang = "zh" if zh_ratio >= 0.7 else "en" - logger.debug(f"[S2-agent] language detection: zh_chars={zh_chars} total={total_chars} ratio={zh_ratio:.2f} → {lang}") - return lang def _build_materials_message(materials: Dict[str, Any], lang: str = "zh") -> str: diff --git a/tests/test_lang_detect.py b/tests/test_lang_detect.py index 727df10a..a1561512 100644 --- a/tests/test_lang_detect.py +++ b/tests/test_lang_detect.py @@ -101,3 +101,48 @@ def test_reconcile_keeps_english_prompt_when_chinese_term_appears_later(): current_date="2026-10-12 16:00", ) assert prompt.startswith("You are a memory management system.") + + +def test_system2_detect_language_handles_tech_heavy_chinese(): + """System2's _detect_language must not misclassify tech-heavy Chinese as en. + + Regression for #243: the old full-text ``zh_ratio >= 0.7`` scan diluted the + Chinese denominator with paths, timestamps, commands and English terms, so + developer-facing Chinese memories were classified "en" and their L6 schemas + emitted in English. + """ + from hy_memory.pipelines.system2_agent import _detect_language + + materials = { + "clusters": [ + { + "facts": [ + {"content": "修复 search_client.py 的 timeout 异常,使用 asyncio.wait_for"}, + {"content": "在 C:/Users/tara/project/src/main.py 中实现记忆功能"}, + {"content": "执行 pytest tests/test_search.py 时出现 TypeError"}, + {"content": "时间戳 2026-11-10 18:00:00 生成的日志文件 log.txt"}, + {"content": "用户偏好使用 git commit -m 提交代码"}, + {"content": "需要调用 API https://example.com/v1/memories 保存记忆"}, + ] + } + ], + "unprocessed_facts": [], + } + assert _detect_language(materials) == "zh" + + +def test_system2_detect_language_english_and_empty(): + from hy_memory.pipelines.system2_agent import _detect_language + + en_materials = { + "clusters": [ + { + "facts": [ + {"content": "The user likes science fiction movies and reading books about space exploration."} + ] + } + ], + "unprocessed_facts": [], + } + assert _detect_language(en_materials) == "en" + assert _detect_language({}) == "en"