From 720edb8f114621bf36ecb1f41783961454e69c1c Mon Sep 17 00:00:00 2001 From: Wang Xinming <1353119036@qq.com> Date: Mon, 2 Nov 2026 12:29:36 +0800 Subject: [PATCH] fix(system2): detect technical Chinese facts before language voting --- CHANGELOG.md | 4 ++ hy_memory/pipelines/system2_agent.py | 53 ++++++++++++------------ tests/test_prompt_language_selection.py | 8 +++- tests/test_system2_language_detection.py | 41 ++++++++++++++++++ 4 files changed, 77 insertions(+), 29 deletions(-) create mode 100644 tests/test_system2_language_detection.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 5aadfcad..280c5165 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,10 @@ Hy Memory and MemHub are versioned and released independently. ## Unreleased +- System 2 automatic prompt selection now detects each fact with the SDK language + detector and uses a majority vote, so code-heavy Chinese facts are not diluted + by long ASCII payloads. Explicit language settings still take precedence. + ## MemHub 1.0.6 — 2026-11-02 Build 37, on `hy-memory==2.0.11`. Linux x64/arm64 Runtime and macOS arm64 diff --git a/hy_memory/pipelines/system2_agent.py b/hy_memory/pipelines/system2_agent.py index 11eaad93..098330d1 100644 --- a/hy_memory/pipelines/system2_agent.py +++ b/hy_memory/pipelines/system2_agent.py @@ -2308,34 +2308,33 @@ def _parse_operations_json(text: str) -> Optional[List[Dict[str, Any]]]: def _detect_language(materials: Dict[str, Any]) -> str: + """Vote over fact languages using the SDK's code-aware language detector. + + Each nonblank fact gets one vote so a long path or code payload cannot + outweigh the prose in other facts. Empty materials and ties use English. + Explicit language overrides are resolved by the caller before detection. """ - 从 materials 的 fact content 检测语言。 - - 扫描所有聚类中的 fact content,计算中文字符占比。 - 中文比例 >= 70% → "zh",否则 → "en"。 - """ - import re - all_text = "" - for cluster in materials.get("clusters", []): - for f in cluster.get("facts", []): - all_text += f.get("content", "") + " " - for f in materials.get("unprocessed_facts", []): - all_text += f.get("content", "") + " " - - if not all_text.strip(): - return "en" - - # 统计中文字符(CJK Unified Ideographs) - zh_chars = len(re.findall(r'[\u4e00-\u9fff]', all_text)) - # 统计所有非空白字符 - total_chars = len(re.findall(r'\S', all_text)) - - if total_chars == 0: - return "en" - - zh_ratio = zh_chars / total_chars - lang = "zh" if zh_ratio >= 0.7 else "en" - logger.debug(f"[S2-agent] language detection: zh_chars={zh_chars} total={total_chars} ratio={zh_ratio:.2f} → {lang}") + from ..utils.lang_detect import detect_language + + facts = [ + fact + for cluster in materials.get("clusters", []) + for fact in cluster.get("facts", []) + ] + facts.extend(materials.get("unprocessed_facts", [])) + zh_votes = total = 0 + for fact in facts: + content = fact.get("content", "") + if not content.strip(): + continue + total += 1 + zh_votes += detect_language(content) == "zh" + + lang = "zh" if zh_votes > total / 2 else "en" + logger.debug( + "[S2-agent] language detection: zh_facts=%d total=%d → %s", + zh_votes, total, lang, + ) return lang diff --git a/tests/test_prompt_language_selection.py b/tests/test_prompt_language_selection.py index f43bebdf..b4820b73 100644 --- a/tests/test_prompt_language_selection.py +++ b/tests/test_prompt_language_selection.py @@ -404,7 +404,11 @@ def test_system2_prompts_follow_language(code): @pytest.mark.asyncio -async def test_system2_agent_uses_explicit_language(monkeypatch): +@pytest.mark.parametrize("fact_content", [ + "用户每天跑步", + "用户使用 Python 开发,运行 pytest tests/test_search_client.py --maxfail=1", +]) +async def test_system2_agent_uses_explicit_language(monkeypatch, fact_content): from hy_memory.pipelines import system2_agent as s2 seen = {} @@ -419,7 +423,7 @@ async def complete_messages(self, messages, **kwargs): monkeypatch.setattr("hy_memory.agent.llm_provider.LLMProvider", _Provider) materials = {"stats": {}, "clusters": [{"facts": [ - {"layer": "L2", "content": "用户每天跑步", "node_id": "f1"}], "centroid_text": "跑步"}]} + {"layer": "L2", "content": fact_content, "node_id": "f1"}], "centroid_text": "跑步"}]} async def _run(**kwargs): # Only the prompt sent to the model matters here; executing the # (empty) operation list needs a real tool executor. diff --git a/tests/test_system2_language_detection.py b/tests/test_system2_language_detection.py new file mode 100644 index 00000000..19bd9e4b --- /dev/null +++ b/tests/test_system2_language_detection.py @@ -0,0 +1,41 @@ +"""System 2 should route technical facts by prose language, not code length.""" + +import pytest + +from hy_memory.pipelines.system2_agent import _detect_language + + +ZH_FACT = "用户使用 Python 开发,运行 pytest tests/test_search_client.py --maxfail=1" +EN_FACT = "The user runs pytest tests/test_search_client.py --maxfail=1" + + +@pytest.mark.parametrize("source", ["clusters", "unprocessed_facts"]) +@pytest.mark.parametrize("content,expected", [ + (ZH_FACT, "zh"), + ("用户喜欢咖啡", "zh"), + (EN_FACT, "en"), + ("Please update the 中文 README file.", "en"), + ("", "en"), + (" \n\t", "en"), +]) +def test_fact_language_ignores_technical_payload_length(source, content, expected): + facts = [{"content": content}] + materials = {source: [{"facts": facts}] if source == "clusters" else facts} + assert _detect_language(materials) == expected + + +@pytest.mark.parametrize("contents,expected", [ + ([EN_FACT * 20, ZH_FACT, ZH_FACT], "zh"), + ([ZH_FACT * 20, EN_FACT, EN_FACT], "en"), + ([ZH_FACT, EN_FACT], "en"), + (["", " ", ZH_FACT], "zh"), + ([], "en"), +]) +def test_materials_vote_across_both_fact_sources(contents, expected): + facts = [{"content": content} for content in contents] + for ordered in (facts, list(reversed(facts))): + materials = { + "clusters": [{"facts": ordered[:1]}], + "unprocessed_facts": ordered[1:], + } + assert _detect_language(materials) == expected