From 04ee55e1c9bfcd91038f8f6ebaac8d7e75c7bb76 Mon Sep 17 00:00:00 2001 From: Zhang Sheng Date: Mon, 20 Jul 2026 08:29:32 +0800 Subject: [PATCH] fix: improve Chinese date parsing and handling MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 1. Added support for more flexible Chinese date formats (missing 月/ 日 suffixes) 2. Fixed potential conflict between size extraction and year-month patterns 3. Added test case to verify size extraction doesn't shadow year-month patterns 4. Enhanced regex patterns to support more date format variations 5. Added checks in SizeExtractor to respect previously matched spans The changes improve Chinese NLP parsing by: - Making common date patterns like "2025年12" (without 月) and "2025年12 月30" (without 日) work - Preventing false positives where size patterns could incorrectly match year components - Supporting more Chinese variations like "年份" suffix and different separators - Adding proper span checking to maintain intent integrity Log: Improved Chinese date parsing with more flexible formats and better conflict resolution Influence: 1. Test various Chinese date formats with and without 月/日 suffixes 2. Verify mixed queries like "2025-5的音乐" correctly parse as date + filetype 3. Test boundary cases like "50KB" vs "2025年5月" 4. Verify all supported date separators (年./-) work correctly fix: 改进中文日期解析和处理功能 1. 添加对更灵活中文日期格式的支持（支持省略月/日后缀） 2. 修复大小提取与年月模式之间的潜在冲突 3. 添加测试用例验证大小提取不会遮盖年月模式 4. 增强正则表达式以支持更多日期格式变化 5. 在SizeExtractor中添加检查以尊重先前匹配的文本片段 这些改进通过以下方式提升了中文NLP解析： - 使常见日期格式如"2025年12"（无月）和"2025年12月30"（无日）能够正确解析 - 防止大小模式误匹配年份组件的错误 - 支持更多中文变体如"年份"后缀和不同分隔符 - 添加正确的文本片段检查以保持意图完整性 Log: 改进中文日期解析，支持更灵活的格式和更好的冲突解决 Influence: 1. 测试带或不带月/日后缀的各种中文日期格式 2. 验证混合查询如"2025-5的音乐"能正确解析为日期+文件类型 3. 测试边界情况如"50KB"与"2025年5月"的区别 4. 验证所有支持的日期分隔符（年./-）都能正常工作 Fixes: #370481 --- .../dfm-search-tests/tst_chinese_nlp.cpp | 54 +++++++++++++++++ .../dfm-search-tests/tst_semantic_search.cpp | 60 +++++++++++++++---- .../semantic/extractors/sizeextractor.cpp | 11 ++++ .../semantic/rules/zh_CN/time_rules.json | 4 +- 4 files changed, 116 insertions(+), 13 deletions(-) diff --git a/autotests/dfm-search-tests/tst_chinese_nlp.cpp b/autotests/dfm-search-tests/tst_chinese_nlp.cpp index 321b1b79..2971e72d 100644 --- a/autotests/dfm-search-tests/tst_chinese_nlp.cpp +++ b/autotests/dfm-search-tests/tst_chinese_nlp.cpp @@ -204,6 +204,7 @@ private Q_SLOTS: void size_suffix_max(); void size_suffix_combined(); void size_suffix_chineseUnits(); + void size_doesNotShadow_yearMonth(); // Relative time tests void timeRelative_justNow(); @@ -971,6 +972,22 @@ void tst_ChineseNLP::timeCustom_yearMonth() QCOMPARE(intent.timeConstraint().customStart().date().day(), 1); QCOMPARE(intent.timeConstraint().customEnd().date().year(), 2025); QCOMPARE(intent.timeConstraint().customEnd().date().month(), 12); + + // "2025年12" — trailing 月 is optional + ParsedIntent intent2; + m_parser->parse(QStringLiteral("2025年12创建的文件"), intent2); + QCOMPARE(intent2.timeConstraint().kind(), TimeConstraintKind::Custom); + QCOMPARE(intent2.timeConstraint().customStart().date().year(), 2025); + QCOMPARE(intent2.timeConstraint().customStart().date().month(), 12); + QCOMPARE(intent2.timeConstraint().customStart().date().day(), 1); + + // "2025年12月份" — support 月份 suffix too + ParsedIntent intent3; + m_parser->parse(QStringLiteral("2025年12月份创建的文件"), intent3); + QCOMPARE(intent3.timeConstraint().kind(), TimeConstraintKind::Custom); + QCOMPARE(intent3.timeConstraint().customStart().date().year(), 2025); + QCOMPARE(intent3.timeConstraint().customStart().date().month(), 12); + QCOMPARE(intent3.timeConstraint().customStart().date().day(), 1); } void tst_ChineseNLP::timeCustom_yearMonth_separators() @@ -1035,6 +1052,14 @@ void tst_ChineseNLP::timeCustom_fullDate() // Verify time boundaries QCOMPARE(intent.timeConstraint().customStart().time().hour(), 0); QCOMPARE(intent.timeConstraint().customEnd().time().hour(), 23); + + // "2025年12月30" — trailing 日号 is optional + ParsedIntent intent2; + m_parser->parse(QStringLiteral("2025年12月30创建的文件"), intent2); + QCOMPARE(intent2.timeConstraint().kind(), TimeConstraintKind::Custom); + QCOMPARE(intent2.timeConstraint().customStart().date().year(), 2025); + QCOMPARE(intent2.timeConstraint().customStart().date().month(), 12); + QCOMPARE(intent2.timeConstraint().customStart().date().day(), 30); } void tst_ChineseNLP::timeCustom_fullDate_separators() @@ -1579,6 +1604,35 @@ void tst_ChineseNLP::size_suffix_chineseUnits() QCOMPARE(intent2.sizeConstraint().maxSize(), 51200LL); // 50KB } +void tst_ChineseNLP::size_doesNotShadow_yearMonth() +{ + ParsedIntent intent; + m_parser->parse(QStringLiteral("2025-5的音乐"), intent); + + QCOMPARE(intent.timeConstraint().kind(), TimeConstraintKind::Custom); + QCOMPARE(intent.timeConstraint().customStart().date().year(), 2025); + QCOMPARE(intent.timeConstraint().customStart().date().month(), 5); + QVERIFY(!intent.sizeConstraint().isValid()); + QVERIFY(setEquals(intent.fileExtensions(), audioExpectedExts())); + + bool hasTimeYearMonth = false; + bool hasAudio = false; + bool hasSizeBetween = false; + for (const MatchSpan &span : intent.consumedSpans()) { + if (span.ruleId() == QLatin1String("time_exact_year_month")) { + hasTimeYearMonth = true; + } else if (span.ruleId() == QLatin1String("filetype_audio")) { + hasAudio = true; + } else if (span.ruleId() == QLatin1String("size_dynamic_between")) { + hasSizeBetween = true; + } + } + + QVERIFY(hasTimeYearMonth); + QVERIFY(hasAudio); + QVERIFY(!hasSizeBetween); +} + // ===== Relative Time Tests ===== void tst_ChineseNLP::timeRelative_justNow() diff --git a/autotests/dfm-search-tests/tst_semantic_search.cpp b/autotests/dfm-search-tests/tst_semantic_search.cpp index b42f76ec..737a51e4 100644 --- a/autotests/dfm-search-tests/tst_semantic_search.cpp +++ b/autotests/dfm-search-tests/tst_semantic_search.cpp @@ -306,15 +306,35 @@ void tst_TimeExtraction::customYearMonth() meta["type"] = "custom"; meta["format"] = "year_month"; QByteArray json = makeRuleJson("time", "time_exact_year_month", - "(?\\d{4})-(?\\d{1,2})", 150, meta); + "(?\\d{2,4})[年\\./\\-](?\\d{1,2})(?:月份?)?", 150, meta); RuleGroup group; QVERIFY(buildGroupFromJson(json, group)); - auto match = group.rules[0].regex.match("2025-12"); - QVERIFY(match.hasMatch()); - QCOMPARE(match.captured("year"), QString("2025")); - QCOMPARE(match.captured("month"), QString("12")); + auto match1 = group.rules[0].regex.match("2025年12月"); + QVERIFY(match1.hasMatch()); + QCOMPARE(match1.captured("year"), QString("2025")); + QCOMPARE(match1.captured("month"), QString("12")); + + auto match2 = group.rules[0].regex.match("2025年12"); + QVERIFY(match2.hasMatch()); + QCOMPARE(match2.captured("year"), QString("2025")); + QCOMPARE(match2.captured("month"), QString("12")); + + auto match3 = group.rules[0].regex.match("2025年12月份"); + QVERIFY(match3.hasMatch()); + QCOMPARE(match3.captured("year"), QString("2025")); + QCOMPARE(match3.captured("month"), QString("12")); + + auto match4 = group.rules[0].regex.match("2025-12"); + QVERIFY(match4.hasMatch()); + QCOMPARE(match4.captured("year"), QString("2025")); + QCOMPARE(match4.captured("month"), QString("12")); + + auto match5 = group.rules[0].regex.match("25.12"); + QVERIFY(match5.hasMatch()); + QCOMPARE(match5.captured("year"), QString("25")); + QCOMPARE(match5.captured("month"), QString("12")); } void tst_TimeExtraction::customFullDate() @@ -323,17 +343,35 @@ void tst_TimeExtraction::customFullDate() meta["type"] = "custom"; meta["format"] = "full_date"; QByteArray json = makeRuleJson("time", "time_exact_full_date", - "(?\\d{4})-(?\\d{1,2})-(?\\d{1,2})", + "(?\\d{2,4})[年\\./\\-](?\\d{1,2})[月\\./\\-](?\\d{1,2})[日号]?", 140, meta); RuleGroup group; QVERIFY(buildGroupFromJson(json, group)); - auto match = group.rules[0].regex.match("2025-03-15"); - QVERIFY(match.hasMatch()); - QCOMPARE(match.captured("year"), QString("2025")); - QCOMPARE(match.captured("month"), QString("03")); - QCOMPARE(match.captured("day"), QString("15")); + auto match1 = group.rules[0].regex.match("2025年12月5日"); + QVERIFY(match1.hasMatch()); + QCOMPARE(match1.captured("year"), QString("2025")); + QCOMPARE(match1.captured("month"), QString("12")); + QCOMPARE(match1.captured("day"), QString("5")); + + auto match2 = group.rules[0].regex.match("2025年12月5"); + QVERIFY(match2.hasMatch()); + QCOMPARE(match2.captured("year"), QString("2025")); + QCOMPARE(match2.captured("month"), QString("12")); + QCOMPARE(match2.captured("day"), QString("5")); + + auto match3 = group.rules[0].regex.match("2025-03-15"); + QVERIFY(match3.hasMatch()); + QCOMPARE(match3.captured("year"), QString("2025")); + QCOMPARE(match3.captured("month"), QString("03")); + QCOMPARE(match3.captured("day"), QString("15")); + + auto match4 = group.rules[0].regex.match("2025/12/5"); + QVERIFY(match4.hasMatch()); + QCOMPARE(match4.captured("year"), QString("2025")); + QCOMPARE(match4.captured("month"), QString("12")); + QCOMPARE(match4.captured("day"), QString("5")); } void tst_TimeExtraction::noMatch() diff --git a/src/dfm-search/dfm-search-lib/semantic/extractors/sizeextractor.cpp b/src/dfm-search/dfm-search-lib/semantic/extractors/sizeextractor.cpp index fd0a70ec..4f0bda5b 100644 --- a/src/dfm-search/dfm-search-lib/semantic/extractors/sizeextractor.cpp +++ b/src/dfm-search/dfm-search-lib/semantic/extractors/sizeextractor.cpp @@ -29,6 +29,17 @@ void SizeExtractor::extract(const QString &input, ParsedIntent &intent) return; } + // Size extraction runs after time/filetype. If this size span is already + // fully consumed by an earlier, higher-confidence dimension such as a + // year-month time expression ("2025-5"), ignore it to avoid overwriting + // the intent with a bogus size constraint. + for (const MatchSpan &existing : intent.consumedSpans()) { + if (match.capturedStart() >= existing.start() + && match.capturedEnd() <= existing.end()) { + return; + } + } + const QVariantMap metadata = m_engine->ruleMetadata("size", ruleId); const QString typeStr = metadata.value("type").toString(); SizeConstraint sc; diff --git a/src/dfm-search/dfm-search-lib/semantic/rules/zh_CN/time_rules.json b/src/dfm-search/dfm-search-lib/semantic/rules/zh_CN/time_rules.json index df4086be..31e13dec 100644 --- a/src/dfm-search/dfm-search-lib/semantic/rules/zh_CN/time_rules.json +++ b/src/dfm-search/dfm-search-lib/semantic/rules/zh_CN/time_rules.json @@ -157,7 +157,7 @@ }, { "id": "time_exact_year_month", - "pattern": "(?\\d{2,4})[年\\./\\-](?\\d{1,2})月份?", + "pattern": "(?\\d{2,4})[年\\./\\-](?\\d{1,2})(?:月份?)?", "description": "Exact year-month (e.g. 2025年12月, 2025-12)", "enabled": true, "priority": 140, @@ -337,4 +337,4 @@ ] } ] -} \ No newline at end of file +}