diff --git a/CHANGELOG.md b/CHANGELOG.md index 8e1b6de..199e418 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added +- “数字与格式(ITN)”新增“中文与英文/数字间空格”开关。开启后在相邻中文与 ASCII 英文或数字之间添加空格,默认关闭。 - Fcitx5 adds `PanelStyle=ultra_minimal`: recording and post-release processing use a compact ellipsis animation while keeping live recognition preview and hiding polish intermediates. Existing `minimal` is relabeled as concise; `animated` is unchanged. ### Changed diff --git a/docs/guides/settings-center.md b/docs/guides/settings-center.md index ed22849..8d24fe0 100644 --- a/docs/guides/settings-center.md +++ b/docs/guides/settings-center.md @@ -40,6 +40,7 @@ bash scripts/install/ibus/install.sh --install-system-deps --download-models 可配置输入设备、采样率、最短录音、实时 preedit、ITN,并管理: +- 识别后的中文与英文/数字之间的空格; - Fcitx 状态面板的 `minimal`(简洁)、`ultra_minimal`(极简)与 `animated` 样式;极简模式在录音和松键后的处理阶段循环显示省略号, 不展示润色中间结果; diff --git a/src/common/include/vocotype/common/spacing.hpp b/src/common/include/vocotype/common/spacing.hpp new file mode 100644 index 0000000..53cd8c9 --- /dev/null +++ b/src/common/include/vocotype/common/spacing.hpp @@ -0,0 +1,86 @@ +#pragma once + +#include +#include +#include + +namespace vocotype::common { +namespace detail { + +inline bool is_cjk_codepoint(std::uint32_t codepoint) { + return (codepoint >= 0x3400 && codepoint <= 0x4DBF) || + (codepoint >= 0x4E00 && codepoint <= 0x9FFF) || + (codepoint >= 0xF900 && codepoint <= 0xFAFF) || codepoint == 0x3007 || + (codepoint >= 0x20000 && codepoint <= 0x2A6DF) || + (codepoint >= 0x2A700 && codepoint <= 0x2B73F) || + (codepoint >= 0x2B740 && codepoint <= 0x2B81F) || + (codepoint >= 0x2B820 && codepoint <= 0x2CEAF) || + (codepoint >= 0x2CEB0 && codepoint <= 0x2EBEF) || + (codepoint >= 0x2EBF0 && codepoint <= 0x2EE5D) || + (codepoint >= 0x2F800 && codepoint <= 0x2FA1F) || + (codepoint >= 0x30000 && codepoint <= 0x3134A) || + (codepoint >= 0x31350 && codepoint <= 0x323AF); +} + +inline bool is_ascii_word_codepoint(std::uint32_t codepoint) { + return (codepoint >= 'A' && codepoint <= 'Z') || + (codepoint >= 'a' && codepoint <= 'z') || + (codepoint >= '0' && codepoint <= '9'); +} + +inline std::size_t utf8_codepoint(const std::string &text, std::size_t offset, + std::uint32_t &codepoint) { + const auto byte = static_cast(text[offset]); + if (byte < 0x80) { + codepoint = byte; + return 1; + } + const std::size_t size = + (byte & 0xE0) == 0xC0 ? 2 : (byte & 0xF0) == 0xE0 ? 3 + : (byte & 0xF8) == 0xF0 ? 4 + : 1; + if (size == 1 || offset + size > text.size()) { + codepoint = byte; + return 1; + } + std::uint32_t value = byte & ((1U << (8U - size - 1U)) - 1U); + for (std::size_t index = 1; index < size; ++index) { + const auto continuation = static_cast(text[offset + index]); + if ((continuation & 0xC0) != 0x80) { + codepoint = byte; + return 1; + } + value = (value << 6U) | (continuation & 0x3FU); + } + codepoint = value; + return size; +} + +} // namespace detail + +// 在相邻中文与 ASCII 英文/数字之间补一个空格,不修改英文、数字和标点内部结构。 +inline std::string space_between_cjk_and_ascii(std::string text) { + enum class Kind { Other, Cjk, AsciiWord }; + std::string result; + result.reserve(text.size() + text.size() / 8); + Kind previous = Kind::Other; + for (std::size_t offset = 0; offset < text.size();) { + std::uint32_t codepoint = 0; + const std::size_t length = detail::utf8_codepoint(text, offset, codepoint); + const Kind current = detail::is_cjk_codepoint(codepoint) + ? Kind::Cjk + : detail::is_ascii_word_codepoint(codepoint) + ? Kind::AsciiWord + : Kind::Other; + if ((previous == Kind::Cjk && current == Kind::AsciiWord) || + (previous == Kind::AsciiWord && current == Kind::Cjk)) { + result.push_back(' '); + } + result.append(text, offset, length); + previous = current; + offset += length; + } + return result; +} + +} // namespace vocotype::common diff --git a/src/core/include/vocotype/core/config.hpp b/src/core/include/vocotype/core/config.hpp index 7a33856..de859a2 100644 --- a/src/core/include/vocotype/core/config.hpp +++ b/src/core/include/vocotype/core/config.hpp @@ -25,6 +25,7 @@ struct NormalizationConfig { bool compact_times = true; bool compact_distances = true; bool currency_symbols = true; + bool space_between_cjk_and_ascii = false; }; struct OfflineAsrConfig { diff --git a/src/core/include/vocotype/core/offline_asr.hpp b/src/core/include/vocotype/core/offline_asr.hpp index 0ad2a92..7b73eb1 100644 --- a/src/core/include/vocotype/core/offline_asr.hpp +++ b/src/core/include/vocotype/core/offline_asr.hpp @@ -24,6 +24,7 @@ class OfflineAsrProcess { [[nodiscard]] Json prepare(const Json &request = Json::object()); [[nodiscard]] Json transcribe(const Json &request); [[nodiscard]] std::string normalize_text(const std::string &text); + [[nodiscard]] std::string format_final_text(std::string text) const; [[nodiscard]] std::string build_native_hotwords(const std::string &extra = ""); @@ -36,6 +37,7 @@ class OfflineAsrProcess { [[nodiscard]] std::vector worker_arguments() const; OfflineAsrConfig config_; + NormalizationConfig normalization_; std::mutex request_mutex_; TextNormalizer normalizer_; JsonLineWorker worker_; diff --git a/src/core/src/config.cpp b/src/core/src/config.cpp index b9443b7..9bbcd72 100644 --- a/src/core/src/config.cpp +++ b/src/core/src/config.cpp @@ -121,6 +121,7 @@ Json default_config_json() { {"compact_times", true}, {"compact_distances", true}, {"currency_symbols", true}, + {"space_between_cjk_and_ascii", false}, }}, {"asr_streaming", { @@ -219,6 +220,9 @@ AppConfig parse_config(const Json &value) { config.normalization.compact_distances); config.normalization.currency_symbols = bool_value_or( normalization, "currency_symbols", config.normalization.currency_symbols); + config.normalization.space_between_cjk_and_ascii = bool_value_or( + normalization, "space_between_cjk_and_ascii", + config.normalization.space_between_cjk_and_ascii); const Json streaming = merged.value("asr_streaming", Json::object()); config.streaming_asr.enabled = diff --git a/src/core/src/dispatcher.cpp b/src/core/src/dispatcher.cpp index 2cddd58..e8ee870 100644 --- a/src/core/src/dispatcher.cpp +++ b/src/core/src/dispatcher.cpp @@ -132,6 +132,9 @@ Json CoreDispatcher::dispatch(const Json &request) const { bool_value_or(value, "compact_distances", config.compact_distances); config.currency_symbols = bool_value_or(value, "currency_symbols", config.currency_symbols); + config.space_between_cjk_and_ascii = bool_value_or( + value, "space_between_cjk_and_ascii", + config.space_between_cjk_and_ascii); TextNormalizer normalizer(config); normalized = normalizer.normalize(text); } else { diff --git a/src/core/src/offline_asr.cpp b/src/core/src/offline_asr.cpp index 7e50d08..1648cca 100644 --- a/src/core/src/offline_asr.cpp +++ b/src/core/src/offline_asr.cpp @@ -1,4 +1,5 @@ #include "vocotype/core/offline_asr.hpp" +#include "vocotype/common/spacing.hpp" #include @@ -62,7 +63,8 @@ Json error_response(const std::string &error) { OfflineAsrProcess::OfflineAsrProcess(OfflineAsrConfig config, NormalizationConfig normalization) - : config_(std::move(config)), normalizer_(normalization) {} + : config_(std::move(config)), normalization_(std::move(normalization)), + normalizer_(normalization_) {} bool OfflineAsrProcess::enabled() const noexcept { return config_.enabled; } @@ -243,6 +245,12 @@ std::string OfflineAsrProcess::normalize_text(const std::string &text) { return normalizer_.normalize(text); } +std::string OfflineAsrProcess::format_final_text(std::string text) const { + return normalization_.space_between_cjk_and_ascii + ? vocotype::common::space_between_cjk_and_ascii(std::move(text)) + : text; +} + std::string OfflineAsrProcess::build_native_hotwords(const std::string &extra) { std::lock_guard lock(request_mutex_); return normalizer_.build_native_hotwords(extra); diff --git a/src/core/src/text_normalizer.cpp b/src/core/src/text_normalizer.cpp index 9262e02..d92b4c7 100644 --- a/src/core/src/text_normalizer.cpp +++ b/src/core/src/text_normalizer.cpp @@ -1,4 +1,5 @@ #include "vocotype/core/text_normalizer.hpp" +#include "vocotype/common/spacing.hpp" #include "vocotype/common/terms_yaml.hpp" #include @@ -1756,14 +1757,20 @@ class TextNormalizer::Impl { if (terms.text.empty()) { return {}; } + std::string result; if (!config_.enabled) { - return encode_utf8(terms.text); - } - const UString numeric = - normalize_chinese_numbers(terms.text, terms.protected_spans); - const TermRewriteResult restyled = rewrite_terms(lexicon, numeric); - return encode_utf8( - apply_written_style(restyled.text, config_, restyled.protected_spans)); + result = encode_utf8(terms.text); + } else { + const UString numeric = + normalize_chinese_numbers(terms.text, terms.protected_spans); + const TermRewriteResult restyled = rewrite_terms(lexicon, numeric); + result = encode_utf8( + apply_written_style(restyled.text, config_, restyled.protected_spans)); + } + return config_.space_between_cjk_and_ascii + ? vocotype::common::space_between_cjk_and_ascii( + std::move(result)) + : result; } std::string build_native_hotwords(const std::string &extra) { diff --git a/src/core/src/transcription_tasks.cpp b/src/core/src/transcription_tasks.cpp index 6123064..571b5c4 100644 --- a/src/core/src/transcription_tasks.cpp +++ b/src/core/src/transcription_tasks.cpp @@ -362,7 +362,7 @@ void TranscriptionTaskManager::run_task(const std::shared_ptr &task, return; } cleanup.remove_now(); - task->mark_final(polished.text, polished.reason); + task->mark_final(asr_.format_final_text(polished.text), polished.reason); } } // namespace vocotype::core diff --git a/src/core/tests/core_tests.cpp b/src/core/tests/core_tests.cpp index cab3b59..abf81f1 100644 --- a/src/core/tests/core_tests.cpp +++ b/src/core/tests/core_tests.cpp @@ -21,6 +21,7 @@ #include "vocotype/common/posix.hpp" #include "vocotype/core/config.hpp" #include "vocotype/core/dispatcher.hpp" +#include "vocotype/core/offline_asr.hpp" #include "vocotype/core/server.hpp" #include "vocotype/core/text_normalizer.hpp" #include "vocotype/core/voice_edit.hpp" @@ -391,6 +392,19 @@ void test_text_normalizer() { require(no_itn.normalize("二百五十六台") == "二百五十六台", "normalization master switch was ignored"); + vocotype::core::NormalizationConfig spaced; + spaced.space_between_cjk_and_ascii = true; + TextNormalizer spaced_style(spaced); + require(spaced_style.normalize("使用Claude Code处理2026年数据") == + "使用 Claude Code 处理 2026 年数据", + "CJK and ASCII spacing style was not applied"); + require(spaced_style.normalize("𠮷A") == "𠮷 A", + "supplementary-plane Han spacing was not applied"); + + vocotype::core::OfflineAsrProcess final_formatter({}, spaced); + require(final_formatter.format_final_text("使用Claude") == "使用 Claude", + "post-polish final spacing was not applied"); + { std::ofstream output(terms); output << R"(terms: @@ -535,7 +549,8 @@ void test_config_merge() { {{"native_enabled", true}, {"intra_op_num_threads", 4}, {"hotword", "VoCoType"}}}, - {"normalization", {{"compact_times", false}}}, + {"normalization", + {{"compact_times", false}, {"space_between_cjk_and_ascii", true}}}, {"asr_streaming", {{"enabled", true}, {"intra_op_num_threads", 3}, @@ -556,6 +571,8 @@ void test_config_merge() { "normalization style override was lost"); require(config.normalization.compact_dates, "normalization default was not preserved"); + require(config.normalization.space_between_cjk_and_ascii, + "CJK and ASCII spacing override was lost"); require(config.streaming_asr.enabled, "streaming ASR enabled override was lost"); require(config.streaming_asr.threads == 3, diff --git a/src/desktop/src/settings_main.cpp b/src/desktop/src/settings_main.cpp index e8d1677..1eb95fa 100644 --- a/src/desktop/src/settings_main.cpp +++ b/src/desktop/src/settings_main.cpp @@ -109,6 +109,7 @@ struct SettingsWindow { GtkSwitch *compact_times = nullptr; GtkSwitch *compact_distances = nullptr; GtkSwitch *currency_symbols = nullptr; + GtkSwitch *space_between_cjk_and_ascii = nullptr; GtkEntry *itn_preview_input = nullptr; GtkLabel *itn_preview_output = nullptr; GtkComboBoxText *fcitx_panel_style = nullptr; @@ -982,6 +983,8 @@ void save_config(SettingsWindow &window) { gtk_switch_get_active(window.compact_distances); normalization["currency_symbols"] = gtk_switch_get_active(window.currency_symbols); + normalization["space_between_cjk_and_ascii"] = + gtk_switch_get_active(window.space_between_cjk_and_ascii); auto &slm = window.config["slm"]; slm["enabled"] = gtk_switch_get_active(window.slm_enabled); @@ -1031,6 +1034,9 @@ void save_config(SettingsWindow &window) { gtk_switch_get_active(window.fcitx_block_composing) ? "True" : "False"}, {"StripTrailingPeriodOnCommit", gtk_switch_get_active(window.fcitx_strip_period) ? "True" : "False"}, + {"SpaceBetweenCjkAndAscii", + gtk_switch_get_active(window.space_between_cjk_and_ascii) ? "True" + : "False"}, }; if (hotkeys_changed) { fcitx_values.insert( @@ -2524,6 +2530,11 @@ void populate_from_config(SettingsWindow &window) { json_bool(normalization, "compact_distances", true)); gtk_switch_set_active(window.currency_symbols, json_bool(normalization, "currency_symbols", true)); + gtk_switch_set_active( + window.space_between_cjk_and_ascii, + json_bool(normalization, "space_between_cjk_and_ascii", + config_value(fcitx_config_path(), "SpaceBetweenCjkAndAscii", + "False") == "True")); const auto &slm = window.config["slm"]; gtk_switch_set_active(window.slm_enabled, json_bool(slm, "enabled", false)); @@ -3109,6 +3120,7 @@ GtkWidget *build_recognition(SettingsWindow &window) { window.compact_times = GTK_SWITCH(sui::make_switch()); window.compact_distances = GTK_SWITCH(sui::make_switch()); window.currency_symbols = GTK_SWITCH(sui::make_switch()); + window.space_between_cjk_and_ascii = GTK_SWITCH(sui::make_switch()); gtk_box_pack_start(GTK_BOX(itn_card), sui::make_row("启用数字与 ITN", "关闭后保留用户词典替换,但不改写中文数字。", @@ -3132,6 +3144,12 @@ GtkWidget *build_recognition(SettingsWindow &window) { sui::make_row("金额符号", "例如:一百二十八元 → ¥128", GTK_WIDGET(window.currency_symbols)), FALSE, FALSE, 0); + gtk_box_pack_start( + GTK_BOX(itn_card), + sui::make_row("中文与英文/数字间空格", + "在相邻中文与英文单词或阿拉伯数字之间加空格,例如“使用 Claude Code 处理 2026 年数据”。", + GTK_WIDGET(window.space_between_cjk_and_ascii)), + FALSE, FALSE, 0); gtk_box_pack_start(GTK_BOX(page.content), itn_card, FALSE, FALSE, 0); GtkWidget *preview_card = sui::make_card(); @@ -3232,6 +3250,8 @@ GtkWidget *build_recognition(SettingsWindow &window) { {"compact_times", gtk_switch_get_active(self->compact_times)}, {"compact_distances", gtk_switch_get_active(self->compact_distances)}, {"currency_symbols", gtk_switch_get_active(self->currency_symbols)}, + {"space_between_cjk_and_ascii", + gtk_switch_get_active(self->space_between_cjk_and_ascii)}, }; const Json result = vocotype::desktop::unix_json_request( vocotype::desktop::backend_socket_path(), diff --git a/src/integrations/fcitx5/module/vocotype_module.cpp b/src/integrations/fcitx5/module/vocotype_module.cpp index 7084fba..d61cb69 100644 --- a/src/integrations/fcitx5/module/vocotype_module.cpp +++ b/src/integrations/fcitx5/module/vocotype_module.cpp @@ -1,5 +1,6 @@ #include "vocotype_module.h" #include "vocotype/common/preview_text.hpp" +#include "vocotype/common/spacing.hpp" #include "recorder_shutdown.hpp" #include "timer_lifetime.hpp" @@ -2151,10 +2152,14 @@ void VoCoTypeModule::commitText(fcitx::InputContext *ic, } const std::string commit_text = strip_trailing_period ? stripTrailingCommitPeriod(text) : text; + const std::string formatted_text = config_.spaceBetweenCjkAndAscii.value() + ? vocotype::common::space_between_cjk_and_ascii( + commit_text) + : commit_text; const uint64_t now = fcitx::now(CLOCK_MONOTONIC); const std::string program = ic->program(); const std::string frontend = ic->frontend() ? ic->frontend() : ""; - if (last_committed_ic_ == ic && last_committed_text_ == commit_text && + if (last_committed_ic_ == ic && last_committed_text_ == formatted_text && last_committed_program_ == program && last_committed_frontend_ == frontend && now >= last_commit_time_us_ && now - last_commit_time_us_ < DUPLICATE_COMMIT_SUPPRESS_US) { @@ -2162,12 +2167,12 @@ void VoCoTypeModule::commitText(fcitx::InputContext *ic, } clearOwnedUI(ic); - ic->commitString(commit_text); + ic->commitString(formatted_text); last_committed_ic_ = ic; last_committed_program_ = program; last_committed_frontend_ = frontend; - last_committed_text_ = commit_text; + last_committed_text_ = formatted_text; last_commit_time_us_ = now; } diff --git a/src/integrations/fcitx5/module/vocotype_module.h b/src/integrations/fcitx5/module/vocotype_module.h index c548832..aafbb5b 100644 --- a/src/integrations/fcitx5/module/vocotype_module.h +++ b/src/integrations/fcitx5/module/vocotype_module.h @@ -84,6 +84,8 @@ FCITX_CONFIGURATION( this, "BlockWhenComposing", "存在未提交预编辑时禁止开始录音", true}; fcitx::HiddenOption stripTrailingPeriodOnCommit{ this, "StripTrailingPeriodOnCommit", "提交时移除尾部句号", false}; + fcitx::HiddenOption spaceBetweenCjkAndAscii{ + this, "SpaceBetweenCjkAndAscii", "中文与英文/数字之间加空格", false}; fcitx::HiddenOption panelStyle{ this, "PanelStyle", "状态提示样式(minimal、ultra_minimal 或 animated)", "minimal"};); diff --git a/src/integrations/macos/VocoTypeApplicationController.mm b/src/integrations/macos/VocoTypeApplicationController.mm index 5d8704c..61ddacd 100644 --- a/src/integrations/macos/VocoTypeApplicationController.mm +++ b/src/integrations/macos/VocoTypeApplicationController.mm @@ -918,6 +918,7 @@ @implementation VocoTypeApplicationController { NSButton *_compactTimes; NSButton *_compactDistances; NSButton *_currencySymbols; + NSButton *_spaceBetweenCjkAndAscii; NSTextField *_normalizationInput; NSTextField *_normalizationOutput; std::vector _inputDevices; @@ -1623,11 +1624,15 @@ - (void)buildGeneralPage { _compactTimes = switch_button(); _compactDistances = switch_button(); _currencySymbols = switch_button(); + _spaceBetweenCjkAndAscii = switch_button(); [normalizationCard addArrangedSubview:settings_row(@"启用文本规范化", @"识别后执行术语替换和中文数字 ITN。", _normalizationEnabled)]; [normalizationCard addArrangedSubview:settings_row(@"紧凑日期", @"例如 2026/07/27。", _compactDates)]; [normalizationCard addArrangedSubview:settings_row(@"紧凑时间", @"例如 15:20。", _compactTimes)]; [normalizationCard addArrangedSubview:settings_row(@"紧凑距离", @"例如 320m。", _compactDistances)]; [normalizationCard addArrangedSubview:settings_row(@"货币符号", @"例如 ¥256。", _currencySymbols)]; + [normalizationCard addArrangedSubview:settings_row( + @"中文与英文/数字间空格", + @"例如“使用 Claude Code 处理 2026 年数据”。", _spaceBetweenCjkAndAscii)]; _normalizationInput = text_field(@"下午三点二十分跑了三百二十米,价格二百五十六元"); NSStackView *preview = horizontal_stack(8.0); [preview addArrangedSubview:_normalizationInput]; @@ -2579,6 +2584,10 @@ - (void)reloadFields { _currencySymbols.state = normalization.value("currency_symbols", true) ? NSControlStateValueOn : NSControlStateValueOff; + _spaceBetweenCjkAndAscii.state = + normalization.value("space_between_cjk_and_ascii", false) + ? NSControlStateValueOn + : NSControlStateValueOff; const Json slm = _config.value("slm", Json::object()); _slmEnabled.state = slm.value("enabled", false) ? NSControlStateValueOn @@ -2674,6 +2683,8 @@ - (BOOL)persistSettings { _compactDistances.state == NSControlStateValueOn; normalization["currency_symbols"] = _currencySymbols.state == NSControlStateValueOn; + normalization["space_between_cjk_and_ascii"] = + _spaceBetweenCjkAndAscii.state == NSControlStateValueOn; Json &slm = shared["slm"]; slm["enabled"] = _slmEnabled.state == NSControlStateValueOn;