Resolve and integrate #65 CJK/ASCII spacing

CHANGELOG.md

@@ -9,6 +9,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ### Added
 
+- “数字与格式（ITN）”新增“中文与英文/数字间空格”开关。开启后在相邻中文与 ASCII 英文或数字之间添加空格，默认关闭。
 - Fcitx5 adds `PanelStyle=ultra_minimal`: recording and post-release processing use a compact ellipsis animation while keeping live recognition preview and hiding polish intermediates. Existing `minimal` is relabeled as concise; `animated` is unchanged.
 
 ### Changed

docs/guides/settings-center.md

@@ -40,6 +40,7 @@ bash scripts/install/ibus/install.sh --install-system-deps --download-models
 
 可配置输入设备、采样率、最短录音、实时 preedit、ITN，并管理：
 
+- 识别后的中文与英文/数字之间的空格；
 - Fcitx 状态面板的 `minimal`（简洁）、`ultra_minimal`（极简）与
   `animated` 样式；极简模式在录音和松键后的处理阶段循环显示省略号，
   不展示润色中间结果；

src/common/include/vocotype/common/spacing.hpp

@@ -0,0 +1,86 @@
+#pragma once
+
+#include <cstddef>
+#include <cstdint>
+#include <string>
+
+namespace vocotype::common {
+namespace detail {
+
+inline bool is_cjk_codepoint(std::uint32_t codepoint) {
+  return (codepoint >= 0x3400 && codepoint <= 0x4DBF) ||
+         (codepoint >= 0x4E00 && codepoint <= 0x9FFF) ||
+         (codepoint >= 0xF900 && codepoint <= 0xFAFF) || codepoint == 0x3007 ||
+         (codepoint >= 0x20000 && codepoint <= 0x2A6DF) ||
+         (codepoint >= 0x2A700 && codepoint <= 0x2B73F) ||
+         (codepoint >= 0x2B740 && codepoint <= 0x2B81F) ||
+         (codepoint >= 0x2B820 && codepoint <= 0x2CEAF) ||
+         (codepoint >= 0x2CEB0 && codepoint <= 0x2EBEF) ||
+         (codepoint >= 0x2EBF0 && codepoint <= 0x2EE5D) ||
+         (codepoint >= 0x2F800 && codepoint <= 0x2FA1F) ||
+         (codepoint >= 0x30000 && codepoint <= 0x3134A) ||
+         (codepoint >= 0x31350 && codepoint <= 0x323AF);
+}
+
+inline bool is_ascii_word_codepoint(std::uint32_t codepoint) {
+  return (codepoint >= 'A' && codepoint <= 'Z') ||
+         (codepoint >= 'a' && codepoint <= 'z') ||
+         (codepoint >= '0' && codepoint <= '9');
+}
+
+inline std::size_t utf8_codepoint(const std::string &text, std::size_t offset,
+                                  std::uint32_t &codepoint) {
+  const auto byte = static_cast<unsigned char>(text[offset]);
+  if (byte < 0x80) {
+    codepoint = byte;
+    return 1;
+  }
+  const std::size_t size =
+      (byte & 0xE0) == 0xC0 ? 2 : (byte & 0xF0) == 0xE0 ? 3
+                                                    : (byte & 0xF8) == 0xF0 ? 4
+                                                                           : 1;
+  if (size == 1 || offset + size > text.size()) {
+    codepoint = byte;
+    return 1;
+  }
+  std::uint32_t value = byte & ((1U << (8U - size - 1U)) - 1U);
+  for (std::size_t index = 1; index < size; ++index) {
+    const auto continuation = static_cast<unsigned char>(text[offset + index]);
+    if ((continuation & 0xC0) != 0x80) {
+      codepoint = byte;
+      return 1;
+    }
+    value = (value << 6U) | (continuation & 0x3FU);
+  }
+  codepoint = value;
+  return size;
+}
+
+} // namespace detail
+
+// 在相邻中文与 ASCII 英文/数字之间补一个空格，不修改英文、数字和标点内部结构。
+inline std::string space_between_cjk_and_ascii(std::string text) {
+  enum class Kind { Other, Cjk, AsciiWord };
+  std::string result;
+  result.reserve(text.size() + text.size() / 8);
+  Kind previous = Kind::Other;
+  for (std::size_t offset = 0; offset < text.size();) {
+    std::uint32_t codepoint = 0;
+    const std::size_t length = detail::utf8_codepoint(text, offset, codepoint);
+    const Kind current = detail::is_cjk_codepoint(codepoint)
+                             ? Kind::Cjk
+                             : detail::is_ascii_word_codepoint(codepoint)
+                                   ? Kind::AsciiWord
+                                   : Kind::Other;
+    if ((previous == Kind::Cjk && current == Kind::AsciiWord) ||
+        (previous == Kind::AsciiWord && current == Kind::Cjk)) {
+      result.push_back(' ');
+    }
+    result.append(text, offset, length);
+    previous = current;
+    offset += length;
+  }
+  return result;
+}
+
+} // namespace vocotype::common

src/core/include/vocotype/core/config.hpp

@@ -25,6 +25,7 @@ struct NormalizationConfig {
   bool compact_times = true;
   bool compact_distances = true;
   bool currency_symbols = true;
+  bool space_between_cjk_and_ascii = false;
 };
 
 struct OfflineAsrConfig {

src/core/include/vocotype/core/offline_asr.hpp

@@ -24,6 +24,7 @@ class OfflineAsrProcess {
   [[nodiscard]] Json prepare(const Json &request = Json::object());
   [[nodiscard]] Json transcribe(const Json &request);
   [[nodiscard]] std::string normalize_text(const std::string &text);
+  [[nodiscard]] std::string format_final_text(std::string text) const;
   [[nodiscard]] std::string
   build_native_hotwords(const std::string &extra = "");
 
@@ -36,6 +37,7 @@ class OfflineAsrProcess {
   [[nodiscard]] std::vector<std::string> worker_arguments() const;
 
   OfflineAsrConfig config_;
+  NormalizationConfig normalization_;
   std::mutex request_mutex_;
   TextNormalizer normalizer_;
   JsonLineWorker worker_;

src/core/src/config.cpp

@@ -121,6 +121,7 @@ Json default_config_json() {
            {"compact_times", true},
            {"compact_distances", true},
            {"currency_symbols", true},
+           {"space_between_cjk_and_ascii", false},
        }},
       {"asr_streaming",
        {
@@ -219,6 +220,9 @@ AppConfig parse_config(const Json &value) {
                     config.normalization.compact_distances);
   config.normalization.currency_symbols = bool_value_or(
       normalization, "currency_symbols", config.normalization.currency_symbols);
+  config.normalization.space_between_cjk_and_ascii = bool_value_or(
+      normalization, "space_between_cjk_and_ascii",
+      config.normalization.space_between_cjk_and_ascii);
 
   const Json streaming = merged.value("asr_streaming", Json::object());
   config.streaming_asr.enabled =

src/core/src/dispatcher.cpp

@@ -132,6 +132,9 @@ Json CoreDispatcher::dispatch(const Json &request) const {
           bool_value_or(value, "compact_distances", config.compact_distances);
       config.currency_symbols =
           bool_value_or(value, "currency_symbols", config.currency_symbols);
+      config.space_between_cjk_and_ascii = bool_value_or(
+          value, "space_between_cjk_and_ascii",
+          config.space_between_cjk_and_ascii);
       TextNormalizer normalizer(config);
       normalized = normalizer.normalize(text);
     } else {

src/core/src/offline_asr.cpp

@@ -1,4 +1,5 @@
 #include "vocotype/core/offline_asr.hpp"
+#include "vocotype/common/spacing.hpp"
 
 #include <unistd.h>
 
@@ -62,7 +63,8 @@ Json error_response(const std::string &error) {
 
 OfflineAsrProcess::OfflineAsrProcess(OfflineAsrConfig config,
                                      NormalizationConfig normalization)
-    : config_(std::move(config)), normalizer_(normalization) {}
+    : config_(std::move(config)), normalization_(std::move(normalization)),
+      normalizer_(normalization_) {}
 
 bool OfflineAsrProcess::enabled() const noexcept { return config_.enabled; }
 
@@ -243,6 +245,12 @@ std::string OfflineAsrProcess::normalize_text(const std::string &text) {
   return normalizer_.normalize(text);
 }
 
+std::string OfflineAsrProcess::format_final_text(std::string text) const {
+  return normalization_.space_between_cjk_and_ascii
+             ? vocotype::common::space_between_cjk_and_ascii(std::move(text))
+             : text;
+}
+
 std::string OfflineAsrProcess::build_native_hotwords(const std::string &extra) {
   std::lock_guard lock(request_mutex_);
   return normalizer_.build_native_hotwords(extra);

src/core/src/text_normalizer.cpp

@@ -1,4 +1,5 @@
 #include "vocotype/core/text_normalizer.hpp"
+#include "vocotype/common/spacing.hpp"
 #include "vocotype/common/terms_yaml.hpp"
 
 #include <algorithm>
@@ -1756,14 +1757,20 @@ class TextNormalizer::Impl {
     if (terms.text.empty()) {
       return {};
     }
+    std::string result;
     if (!config_.enabled) {
-      return encode_utf8(terms.text);
-    }
-    const UString numeric =
-        normalize_chinese_numbers(terms.text, terms.protected_spans);
-    const TermRewriteResult restyled = rewrite_terms(lexicon, numeric);
-    return encode_utf8(
-        apply_written_style(restyled.text, config_, restyled.protected_spans));
+      result = encode_utf8(terms.text);
+    } else {
+      const UString numeric =
+          normalize_chinese_numbers(terms.text, terms.protected_spans);
+      const TermRewriteResult restyled = rewrite_terms(lexicon, numeric);
+      result = encode_utf8(
+          apply_written_style(restyled.text, config_, restyled.protected_spans));
+    }
+    return config_.space_between_cjk_and_ascii
+               ? vocotype::common::space_between_cjk_and_ascii(
+                     std::move(result))
+               : result;
   }
 
   std::string build_native_hotwords(const std::string &extra) {

src/core/src/transcription_tasks.cpp

@@ -362,7 +362,7 @@ void TranscriptionTaskManager::run_task(const std::shared_ptr<Task> &task,
     return;
   }
   cleanup.remove_now();
-  task->mark_final(polished.text, polished.reason);
+  task->mark_final(asr_.format_final_text(polished.text), polished.reason);
 }
 
 } // namespace vocotype::core

src/core/tests/core_tests.cpp

@@ -21,6 +21,7 @@
 #include "vocotype/common/posix.hpp"
 #include "vocotype/core/config.hpp"
 #include "vocotype/core/dispatcher.hpp"
+#include "vocotype/core/offline_asr.hpp"
 #include "vocotype/core/server.hpp"
 #include "vocotype/core/text_normalizer.hpp"
 #include "vocotype/core/voice_edit.hpp"
@@ -391,6 +392,19 @@ void test_text_normalizer() {
     require(no_itn.normalize("二百五十六台") == "二百五十六台",
             "normalization master switch was ignored");
 
+    vocotype::core::NormalizationConfig spaced;
+    spaced.space_between_cjk_and_ascii = true;
+    TextNormalizer spaced_style(spaced);
+    require(spaced_style.normalize("使用Claude Code处理2026年数据") ==
+                "使用 Claude Code 处理 2026 年数据",
+            "CJK and ASCII spacing style was not applied");
+    require(spaced_style.normalize("𠮷A") == "𠮷 A",
+            "supplementary-plane Han spacing was not applied");
+
+    vocotype::core::OfflineAsrProcess final_formatter({}, spaced);
+    require(final_formatter.format_final_text("使用Claude") == "使用 Claude",
+            "post-polish final spacing was not applied");
+
     {
       std::ofstream output(terms);
       output << R"(terms:
@@ -535,7 +549,8 @@ void test_config_merge() {
         {{"native_enabled", true},
          {"intra_op_num_threads", 4},
          {"hotword", "VoCoType"}}},
-       {"normalization", {{"compact_times", false}}},
+       {"normalization",
+        {{"compact_times", false}, {"space_between_cjk_and_ascii", true}}},
        {"asr_streaming",
         {{"enabled", true},
          {"intra_op_num_threads", 3},
@@ -556,6 +571,8 @@ void test_config_merge() {
           "normalization style override was lost");
   require(config.normalization.compact_dates,
           "normalization default was not preserved");
+  require(config.normalization.space_between_cjk_and_ascii,
+          "CJK and ASCII spacing override was lost");
   require(config.streaming_asr.enabled,
           "streaming ASR enabled override was lost");
   require(config.streaming_asr.threads == 3,

src/desktop/src/settings_main.cpp

@@ -109,6 +109,7 @@ struct SettingsWindow {
   GtkSwitch *compact_times = nullptr;
   GtkSwitch *compact_distances = nullptr;
   GtkSwitch *currency_symbols = nullptr;
+  GtkSwitch *space_between_cjk_and_ascii = nullptr;
   GtkEntry *itn_preview_input = nullptr;
   GtkLabel *itn_preview_output = nullptr;
   GtkComboBoxText *fcitx_panel_style = nullptr;
@@ -982,6 +983,8 @@ void save_config(SettingsWindow &window) {
       gtk_switch_get_active(window.compact_distances);
   normalization["currency_symbols"] =
       gtk_switch_get_active(window.currency_symbols);
+  normalization["space_between_cjk_and_ascii"] =
+      gtk_switch_get_active(window.space_between_cjk_and_ascii);
 
   auto &slm = window.config["slm"];
   slm["enabled"] = gtk_switch_get_active(window.slm_enabled);
@@ -1031,6 +1034,9 @@ void save_config(SettingsWindow &window) {
        gtk_switch_get_active(window.fcitx_block_composing) ? "True" : "False"},
       {"StripTrailingPeriodOnCommit",
        gtk_switch_get_active(window.fcitx_strip_period) ? "True" : "False"},
+      {"SpaceBetweenCjkAndAscii",
+       gtk_switch_get_active(window.space_between_cjk_and_ascii) ? "True"
+                                                                  : "False"},
   };
   if (hotkeys_changed) {
     fcitx_values.insert(
@@ -2524,6 +2530,11 @@ void populate_from_config(SettingsWindow &window) {
                         json_bool(normalization, "compact_distances", true));
   gtk_switch_set_active(window.currency_symbols,
                         json_bool(normalization, "currency_symbols", true));
+  gtk_switch_set_active(
+      window.space_between_cjk_and_ascii,
+      json_bool(normalization, "space_between_cjk_and_ascii",
+                config_value(fcitx_config_path(), "SpaceBetweenCjkAndAscii",
+                             "False") == "True"));
 
   const auto &slm = window.config["slm"];
   gtk_switch_set_active(window.slm_enabled, json_bool(slm, "enabled", false));
@@ -3109,6 +3120,7 @@ GtkWidget *build_recognition(SettingsWindow &window) {
   window.compact_times = GTK_SWITCH(sui::make_switch());
   window.compact_distances = GTK_SWITCH(sui::make_switch());
   window.currency_symbols = GTK_SWITCH(sui::make_switch());
+  window.space_between_cjk_and_ascii = GTK_SWITCH(sui::make_switch());
   gtk_box_pack_start(GTK_BOX(itn_card),
                      sui::make_row("启用数字与 ITN",
                                    "关闭后保留用户词典替换，但不改写中文数字。",
@@ -3132,6 +3144,12 @@ GtkWidget *build_recognition(SettingsWindow &window) {
                      sui::make_row("金额符号", "例如：一百二十八元 → ¥128",
                                    GTK_WIDGET(window.currency_symbols)),
                      FALSE, FALSE, 0);
+  gtk_box_pack_start(
+      GTK_BOX(itn_card),
+      sui::make_row("中文与英文/数字间空格",
+                    "在相邻中文与英文单词或阿拉伯数字之间加空格，例如“使用 Claude Code 处理 2026 年数据”。",
+                    GTK_WIDGET(window.space_between_cjk_and_ascii)),
+      FALSE, FALSE, 0);
   gtk_box_pack_start(GTK_BOX(page.content), itn_card, FALSE, FALSE, 0);
 
   GtkWidget *preview_card = sui::make_card();
@@ -3232,6 +3250,8 @@ GtkWidget *build_recognition(SettingsWindow &window) {
           {"compact_times", gtk_switch_get_active(self->compact_times)},
           {"compact_distances", gtk_switch_get_active(self->compact_distances)},
           {"currency_symbols", gtk_switch_get_active(self->currency_symbols)},
+          {"space_between_cjk_and_ascii",
+           gtk_switch_get_active(self->space_between_cjk_and_ascii)},
       };
       const Json result = vocotype::desktop::unix_json_request(
           vocotype::desktop::backend_socket_path(),

src/integrations/fcitx5/module/vocotype_module.cpp

@@ -1,5 +1,6 @@
 #include "vocotype_module.h"
 #include "vocotype/common/preview_text.hpp"
+#include "vocotype/common/spacing.hpp"
 #include "recorder_shutdown.hpp"
 #include "timer_lifetime.hpp"
 
@@ -2151,23 +2152,27 @@ void VoCoTypeModule::commitText(fcitx::InputContext *ic,
     }
   const std::string commit_text =
       strip_trailing_period ? stripTrailingCommitPeriod(text) : text;
+  const std::string formatted_text = config_.spaceBetweenCjkAndAscii.value()
+                                         ? vocotype::common::space_between_cjk_and_ascii(
+                                               commit_text)
+                                         : commit_text;
     const uint64_t now = fcitx::now(CLOCK_MONOTONIC);
     const std::string program = ic->program();
     const std::string frontend = ic->frontend() ? ic->frontend() : "";
-    if (last_committed_ic_ == ic && last_committed_text_ == commit_text &&
+    if (last_committed_ic_ == ic && last_committed_text_ == formatted_text &&
         last_committed_program_ == program &&
         last_committed_frontend_ == frontend && now >= last_commit_time_us_ &&
         now - last_commit_time_us_ < DUPLICATE_COMMIT_SUPPRESS_US) {
         return;
     }
 
     clearOwnedUI(ic);
-    ic->commitString(commit_text);
+    ic->commitString(formatted_text);
 
     last_committed_ic_ = ic;
     last_committed_program_ = program;
     last_committed_frontend_ = frontend;
-    last_committed_text_ = commit_text;
+    last_committed_text_ = formatted_text;
     last_commit_time_us_ = now;
 }
 

src/integrations/fcitx5/module/vocotype_module.h

@@ -84,6 +84,8 @@ FCITX_CONFIGURATION(
         this, "BlockWhenComposing", "存在未提交预编辑时禁止开始录音", true};
     fcitx::HiddenOption<bool> stripTrailingPeriodOnCommit{
         this, "StripTrailingPeriodOnCommit", "提交时移除尾部句号", false};
+    fcitx::HiddenOption<bool> spaceBetweenCjkAndAscii{
+        this, "SpaceBetweenCjkAndAscii", "中文与英文/数字之间加空格", false};
     fcitx::HiddenOption<std::string> panelStyle{
         this, "PanelStyle", "状态提示样式（minimal、ultra_minimal 或 animated）", "minimal"};);