From c76453a68a745082ee1c6a42c517b05e3e5e5766 Mon Sep 17 00:00:00 2001 From: Next Alone <12210746+NextAlone@users.noreply.github.com> Date: Sun, 8 Feb 2026 00:23:34 +0800 Subject: [PATCH 1/2] feat: pangu sending and receiving text for CJK Signed-off-by: Next Alone <12210746+NextAlone@users.noreply.github.com> --- Telegram/CMakeLists.txt | 2 + Telegram/Resources/langs/lang.strings | 2 + Telegram/SourceFiles/api/api_sending.cpp | 9 +- Telegram/SourceFiles/apiwrap.cpp | 4 + .../SourceFiles/core/enhanced_settings.cpp | 4 + Telegram/SourceFiles/core/pangu_spacing.cpp | 184 ++++++++++++++++++ Telegram/SourceFiles/core/pangu_spacing.h | 19 ++ Telegram/SourceFiles/history/history_item.cpp | 5 + .../settings/settings_enhanced.cpp | 29 ++- 9 files changed, 256 insertions(+), 2 deletions(-) create mode 100644 Telegram/SourceFiles/core/pangu_spacing.cpp create mode 100644 Telegram/SourceFiles/core/pangu_spacing.h diff --git a/Telegram/CMakeLists.txt b/Telegram/CMakeLists.txt index 2402ad43b6..92c170a563 100644 --- a/Telegram/CMakeLists.txt +++ b/Telegram/CMakeLists.txt @@ -497,6 +497,8 @@ PRIVATE core/deadlock_detector.h core/enhanced_settings.cpp core/enhanced_settings.h + core/pangu_spacing.cpp + core/pangu_spacing.h core/file_utilities.cpp core/file_utilities.h core/launcher.cpp diff --git a/Telegram/Resources/langs/lang.strings b/Telegram/Resources/langs/lang.strings index 349723bf60..c1ffabe7b1 100644 --- a/Telegram/Resources/langs/lang.strings +++ b/Telegram/Resources/langs/lang.strings @@ -7525,6 +7525,8 @@ https://github.com/telegramdesktop/tdesktop/blob/master/LEGAL "lng_settings_translate_to_tc" = "使用正體中文翻譯訊息"; "lng_settings_screen_shot_mode" = "Screenshot Mode"; "lng_settings_restart_hint" = "WARNING: Changing options that are colored in red will take effect only after the app restarts."; +"lng_settings_pangu_spacing_send" = "Auto-space CJK when sending"; +"lng_settings_pangu_spacing_receive" = "Auto-space CJK when displaying"; "lng_settings_skip_message" = "Skip to next unread chat"; "lng_settings_skip_message_desc" = "Note: Allow ALT+↑/↓ shortcut to skip to next unread chat"; diff --git a/Telegram/SourceFiles/api/api_sending.cpp b/Telegram/SourceFiles/api/api_sending.cpp index 0799707617..a53314586e 100644 --- a/Telegram/SourceFiles/api/api_sending.cpp +++ b/Telegram/SourceFiles/api/api_sending.cpp @@ -33,7 +33,8 @@ For license and copyright information please follow this link: #include "storage/file_upload.h" #include "mainwidget.h" #include "apiwrap.h" - +#include "core/pangu_spacing.h" +#include "settings.h" namespace Api { namespace { @@ -196,6 +197,9 @@ void SendExistingMedia( message.textWithTags.text, TextUtilities::ConvertTextTagsToEntities(message.textWithTags.tags) }; + if (GetEnhancedBool("pangu_spacing_send")) { + caption = PanguSpacing::SpacingTextWithEntities(caption); + } TextUtilities::Trim(caption); auto sentEntities = EntitiesToMTP( session, @@ -574,6 +578,9 @@ void SendConfirmedFile( file->caption.text, TextUtilities::ConvertTextTagsToEntities(file->caption.tags) }; + if (GetEnhancedBool("pangu_spacing_send")) { + caption = PanguSpacing::SpacingTextWithEntities(caption); + } const auto prepareFlags = Ui::ItemTextOptions( history, session->user()).flags; diff --git a/Telegram/SourceFiles/apiwrap.cpp b/Telegram/SourceFiles/apiwrap.cpp index f0ef07ad53..c272d22754 100644 --- a/Telegram/SourceFiles/apiwrap.cpp +++ b/Telegram/SourceFiles/apiwrap.cpp @@ -56,6 +56,7 @@ For license and copyright information please follow this link: #include "data/data_history_messages.h" #include "core/core_cloud_password.h" #include "core/application.h" +#include "core/pangu_spacing.h" #include "base/unixtime.h" #include "base/random.h" #include "base/call_delayed.h" @@ -4009,6 +4010,9 @@ void ApiWrap::sendMessage( textWithTags.text, TextUtilities::ConvertTextTagsToEntities(textWithTags.tags) }; + if (GetEnhancedBool("pangu_spacing_send")) { + left = PanguSpacing::SpacingTextWithEntities(left); + } auto prepareFlags = Ui::ItemTextOptions( history, _session->user()).flags; diff --git a/Telegram/SourceFiles/core/enhanced_settings.cpp b/Telegram/SourceFiles/core/enhanced_settings.cpp index 54a1163e58..13b8f85e1c 100644 --- a/Telegram/SourceFiles/core/enhanced_settings.cpp +++ b/Telegram/SourceFiles/core/enhanced_settings.cpp @@ -319,6 +319,8 @@ namespace EnhancedSettings { settings.insert(qsl("recent_display_limit"), 20); settings.insert(qsl("screenshot_mode"), false); settings.insert(qsl("update_url"), ""); + settings.insert(qsl("pangu_spacing_send"), false); + settings.insert(qsl("pangu_spacing_receive"), false); auto document = QJsonDocument(); document.setObject(settings); @@ -374,6 +376,8 @@ namespace EnhancedSettings { settings.insert(qsl("recent_display_limit"), GetEnhancedInt("recent_display_limit")); settings.insert(qsl("screenshot_mode"), GetEnhancedBool("screenshot_mode")); settings.insert(qsl("update_url"), GetEnhancedString("update_url")); + settings.insert(qsl("pangu_spacing_send"), GetEnhancedBool("pangu_spacing_send")); + settings.insert(qsl("pangu_spacing_receive"), GetEnhancedBool("pangu_spacing_receive")); auto document = QJsonDocument(); document.setObject(settings); diff --git a/Telegram/SourceFiles/core/pangu_spacing.cpp b/Telegram/SourceFiles/core/pangu_spacing.cpp new file mode 100644 index 0000000000..3e34eae0ef --- /dev/null +++ b/Telegram/SourceFiles/core/pangu_spacing.cpp @@ -0,0 +1,184 @@ +/* +This file is part of Yukigram Desktop, +the unofficial app based on Telegram Desktop. + +Pangu spacing algorithm ported from pangu.js (https://github.com/vinta/pangu.js) +Licensed under MIT License. +*/ +#include "core/pangu_spacing.h" + +#include + +namespace PanguSpacing { +namespace { + +bool IsCJK(uint ch) { + return (ch >= 0x2E80 && ch <= 0x2EFF) // CJK Radicals Supplement + || (ch >= 0x2F00 && ch <= 0x2FDF) // Kangxi Radicals + || (ch >= 0x3040 && ch <= 0x309F) // Hiragana + || (ch >= 0x30A0 && ch <= 0x30FF) // Katakana + || (ch >= 0x3100 && ch <= 0x312F) // Bopomofo + || (ch >= 0x3200 && ch <= 0x32FF) // Enclosed CJK Letters and Months + || (ch >= 0x3400 && ch <= 0x4DBF) // CJK Unified Ideographs Extension A + || (ch >= 0x4E00 && ch <= 0x9FFF) // CJK Unified Ideographs + || (ch >= 0xF900 && ch <= 0xFAFF) // CJK Compatibility Ideographs + || (ch >= 0xFE30 && ch <= 0xFE4F) // CJK Compatibility Forms + || (ch >= 0x1F200 && ch <= 0x1F2FF) // Enclosed Ideographic Supplement + || (ch >= 0x20000 && ch <= 0x2A6DF) // CJK Unified Ideographs Extension B + || (ch >= 0x2A700 && ch <= 0x2B73F) // CJK Unified Ideographs Extension C + || (ch >= 0x2B740 && ch <= 0x2B81F) // CJK Unified Ideographs Extension D + || (ch >= 0x2B820 && ch <= 0x2CEAF) // CJK Unified Ideographs Extension E + || (ch >= 0x2CEB0 && ch <= 0x2EBEF) // CJK Unified Ideographs Extension F + || (ch >= 0x30000 && ch <= 0x3134F); // CJK Unified Ideographs Extension G +} + +bool IsHalfWidthAlphaNumeric(uint ch) { + return (ch >= 'A' && ch <= 'Z') + || (ch >= 'a' && ch <= 'z') + || (ch >= '0' && ch <= '9'); +} + +// Get the Unicode code point at a given position in the string, +// handling surrogate pairs. Returns the code point and advances pos +// past the surrogate pair if applicable. +uint CodePointAt(const QString &text, int &pos) { + const auto high = text.at(pos).unicode(); + if (QChar::isHighSurrogate(high) && (pos + 1) < text.size()) { + const auto low = text.at(pos + 1).unicode(); + if (QChar::isLowSurrogate(low)) { + ++pos; // consume the low surrogate + return QChar::surrogateToUcs4(high, low); + } + } + return high; +} + +} // namespace + +QString SpacingText(const QString &text) { + if (text.isEmpty()) { + return text; + } + + auto result = QString(); + result.reserve(text.size() + text.size() / 4); + + uint prevCodePoint = 0; + auto hasPrev = false; + + auto i = 0; + while (i < text.size()) { + const auto startPos = i; + const auto codePoint = CodePointAt(text, i); + + if (hasPrev) { + const auto prevIsCJK = IsCJK(prevCodePoint); + const auto currIsCJK = IsCJK(codePoint); + const auto prevIsAN = IsHalfWidthAlphaNumeric(prevCodePoint); + const auto currIsAN = IsHalfWidthAlphaNumeric(codePoint); + + if ((prevIsCJK && currIsAN) || (prevIsAN && currIsCJK)) { + result.append(' '); + } + } + + // Append the current character(s). + if (codePoint > 0xFFFF) { + result.append(text.at(startPos)); + result.append(text.at(startPos + 1)); + } else { + result.append(text.at(startPos)); + } + + prevCodePoint = codePoint; + hasPrev = true; + ++i; + } + + return result; +} + +TextWithEntities SpacingTextWithEntities(const TextWithEntities &textWithEntities) { + if (textWithEntities.text.isEmpty()) { + return textWithEntities; + } + + const auto &originalText = textWithEntities.text; + const auto &originalEntities = textWithEntities.entities; + + // First pass: find all UTF-16 positions where a space should be inserted. + auto insertPositions = std::vector(); + + uint prevCodePoint = 0; + auto hasPrev = false; + + auto i = 0; + while (i < originalText.size()) { + const auto startPos = i; + const auto codePoint = CodePointAt(originalText, i); + + if (hasPrev) { + const auto prevIsCJK = IsCJK(prevCodePoint); + const auto currIsCJK = IsCJK(codePoint); + const auto prevIsAN = IsHalfWidthAlphaNumeric(prevCodePoint); + const auto currIsAN = IsHalfWidthAlphaNumeric(codePoint); + + if ((prevIsCJK && currIsAN) || (prevIsAN && currIsCJK)) { + insertPositions.push_back(startPos); + } + } + + prevCodePoint = codePoint; + hasPrev = true; + ++i; + } + + if (insertPositions.empty()) { + return textWithEntities; + } + + // Build the new text with spaces inserted. + auto newText = QString(); + newText.reserve(originalText.size() + int(insertPositions.size())); + + auto insertIdx = 0; + for (auto j = 0; j < originalText.size(); ++j) { + if (insertIdx < int(insertPositions.size()) + && j == insertPositions[insertIdx]) { + newText.append(' '); + ++insertIdx; + } + newText.append(originalText.at(j)); + } + + // Adjust entity offsets and lengths using binary search. + auto newEntities = EntitiesInText(); + newEntities.reserve(originalEntities.size()); + + const auto begin = insertPositions.begin(); + const auto end = insertPositions.end(); + + for (const auto &entity : originalEntities) { + const auto oldOffset = entity.offset(); + const auto oldEnd = oldOffset + entity.length(); + + // Spaces inserted before entity start shift the offset. + const auto spacesBeforeStart = int( + std::lower_bound(begin, end, oldOffset) - begin); + + // Spaces inserted within entity range expand the length. + const auto spacesBeforeEnd = int( + std::lower_bound(begin, end, oldEnd) - begin); + const auto spacesWithin = spacesBeforeEnd - spacesBeforeStart; + + newEntities.push_back(EntityInText( + entity.type(), + oldOffset + spacesBeforeStart, + entity.length() + spacesWithin, + entity.data())); + } + + return { newText, newEntities }; +} + +} // namespace PanguSpacing diff --git a/Telegram/SourceFiles/core/pangu_spacing.h b/Telegram/SourceFiles/core/pangu_spacing.h new file mode 100644 index 0000000000..bc641c21ff --- /dev/null +++ b/Telegram/SourceFiles/core/pangu_spacing.h @@ -0,0 +1,19 @@ +/* +This file is part of Yukigram Desktop, +the unofficial app based on Telegram Desktop. +For license and copyright information please follow this link: +https://github.com/nicennnnnnnlee/nicennnnnnnlee.github.io/blob/master/LICENSE +*/ +#pragma once + +#include "ui/text/text_entity.h" + +namespace PanguSpacing { + +// Apply pangu spacing to plain text (insert spaces between CJK and half-width characters). +[[nodiscard]] QString SpacingText(const QString &text); + +// Apply pangu spacing to TextWithEntities, adjusting entity offsets accordingly. +[[nodiscard]] TextWithEntities SpacingTextWithEntities(const TextWithEntities &textWithEntities); + +} // namespace PanguSpacing diff --git a/Telegram/SourceFiles/history/history_item.cpp b/Telegram/SourceFiles/history/history_item.cpp index e4e4de719b..5dad0af978 100644 --- a/Telegram/SourceFiles/history/history_item.cpp +++ b/Telegram/SourceFiles/history/history_item.cpp @@ -34,6 +34,7 @@ For license and copyright information please follow this link: #include "apiwrap.h" #include "media/audio/media_audio.h" #include "core/application.h" +#include "core/pangu_spacing.h" #include "window/window_controller.h" #include "window/window_session_controller.h" #include "core/click_handler_types.h" @@ -503,6 +504,10 @@ HistoryItem::HistoryItem( }; } + if (GetEnhancedBool("pangu_spacing_receive")) { + textWithEntities = + PanguSpacing::SpacingTextWithEntities(textWithEntities); + } setText(_media ? textWithEntities : EnsureNonEmpty(textWithEntities)); if (const auto groupedId = data.vgrouped_id()) { setGroupId( diff --git a/Telegram/SourceFiles/settings/settings_enhanced.cpp b/Telegram/SourceFiles/settings/settings_enhanced.cpp index 228dbcd253..de147b4a3c 100644 --- a/Telegram/SourceFiles/settings/settings_enhanced.cpp +++ b/Telegram/SourceFiles/settings/settings_enhanced.cpp @@ -214,6 +214,34 @@ namespace Settings { EnhancedSettings::Write(); }, container->lifetime()); + AddButtonWithIcon( + inner, + tr::lng_settings_pangu_spacing_send(), + st::settingsButtonNoIcon + )->toggleOn( + rpl::single(GetEnhancedBool("pangu_spacing_send")) + )->toggledChanges( + ) | rpl::filter([=](bool toggled) { + return (toggled != GetEnhancedBool("pangu_spacing_send")); + }) | rpl::on_next([=](bool toggled) { + SetEnhancedValue("pangu_spacing_send", toggled); + EnhancedSettings::Write(); + }, container->lifetime()); + + AddButtonWithIcon( + inner, + tr::lng_settings_pangu_spacing_receive(), + st::settingsButtonNoIcon + )->toggleOn( + rpl::single(GetEnhancedBool("pangu_spacing_receive")) + )->toggledChanges( + ) | rpl::filter([=](bool toggled) { + return (toggled != GetEnhancedBool("pangu_spacing_receive")); + }) | rpl::on_next([=](bool toggled) { + SetEnhancedValue("pangu_spacing_receive", toggled); + EnhancedSettings::Write(); + }, container->lifetime()); + AddSkip(container); AddButtonWithIcon( @@ -622,4 +650,3 @@ namespace Settings { Ui::ResizeFitChild(this, content); } } // namespace Settings - From 75e9ed58b106ae3adbfc6ed5b1a93a9ea2f98ec7 Mon Sep 17 00:00:00 2001 From: Next Alone <12210746+NextAlone@users.noreply.github.com> Date: Thu, 12 Feb 2026 14:30:20 +0800 Subject: [PATCH 2/2] Update Telegram/SourceFiles/core/pangu_spacing.h Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- Telegram/SourceFiles/core/pangu_spacing.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Telegram/SourceFiles/core/pangu_spacing.h b/Telegram/SourceFiles/core/pangu_spacing.h index bc641c21ff..f291fb3faa 100644 --- a/Telegram/SourceFiles/core/pangu_spacing.h +++ b/Telegram/SourceFiles/core/pangu_spacing.h @@ -10,10 +10,10 @@ For license and copyright information please follow this link: namespace PanguSpacing { -// Apply pangu spacing to plain text (insert spaces between CJK and half-width characters). +// Apply pangu spacing to plain text (insert spaces between CJK and ASCII alphanumeric characters [A-Za-z0-9]). [[nodiscard]] QString SpacingText(const QString &text); -// Apply pangu spacing to TextWithEntities, adjusting entity offsets accordingly. +// Apply pangu spacing to TextWithEntities (CJK vs. ASCII alphanumerics), adjusting entity offsets accordingly. [[nodiscard]] TextWithEntities SpacingTextWithEntities(const TextWithEntities &textWithEntities); } // namespace PanguSpacing