feat: pangu sending and receiving text for CJK

Telegram/CMakeLists.txt

@@ -507,6 +507,8 @@ PRIVATE
     core/deadlock_detector.h
     core/enhanced_settings.cpp
     core/enhanced_settings.h
+    core/pangu_spacing.cpp
+    core/pangu_spacing.h
     core/deep_links/deep_links_chats.cpp
     core/deep_links/deep_links_chats.h
     core/deep_links/deep_links_contacts.cpp

Telegram/Resources/langs/lang.strings

@@ -7635,6 +7635,8 @@ https://github.com/telegramdesktop/tdesktop/blob/master/LEGAL
 "lng_settings_translate_to_tc" = "使用正體中文翻譯訊息";
 "lng_settings_screen_shot_mode" = "Screenshot Mode";
 "lng_settings_restart_hint" = "WARNING: Changing options that are colored in red will take effect only after the app restarts.";
+"lng_settings_pangu_spacing_send" = "Auto-space CJK when sending";
+"lng_settings_pangu_spacing_receive" = "Auto-space CJK when displaying";
 
 "lng_settings_skip_message" = "Skip to next unread chat";
 "lng_settings_skip_message_desc" = "Note: Allow ALT+↑/↓ shortcut to skip to next unread chat";

Telegram/SourceFiles/api/api_sending.cpp

@@ -33,7 +33,8 @@ For license and copyright information please follow this link:
 #include "storage/file_upload.h"
 #include "mainwidget.h"
 #include "apiwrap.h"
-
+#include "core/pangu_spacing.h"
+#include "settings.h"
 namespace Api {
 namespace {
 
@@ -196,6 +197,9 @@ void SendExistingMedia(
 		message.textWithTags.text,
 		TextUtilities::ConvertTextTagsToEntities(message.textWithTags.tags)
 	};
+	if (GetEnhancedBool("pangu_spacing_send")) {
+		caption = PanguSpacing::SpacingTextWithEntities(caption);
+	}
 	TextUtilities::Trim(caption);
 	auto sentEntities = EntitiesToMTP(
 		session,
@@ -574,6 +578,9 @@ void SendConfirmedFile(
 		file->caption.text,
 		TextUtilities::ConvertTextTagsToEntities(file->caption.tags)
 	};
+	if (GetEnhancedBool("pangu_spacing_send")) {
+		caption = PanguSpacing::SpacingTextWithEntities(caption);
+	}
 	const auto prepareFlags = Ui::ItemTextOptions(
 		history,
 		session->user()).flags;

Telegram/SourceFiles/apiwrap.cpp

@@ -56,6 +56,7 @@ For license and copyright information please follow this link:
 #include "data/data_history_messages.h"
 #include "core/core_cloud_password.h"
 #include "core/application.h"
+#include "core/pangu_spacing.h"
 #include "base/unixtime.h"
 #include "base/random.h"
 #include "base/call_delayed.h"
@@ -4009,6 +4010,9 @@ void ApiWrap::sendMessage(
 		textWithTags.text,
 		TextUtilities::ConvertTextTagsToEntities(textWithTags.tags)
 	};
+	if (GetEnhancedBool("pangu_spacing_send")) {
+		left = PanguSpacing::SpacingTextWithEntities(left);
+	}
 	auto prepareFlags = Ui::ItemTextOptions(
 		history,
 		_session->user()).flags;

Telegram/SourceFiles/core/enhanced_settings.cpp

@@ -319,6 +319,8 @@ namespace EnhancedSettings {
 		settings.insert(qsl("recent_display_limit"), 20);
 		settings.insert(qsl("screenshot_mode"), false);
 		settings.insert(qsl("update_url"), "");
+		settings.insert(qsl("pangu_spacing_send"), false);
+		settings.insert(qsl("pangu_spacing_receive"), false);
 
 		auto document = QJsonDocument();
 		document.setObject(settings);
@@ -374,6 +376,8 @@ namespace EnhancedSettings {
 		settings.insert(qsl("recent_display_limit"), GetEnhancedInt("recent_display_limit"));
 		settings.insert(qsl("screenshot_mode"), GetEnhancedBool("screenshot_mode"));
 		settings.insert(qsl("update_url"), GetEnhancedString("update_url"));
+		settings.insert(qsl("pangu_spacing_send"), GetEnhancedBool("pangu_spacing_send"));
+		settings.insert(qsl("pangu_spacing_receive"), GetEnhancedBool("pangu_spacing_receive"));
 
 		auto document = QJsonDocument();
 		document.setObject(settings);

Telegram/SourceFiles/core/pangu_spacing.cpp

@@ -0,0 +1,184 @@
+/*
+This file is part of Yukigram Desktop,
+the unofficial app based on Telegram Desktop.
+
+Pangu spacing algorithm ported from pangu.js (https://github.com/vinta/pangu.js)
+Licensed under MIT License.
+*/
+#include "core/pangu_spacing.h"
+
+#include <algorithm>
+
+namespace PanguSpacing {
+namespace {
+
+bool IsCJK(uint ch) {
+	return (ch >= 0x2E80 && ch <= 0x2EFF)   // CJK Radicals Supplement
+		|| (ch >= 0x2F00 && ch <= 0x2FDF)    // Kangxi Radicals
+		|| (ch >= 0x3040 && ch <= 0x309F)    // Hiragana
+		|| (ch >= 0x30A0 && ch <= 0x30FF)    // Katakana
+		|| (ch >= 0x3100 && ch <= 0x312F)    // Bopomofo
+		|| (ch >= 0x3200 && ch <= 0x32FF)    // Enclosed CJK Letters and Months
+		|| (ch >= 0x3400 && ch <= 0x4DBF)    // CJK Unified Ideographs Extension A
+		|| (ch >= 0x4E00 && ch <= 0x9FFF)    // CJK Unified Ideographs
+		|| (ch >= 0xF900 && ch <= 0xFAFF)    // CJK Compatibility Ideographs
+		|| (ch >= 0xFE30 && ch <= 0xFE4F)    // CJK Compatibility Forms
+		|| (ch >= 0x1F200 && ch <= 0x1F2FF)  // Enclosed Ideographic Supplement
+		|| (ch >= 0x20000 && ch <= 0x2A6DF)  // CJK Unified Ideographs Extension B
+		|| (ch >= 0x2A700 && ch <= 0x2B73F)  // CJK Unified Ideographs Extension C
+		|| (ch >= 0x2B740 && ch <= 0x2B81F)  // CJK Unified Ideographs Extension D
+		|| (ch >= 0x2B820 && ch <= 0x2CEAF)  // CJK Unified Ideographs Extension E
+		|| (ch >= 0x2CEB0 && ch <= 0x2EBEF)  // CJK Unified Ideographs Extension F
+		|| (ch >= 0x30000 && ch <= 0x3134F); // CJK Unified Ideographs Extension G
+}
+
+bool IsHalfWidthAlphaNumeric(uint ch) {
+	return (ch >= 'A' && ch <= 'Z')
+		|| (ch >= 'a' && ch <= 'z')
+		|| (ch >= '0' && ch <= '9');
+}
+
+// Get the Unicode code point at a given position in the string,
+// handling surrogate pairs. Returns the code point and advances pos
+// past the surrogate pair if applicable.
+uint CodePointAt(const QString &text, int &pos) {
+	const auto high = text.at(pos).unicode();
+	if (QChar::isHighSurrogate(high) && (pos + 1) < text.size()) {
+		const auto low = text.at(pos + 1).unicode();
+		if (QChar::isLowSurrogate(low)) {
+			++pos; // consume the low surrogate
+			return QChar::surrogateToUcs4(high, low);
+		}
+	}
+	return high;
+}
+
+} // namespace
+
+QString SpacingText(const QString &text) {
+	if (text.isEmpty()) {
+		return text;
+	}
+
+	auto result = QString();
+	result.reserve(text.size() + text.size() / 4);
+
+	uint prevCodePoint = 0;
+	auto hasPrev = false;
+
+	auto i = 0;
+	while (i < text.size()) {
+		const auto startPos = i;
+		const auto codePoint = CodePointAt(text, i);
+
+		if (hasPrev) {
+			const auto prevIsCJK = IsCJK(prevCodePoint);
+			const auto currIsCJK = IsCJK(codePoint);
+			const auto prevIsAN = IsHalfWidthAlphaNumeric(prevCodePoint);
+			const auto currIsAN = IsHalfWidthAlphaNumeric(codePoint);
+
+			if ((prevIsCJK && currIsAN) || (prevIsAN && currIsCJK)) {
+				result.append(' ');
+			}
+		}
+
+		// Append the current character(s).
+		if (codePoint > 0xFFFF) {
+			result.append(text.at(startPos));
+			result.append(text.at(startPos + 1));
+		} else {
+			result.append(text.at(startPos));
+		}
+
+		prevCodePoint = codePoint;
+		hasPrev = true;
+		++i;
+	}
+
+	return result;
+}
+
+TextWithEntities SpacingTextWithEntities(const TextWithEntities &textWithEntities) {
+	if (textWithEntities.text.isEmpty()) {
+		return textWithEntities;
+	}
+
+	const auto &originalText = textWithEntities.text;
+	const auto &originalEntities = textWithEntities.entities;
+
+	// First pass: find all UTF-16 positions where a space should be inserted.
+	auto insertPositions = std::vector<int>();
+
+	uint prevCodePoint = 0;
+	auto hasPrev = false;
+
+	auto i = 0;
+	while (i < originalText.size()) {
+		const auto startPos = i;
+		const auto codePoint = CodePointAt(originalText, i);
+
+		if (hasPrev) {
+			const auto prevIsCJK = IsCJK(prevCodePoint);
+			const auto currIsCJK = IsCJK(codePoint);
+			const auto prevIsAN = IsHalfWidthAlphaNumeric(prevCodePoint);
+			const auto currIsAN = IsHalfWidthAlphaNumeric(codePoint);
+
+			if ((prevIsCJK && currIsAN) || (prevIsAN && currIsCJK)) {
+				insertPositions.push_back(startPos);
+			}
+		}
+
+		prevCodePoint = codePoint;
+		hasPrev = true;
+		++i;
+	}
+
+	if (insertPositions.empty()) {
+		return textWithEntities;
+	}
+
+	// Build the new text with spaces inserted.
+	auto newText = QString();
+	newText.reserve(originalText.size() + int(insertPositions.size()));
+
+	auto insertIdx = 0;
+	for (auto j = 0; j < originalText.size(); ++j) {
+		if (insertIdx < int(insertPositions.size())
+			&& j == insertPositions[insertIdx]) {
+			newText.append(' ');
+			++insertIdx;
+		}
+		newText.append(originalText.at(j));
+	}
+
+	// Adjust entity offsets and lengths using binary search.
+	auto newEntities = EntitiesInText();
+	newEntities.reserve(originalEntities.size());
+
+	const auto begin = insertPositions.begin();
+	const auto end = insertPositions.end();
+
+	for (const auto &entity : originalEntities) {
+		const auto oldOffset = entity.offset();
+		const auto oldEnd = oldOffset + entity.length();
+
+		// Spaces inserted before entity start shift the offset.
+		const auto spacesBeforeStart = int(
+			std::lower_bound(begin, end, oldOffset) - begin);
+
+		// Spaces inserted within entity range expand the length.
+		const auto spacesBeforeEnd = int(
+			std::lower_bound(begin, end, oldEnd) - begin);
+		const auto spacesWithin = spacesBeforeEnd - spacesBeforeStart;
+
+		newEntities.push_back(EntityInText(
+			entity.type(),
+			oldOffset + spacesBeforeStart,
+			entity.length() + spacesWithin,
+			entity.data()));
+	}
+
+	return { newText, newEntities };
+}
+
+} // namespace PanguSpacing

Telegram/SourceFiles/core/pangu_spacing.h

@@ -0,0 +1,19 @@
+/*
+This file is part of Yukigram Desktop,
+the unofficial app based on Telegram Desktop.
+For license and copyright information please follow this link:
+https://github.com/nicennnnnnnlee/nicennnnnnnlee.github.io/blob/master/LICENSE
+*/
+#pragma once
+
+#include "ui/text/text_entity.h"
+
+namespace PanguSpacing {
+
+// Apply pangu spacing to plain text (insert spaces between CJK and ASCII alphanumeric characters [A-Za-z0-9]).
+[[nodiscard]] QString SpacingText(const QString &text);
+
+// Apply pangu spacing to TextWithEntities (CJK vs. ASCII alphanumerics), adjusting entity offsets accordingly.
+[[nodiscard]] TextWithEntities SpacingTextWithEntities(const TextWithEntities &textWithEntities);
+
+} // namespace PanguSpacing

Telegram/SourceFiles/history/history_item.cpp

@@ -34,6 +34,7 @@ For license and copyright information please follow this link:
 #include "apiwrap.h"
 #include "media/audio/media_audio.h"
 #include "core/application.h"
+#include "core/pangu_spacing.h"
 #include "window/window_controller.h"
 #include "window/window_session_controller.h"
 #include "core/click_handler_types.h"
@@ -503,6 +504,10 @@ HistoryItem::HistoryItem(
 			};
 		}
 
+		if (GetEnhancedBool("pangu_spacing_receive")) {
+			textWithEntities =
+					PanguSpacing::SpacingTextWithEntities(textWithEntities);
+		}
 		setText(_media ? textWithEntities : EnsureNonEmpty(textWithEntities));
 		if (const auto groupedId = data.vgrouped_id()) {
 			setGroupId(

Telegram/SourceFiles/settings/settings_enhanced.cpp

@@ -213,6 +213,34 @@ namespace Settings {
 			EnhancedSettings::Write();
 		}, container->lifetime());
 
+		AddButtonWithIcon(
+				inner,
+				tr::lng_settings_pangu_spacing_send(),
+				st::settingsButtonNoIcon
+		)->toggleOn(
+				rpl::single(GetEnhancedBool("pangu_spacing_send"))
+		)->toggledChanges(
+		) | rpl::filter([=](bool toggled) {
+			return (toggled != GetEnhancedBool("pangu_spacing_send"));
+		}) | rpl::on_next([=](bool toggled) {
+			SetEnhancedValue("pangu_spacing_send", toggled);
+			EnhancedSettings::Write();
+		}, container->lifetime());
+
+		AddButtonWithIcon(
+				inner,
+				tr::lng_settings_pangu_spacing_receive(),
+				st::settingsButtonNoIcon
+		)->toggleOn(
+				rpl::single(GetEnhancedBool("pangu_spacing_receive"))
+		)->toggledChanges(
+		) | rpl::filter([=](bool toggled) {
+			return (toggled != GetEnhancedBool("pangu_spacing_receive"));
+		}) | rpl::on_next([=](bool toggled) {
+			SetEnhancedValue("pangu_spacing_receive", toggled);
+			EnhancedSettings::Write();
+		}, container->lifetime());
+
 		AddSkip(container);
 
 		AddButtonWithIcon(
@@ -621,4 +649,3 @@ namespace Settings {
 		Ui::ResizeFitChild(this, content);
 	}
 } // namespace Settings
-