From 98181ae3164c01f3e730c9da85b0b198ef61013f Mon Sep 17 00:00:00 2001 From: Oxydox <96330449+Oxydox@users.noreply.github.com> Date: Sat, 5 Sep 2026 22:50:22 +0700 Subject: [PATCH 01/11] Refactor VietnameseTelexProcessor for trailing consonant handling fix(telex): tone placement on ao/eo clusters, d-adjacency, closed-syllable vowel reach-through --- .../telex/VietnameseTelexProcessor.kt | 41 +++++++++++++------ 1 file changed, 28 insertions(+), 13 deletions(-) diff --git a/app/src/main/java/it/palsoftware/pastiera/inputmethod/telex/VietnameseTelexProcessor.kt b/app/src/main/java/it/palsoftware/pastiera/inputmethod/telex/VietnameseTelexProcessor.kt index e81d9112c..6bb50d976 100644 --- a/app/src/main/java/it/palsoftware/pastiera/inputmethod/telex/VietnameseTelexProcessor.kt +++ b/app/src/main/java/it/palsoftware/pastiera/inputmethod/telex/VietnameseTelexProcessor.kt @@ -79,30 +79,41 @@ internal object VietnameseTelexProcessor { val parts = Parts.fromChar(chars[idx]) val trailing = chars.subList(idx + 1, chars.size).joinToString("") val replacement = when (key) { + // For self-doubling shape keys (aa->â, ee->ê, oo->ô), only allow reaching back + // through a trailing consonant CODA if that coda is still "open" (i.e. we're not + // crossing a completed syllable boundary). We approximate this by requiring the + // trailing text be empty or made up entirely of vowels (e.g. "dau"+a->"dau"-with- + // circumflex, where trailing is the vowel "u"). A trailing consonant (e.g. "mam"+a) + // means an earlier syllable already closed, so the new keystroke starts a fresh one. 'a' -> when { - trailing.isNotEmpty() && !isValidVietnameseTail(trailing) -> null + trailing.isNotEmpty() && !trailing.all { Parts.fromChar(it).isVietnameseVowel() } -> null parts.isBase('a') && !parts.hasShape() -> parts.withShape(CIRCUMFLEX).toChar().toString() parts.isBase('a') && parts.hasShape(CIRCUMFLEX) -> "${caseOf(parts.base)}$keyChar" else -> null } 'e' -> when { - trailing.isNotEmpty() && !isValidVietnameseTail(trailing) -> null + trailing.isNotEmpty() && !trailing.all { Parts.fromChar(it).isVietnameseVowel() } -> null parts.isBase('e') && !parts.hasShape() -> parts.withShape(CIRCUMFLEX).toChar().toString() parts.isBase('e') && parts.hasShape(CIRCUMFLEX) -> "${caseOf(parts.base)}$keyChar" else -> null } 'o' -> when { - trailing.isNotEmpty() && !isValidVietnameseTail(trailing) -> null + trailing.isNotEmpty() && !trailing.all { Parts.fromChar(it).isVietnameseVowel() } -> null parts.isBase('o') && !parts.hasShape() -> parts.withShape(CIRCUMFLEX).toChar().toString() parts.isBase('o') && parts.hasShape(CIRCUMFLEX) -> "${caseOf(parts.base)}$keyChar" else -> null } - 'd' -> when (chars[idx]) { - 'd' -> "đ" - 'D' -> "Đ" - // For an already transformed đ/Đ, treat a new d/D as a literal append - // instead of toggling back, so users can continue typing the next letter. - 'đ', 'Đ' -> return syllable + keyChar + // 'd' doubling (dd->đ) requires the two d's to be strictly adjacent: no reaching + // back through any other letter at all (unlike a/e/o/w, there's no legitimate + // Vietnamese case where a 'd' modifier applies at a distance). A third 'd' press + // reverts đ fully back to the literal double letter, matching how the other shape + // keys escape (e.g. ô + o -> oo), confirmed against real device behavior. + 'd' -> when { + trailing.isNotEmpty() -> null + chars[idx] == 'd' -> "đ" + chars[idx] == 'D' -> "Đ" + chars[idx] == 'đ' -> "dd" + chars[idx] == 'Đ' -> "DD" else -> null } 'w' -> when { @@ -199,13 +210,17 @@ internal object VietnameseTelexProcessor { val prevPos = lastPos - 1 val prev = vowelParts.getOrNull(prevPos) - // "oa"/"oe" commonly take tone on 'o'. + // "oa"/"oe" take tone on 'o' only in an OPEN syllable (nothing after the vowel cluster, + // e.g. "hoa"->"hòa"). In a CLOSED syllable with a trailing coda (e.g. "toan"->"toán", + // "thoat"->"thoát"), the tone moves onto the second vowel instead. + val hasCoda = vowelIndices[lastPos] < chars.lastIndex if (prev != null && prev.base.lowercaseChar() == 'o' && last.base.lowercaseChar() in setOf('a', 'e')) { - return vowelIndices[prevPos] + return if (hasCoda) vowelIndices[lastPos] else vowelIndices[prevPos] } - // If the final vowel is a semivowel, prefer the previous vowel. - if (prev != null && last.base.lowercaseChar() in setOf('i', 'y', 'u')) { + // If the final vowel is a semivowel-like glide (i/y/u, and o as in "ao"/"eo"), prefer + // the previous vowel. + if (prev != null && last.base.lowercaseChar() in setOf('i', 'y', 'u', 'o')) { // Exception: "uy" usually carries tone on y. if (!(prev.base.lowercaseChar() == 'u' && last.base.lowercaseChar() == 'y')) { return vowelIndices[prevPos] From 807b783c40d0cfe0787886678fd4be034d4025e3 Mon Sep 17 00:00:00 2001 From: Oxydox <96330449+Oxydox@users.noreply.github.com> Date: Sat, 5 Sep 2026 23:23:01 +0700 Subject: [PATCH 02/11] Enhance VietnameseTelexProcessor for better conversion Refactor VietnameseTelexProcessor to handle non-Vietnamese syllables and improve tone application logic. --- .../telex/VietnameseTelexProcessor.kt | 170 ++++++++++++++++-- 1 file changed, 158 insertions(+), 12 deletions(-) diff --git a/app/src/main/java/it/palsoftware/pastiera/inputmethod/telex/VietnameseTelexProcessor.kt b/app/src/main/java/it/palsoftware/pastiera/inputmethod/telex/VietnameseTelexProcessor.kt index 6bb50d976..de00e154a 100644 --- a/app/src/main/java/it/palsoftware/pastiera/inputmethod/telex/VietnameseTelexProcessor.kt +++ b/app/src/main/java/it/palsoftware/pastiera/inputmethod/telex/VietnameseTelexProcessor.kt @@ -29,22 +29,55 @@ internal object VietnameseTelexProcessor { fun isActiveForLayout(layoutName: String?): Boolean = layoutName == VIETNAMESE_TELEX_LAYOUT_ID + // The one piece of state that survives across keystrokes: once a syllable has been + // confirmed non-Vietnamese (via a tone-cancel or a plausibility rollback), we remember its + // text and stop attempting any further Vietnamese conversion on it, until a new word + // starts. This is the minimal version of the "raw keystroke memory" a full rollback + // architecture needs -- not a full keystroke log, just a bail-out marker. + private var bailoutSyllable: String = "" + fun rewrite(textBeforeCursor: String, keyChar: Char): Rewrite? { + if (!keyChar.isLetter()) return null val lowerKey = keyChar.lowercaseChar() - if (lowerKey !in toneKeys && lowerKey !in shapeKeys) return null val syllableStart = findSyllableStart(textBeforeCursor) - if (syllableStart == textBeforeCursor.length) return null - + if (syllableStart == textBeforeCursor.length) { + bailoutSyllable = "" + return null + } val syllable = textBeforeCursor.substring(syllableStart) - val rewritten = when { - lowerKey == 'z' -> clearDiacritics(syllable) - lowerKey in toneByKey -> applyToneKey(syllable, keyChar) - else -> applyShapeKey(syllable, keyChar) - } ?: return null - - if (rewritten == syllable) return null - return Rewrite(replaceCount = syllable.length, replacement = rewritten) + + if (bailoutSyllable.isNotEmpty() && syllable.startsWith(bailoutSyllable)) { + return null // this word was already confirmed non-Vietnamese; stop converting it + } + bailoutSyllable = "" + + if ((lowerKey in toneKeys || lowerKey in shapeKeys) && (lowerKey == 'd' || establishedOnsetIsValid(syllable))) { + val rewritten = when { + lowerKey == 'z' -> clearDiacritics(syllable) + lowerKey in toneByKey -> applyToneKey(syllable, keyChar) + else -> applyShapeKey(syllable, keyChar) + } + if (rewritten != null && rewritten != syllable) { + return Rewrite(replaceCount = syllable.length, replacement = rewritten) + } + } + + // Nothing converted this keystroke (either it's a plain consonant, or a shape/tone key + // that didn't find anything to convert). Check whether accepting it literally still + // keeps this syllable a plausible, still-forming Vietnamese syllable. If not, this + // syllable was never really Vietnamese (e.g. an English word) -- roll back every + // conversion applied so far in it back to the raw keys that produced them. + return checkPlausibilityRollback(syllable, keyChar) + } + + private fun checkPlausibilityRollback(syllable: String, keyChar: Char): Rewrite? { + if (isPlausiblePrefix(syllable, keyChar)) return null + val rawSoFar = rawKeysForSyllable(syllable) + if (rawSoFar == syllable) return null + val result = rawSoFar + keyChar + bailoutSyllable = result + return Rewrite(replaceCount = syllable.length, replacement = result) } private fun findSyllableStart(text: String): Int { @@ -169,9 +202,24 @@ internal object VietnameseTelexProcessor { val targetIndex = findToneTargetIndex(chars) ?: return null val parts = Parts.fromChar(chars[targetIndex]) + // A DIFFERENT tone key trying to overwrite an already-applied tone, once the coda + // after it is already a complete, sealed Vietnamese final (t, p, m, c, ch, ng, nh), is + // not really re-toning this syllable -- it's a literal letter (e.g. "de"+s->"dé", then + // "t" completes the coda "det", then "r" should just be the letter r, not overwrite + // the tone). A FRESH tone application (no existing tone yet) is always still allowed, + // even after a sealed coda -- that's the normal, expected "tone typed last" usage + // (e.g. "thich"+s->"thích"). + if (parts.tone != null && parts.tone != toneMark) { + val consonantTail = syllable.substring(targetIndex + 1) + .dropWhile { Parts.fromChar(it).isVietnameseVowel() } + if (consonantTail.isNotEmpty() && isSealedTerminalCoda(consonantTail)) return null + } + return if (parts.tone == toneMark) { val cleared = parts.withTone(null).toChar() - syllable.substring(0, targetIndex) + cleared + syllable.substring(targetIndex + 1) + keyChar + val result = syllable.substring(0, targetIndex) + cleared + syllable.substring(targetIndex + 1) + keyChar + bailoutSyllable = result + result } else { val toned = parts.withTone(toneMark).toChar() syllable.substring(0, targetIndex) + toned + syllable.substring(targetIndex + 1) @@ -290,6 +338,104 @@ internal object VietnameseTelexProcessor { return isValidVietnameseCoda(trailing) } + // Codas that cannot grow into anything longer: once one of these is complete, the + // syllable is "sealed" and any further keystroke starts something new instead of + // continuing this syllable's coda. + private val sealedCodas = setOf("t", "p", "m", "c", "ch", "ng", "nh") + + private fun isSealedTerminalCoda(text: String): Boolean = text.lowercase() in sealedCodas + + // A coda-so-far is still plausible if it's already a complete valid final, OR if it's a + // prefix that could still grow into one (e.g. "n" -> "ng"/"nh"). + private fun isGrowableCodaPrefix(text: String): Boolean { + if (text.isEmpty()) return true + val t = text.lowercase() + return isValidVietnameseCoda(t) || setOf("c", "ch", "m", "n", "ng", "nh", "p", "t").any { it.startsWith(t) } + } + + // The (~25) legal Vietnamese syllable-initial consonant clusters. Anything else (pr, dr, + // str, sh, bl, fr...) never legitimately starts a Vietnamese syllable. + private val validOnsets = setOf( + "b", "c", "ch", "d", "đ", "g", "gh", "gi", "h", "k", "kh", "l", "m", "n", "ng", "ngh", + "nh", "p", "ph", "q", "r", "s", "t", "th", "tr", "v", "x" + ) + + private fun isValidOnsetPrefix(text: String): Boolean { + if (text.isEmpty()) return true + val t = text.lowercase() + return validOnsets.any { it == t || it.startsWith(t) } + } + + private fun codaOf(syllable: String): String { + val lastVowelIdx = syllable.indices.lastOrNull { Parts.fromChar(syllable[it]).isVietnameseVowel() } + ?: return "" + return syllable.substring(lastVowelIdx + 1) + } + + private fun isCompleteValidOnset(text: String): Boolean { + if (text.isEmpty()) return true // onsetless syllable (e.g. "anh", "em") is valid + return text.lowercase() in validOnsets + } + + private fun extractOnset(syllable: String): String { + val firstVowelIdx = syllable.indices.firstOrNull { Parts.fromChar(syllable[it]).isVietnameseVowel() } + ?: return syllable + return syllable.substring(0, firstVowelIdx) + } + + // Whether the syllable's already-typed onset (the consonants before its first vowel, if + // any) is still a legal Vietnamese onset. If a vowel has already appeared, the onset is + // "closed" and must be a COMPLETE valid onset; otherwise it just needs to still be a + // growable prefix of one. This runs before ANY shape/tone conversion is attempted -- an + // invalid onset (e.g. "w", "z", "str") means this syllable was never really Vietnamese, so + // we skip straight to plain literal insertion / rollback instead of converting anything. + private fun establishedOnsetIsValid(syllable: String): Boolean { + val onset = extractOnset(syllable) + val hasVowel = syllable.length > onset.length + return if (hasVowel) isCompleteValidOnset(onset) else isValidOnsetPrefix(onset) + } + + // Would accepting `extra` as the next character keep `syllable` a plausible, still-forming + // Vietnamese syllable? We validate the onset once it's about to close off (the first vowel + // arrives) and the coda as it grows -- we don't validate the vowel nucleus itself here. + private fun isPlausiblePrefix(syllable: String, extra: Char): Boolean { + val hasVowelAlready = syllable.any { Parts.fromChar(it).isVietnameseVowel() } + return if (Parts.fromChar(extra).isVietnameseVowel()) { + if (!hasVowelAlready) isCompleteValidOnset(syllable) else true + } else if (!hasVowelAlready) { + isValidOnsetPrefix(syllable + extra) + } else { + isGrowableCodaPrefix(codaOf(syllable) + extra) + } + } + + private val keyForTone: Map = toneByKey.entries.associate { (k, v) -> v to k } + + // Reconstructs the literal keys that would have produced a single rendered character, + // e.g. 'ă' -> "aw", 'á' -> "as", 'đ' -> "dd". Used only when rolling back a syllable that + // turned out not to be Vietnamese after all. + private fun rawKeysForChar(ch: Char): String { + if (ch == 'đ') return "dd" + if (ch == 'Đ') return "DD" + val parts = Parts.fromChar(ch) + val matchCase: (Char) -> Char = { m -> if (parts.base.isUpperCase()) m.uppercaseChar() else m } + val shapeKey = when (parts.shape) { + CIRCUMFLEX -> matchCase(parts.base.lowercaseChar()) + BREVE, HORN -> matchCase('w') + else -> null + } + val toneKeyChar = parts.tone?.let { keyForTone[it] }?.let(matchCase) + return buildString { + append(parts.base) + shapeKey?.let { append(it) } + toneKeyChar?.let { append(it) } + } + } + + private fun rawKeysForSyllable(syllable: String): String = buildString { + for (ch in syllable) append(rawKeysForChar(ch)) + } + private data class Parts( val base: Char, val shape: Char? = null, From e05bd3e9d2a3fcfcee751d8f721e73f9af7c5e04 Mon Sep 17 00:00:00 2001 From: Oxydox <96330449+Oxydox@users.noreply.github.com> Date: Sat, 5 Sep 2026 23:32:07 +0700 Subject: [PATCH 03/11] =?UTF-8?q?Fix=20test=20assertion=20for=20'=C4=91'?= =?UTF-8?q?=20to=20'dd'=20rewrite?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../pastiera/inputmethod/telex/VietnameseTelexProcessorTest.kt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/src/test/java/it/palsoftware/pastiera/inputmethod/telex/VietnameseTelexProcessorTest.kt b/app/src/test/java/it/palsoftware/pastiera/inputmethod/telex/VietnameseTelexProcessorTest.kt index a1ccd11da..c7eb6489b 100644 --- a/app/src/test/java/it/palsoftware/pastiera/inputmethod/telex/VietnameseTelexProcessorTest.kt +++ b/app/src/test/java/it/palsoftware/pastiera/inputmethod/telex/VietnameseTelexProcessorTest.kt @@ -14,7 +14,7 @@ class VietnameseTelexProcessorTest { assertRewrite("mo", 'w', "mơ") assertRewrite("tu", 'w', "tư") assertRewrite("d", 'd', "đ") - assertRewrite("đ", 'd', "đd") + assertRewrite("đ", 'd', "dd") } @Test From 8b1d22c558674e1caa85591c207475aebf83c2ee Mon Sep 17 00:00:00 2001 From: Oxydox <96330449+Oxydox@users.noreply.github.com> Date: Sun, 6 Sep 2026 01:03:47 +0700 Subject: [PATCH 04/11] Refactor accent detection in SuggestionEngine --- .../palsoftware/pastiera/core/suggestions/SuggestionEngine.kt | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/app/src/main/java/it/palsoftware/pastiera/core/suggestions/SuggestionEngine.kt b/app/src/main/java/it/palsoftware/pastiera/core/suggestions/SuggestionEngine.kt index 0083ca2a2..fcbdf1aa3 100644 --- a/app/src/main/java/it/palsoftware/pastiera/core/suggestions/SuggestionEngine.kt +++ b/app/src/main/java/it/palsoftware/pastiera/core/suggestions/SuggestionEngine.kt @@ -28,7 +28,6 @@ class SuggestionEngine( private val accentCache: MutableMap = mutableMapOf() private val tag = "SuggestionEngine" private val wordNormalizeCache: MutableMap = mutableMapOf() - private val accentChars = setOf('à', 'è', 'é', 'ì', 'ò', 'ó', 'ù', 'À', 'È', 'É', 'Ì', 'Ò', 'Ó', 'Ù') // Keyboard layout positions - built dynamically based on layout type private var keyboardPositions: Map> = buildKeyboardPositions("qwerty") @@ -526,7 +525,7 @@ class SuggestionEngine( } } - val hasAccent = entry.word.any { it in accentChars } + val hasAccent = stripAccents(entry.word) != entry.word val hasDigit = entry.word.any { it.isDigit() } val hasSymbol = entry.word.any { !it.isLetterOrDigit() && it != '\'' } val isSameBaseLetter = entry.word.equals(currentWord, ignoreCase = true) From 64da917160c24394f566facf5ea9abf464b610cb Mon Sep 17 00:00:00 2001 From: Oxydox <96330449+Oxydox@users.noreply.github.com> Date: Sun, 6 Sep 2026 01:04:22 +0700 Subject: [PATCH 05/11] Implement Vietnamese bigram seeding for suggestions Added a method to pre-populate Vietnamese bigrams for next-word suggestions, ensuring new installs have initial predictions. This method seeds common word pairs with a low baseline count to encourage learning from user input. --- .../core/suggestions/UserNGramStore.kt | 163 ++++++++++++++++++ 1 file changed, 163 insertions(+) diff --git a/app/src/main/java/it/palsoftware/pastiera/core/suggestions/UserNGramStore.kt b/app/src/main/java/it/palsoftware/pastiera/core/suggestions/UserNGramStore.kt index 8e8d3df23..6c029ae2a 100644 --- a/app/src/main/java/it/palsoftware/pastiera/core/suggestions/UserNGramStore.kt +++ b/app/src/main/java/it/palsoftware/pastiera/core/suggestions/UserNGramStore.kt @@ -43,6 +43,37 @@ class UserNGramStore(context: Context) : SQLiteOpenHelper( "CREATE INDEX ${TABLE_BIGRAMS}_lookup ON $TABLE_BIGRAMS " + "($COL_LOCALE, $COL_PREFIX, $COL_COUNT DESC, $COL_LAST_USED DESC)" ) + seedDefaultBigrams(db) + } + + /** + * Pre-populates a small set of very common Vietnamese word-pair predictions, so next-word + * suggestions aren't completely empty for a brand-new install. Seeded at a low baseline + * count (1) so genuinely learned usage (which increments on every real use) naturally + * overtakes it over time rather than permanently dominating. + */ + private fun seedDefaultBigrams(db: SQLiteDatabase) { + val nowMs = System.currentTimeMillis() + db.beginTransaction() + try { + for ((prefix, nextWord) in DEFAULT_VI_BIGRAMS) { + db.insertWithOnConflict( + TABLE_BIGRAMS, + null, + ContentValues().apply { + put(COL_LOCALE, "vi") + put(COL_PREFIX, prefix) + put(COL_NEXT_WORD, nextWord) + put(COL_COUNT, 1) + put(COL_LAST_USED, nowMs) + }, + SQLiteDatabase.CONFLICT_IGNORE + ) + } + db.setTransactionSuccessful() + } finally { + db.endTransaction() + } } override fun onUpgrade(db: SQLiteDatabase, oldVersion: Int, newVersion: Int) { @@ -144,5 +175,137 @@ class UserNGramStore(context: Context) : SQLiteOpenHelper( private const val COL_NEXT_WORD = "next_word" private const val COL_COUNT = "count" private const val COL_LAST_USED = "last_used" + + // prefix is the accent-stripped, lowercase normalized form of the previous word + // (matching NextWordPredictor.normalizedKey); next_word is the real display form. + private val DEFAULT_VI_BIGRAMS: List> = listOf( + "khong" to "biết", + "khong" to "có", + "khong" to "phải", + "khong" to "được", + "khong" to "thể", + "khong" to "sao", + "khong" to "muốn", + "khong" to "còn", + "khong" to "ai", + "khong" to "gì", + "khong" to "đâu", + "khong" to "hiểu", + "khong" to "thích", + "khong" to "bao giờ", + "khong" to "dám", + "toi" to "là", + "toi" to "có", + "toi" to "muốn", + "toi" to "nghĩ", + "toi" to "thích", + "toi" to "đi", + "toi" to "làm", + "toi" to "biết", + "toi" to "không", + "toi" to "sẽ", + "toi" to "đã", + "minh" to "là", + "minh" to "có", + "minh" to "muốn", + "minh" to "đi", + "minh" to "nghĩ", + "minh" to "không", + "ban" to "có", + "ban" to "là", + "ban" to "muốn", + "ban" to "đi", + "ban" to "làm", + "ban" to "ơi", + "anh" to "có", + "anh" to "là", + "anh" to "muốn", + "anh" to "đi", + "anh" to "ơi", + "chi" to "có", + "chi" to "là", + "chi" to "ơi", + "em" to "có", + "em" to "là", + "em" to "muốn", + "em" to "ơi", + "rat" to "vui", + "rat" to "tốt", + "rat" to "nhiều", + "rat" to "đẹp", + "rat" to "thích", + "rat" to "mệt", + "rat" to "tiếc", + "rat" to "khó", + "rat" to "quan trọng", + "kha" to "vui", + "kha" to "tốt", + "kha" to "nhiều", + "qua" to "nhiều", + "qua" to "tốt", + "qua" to "vui", + "co" to "thể", + "co" to "lẽ", + "co" to "người", + "co" to "một", + "co" to "nhiều", + "co" to "vẻ", + "co" to "khi", + "co" to "lúc", + "la" to "một", + "la" to "người", + "la" to "gì", + "la" to "ai", + "hom" to "nay", + "hom" to "qua", + "hom" to "sau", + "bay" to "giờ", + "va" to "tôi", + "va" to "anh", + "va" to "em", + "va" to "các", + "nhung" to "tôi", + "nhung" to "anh", + "nhung" to "không", + "nhung" to "mà", + "xin" to "chào", + "xin" to "lỗi", + "xin" to "cảm ơn", + "cam" to "ơn", + "cam" to "thấy", + "dang" to "làm", + "dang" to "đi", + "dang" to "học", + "se" to "có", + "se" to "là", + "se" to "đi", + "se" to "làm", + "se" to "không", + "da" to "có", + "da" to "là", + "da" to "đi", + "da" to "làm", + "da" to "xong", + "duoc" to "không", + "duoc" to "rồi", + "muon" to "đi", + "muon" to "làm", + "muon" to "biết", + "muon" to "nói", + "noi" to "chuyện", + "noi" to "gì", + "noi" to "với", + "lam" to "gì", + "lam" to "sao", + "lam" to "việc", + "di" to "đâu", + "di" to "học", + "di" to "làm", + "di" to "ngủ", + "an" to "cơm", + "an" to "sáng", + "an" to "trưa", + "an" to "tối" + ) } } From 5341cb583c2ed231c3a972b4476b8744aaffebbc Mon Sep 17 00:00:00 2001 From: Oxydox <96330449+Oxydox@users.noreply.github.com> Date: Sun, 6 Sep 2026 12:05:23 +0700 Subject: [PATCH 06/11] Refactor VietnameseTelexProcessor to manage tone cancellation Removed bailoutSyllable state management and replaced it with toneCancelledPrefix to handle tone cancellation more effectively. Updated logic in rewrite and applyToneKey methods to accommodate the new state. --- .../telex/VietnameseTelexProcessor.kt | 126 +++--------------- 1 file changed, 22 insertions(+), 104 deletions(-) diff --git a/app/src/main/java/it/palsoftware/pastiera/inputmethod/telex/VietnameseTelexProcessor.kt b/app/src/main/java/it/palsoftware/pastiera/inputmethod/telex/VietnameseTelexProcessor.kt index de00e154a..d2fea63b0 100644 --- a/app/src/main/java/it/palsoftware/pastiera/inputmethod/telex/VietnameseTelexProcessor.kt +++ b/app/src/main/java/it/palsoftware/pastiera/inputmethod/telex/VietnameseTelexProcessor.kt @@ -29,29 +29,28 @@ internal object VietnameseTelexProcessor { fun isActiveForLayout(layoutName: String?): Boolean = layoutName == VIETNAMESE_TELEX_LAYOUT_ID - // The one piece of state that survives across keystrokes: once a syllable has been - // confirmed non-Vietnamese (via a tone-cancel or a plausibility rollback), we remember its - // text and stop attempting any further Vietnamese conversion on it, until a new word - // starts. This is the minimal version of the "raw keystroke memory" a full rollback - // architecture needs -- not a full keystroke log, just a bail-out marker. - private var bailoutSyllable: String = "" - fun rewrite(textBeforeCursor: String, keyChar: Char): Rewrite? { if (!keyChar.isLetter()) return null val lowerKey = keyChar.lowercaseChar() val syllableStart = findSyllableStart(textBeforeCursor) if (syllableStart == textBeforeCursor.length) { - bailoutSyllable = "" + toneCancelledPrefix = "" return null } val syllable = textBeforeCursor.substring(syllableStart) - - if (bailoutSyllable.isNotEmpty() && syllable.startsWith(bailoutSyllable)) { - return null // this word was already confirmed non-Vietnamese; stop converting it + if (toneCancelledPrefix.isNotEmpty() && !syllable.startsWith(toneCancelledPrefix)) { + toneCancelledPrefix = "" } - bailoutSyllable = "" + // Only attempt a Vietnamese conversion while this syllable's onset (the consonants + // before its first vowel) is still a plausible Vietnamese onset. This is what protects + // words like "wolf"/"zoo" (invalid onset "w"/"z") from ever getting converted in the + // first place -- and because nothing gets converted for them, there's nothing to correct + // or roll back later. Once a syllable already contains a REAL conversion (a diacritic + // actually applied), we never undo it: a keystroke that doesn't fit just becomes a + // literal character appended after it, so an already-correct word is never corrupted + // (e.g. "biết" + n -> "biếtn", not a reversion to raw letters). if ((lowerKey in toneKeys || lowerKey in shapeKeys) && (lowerKey == 'd' || establishedOnsetIsValid(syllable))) { val rewritten = when { lowerKey == 'z' -> clearDiacritics(syllable) @@ -63,21 +62,7 @@ internal object VietnameseTelexProcessor { } } - // Nothing converted this keystroke (either it's a plain consonant, or a shape/tone key - // that didn't find anything to convert). Check whether accepting it literally still - // keeps this syllable a plausible, still-forming Vietnamese syllable. If not, this - // syllable was never really Vietnamese (e.g. an English word) -- roll back every - // conversion applied so far in it back to the raw keys that produced them. - return checkPlausibilityRollback(syllable, keyChar) - } - - private fun checkPlausibilityRollback(syllable: String, keyChar: Char): Rewrite? { - if (isPlausiblePrefix(syllable, keyChar)) return null - val rawSoFar = rawKeysForSyllable(syllable) - if (rawSoFar == syllable) return null - val result = rawSoFar + keyChar - bailoutSyllable = result - return Rewrite(replaceCount = syllable.length, replacement = result) + return null } private fun findSyllableStart(text: String): Int { @@ -194,7 +179,15 @@ internal object VietnameseTelexProcessor { return null } + // Once a tone is explicitly cancelled (same tone key pressed twice in a row), this syllable + // is remembered as "tone-cancelled" -- the person deliberately said "not that", so no + // further tone key (fresh or cycling) should touch it again, even though the visible text + // is now plain and indistinguishable from a vowel that never had a tone attempt at all. + // Cleared at the next word boundary. + private var toneCancelledPrefix: String = "" + private fun applyToneKey(syllable: String, keyChar: Char): String? { + if (toneCancelledPrefix.isNotEmpty() && syllable.startsWith(toneCancelledPrefix)) return null if (hasSeparatedVowelGroups(syllable)) return null val key = keyChar.lowercaseChar() val toneMark = toneByKey[key] ?: return null @@ -202,23 +195,10 @@ internal object VietnameseTelexProcessor { val targetIndex = findToneTargetIndex(chars) ?: return null val parts = Parts.fromChar(chars[targetIndex]) - // A DIFFERENT tone key trying to overwrite an already-applied tone, once the coda - // after it is already a complete, sealed Vietnamese final (t, p, m, c, ch, ng, nh), is - // not really re-toning this syllable -- it's a literal letter (e.g. "de"+s->"dé", then - // "t" completes the coda "det", then "r" should just be the letter r, not overwrite - // the tone). A FRESH tone application (no existing tone yet) is always still allowed, - // even after a sealed coda -- that's the normal, expected "tone typed last" usage - // (e.g. "thich"+s->"thích"). - if (parts.tone != null && parts.tone != toneMark) { - val consonantTail = syllable.substring(targetIndex + 1) - .dropWhile { Parts.fromChar(it).isVietnameseVowel() } - if (consonantTail.isNotEmpty() && isSealedTerminalCoda(consonantTail)) return null - } - return if (parts.tone == toneMark) { val cleared = parts.withTone(null).toChar() val result = syllable.substring(0, targetIndex) + cleared + syllable.substring(targetIndex + 1) + keyChar - bailoutSyllable = result + toneCancelledPrefix = result result } else { val toned = parts.withTone(toneMark).toChar() @@ -338,21 +318,6 @@ internal object VietnameseTelexProcessor { return isValidVietnameseCoda(trailing) } - // Codas that cannot grow into anything longer: once one of these is complete, the - // syllable is "sealed" and any further keystroke starts something new instead of - // continuing this syllable's coda. - private val sealedCodas = setOf("t", "p", "m", "c", "ch", "ng", "nh") - - private fun isSealedTerminalCoda(text: String): Boolean = text.lowercase() in sealedCodas - - // A coda-so-far is still plausible if it's already a complete valid final, OR if it's a - // prefix that could still grow into one (e.g. "n" -> "ng"/"nh"). - private fun isGrowableCodaPrefix(text: String): Boolean { - if (text.isEmpty()) return true - val t = text.lowercase() - return isValidVietnameseCoda(t) || setOf("c", "ch", "m", "n", "ng", "nh", "p", "t").any { it.startsWith(t) } - } - // The (~25) legal Vietnamese syllable-initial consonant clusters. Anything else (pr, dr, // str, sh, bl, fr...) never legitimately starts a Vietnamese syllable. private val validOnsets = setOf( @@ -366,12 +331,6 @@ internal object VietnameseTelexProcessor { return validOnsets.any { it == t || it.startsWith(t) } } - private fun codaOf(syllable: String): String { - val lastVowelIdx = syllable.indices.lastOrNull { Parts.fromChar(syllable[it]).isVietnameseVowel() } - ?: return "" - return syllable.substring(lastVowelIdx + 1) - } - private fun isCompleteValidOnset(text: String): Boolean { if (text.isEmpty()) return true // onsetless syllable (e.g. "anh", "em") is valid return text.lowercase() in validOnsets @@ -388,54 +347,13 @@ internal object VietnameseTelexProcessor { // "closed" and must be a COMPLETE valid onset; otherwise it just needs to still be a // growable prefix of one. This runs before ANY shape/tone conversion is attempted -- an // invalid onset (e.g. "w", "z", "str") means this syllable was never really Vietnamese, so - // we skip straight to plain literal insertion / rollback instead of converting anything. + // every subsequent letter just inserts literally instead of being converted. private fun establishedOnsetIsValid(syllable: String): Boolean { val onset = extractOnset(syllable) val hasVowel = syllable.length > onset.length return if (hasVowel) isCompleteValidOnset(onset) else isValidOnsetPrefix(onset) } - // Would accepting `extra` as the next character keep `syllable` a plausible, still-forming - // Vietnamese syllable? We validate the onset once it's about to close off (the first vowel - // arrives) and the coda as it grows -- we don't validate the vowel nucleus itself here. - private fun isPlausiblePrefix(syllable: String, extra: Char): Boolean { - val hasVowelAlready = syllable.any { Parts.fromChar(it).isVietnameseVowel() } - return if (Parts.fromChar(extra).isVietnameseVowel()) { - if (!hasVowelAlready) isCompleteValidOnset(syllable) else true - } else if (!hasVowelAlready) { - isValidOnsetPrefix(syllable + extra) - } else { - isGrowableCodaPrefix(codaOf(syllable) + extra) - } - } - - private val keyForTone: Map = toneByKey.entries.associate { (k, v) -> v to k } - - // Reconstructs the literal keys that would have produced a single rendered character, - // e.g. 'ă' -> "aw", 'á' -> "as", 'đ' -> "dd". Used only when rolling back a syllable that - // turned out not to be Vietnamese after all. - private fun rawKeysForChar(ch: Char): String { - if (ch == 'đ') return "dd" - if (ch == 'Đ') return "DD" - val parts = Parts.fromChar(ch) - val matchCase: (Char) -> Char = { m -> if (parts.base.isUpperCase()) m.uppercaseChar() else m } - val shapeKey = when (parts.shape) { - CIRCUMFLEX -> matchCase(parts.base.lowercaseChar()) - BREVE, HORN -> matchCase('w') - else -> null - } - val toneKeyChar = parts.tone?.let { keyForTone[it] }?.let(matchCase) - return buildString { - append(parts.base) - shapeKey?.let { append(it) } - toneKeyChar?.let { append(it) } - } - } - - private fun rawKeysForSyllable(syllable: String): String = buildString { - for (ch in syllable) append(rawKeysForChar(ch)) - } - private data class Parts( val base: Char, val shape: Char? = null, From 49a43cecadfcc60ffc22284df5e380bc9fcd920e Mon Sep 17 00:00:00 2001 From: Oxydox <96330449+Oxydox@users.noreply.github.com> Date: Thu, 10 Sep 2026 23:36:34 +0700 Subject: [PATCH 07/11] Refactor activity setup in CandidatesBarControllerTest test: fix window-visibility timing in CandidatesBarControllerTest setContentView() was called after the activity was already visible, so Robolectric never dispatched windowVisibility=VISIBLE to the new content view, failing the stricter isActuallyRendered() check added in the latest upstream release. Reordered to setContentView before the activity becomes visible, matching real Android lifecycle order. --- .../pastiera/inputmethod/CandidatesBarControllerTest.kt | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/app/src/test/java/it/palsoftware/pastiera/inputmethod/CandidatesBarControllerTest.kt b/app/src/test/java/it/palsoftware/pastiera/inputmethod/CandidatesBarControllerTest.kt index 46cb8ba06..5cf2e3802 100644 --- a/app/src/test/java/it/palsoftware/pastiera/inputmethod/CandidatesBarControllerTest.kt +++ b/app/src/test/java/it/palsoftware/pastiera/inputmethod/CandidatesBarControllerTest.kt @@ -53,17 +53,19 @@ class CandidatesBarControllerTest { @Test fun attachedAndLaidOutInputViewIsReportedAsRendered() { - val activity = Robolectric.buildActivity(Activity::class.java).setup().visible().get() + val activityController = Robolectric.buildActivity(Activity::class.java) + val activity = activityController.create().get() val controller = CandidatesBarController(activity) val inputView = controller.getInputView() activity.setContentView(inputView) + activityController.start().postCreate(null).resume().visible() val decorView = activity.window.decorView decorView.measure( View.MeasureSpec.makeMeasureSpec(1080, View.MeasureSpec.EXACTLY), View.MeasureSpec.makeMeasureSpec(2400, View.MeasureSpec.EXACTLY) ) decorView.layout(0, 0, 1080, 2400) - + assertTrue(controller.isInputViewActuallyRendered()) } From 1d4111b19742a856c79ffc5789683e2c2a9b8a27 Mon Sep 17 00:00:00 2001 From: Oxydox <96330449+Oxydox@users.noreply.github.com> Date: Thu, 10 Sep 2026 23:50:59 +0700 Subject: [PATCH 08/11] Fix input view rendering in CandidatesBarControllerTest Updated activity controller setup in test to ensure proper visibility and rendering of input view. --- .../inputmethod/CandidatesBarControllerTest.kt | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/app/src/test/java/it/palsoftware/pastiera/inputmethod/CandidatesBarControllerTest.kt b/app/src/test/java/it/palsoftware/pastiera/inputmethod/CandidatesBarControllerTest.kt index 5cf2e3802..0171b21e4 100644 --- a/app/src/test/java/it/palsoftware/pastiera/inputmethod/CandidatesBarControllerTest.kt +++ b/app/src/test/java/it/palsoftware/pastiera/inputmethod/CandidatesBarControllerTest.kt @@ -53,22 +53,26 @@ class CandidatesBarControllerTest { @Test fun attachedAndLaidOutInputViewIsReportedAsRendered() { - val activityController = Robolectric.buildActivity(Activity::class.java) - val activity = activityController.create().get() + val activity = Robolectric.buildActivity(Activity::class.java).setup().visible().get() val controller = CandidatesBarController(activity) val inputView = controller.getInputView() activity.setContentView(inputView) - activityController.start().postCreate(null).resume().visible() val decorView = activity.window.decorView decorView.measure( View.MeasureSpec.makeMeasureSpec(1080, View.MeasureSpec.EXACTLY), View.MeasureSpec.makeMeasureSpec(2400, View.MeasureSpec.EXACTLY) ) decorView.layout(0, 0, 1080, 2400) + // isActuallyRendered() now also requires windowVisibility == VISIBLE (added upstream). + // setContentView() swapped the view hierarchy in after the activity was already made + // visible, so the window-visibility dispatch that happened during .visible() never + // reached this new content view. Dispatch it explicitly so it propagates correctly + // to inputView and its descendants. + decorView.dispatchWindowVisibilityChanged(View.VISIBLE) assertTrue(controller.isInputViewActuallyRendered()) } - + @Test fun candidatesViewIsNotCollapsedByConfiguredSoftwareKeyboardMode() { SettingsManager.setSoftwareKeyboardMode( From 4aec915749da162780aa46b60fe69ae903c61950 Mon Sep 17 00:00:00 2001 From: Oxydox <96330449+Oxydox@users.noreply.github.com> Date: Thu, 10 Sep 2026 23:58:55 +0700 Subject: [PATCH 09/11] Refactor window visibility handling in test case Updated window visibility dispatch logic in test. --- .../inputmethod/CandidatesBarControllerTest.kt | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/app/src/test/java/it/palsoftware/pastiera/inputmethod/CandidatesBarControllerTest.kt b/app/src/test/java/it/palsoftware/pastiera/inputmethod/CandidatesBarControllerTest.kt index 0171b21e4..6d0c4d9b3 100644 --- a/app/src/test/java/it/palsoftware/pastiera/inputmethod/CandidatesBarControllerTest.kt +++ b/app/src/test/java/it/palsoftware/pastiera/inputmethod/CandidatesBarControllerTest.kt @@ -63,12 +63,15 @@ class CandidatesBarControllerTest { View.MeasureSpec.makeMeasureSpec(2400, View.MeasureSpec.EXACTLY) ) decorView.layout(0, 0, 1080, 2400) - // isActuallyRendered() now also requires windowVisibility == VISIBLE (added upstream). - // setContentView() swapped the view hierarchy in after the activity was already made - // visible, so the window-visibility dispatch that happened during .visible() never - // reached this new content view. Dispatch it explicitly so it propagates correctly - // to inputView and its descendants. + + // Dispatch window visibility to decorView first decorView.dispatchWindowVisibilityChanged(View.VISIBLE) + + // Then explicitly dispatch to inputView and its children to ensure propagation + inputView.dispatchWindowVisibilityChanged(View.VISIBLE) + + // Also ensure inputView is marked as visible + inputView.visibility = View.VISIBLE assertTrue(controller.isInputViewActuallyRendered()) } From d5dc3ef509626fcb26fc6ff7434f0a45d4e59ab2 Mon Sep 17 00:00:00 2001 From: Oxydox <96330449+Oxydox@users.noreply.github.com> Date: Fri, 11 Sep 2026 00:01:05 +0700 Subject: [PATCH 10/11] Refactor visibility dispatching in CandidatesBarControllerTest --- .../inputmethod/CandidatesBarControllerTest.kt | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/app/src/test/java/it/palsoftware/pastiera/inputmethod/CandidatesBarControllerTest.kt b/app/src/test/java/it/palsoftware/pastiera/inputmethod/CandidatesBarControllerTest.kt index 6d0c4d9b3..9b631887a 100644 --- a/app/src/test/java/it/palsoftware/pastiera/inputmethod/CandidatesBarControllerTest.kt +++ b/app/src/test/java/it/palsoftware/pastiera/inputmethod/CandidatesBarControllerTest.kt @@ -64,14 +64,18 @@ class CandidatesBarControllerTest { ) decorView.layout(0, 0, 1080, 2400) - // Dispatch window visibility to decorView first + // Dispatch window visibility to decorView decorView.dispatchWindowVisibilityChanged(View.VISIBLE) - // Then explicitly dispatch to inputView and its children to ensure propagation - inputView.dispatchWindowVisibilityChanged(View.VISIBLE) + // Measure and layout the inputView itself + inputView.measure( + View.MeasureSpec.makeMeasureSpec(1080, View.MeasureSpec.EXACTLY), + View.MeasureSpec.makeMeasureSpec(2400, View.MeasureSpec.UNSPECIFIED) + ) + inputView.layout(0, 0, inputView.measuredWidth, inputView.measuredHeight) - // Also ensure inputView is marked as visible - inputView.visibility = View.VISIBLE + // Dispatch visibility to inputView + inputView.dispatchWindowVisibilityChanged(View.VISIBLE) assertTrue(controller.isInputViewActuallyRendered()) } From d9e65781fc7ed370cb39ad4ab14e67ff7113cb1b Mon Sep 17 00:00:00 2001 From: Oxydox <96330449+Oxydox@users.noreply.github.com> Date: Fri, 11 Sep 2026 00:09:10 +0700 Subject: [PATCH 11/11] Set window to fit system windows in test Ensure window visibility is set for system-level rendering. --- .../pastiera/inputmethod/CandidatesBarControllerTest.kt | 3 +++ 1 file changed, 3 insertions(+) diff --git a/app/src/test/java/it/palsoftware/pastiera/inputmethod/CandidatesBarControllerTest.kt b/app/src/test/java/it/palsoftware/pastiera/inputmethod/CandidatesBarControllerTest.kt index 9b631887a..07cc31fa0 100644 --- a/app/src/test/java/it/palsoftware/pastiera/inputmethod/CandidatesBarControllerTest.kt +++ b/app/src/test/java/it/palsoftware/pastiera/inputmethod/CandidatesBarControllerTest.kt @@ -76,6 +76,9 @@ class CandidatesBarControllerTest { // Dispatch visibility to inputView inputView.dispatchWindowVisibilityChanged(View.VISIBLE) + + // Ensure the window is visible at the system level + activity.window.setDecorFitsSystemWindows(true) assertTrue(controller.isInputViewActuallyRendered()) }