Skip to content

Fix Vietnamese Telex tone/shape-key bugs, correct accent detection for non-Italian locales, seed next-word predictions #297

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
Oxydox wants to merge 5 commits into from
Closed

Fix Vietnamese Telex tone/shape-key bugs, correct accent detection for non-Italian locales, seed next-word predictions #297

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
98181ae
Refactor VietnameseTelexProcessor for trailing consonant handling
Oxydox Sep 5, 2026
807b783
Enhance VietnameseTelexProcessor for better conversion
Oxydox Sep 5, 2026
e05bd3e
Fix test assertion for 'đ' to 'dd' rewrite
Oxydox Sep 5, 2026
8b1d22c
Refactor accent detection in SuggestionEngine
Oxydox Sep 5, 2026
64da917
Implement Vietnamese bigram seeding for suggestions
Oxydox Sep 5, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 1 addition & 2 deletions app/src/main/java/it/palsoftware/pastiera/core/suggestions/SuggestionEngine.kt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,6 @@ class SuggestionEngine(
private val accentCache: MutableMap<String, String> = mutableMapOf()
private val tag = "SuggestionEngine"
private val wordNormalizeCache: MutableMap<String, String> = mutableMapOf()
private val accentChars = setOf('à', 'è', 'é', 'ì', 'ò', 'ó', 'ù', 'À', 'È', 'É', 'Ì', 'Ò', 'Ó', 'Ù')

// Keyboard layout positions - built dynamically based on layout type
private var keyboardPositions: Map<Char, Pair<Int, Int>> = buildKeyboardPositions("qwerty")
Expand Down Expand Up @@ -526,7 +525,7 @@ class SuggestionEngine(
}
}

val hasAccent = entry.word.any { it in accentChars }
val hasAccent = stripAccents(entry.word) != entry.word
val hasDigit = entry.word.any { it.isDigit() }
val hasSymbol = entry.word.any { !it.isLetterOrDigit() && it != '\'' }
val isSameBaseLetter = entry.word.equals(currentWord, ignoreCase = true)
Expand Down
163 changes: 163 additions & 0 deletions app/src/main/java/it/palsoftware/pastiera/core/suggestions/UserNGramStore.kt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -43,6 +43,37 @@ class UserNGramStore(context: Context) : SQLiteOpenHelper(
"CREATE INDEX ${TABLE_BIGRAMS}_lookup ON $TABLE_BIGRAMS " +
"($COL_LOCALE, $COL_PREFIX, $COL_COUNT DESC, $COL_LAST_USED DESC)"
)
seedDefaultBigrams(db)
}

/**
* Pre-populates a small set of very common Vietnamese word-pair predictions, so next-word
* suggestions aren't completely empty for a brand-new install. Seeded at a low baseline
* count (1) so genuinely learned usage (which increments on every real use) naturally
* overtakes it over time rather than permanently dominating.
*/
private fun seedDefaultBigrams(db: SQLiteDatabase) {
val nowMs = System.currentTimeMillis()
db.beginTransaction()
try {
for ((prefix, nextWord) in DEFAULT_VI_BIGRAMS) {
db.insertWithOnConflict(
TABLE_BIGRAMS,
null,
ContentValues().apply {
put(COL_LOCALE, "vi")
put(COL_PREFIX, prefix)
put(COL_NEXT_WORD, nextWord)
put(COL_COUNT, 1)
put(COL_LAST_USED, nowMs)
},
SQLiteDatabase.CONFLICT_IGNORE
)
}
db.setTransactionSuccessful()
} finally {
db.endTransaction()
}
}

override fun onUpgrade(db: SQLiteDatabase, oldVersion: Int, newVersion: Int) {
Expand Down Expand Up @@ -144,5 +175,137 @@ class UserNGramStore(context: Context) : SQLiteOpenHelper(
private const val COL_NEXT_WORD = "next_word"
private const val COL_COUNT = "count"
private const val COL_LAST_USED = "last_used"

// prefix is the accent-stripped, lowercase normalized form of the previous word
// (matching NextWordPredictor.normalizedKey); next_word is the real display form.
private val DEFAULT_VI_BIGRAMS: List<Pair<String, String>> = listOf(
"khong" to "biết",
"khong" to "có",
"khong" to "phải",
"khong" to "được",
"khong" to "thể",
"khong" to "sao",
"khong" to "muốn",
"khong" to "còn",
"khong" to "ai",
"khong" to "gì",
"khong" to "đâu",
"khong" to "hiểu",
"khong" to "thích",
"khong" to "bao giờ",
"khong" to "dám",
"toi" to "là",
"toi" to "có",
"toi" to "muốn",
"toi" to "nghĩ",
"toi" to "thích",
"toi" to "đi",
"toi" to "làm",
"toi" to "biết",
"toi" to "không",
"toi" to "sẽ",
"toi" to "đã",
"minh" to "là",
"minh" to "có",
"minh" to "muốn",
"minh" to "đi",
"minh" to "nghĩ",
"minh" to "không",
"ban" to "có",
"ban" to "là",
"ban" to "muốn",
"ban" to "đi",
"ban" to "làm",
"ban" to "ơi",
"anh" to "có",
"anh" to "là",
"anh" to "muốn",
"anh" to "đi",
"anh" to "ơi",
"chi" to "có",
"chi" to "là",
"chi" to "ơi",
"em" to "có",
"em" to "là",
"em" to "muốn",
"em" to "ơi",
"rat" to "vui",
"rat" to "tốt",
"rat" to "nhiều",
"rat" to "đẹp",
"rat" to "thích",
"rat" to "mệt",
"rat" to "tiếc",
"rat" to "khó",
"rat" to "quan trọng",
"kha" to "vui",
"kha" to "tốt",
"kha" to "nhiều",
"qua" to "nhiều",
"qua" to "tốt",
"qua" to "vui",
"co" to "thể",
"co" to "lẽ",
"co" to "người",
"co" to "một",
"co" to "nhiều",
"co" to "vẻ",
"co" to "khi",
"co" to "lúc",
"la" to "một",
"la" to "người",
"la" to "gì",
"la" to "ai",
"hom" to "nay",
"hom" to "qua",
"hom" to "sau",
"bay" to "giờ",
"va" to "tôi",
"va" to "anh",
"va" to "em",
"va" to "các",
"nhung" to "tôi",
"nhung" to "anh",
"nhung" to "không",
"nhung" to "mà",
"xin" to "chào",
"xin" to "lỗi",
"xin" to "cảm ơn",
"cam" to "ơn",
"cam" to "thấy",
"dang" to "làm",
"dang" to "đi",
"dang" to "học",
"se" to "có",
"se" to "là",
"se" to "đi",
"se" to "làm",
"se" to "không",
"da" to "có",
"da" to "là",
"da" to "đi",
"da" to "làm",
"da" to "xong",
"duoc" to "không",
"duoc" to "rồi",
"muon" to "đi",
"muon" to "làm",
"muon" to "biết",
"muon" to "nói",
"noi" to "chuyện",
"noi" to "gì",
"noi" to "với",
"lam" to "gì",
"lam" to "sao",
"lam" to "việc",
"di" to "đâu",
"di" to "học",
"di" to "làm",
"di" to "ngủ",
"an" to "cơm",
"an" to "sáng",
"an" to "trưa",
"an" to "tối"
)
}
}
Loading