diff --git a/app/src/main/assets/common/predictions/en_next_words.json b/app/src/main/assets/common/predictions/en_next_words.json new file mode 100644 index 000000000..273a74f87 --- /dev/null +++ b/app/src/main/assets/common/predictions/en_next_words.json @@ -0,0 +1,129 @@ +{ + "fallback": [ + "the", + "to", + "and", + "you", + "it", + "that", + "for", + "with", + "this", + "have", + "your", + "from" + ], + "bigrams": { + "a": ["lot", "little", "few", "bit", "good", "great", "new"], + "about": ["the", "this", "that", "it", "what", "how"], + "all": ["the", "of", "you", "this", "that"], + "am": ["not", "going", "trying", "glad", "sure"], + "and": ["the", "i", "you", "we", "it", "then", "that"], + "are": ["you", "we", "they", "there", "not", "going"], + "as": ["a", "well", "soon", "long", "much", "if"], + "at": ["the", "all", "least", "first", "home", "work"], + "be": ["able", "there", "sure", "back", "careful", "good"], + "been": ["a", "there", "trying", "working", "doing"], + "but": ["i", "it", "you", "we", "the", "that"], + "by": ["the", "far", "then", "default", "my"], + "can": ["you", "we", "i", "do", "get", "see", "help"], + "could": ["you", "we", "be", "do", "get", "try"], + "did": ["you", "we", "they", "it", "not"], + "do": ["you", "we", "i", "not", "that", "this"], + "does": ["it", "that", "this", "not"], + "for": ["the", "you", "me", "this", "that", "now"], + "from": ["the", "you", "my", "there", "here"], + "get": ["the", "it", "back", "started", "to", "a"], + "good": ["luck", "idea", "morning", "night", "to"], + "got": ["it", "to", "the", "a", "some"], + "had": ["to", "a", "been", "no", "the"], + "has": ["been", "to", "a", "the", "no"], + "have": ["a", "to", "been", "the", "you", "we"], + "he": ["was", "is", "said", "has", "will"], + "hello": ["there", "everyone", "again"], + "here": ["is", "are", "we", "you", "it"], + "how": ["are", "do", "to", "can", "would", "much", "long"], + "i": ["am", "have", "think", "need", "want", "will", "can", "was"], + "if": ["you", "we", "it", "that", "there"], + "in": ["the", "a", "my", "this", "that"], + "is": ["the", "a", "it", "not", "there", "that"], + "it": ["is", "was", "would", "will", "has", "can", "should"], + "just": ["a", "the", "to", "let", "be", "wanted"], + "know": ["if", "what", "that", "how", "you"], + "let": ["me", "us", "it", "them"], + "like": ["to", "a", "the", "this", "that", "you"], + "me": ["know", "if", "when", "what", "and"], + "my": ["phone", "keyboard", "app", "friend", "device"], + "need": ["to", "a", "the", "help", "more"], + "new": ["one", "version", "build", "feature", "keyboard"], + "not": ["sure", "working", "bad", "the", "a"], + "of": ["the", "this", "that", "it", "course"], + "on": ["the", "my", "this", "that", "your"], + "or": ["the", "you", "we", "not", "maybe"], + "please": ["let", "check", "send", "help"], + "should": ["be", "have", "work", "not"], + "so": ["i", "you", "we", "it", "that"], + "thanks": ["for", "again", "so", "you"], + "that": ["is", "was", "would", "should", "you"], + "the": ["same", "best", "app", "keyboard", "phone", "issue", "other"], + "there": ["is", "are", "was", "will", "should"], + "they": ["are", "have", "will", "can", "were"], + "this": ["is", "was", "would", "should", "one"], + "to": ["the", "be", "do", "get", "make", "use", "see"], + "want": ["to", "a", "the", "it", "more"], + "was": ["the", "a", "not", "going", "just"], + "we": ["can", "should", "are", "need", "have", "will"], + "what": ["is", "do", "are", "you", "we"], + "when": ["you", "we", "it", "the", "i"], + "where": ["is", "are", "do", "can"], + "why": ["is", "does", "do", "are"], + "will": ["be", "have", "not", "the", "it"], + "with": ["the", "this", "that", "you", "a"], + "would": ["be", "like", "have", "you", "it"], + "you": ["are", "can", "should", "will", "have", "need", "know", "want"], + "your": ["phone", "keyboard", "app", "settings", "device"] + }, + "trigrams": { + "a lot": ["of"], + "all of": ["the", "this", "that"], + "can i": ["get", "use", "have", "see"], + "can you": ["help", "check", "send", "see", "try"], + "do you": ["want", "know", "think", "have", "need"], + "for the": ["same", "keyboard", "app", "phone"], + "how are": ["you", "we", "things"], + "how do": ["you", "we", "i"], + "i am": ["not", "going", "trying", "glad", "sure"], + "i can": ["do", "see", "try", "help"], + "i did": ["not", "see", "try"], + "i do": ["not", "think", "want"], + "i had": ["to", "a", "been"], + "i have": ["to", "a", "been", "no"], + "i just": ["wanted", "need", "have"], + "i know": ["that", "what", "you"], + "i need": ["to", "a", "the", "help"], + "i think": ["it", "we", "you", "that"], + "i want": ["to", "a", "the", "it"], + "i will": ["be", "have", "try", "check"], + "if you": ["want", "need", "can", "have"], + "in the": ["same", "app", "keyboard", "settings"], + "is the": ["same", "best", "keyboard", "app"], + "it is": ["not", "the", "a", "just"], + "let me": ["know", "check", "see", "try"], + "need to": ["be", "do", "get", "make"], + "one of": ["the", "my", "your"], + "please let": ["me"], + "so i": ["can", "have", "know"], + "thank you": ["for", "so", "again"], + "there is": ["a", "no", "the", "nothing"], + "this is": ["the", "a", "not", "what"], + "to the": ["same", "app", "keyboard"], + "want to": ["be", "do", "get", "make"], + "we are": ["not", "going", "trying", "ready"], + "we can": ["do", "try", "see", "make"], + "what do": ["you", "we", "i"], + "you are": ["the", "not", "going"], + "you can": ["do", "see", "try", "use"], + "you have": ["to", "a", "the"], + "you need": ["to", "a", "the"] + } +} diff --git a/app/src/main/java/it/palsoftware/pastiera/core/suggestions/BundledPhrasePredictor.kt b/app/src/main/java/it/palsoftware/pastiera/core/suggestions/BundledPhrasePredictor.kt new file mode 100644 index 000000000..28c4ad2ee --- /dev/null +++ b/app/src/main/java/it/palsoftware/pastiera/core/suggestions/BundledPhrasePredictor.kt @@ -0,0 +1,116 @@ +package it.palsoftware.pastiera.core.suggestions + +import android.content.Context +import org.json.JSONArray +import org.json.JSONObject +import java.util.Locale +import java.util.concurrent.ConcurrentHashMap + +class BundledPhrasePredictor( + context: Context, + private val locale: Locale = Locale.getDefault(), + private val isValidWord: ((String) -> Boolean)? = null +) { + private val appContext = context.applicationContext + + companion object { + private const val PREDICTION_ASSET_PREFIX = "common/predictions" + private val normalizeRegex = Regex("[^\\p{L}']") + private val cache = ConcurrentHashMap() + private val emptyPhraseData = PhraseData(emptyMap(), emptyMap(), emptyList()) + } + + fun predictContextual(previousWord: String, previousPreviousWord: String?, limit: Int): List { + if (limit <= 0) return emptyList() + val data = loadData() + if (data === emptyPhraseData) return emptyList() + + val previous = normalize(previousWord) ?: return emptyList() + val previousPrevious = normalize(previousPreviousWord) + val results = linkedSetOf() + + if (previousPrevious != null) { + data.trigrams["$previousPrevious $previous"] + ?.asSequence() + ?.filter(::isPlausibleWord) + ?.forEach(results::add) + } + data.bigrams[previous] + ?.asSequence() + ?.filter(::isPlausibleWord) + ?.forEach(results::add) + + return results.take(limit) + } + + fun fallbackWords(limit: Int): List { + if (limit <= 0) return emptyList() + return loadData().fallback + .asSequence() + .filter(::isPlausibleWord) + .take(limit) + .toList() + } + + private fun loadData(): PhraseData { + val language = locale.language.lowercase(Locale.ROOT) + if (language != "en") return emptyPhraseData + return cache.getOrPut(language) { + val path = "$PREDICTION_ASSET_PREFIX/${language}_next_words.json" + runCatching { + appContext.assets.open(path).bufferedReader().use { reader -> + val root = JSONObject(reader.readText()) + PhraseData( + bigrams = parseMap(root.optJSONObject("bigrams")), + trigrams = parseMap(root.optJSONObject("trigrams")), + fallback = parseArray(root.optJSONArray("fallback")) + ) + } + }.getOrElse { + emptyPhraseData + } + } + } + + private fun parseMap(source: JSONObject?): Map> { + if (source == null) return emptyMap() + return source.keys().asSequence().associateWith { key -> + parseArray(source.optJSONArray(key)) + } + } + + private fun parseArray(source: JSONArray?): List { + if (source == null) return emptyList() + return buildList { + for (index in 0 until source.length()) { + source.optString(index) + .takeIf { it.isNotBlank() } + ?.let(::add) + } + } + } + + private fun normalize(word: String?): String? { + if (word.isNullOrBlank()) return null + val normalized = word + .replace('’', '\'') + .replace('‘', '\'') + .replace('ʼ', '\'') + .lowercase(locale) + .replace(normalizeRegex, "") + .trim('\'') + return normalized.takeIf { it.isNotBlank() } + } + + private fun isPlausibleWord(word: String): Boolean { + val normalized = normalize(word) ?: return false + if (normalized.length < 2) return false + return isValidWord?.invoke(normalized) ?: true + } + + private data class PhraseData( + val bigrams: Map>, + val trigrams: Map>, + val fallback: List + ) +} diff --git a/app/src/main/java/it/palsoftware/pastiera/core/suggestions/SuggestionController.kt b/app/src/main/java/it/palsoftware/pastiera/core/suggestions/SuggestionController.kt index 660926787..731e7f8a7 100644 --- a/app/src/main/java/it/palsoftware/pastiera/core/suggestions/SuggestionController.kt +++ b/app/src/main/java/it/palsoftware/pastiera/core/suggestions/SuggestionController.kt @@ -60,6 +60,7 @@ class SuggestionController( ) private var autoReplaceController = createAutoReplaceController() private val nextWordPredictor = nextWordPredictorOverride ?: NextWordPredictor(UserNGramStore(appContext)) + private var bundledPhrasePredictor = buildBundledPhrasePredictor(currentLocale) private val extraSuggestionEngines = mutableMapOf() private data class SuggestionLanguageEngine( @@ -108,6 +109,14 @@ class SuggestionController( } ) } + + private fun buildBundledPhrasePredictor(locale: Locale): BundledPhrasePredictor { + return BundledPhrasePredictor( + context = appContext, + locale = locale, + isValidWord = { candidate -> isKnownWordInActiveDictionaries(candidate) } + ) + } /** * Updates the locale and reloads the dictionary for the new language. @@ -129,6 +138,7 @@ class SuggestionController( setKeyboardLayout(keyboardLayoutProvider()) } autoReplaceController = createAutoReplaceController() + bundledPhrasePredictor = buildBundledPhrasePredictor(currentLocale) extraSuggestionEngines.clear() // Recreate tracker to use new engine (tracker captures suggestionEngine in closure) @@ -150,6 +160,7 @@ class SuggestionController( // Reset tracker and clear suggestions previousCompletedWord = null + previousPreviousCompletedWord = null sentenceStartPending = true tracker.reset() suggestionsListener?.invoke(emptyList()) @@ -174,6 +185,7 @@ class SuggestionController( private val cursorDebounceMs = 120L private var pendingAddUserWord: String? = null private var previousCompletedWord: String? = null + private var previousPreviousCompletedWord: String? = null private var pendingInitialContextConnection: InputConnection? = null @Volatile private var pendingPrimaryRefreshAfterLoad: Boolean = false @Volatile private var pendingExtraRefreshAfterLoad: Boolean = false @@ -363,6 +375,7 @@ class SuggestionController( if (inputConnection == null) { tracker.reset() previousCompletedWord = null + previousPreviousCompletedWord = null sentenceStartPending = true suggestionsListener?.invoke(emptyList()) return @@ -371,6 +384,7 @@ class SuggestionController( if (!dictionaryRepository.isReady) { tracker.reset() previousCompletedWord = null + previousPreviousCompletedWord = null suggestionsListener?.invoke(emptyList()) return@Runnable } @@ -382,9 +396,10 @@ class SuggestionController( val previous = previousCompletedWord val lastChar = lastCharBeforeCursor(inputConnection) if (previous != null && isSoftPredictionBoundary(lastChar)) { - publishNextWordPredictions(previous) + publishNextWordPredictions(previous, previousPreviousCompletedWord) } else { previousCompletedWord = null + previousPreviousCompletedWord = null sentenceStartPending = true publishSentenceStartPredictionsOrStarter() } @@ -398,6 +413,7 @@ class SuggestionController( tracker.onContextChanged() pendingAddUserWord = null previousCompletedWord = null + previousPreviousCompletedWord = null sentenceStartPending = true suggestionsListener?.invoke(emptyList()) } @@ -406,6 +422,7 @@ class SuggestionController( if (!isEnabled()) return tracker.onContextChanged() previousCompletedWord = null + previousPreviousCompletedWord = null sentenceStartPending = true } @@ -456,6 +473,40 @@ class SuggestionController( fun currentSuggestions(): List = latestSuggestions.get() + fun onSuggestionAccepted(acceptedWord: String) { + if (!isEnabled()) return + tracker.reset() + pendingAddUserWord = null + + val settings = settingsProvider() + if (!settings.suggestionsEnabled) { + previousCompletedWord = null + previousPreviousCompletedWord = null + latestSuggestions.set(emptyList()) + suggestionsListener?.invoke(emptyList()) + return + } + + val cleanWord = acceptedWord.trim().takeIf { it.any { ch -> ch.isLetterOrDigit() } } + if (cleanWord == null) { + latestSuggestions.set(emptyList()) + suggestionsListener?.invoke(emptyList()) + return + } + + val previousWord = previousCompletedWord + if (sentenceStartPending) { + nextWordPredictor.learnSentenceStart(currentLocale, cleanWord) + } + previousWord?.let { previous -> + nextWordPredictor.learn(currentLocale, previous, cleanWord) + } + previousPreviousCompletedWord = previousWord + previousCompletedWord = cleanWord + sentenceStartPending = false + publishNextWordPredictions(cleanWord, previousWord) + } + fun userDictionarySnapshot(): List = userDictionaryStore.getSnapshot() fun dismissSuggestion(candidate: String, hardDeleteUserWord: Boolean = false) { @@ -536,6 +587,7 @@ class SuggestionController( internal fun clearLearnedNextWordsForTests() { nextWordPredictor.clearAll() previousCompletedWord = null + previousPreviousCompletedWord = null } internal fun flushNextWordLearningForTests() { @@ -553,6 +605,7 @@ class SuggestionController( val settings = settingsProvider() if (!settings.suggestionsEnabled) { previousCompletedWord = null + previousPreviousCompletedWord = null latestSuggestions.set(emptyList()) suggestionsListener?.invoke(emptyList()) return @@ -570,14 +623,16 @@ class SuggestionController( when { cleanWord != null && isSoftPredictionBoundary(boundaryChar) -> { + val previousWord = previousCompletedWord + previousPreviousCompletedWord = previousWord previousCompletedWord = cleanWord sentenceStartPending = false - publishNextWordPredictions(cleanWord) + publishNextWordPredictions(cleanWord, previousWord) } cleanWord == null && isSoftPredictionBoundary(boundaryChar) -> { val previous = previousCompletedWord if (previous != null) { - publishNextWordPredictions(previous) + publishNextWordPredictions(previous, previousPreviousCompletedWord) } else { latestSuggestions.set(emptyList()) suggestionsListener?.invoke(emptyList()) @@ -585,6 +640,7 @@ class SuggestionController( } else -> { previousCompletedWord = null + previousPreviousCompletedWord = null sentenceStartPending = true latestSuggestions.set(emptyList()) suggestionsListener?.invoke(emptyList()) @@ -592,7 +648,7 @@ class SuggestionController( } } - private fun publishNextWordPredictions(previousWord: String) { + private fun publishNextWordPredictions(previousWord: String, previousPreviousWord: String?) { val settings = settingsProvider() val primary = nextWordPredictor.predict( currentLocale, @@ -602,7 +658,14 @@ class SuggestionController( val extras = activeExtraLocales().flatMap { locale -> nextWordPredictor.predict(locale, previousWord, settings.maxSuggestions) } - val predictions = mergeSuggestionResults(primary, extras, settings.maxSuggestions) + val learnedPredictions = mergeSuggestionResults(primary, extras, settings.maxSuggestions) + val bundledPredictions = bundledNextWordPredictions( + previousWord = previousWord, + previousPreviousWord = previousPreviousWord, + existing = learnedPredictions, + limit = settings.maxSuggestions + ) + val predictions = mergeSuggestionResults(learnedPredictions, bundledPredictions, settings.maxSuggestions) val suggestions = fillWithStarterSuggestions(predictions, settings) if (suggestions.isNotEmpty()) { latestSuggestions.set(suggestions) @@ -612,6 +675,42 @@ class SuggestionController( } } + private fun bundledNextWordPredictions( + previousWord: String, + previousPreviousWord: String?, + existing: List, + limit: Int + ): List { + val remaining = limit - existing.size + if (remaining <= 0) return emptyList() + + val seen = existing.mapTo(HashSet()) { it.candidate.lowercase(currentLocale) } + val contextual = bundledPhrasePredictor + .predictContextual(previousWord, previousPreviousWord, remaining * 2) + val candidates = if (contextual.isNotEmpty()) { + contextual + } else { + bundledPhrasePredictor.fallbackWords(remaining * 2) + } + + return candidates.asSequence() + .filter { candidate -> + !candidate.equals(previousWord, ignoreCase = true) && + seen.add(candidate.lowercase(currentLocale)) + } + .mapIndexed { index, candidate -> + SuggestionResult( + candidate = candidate, + distance = 0, + score = (remaining - index).coerceAtLeast(1).toDouble(), + source = SuggestionSource.MAIN, + kind = SuggestionKind.NEXT_WORD + ) + } + .take(remaining) + .toList() + } + private fun publishSentenceStartPredictionsOrStarter() { val settings = settingsProvider() val primary = nextWordPredictor.predictSentenceStart(currentLocale, settings.maxSuggestions) diff --git a/app/src/main/java/it/palsoftware/pastiera/inputmethod/CandidatesBarController.kt b/app/src/main/java/it/palsoftware/pastiera/inputmethod/CandidatesBarController.kt index c4b1de95e..65e373289 100644 --- a/app/src/main/java/it/palsoftware/pastiera/inputmethod/CandidatesBarController.kt +++ b/app/src/main/java/it/palsoftware/pastiera/inputmethod/CandidatesBarController.kt @@ -56,7 +56,7 @@ class CandidatesBarController( candidatesStatusBar.onAddUserWordSubstitutionRequested = value } - var onSuggestionCommitted: (() -> Unit)? = null + var onSuggestionCommitted: ((String) -> Unit)? = null set(value) { field = value inputStatusBar.onSuggestionCommitted = value diff --git a/app/src/main/java/it/palsoftware/pastiera/inputmethod/PhysicalKeyboardInputMethodService.kt b/app/src/main/java/it/palsoftware/pastiera/inputmethod/PhysicalKeyboardInputMethodService.kt index 22e279a83..58f17cb21 100644 --- a/app/src/main/java/it/palsoftware/pastiera/inputmethod/PhysicalKeyboardInputMethodService.kt +++ b/app/src/main/java/it/palsoftware/pastiera/inputmethod/PhysicalKeyboardInputMethodService.kt @@ -1437,7 +1437,7 @@ class PhysicalKeyboardInputMethodService : InputMethodService() { candidatesBarController.onAddUserWordSubstitutionRequested = { word -> showAddSubstitutionDialog(word) } - candidatesBarController.onSuggestionCommitted = { + candidatesBarController.onSuggestionCommitted = { suggestion -> if (shiftLayerLatched || altLayerLatched) { shiftLayerLatched = false altLayerLatched = false @@ -1448,7 +1448,7 @@ class PhysicalKeyboardInputMethodService : InputMethodService() { modifierStateController.consumeShiftOneShot() } variationInteractedDuringHold = true - suggestionController.readInitialContext(currentInputConnection) + suggestionController.onSuggestionAccepted(suggestion) updateStatusBarText() } candidatesBarController.onHideSuggestion = { suggestion -> @@ -4888,9 +4888,7 @@ class PhysicalKeyboardInputMethodService : InputMethodService() { it.palsoftware.pastiera.core.AutoSpaceTracker.markAutoSpace() } - // CRITICAL FIX: Reset tracker after accepting suggestion to prevent duplicate letters - // The cursor debounce can cause tracker to be out of sync when user types quickly after accepting - suggestionController.onContextReset() + suggestionController.onSuggestionAccepted(suggestion) NotificationHelper.triggerHapticFeedback(this) Log.d(TAG, "Suggestion '$suggestion' inserted successfully") } diff --git a/app/src/main/java/it/palsoftware/pastiera/inputmethod/StatusBarController.kt b/app/src/main/java/it/palsoftware/pastiera/inputmethod/StatusBarController.kt index 6d5f7d2f5..3a9238a8a 100644 --- a/app/src/main/java/it/palsoftware/pastiera/inputmethod/StatusBarController.kt +++ b/app/src/main/java/it/palsoftware/pastiera/inputmethod/StatusBarController.kt @@ -102,7 +102,7 @@ class StatusBarController( variationBarView?.onAddUserWordSubstitutionRequested = value } - var onSuggestionCommitted: (() -> Unit)? = null + var onSuggestionCommitted: ((String) -> Unit)? = null var onHideSuggestion: ((String) -> Unit)? = null diff --git a/app/src/main/java/it/palsoftware/pastiera/inputmethod/suggestions/SuggestionButtonHandler.kt b/app/src/main/java/it/palsoftware/pastiera/inputmethod/suggestions/SuggestionButtonHandler.kt index fd9fa6ae4..94d9a0e5c 100644 --- a/app/src/main/java/it/palsoftware/pastiera/inputmethod/suggestions/SuggestionButtonHandler.kt +++ b/app/src/main/java/it/palsoftware/pastiera/inputmethod/suggestions/SuggestionButtonHandler.kt @@ -23,7 +23,7 @@ object SuggestionButtonHandler { inputConnection: InputConnection?, listener: VariationButtonHandler.OnVariationSelectedListener? = null, shouldDisableAutoCapitalize: Boolean, - onSuggestionCommitted: (() -> Unit)? = null + onSuggestionCommitted: ((String) -> Unit)? = null ): View.OnClickListener { return View.OnClickListener { Log.d(TAG, "Click on suggestion button: $suggestion") @@ -42,7 +42,7 @@ object SuggestionButtonHandler { val committed = replaceCurrentWord(inputConnection, suggestion, forceLeadingCapital) if (committed) { - onSuggestionCommitted?.invoke() + onSuggestionCommitted?.invoke(suggestion) } listener?.onVariationSelected(suggestion) } diff --git a/app/src/main/java/it/palsoftware/pastiera/inputmethod/suggestions/ui/FullSuggestionsBar.kt b/app/src/main/java/it/palsoftware/pastiera/inputmethod/suggestions/ui/FullSuggestionsBar.kt index c688c6f71..686953728 100644 --- a/app/src/main/java/it/palsoftware/pastiera/inputmethod/suggestions/ui/FullSuggestionsBar.kt +++ b/app/src/main/java/it/palsoftware/pastiera/inputmethod/suggestions/ui/FullSuggestionsBar.kt @@ -358,7 +358,7 @@ class FullSuggestionsBar( addWordCandidate: String?, onAddUserWord: ((String) -> Unit)?, onAddUserWordSubstitutionRequested: ((String) -> Unit)?, - onSuggestionCommitted: (() -> Unit)?, + onSuggestionCommitted: ((String) -> Unit)?, onHideSuggestion: ((String) -> Unit)?, onDeleteUserSuggestion: ((String) -> Unit)?, canDeleteUserSuggestion: ((String) -> Boolean)? @@ -609,7 +609,7 @@ class FullSuggestionsBar( addWordCandidate: String?, onAddUserWord: ((String) -> Unit)?, onAddUserWordSubstitutionRequested: ((String) -> Unit)?, - onSuggestionCommitted: (() -> Unit)?, + onSuggestionCommitted: ((String) -> Unit)?, onHideSuggestion: ((String) -> Unit)?, onDeleteUserSuggestion: ((String) -> Unit)?, canDeleteUserSuggestion: ((String) -> Boolean)?