From eb8680a79b68a666e41fcaa6b5af85bf4ad0a38a Mon Sep 17 00:00:00 2001 From: Wells Riley <884715+wr@users.noreply.github.com> Date: Tue, 4 Aug 2026 15:17:25 -0400 Subject: [PATCH 1/5] W-569: stem query suffixes so inflected words reach their keywords MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `:ghosted:` returned nothing while `:ghosting:` returned 👻. fzy rejects a needle longer than its haystack, so the match died before scoring — no amount of tuning reached it. Strip common English inflections and retry, but only when the query as typed found nothing. That gating is what makes the rules safe to keep crude: a wrong guess leaves an empty picker empty, and a query that already works keeps its exact ranking and pays nothing. Candidates go out most-likely-first, so in practice an inflected query costs one extra corpus scan, not one per rule. Refs: W-569 --- Mojito.xcodeproj/project.pbxproj | 8 +++ Sources/Mojito/EmojiDB/FuzzyMatcher.swift | 20 +++++- Sources/Mojito/EmojiDB/QueryStemmer.swift | 74 +++++++++++++++++++++ Tests/MojitoTests/FuzzyMatcherTests.swift | 23 ++++++- Tests/MojitoTests/QueryStemmerTests.swift | 81 +++++++++++++++++++++++ 5 files changed, 204 insertions(+), 2 deletions(-) create mode 100644 Sources/Mojito/EmojiDB/QueryStemmer.swift create mode 100644 Tests/MojitoTests/QueryStemmerTests.swift diff --git a/Mojito.xcodeproj/project.pbxproj b/Mojito.xcodeproj/project.pbxproj index 9cee684..af6b488 100644 --- a/Mojito.xcodeproj/project.pbxproj +++ b/Mojito.xcodeproj/project.pbxproj @@ -110,6 +110,7 @@ 9F45015A816323AF275F23ED /* DebugReport.swift in Sources */ = {isa = PBXBuildFile; fileRef = 5C9E01A82260E8A38DF8D7EA /* DebugReport.swift */; }; 9F550EAD8B9A90F2874E8F22 /* TelemetryUploader.swift in Sources */ = {isa = PBXBuildFile; fileRef = AEA05D792327D50DA2FCF35B /* TelemetryUploader.swift */; }; 9FE6DD159CABAE6FAC74E6BD /* TriggerConfigStoreTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = F335AAB3008F02F987CB8DC2 /* TriggerConfigStoreTests.swift */; }; + A11509496D8522DC13BF31F3 /* QueryStemmerTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = 6401923F7190AFC65A7B6959 /* QueryStemmerTests.swift */; }; A1C87132DCD9B263CA32BAAE /* WordleSounds.swift in Sources */ = {isa = PBXBuildFile; fileRef = 31F82574CC8A6D051CF87D9B /* WordleSounds.swift */; }; A637DCA2905238217427E806 /* EmojiBrowserCatalog.swift in Sources */ = {isa = PBXBuildFile; fileRef = 96C001583E27CC71D9959055 /* EmojiBrowserCatalog.swift */; }; A9A15B339C05195BB45CC09F /* MyLegSound.swift in Sources */ = {isa = PBXBuildFile; fileRef = D2C4C16E01184684E33C3437 /* MyLegSound.swift */; }; @@ -161,6 +162,7 @@ E539CB425388907B1CA7D3CE /* GifSearcher.swift in Sources */ = {isa = PBXBuildFile; fileRef = 1C3CAFD7B83168FF01538FF4 /* GifSearcher.swift */; }; E5C85B09298DAB6F6594B181 /* BouncingDVD.swift in Sources */ = {isa = PBXBuildFile; fileRef = 0836B5ACF7E24509928DA7B0 /* BouncingDVD.swift */; }; E62A74A5CA784B452A59CE00 /* DiscoveryNotifier.swift in Sources */ = {isa = PBXBuildFile; fileRef = E6F0EF28CC67A6AB9FFF203F /* DiscoveryNotifier.swift */; }; + E8715EE20B3652E19AEB690D /* QueryStemmer.swift in Sources */ = {isa = PBXBuildFile; fileRef = 276275D749C5A73951269AF7 /* QueryStemmer.swift */; }; E9340C650BD416393A44ACC3 /* DockIconManager.swift in Sources */ = {isa = PBXBuildFile; fileRef = C02F1996A3DED7A0A133E24C /* DockIconManager.swift */; }; EAF2B984187615EBAF67954E /* DiskOptimizer.swift in Sources */ = {isa = PBXBuildFile; fileRef = CDDFF6254C8A58E63AC223D5 /* DiskOptimizer.swift */; }; ECC3936D06DC519BCE249F8D /* PermissionsCoordinator.swift in Sources */ = {isa = PBXBuildFile; fileRef = 2027256AFD59D65C41FAA2AD /* PermissionsCoordinator.swift */; }; @@ -219,6 +221,7 @@ 21B80C892257DEA88108EE93 /* TriggerConfig.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = TriggerConfig.swift; sourceTree = ""; }; 22B06D1C751C2161A7A99DA1 /* AppDelegate.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = AppDelegate.swift; sourceTree = ""; }; 2361376F2801DB575F27B480 /* DialupSound.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = DialupSound.swift; sourceTree = ""; }; + 276275D749C5A73951269AF7 /* QueryStemmer.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = QueryStemmer.swift; sourceTree = ""; }; 2928BC07BE26CF9B15F23D70 /* DebugRecorder.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = DebugRecorder.swift; sourceTree = ""; }; 2C4CF18CE5AC46767EFCA557 /* v03.bin */ = {isa = PBXFileReference; lastKnownFileType = archive.macbinary; path = v03.bin; sourceTree = ""; }; 2C66984ECCB4DB6B1A018452 /* PrefsKey.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = PrefsKey.swift; sourceTree = ""; }; @@ -253,6 +256,7 @@ 5C9E01A82260E8A38DF8D7EA /* DebugReport.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = DebugReport.swift; sourceTree = ""; }; 5FB7AB3DB0E63AF30880FE66 /* BrowserURL.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = BrowserURL.swift; sourceTree = ""; }; 625433055302CF9B9053F885 /* ExclusionStoreTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = ExclusionStoreTests.swift; sourceTree = ""; }; + 6401923F7190AFC65A7B6959 /* QueryStemmerTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = QueryStemmerTests.swift; sourceTree = ""; }; 6666150B793721D5095E72A8 /* SymbolsDatabase.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = SymbolsDatabase.swift; sourceTree = ""; }; 66C166F241AD0718D2067AF1 /* SystemEmojiPicker.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = SystemEmojiPicker.swift; sourceTree = ""; }; 682479ADBA3E221825BADC83 /* TriggersSettings.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = TriggersSettings.swift; sourceTree = ""; }; @@ -404,6 +408,7 @@ 42FA18E36A4027D4CB96ADA5 /* FuzzyMatcherTests.swift */, 69CD38087938C1E258718C69 /* FzyScorerTests.swift */, 12F71260B63547A6DF1B01AD /* PickerViewModelTests.swift */, + 6401923F7190AFC65A7B6959 /* QueryStemmerTests.swift */, 9E0C646543028E968FB6287E /* SeasonalGatesTests.swift */, D74BDC060E2EAACF7A71A3A6 /* SkinToneTests.swift */, E25EC50F4D58E03E218947DF /* SymbolsDatabaseTests.swift */, @@ -705,6 +710,7 @@ 6E2009C3B36DF9228B873244 /* EmoticonTable.swift */, 5A8851906565F70B7815640F /* FuzzyMatcher.swift */, FE5EDBB8F234B21B8D4A3324 /* FzyScorer.swift */, + 276275D749C5A73951269AF7 /* QueryStemmer.swift */, 964D290D66BB33D1F21333A0 /* QuickAccessStore.swift */, F5C5BC5F015CABDD85262258 /* SeasonalGates.swift */, B375DE27AC0AE1D8DE965403 /* SkinTone.swift */, @@ -1007,6 +1013,7 @@ ABF54441D03140C8174E13A9 /* PrefsKey.swift in Sources */, BA163217ADDEBAF751D5ABA2 /* PrideWave.swift in Sources */, CEAF9DB89294B0652A8A16F7 /* PrivacyPermissionsSettings.swift in Sources */, + E8715EE20B3652E19AEB690D /* QueryStemmer.swift in Sources */, D4E80136FD448E4ACC27EC58 /* QuickAccessSettings.swift in Sources */, 1A80F47103BB59733D6292B7 /* QuickAccessStore.swift in Sources */, FD15928FE17F78588821135E /* Rickroll.swift in Sources */, @@ -1067,6 +1074,7 @@ 4F9E4BA7A275583B07011809 /* FuzzyMatcherTests.swift in Sources */, B32AE99482DA334BDD7CEEAC /* FzyScorerTests.swift in Sources */, 2C2FD54F6E35CCD0374213BB /* PickerViewModelTests.swift in Sources */, + A11509496D8522DC13BF31F3 /* QueryStemmerTests.swift in Sources */, C025CD4AE30320C3FE69028A /* SeasonalGatesTests.swift in Sources */, 917B8512B30F3B6E8C9D2B8C /* SkinToneTests.swift in Sources */, 91A5F2A800EA1FC0700394EA /* SymbolsDatabaseTests.swift in Sources */, diff --git a/Sources/Mojito/EmojiDB/FuzzyMatcher.swift b/Sources/Mojito/EmojiDB/FuzzyMatcher.swift index ce3c275..fad0316 100644 --- a/Sources/Mojito/EmojiDB/FuzzyMatcher.swift +++ b/Sources/Mojito/EmojiDB/FuzzyMatcher.swift @@ -181,7 +181,7 @@ struct FuzzyMatcher { pool = SymbolsCorpus.entries } - let trimmed = rankedResults( + var trimmed = rankedResults( needle: needle, pool: pool, usage: usage, @@ -190,6 +190,24 @@ struct FuzzyMatcher { limit: limit ) + // Only when the query as typed found nothing: fzy rejects a needle + // longer than its haystack, so `:ghosted:` can't reach the `ghost` + // keyword until the suffix comes off. Gated on empty so a query that + // already works keeps its exact ranking and pays nothing. + if trimmed.isEmpty { + for stem in QueryStemmer.stems(of: needle) { + trimmed = rankedResults( + needle: stem, + pool: pool, + usage: usage, + useFrequencyBoost: useFrequencyBoost, + scanTags: stem.count >= tagMinNeedle, + limit: limit + ) + if !trimmed.isEmpty { break } + } + } + let lowercased = query.lowercased() // Discovery hint sits just below the top real match — a normal diff --git a/Sources/Mojito/EmojiDB/QueryStemmer.swift b/Sources/Mojito/EmojiDB/QueryStemmer.swift new file mode 100644 index 0000000..aebf24b --- /dev/null +++ b/Sources/Mojito/EmojiDB/QueryStemmer.swift @@ -0,0 +1,74 @@ +import Foundation + +/// Strips common English inflections so an inflected query can reach an +/// uninflected keyword. +/// +/// `FzyScorer` rejects a needle longer than its haystack, so `:ghosted:` can +/// never match the `ghost` / `ghosting` keywords no matter how scoring is +/// tuned — the match dies before the DP runs. Trimming the suffix is the only +/// way through. +/// +/// Deliberately cruder than a real stemmer (Porter, Snowball). It runs only as +/// a fallback when the raw query found nothing, so a wrong guess costs an empty +/// picker staying empty — never a worse result for a query that already works. +/// That asymmetry is what lets the rules be this loose. +enum QueryStemmer { + + /// Below this a stem is too generic to mean anything. + private static let minStem = 3 + + /// Candidate stems for `needle`, most likely first; empty when no rule + /// applies. Callers try each in order and stop at the first with results. + /// + /// Several spellings are offered at once on purpose: `-ing` alone can't + /// tell `shipping` (undo a doubled consonant) from `celebrating` (restore a + /// dropped `e`) from `blocking` (plain trim), so all three go out and the + /// corpus decides which one exists. + static func stems(of needle: [Character]) -> [[Character]] { + var out: [[Character]] = [] + + func add(_ candidate: [Character]) { + guard candidate.count >= minStem, !out.contains(candidate) else { return } + out.append(candidate) + } + + /// `shipping` → `shipp` → `ship`. Doubled consonants only, so `pressed` + /// yields `press` before the lossy `pres`. + func undoubled(_ stem: [Character]) -> [Character]? { + guard let last = stem.last, + stem.count > minStem, + last == stem[stem.count - 2], + !"aeiou".contains(last) + else { return nil } + return Array(stem.dropLast()) + } + + func hasSuffix(_ chars: [Character]) -> Bool { + needle.count > chars.count && needle.suffix(chars.count).elementsEqual(chars) + } + + func trim(_ suffix: String, restoringE: Bool = false) { + let chars = Array(suffix) + guard hasSuffix(chars) else { return } + let stem = Array(needle.dropLast(chars.count)) + add(stem) + // `celebrat` is nothing; `celebrate` is the word. + if restoringE { add(stem + ["e"]) } + if let shorter = undoubled(stem) { add(shorter) } + } + + // `parties` → `party`, `carried` → `carry`. Checked before the generic + // `-es` / `-ed` rules, which would leave the meaningless `parti`. + for suffix in ["ies", "ied"] where hasSuffix(Array(suffix)) { + add(Array(needle.dropLast(3)) + ["y"]) + } + + trim("ing", restoringE: true) + trim("ed", restoringE: true) + trim("es") + // `press` is not `pres`. + if !hasSuffix(["s", "s"]) { trim("s") } + + return out + } +} diff --git a/Tests/MojitoTests/FuzzyMatcherTests.swift b/Tests/MojitoTests/FuzzyMatcherTests.swift index 94dd43e..a5abcad 100644 --- a/Tests/MojitoTests/FuzzyMatcherTests.swift +++ b/Tests/MojitoTests/FuzzyMatcherTests.swift @@ -156,7 +156,7 @@ struct FuzzyMatcherTests { #expect(happyIdx < wheelchairIdx) } - @Test(arguments: ["yeet", "lfg", "cursed"]) + @Test(arguments: ["yeet", "lfg", "qwrtz"]) func queryWithNoRealMatchReturnsNothing(query: String) { // Across ~23k haystacks something always matches as a scattered // subsequence — 🐞 for "yeet", 🥬 for "lfg". None of these words is in @@ -176,6 +176,27 @@ struct FuzzyMatcherTests { #expect(search(query, limit: 12).contains { $0.emoji.hexcode == hexcode }) } + @Test(arguments: [ + ("ghosted", "1F47B"), // 👻 — the keyword is "ghosting"/"ghost" + ("deployed", "1F680"), // 🚀 + ("cursed", "1F92C"), // 🤬 — via "curse" + ("launching", "1F680"), // 🚀 + ]) + func inflectedQueryReachesItsKeyword(query: String, hexcode: String) { + // fzy rejects a needle longer than its haystack, so these are + // unreachable until the suffix comes off. + #expect(search(query, limit: 12).contains { $0.emoji.hexcode == hexcode }) + } + + @Test func stemmingOnlyRunsWhenTheQueryFoundNothing() { + // ":cats" matches 🐱 directly (shortcode "cats"), so the `-s` stem must + // not run and reshuffle the ranking. + let direct = search("cats", limit: 12).map(\.emoji.hexcode) + let bare = search("cat", limit: 12).map(\.emoji.hexcode) + #expect(!direct.isEmpty) + #expect(direct != bare) + } + @Test func floorSpares2CharQueries() { // Short needles score low by construction, so the floor is off below 3 // characters — the prefix tier carries them instead. diff --git a/Tests/MojitoTests/QueryStemmerTests.swift b/Tests/MojitoTests/QueryStemmerTests.swift new file mode 100644 index 0000000..394b8f0 --- /dev/null +++ b/Tests/MojitoTests/QueryStemmerTests.swift @@ -0,0 +1,81 @@ +import Testing +@testable import Mojito + +/// `QueryStemmer` produces *candidate* spellings, not one canonical stem — the +/// corpus decides which candidate exists. So these assert that the right +/// spelling is offered, not that it's the only one or that it comes first. +struct QueryStemmerTests { + + private func stems(_ query: String) -> [String] { + QueryStemmer.stems(of: Array(query)).map { String($0) } + } + + @Test(arguments: [ + ("ghosted", "ghost"), + ("deployed", "deploy"), + ("launching", "launch"), + ("blocking", "block"), + ("hearts", "heart"), + ("wishes", "wish"), + ]) + func offersThePlainStem(query: String, stem: String) { + #expect(stems(query).contains(stem)) + } + + @Test(arguments: [ + ("parties", "party"), + ("carried", "carry"), + ]) + func restoresYForIesAndIed(query: String, stem: String) { + #expect(stems(query).contains(stem)) + } + + @Test(arguments: [ + ("celebrating", "celebrate"), + ("smiled", "smile"), + ]) + func restoresADroppedE(query: String, stem: String) { + #expect(stems(query).contains(stem)) + } + + @Test(arguments: [ + ("shipping", "ship"), + ("shipped", "ship"), + ]) + func undoesConsonantDoubling(query: String, stem: String) { + #expect(stems(query).contains(stem)) + } + + @Test func keepsTheLosslessStemAheadOfTheLossyOne() throws { + // "pressed" → "press" (real) must be offered before "pres" (junk from + // the doubled-consonant rule), since the caller stops at the first hit. + let candidates = stems("pressed") + let press = try #require(candidates.firstIndex(of: "press")) + if let pres = candidates.firstIndex(of: "pres") { + #expect(press < pres) + } + } + + @Test(arguments: ["press", "glass", "grass"]) + func neverStripsSFromADoubleS(query: String) { + #expect(!stems(query).contains(String(query.dropLast()))) + } + + @Test(arguments: ["ghost", "heart", "smile", "rocket", "cat"]) + func uninflectedQueryYieldsNothing(query: String) { + // Nothing to strip — the fallback should never fire on these. + #expect(stems(query).isEmpty) + } + + @Test(arguments: ["is", "as", "ing", "ed", "es"]) + func tooShortToStem(query: String) { + // A 2–3 char query would stem to something meaninglessly generic. + #expect(stems(query).isEmpty) + } + + @Test func neverEmitsAStemBelowThreeCharacters() { + for query in ["ties", "toes", "pies", "dyes", "axes", "owed", "ring"] { + #expect(stems(query).allSatisfy { $0.count >= 3 }) + } + } +} From a25c0830364a41eac8e0b71e663435001b5eded4 Mon Sep 17 00:00:00 2001 From: Wells Riley <884715+wr@users.noreply.github.com> Date: Tue, 4 Aug 2026 15:46:04 -0400 Subject: [PATCH 2/5] W-569: only search a stem that's a real term in the corpus MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Stemming guesses several spellings and can't know which is a word, so taking the first candidate that returned anything let a nonsense stem win on an accidental fuzzy hit: `movies` → `movy` finds 🎑 (m-o-v-y inside "moon_viewing_ceremony") and shadowed `movie` → 🎥 behind it. `untied` → `unty` surfaced 📍 the same way. Gate each candidate on whether it starts some haystack — a plain character compare, no DP, so it costs far less than the fzy pass it guards. Refs: W-569 --- Sources/Mojito/EmojiDB/FuzzyMatcher.swift | 20 +++++++++++++++++++- Tests/MojitoTests/FuzzyMatcherTests.swift | 21 +++++++++++++++++++++ 2 files changed, 40 insertions(+), 1 deletion(-) diff --git a/Sources/Mojito/EmojiDB/FuzzyMatcher.swift b/Sources/Mojito/EmojiDB/FuzzyMatcher.swift index fad0316..7dc8aef 100644 --- a/Sources/Mojito/EmojiDB/FuzzyMatcher.swift +++ b/Sources/Mojito/EmojiDB/FuzzyMatcher.swift @@ -194,8 +194,14 @@ struct FuzzyMatcher { // longer than its haystack, so `:ghosted:` can't reach the `ghost` // keyword until the suffix comes off. Gated on empty so a query that // already works keeps its exact ranking and pays nothing. + // + // A candidate has to be a real term in the corpus before it's searched. + // Stemming guesses several spellings and can't know which is a word, so + // without the check the first guess wins on any fuzzy hit at all — + // `movies` → `movy` finds 🎑 (m‑o‑v‑y inside "moon_viewing_ceremony") + // and shadows `movie` → 🎥 behind it. if trimmed.isEmpty { - for stem in QueryStemmer.stems(of: needle) { + for stem in QueryStemmer.stems(of: needle) where isCorpusTerm(stem, in: pool) { trimmed = rankedResults( needle: stem, pool: pool, @@ -262,6 +268,18 @@ struct FuzzyMatcher { return output } + /// Whether `stem` starts some haystack in `pool` — i.e. whether it's a real + /// term here rather than a spelling the stemmer invented. Plain character + /// compares, no DP, so it's far cheaper than the fzy pass it gates. + static func isCorpusTerm(_ stem: [Character], in pool: [IndexedEmoji]) -> Bool { + for indexed in pool { + for haystack in indexed.haystacks where haystack.chars.starts(with: stem) { + return true + } + } + return false + } + /// The scoring core: rank a haystack pool against `needle` and return the /// top `limit` rows. Split out from `search` so the ranking (including the /// alias bonus) is testable with a synthetic pool, without the easter-egg diff --git a/Tests/MojitoTests/FuzzyMatcherTests.swift b/Tests/MojitoTests/FuzzyMatcherTests.swift index a5abcad..a9d5b92 100644 --- a/Tests/MojitoTests/FuzzyMatcherTests.swift +++ b/Tests/MojitoTests/FuzzyMatcherTests.swift @@ -188,6 +188,27 @@ struct FuzzyMatcherTests { #expect(search(query, limit: 12).contains { $0.emoji.hexcode == hexcode }) } + @Test func inventedStemDoesNotShadowTheRealOne() throws { + // "movies" offers movy → movi → movie. `movy` isn't a word, but it + // fuzzy-matches 🎑 (m‑o‑v‑y inside "moon_viewing_ceremony") well enough + // to clear the floor — so without the is-it-a-real-term gate it wins + // the race and 🎥 never surfaces. + let results = search("movies", limit: 12).map(\.emoji.hexcode) + let camera = try #require(results.firstIndex(of: "1F3A5")) // 🎥 movie_camera + // 🎑 may still show up as a weak match on the accepted stem — it just + // can't be the reason the better stem was never tried. + if let moon = results.firstIndex(of: "1F391") { // 🎑 + #expect(camera < moon) + } + } + + @Test func stemsThatAreNotWordsYieldNothing() { + // "untied" offers unty → unti → untie. None is a term in the corpus + // (there's no untie emoji), so every candidate is rejected and the + // picker stays empty rather than showing 📍 via a loose `unty` match. + #expect(realResults(search("untied")).isEmpty) + } + @Test func stemmingOnlyRunsWhenTheQueryFoundNothing() { // ":cats" matches 🐱 directly (shortcode "cats"), so the `-s` stem must // not run and reshuffle the ranking. From f055fe7c1f3de67805d3ba95adf9bfdcf21e2c38 Mon Sep 17 00:00:00 2001 From: Wells Riley <884715+wr@users.noreply.github.com> Date: Tue, 4 Aug 2026 15:53:12 -0400 Subject: [PATCH 3/5] W-569: prefer the stem that keeps more of the typed query MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The corpus check rejects invented spellings, but not a trim that lands on a different real word: `hoped` → `hop` returns rabbits and shadows `hope`, `bared` → `bar` shadows `bare`. Order accepted candidates by length so the one retaining most of what was typed is searched first, keeping the stemmer's own order for ties. Refs: W-569 --- Sources/Mojito/EmojiDB/FuzzyMatcher.swift | 24 +++++++++++++++++++++- Tests/MojitoTests/FuzzyMatcherTests.swift | 25 +++++++++++++++++++++++ 2 files changed, 48 insertions(+), 1 deletion(-) diff --git a/Sources/Mojito/EmojiDB/FuzzyMatcher.swift b/Sources/Mojito/EmojiDB/FuzzyMatcher.swift index 7dc8aef..fdf30b7 100644 --- a/Sources/Mojito/EmojiDB/FuzzyMatcher.swift +++ b/Sources/Mojito/EmojiDB/FuzzyMatcher.swift @@ -201,7 +201,7 @@ struct FuzzyMatcher { // `movies` → `movy` finds 🎑 (m‑o‑v‑y inside "moon_viewing_ceremony") // and shadows `movie` → 🎥 behind it. if trimmed.isEmpty { - for stem in QueryStemmer.stems(of: needle) where isCorpusTerm(stem, in: pool) { + for stem in acceptedStems(for: needle, in: pool) { trimmed = rankedResults( needle: stem, pool: pool, @@ -268,6 +268,28 @@ struct FuzzyMatcher { return output } + /// Stem candidates worth searching, best first. + /// + /// Keeps only spellings that exist here, then prefers the one retaining + /// most of what the user typed. Both matter, and for different reasons: + /// without the corpus check `movies` → `movy` matches 🎑 by accident and + /// shadows `movie`; without the length preference `hoping` → `hop` is a + /// real term that shadows `hope`, and `caring` → `car` shadows `care`. + /// + /// Ties keep the stemmer's own order, which is why this sorts on a + /// (length, position) key rather than calling the non-stable `sort`. + static func acceptedStems(for needle: [Character], in pool: [IndexedEmoji]) -> [[Character]] { + QueryStemmer.stems(of: needle) + .enumerated() + .filter { isCorpusTerm($0.element, in: pool) } + .sorted { lhs, rhs in + lhs.element.count != rhs.element.count + ? lhs.element.count > rhs.element.count + : lhs.offset < rhs.offset + } + .map(\.element) + } + /// Whether `stem` starts some haystack in `pool` — i.e. whether it's a real /// term here rather than a spelling the stemmer invented. Plain character /// compares, no DP, so it's far cheaper than the fzy pass it gates. diff --git a/Tests/MojitoTests/FuzzyMatcherTests.swift b/Tests/MojitoTests/FuzzyMatcherTests.swift index a9d5b92..8ce6e10 100644 --- a/Tests/MojitoTests/FuzzyMatcherTests.swift +++ b/Tests/MojitoTests/FuzzyMatcherTests.swift @@ -202,6 +202,31 @@ struct FuzzyMatcherTests { } } + @Test(arguments: [ + ("hoped", ["hope", "hop"]), + ("bared", ["bare", "bar"]), + ("smiled", ["smile", "smil"]), + ]) + func longerStemIsTriedFirstWhenBothAreRealTerms(query: String, expected: [String]) { + // Trimming `-ed` off "hoped" leaves "hop", a corpus term in its own + // right — so the is-it-a-real-term gate can't reject it, and the + // candidate retaining more of what was typed has to win on order. + let stems = FuzzyMatcher.acceptedStems( + for: Array(query), in: EmojiDatabase.shared.indexed + ).map { String($0) } + #expect(stems == expected) + } + + @Test func longerStemChangesWhatSurfaces() throws { + // End-to-end: ":hoped" must reach 🤞 (tagged "hope") rather than the + // rabbits that "hop" would have returned. + let results = search("hoped", limit: 12).map(\.emoji.hexcode) + let hope = try #require(results.firstIndex(of: "1F91E")) // 🤞 + if let rabbit = results.firstIndex(of: "1F430") { // 🐰 + #expect(hope < rabbit) + } + } + @Test func stemsThatAreNotWordsYieldNothing() { // "untied" offers unty → unti → untie. None is a term in the corpus // (there's no untie emoji), so every candidate is rejected and the From c267c88cc88d970103275ea8bb660d59196c4307 Mon Sep 17 00:00:00 2001 From: Wells Riley <884715+wr@users.noreply.github.com> Date: Tue, 4 Aug 2026 15:58:47 -0400 Subject: [PATCH 4/5] W-569: rank stem candidates by how solidly they exist in the corpus MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Ordering purely by length broke `skies`: the longest accepted candidate is `skie`, which isn't a word but prefixes `skier`, so ⛷️ won and `sky` was never tried. Length was standing in for "is this real", and it's a bad proxy. Rank by match strength first — an exact corpus term beats one that merely prefixes a longer word — then by length, then by the stemmer's own order. `skies` → sky, ski, skie; `hoped` → hope, hop; `movies` → movie, movi. Refs: W-569 --- Sources/Mojito/EmojiDB/FuzzyMatcher.swift | 53 +++++++++++++++-------- Tests/MojitoTests/FuzzyMatcherTests.swift | 13 ++++-- 2 files changed, 44 insertions(+), 22 deletions(-) diff --git a/Sources/Mojito/EmojiDB/FuzzyMatcher.swift b/Sources/Mojito/EmojiDB/FuzzyMatcher.swift index fdf30b7..5a4c6fa 100644 --- a/Sources/Mojito/EmojiDB/FuzzyMatcher.swift +++ b/Sources/Mojito/EmojiDB/FuzzyMatcher.swift @@ -268,38 +268,55 @@ struct FuzzyMatcher { return output } + /// How well a stem candidate exists in the corpus. + enum StemMatch: Int, Comparable { + case none = 0 // invented spelling — `movy` + case prefix = 1 // only the start of some longer term — `skie` in `skier` + case exact = 2 // a term in its own right — `sky` + + static func < (lhs: StemMatch, rhs: StemMatch) -> Bool { lhs.rawValue < rhs.rawValue } + } + /// Stem candidates worth searching, best first. /// - /// Keeps only spellings that exist here, then prefers the one retaining - /// most of what the user typed. Both matter, and for different reasons: - /// without the corpus check `movies` → `movy` matches 🎑 by accident and - /// shadows `movie`; without the length preference `hoping` → `hop` is a - /// real term that shadows `hope`, and `caring` → `car` shadows `care`. + /// The stemmer guesses several spellings and can't know which is a word, so + /// the corpus decides. Ranked by how solidly a candidate exists here, then + /// by how much of the typed query it keeps: + /// + /// - `movies` → `movy` is invented (it only matches 🎑 by accident, via + /// m‑o‑v‑y inside "moon_viewing_ceremony") and is dropped outright. + /// - `skies` → `skie` isn't a word, but it *prefixes* `skier`, so it can't + /// be dropped — it just has to lose to the exact terms `sky` and `ski`. + /// - `hoped` → both `hope` and `hop` are exact terms, so the longer one + /// wins; same for `bared` → `bare` over `bar`. /// - /// Ties keep the stemmer's own order, which is why this sorts on a - /// (length, position) key rather than calling the non-stable `sort`. + /// Ties keep the stemmer's own order, hence sorting on an explicit + /// (match, length, position) key rather than the non-stable `sort`. static func acceptedStems(for needle: [Character], in pool: [IndexedEmoji]) -> [[Character]] { QueryStemmer.stems(of: needle) .enumerated() - .filter { isCorpusTerm($0.element, in: pool) } + .map { (offset: $0.offset, stem: $0.element, match: stemMatch($0.element, in: pool)) } + .filter { $0.match > .none } .sorted { lhs, rhs in - lhs.element.count != rhs.element.count - ? lhs.element.count > rhs.element.count - : lhs.offset < rhs.offset + if lhs.match != rhs.match { return lhs.match > rhs.match } + if lhs.stem.count != rhs.stem.count { return lhs.stem.count > rhs.stem.count } + return lhs.offset < rhs.offset } - .map(\.element) + .map(\.stem) } - /// Whether `stem` starts some haystack in `pool` — i.e. whether it's a real - /// term here rather than a spelling the stemmer invented. Plain character - /// compares, no DP, so it's far cheaper than the fzy pass it gates. - static func isCorpusTerm(_ stem: [Character], in pool: [IndexedEmoji]) -> Bool { + /// Whether `stem` is a haystack in `pool`, merely starts one, or neither. + /// Plain character compares, no DP, so it costs far less than the fzy pass + /// it gates. + static func stemMatch(_ stem: [Character], in pool: [IndexedEmoji]) -> StemMatch { + var best = StemMatch.none for indexed in pool { for haystack in indexed.haystacks where haystack.chars.starts(with: stem) { - return true + if haystack.chars.count == stem.count { return .exact } + best = .prefix } } - return false + return best } /// The scoring core: rank a haystack pool against `needle` and return the diff --git a/Tests/MojitoTests/FuzzyMatcherTests.swift b/Tests/MojitoTests/FuzzyMatcherTests.swift index 8ce6e10..4e58cc0 100644 --- a/Tests/MojitoTests/FuzzyMatcherTests.swift +++ b/Tests/MojitoTests/FuzzyMatcherTests.swift @@ -203,14 +203,19 @@ struct FuzzyMatcherTests { } @Test(arguments: [ + // Both exact terms — the longer one keeps more of the query. ("hoped", ["hope", "hop"]), ("bared", ["bare", "bar"]), + // "smil" only prefixes "smile", so the exact term wins despite the tie + // in neither being longer by much. ("smiled", ["smile", "smil"]), + // "skie" is longer than both, but only prefixes "skier" — the exact + // terms have to outrank it or ":skies" returns ⛷️. + ("skies", ["sky", "ski", "skie"]), + // "movi" prefixes "movie_camera"; "movie" is exact. + ("movies", ["movie", "movi"]), ]) - func longerStemIsTriedFirstWhenBothAreRealTerms(query: String, expected: [String]) { - // Trimming `-ed` off "hoped" leaves "hop", a corpus term in its own - // right — so the is-it-a-real-term gate can't reject it, and the - // candidate retaining more of what was typed has to win on order. + func stemsAreOrderedByHowSolidlyTheyExist(query: String, expected: [String]) { let stems = FuzzyMatcher.acceptedStems( for: Array(query), in: EmojiDatabase.shared.indexed ).map { String($0) } From 608b3c963ce04fc334a18e07cc26c809ffaed5d7 Mon Sep 17 00:00:00 2001 From: Wells Riley <884715+wr@users.noreply.github.com> Date: Tue, 4 Aug 2026 16:05:56 -0400 Subject: [PATCH 5/5] W-569: split the -ies and -ied rules apart MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit They aren't equally trustworthy. `-ies` is reliably the plural of a `-y` noun — `skies` is sky + s, never ski + es, since the plural of ski is skis — so it outranks the generic trims. `-ied` carries no such guarantee: `carried` is carry + ied but `skied` is ski + ed, so it goes last and wins only when the plain trim found no real word. Treating them as one rule meant `skied` resolved to sky. Refs: W-569 --- Sources/Mojito/EmojiDB/QueryStemmer.swift | 16 +++++++++++++--- Tests/MojitoTests/FuzzyMatcherTests.swift | 12 +++++++++--- 2 files changed, 22 insertions(+), 6 deletions(-) diff --git a/Sources/Mojito/EmojiDB/QueryStemmer.swift b/Sources/Mojito/EmojiDB/QueryStemmer.swift index aebf24b..e563a45 100644 --- a/Sources/Mojito/EmojiDB/QueryStemmer.swift +++ b/Sources/Mojito/EmojiDB/QueryStemmer.swift @@ -57,18 +57,28 @@ enum QueryStemmer { if let shorter = undoubled(stem) { add(shorter) } } - // `parties` → `party`, `carried` → `carry`. Checked before the generic - // `-es` / `-ed` rules, which would leave the meaningless `parti`. - for suffix in ["ies", "ied"] where hasSuffix(Array(suffix)) { + /// `parties` → `party`. A constructed spelling, not a substring of what + /// was typed. + func restoreY() { add(Array(needle.dropLast(3)) + ["y"]) } + // `-ies` is reliably the plural of a `-y` noun, so it outranks the + // generic trims: `skies` is sky + s, never ski + es (that's `skis`). + if hasSuffix(["i", "e", "s"]) { restoreY() } + trim("ing", restoringE: true) trim("ed", restoringE: true) trim("es") // `press` is not `pres`. if !hasSuffix(["s", "s"]) { trim("s") } + // `-ied` carries no such guarantee — `carried` is carry + ied but + // `skied` is ski + ed — so it goes last and only wins when the plain + // trim didn't land on a real word. Callers rank corpus-exact candidates + // above prefix-only ones, so `carry` still beats `carri` from here. + if hasSuffix(["i", "e", "d"]) { restoreY() } + return out } } diff --git a/Tests/MojitoTests/FuzzyMatcherTests.swift b/Tests/MojitoTests/FuzzyMatcherTests.swift index 4e58cc0..7cedaad 100644 --- a/Tests/MojitoTests/FuzzyMatcherTests.swift +++ b/Tests/MojitoTests/FuzzyMatcherTests.swift @@ -209,11 +209,17 @@ struct FuzzyMatcherTests { // "smil" only prefixes "smile", so the exact term wins despite the tie // in neither being longer by much. ("smiled", ["smile", "smil"]), - // "skie" is longer than both, but only prefixes "skier" — the exact - // terms have to outrank it or ":skies" returns ⛷️. - ("skies", ["sky", "ski", "skie"]), // "movi" prefixes "movie_camera"; "movie" is exact. ("movies", ["movie", "movi"]), + // "ski" and "sky" are both exact, so match rank and length tie and the + // suffix decides: "skies" is sky + s (skis is the plural of ski), while + // "skied" is ski + ed. + ("skies", ["sky", "ski", "skie"]), + ("skied", ["ski", "sky", "skie"]), + // …but where only the constructed form is exact, it still wins. + ("parties", ["party", "parti"]), + ("bodies", ["body"]), + ("copies", ["copy"]), ]) func stemsAreOrderedByHowSolidlyTheyExist(query: String, expected: [String]) { let stems = FuzzyMatcher.acceptedStems(