W-569: stem query suffixes so inflected words reach their keywords

Mojito.xcodeproj/project.pbxproj

@@ -110,6 +110,7 @@
 		9F45015A816323AF275F23ED /* DebugReport.swift in Sources */ = {isa = PBXBuildFile; fileRef = 5C9E01A82260E8A38DF8D7EA /* DebugReport.swift */; };
 		9F550EAD8B9A90F2874E8F22 /* TelemetryUploader.swift in Sources */ = {isa = PBXBuildFile; fileRef = AEA05D792327D50DA2FCF35B /* TelemetryUploader.swift */; };
 		9FE6DD159CABAE6FAC74E6BD /* TriggerConfigStoreTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = F335AAB3008F02F987CB8DC2 /* TriggerConfigStoreTests.swift */; };
+		A11509496D8522DC13BF31F3 /* QueryStemmerTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = 6401923F7190AFC65A7B6959 /* QueryStemmerTests.swift */; };
 		A1C87132DCD9B263CA32BAAE /* WordleSounds.swift in Sources */ = {isa = PBXBuildFile; fileRef = 31F82574CC8A6D051CF87D9B /* WordleSounds.swift */; };
 		A637DCA2905238217427E806 /* EmojiBrowserCatalog.swift in Sources */ = {isa = PBXBuildFile; fileRef = 96C001583E27CC71D9959055 /* EmojiBrowserCatalog.swift */; };
 		A9A15B339C05195BB45CC09F /* MyLegSound.swift in Sources */ = {isa = PBXBuildFile; fileRef = D2C4C16E01184684E33C3437 /* MyLegSound.swift */; };
@@ -161,6 +162,7 @@
 		E539CB425388907B1CA7D3CE /* GifSearcher.swift in Sources */ = {isa = PBXBuildFile; fileRef = 1C3CAFD7B83168FF01538FF4 /* GifSearcher.swift */; };
 		E5C85B09298DAB6F6594B181 /* BouncingDVD.swift in Sources */ = {isa = PBXBuildFile; fileRef = 0836B5ACF7E24509928DA7B0 /* BouncingDVD.swift */; };
 		E62A74A5CA784B452A59CE00 /* DiscoveryNotifier.swift in Sources */ = {isa = PBXBuildFile; fileRef = E6F0EF28CC67A6AB9FFF203F /* DiscoveryNotifier.swift */; };
+		E8715EE20B3652E19AEB690D /* QueryStemmer.swift in Sources */ = {isa = PBXBuildFile; fileRef = 276275D749C5A73951269AF7 /* QueryStemmer.swift */; };
 		E9340C650BD416393A44ACC3 /* DockIconManager.swift in Sources */ = {isa = PBXBuildFile; fileRef = C02F1996A3DED7A0A133E24C /* DockIconManager.swift */; };
 		EAF2B984187615EBAF67954E /* DiskOptimizer.swift in Sources */ = {isa = PBXBuildFile; fileRef = CDDFF6254C8A58E63AC223D5 /* DiskOptimizer.swift */; };
 		ECC3936D06DC519BCE249F8D /* PermissionsCoordinator.swift in Sources */ = {isa = PBXBuildFile; fileRef = 2027256AFD59D65C41FAA2AD /* PermissionsCoordinator.swift */; };
@@ -219,6 +221,7 @@
 		21B80C892257DEA88108EE93 /* TriggerConfig.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = TriggerConfig.swift; sourceTree = "<group>"; };
 		22B06D1C751C2161A7A99DA1 /* AppDelegate.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = AppDelegate.swift; sourceTree = "<group>"; };
 		2361376F2801DB575F27B480 /* DialupSound.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = DialupSound.swift; sourceTree = "<group>"; };
+		276275D749C5A73951269AF7 /* QueryStemmer.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = QueryStemmer.swift; sourceTree = "<group>"; };
 		2928BC07BE26CF9B15F23D70 /* DebugRecorder.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = DebugRecorder.swift; sourceTree = "<group>"; };
 		2C4CF18CE5AC46767EFCA557 /* v03.bin */ = {isa = PBXFileReference; lastKnownFileType = archive.macbinary; path = v03.bin; sourceTree = "<group>"; };
 		2C66984ECCB4DB6B1A018452 /* PrefsKey.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = PrefsKey.swift; sourceTree = "<group>"; };
@@ -253,6 +256,7 @@
 		5C9E01A82260E8A38DF8D7EA /* DebugReport.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = DebugReport.swift; sourceTree = "<group>"; };
 		5FB7AB3DB0E63AF30880FE66 /* BrowserURL.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = BrowserURL.swift; sourceTree = "<group>"; };
 		625433055302CF9B9053F885 /* ExclusionStoreTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = ExclusionStoreTests.swift; sourceTree = "<group>"; };
+		6401923F7190AFC65A7B6959 /* QueryStemmerTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = QueryStemmerTests.swift; sourceTree = "<group>"; };
 		6666150B793721D5095E72A8 /* SymbolsDatabase.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = SymbolsDatabase.swift; sourceTree = "<group>"; };
 		66C166F241AD0718D2067AF1 /* SystemEmojiPicker.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = SystemEmojiPicker.swift; sourceTree = "<group>"; };
 		682479ADBA3E221825BADC83 /* TriggersSettings.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = TriggersSettings.swift; sourceTree = "<group>"; };
@@ -404,6 +408,7 @@
 				42FA18E36A4027D4CB96ADA5 /* FuzzyMatcherTests.swift */,
 				69CD38087938C1E258718C69 /* FzyScorerTests.swift */,
 				12F71260B63547A6DF1B01AD /* PickerViewModelTests.swift */,
+				6401923F7190AFC65A7B6959 /* QueryStemmerTests.swift */,
 				9E0C646543028E968FB6287E /* SeasonalGatesTests.swift */,
 				D74BDC060E2EAACF7A71A3A6 /* SkinToneTests.swift */,
 				E25EC50F4D58E03E218947DF /* SymbolsDatabaseTests.swift */,
@@ -705,6 +710,7 @@
 				6E2009C3B36DF9228B873244 /* EmoticonTable.swift */,
 				5A8851906565F70B7815640F /* FuzzyMatcher.swift */,
 				FE5EDBB8F234B21B8D4A3324 /* FzyScorer.swift */,
+				276275D749C5A73951269AF7 /* QueryStemmer.swift */,
 				964D290D66BB33D1F21333A0 /* QuickAccessStore.swift */,
 				F5C5BC5F015CABDD85262258 /* SeasonalGates.swift */,
 				B375DE27AC0AE1D8DE965403 /* SkinTone.swift */,
@@ -1007,6 +1013,7 @@
 				ABF54441D03140C8174E13A9 /* PrefsKey.swift in Sources */,
 				BA163217ADDEBAF751D5ABA2 /* PrideWave.swift in Sources */,
 				CEAF9DB89294B0652A8A16F7 /* PrivacyPermissionsSettings.swift in Sources */,
+				E8715EE20B3652E19AEB690D /* QueryStemmer.swift in Sources */,
 				D4E80136FD448E4ACC27EC58 /* QuickAccessSettings.swift in Sources */,
 				1A80F47103BB59733D6292B7 /* QuickAccessStore.swift in Sources */,
 				FD15928FE17F78588821135E /* Rickroll.swift in Sources */,
@@ -1067,6 +1074,7 @@
 				4F9E4BA7A275583B07011809 /* FuzzyMatcherTests.swift in Sources */,
 				B32AE99482DA334BDD7CEEAC /* FzyScorerTests.swift in Sources */,
 				2C2FD54F6E35CCD0374213BB /* PickerViewModelTests.swift in Sources */,
+				A11509496D8522DC13BF31F3 /* QueryStemmerTests.swift in Sources */,
 				C025CD4AE30320C3FE69028A /* SeasonalGatesTests.swift in Sources */,
 				917B8512B30F3B6E8C9D2B8C /* SkinToneTests.swift in Sources */,
 				91A5F2A800EA1FC0700394EA /* SymbolsDatabaseTests.swift in Sources */,

Sources/Mojito/EmojiDB/FuzzyMatcher.swift

@@ -181,7 +181,7 @@ struct FuzzyMatcher {
             pool = SymbolsCorpus.entries
         }
 
-        let trimmed = rankedResults(
+        var trimmed = rankedResults(
             needle: needle,
             pool: pool,
             usage: usage,
@@ -190,6 +190,30 @@ struct FuzzyMatcher {
             limit: limit
         )
 
+        // Only when the query as typed found nothing: fzy rejects a needle
+        // longer than its haystack, so `:ghosted:` can't reach the `ghost`
+        // keyword until the suffix comes off. Gated on empty so a query that
+        // already works keeps its exact ranking and pays nothing.
+        //
+        // A candidate has to be a real term in the corpus before it's searched.
+        // Stemming guesses several spellings and can't know which is a word, so
+        // without the check the first guess wins on any fuzzy hit at all —
+        // `movies` → `movy` finds 🎑 (m‑o‑v‑y inside "moon_viewing_ceremony")
+        // and shadows `movie` → 🎥 behind it.
+        if trimmed.isEmpty {
+            for stem in acceptedStems(for: needle, in: pool) {
+                trimmed = rankedResults(
+                    needle: stem,
+                    pool: pool,
+                    usage: usage,
+                    useFrequencyBoost: useFrequencyBoost,
+                    scanTags: stem.count >= tagMinNeedle,
+                    limit: limit
+                )
+                if !trimmed.isEmpty { break }
+            }
+        }
+
         let lowercased = query.lowercased()
 
         // Discovery hint sits just below the top real match — a normal
@@ -244,6 +268,57 @@ struct FuzzyMatcher {
         return output
     }
 
+    /// How well a stem candidate exists in the corpus.
+    enum StemMatch: Int, Comparable {
+        case none = 0       // invented spelling — `movy`
+        case prefix = 1     // only the start of some longer term — `skie` in `skier`
+        case exact = 2      // a term in its own right — `sky`
+
+        static func < (lhs: StemMatch, rhs: StemMatch) -> Bool { lhs.rawValue < rhs.rawValue }
+    }
+
+    /// Stem candidates worth searching, best first.
+    ///
+    /// The stemmer guesses several spellings and can't know which is a word, so
+    /// the corpus decides. Ranked by how solidly a candidate exists here, then
+    /// by how much of the typed query it keeps:
+    ///
+    /// - `movies` → `movy` is invented (it only matches 🎑 by accident, via
+    ///   m‑o‑v‑y inside "moon_viewing_ceremony") and is dropped outright.
+    /// - `skies` → `skie` isn't a word, but it *prefixes* `skier`, so it can't
+    ///   be dropped — it just has to lose to the exact terms `sky` and `ski`.
+    /// - `hoped` → both `hope` and `hop` are exact terms, so the longer one
+    ///   wins; same for `bared` → `bare` over `bar`.
+    ///
+    /// Ties keep the stemmer's own order, hence sorting on an explicit
+    /// (match, length, position) key rather than the non-stable `sort`.
+    static func acceptedStems(for needle: [Character], in pool: [IndexedEmoji]) -> [[Character]] {
+        QueryStemmer.stems(of: needle)
+            .enumerated()
+            .map { (offset: $0.offset, stem: $0.element, match: stemMatch($0.element, in: pool)) }
+            .filter { $0.match > .none }
+            .sorted { lhs, rhs in
+                if lhs.match != rhs.match { return lhs.match > rhs.match }
+                if lhs.stem.count != rhs.stem.count { return lhs.stem.count > rhs.stem.count }
+                return lhs.offset < rhs.offset
+            }
+            .map(\.stem)
+    }
+
+    /// Whether `stem` is a haystack in `pool`, merely starts one, or neither.
+    /// Plain character compares, no DP, so it costs far less than the fzy pass
+    /// it gates.
+    static func stemMatch(_ stem: [Character], in pool: [IndexedEmoji]) -> StemMatch {
+        var best = StemMatch.none
+        for indexed in pool {
+            for haystack in indexed.haystacks where haystack.chars.starts(with: stem) {
+                if haystack.chars.count == stem.count { return .exact }
+                best = .prefix
+            }
+        }
+        return best
+    }
+
     /// The scoring core: rank a haystack pool against `needle` and return the
     /// top `limit` rows. Split out from `search` so the ranking (including the
     /// alias bonus) is testable with a synthetic pool, without the easter-egg

Sources/Mojito/EmojiDB/QueryStemmer.swift

@@ -0,0 +1,84 @@
+import Foundation
+
+/// Strips common English inflections so an inflected query can reach an
+/// uninflected keyword.
+///
+/// `FzyScorer` rejects a needle longer than its haystack, so `:ghosted:` can
+/// never match the `ghost` / `ghosting` keywords no matter how scoring is
+/// tuned — the match dies before the DP runs. Trimming the suffix is the only
+/// way through.
+///
+/// Deliberately cruder than a real stemmer (Porter, Snowball). It runs only as
+/// a fallback when the raw query found nothing, so a wrong guess costs an empty
+/// picker staying empty — never a worse result for a query that already works.
+/// That asymmetry is what lets the rules be this loose.
+enum QueryStemmer {
+
+    /// Below this a stem is too generic to mean anything.
+    private static let minStem = 3
+
+    /// Candidate stems for `needle`, most likely first; empty when no rule
+    /// applies. Callers try each in order and stop at the first with results.
+    ///
+    /// Several spellings are offered at once on purpose: `-ing` alone can't
+    /// tell `shipping` (undo a doubled consonant) from `celebrating` (restore a
+    /// dropped `e`) from `blocking` (plain trim), so all three go out and the
+    /// corpus decides which one exists.
+    static func stems(of needle: [Character]) -> [[Character]] {
+        var out: [[Character]] = []
+
+        func add(_ candidate: [Character]) {
+            guard candidate.count >= minStem, !out.contains(candidate) else { return }
+            out.append(candidate)
+        }
+
+        /// `shipping` → `shipp` → `ship`. Doubled consonants only, so `pressed`
+        /// yields `press` before the lossy `pres`.
+        func undoubled(_ stem: [Character]) -> [Character]? {
+            guard let last = stem.last,
+                  stem.count > minStem,
+                  last == stem[stem.count - 2],
+                  !"aeiou".contains(last)
+            else { return nil }
+            return Array(stem.dropLast())
+        }
+
+        func hasSuffix(_ chars: [Character]) -> Bool {
+            needle.count > chars.count && needle.suffix(chars.count).elementsEqual(chars)
+        }
+
+        func trim(_ suffix: String, restoringE: Bool = false) {
+            let chars = Array(suffix)
+            guard hasSuffix(chars) else { return }
+            let stem = Array(needle.dropLast(chars.count))
+            add(stem)
+            // `celebrat` is nothing; `celebrate` is the word.
+            if restoringE { add(stem + ["e"]) }
+            if let shorter = undoubled(stem) { add(shorter) }
+        }
+
+        /// `parties` → `party`. A constructed spelling, not a substring of what
+        /// was typed.
+        func restoreY() {
+            add(Array(needle.dropLast(3)) + ["y"])
+        }
+
+        // `-ies` is reliably the plural of a `-y` noun, so it outranks the
+        // generic trims: `skies` is sky + s, never ski + es (that's `skis`).
+        if hasSuffix(["i", "e", "s"]) { restoreY() }
+
+        trim("ing", restoringE: true)
+        trim("ed", restoringE: true)
+        trim("es")
+        // `press` is not `pres`.
+        if !hasSuffix(["s", "s"]) { trim("s") }
+
+        // `-ied` carries no such guarantee — `carried` is carry + ied but
+        // `skied` is ski + ed — so it goes last and only wins when the plain
+        // trim didn't land on a real word. Callers rank corpus-exact candidates
+        // above prefix-only ones, so `carry` still beats `carri` from here.
+        if hasSuffix(["i", "e", "d"]) { restoreY() }
+
+        return out
+    }
+}

Tests/MojitoTests/FuzzyMatcherTests.swift

@@ -156,7 +156,7 @@ struct FuzzyMatcherTests {
         #expect(happyIdx < wheelchairIdx)
     }
 
-    @Test(arguments: ["yeet", "lfg", "cursed"])
+    @Test(arguments: ["yeet", "lfg", "qwrtz"])
     func queryWithNoRealMatchReturnsNothing(query: String) {
         // Across ~23k haystacks something always matches as a scattered
         // subsequence — 🐞 for "yeet", 🥬 for "lfg". None of these words is in
@@ -176,6 +176,84 @@ struct FuzzyMatcherTests {
         #expect(search(query, limit: 12).contains { $0.emoji.hexcode == hexcode })
     }
 
+    @Test(arguments: [
+        ("ghosted", "1F47B"),    // 👻 — the keyword is "ghosting"/"ghost"
+        ("deployed", "1F680"),   // 🚀
+        ("cursed", "1F92C"),     // 🤬 — via "curse"
+        ("launching", "1F680"),  // 🚀
+    ])
+    func inflectedQueryReachesItsKeyword(query: String, hexcode: String) {
+        // fzy rejects a needle longer than its haystack, so these are
+        // unreachable until the suffix comes off.
+        #expect(search(query, limit: 12).contains { $0.emoji.hexcode == hexcode })
+    }
+
+    @Test func inventedStemDoesNotShadowTheRealOne() throws {
+        // "movies" offers movy → movi → movie. `movy` isn't a word, but it
+        // fuzzy-matches 🎑 (m‑o‑v‑y inside "moon_viewing_ceremony") well enough
+        // to clear the floor — so without the is-it-a-real-term gate it wins
+        // the race and 🎥 never surfaces.
+        let results = search("movies", limit: 12).map(\.emoji.hexcode)
+        let camera = try #require(results.firstIndex(of: "1F3A5"))  // 🎥 movie_camera
+        // 🎑 may still show up as a weak match on the accepted stem — it just
+        // can't be the reason the better stem was never tried.
+        if let moon = results.firstIndex(of: "1F391") {             // 🎑
+            #expect(camera < moon)
+        }
+    }
+
+    @Test(arguments: [
+        // Both exact terms — the longer one keeps more of the query.
+        ("hoped", ["hope", "hop"]),
+        ("bared", ["bare", "bar"]),
+        // "smil" only prefixes "smile", so the exact term wins despite the tie
+        // in neither being longer by much.
+        ("smiled", ["smile", "smil"]),
+        // "movi" prefixes "movie_camera"; "movie" is exact.
+        ("movies", ["movie", "movi"]),
+        // "ski" and "sky" are both exact, so match rank and length tie and the
+        // suffix decides: "skies" is sky + s (skis is the plural of ski), while
+        // "skied" is ski + ed.
+        ("skies", ["sky", "ski", "skie"]),
+        ("skied", ["ski", "sky", "skie"]),
+        // …but where only the constructed form is exact, it still wins.
+        ("parties", ["party", "parti"]),
+        ("bodies", ["body"]),
+        ("copies", ["copy"]),
+    ])
+    func stemsAreOrderedByHowSolidlyTheyExist(query: String, expected: [String]) {
+        let stems = FuzzyMatcher.acceptedStems(
+            for: Array(query), in: EmojiDatabase.shared.indexed
+        ).map { String($0) }
+        #expect(stems == expected)
+    }
+
+    @Test func longerStemChangesWhatSurfaces() throws {
+        // End-to-end: ":hoped" must reach 🤞 (tagged "hope") rather than the
+        // rabbits that "hop" would have returned.
+        let results = search("hoped", limit: 12).map(\.emoji.hexcode)
+        let hope = try #require(results.firstIndex(of: "1F91E"))   // 🤞
+        if let rabbit = results.firstIndex(of: "1F430") {          // 🐰
+            #expect(hope < rabbit)
+        }
+    }
+
+    @Test func stemsThatAreNotWordsYieldNothing() {
+        // "untied" offers unty → unti → untie. None is a term in the corpus
+        // (there's no untie emoji), so every candidate is rejected and the
+        // picker stays empty rather than showing 📍 via a loose `unty` match.
+        #expect(realResults(search("untied")).isEmpty)
+    }
+
+    @Test func stemmingOnlyRunsWhenTheQueryFoundNothing() {
+        // ":cats" matches 🐱 directly (shortcode "cats"), so the `-s` stem must
+        // not run and reshuffle the ranking.
+        let direct = search("cats", limit: 12).map(\.emoji.hexcode)
+        let bare = search("cat", limit: 12).map(\.emoji.hexcode)
+        #expect(!direct.isEmpty)
+        #expect(direct != bare)
+    }
+
     @Test func floorSpares2CharQueries() {
         // Short needles score low by construction, so the floor is off below 3
         // characters — the prefix tier carries them instead.