diff --git a/Mojito.xcodeproj/project.pbxproj b/Mojito.xcodeproj/project.pbxproj index 9cee684..af6b488 100644 --- a/Mojito.xcodeproj/project.pbxproj +++ b/Mojito.xcodeproj/project.pbxproj @@ -110,6 +110,7 @@ 9F45015A816323AF275F23ED /* DebugReport.swift in Sources */ = {isa = PBXBuildFile; fileRef = 5C9E01A82260E8A38DF8D7EA /* DebugReport.swift */; }; 9F550EAD8B9A90F2874E8F22 /* TelemetryUploader.swift in Sources */ = {isa = PBXBuildFile; fileRef = AEA05D792327D50DA2FCF35B /* TelemetryUploader.swift */; }; 9FE6DD159CABAE6FAC74E6BD /* TriggerConfigStoreTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = F335AAB3008F02F987CB8DC2 /* TriggerConfigStoreTests.swift */; }; + A11509496D8522DC13BF31F3 /* QueryStemmerTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = 6401923F7190AFC65A7B6959 /* QueryStemmerTests.swift */; }; A1C87132DCD9B263CA32BAAE /* WordleSounds.swift in Sources */ = {isa = PBXBuildFile; fileRef = 31F82574CC8A6D051CF87D9B /* WordleSounds.swift */; }; A637DCA2905238217427E806 /* EmojiBrowserCatalog.swift in Sources */ = {isa = PBXBuildFile; fileRef = 96C001583E27CC71D9959055 /* EmojiBrowserCatalog.swift */; }; A9A15B339C05195BB45CC09F /* MyLegSound.swift in Sources */ = {isa = PBXBuildFile; fileRef = D2C4C16E01184684E33C3437 /* MyLegSound.swift */; }; @@ -161,6 +162,7 @@ E539CB425388907B1CA7D3CE /* GifSearcher.swift in Sources */ = {isa = PBXBuildFile; fileRef = 1C3CAFD7B83168FF01538FF4 /* GifSearcher.swift */; }; E5C85B09298DAB6F6594B181 /* BouncingDVD.swift in Sources */ = {isa = PBXBuildFile; fileRef = 0836B5ACF7E24509928DA7B0 /* BouncingDVD.swift */; }; E62A74A5CA784B452A59CE00 /* DiscoveryNotifier.swift in Sources */ = {isa = PBXBuildFile; fileRef = E6F0EF28CC67A6AB9FFF203F /* DiscoveryNotifier.swift */; }; + E8715EE20B3652E19AEB690D /* QueryStemmer.swift in Sources */ = {isa = PBXBuildFile; fileRef = 276275D749C5A73951269AF7 /* QueryStemmer.swift */; }; E9340C650BD416393A44ACC3 /* DockIconManager.swift in Sources */ = {isa = PBXBuildFile; fileRef = C02F1996A3DED7A0A133E24C /* DockIconManager.swift */; }; EAF2B984187615EBAF67954E /* DiskOptimizer.swift in Sources */ = {isa = PBXBuildFile; fileRef = CDDFF6254C8A58E63AC223D5 /* DiskOptimizer.swift */; }; ECC3936D06DC519BCE249F8D /* PermissionsCoordinator.swift in Sources */ = {isa = PBXBuildFile; fileRef = 2027256AFD59D65C41FAA2AD /* PermissionsCoordinator.swift */; }; @@ -219,6 +221,7 @@ 21B80C892257DEA88108EE93 /* TriggerConfig.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = TriggerConfig.swift; sourceTree = ""; }; 22B06D1C751C2161A7A99DA1 /* AppDelegate.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = AppDelegate.swift; sourceTree = ""; }; 2361376F2801DB575F27B480 /* DialupSound.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = DialupSound.swift; sourceTree = ""; }; + 276275D749C5A73951269AF7 /* QueryStemmer.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = QueryStemmer.swift; sourceTree = ""; }; 2928BC07BE26CF9B15F23D70 /* DebugRecorder.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = DebugRecorder.swift; sourceTree = ""; }; 2C4CF18CE5AC46767EFCA557 /* v03.bin */ = {isa = PBXFileReference; lastKnownFileType = archive.macbinary; path = v03.bin; sourceTree = ""; }; 2C66984ECCB4DB6B1A018452 /* PrefsKey.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = PrefsKey.swift; sourceTree = ""; }; @@ -253,6 +256,7 @@ 5C9E01A82260E8A38DF8D7EA /* DebugReport.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = DebugReport.swift; sourceTree = ""; }; 5FB7AB3DB0E63AF30880FE66 /* BrowserURL.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = BrowserURL.swift; sourceTree = ""; }; 625433055302CF9B9053F885 /* ExclusionStoreTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = ExclusionStoreTests.swift; sourceTree = ""; }; + 6401923F7190AFC65A7B6959 /* QueryStemmerTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = QueryStemmerTests.swift; sourceTree = ""; }; 6666150B793721D5095E72A8 /* SymbolsDatabase.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = SymbolsDatabase.swift; sourceTree = ""; }; 66C166F241AD0718D2067AF1 /* SystemEmojiPicker.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = SystemEmojiPicker.swift; sourceTree = ""; }; 682479ADBA3E221825BADC83 /* TriggersSettings.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = TriggersSettings.swift; sourceTree = ""; }; @@ -404,6 +408,7 @@ 42FA18E36A4027D4CB96ADA5 /* FuzzyMatcherTests.swift */, 69CD38087938C1E258718C69 /* FzyScorerTests.swift */, 12F71260B63547A6DF1B01AD /* PickerViewModelTests.swift */, + 6401923F7190AFC65A7B6959 /* QueryStemmerTests.swift */, 9E0C646543028E968FB6287E /* SeasonalGatesTests.swift */, D74BDC060E2EAACF7A71A3A6 /* SkinToneTests.swift */, E25EC50F4D58E03E218947DF /* SymbolsDatabaseTests.swift */, @@ -705,6 +710,7 @@ 6E2009C3B36DF9228B873244 /* EmoticonTable.swift */, 5A8851906565F70B7815640F /* FuzzyMatcher.swift */, FE5EDBB8F234B21B8D4A3324 /* FzyScorer.swift */, + 276275D749C5A73951269AF7 /* QueryStemmer.swift */, 964D290D66BB33D1F21333A0 /* QuickAccessStore.swift */, F5C5BC5F015CABDD85262258 /* SeasonalGates.swift */, B375DE27AC0AE1D8DE965403 /* SkinTone.swift */, @@ -1007,6 +1013,7 @@ ABF54441D03140C8174E13A9 /* PrefsKey.swift in Sources */, BA163217ADDEBAF751D5ABA2 /* PrideWave.swift in Sources */, CEAF9DB89294B0652A8A16F7 /* PrivacyPermissionsSettings.swift in Sources */, + E8715EE20B3652E19AEB690D /* QueryStemmer.swift in Sources */, D4E80136FD448E4ACC27EC58 /* QuickAccessSettings.swift in Sources */, 1A80F47103BB59733D6292B7 /* QuickAccessStore.swift in Sources */, FD15928FE17F78588821135E /* Rickroll.swift in Sources */, @@ -1067,6 +1074,7 @@ 4F9E4BA7A275583B07011809 /* FuzzyMatcherTests.swift in Sources */, B32AE99482DA334BDD7CEEAC /* FzyScorerTests.swift in Sources */, 2C2FD54F6E35CCD0374213BB /* PickerViewModelTests.swift in Sources */, + A11509496D8522DC13BF31F3 /* QueryStemmerTests.swift in Sources */, C025CD4AE30320C3FE69028A /* SeasonalGatesTests.swift in Sources */, 917B8512B30F3B6E8C9D2B8C /* SkinToneTests.swift in Sources */, 91A5F2A800EA1FC0700394EA /* SymbolsDatabaseTests.swift in Sources */, diff --git a/Sources/Mojito/EmojiDB/FuzzyMatcher.swift b/Sources/Mojito/EmojiDB/FuzzyMatcher.swift index ce3c275..5a4c6fa 100644 --- a/Sources/Mojito/EmojiDB/FuzzyMatcher.swift +++ b/Sources/Mojito/EmojiDB/FuzzyMatcher.swift @@ -181,7 +181,7 @@ struct FuzzyMatcher { pool = SymbolsCorpus.entries } - let trimmed = rankedResults( + var trimmed = rankedResults( needle: needle, pool: pool, usage: usage, @@ -190,6 +190,30 @@ struct FuzzyMatcher { limit: limit ) + // Only when the query as typed found nothing: fzy rejects a needle + // longer than its haystack, so `:ghosted:` can't reach the `ghost` + // keyword until the suffix comes off. Gated on empty so a query that + // already works keeps its exact ranking and pays nothing. + // + // A candidate has to be a real term in the corpus before it's searched. + // Stemming guesses several spellings and can't know which is a word, so + // without the check the first guess wins on any fuzzy hit at all — + // `movies` → `movy` finds 🎑 (m‑o‑v‑y inside "moon_viewing_ceremony") + // and shadows `movie` → 🎥 behind it. + if trimmed.isEmpty { + for stem in acceptedStems(for: needle, in: pool) { + trimmed = rankedResults( + needle: stem, + pool: pool, + usage: usage, + useFrequencyBoost: useFrequencyBoost, + scanTags: stem.count >= tagMinNeedle, + limit: limit + ) + if !trimmed.isEmpty { break } + } + } + let lowercased = query.lowercased() // Discovery hint sits just below the top real match — a normal @@ -244,6 +268,57 @@ struct FuzzyMatcher { return output } + /// How well a stem candidate exists in the corpus. + enum StemMatch: Int, Comparable { + case none = 0 // invented spelling — `movy` + case prefix = 1 // only the start of some longer term — `skie` in `skier` + case exact = 2 // a term in its own right — `sky` + + static func < (lhs: StemMatch, rhs: StemMatch) -> Bool { lhs.rawValue < rhs.rawValue } + } + + /// Stem candidates worth searching, best first. + /// + /// The stemmer guesses several spellings and can't know which is a word, so + /// the corpus decides. Ranked by how solidly a candidate exists here, then + /// by how much of the typed query it keeps: + /// + /// - `movies` → `movy` is invented (it only matches 🎑 by accident, via + /// m‑o‑v‑y inside "moon_viewing_ceremony") and is dropped outright. + /// - `skies` → `skie` isn't a word, but it *prefixes* `skier`, so it can't + /// be dropped — it just has to lose to the exact terms `sky` and `ski`. + /// - `hoped` → both `hope` and `hop` are exact terms, so the longer one + /// wins; same for `bared` → `bare` over `bar`. + /// + /// Ties keep the stemmer's own order, hence sorting on an explicit + /// (match, length, position) key rather than the non-stable `sort`. + static func acceptedStems(for needle: [Character], in pool: [IndexedEmoji]) -> [[Character]] { + QueryStemmer.stems(of: needle) + .enumerated() + .map { (offset: $0.offset, stem: $0.element, match: stemMatch($0.element, in: pool)) } + .filter { $0.match > .none } + .sorted { lhs, rhs in + if lhs.match != rhs.match { return lhs.match > rhs.match } + if lhs.stem.count != rhs.stem.count { return lhs.stem.count > rhs.stem.count } + return lhs.offset < rhs.offset + } + .map(\.stem) + } + + /// Whether `stem` is a haystack in `pool`, merely starts one, or neither. + /// Plain character compares, no DP, so it costs far less than the fzy pass + /// it gates. + static func stemMatch(_ stem: [Character], in pool: [IndexedEmoji]) -> StemMatch { + var best = StemMatch.none + for indexed in pool { + for haystack in indexed.haystacks where haystack.chars.starts(with: stem) { + if haystack.chars.count == stem.count { return .exact } + best = .prefix + } + } + return best + } + /// The scoring core: rank a haystack pool against `needle` and return the /// top `limit` rows. Split out from `search` so the ranking (including the /// alias bonus) is testable with a synthetic pool, without the easter-egg diff --git a/Sources/Mojito/EmojiDB/QueryStemmer.swift b/Sources/Mojito/EmojiDB/QueryStemmer.swift new file mode 100644 index 0000000..e563a45 --- /dev/null +++ b/Sources/Mojito/EmojiDB/QueryStemmer.swift @@ -0,0 +1,84 @@ +import Foundation + +/// Strips common English inflections so an inflected query can reach an +/// uninflected keyword. +/// +/// `FzyScorer` rejects a needle longer than its haystack, so `:ghosted:` can +/// never match the `ghost` / `ghosting` keywords no matter how scoring is +/// tuned — the match dies before the DP runs. Trimming the suffix is the only +/// way through. +/// +/// Deliberately cruder than a real stemmer (Porter, Snowball). It runs only as +/// a fallback when the raw query found nothing, so a wrong guess costs an empty +/// picker staying empty — never a worse result for a query that already works. +/// That asymmetry is what lets the rules be this loose. +enum QueryStemmer { + + /// Below this a stem is too generic to mean anything. + private static let minStem = 3 + + /// Candidate stems for `needle`, most likely first; empty when no rule + /// applies. Callers try each in order and stop at the first with results. + /// + /// Several spellings are offered at once on purpose: `-ing` alone can't + /// tell `shipping` (undo a doubled consonant) from `celebrating` (restore a + /// dropped `e`) from `blocking` (plain trim), so all three go out and the + /// corpus decides which one exists. + static func stems(of needle: [Character]) -> [[Character]] { + var out: [[Character]] = [] + + func add(_ candidate: [Character]) { + guard candidate.count >= minStem, !out.contains(candidate) else { return } + out.append(candidate) + } + + /// `shipping` → `shipp` → `ship`. Doubled consonants only, so `pressed` + /// yields `press` before the lossy `pres`. + func undoubled(_ stem: [Character]) -> [Character]? { + guard let last = stem.last, + stem.count > minStem, + last == stem[stem.count - 2], + !"aeiou".contains(last) + else { return nil } + return Array(stem.dropLast()) + } + + func hasSuffix(_ chars: [Character]) -> Bool { + needle.count > chars.count && needle.suffix(chars.count).elementsEqual(chars) + } + + func trim(_ suffix: String, restoringE: Bool = false) { + let chars = Array(suffix) + guard hasSuffix(chars) else { return } + let stem = Array(needle.dropLast(chars.count)) + add(stem) + // `celebrat` is nothing; `celebrate` is the word. + if restoringE { add(stem + ["e"]) } + if let shorter = undoubled(stem) { add(shorter) } + } + + /// `parties` → `party`. A constructed spelling, not a substring of what + /// was typed. + func restoreY() { + add(Array(needle.dropLast(3)) + ["y"]) + } + + // `-ies` is reliably the plural of a `-y` noun, so it outranks the + // generic trims: `skies` is sky + s, never ski + es (that's `skis`). + if hasSuffix(["i", "e", "s"]) { restoreY() } + + trim("ing", restoringE: true) + trim("ed", restoringE: true) + trim("es") + // `press` is not `pres`. + if !hasSuffix(["s", "s"]) { trim("s") } + + // `-ied` carries no such guarantee — `carried` is carry + ied but + // `skied` is ski + ed — so it goes last and only wins when the plain + // trim didn't land on a real word. Callers rank corpus-exact candidates + // above prefix-only ones, so `carry` still beats `carri` from here. + if hasSuffix(["i", "e", "d"]) { restoreY() } + + return out + } +} diff --git a/Tests/MojitoTests/FuzzyMatcherTests.swift b/Tests/MojitoTests/FuzzyMatcherTests.swift index 94dd43e..7cedaad 100644 --- a/Tests/MojitoTests/FuzzyMatcherTests.swift +++ b/Tests/MojitoTests/FuzzyMatcherTests.swift @@ -156,7 +156,7 @@ struct FuzzyMatcherTests { #expect(happyIdx < wheelchairIdx) } - @Test(arguments: ["yeet", "lfg", "cursed"]) + @Test(arguments: ["yeet", "lfg", "qwrtz"]) func queryWithNoRealMatchReturnsNothing(query: String) { // Across ~23k haystacks something always matches as a scattered // subsequence — 🐞 for "yeet", 🥬 for "lfg". None of these words is in @@ -176,6 +176,84 @@ struct FuzzyMatcherTests { #expect(search(query, limit: 12).contains { $0.emoji.hexcode == hexcode }) } + @Test(arguments: [ + ("ghosted", "1F47B"), // 👻 — the keyword is "ghosting"/"ghost" + ("deployed", "1F680"), // 🚀 + ("cursed", "1F92C"), // 🤬 — via "curse" + ("launching", "1F680"), // 🚀 + ]) + func inflectedQueryReachesItsKeyword(query: String, hexcode: String) { + // fzy rejects a needle longer than its haystack, so these are + // unreachable until the suffix comes off. + #expect(search(query, limit: 12).contains { $0.emoji.hexcode == hexcode }) + } + + @Test func inventedStemDoesNotShadowTheRealOne() throws { + // "movies" offers movy → movi → movie. `movy` isn't a word, but it + // fuzzy-matches 🎑 (m‑o‑v‑y inside "moon_viewing_ceremony") well enough + // to clear the floor — so without the is-it-a-real-term gate it wins + // the race and 🎥 never surfaces. + let results = search("movies", limit: 12).map(\.emoji.hexcode) + let camera = try #require(results.firstIndex(of: "1F3A5")) // 🎥 movie_camera + // 🎑 may still show up as a weak match on the accepted stem — it just + // can't be the reason the better stem was never tried. + if let moon = results.firstIndex(of: "1F391") { // 🎑 + #expect(camera < moon) + } + } + + @Test(arguments: [ + // Both exact terms — the longer one keeps more of the query. + ("hoped", ["hope", "hop"]), + ("bared", ["bare", "bar"]), + // "smil" only prefixes "smile", so the exact term wins despite the tie + // in neither being longer by much. + ("smiled", ["smile", "smil"]), + // "movi" prefixes "movie_camera"; "movie" is exact. + ("movies", ["movie", "movi"]), + // "ski" and "sky" are both exact, so match rank and length tie and the + // suffix decides: "skies" is sky + s (skis is the plural of ski), while + // "skied" is ski + ed. + ("skies", ["sky", "ski", "skie"]), + ("skied", ["ski", "sky", "skie"]), + // …but where only the constructed form is exact, it still wins. + ("parties", ["party", "parti"]), + ("bodies", ["body"]), + ("copies", ["copy"]), + ]) + func stemsAreOrderedByHowSolidlyTheyExist(query: String, expected: [String]) { + let stems = FuzzyMatcher.acceptedStems( + for: Array(query), in: EmojiDatabase.shared.indexed + ).map { String($0) } + #expect(stems == expected) + } + + @Test func longerStemChangesWhatSurfaces() throws { + // End-to-end: ":hoped" must reach 🤞 (tagged "hope") rather than the + // rabbits that "hop" would have returned. + let results = search("hoped", limit: 12).map(\.emoji.hexcode) + let hope = try #require(results.firstIndex(of: "1F91E")) // 🤞 + if let rabbit = results.firstIndex(of: "1F430") { // 🐰 + #expect(hope < rabbit) + } + } + + @Test func stemsThatAreNotWordsYieldNothing() { + // "untied" offers unty → unti → untie. None is a term in the corpus + // (there's no untie emoji), so every candidate is rejected and the + // picker stays empty rather than showing 📍 via a loose `unty` match. + #expect(realResults(search("untied")).isEmpty) + } + + @Test func stemmingOnlyRunsWhenTheQueryFoundNothing() { + // ":cats" matches 🐱 directly (shortcode "cats"), so the `-s` stem must + // not run and reshuffle the ranking. + let direct = search("cats", limit: 12).map(\.emoji.hexcode) + let bare = search("cat", limit: 12).map(\.emoji.hexcode) + #expect(!direct.isEmpty) + #expect(direct != bare) + } + @Test func floorSpares2CharQueries() { // Short needles score low by construction, so the floor is off below 3 // characters — the prefix tier carries them instead. diff --git a/Tests/MojitoTests/QueryStemmerTests.swift b/Tests/MojitoTests/QueryStemmerTests.swift new file mode 100644 index 0000000..394b8f0 --- /dev/null +++ b/Tests/MojitoTests/QueryStemmerTests.swift @@ -0,0 +1,81 @@ +import Testing +@testable import Mojito + +/// `QueryStemmer` produces *candidate* spellings, not one canonical stem — the +/// corpus decides which candidate exists. So these assert that the right +/// spelling is offered, not that it's the only one or that it comes first. +struct QueryStemmerTests { + + private func stems(_ query: String) -> [String] { + QueryStemmer.stems(of: Array(query)).map { String($0) } + } + + @Test(arguments: [ + ("ghosted", "ghost"), + ("deployed", "deploy"), + ("launching", "launch"), + ("blocking", "block"), + ("hearts", "heart"), + ("wishes", "wish"), + ]) + func offersThePlainStem(query: String, stem: String) { + #expect(stems(query).contains(stem)) + } + + @Test(arguments: [ + ("parties", "party"), + ("carried", "carry"), + ]) + func restoresYForIesAndIed(query: String, stem: String) { + #expect(stems(query).contains(stem)) + } + + @Test(arguments: [ + ("celebrating", "celebrate"), + ("smiled", "smile"), + ]) + func restoresADroppedE(query: String, stem: String) { + #expect(stems(query).contains(stem)) + } + + @Test(arguments: [ + ("shipping", "ship"), + ("shipped", "ship"), + ]) + func undoesConsonantDoubling(query: String, stem: String) { + #expect(stems(query).contains(stem)) + } + + @Test func keepsTheLosslessStemAheadOfTheLossyOne() throws { + // "pressed" → "press" (real) must be offered before "pres" (junk from + // the doubled-consonant rule), since the caller stops at the first hit. + let candidates = stems("pressed") + let press = try #require(candidates.firstIndex(of: "press")) + if let pres = candidates.firstIndex(of: "pres") { + #expect(press < pres) + } + } + + @Test(arguments: ["press", "glass", "grass"]) + func neverStripsSFromADoubleS(query: String) { + #expect(!stems(query).contains(String(query.dropLast()))) + } + + @Test(arguments: ["ghost", "heart", "smile", "rocket", "cat"]) + func uninflectedQueryYieldsNothing(query: String) { + // Nothing to strip — the fallback should never fire on these. + #expect(stems(query).isEmpty) + } + + @Test(arguments: ["is", "as", "ing", "ed", "es"]) + func tooShortToStem(query: String) { + // A 2–3 char query would stem to something meaninglessly generic. + #expect(stems(query).isEmpty) + } + + @Test func neverEmitsAStemBelowThreeCharacters() { + for query in ["ties", "toes", "pies", "dyes", "axes", "owed", "ring"] { + #expect(stems(query).allSatisfy { $0.count >= 3 }) + } + } +}