diff --git a/locales/en.json b/locales/en.json index 5d4cf5e..c52754e 100644 --- a/locales/en.json +++ b/locales/en.json @@ -43,6 +43,7 @@ "dictionary": { "autoLearnTitle": "Auto-learn from corrections", "autoLearnDescription": "When you fix a transcript in another app, Whisper learns how the word was misheard and corrects it next time", + "autoLearnRequiresCleanup": "Enable and configure text cleanup to use auto-learn. Learned data is kept, and manually added dictionary words stay active.", "title": "Dictionary" }, "general": { @@ -727,6 +728,19 @@ } }, "addedToDict": "Added {{words}} to your dictionary", + "learningCorrection": "Learned correction {{corrections}}", + "learningContext": { + "title": "Remembered {{corrections}}", + "description": "This correction depends on context and will be judged in future dictations." + }, + "learningUnreadable": { + "title": "Cannot read this input field right now", + "description": "Auto-learn is temporarily unavailable. You can still dictate." + }, + "learningRecovered": { + "title": "Auto-learn is available again", + "description": "Corrections in this input field can now be learned." + }, "undo": "Undo", "cleanupFailed": { "title": "Cleanup failed", diff --git a/locales/zh-CN.json b/locales/zh-CN.json index 34ca9e1..8149dd8 100644 --- a/locales/zh-CN.json +++ b/locales/zh-CN.json @@ -43,6 +43,7 @@ "dictionary": { "autoLearnTitle": "从纠正中自动学习", "autoLearnDescription": "你在其他应用里改正转录内容后,Whisper 会记住这个词被误听成什么,下次自动修正", + "autoLearnRequiresCleanup": "开启并配置文本润色后可用。已学内容会保留,手动添加的词典仍可使用。", "title": "词典" }, "general": { @@ -706,6 +707,19 @@ } }, "addedToDict": "已将 {{words}} 添加到你的词典", + "learningCorrection": "已记住纠正 {{corrections}}", + "learningContext": { + "title": "已记下 {{corrections}}", + "description": "这处纠正有歧义,后续结合语境使用。" + }, + "learningUnreadable": { + "title": "暂时无法读取这个输入框", + "description": "自动学习暂时不可用,口述仍可使用。" + }, + "learningRecovered": { + "title": "已恢复自动学习", + "description": "接下来对这个输入框的纠正会继续学习。" + }, "undo": "撤消", "cleanupFailed": { "title": "清理失败", diff --git a/resources/native/macos-text-monitor.swift b/resources/native/macos-text-monitor.swift index 1988c9d..d9980aa 100644 --- a/resources/native/macos-text-monitor.swift +++ b/resources/native/macos-text-monitor.swift @@ -1,12 +1,8 @@ import Cocoa import Foundation import Darwin +import Carbon -let MAX_OUTPUT_BYTES = 10240 -let MAX_BASE64_SOURCE_CHARS = 7000 - -var monitoredElement: AXUIElement? -var observer: AXObserver? var monitoredPid: pid_t = 0 func writeOutput(_ message: String) { @@ -18,18 +14,6 @@ func writeError(_ message: String) { FileHandle.standardError.write((message + "\n").data(using: .utf8)!) } -func writeTextOutput(_ prefix: String, _ value: String) { - let truncated = String(value.prefix(MAX_OUTPUT_BYTES)) - if truncated.contains("\n") || truncated.contains("\r") { - let capped = String(truncated.prefix(MAX_BASE64_SOURCE_CHARS)) - let encoded = Data(capped.utf8).base64EncodedString() - writeOutput("\(prefix)_B64:\(encoded)") - return - } - - writeOutput("\(prefix):\(truncated)") -} - func isEditableTextElement(_ element: AXUIElement) -> Bool { var enabledValue: AnyObject? if AXUIElementCopyAttributeValue( @@ -283,14 +267,6 @@ func pasteTargetStatus(for targetPid: pid_t) -> PasteTargetStatus { return status } -func readCurrentValue() -> String? { - guard let element = monitoredElement else { return nil } - var value: AnyObject? - let result = AXUIElementCopyAttributeValue(element, kAXValueAttribute as CFString, &value) - guard result == .success, let str = value as? String else { return nil } - return str -} - func resolveFocusedElement(_ appElement: AXUIElement, pid: pid_t, maxRetries: Int = 5) -> AXUIElement? { for attempt in 1...maxRetries { var elementValue: AnyObject? @@ -326,32 +302,308 @@ func resolveFocusedElement(_ appElement: AXUIElement, pid: pid_t, maxRetries: In return nil } -func observerCallback( - _ observer: AXObserver, - _ element: AXUIElement, - _ notification: CFString, - _ refcon: UnsafeMutableRawPointer? -) { - if (notification as String) == kAXFocusedUIElementChangedNotification { - // The user moved on from the field; the session is over. - var focused: AnyObject? - if AXUIElementCopyAttributeValue( - AXUIElementCreateApplication(monitoredPid), - kAXFocusedUIElementAttribute as CFString, - &focused - ) == .success, let element = pasteTargetElement(focused), - let monitored = monitoredElement, !CFEqual(element, monitored) { - writeError("Focus left the monitored element") - CFRunLoopStop(CFRunLoopGetCurrent()) +// A read-only, multi-field AX server. Each identity is scoped to this helper +// process and follows the same AX element across application/focus switches. +final class ObservedInput { + let id: String + let element: AXUIElement + var active = false + var healthOnly = false + var hadContent = false + var lastValue: String? + var lastComposing = false + var compositionHint = false + var unreadable = false + var lastSendGesture: TimeInterval = 0 + var lastFocusedAt: TimeInterval = 0 + var expires = ProcessInfo.processInfo.systemUptime + 300 + init(_ id: String, _ element: AXUIElement) { self.id = id; self.element = element } +} + +var observedInputs: [String: ObservedInput] = [:] +var preparedInputs: [Int: (String, Bool, TimeInterval)] = [:] +var observationApp: AXUIElement? +var observationAX: AXObserver? +var unboundHealthUntil: TimeInterval = 0 +let maxObservedUTF16 = 256 * 1024 + +func observationOutput(_ value: [String: Any]) { + guard let data = try? JSONSerialization.data(withJSONObject: value, options: [.sortedKeys]), + let line = String(data: data, encoding: .utf8) else { return } + writeOutput(line) +} + +func observationValue(_ element: AXUIElement) -> String? { + if let value = pasteTargetAttribute(element, kAXValueAttribute) as? String { + return value.utf16.count <= maxObservedUTF16 ? value : nil + } + // Web contenteditable can omit AXValue while providing the standard + // parameterized text interface. Read the whole bounded field, never a + // prefix that silently omits the dictated passage near the end. + if let count = pasteTargetAttribute(element, kAXNumberOfCharactersAttribute) as? Int, + count >= 0, count <= maxObservedUTF16 { + var range = CFRange(location: 0, length: count) + if let parameter = AXValueCreate(.cfRange, &range) { + var result: AnyObject? + if AXUIElementCopyParameterizedAttributeValue( + element, kAXStringForRangeParameterizedAttribute as CFString, parameter, &result + ) == .success, let value = result as? String { return value } + } + } + // Chromium contenteditable trees sometimes expose only static text + // descendants. Restrict this walk to the bound editor, with hard budgets. + var pending: [(AXUIElement, Int)] = [(element, 0)] + var chunks: [String] = [] + var count = 0 + var visited = 0 + let deadline = ProcessInfo.processInfo.systemUptime + 0.05 + while let (node, depth) = pending.popLast() { + visited += 1 + guard visited <= 2000, ProcessInfo.processInfo.systemUptime < deadline else { return nil } + if pasteTargetAttribute(node, kAXRoleAttribute) as? String == "AXStaticText", + let value = pasteTargetAttribute(node, kAXValueAttribute) as? String { + count += value.utf16.count + guard count <= maxObservedUTF16 else { return nil } + chunks.append(value) + continue + } + if depth < 16, let children = pasteTargetAttribute(node, kAXChildrenAttribute) as? [AXUIElement] { + pending.append(contentsOf: children.reversed().map { ($0, depth + 1) }) + } + } + // Missing data must not masquerade as an empty, readable field. + return chunks.isEmpty ? nil : chunks.joined(separator: "\n") +} + +func observationSelection(_ element: AXUIElement) -> CFRange? { + guard let value = pasteTargetAttribute(element, kAXSelectedTextRangeAttribute), + CFGetTypeID(value) == AXValueGetTypeID() else { return nil } + var range = CFRange() + return AXValueGetValue(value as! AXValue, .cfRange, &range) ? range : nil +} + +func observationComposing(_ element: AXUIElement) -> Bool { + // Native input methods expose a marked range. Chromium versions that + // expose AXMarkedText use that string instead. Neither attribute is set. + if let value = pasteTargetAttribute(element, "AXMarkedTextRange"), + CFGetTypeID(value) == AXValueGetTypeID() { + var range = CFRange() + if AXValueGetValue(value as! AXValue, .cfRange, &range), range.location >= 0, range.length > 0 { return true } + } + return !(pasteTargetAttribute(element, "AXMarkedText") as? String ?? "").isEmpty +} + +func observationIsFocused(_ input: ObservedInput) -> Bool { + guard keyboardFocusPid() == monitoredPid, let app = observationApp, + let focused = pasteTargetElement(pasteTargetAttribute(app, kAXFocusedUIElementAttribute)) else { return false } + return CFEqual(focused, input.element) +} + +func observationIsSendButton(_ element: AXUIElement) -> Bool { + guard pasteTargetAttribute(element, kAXRoleAttribute) as? String == "AXButton" else { return false } + let label = [kAXTitleAttribute, kAXDescriptionAttribute, kAXHelpAttribute] + .compactMap { pasteTargetAttribute(element, $0) as? String }.joined(separator: " ").lowercased() + return ["send", "submit", "发送", "提交"].contains(where: { label.contains($0) }) +} + +func observationSample(_ input: ObservedInput) { + guard input.active else { return } + let now = ProcessInfo.processInfo.systemUptime + let focused = observationIsFocused(input) + if focused { input.lastFocusedAt = now } + else { + // Accessibility activation of a Send button need not emit a mouse + // event. A just-focused Send button can confirm a subsequent clear. + if now - input.lastFocusedAt < 0.7, keyboardFocusPid() == monitoredPid, + let app = observationApp, + let button = pasteTargetElement(pasteTargetAttribute(app, kAXFocusedUIElementAttribute)), + observationIsSendButton(button) { + input.lastSendGesture = now } + if now - input.lastSendGesture >= 0.7 { return } + } + guard let value = observationValue(input.element) else { + if !input.unreadable { + observationOutput(["kind": "unreadable", "field": input.id, "reason": "ax_text_unavailable_or_exceeds_256k_utf16"]) + } + input.unreadable = true return } - if let value = readCurrentValue() { - writeTextOutput("CHANGED", value) + let composing = observationComposing(input.element) || input.compositionHint + let empty = value.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty + if empty, input.hadContent, !input.healthOnly, + ProcessInfo.processInfo.systemUptime - input.lastSendGesture < 0.7 { + observationOutput(["kind": "sent", "field": input.id]) + input.active = false + input.hadContent = false + input.lastValue = nil + return } + // A Send action may briefly move focus away. Only its clear is relevant; + // normal edits in an unfocused field remain paused. + guard focused else { return } + guard input.unreadable || value != input.lastValue || composing != input.lastComposing else { return } + input.unreadable = false + observationOutput(["kind": "value", "field": input.id, "value": value, "composing": composing]) + input.lastValue = value + if !empty { input.hadContent = true } + input.lastComposing = composing } -// Usage: macos-text-monitor +func observationCallback(_ observer: AXObserver, _ element: AXUIElement, + _ notification: CFString, _ refcon: UnsafeMutableRawPointer?) { + if notification as String == kAXUIElementDestroyedNotification { + for input in Array(observedInputs.values) where CFEqual(input.element, element) { + observationOutput(["kind": "closed", "field": input.id]) + observedInputs.removeValue(forKey: input.id) + } + return + } + for input in observedInputs.values where CFEqual(input.element, element) { observationSample(input) } +} + +func observationCommand(_ command: [String: Any]) { + let name = command["command"] as? String ?? "" + let ticket = command["ticket"] as? Int ?? 0 + switch name { + case "prepare": + guard let app = observationApp, + keyboardFocusPid() == monitoredPid, + let element = resolveFocusedElement(app, pid: monitoredPid, maxRetries: 1) else { + observationOutput(["kind": "prepared", "ticket": ticket, "reason": "ax_focused_element_unavailable"]) + return + } + let input = observedInputs.values.first { CFEqual($0.element, element) } ?? + ObservedInput(UUID().uuidString, element) + observedInputs[input.id] = input + preparedInputs[ticket] = (input.id, input.active, input.expires) + if let observer = observationAX { + // Polling remains active if a web editor rejects notifications. + _ = AXObserverAddNotification(observer, element, kAXValueChangedNotification as CFString, nil) + _ = AXObserverAddNotification(observer, element, kAXSelectedTextChangedNotification as CFString, nil) + _ = AXObserverAddNotification(observer, element, kAXUIElementDestroyedNotification as CFString, nil) + } + guard let value = observationValue(element), + let selection = observationSelection(element) ?? (value.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty ? CFRange(location: 0, length: 0) : nil) else { + observationOutput(["kind": "prepared", "ticket": ticket, "field": input.id, + "reason": "ax_value_or_selection_unavailable"]) + return + } + observationSample(input) + // Pause this field until commit so Command-V cannot be mistaken for + // a correction to the preceding dictation while the ticket is open. + input.active = false + observationOutput(["kind": "prepared", "ticket": ticket, "field": input.id, + "value": value, "start": selection.location, "length": selection.length]) + case "commit", "cancel", "health": + if let (id, wasActive, oldExpiry) = preparedInputs.removeValue(forKey: ticket), let input = observedInputs[id] { + if name == "commit" { + input.active = true + input.healthOnly = false + input.expires = ProcessInfo.processInfo.systemUptime + 300 + } else if name == "health" { + input.active = true + input.healthOnly = input.healthOnly || !wasActive + input.expires = wasActive ? oldExpiry : ProcessInfo.processInfo.systemUptime + 300 + } else { + input.active = wasActive + input.expires = oldExpiry + if !wasActive { observedInputs.removeValue(forKey: id) } + } + observationSample(input) + } else if name == "health" { + unboundHealthUntil = ProcessInfo.processInfo.systemUptime + 300 + } + case "snapshot": + for input in observedInputs.values { observationSample(input) } + observationOutput(["kind": "synced", "ticket": ticket]) + case "forget": + if let id = command["field"] as? String { observedInputs.removeValue(forKey: id) } + default: break + } +} + +func runObservationServer(_ pid: pid_t) -> Never { + monitoredPid = pid + observationApp = AXUIElementCreateApplication(pid) + var created: AXObserver? + if AXObserverCreate(pid, observationCallback, &created) == .success, let created = created { + observationAX = created + CFRunLoopAddSource(CFRunLoopGetCurrent(), AXObserverGetRunLoopSource(created), .commonModes) + } + let events = NSEvent.addGlobalMonitorForEvents(matching: [.keyDown, .leftMouseDown]) { event in + guard keyboardFocusPid() == pid else { return } + let now = ProcessInfo.processInfo.systemUptime + let inputSource = TISCopyCurrentKeyboardInputSource().takeRetainedValue() + let asciiProperty = TISGetInputSourceProperty(inputSource, kTISPropertyInputSourceIsASCIICapable) + let asciiCapable = asciiProperty.map { CFBooleanGetValue(Unmanaged.fromOpaque($0).takeUnretainedValue()) } ?? true + var sendButtonClick = false + if event.type == .leftMouseDown { + var hit: AXUIElement? + let point = event.cgEvent?.location ?? .zero + if AXUIElementCopyElementAtPosition(AXUIElementCreateSystemWide(), Float(point.x), Float(point.y), &hit) == .success, + let hit = hit { + sendButtonClick = observationIsSendButton(hit) + } + } + for input in observedInputs.values where input.active && + (observationIsFocused(input) || (sendButtonClick && now - input.lastFocusedAt < 0.3)) { + let wasComposing = input.compositionHint || observationComposing(input.element) + // When an input method omits AX marked-text attributes, retain a + // conservative composing hint until its commit key or candidate + // click. This avoids learning a paused Chinese preedit. + if event.type == .leftMouseDown || [36, 76, 49, 53].contains(event.keyCode) { + input.compositionHint = false + } else if event.type == .keyDown && !asciiCapable && + event.modifierFlags.intersection([.command, .control]).isEmpty && + !(event.characters ?? "").isEmpty { + input.compositionHint = true + } + // Return used to commit an input-method candidate is not Send. + // A caret click is never a send hint, nor is Delete after a click. + if event.type == .keyDown && [51, 117].contains(event.keyCode) { + input.lastSendGesture = 0 + } else if event.type == .keyDown && [36, 76].contains(event.keyCode) && !wasComposing { + input.lastSendGesture = now + } else if sendButtonClick { + input.lastSendGesture = now + } + observationSample(input) + } + } + let timer = Timer(timeInterval: 0.1, repeats: true) { _ in + let now = ProcessInfo.processInfo.systemUptime + if now < unboundHealthUntil, keyboardFocusPid() == pid, let app = observationApp, + let focused = pasteTargetElement(pasteTargetAttribute(app, kAXFocusedUIElementAttribute)), + isEditableTextElement(focused) || isEditableBrowserPasteTarget(focused), + let value = observationValue(focused), observationSelection(focused) != nil || value.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty { + unboundHealthUntil = 0 + observationOutput(["kind": "health_recovered"]) + } + for input in Array(observedInputs.values) { + if now >= input.expires { + observationOutput(["kind": "closed", "field": input.id]) + observedInputs.removeValue(forKey: input.id) + } else { observationSample(input) } + } + } + RunLoop.main.add(timer, forMode: .common) + // Read complete newline-delimited commands off the AX run loop. + DispatchQueue.global(qos: .utility).async { + while let line = readLine() { + guard let data = line.data(using: .utf8), + let command = try? JSONSerialization.jsonObject(with: data) as? [String: Any] else { continue } + DispatchQueue.main.async { observationCommand(command) } + } + DispatchQueue.main.async { CFRunLoopStop(CFRunLoopGetCurrent()) } + } + CFRunLoopRun() + if let events = events { NSEvent.removeMonitor(events) } + exit(0) +} + + +// Usage: macos-text-monitor --observe // macos-text-monitor --paste-target // macos-text-monitor --focused-app let pasteTargetMode = CommandLine.arguments.count >= 2 && @@ -372,105 +624,16 @@ if CommandLine.arguments.count >= 2 && CommandLine.arguments[1] == "--focused-ap exit(0) } -let pidArgumentIndex = pasteTargetMode ? 2 : 1 - -guard CommandLine.arguments.count > pidArgumentIndex, - let targetPid = Int32(CommandLine.arguments[pidArgumentIndex]), - targetPid > 0 else { - writeError("Usage: macos-text-monitor [--paste-target] | --focused-app") - writeOutput(pasteTargetMode ? "UNKNOWN" : "NO_ELEMENT") - exit(1) +if CommandLine.arguments.count >= 3, CommandLine.arguments[1] == "--observe", + let pid = Int32(CommandLine.arguments[2]), pid > 0 { + runObservationServer(pid) } -monitoredPid = targetPid - -// Dictation probing does not use the monitor's retry ladder. -if pasteTargetMode { - writeOutput(pasteTargetStatus(for: targetPid).rawValue) +if pasteTargetMode, CommandLine.arguments.count >= 3, + let pid = Int32(CommandLine.arguments[2]), pid > 0 { + writeOutput(pasteTargetStatus(for: pid).rawValue) exit(0) } - -// Target the specific application by PID (passed from the app, which captures -// it BEFORE the overlay steals focus). -guard CommandLine.arguments.count > 2, - let timeoutSeconds = Double(CommandLine.arguments[2]) else { - writeError("Usage: macos-text-monitor ") - writeOutput("NO_ELEMENT") - exit(1) -} - -let appElement = AXUIElementCreateApplication(monitoredPid) -guard let resolvedElement = resolveFocusedElement(appElement, pid: monitoredPid) else { - writeOutput("NO_ELEMENT") - exit(1) -} - -monitoredElement = resolvedElement -writeError("Monitoring element in PID \(monitoredPid)") - -// Only a field with a text value can be monitored. -guard readCurrentValue() != nil else { - writeError("Focused element has no text value") - writeOutput("NO_VALUE") - exit(0) -} - -// Create AXObserver for the target application's PID -var createdObserver: AXObserver? -let observerResult = AXObserverCreate(monitoredPid, observerCallback, &createdObserver) - -guard observerResult == .success, let obs = createdObserver else { - writeError("Failed to create AXObserver (error: \(observerResult.rawValue))") - exit(1) -} - -observer = obs - -// Watch for value changes -let addResult = AXObserverAddNotification( - obs, - monitoredElement!, - kAXValueChangedNotification as CFString, - nil -) - -if addResult != .success { - writeError("Failed to add notification (error: \(addResult.rawValue))") - exit(1) -} - -// Best effort: without it the session just runs to its timeout. -_ = AXObserverAddNotification( - obs, - appElement, - kAXFocusedUIElementChangedNotification as CFString, - nil -) - -// Everything from here on arrives as CHANGED, so nothing falls between this -// read and the first notification. -if let value = readCurrentValue() { - writeTextOutput("BASELINE", value) -} - -// Add observer to run loop -CFRunLoopAddSource( - CFRunLoopGetCurrent(), - AXObserverGetRunLoopSource(obs), - .commonModes -) - -// Schedule auto-exit after timeout -DispatchQueue.main.asyncAfter(deadline: .now() + timeoutSeconds) { - CFRunLoopStop(CFRunLoopGetCurrent()) -} - -// Handle SIGTERM for clean exit -let signalSource = DispatchSource.makeSignalSource(signal: SIGTERM, queue: .main) -signal(SIGTERM, SIG_IGN) -signalSource.setEventHandler { - CFRunLoopStop(CFRunLoopGetCurrent()) -} -signalSource.resume() - -CFRunLoopRun() +writeError("Usage: macos-text-monitor --observe | --paste-target | --focused-app") +writeOutput("UNKNOWN") +exit(1) diff --git a/src/app.rs b/src/app.rs index b150381..7e9cc1c 100644 --- a/src/app.rs +++ b/src/app.rs @@ -307,6 +307,9 @@ pub fn run() { AppState::init(settings, db, cx); ui::overlays::Overlays::init(cx); Dictation::init(cx); + crate::native::text_monitor::TextEditMonitor::shared() + .set_enabled(AppState::settings(cx).auto_learn_enabled()); + backend::start_learning_listener(cx); cx.bind_keys([ KeyBinding::new("cmd-k", OpenSearch, Some("ControlPanel")), KeyBinding::new("cmd-,", ShowSettings, None), @@ -318,6 +321,7 @@ pub fn run() { ]); cx.on_app_quit(|cx| { AppState::global(cx).update(cx, |state, _| state.invalidate_learning()); + crate::native::text_monitor::TextEditMonitor::shared().set_enabled(false); if let Some(manager) = backend::hotkeys() { manager.shutdown(); } diff --git a/src/app_state.rs b/src/app_state.rs index 9412e04..b8d8af4 100644 --- a/src/app_state.rs +++ b/src/app_state.rs @@ -15,7 +15,10 @@ use std::sync::atomic::{AtomicU64, Ordering}; use gpui_kit::*; use crate::audio_storage::{AudioStorage, RetentionCleanupReport}; -use crate::db::{Database, DictionaryEntry, DictionarySource, Snippet, TranscriptionItem}; +use crate::db::{ + CorrectionLearningOutcome, Database, DictionaryEntry, DictionarySource, Snippet, + TranscriptionItem, +}; use crate::history::{HistoryStore, HistoryWrite}; use crate::i18n::{self, Language}; use crate::native::text_monitor::SessionReport; @@ -167,8 +170,10 @@ impl AppState { } }) .detach(); - if before.auto_learn_corrections != self.settings.auto_learn_corrections { + if before.auto_learn_enabled() != self.settings.auto_learn_enabled() { self.dictionary_revision.fetch_add(1, Ordering::SeqCst); + crate::native::text_monitor::TextEditMonitor::shared() + .set_enabled(self.settings.auto_learn_enabled()); } if before.theme != self.settings.theme { theme::apply(self.settings.theme, cx); @@ -461,23 +466,27 @@ impl AppState { .detach(); } - /// Learns from a finished post-paste session. Returns the words added to - /// the Dictionary. + /// Replaces a session's learning contribution with its latest completed edit. pub fn learn_from_session( &mut self, report: SessionReport, cx: &mut Context, - ) -> Task> { + ) -> Task { let revision = self.dictionary_revision.load(Ordering::SeqCst); let token = self.dictionary_revision.clone(); cx.spawn(async move |this, cx| { - let SessionReport { edit, .. } = report; + let SessionReport { + edit, + session_id, + revision: session_revision, + .. + } = report; let corrections = if let Some(edit) = edit { crate::services::runtime::blocking(move || { - crate::text::correction_learner::extract_corrections( + crate::text::correction_learner::extract_corrections_in_regions( &edit.baseline, &edit.edited, - &edit.pasted, + &edit.pasted_ranges, ) }) .await @@ -487,27 +496,32 @@ impl AppState { }; let Ok(Some(db)) = this.update(cx, |this, _| { (this.dictionary_revision.load(Ordering::SeqCst) == revision - && this.settings.auto_learn_corrections) - .then(|| this.db.clone()) + && this.settings.auto_learn_enabled()) + .then(|| this.db.clone()) }) else { - return Vec::new(); + return CorrectionLearningOutcome::default(); }; let (learned, dictionary) = crate::services::runtime::storage(move || { let learned = db - .record_corrections_if_current(&corrections, &token, revision) + .record_session_corrections_if_current( + &session_id, + session_revision, + &corrections, + &token, + revision, + ) .inspect_err(|err| log::warn!("failed to save learned words: {err}")) .unwrap_or_default(); - let dictionary = (!corrections.is_empty()).then(|| DictionarySnapshot::read(&db)); + // An empty replacement can revoke an earlier learned correction. + let dictionary = DictionarySnapshot::read(&db); (learned, dictionary) }) .await; this.update(cx, |this, cx| { if this.dictionary_revision.load(Ordering::SeqCst) != revision { - return Vec::new(); - } - if let Some(dictionary) = dictionary { - this.apply_dictionary_snapshot(dictionary, cx); + return CorrectionLearningOutcome::default(); } + this.apply_dictionary_snapshot(dictionary, cx); learned }) .unwrap_or_default() diff --git a/src/backend.rs b/src/backend.rs index 9c280e4..b2e3725 100644 --- a/src/backend.rs +++ b/src/backend.rs @@ -296,14 +296,28 @@ pub fn paste_text( text: String, target_pid: Option, output_mode: crate::settings::OutputMode, + learn: bool, ) -> impl std::future::Future + 'static { let (tx, rx) = futures::channel::oneshot::channel(); std::thread::spawn(move || { - crate::native::text_monitor::TextEditMonitor::shared().stop_monitoring(); - let outcome = crate::native::paste::paste_text( - &text, - crate::native::paste::PasteOptions::new(target_pid, output_mode), - ); + let options = crate::native::paste::PasteOptions::new(target_pid, output_mode); + let outcome = if learn { + crate::native::paste::Paster::shared().paste_text_observed( + &text, + options, + |text, pid| { + let prepared = crate::native::text_monitor::TextEditMonitor::shared() + .prepare_paste(text, pid); + move |pasted| { + if let Some(prepared) = prepared { + prepared.commit(pasted); + } + } + }, + ) + } else { + crate::native::paste::paste_text(&text, options) + }; if let Some(failure) = &outcome.failure { log::warn!("automatic paste failed: {failure:?}"); } @@ -312,26 +326,123 @@ pub fn paste_text( async move { rx.await.unwrap_or(false) } } -/// Watches the target field after an automatic paste and adds learned -/// corrections to the Dictionary (auto-learn). Resolves to the words added -/// once the session ends. -pub fn monitor_corrections( - text: String, - target_pid: Option, - cx: &mut gpui_kit::App, -) -> gpui_kit::Task> { - let reports = crate::native::text_monitor::TextEditMonitor::shared() - .start_monitoring_after_paste(&text, target_pid); - cx.spawn(async move |cx| { - let Ok(report) = reports.recv().await else { - return Vec::new(); - }; - cx.update(|cx| { - crate::app_state::AppState::global(cx) - .update(cx, |s, cx| s.learn_from_session(report, cx)) +/// Consumes completed edits and observation health independently of dictation +/// generations, so later edits to earlier passages are not discarded. +struct LearningBarriers(async_channel::Sender>); +impl gpui_kit::Global for LearningBarriers {} + +/// Waits for completed native edits and their queued dictionary writes before +/// taking the next pipeline snapshot. Unfinished composition remains pending. +pub fn flush_learning(cx: &mut gpui_kit::App) -> gpui_kit::Task<()> { + let barriers = cx.global::().0.clone(); + cx.spawn(async move |_| { + let _ = crate::services::runtime::blocking(|| { + crate::native::text_monitor::TextEditMonitor::shared().flush_completed_edits(); }) - .await + .await; + let (done, completed) = futures::channel::oneshot::channel(); + if barriers.send(done).await.is_ok() { + let _ = completed.await; + } + }) +} + +pub fn start_learning_listener(cx: &mut gpui_kit::App) { + use crate::app_state::AppState; + use crate::dictation::{Dictation, DictationNotice}; + use crate::native::text_monitor::{ObservationOutcome, TextEditMonitor}; + use futures::FutureExt; + + let reports = TextEditMonitor::shared().subscribe(); + let (barrier_sender, barriers) = async_channel::unbounded(); + cx.set_global(LearningBarriers(barrier_sender)); + cx.spawn(async move |cx| { + loop { + let report = reports.recv().fuse(); + let barrier = barriers.recv().fuse(); + futures::pin_mut!(report, barrier); + let report = futures::select_biased! { + report = report => match report { Ok(report) => report, Err(_) => break }, + barrier = barrier => { + if let Ok(barrier) = barrier { + let _ = barrier.send(()); + } + continue; + } + }; + if let Some(diagnostic) = &report.diagnostic { + log::debug!( + "[auto-learn] outcome={:?} app={:?} field={:?} reason={diagnostic}", + report.outcome, + report.app, + report.field + ); + } + if !cx.update(|cx| AppState::settings(cx).auto_learn_enabled()) { + continue; + } + match report.outcome { + ObservationOutcome::Unreadable | ObservationOutcome::Recovered => { + let notice = if report.outcome == ObservationOutcome::Unreadable { + DictationNotice::LearningUnreadable + } else { + DictationNotice::LearningRecovered + }; + cx.update(|cx| { + Dictation::global(cx).update(cx, |_, cx| cx.emit(notice)); + }); + } + ObservationOutcome::CompletedEdit => { + let learning = cx.update(|cx| { + AppState::global(cx) + .update(cx, |state, cx| state.learn_from_session(report, cx)) + }); + let outcome = learning.await; + cx.update(|cx| { + if !AppState::settings(cx).auto_learn_enabled() { + return; + } + let contextual: std::collections::HashSet<_> = outcome + .context_dependent_terms + .iter() + .map(String::as_str) + .collect(); + let clear: Vec<_> = outcome + .added_terms + .iter() + .filter(|term| !contextual.contains(term.as_str())) + .cloned() + .collect(); + let existing_word_pairs: Vec<_> = outcome + .clear_correction_pairs + .into_iter() + .filter(|pair| !outcome.added_terms.contains(&pair.term)) + .map(|pair| (pair.misheard, pair.term)) + .collect(); + Dictation::global(cx).update(cx, |_, cx| { + if !clear.is_empty() { + cx.emit(DictationNotice::LearnedWords(clear)); + } + if !existing_word_pairs.is_empty() { + cx.emit(DictationNotice::LearnedCorrection(existing_word_pairs)); + } + if !outcome.context_dependent_pairs.is_empty() { + cx.emit(DictationNotice::ContextDependentCorrection( + outcome + .context_dependent_pairs + .into_iter() + .map(|pair| (pair.misheard, pair.term)) + .collect(), + )); + } + }); + }); + } + ObservationOutcome::Readable => {} + } + } }) + .detach(); } /// Removes words auto-learn just added (the toast's Undo). diff --git a/src/db.rs b/src/db.rs index 4b6a997..d0c6ed2 100644 --- a/src/db.rs +++ b/src/db.rs @@ -13,7 +13,7 @@ use chrono::{DateTime, Duration, Utc}; use parking_lot::{Mutex, MutexGuard}; use rusqlite::{Connection, OptionalExtension, Row, TransactionBehavior, params}; -pub use dictionary::{DictionaryEntry, DictionarySource}; +pub use dictionary::{CorrectionLearningOutcome, DictionaryEntry, DictionarySource}; pub use snippets::{MAX_SNIPPET_TRIGGER_LENGTH, Snippet}; /// Shared handle to the database; cheap to share behind an `Arc`. diff --git a/src/db/dictionary.rs b/src/db/dictionary.rs index 4850b2d..f4aae0d 100644 --- a/src/db/dictionary.rs +++ b/src/db/dictionary.rs @@ -8,7 +8,7 @@ use std::sync::atomic::{AtomicU64, Ordering}; use anyhow::Result; use chrono::{DateTime, Utc}; -use rusqlite::{Connection, Transaction, TransactionBehavior, params}; +use rusqlite::{Connection, OptionalExtension, Transaction, TransactionBehavior, params}; use unicode_normalization::UnicodeNormalization; use super::Database; @@ -52,6 +52,15 @@ pub struct DictionaryEntry { pub last_used_at: Option>, } +/// Newly remembered vocabulary and corrections that need the current context. +#[derive(Debug, Clone, Default, PartialEq, Eq)] +pub struct CorrectionLearningOutcome { + pub added_terms: Vec, + pub context_dependent_terms: Vec, + pub context_dependent_pairs: Vec, + pub clear_correction_pairs: Vec, +} + impl DictionaryEntry { /// Projects a queued edit without waiting for SQLite. Normalization and /// insertion order match `apply_dictionary_changes`; stored metadata is @@ -291,6 +300,8 @@ fn upsert_word( // SQLite NOCASE cannot follow a Unicode case-only respelling. tx.prepare_cached("UPDATE dictionary_corrections SET term = ? WHERE term = ?")? .execute(params![word, existing.word])?; + tx.prepare_cached("UPDATE dictionary_session_corrections SET term = ? WHERE term = ?")? + .execute(params![word, existing.word])?; } Ok(()) } @@ -302,6 +313,19 @@ fn rejected_words(conn: &Connection) -> Result> { .collect::>()?) } +fn respell_case_correction( + tx: &Transaction<'_>, + existing: &mut ExistingRow, + misheard: &str, + term: &str, +) -> Result<()> { + if misheard.to_lowercase() == term.to_lowercase() { + upsert_word(tx, term, Some(existing), DictionarySource::Learned.as_str())?; + existing.word = term.to_string(); + } + Ok(()) +} + fn delete_row(tx: &Transaction<'_>, existing: Option<&ExistingRow>) -> Result<()> { let Some(existing) = existing else { return Ok(()); @@ -315,31 +339,15 @@ fn delete_row(tx: &Transaction<'_>, existing: Option<&ExistingRow>) -> Result<() // What the user took out stays out, and its mishearings stop applying. tx.prepare_cached("DELETE FROM dictionary_corrections WHERE term = ?")? .execute([&existing.word])?; + tx.prepare_cached("DELETE FROM dictionary_session_corrections WHERE term = ?")? + .execute([&existing.word])?; + tx.prepare_cached("DELETE FROM dictionary_session_terms WHERE word_id = ?")? + .execute([existing.id])?; tx.prepare_cached("INSERT OR IGNORE INTO dictionary_rejected_words (word) VALUES (?)")? .execute([&existing.word])?; Ok(()) } -/// Counts one sighting of `correction` and returns the sightings of its term. -/// A correction that reverses a stored one replaces it. -fn record_correction(tx: &Transaction<'_>, correction: &Correction) -> Result { - let Correction { misheard, term, .. } = correction; - if misheard.to_lowercase() != term.to_lowercase() { - tx.prepare_cached("DELETE FROM dictionary_corrections WHERE misheard = ? AND term = ?")? - .execute(params![term, misheard])?; - } - tx.prepare_cached( - "INSERT INTO dictionary_corrections (misheard, term) VALUES (?, ?) - ON CONFLICT (misheard, term) DO UPDATE SET - count = count + 1, term = excluded.term, updated_at = excluded.updated_at", - )? - .execute(params![misheard, term])?; - let sightings = tx - .prepare_cached("SELECT SUM(count) FROM dictionary_corrections WHERE term = ?")? - .query_row([term], |row| row.get(0))?; - Ok(sightings) -} - impl Database { /// Words in insertion order. #[cfg(test)] @@ -499,72 +507,196 @@ impl Database { Ok(()) } - /// Stores what the user corrected and adds the terms that earned a place - /// in the Dictionary: a strong one at once, a weak one on its second - /// sighting. Words the user removed are ignored. A term already in the + /// Stores the correction after its first completed edit. Context-dependent + /// pairs stay subject to the cleanup model. Words the user removed are + /// ignored. A term already in the /// Dictionary only takes the capitalization the user just gave it. /// Returns the words added. #[cfg(test)] pub fn record_corrections(&self, corrections: &[Correction]) -> Result> { + Ok(self + .record_corrections_with_outcome(corrections)? + .added_terms) + } + + #[cfg(test)] + pub fn record_corrections_with_outcome( + &self, + corrections: &[Correction], + ) -> Result { self.record_corrections_checked(corrections, None) } /// Stores a background learning result only while its dictionary generation /// is still current. Check under the connection lock so a pending result /// cannot undo a clear or another dictionary edit that invalidated it. + #[cfg(test)] pub fn record_corrections_if_current( &self, corrections: &[Correction], generation: &AtomicU64, expected: u64, ) -> Result> { - self.record_corrections_checked(corrections, Some((generation, expected))) + Ok(self + .record_corrections_checked(corrections, Some((generation, expected)))? + .added_terms) } - fn record_corrections_checked( + /// Reconciles the latest completed edit of one observation session. + /// Repeated snapshots and stale revisions cannot add evidence or feedback. + pub fn record_session_corrections_if_current( &self, + session_id: &str, + revision: u64, corrections: &[Correction], - generation: Option<(&AtomicU64, u64)>, - ) -> Result> { - if corrections.is_empty() { - return Ok(Vec::new()); - } + generation: &AtomicU64, + expected: u64, + ) -> Result { let mut conn = self.lock(); - if generation.is_some_and(|(current, expected)| current.load(Ordering::Acquire) != expected) - { - return Ok(Vec::new()); + if generation.load(Ordering::Acquire) != expected { + return Ok(CorrectionLearningOutcome::default()); } - let mut by_lower = dictionary_rows(&conn)?; + let revision = i64::try_from(revision)?; let tx = conn.transaction_with_behavior(TransactionBehavior::Immediate)?; + let previous_revision: Option = tx + .query_row( + "SELECT revision FROM dictionary_learning_sessions WHERE session_id = ?", + [session_id], + |row| row.get(0), + ) + .optional()?; + if previous_revision.is_some_and(|previous| revision <= previous) { + return Ok(CorrectionLearningOutcome::default()); + } + let previous: Vec<(String, String)> = tx + .prepare( + "SELECT misheard, term FROM dictionary_session_corrections WHERE session_id = ?", + )? + .query_map([session_id], |row| Ok((row.get(0)?, row.get(1)?)))? + .collect::>()?; + let previous_keys: HashSet<_> = previous + .iter() + .map(|(misheard, term)| (misheard.to_lowercase(), term.to_lowercase())) + .collect(); let rejected: HashSet = rejected_words(&tx)? .into_iter() .map(|word| word.to_lowercase()) .collect(); - let mut added: Vec = Vec::new(); + let mut by_lower = dictionary_rows(&tx)?; + let mut outcome = CorrectionLearningOutcome::default(); + let mut seen = HashSet::new(); for correction in corrections { let term = trim_text(&correction.term); let lower = term.to_lowercase(); + let key = (correction.misheard.to_lowercase(), lower.clone()); if term.is_empty() || rejected.contains(&lower) { continue; } - let sightings = record_correction(&tx, correction)?; - let source = DictionarySource::Learned.as_str(); - if let Some(existing) = by_lower.get_mut(&lower) { - if correction.misheard.to_lowercase() == lower { - upsert_word(&tx, term, Some(existing), source)?; - existing.word = term.to_string(); + if !seen.insert(key.clone()) || previous_keys.contains(&key) { + if let Some(existing) = by_lower.get_mut(&lower) { + respell_case_correction(&tx, existing, &correction.misheard, term)?; } continue; } - let earned = !correction.needs_repeat || sightings >= 2; - let is_new = !added.iter().any(|word| word.to_lowercase() == lower); - if earned && is_new { - upsert_word(&tx, term, None, source)?; - added.push(term.to_string()); + let known = tx + .query_row( + "SELECT 1 FROM dictionary_corrections WHERE misheard = ? AND term = ?", + params![correction.misheard, term], + |_| Ok(()), + ) + .optional()? + .is_some(); + tx.execute( + "INSERT INTO dictionary_corrections (misheard, term) VALUES (?, ?) + ON CONFLICT (misheard, term) DO UPDATE SET count = count + 1, updated_at = excluded.updated_at", + params![correction.misheard, term], + )?; + tx.execute( + "INSERT INTO dictionary_session_corrections (session_id, misheard, term) VALUES (?, ?, ?)", + params![session_id, correction.misheard, term], + )?; + let automatic = (KnownCorrection { + misheard: correction.misheard.clone(), + term: term.to_string(), + count: 1, + }) + .applies_automatically(); + if !known && automatic { + outcome.clear_correction_pairs.push(correction.clone()); + } + if !known && !automatic { + if !outcome + .context_dependent_terms + .iter() + .any(|word| word.to_lowercase() == lower) + { + outcome.context_dependent_terms.push(term.to_string()); + } + outcome.context_dependent_pairs.push(correction.clone()); + } + if let Some(existing) = by_lower.get_mut(&lower) { + respell_case_correction(&tx, existing, &correction.misheard, term)?; + } else { + upsert_word(&tx, term, None, DictionarySource::Learned.as_str())?; + tx.execute( + "INSERT INTO dictionary_session_terms (word_id) VALUES (?)", + [tx.last_insert_rowid()], + )?; + outcome.added_terms.push(term.to_string()); + by_lower = dictionary_rows(&tx)?; } } + for (misheard, term) in previous { + if seen.contains(&(misheard.to_lowercase(), term.to_lowercase())) { + continue; + } + tx.execute( + "UPDATE dictionary_corrections SET count = count - 1 WHERE misheard = ? AND term = ?", + params![misheard, term], + )?; + tx.execute( + "DELETE FROM dictionary_corrections WHERE misheard = ? AND term = ? AND count <= 0", + params![misheard, term], + )?; + tx.execute( + "DELETE FROM dictionary_session_corrections WHERE session_id = ? AND misheard = ? AND term = ?", + params![session_id, misheard, term], + )?; + } + tx.execute( + "DELETE FROM custom_dictionary WHERE source = 'learned' + AND id IN (SELECT word_id FROM dictionary_session_terms) + AND NOT EXISTS (SELECT 1 FROM dictionary_corrections c WHERE c.term = custom_dictionary.word)", + [], + )?; + tx.execute( + "DELETE FROM dictionary_session_terms WHERE word_id NOT IN (SELECT id FROM custom_dictionary)", + [], + )?; + tx.execute( + "INSERT INTO dictionary_learning_sessions (session_id, revision) VALUES (?, ?) + ON CONFLICT (session_id) DO UPDATE SET revision = excluded.revision", + params![session_id, revision], + )?; tx.commit()?; - Ok(added) + Ok(outcome) + } + + #[cfg(test)] + fn record_corrections_checked( + &self, + corrections: &[Correction], + generation: Option<(&AtomicU64, u64)>, + ) -> Result { + let default_generation = AtomicU64::new(0); + let (generation, expected) = generation.unwrap_or((&default_generation, 0)); + self.record_session_corrections_if_current( + &uuid::Uuid::new_v4().to_string(), + 1, + corrections, + generation, + expected, + ) } /// Corrections to terms in the Dictionary, most repeated first. A mishearing @@ -601,16 +733,32 @@ impl Database { tx.execute("DELETE FROM custom_dictionary", [])?; tx.execute("DELETE FROM dictionary_corrections", [])?; tx.execute("DELETE FROM dictionary_rejected_words", [])?; + tx.execute("DELETE FROM dictionary_session_corrections", [])?; + tx.execute("DELETE FROM dictionary_session_terms", [])?; + tx.execute("DELETE FROM dictionary_learning_sessions", [])?; tx.commit()?; Ok(()) } /// Forgets one mishearing of `term`. pub fn remove_correction(&self, misheard: &str, term: &str) -> Result<()> { - self.lock().execute( + let mut conn = self.lock(); + let tx = conn.transaction_with_behavior(TransactionBehavior::Immediate)?; + tx.execute( "DELETE FROM dictionary_corrections WHERE misheard = ? AND term = ?", params![misheard, term], )?; + tx.execute( + "DELETE FROM dictionary_session_corrections WHERE misheard = ? AND term = ?", + params![misheard, term], + )?; + // Explicitly keeping the word while forgetting its correction makes it + // independent of any older observation session. + tx.execute( + "DELETE FROM dictionary_session_terms WHERE word_id IN (SELECT id FROM custom_dictionary WHERE word = ?)", + [term], + )?; + tx.commit()?; Ok(()) } @@ -1054,6 +1202,219 @@ mod tests { ); } + #[test] + fn a_context_dependent_correction_is_remembered_on_the_first_edit() { + let db = db(); + let outcome = db + .record_corrections_with_outcome(&[correction("cloud", "Claude", true)]) + .unwrap(); + assert_eq!(outcome.context_dependent_terms, ["Claude"]); + assert_eq!(db.get_known_corrections().unwrap()[0].count, 1); + assert!(!db.get_known_corrections().unwrap()[0].applies_automatically()); + } + + #[test] + fn repeated_snapshots_of_one_session_count_as_one_correction() { + let db = db(); + let generation = AtomicU64::new(1); + let corrections = [correction("cloud", "Claude", true)]; + db.record_session_corrections_if_current("field-session", 1, &corrections, &generation, 1) + .unwrap(); + let repeated = db + .record_session_corrections_if_current("field-session", 2, &corrections, &generation, 1) + .unwrap(); + assert_eq!(repeated, CorrectionLearningOutcome::default()); + assert_eq!(db.get_known_corrections().unwrap()[0].count, 1); + } + + #[test] + fn a_new_clear_correction_reports_success_without_readding_a_manual_word() { + let db = db(); + db.apply_dictionary_changes(&["Claude Code"], NONE, manual()) + .unwrap(); + let generation = AtomicU64::new(1); + let corrections = [correction("cloud code", "Claude Code", false)]; + let outcome = db + .record_session_corrections_if_current("session", 1, &corrections, &generation, 1) + .unwrap(); + assert!(outcome.added_terms.is_empty()); + assert_eq!(outcome.clear_correction_pairs, corrections); + assert_eq!( + db.dictionary_word_source("Claude Code").unwrap().as_deref(), + Some("manual") + ); + } + + #[test] + fn the_latest_session_spelling_wins_without_adding_a_sighting() { + let db = db(); + let generation = AtomicU64::new(1); + db.record_session_corrections_if_current( + "session", + 1, + &[correction("github", "GitHub", false)], + &generation, + 1, + ) + .unwrap(); + db.record_session_corrections_if_current( + "session", + 2, + &[correction("github", "GITHUB", false)], + &generation, + 1, + ) + .unwrap(); + assert_eq!( + db.get_known_corrections().unwrap(), + [KnownCorrection { + misheard: "github".into(), + term: "GITHUB".into(), + count: 1, + }] + ); + } + + #[test] + fn the_latest_session_edit_replaces_only_its_own_learning() { + let db = db(); + let generation = AtomicU64::new(1); + db.apply_dictionary_changes(&["Shared"], NONE, manual()) + .unwrap(); + db.record_corrections(&[correction("historic", "Historical", false)]) + .unwrap(); + db.record_session_corrections_if_current( + "first", + 1, + &[ + correction("cloud", "Claude", true), + correction("Shunade", "Sinead", false), + correction("shared", "Shared", true), + correction("historic", "Historical", false), + ], + &generation, + 1, + ) + .unwrap(); + db.record_session_corrections_if_current( + "second", + 1, + &[correction("cloud", "Claude", true)], + &generation, + 1, + ) + .unwrap(); + db.record_session_corrections_if_current( + "first", + 2, + &[correction("cloud", "Claudette", true)], + &generation, + 1, + ) + .unwrap(); + let mut words = db.get_dictionary().unwrap(); + words.sort(); + assert_eq!(words, ["Claude", "Claudette", "Historical", "Shared"]); + let mut pairs: Vec<_> = db + .get_known_corrections() + .unwrap() + .into_iter() + .map(|c| (c.misheard, c.term, c.count)) + .collect(); + pairs.sort(); + assert_eq!( + pairs, + [ + ("cloud".into(), "Claude".into(), 1), + ("cloud".into(), "Claudette".into(), 1), + ("historic".into(), "Historical".into(), 1), + ] + ); + } + + #[test] + fn clearing_learned_data_allows_a_current_session_to_start_fresh() { + let db = db(); + let generation = AtomicU64::new(1); + let corrections = [correction("cloud code", "Claude Code", false)]; + db.record_session_corrections_if_current("session", 1, &corrections, &generation, 1) + .unwrap(); + generation.store(2, Ordering::Release); + db.clear_dictionary().unwrap(); + assert!( + db.record_session_corrections_if_current("session", 2, &corrections, &generation, 1) + .unwrap() + .added_terms + .is_empty() + ); + assert_eq!( + db.record_session_corrections_if_current("session", 3, &corrections, &generation, 2) + .unwrap() + .added_terms, + ["Claude Code"] + ); + } + + #[test] + fn revising_a_session_after_a_unicode_respelling_retracts_its_correction() { + let db = db(); + let generation = AtomicU64::new(1); + db.record_session_corrections_if_current( + "session", + 1, + &[correction("eclair", "Éclair", true)], + &generation, + 1, + ) + .unwrap(); + db.apply_dictionary_changes(&["éclair"], &["Éclair"], manual()) + .unwrap(); + db.record_session_corrections_if_current("session", 2, &[], &generation, 1) + .unwrap(); + assert!(db.get_known_corrections().unwrap().is_empty()); + assert_eq!(db.get_dictionary().unwrap(), ["éclair"]); + } + + #[test] + fn a_removed_word_does_not_let_an_old_session_erase_new_learning() { + let db = db(); + let generation = AtomicU64::new(1); + let corrections = [correction("cloud", "Claude", true)]; + db.record_session_corrections_if_current("old", 1, &corrections, &generation, 1) + .unwrap(); + db.apply_dictionary_changes(NONE, &["Claude"], manual()) + .unwrap(); + db.apply_dictionary_changes(&["Claude"], NONE, manual()) + .unwrap(); + db.record_session_corrections_if_current("new", 1, &corrections, &generation, 1) + .unwrap(); + db.record_session_corrections_if_current("old", 2, &[], &generation, 1) + .unwrap(); + assert_eq!( + db.get_known_corrections().unwrap(), + [KnownCorrection { + misheard: "cloud".into(), + term: "Claude".into(), + count: 1, + }] + ); + } + + #[test] + fn forgetting_a_correction_detaches_its_old_session_contributions() { + let db = db(); + let generation = AtomicU64::new(1); + let corrections = [correction("cloud", "Claude", true)]; + db.record_session_corrections_if_current("old", 1, &corrections, &generation, 1) + .unwrap(); + db.remove_correction("cloud", "Claude").unwrap(); + db.record_session_corrections_if_current("new", 1, &corrections, &generation, 1) + .unwrap(); + db.record_session_corrections_if_current("old", 2, &[], &generation, 1) + .unwrap(); + assert_eq!(db.get_known_corrections().unwrap()[0].count, 1); + } + #[test] fn an_invalidated_learning_result_cannot_restore_a_cleared_dictionary() { let db = db(); @@ -1082,29 +1443,32 @@ mod tests { } #[test] - fn a_weak_correction_needs_a_second_sighting_of_its_term() { + fn a_context_dependent_term_is_added_without_deliberate_repetition() { let db = db(); let weak = correction("affect", "effect", true); - assert!( - db.record_corrections(std::slice::from_ref(&weak)) - .unwrap() - .is_empty() + assert_eq!( + db.record_corrections(std::slice::from_ref(&weak)).unwrap(), + ["effect"] ); - assert!(db.get_dictionary().unwrap().is_empty()); - assert!(db.get_known_corrections().unwrap().is_empty()); - assert_eq!(db.record_corrections(&[weak]).unwrap(), ["effect"]); + assert_eq!(db.get_dictionary().unwrap(), ["effect"]); + assert!(!db.get_known_corrections().unwrap()[0].applies_automatically()); + assert!(db.record_corrections(&[weak]).unwrap().is_empty()); } #[test] fn different_mishearings_of_a_term_add_up() { let db = db(); - db.record_corrections(&[correction("克劳", "Claude", true)]) - .unwrap(); assert_eq!( - db.record_corrections(&[correction("cloud", "Claude", true)]) + db.record_corrections(&[correction("克劳", "Claude", true)]) .unwrap(), ["Claude"] ); + assert!( + db.record_corrections(&[correction("cloud", "Claude", true)]) + .unwrap() + .is_empty() + ); + assert_eq!(db.get_known_corrections().unwrap().len(), 2); } #[test] @@ -1255,21 +1619,46 @@ mod tests { } #[test] - fn a_reversed_correction_replaces_the_old_one() { - let db = db(); - db.apply_dictionary_changes(&["Alpha", "Beta"], NONE, manual()) - .unwrap(); - db.record_corrections(&[correction("Alpha", "Beta", false)]) - .unwrap(); - db.record_corrections(&[correction("Beta", "Alpha", false)]) + fn opposite_contexts_keep_independent_session_contributions() { + for removed in ["first", "second"] { + let db = db(); + db.apply_dictionary_changes(&["Alpha", "Beta"], NONE, manual()) + .unwrap(); + let generation = AtomicU64::new(1); + db.record_session_corrections_if_current( + "first", + 1, + &[correction("Alpha", "Beta", false)], + &generation, + 1, + ) .unwrap(); - let rows: i64 = db - .lock() - .query_row("SELECT COUNT(*) FROM dictionary_corrections", [], |r| { - r.get(0) - }) + db.record_session_corrections_if_current( + "second", + 1, + &[correction("Beta", "Alpha", false)], + &generation, + 1, + ) .unwrap(); - assert_eq!(rows, 1); + db.record_session_corrections_if_current(removed, 2, &[], &generation, 1) + .unwrap(); + let (misheard, term) = if removed == "first" { + ("Beta", "Alpha") + } else { + ("Alpha", "Beta") + }; + db.apply_dictionary_changes(NONE, &[misheard], manual()) + .unwrap(); + assert_eq!( + db.get_known_corrections().unwrap(), + [KnownCorrection { + misheard: misheard.into(), + term: term.into(), + count: 1 + }] + ); + } } #[test] diff --git a/src/db/schema.rs b/src/db/schema.rs index f320f2b..6a12961 100644 --- a/src/db/schema.rs +++ b/src/db/schema.rs @@ -5,7 +5,7 @@ use rusqlite::Connection; use super::timestamp::SQL_NOW; -const SCHEMA_VERSION: i64 = 3; +const SCHEMA_VERSION: i64 = 4; pub(super) fn init(conn: &Connection) -> Result<()> { // Wait briefly rather than fail when another connection holds the lock. @@ -57,6 +57,22 @@ pub(super) fn init(conn: &Connection) -> Result<()> { word TEXT PRIMARY KEY COLLATE NOCASE ); + -- A stable editor observation contributes at most one sighting per pair. + CREATE TABLE IF NOT EXISTS dictionary_learning_sessions ( + session_id TEXT PRIMARY KEY, + revision INTEGER NOT NULL + ); + CREATE TABLE IF NOT EXISTS dictionary_session_corrections ( + session_id TEXT NOT NULL, + misheard TEXT NOT NULL COLLATE NOCASE, + term TEXT NOT NULL COLLATE NOCASE, + PRIMARY KEY (session_id, misheard, term) + ); + -- Only words created by session learning may be retracted with its edits. + CREATE TABLE IF NOT EXISTS dictionary_session_terms ( + word_id INTEGER PRIMARY KEY + ); + -- Retained for compatibility; auto-learn no longer writes statistics. CREATE TABLE IF NOT EXISTS auto_learn_stats ( app TEXT PRIMARY KEY, diff --git a/src/db/tests.rs b/src/db/tests.rs index a6a8277..328269d 100644 --- a/src/db/tests.rs +++ b/src/db/tests.rs @@ -294,7 +294,7 @@ fn schema_matches_the_expected_layout() { "spoken_duration_ms", ] ); - assert_eq!(count(&db, "PRAGMA user_version"), 3); + assert_eq!(count(&db, "PRAGMA user_version"), 4); } fn legacy_connection(path: &Path) -> Connection { @@ -375,7 +375,7 @@ fn upgrades_legacy_metadata_without_losing_history_or_counters() { assert_eq!(summary.total_dictations, 1); assert_eq!(summary.average_wpm, Some(120)); assert_eq!(count(&upgraded, "SELECT id FROM analytics_events"), 17); - assert_eq!(count(&upgraded, "PRAGMA user_version"), 3); + assert_eq!(count(&upgraded, "PRAGMA user_version"), 4); assert_eq!( count( &upgraded, @@ -437,12 +437,12 @@ fn failed_upgrade_rolls_back_schema_and_version() { fn refuses_a_newer_schema_without_downgrading_it() { let db = db(); let conn = db.lock(); - conn.pragma_update(None, "user_version", 4).unwrap(); + conn.pragma_update(None, "user_version", 5).unwrap(); assert!(schema::init(&conn).is_err()); assert_eq!( conn.query_row("PRAGMA user_version", [], |r| r.get::<_, i64>(0)) .unwrap(), - 4 + 5 ); } diff --git a/src/dictation.rs b/src/dictation.rs index dd75f79..2ed41da 100644 --- a/src/dictation.rs +++ b/src/dictation.rs @@ -143,6 +143,10 @@ pub enum DictationNotice { description: Option, }, LearnedWords(Vec), + LearnedCorrection(Vec<(String, String)>), + ContextDependentCorrection(Vec<(String, String)>), + LearningUnreadable, + LearningRecovered, /// A clean press that did nothing; explains how the mode starts Dictation. HotkeyHint { hold: bool, @@ -594,10 +598,15 @@ impl Dictation { fn pipeline_input( audio: bytes::Bytes, - cx: &App, + cx: &mut App, ) -> impl std::future::Future + Send + 'static { let state = AppState::global(cx).read(cx); - capture_pipeline_input(audio, state.settings.clone(), state.db.clone()) + let (settings, db) = (state.settings.clone(), state.db.clone()); + let learning = backend::flush_learning(cx); + async move { + learning.await; + capture_pipeline_input(audio, settings, db).await + } } fn process(&mut self, audio: RecordedAudio, cx: &mut Context) { @@ -905,8 +914,13 @@ impl Dictation { return; } let paste_started = Instant::now(); - let pasted = - backend::paste_text(text.clone(), target_pid, settings.output_mode).await; + let pasted = backend::paste_text( + text.clone(), + target_pid, + settings.output_mode, + settings.auto_learn_enabled(), + ) + .await; log::info!( "[perf] operation={generation} stage=paste_complete elapsed_ms={:.3}", paste_started.elapsed().as_secs_f64() * 1000.0 @@ -933,20 +947,6 @@ impl Dictation { description: Some(t("app.toasts.cleanupFailed.description")), }); } - let s = AppState::settings(cx); - if s.auto_learn_corrections { - let learned = - backend::monitor_corrections(text.clone(), target_pid, cx); - cx.spawn(async move |this, cx| { - let words = learned.await; - if !words.is_empty() { - let _ = this.update(cx, |_, cx| { - cx.emit(DictationNotice::LearnedWords(words)) - }); - } - }) - .detach(); - } } else { let copied = backend::write_clipboard(&text); cx.emit(DictationNotice::DeliveryFallback { @@ -1018,24 +1018,39 @@ fn capture_pipeline_input( settings: Settings, db: Arc, ) -> impl std::future::Future + Send + 'static { - crate::services::runtime::storage(move || PipelineInput { - audio, - dictionary: db + crate::services::runtime::storage(move || { + let auto_learn_enabled = settings.auto_learn_enabled(); + let dictionary: Vec = db .get_dictionary_entries() .unwrap_or_default() .into_iter() + .filter(|entry| { + auto_learn_enabled || entry.source == crate::db::DictionarySource::Manual + }) .map(|entry| entry.word) - .collect(), - dictionary_priority: db.get_dictionary_by_priority().unwrap_or_default(), - known_corrections: if settings.auto_learn_corrections { - db.get_known_corrections().unwrap_or_default() - } else { - Vec::new() - }, - snippets: db.get_snippets().unwrap_or_default(), - settings, - cleanup_api_key: None, - cleanup_stage: None, + .collect(); + let allowed_words: std::collections::HashSet<&str> = + dictionary.iter().map(String::as_str).collect(); + let dictionary_priority = db + .get_dictionary_by_priority() + .unwrap_or_default() + .into_iter() + .filter(|word| allowed_words.contains(word.as_str())) + .collect(); + PipelineInput { + audio, + dictionary, + dictionary_priority, + known_corrections: if auto_learn_enabled { + db.get_known_corrections().unwrap_or_default() + } else { + Vec::new() + }, + snippets: db.get_snippets().unwrap_or_default(), + settings, + cleanup_api_key: None, + cleanup_stage: None, + } }) } @@ -1539,6 +1554,57 @@ mod tests { }); } + #[test] + fn pipeline_suspends_learned_influence_until_cleanup_and_auto_learn_are_enabled() { + use crate::text::correction_learner::Correction; + let db = Arc::new(Database::open_in_memory().unwrap()); + db.apply_dictionary_changes(&["Whisper"], &[] as &[&str], DictionarySource::Manual) + .unwrap(); + db.record_corrections(&[Correction { + misheard: "cloud code".into(), + term: "Claude Code".into(), + needs_repeat: false, + }]) + .unwrap(); + let configured = Settings { + cleanup_model: "cleanup".into(), + cleanup_remote_url: "http://localhost:8080/v1".into(), + ..Settings::default() + }; + run(async { + for settings in [ + Settings { + use_cleanup_model: false, + ..configured.clone() + }, + Settings { + cleanup_model: String::new(), + ..configured.clone() + }, + Settings { + cleanup_remote_url: String::new(), + ..configured.clone() + }, + Settings { + auto_learn_corrections: false, + ..configured.clone() + }, + ] { + let input = + capture_pipeline_input(bytes::Bytes::new(), settings.clone(), db.clone()).await; + assert_eq!(input.dictionary, ["Whisper"]); + assert_eq!(input.dictionary_priority, ["Whisper"]); + assert!(input.known_corrections.is_empty()); + assert_eq!(input.settings, settings); + } + + let resumed = capture_pipeline_input(bytes::Bytes::new(), configured, db).await; + assert_eq!(resumed.dictionary, ["Whisper", "Claude Code"]); + assert_eq!(resumed.dictionary_priority, ["Whisper", "Claude Code"]); + assert_eq!(resumed.known_corrections.len(), 1); + }); + } + #[test] fn pipeline_rules_capture_prior_writes_and_exclude_later_edits() { use crate::text::correction_learner::Correction; @@ -1564,7 +1630,15 @@ mod tests { db.record_corrections(&[correction]).unwrap(); }) }; - let enabled = capture_pipeline_input(bytes::Bytes::new(), Settings::default(), db.clone()); + let enabled = capture_pipeline_input( + bytes::Bytes::new(), + Settings { + cleanup_model: "cleanup".into(), + cleanup_remote_url: "http://localhost:8080/v1".into(), + ..Settings::default() + }, + db.clone(), + ); let disabled = capture_pipeline_input( bytes::Bytes::new(), Settings { diff --git a/src/native/paste.rs b/src/native/paste.rs index 85c02e5..b707613 100644 --- a/src/native/paste.rs +++ b/src/native/paste.rs @@ -343,13 +343,24 @@ impl Paster { /// Activates the Target app and pastes `text` (with smart spacing) into it. pub fn paste_text(&self, text: &str, options: PasteOptions) -> PasteOutcome { + self.paste_text_observed(text, options, |_, _| |_| {}) + } + + /// Binds edit observation after target validation, before Command-V, and + /// confirms whether delivery succeeded. Preparation stays inside the queue. + pub fn paste_text_observed( + &self, + text: &str, + options: PasteOptions, + prepare: impl FnOnce(&str, Option) -> F, + ) -> PasteOutcome { // Activation stays outside the clipboard queue. if options.target_pid.is_some() { (self.hooks.activate_target)(options.target_pid); } let spaced = apply_smart_spacing(text); let ticket = self.queue.acquire(); - let (outcome, restore) = self.paste_locked(&spaced, options); + let (outcome, restore) = self.paste_locked(&spaced, options, prepare); match restore { // The ticket is held until the restore settles. Some(restore) => { @@ -377,10 +388,11 @@ impl Paster { operation() } - fn paste_locked( + fn paste_locked( &self, text: &str, options: PasteOptions, + prepare: impl FnOnce(&str, Option) -> F, ) -> (PasteOutcome, Option) { let pasteboard = &self.hooks.pasteboard; let original = options @@ -408,6 +420,7 @@ impl Paster { return (PasteOutcome::clipboard_only(), None); } + let complete = prepare(text, options.target_pid); let result = match self.paste_command() { Ok(()) => Ok(()), Err(PasteFailure::KeyboardLayoutUnresolved) => { @@ -421,6 +434,7 @@ impl Paster { self.paste_command() } }; + complete(result.is_ok()); match result { Ok(()) => ( PasteOutcome { @@ -514,8 +528,7 @@ fn restore_after_delay( pasteboard.restore(original); } -/// Pastes through the shared [`Paster`]. After `pasted`, start -/// `TextEditMonitor::shared().start_monitoring_after_paste` for auto-learn. +/// Pastes through the shared [`Paster`] without observing subsequent edits. pub fn paste_text(text: &str, options: PasteOptions) -> PasteOutcome { Paster::shared().paste_text(text, options) } @@ -702,6 +715,30 @@ mod tests { f.paster.run_clipboard_operation(|| ()); } + #[test] + fn observation_binds_the_validated_field_before_paste_and_confirms_delivery() { + let f = fixture(true, true); + let prepared = Arc::new(Mutex::new(None)); + let completed = Arc::new(Mutex::new(None)); + let prepared_result = prepared.clone(); + let completed_result = completed.clone(); + let commands = f.runs.clone(); + let outcome = f + .paster + .paste_text_observed("dictated", opts(Some(42)), move |text, pid| { + assert!( + lock(&commands).is_empty(), + "observation must precede Command-V" + ); + *lock(&prepared_result) = Some((text.to_string(), pid)); + move |pasted| *lock(&completed_result) = Some(pasted) + }); + assert!(outcome.pasted); + assert_eq!(*lock(&prepared), Some(("dictated ".into(), Some(42)))); + assert_eq!(*lock(&completed), Some(true)); + settle(&f); + } + #[test] fn restore_skipped_when_another_write_wins() { let pasteboard = FakePasteboard::default(); diff --git a/src/native/text_monitor.rs b/src/native/text_monitor.rs index 9f56bc3..c94a1fb 100644 --- a/src/native/text_monitor.rs +++ b/src/native/text_monitor.rs @@ -4,31 +4,26 @@ //! focus at a hotkey press (the Target app), before the overlay steals focus. //! - [`TextEditMonitor::activate_pid`] / [`TextEditMonitor::can_paste_at_target`] //! gate automatic paste. -//! - [`TextEditMonitor::start_monitoring_after_paste`] watches the focused field after an -//! automatic paste. The session ends when the field empties (the message was -//! sent), focus leaves it, another paste starts or [`MONITOR_TIMEOUT`] runs -//! out, and reports the field's last text as one [`SessionReport`] for the -//! correction learner, so half-typed fixes are never learned. +//! - [`TextEditMonitor::prepare_paste`] anchors a field before Command-V. +//! - [`TextEditMonitor::subscribe`] streams completed edits and read health +//! while each field remains eligible for five minutes after delivery. //! //! Monitoring is strictly read-only: never write AX attributes such as //! AXEnhancedUserInterface on the target app (it blurs Chromium editors). //! All methods block (helper spawns, short sleeps); call them off the main //! thread. -use std::collections::{HashSet, VecDeque}; +use std::ops::Range; use std::path::PathBuf; -use std::sync::atomic::{AtomicU64, Ordering}; use std::sync::{Arc, Condvar, Mutex, OnceLock, Weak}; use std::time::{Duration, Instant}; -use base64::Engine; - use super::helper::{probe_first_line, resolve_helper}; use super::supervised_child::{ ChildOptions, Control, Down, HelperSpawner, ProcessSpawner, Protocol, Recovery, RestartPolicy, SupervisedChild, }; -use super::timer::{RealTimers, TimerId, Timers, lock}; +use super::timer::{RealTimers, Timers, lock}; const ACTIVATE_CONFIRM_RETRIES: u32 = 6; const ACTIVATE_CONFIRM_DELAY: Duration = Duration::from_millis(25); @@ -37,33 +32,27 @@ const TARGET_PROBE_TIMEOUT: Duration = Duration::from_millis(700); /// hundred ms; reuse the result across that burst only. const TARGET_CAPTURE_FRESHNESS: Duration = Duration::from_millis(250); /// Long enough to read a dictation back and fix it. -pub const MONITOR_TIMEOUT: Duration = Duration::from_secs(90); -/// Wait for the paste to settle in the target app before monitoring. -pub const MONITOR_START_DELAY: Duration = Duration::from_secs(1); -/// Dictating several times in a row and then fixing an earlier part is normal, -/// so a session accepts edits to the app's recent pastes. -const RECENT_PASTES: usize = 5; -const RECENT_PASTE_WINDOW: Duration = Duration::from_secs(5 * 60); - +pub const MONITOR_TIMEOUT: Duration = Duration::from_secs(5 * 60); pub const MONITOR_BINARY: &str = "macos-text-monitor"; -/// The helper caps a reported value at 10240 characters. -const MAX_LINE_BYTES: usize = 64 * 1024; -/// Enough for the helper's diagnostics, which only feed -/// [`is_persistent_ax_no_value_failure`]. -const MAX_STDERR_BYTES: usize = 16 * 1024; - -/// The field right after the paste and its last text before the session ended. +/// The original dictated field text and its latest completed revision. #[derive(Debug, Clone, PartialEq, Eq)] pub struct TextEdit { pub baseline: String, pub edited: String, - /// The transcripts this session may have corrected, oldest first. - pub pasted: Vec, + /// Exact delivered UTF-8 byte ranges in the cumulative original baseline. + pub pasted_ranges: Vec>, } -/// How one monitored paste went. +/// A completed field revision or a change in observation availability. #[derive(Debug, Clone, PartialEq, Eq)] pub struct SessionReport { + pub outcome: ObservationOutcome, + pub diagnostic: Option, + pub field: Option, + /// Stable across repeated dictation in one eligible field lifecycle. + pub session_id: String, + /// Increases only for completed edits; an empty diff revokes prior learning. + pub revision: u64, /// Bundle identifier of the target app. pub app: Option, /// Whether the field's text could be read at all. @@ -122,102 +111,12 @@ struct Capture { last_capture_at: Option, } -struct RecentPaste { - pid: Option, - text: String, - at: Instant, -} - -/// One monitored paste. -struct Session { - app: Option, - pasted: Vec, - baseline: Option, - /// The last non-empty text after the baseline: pressing Send clears the - /// field right after a fix, and the fix must survive that. - latest: Option, - report: async_channel::Sender, -} - -impl Session { - fn finish(self) { - let readable = self.baseline.is_some(); - let edit = match (self.baseline, self.latest) { - (Some(baseline), Some(edited)) if baseline != edited => Some(TextEdit { - baseline, - edited, - pasted: self.pasted, - }), - _ => None, - }; - let _ = self.report.try_send(SessionReport { - app: self.app, - readable, - edit, - }); - } - - fn on_baseline(&mut self, value: String) { - self.baseline.get_or_insert(value); - } - - /// A slow app takes the paste only after the baseline was read; its text - /// then becomes the baseline. Returns false when the field was emptied. - fn on_changed(&mut self, value: String) -> bool { - let Some(baseline) = &self.baseline else { - return true; - }; - if value.trim().is_empty() { - return baseline.trim().is_empty(); - } - let paste_landed = self - .pasted - .last() - .is_some_and(|text| !baseline.contains(text.as_str()) && value.contains(text.as_str())); - if paste_landed { - self.baseline = Some(value); - self.latest = None; - } else { - self.latest = Some(value); - } - true - } -} - -#[derive(Default)] -struct Monitoring { - session: Option, - run: Option>, - timeout: Option, - recent: VecDeque, -} - -impl Monitoring { - /// Records a paste and returns the app's recent ones, oldest first. - fn remember_paste(&mut self, pid: Option, text: &str) -> Vec { - self.recent - .retain(|paste| paste.pid == pid && paste.at.elapsed() < RECENT_PASTE_WINDOW); - if self.recent.len() == RECENT_PASTES { - self.recent.pop_front(); - } - self.recent.push_back(RecentPaste { - pid, - text: text.to_string(), - at: Instant::now(), - }); - self.recent.iter().map(|p| p.text.clone()).collect() - } -} - struct Inner { + lifecycle: Mutex<()>, + observations: Mutex, hooks: MonitorHooks, timers: Arc, capture: Mutex, - monitoring: Mutex, - // Bumped by every start/stop; a run whose generation is stale gives up. - generation: AtomicU64, - // PIDs whose AX tree never yields a focused element this session. - unsupported_pids: Mutex>, this: Weak, } @@ -242,12 +141,11 @@ impl TextEditMonitor { fn build(hooks: MonitorHooks, timers: Arc) -> Self { Self { inner: Arc::new_cyclic(|this| Inner { + lifecycle: Mutex::new(()), + observations: Mutex::new(Observations::default()), hooks, timers, capture: Mutex::new(Capture::default()), - monitoring: Mutex::new(Monitoring::default()), - generation: AtomicU64::new(0), - unsupported_pids: Mutex::new(HashSet::new()), this: this.clone(), }), } @@ -347,110 +245,39 @@ impl TextEditMonitor { } } - /// Waits [`MONITOR_START_DELAY`] (on a background thread) and then - /// monitors, as automatic paste does after a successful paste. The - /// receiver gets one report when the session ends. - pub fn start_monitoring_after_paste( - &self, - original_text: &str, - target_pid: Option, - ) -> async_channel::Receiver { - let (report, receiver) = async_channel::bounded(1); - let generation = self.prepare_start(); - let monitor = self.clone(); - let text = original_text.to_string(); - self.inner.timers.schedule( - MONITOR_START_DELAY, - Box::new(move || { - // Helper resolution and AppKit lookups must not block the timer thread. - std::thread::spawn(move || { - monitor.start_reserved(generation, &text, MONITOR_TIMEOUT, target_pid, report); - }); - }), - ); - receiver - } - - /// Watches the focused field of `target_pid` for `timeout`. Ends any - /// previous session first. Returns immediately; `report` always receives - /// the outcome. + /// Discards all observation state and stops helpers when learning is disabled. #[cfg(test)] - pub fn start_monitoring( - &self, - original_text: &str, - timeout: Duration, - target_pid: Option, - report: async_channel::Sender, - ) { - let generation = self.prepare_start(); - self.start_reserved(generation, original_text, timeout, target_pid, report); - } - - fn prepare_start(&self) -> u64 { - self.inner - .stop(None) - .expect("unconditional stop reserves a generation") - } - - fn start_reserved( - &self, - generation: u64, - original_text: &str, - timeout: Duration, - target_pid: Option, - report: async_channel::Sender, - ) { - let inner = &self.inner; - let app = target_pid.and_then(appkit::bundle_identifier); - { - let mut monitoring = lock(&inner.monitoring); - if inner.generation.load(Ordering::SeqCst) != generation { - return; - } - let pasted = monitoring.remember_paste(target_pid, original_text); - monitoring.session = Some(Session { - app, - pasted, - baseline: None, - latest: None, - report, - }); - } - - let Some(pid) = target_pid.filter(|p| *p > 0) else { - log::debug!("[text-monitor] no target pid"); - inner.stop(Some(generation)); - return; - }; - let Some(helper) = (inner.hooks.resolve_helper)() else { - log::debug!("[text-monitor] helper unavailable"); - inner.stop(Some(generation)); - return; + pub fn stop_monitoring(&self) { + let _lifecycle = lock(&self.inner.lifecycle); + self.stop_observers(); + } + + /// Gates all future preparations and invalidates tickets already in flight. + pub fn set_enabled(&self, enabled: bool) { + let _lifecycle = lock(&self.inner.lifecycle); + let changed = { + let mut observations = lock(&self.inner.observations); + let changed = observations.enabled != enabled; + observations.enabled = enabled; + changed }; - if lock(&inner.unsupported_pids).contains(&pid) { - log::debug!("[text-monitor] AX reads unsupported for {pid} this session"); - inner.stop(Some(generation)); - return; + if changed || !enabled { + self.stop_observers(); } - - let weak = inner.this.clone(); - inner.arm_timeout(generation, timeout); - std::thread::spawn(move || { - let Some(inner) = weak.upgrade() else { return }; - inner.run_native(generation, pid, helper, timeout); - }); } - /// Ends the session, reporting what it saw. A paste ends the previous - /// session before it lands, so the new text is not read as an edit. - pub fn stop_monitoring(&self) { - self.inner.stop(None); - } - - /// True while a monitoring run is in progress. - #[cfg(test)] - pub fn is_monitoring(&self) -> bool { - lock(&self.inner.monitoring).session.is_some() + fn stop_observers(&self) { + let runs = { + let mut observations = lock(&self.inner.observations); + observations.epoch += 1; + observations.fields.clear(); + observations.pending.clear(); + observations.failures.clear(); + std::mem::take(&mut observations.runs) + }; + for (_, run) in runs { + run.stop(); + } } } @@ -472,236 +299,992 @@ impl Inner { } (self.hooks.frontmost_pid)() } +} + +/// The observation result is independent from whether an edit was found. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)] +pub enum ObservationOutcome { + #[default] + Readable, + Unreadable, + Recovered, + CompletedEdit, +} + +struct Observations { + enabled: bool, + epoch: u64, + runs: std::collections::HashMap>, + fields: std::collections::HashMap<(i32, String), ObservedField>, + pending: std::collections::HashMap>, + subscribers: Vec>, + next_ticket: u64, + failures: std::collections::HashSet<(i32, String)>, +} + +impl Default for Observations { + fn default() -> Self { + Self { + enabled: true, + epoch: 0, + runs: Default::default(), + fields: Default::default(), + pending: Default::default(), + subscribers: Vec::new(), + next_ticket: 0, + failures: Default::default(), + } + } +} + +struct ObservedField { + app: Option, + session_id: String, + revision: u64, + baseline: String, + latest: String, + reported: String, + pasted_ranges: Vec>, + composing: bool, + health_only: bool, + alignment_blocked: bool, + empty: bool, + unreadable: bool, + change: u64, + last_change_at: u64, + expiry: u64, +} + +#[derive(Debug, Clone, serde::Deserialize)] +struct NativeObservation { + kind: String, + #[serde(default)] + ticket: u64, + #[serde(default)] + field: String, + value: Option, + #[serde(default)] + start: usize, + #[serde(default)] + length: usize, + #[serde(default)] + composing: bool, + reason: Option, +} + +/// A target field anchored before Command-V. Dropping an uncommitted ticket +/// cancels it. A failed preparation never prevents the ordinary paste. +pub struct PreparedPaste { + monitor: TextEditMonitor, + pid: i32, + ticket: u64, + epoch: u64, + prepared: NativeObservation, + text: String, + finished: bool, +} - fn is_current(&self, generation: u64) -> bool { - self.generation.load(Ordering::SeqCst) == generation - && lock(&self.monitoring).session.is_some() +impl PreparedPaste { + pub fn commit(mut self, pasted: bool) { + let committed = pasted + && self.monitor.inner.commit_observation( + self.pid, + self.ticket, + self.epoch, + &self.prepared, + &self.text, + ); + self.monitor.inner.observation_command( + self.pid, + serde_json::json!({ + "command": if committed { "commit" } else { "cancel" }, "ticket": self.ticket + }), + ); + self.finished = true; } +} - /// Stops monitoring; with `Some(generation)` only if that run is current. - fn stop(&self, generation: Option) -> Option { - let (session, next_generation) = { - let mut monitoring = lock(&self.monitoring); - if let Some(generation) = generation - && self.generation.load(Ordering::SeqCst) != generation - { +impl Drop for PreparedPaste { + fn drop(&mut self) { + if !self.finished { + self.monitor.inner.observation_command( + self.pid, + serde_json::json!({ + "command": "cancel", "ticket": self.ticket + }), + ); + } + } +} + +impl TextEditMonitor { + /// Each subscriber receives all reports; receivers do not steal each + /// other's messages. Install the application subscriber before pasting. + pub fn subscribe(&self) -> async_channel::Receiver { + let (sender, receiver) = async_channel::unbounded(); + lock(&self.inner.observations).subscribers.push(sender); + receiver + } + + /// Binds the intended AX element and selection before the paste event. + /// This bounded operation belongs inside the serialized paste queue. + pub fn prepare_paste(&self, text: &str, pid: Option) -> Option { + let pid = pid.filter(|pid| *pid > 0)?; + let epoch = { + let observations = lock(&self.inner.observations); + if !observations.enabled { + return None; + } + observations.epoch + }; + if self.inner.ensure_observer(pid, epoch).is_none() { + self.inner + .observation_failure(pid, None, "helper_unavailable"); + return None; + } + let (sender, receiver) = async_channel::bounded(1); + let ticket = { + let mut observations = lock(&self.inner.observations); + observations.next_ticket += 1; + let ticket = observations.next_ticket; + observations.pending.insert(ticket, sender); + ticket + }; + if !self.inner.observation_command( + pid, + serde_json::json!({"command":"prepare", "ticket":ticket}), + ) { + lock(&self.inner.observations).pending.remove(&ticket); + self.inner + .observation_failure(pid, None, "helper_command_unavailable"); + return None; + } + // recv_blocking has no deadline; use a condition-independent bounded + // wait so an unresponsive AX server cannot hold the paste queue. + let deadline = Instant::now() + TARGET_PROBE_TIMEOUT; + let prepared = loop { + match receiver.try_recv() { + Ok(prepared) => break Some(prepared), + Err(async_channel::TryRecvError::Closed) => break None, + Err(async_channel::TryRecvError::Empty) if Instant::now() < deadline => { + std::thread::sleep(Duration::from_millis(5)); + } + Err(_) => break None, + } + }; + lock(&self.inner.observations).pending.remove(&ticket); + { + let observations = lock(&self.inner.observations); + if !observations.enabled || observations.epoch != epoch { return None; } - let next_generation = self.generation.fetch_add(1, Ordering::SeqCst) + 1; - if let Some(timer) = monitoring.timeout.take() { - self.timers.cancel(timer); + } + let Some(prepared) = prepared else { + self.inner.observation_command( + pid, + serde_json::json!({"command":"cancel", "ticket":ticket}), + ); + self.inner + .observation_failure(pid, None, "ax_prepare_timeout"); + return None; + }; + if prepared.value.is_none() { + self.inner.observation_failure( + pid, + (!prepared.field.is_empty()).then_some(prepared.field.clone()), + prepared.reason.as_deref().unwrap_or("ax_text_unavailable"), + ); + self.inner.observation_command( + pid, + serde_json::json!({"command":"health", "ticket":ticket}), + ); + return None; + } + Some(PreparedPaste { + monitor: self.clone(), + pid, + ticket, + epoch, + prepared, + text: text.into(), + finished: false, + }) + } + + /// Synchronizes helper output before the next dictation reads learned + /// data. This does not turn unfinished composition into a correction. + pub fn flush_completed_edits(&self) { + let runs: Vec<_> = lock(&self.inner.observations) + .runs + .keys() + .copied() + .collect(); + for pid in runs { + let (sender, receiver) = async_channel::bounded(1); + let ticket = { + let mut observations = lock(&self.inner.observations); + observations.next_ticket += 1; + let ticket = observations.next_ticket; + observations.pending.insert(ticket, sender); + ticket + }; + self.inner.observation_command( + pid, + serde_json::json!({"command":"snapshot", "ticket":ticket}), + ); + let deadline = Instant::now() + Duration::from_millis(250); + while receiver.try_recv().is_err() && Instant::now() < deadline { + std::thread::sleep(Duration::from_millis(5)); + } + lock(&self.inner.observations).pending.remove(&ticket); + } + } +} + +impl Inner { + fn observation_failure(&self, pid: i32, field: Option, reason: &str) { + if !lock(&self.observations).enabled { + return; + } + let key = (pid, field.clone().unwrap_or_default()); + let notify = { + let mut observations = lock(&self.observations); + if field.is_some() && !observations.fields.contains_key(&key) { + observations.fields.insert( + key.clone(), + ObservedField { + app: appkit::bundle_identifier(pid), + session_id: uuid::Uuid::new_v4().to_string(), + revision: 0, + baseline: String::new(), + latest: String::new(), + reported: String::new(), + pasted_ranges: Vec::new(), + composing: false, + health_only: true, + alignment_blocked: false, + empty: false, + unreadable: true, + change: 0, + last_change_at: self.timers.now_ms(), + expiry: 0, + }, + ); } - if let Some(run) = monitoring.run.take() { - run.stop(); + if let Some(observed) = observations.fields.get_mut(&key) { + observed.unreadable = true; } - (monitoring.session.take(), next_generation) + observations.failures.insert(key.clone()) }; - if let Some(session) = session { - session.finish(); + if !notify { + return; + } + if field.is_none() { + let epoch = lock(&self.observations).epoch; + self.retry_observation_health( + pid, + epoch, + self.timers + .now_ms() + .saturating_add(MONITOR_TIMEOUT.as_millis() as u64), + ); } - Some(next_generation) + let report = self + .field_report( + &key, + ObservationOutcome::Unreadable, + Some(reason.into()), + false, + ) + .unwrap_or(SessionReport { + app: appkit::bundle_identifier(pid), + readable: false, + edit: None, + outcome: ObservationOutcome::Unreadable, + diagnostic: Some(reason.into()), + field, + session_id: String::new(), + revision: 0, + }); + self.publish_observation(report); } - fn arm_timeout(&self, generation: u64, timeout: Duration) { + fn retry_observation_health(&self, pid: i32, epoch: u64, deadline: u64) { let weak = self.this.clone(); - let timer = self.timers.schedule( - timeout, + self.timers.schedule( + Duration::from_secs(1), Box::new(move || { - if let Some(inner) = weak.upgrade() { - inner.stop(Some(generation)); + let Some(inner) = weak.upgrade() else { return }; + { + let observations = lock(&inner.observations); + if !observations.enabled + || observations.epoch != epoch + || !observations.failures.contains(&(pid, String::new())) + || inner.timers.now_ms() >= deadline + { + return; + } } + if inner.ensure_observer(pid, epoch).is_some() + && inner.observation_command( + pid, + serde_json::json!({"command":"health", "ticket":0}), + ) + { + return; + } + inner.retry_observation_health(pid, epoch, deadline); }), ); - let mut monitoring = lock(&self.monitoring); - if self.generation.load(Ordering::SeqCst) == generation { - monitoring.timeout = Some(timer); - } else { - self.timers.cancel(timer); - } } - fn update_session(&self, generation: u64, update: impl FnOnce(&mut Session) -> bool) { - let keep_going = { - let mut monitoring = lock(&self.monitoring); - if self.generation.load(Ordering::SeqCst) != generation { - return; - } - monitoring.session.as_mut().is_none_or(update) - }; - if !keep_going { - log::debug!("[text-monitor] field emptied"); - self.stop(Some(generation)); - } + fn publish_observation(&self, report: SessionReport) { + lock(&self.observations) + .subscribers + .retain(|sender| sender.try_send(report.clone()).is_ok()); } - fn handle_line(&self, generation: u64, line: &str) { - if let Some(value) = payload(line, "CHANGED") { - self.update_session(generation, |session| session.on_changed(value)); - } else if let Some(value) = payload(line, "BASELINE") { - self.update_session(generation, |session| { - session.on_baseline(value); - true - }); - } else if line == "NO_ELEMENT" || line == "NO_VALUE" { - // The helper exits right after; its end stops monitoring. - log::debug!("[text-monitor] no target element ({line})"); - } + fn observation_command(&self, pid: i32, command: serde_json::Value) -> bool { + let run = lock(&self.observations).runs.get(&pid).cloned(); + run.is_some_and(|run| run.update(|_, cx| cx.write_line(&command.to_string()))) } - fn on_watch_event(&self, generation: u64, pid: i32, event: WatchEvent) { - match event { - WatchEvent::Line(line) => self.handle_line(generation, &line), - WatchEvent::Ended { ax_unsupported } => { - // NO_ELEMENT after a uniform -25212 run is terminal for this app. - if ax_unsupported { - lock(&self.unsupported_pids).insert(pid); - } - self.stop(Some(generation)); + fn ensure_observer(&self, pid: i32, epoch: u64) -> Option<()> { + let _lifecycle = lock(&self.lifecycle); + { + let observations = lock(&self.observations); + if !observations.enabled || observations.epoch != epoch { + return None; } } - } - - /// Event-based monitoring through the helper's AXObserver. - fn run_native(&self, generation: u64, pid: i32, helper: HelperCommand, timeout: Duration) { - let program = helper.program; + if lock(&self.observations).runs.contains_key(&pid) { + return Some(()); + } + let helper = (self.hooks.resolve_helper)()?; let mut args = helper.args; - args.push(pid.to_string()); - args.push(timeout.as_secs().to_string()); + args.extend(["--observe".into(), pid.to_string()]); let weak = self.this.clone(); let run = SupervisedChild::new( ChildOptions { - label: "Text monitor", - resolve_binary: Arc::new(move || Some(program.clone())), - spawner: Arc::clone(&self.hooks.spawner), - timers: Arc::clone(&self.timers), + label: "Field observer", + resolve_binary: Arc::new(move || Some(helper.program.clone())), + spawner: self.hooks.spawner.clone(), + timers: self.timers.clone(), restart: RestartPolicy::Never, heartbeat: None, - max_line: MAX_LINE_BYTES, - }, - EditWatch { - args, - stderr: String::new(), - no_element: false, + max_line: 8 * 1024 * 1024, }, + ObservationProtocol { args }, Arc::new(move |event| { if let Some(inner) = weak.upgrade() { - inner.on_watch_event(generation, pid, event); + let current = { + let observations = lock(&inner.observations); + observations.enabled && observations.epoch == epoch + }; + if current { + inner.observe_native(pid, event); + } } }), ); - { - let mut monitoring = lock(&self.monitoring); - if self.generation.load(Ordering::SeqCst) != generation || monitoring.session.is_none() - { - return; - } - monitoring.run = Some(run.clone()); - } + lock(&self.observations).runs.insert(pid, run.clone()); run.start(); - // A stop between storing the run and starting it had nothing to end. - if !self.is_current(generation) { - run.stop(); - } + Some(()) } -} - -enum WatchEvent { - Line(String), - /// The helper ended; `ax_unsupported` once the target app's AX tree - /// proved permanently unreadable. - Ended { - ax_unsupported: bool, - }, -} -/// The text of a `:` or, for multi-line text, `_B64:` line. -fn payload(line: &str, prefix: &str) -> Option { - let rest = line.strip_prefix(prefix)?; - if let Some(encoded) = rest.strip_prefix("_B64:") { - return match base64::engine::general_purpose::STANDARD.decode(encoded.trim()) { - Ok(bytes) => Some(String::from_utf8_lossy(&bytes).into_owned()), - Err(error) => { - log::debug!("[text-monitor] bad base64 payload: {error}"); - None + fn commit_observation( + &self, + pid: i32, + ticket: u64, + epoch: u64, + prepared: &NativeObservation, + text: &str, + ) -> bool { + let _lifecycle = lock(&self.lifecycle); + { + let observations = lock(&self.observations); + if !observations.enabled || observations.epoch != epoch { + return false; + } + } + let key = (pid, prepared.field.clone()); + let pre = prepared.value.as_deref().unwrap_or_default(); + let Some(selected) = utf16_range(pre, prepared.start, prepared.length) else { + self.observation_failure( + pid, + Some(key.1.clone()), + "ax_selection_not_on_text_boundaries", + ); + return false; + }; + let predicted = pre[..selected.start].to_string() + text + &pre[selected.end..]; + let (expiry, change, recovered) = { + let mut observations = lock(&self.observations); + if observations + .fields + .get(&key) + .is_some_and(|field| field.health_only) + { + observations.fields.remove(&key); + } + let recovered = observations.failures.remove(&key) + | observations.failures.remove(&(pid, String::new())); + let field = observations + .fields + .entry(key.clone()) + .or_insert_with(|| ObservedField { + app: appkit::bundle_identifier(pid), + session_id: uuid::Uuid::new_v4().to_string(), + revision: 0, + baseline: pre.into(), + latest: pre.into(), + reported: pre.into(), + pasted_ranges: Vec::new(), + composing: false, + health_only: false, + alignment_blocked: false, + empty: false, + unreadable: false, + change: 0, + last_change_at: self.timers.now_ms(), + expiry: 0, + }); + if field.alignment_blocked { + return false; + } + let selections = aligned_range(pre, &field.baseline, selected.clone()) + .zip(aligned_range(pre, &field.reported, selected.clone())); + let Some((baseline_range, reported_range)) = selections else { + // Preserve the preceding scope instead of inventing a location + // for this delivered text. Further values cannot safely be + // attributed until the field lifecycle is reset. + field.alignment_blocked = true; + field.change += 1; + drop(observations); + self.observation_failure( + pid, + Some(key.1.clone()), + "text_alignment_budget_or_ambiguity", + ); + return false; + }; + field.pasted_ranges = + replaced_regions(&field.pasted_ranges, baseline_range.clone(), text.len()); + if !text.is_empty() { + field + .pasted_ranges + .push(baseline_range.start..baseline_range.start + text.len()); } + field.baseline.replace_range(baseline_range, text); + field.reported.replace_range(reported_range, text); + let recovered = recovered || field.unreadable; + field.unreadable = false; + field.latest = predicted; + field.empty = false; + field.change += 1; + field.last_change_at = self.timers.now_ms(); + field.expiry = ticket; + (field.expiry, field.change, recovered) }; - } - rest.strip_prefix(':').map(str::to_string) -} - -/// One `macos-text-monitor ` run: forwards the helper's lines and -/// judges its diagnostics when it ends. -struct EditWatch { - args: Vec, - stderr: String, - no_element: bool, -} - -impl Protocol for EditWatch { - type Event = WatchEvent; - - fn args(&self) -> Vec { - self.args.clone() + let stable_key = key.clone(); + let weak = self.this.clone(); + self.timers.schedule( + Duration::from_secs(1), + Box::new(move || { + if let Some(inner) = weak.upgrade() { + inner.complete_observation(&stable_key, Some(change)); + } + }), + ); + self.publish_observation( + self.field_report( + &key, + if recovered { + ObservationOutcome::Recovered + } else { + ObservationOutcome::Readable + }, + None, + false, + ) + .unwrap(), + ); + let weak = self.this.clone(); + self.timers.schedule( + MONITOR_TIMEOUT, + Box::new(move || { + if let Some(inner) = weak.upgrade() { + let mut observations = lock(&inner.observations); + if observations + .fields + .get(&key) + .is_some_and(|field| field.expiry == expiry) + { + observations.fields.remove(&key); + drop(observations); + inner.observation_command( + pid, + serde_json::json!({"command":"forget", "field":key.1}), + ); + } + } + }), + ); + true } - fn on_stdout(&mut self, line: &str, cx: &mut Control<'_, WatchEvent>) { - if line == "NO_ELEMENT" { - self.no_element = true; - } - cx.emit(WatchEvent::Line(line.to_string())); + fn field_report( + &self, + key: &(i32, String), + outcome: ObservationOutcome, + diagnostic: Option, + edited: bool, + ) -> Option { + let observations = lock(&self.observations); + let field = observations.fields.get(key)?; + Some(SessionReport { + app: field.app.clone(), + readable: !field.unreadable, + edit: edited.then(|| TextEdit { + baseline: field.baseline.clone(), + edited: field.latest.clone(), + pasted_ranges: field.pasted_ranges.clone(), + }), + outcome, + diagnostic, + field: Some(key.1.clone()), + session_id: field.session_id.clone(), + revision: field.revision, + }) } - fn on_stderr(&mut self, text: &str, _cx: &mut Control<'_, WatchEvent>) { - if self.stderr.len() < MAX_STDERR_BYTES { - self.stderr.push_str(text); - } + fn complete_observation(&self, key: &(i32, String), expected_change: Option) { + let report = { + let mut observations = lock(&self.observations); + let Some(field) = observations.fields.get_mut(key) else { + return; + }; + if field.health_only + || field.alignment_blocked + || field.composing + || (field.empty && expected_change.is_some()) + || field.unreadable + || field.latest.trim().is_empty() + || expected_change.is_some_and(|change| field.change != change) + || field.reported == field.latest + { + return; + } + field.reported = field.latest.clone(); + field.revision += 1; + // Capture exactly the completed revision under the same lock: a + // new composing value must never replace the text being reported. + SessionReport { + app: field.app.clone(), + readable: true, + outcome: ObservationOutcome::CompletedEdit, + diagnostic: None, + field: Some(key.1.clone()), + session_id: field.session_id.clone(), + revision: field.revision, + edit: Some(TextEdit { + baseline: field.baseline.clone(), + edited: field.latest.clone(), + pasted_ranges: field.pasted_ranges.clone(), + }), + } + }; + // An empty diff revokes the preceding contribution after a restoration. + self.publish_observation(report); } - fn on_down(&mut self, down: &Down, _recovery: Recovery, cx: &mut Control<'_, WatchEvent>) { - match down { - Down::Stopped => {} - Down::Missing | Down::NotStarted(_) => { - log::debug!("[text-monitor] helper failed to start ({down:?})"); - cx.emit(WatchEvent::Ended { - ax_unsupported: false, + fn observe_native(&self, pid: i32, event: NativeObservation) { + if event.kind == "health_recovered" { + if lock(&self.observations) + .failures + .remove(&(pid, String::new())) + { + self.publish_observation(SessionReport { + app: appkit::bundle_identifier(pid), + readable: true, + edit: None, + outcome: ObservationOutcome::Recovered, + diagnostic: None, + field: None, + session_id: String::new(), + revision: 0, }); } - Down::Failed(_) | Down::Unavailable(_) | Down::Exited { .. } => { - cx.emit(WatchEvent::Ended { - ax_unsupported: self.no_element - && is_persistent_ax_no_value_failure(&self.stderr), + return; + } + if event.kind == "helper_ended" { + lock(&self.observations).runs.remove(&pid); + let keys: Vec<_> = lock(&self.observations) + .fields + .keys() + .filter(|key| key.0 == pid) + .cloned() + .collect(); + for key in keys { + self.observe_native( + pid, + NativeObservation { + kind: "unreadable".into(), + field: key.1, + ..event.clone() + }, + ); + } + return; + } + if event.kind == "synced" { + let stable: Vec<_> = lock(&self.observations) + .fields + .iter() + .filter(|(key, field)| { + key.0 == pid + && self.timers.now_ms().saturating_sub(field.last_change_at) >= 1_000 }) + .map(|(key, field)| (key.clone(), field.change)) + .collect(); + for (key, change) in stable { + self.complete_observation(&key, Some(change)); } } - } -} - -/// AXError -25212 (kAXErrorNoValue) on every attempt, and the element never -/// resolved: a Chromium/Electron app whose AX tree stays dormant. -fn is_persistent_ax_no_value_failure(stderr: &str) -> bool { - if stderr.contains("Got focused element") { - return false; - } - let mut codes = Vec::new(); - let mut rest = stderr; - while let Some(start) = rest.find("(error: ") { - let after = &rest[start + 8..]; - let Some(end) = after.find(')') else { break }; - let code = &after[..end]; - if code - .trim_start_matches('-') - .chars() - .all(|c| c.is_ascii_digit()) - && !code.is_empty() + if event.kind == "prepared" || event.kind == "synced" { + if let Some(sender) = lock(&self.observations).pending.remove(&event.ticket) { + let _ = sender.try_send(event); + } + return; + } + let key = (pid, event.field.clone()); + if event.kind == "sent" { + self.complete_observation(&key, None); + lock(&self.observations).fields.remove(&key); + return; + } + if event.kind == "closed" { + lock(&self.observations).fields.remove(&key); + return; + } + if event.kind == "unreadable" { + let notify = { + let mut observations = lock(&self.observations); + if let Some(field) = observations.fields.get_mut(&key) { + let notify = !field.unreadable; + field.unreadable = true; + notify + } else { + false + } + }; + if notify + && let Some(report) = + self.field_report(&key, ObservationOutcome::Unreadable, event.reason, false) + { + self.publish_observation(report); + } + return; + } + let Some(value) = event.value else { return }; + let (change, recovered) = { + let mut observations = lock(&self.observations); + let Some(field) = observations.fields.get_mut(&key) else { + return; + }; + if field.alignment_blocked { + return; + } + let recovered = field.unreadable; + field.unreadable = false; + // Empty values are never evidence of submission. The helper sends + // a separate sent event after a key/mouse send gesture and clear. + if value.trim().is_empty() { + if !field.empty { + field.change += 1; + field.last_change_at = self.timers.now_ms(); + } + field.empty = true; + (None, recovered) + } else { + let was_empty = field.empty; + field.empty = false; + if value == field.latest + && event.composing == field.composing + && !recovered + && !was_empty + { + return; + } + field.latest = value; + field.composing = event.composing; + field.change += 1; + field.last_change_at = self.timers.now_ms(); + (Some(field.change), recovered) + } + }; + if recovered { + lock(&self.observations).failures.remove(&key); + } + if recovered + && let Some(report) = + self.field_report(&key, ObservationOutcome::Recovered, None, false) { - codes.push(code.to_string()); + self.publish_observation(report); } - rest = &after[end..]; + let Some(change) = change else { return }; + let weak = self.this.clone(); + self.timers.schedule( + Duration::from_secs(1), + Box::new(move || { + if let Some(inner) = weak.upgrade() { + inner.complete_observation(&key, Some(change)); + } + }), + ); } - !codes.is_empty() && codes.iter().all(|code| code == "-25212") } -/// In-process AppKit lookups. +fn utf16_range(value: &str, start: usize, length: usize) -> Option> { + fn offset(value: &str, requested: usize) -> Option { + let mut units = 0; + for (byte, character) in value.char_indices() { + if units == requested { + return Some(byte); + } + units += character.len_utf16(); + if units > requested { + return None; + } + } + (units == requested).then_some(value.len()) + } + Some(offset(value, start)?..offset(value, start.checked_add(length)?)?) +} + +fn replaced_regions( + regions: &[Range], + replaced: Range, + inserted: usize, +) -> Vec> { + let mut updated = Vec::new(); + for region in regions { + let left_end = region.end.min(replaced.start); + if region.start < left_end { + updated.push(region.start..left_end); + } + let right_start = region.start.max(replaced.end); + if right_start < region.end { + let start = replaced.start + inserted + right_start - replaced.end; + let end = replaced.start + inserted + region.end - replaced.end; + updated.push(start..end); + } + } + updated +} + +/// Map boundaries through every optimal character edit alignment. A bounded +/// band makes small corrections in long editors cheap without greedily +/// anchoring repeated prefixes/suffixes to an arbitrary occurrence. +fn aligned_range(from: &str, to: &str, selected: Range) -> Option> { + const CELL_BUDGET: usize = 1_000_000; + const WORK_BUDGET: usize = 2 * CELL_BUDGET; + const UNREACHABLE: u32 = u32::MAX / 4; + struct Grid { + rows: usize, + columns: usize, + width: usize, + band: usize, + costs: Vec, + } + impl Grid { + fn index(&self, row: usize, column: usize) -> Option { + if row >= self.rows || column >= self.columns || row.abs_diff(column) > self.band { + return None; + } + let offset = if self.width == self.columns { + column + } else { + column + self.band - row + }; + Some(row * self.width + offset) + } + fn cost(&self, row: usize, column: usize) -> u32 { + self.index(row, column) + .map_or(UNREACHABLE, |index| self.costs[index]) + } + } + fn characters(value: &str) -> (Vec, Vec) { + let (characters, mut offsets): (Vec<_>, Vec<_>) = value + .char_indices() + .map(|(offset, character)| (character, offset)) + .unzip(); + offsets.push(value.len()); + (characters, offsets) + } + if selected.start > selected.end + || selected.end > from.len() + || !from.is_char_boundary(selected.start) + || !from.is_char_boundary(selected.end) + { + return None; + } + if from == to { + return Some(selected); + } + // An emptied field supplies no anchor for a position in its old text. + if from.is_empty() { + return None; + } + // Physical field edges do not depend on the alignment of interior edits. + // This also keeps appending to a long corrected field within paste latency. + let edge = |offset| { + if offset == 0 { + Some(0) + } else if offset == from.len() { + Some(to.len()) + } else { + None + } + }; + if let (Some(start), Some(end)) = (edge(selected.start), edge(selected.end)) { + return Some(start..end); + } + let (source, source_offsets) = characters(from); + let (target, target_offsets) = characters(to); + let boundaries = [ + source_offsets.binary_search(&selected.start).ok()?, + source_offsets.binary_search(&selected.end).ok()?, + ]; + let mut band = source.len().abs_diff(target.len()).max(1); + let mut work = WORK_BUDGET; + let grid = loop { + let rows = source.len().checked_add(1)?; + let columns = target.len().checked_add(1)?; + let width = band.checked_mul(2)?.checked_add(1)?.min(columns); + let cells = rows.checked_mul(width)?; + if cells > CELL_BUDGET { + return None; + } + work = work.checked_sub(cells)?; + let mut grid = Grid { + rows, + columns, + width, + band, + costs: vec![UNREACHABLE; cells], + }; + for row in 0..rows { + let first = row.saturating_sub(band); + let last = row.saturating_add(band).min(target.len()); + for column in first..=last { + let cost = if row == 0 { + column as u32 + } else if column == 0 { + row as u32 + } else { + (grid.cost(row - 1, column) + 1) + .min(grid.cost(row, column - 1) + 1) + .min( + grid.cost(row - 1, column - 1) + + u32::from(source[row - 1] != target[column - 1]), + ) + }; + let index = grid.index(row, column)?; + grid.costs[index] = cost; + } + } + // Every path of cost D stays within a band of width D. Once D fits, + // this matrix includes all globally optimal paths, including ties. + if grid.cost(source.len(), target.len()) as usize <= band { + break grid; + } + band = band.checked_mul(2)?.min(source.len().max(target.len())); + }; + let mut mapped = [None, None]; + for (index, &boundary) in boundaries.iter().enumerate() { + if boundary == 0 { + mapped[index] = Some(0); + } else if boundary == source.len() { + mapped[index] = Some(target.len()); + } + } + let mut visited = vec![false; grid.costs.len()]; + let mut pending = vec![(source.len(), target.len())]; + visited[grid.index(source.len(), target.len())?] = true; + while let Some((row, column)) = pending.pop() { + for index in 0..2 { + if boundaries[index] > 0 && boundaries[index] < source.len() && row == boundaries[index] + { + if mapped[index].is_some_and(|previous| previous != column) { + return None; + } + mapped[index] = Some(column); + } + } + let cost = grid.cost(row, column); + let mut visit = |row: usize, column: usize| { + if let Some(index) = grid.index(row, column) + && !visited[index] + { + visited[index] = true; + pending.push((row, column)); + } + }; + if row > 0 && cost == grid.cost(row - 1, column) + 1 { + visit(row - 1, column); + } + if column > 0 && cost == grid.cost(row, column - 1) + 1 { + visit(row, column - 1); + } + if row > 0 + && column > 0 + && cost + == grid.cost(row - 1, column - 1) + u32::from(source[row - 1] != target[column - 1]) + { + visit(row - 1, column - 1); + } + } + let start = target_offsets[mapped[0]?]; + let end = target_offsets[mapped[1]?]; + (start <= end).then_some(start..end) +} + +struct ObservationProtocol { + args: Vec, +} +impl Protocol for ObservationProtocol { + type Event = NativeObservation; + fn args(&self) -> Vec { + self.args.clone() + } + fn on_stdout(&mut self, line: &str, cx: &mut Control<'_, NativeObservation>) { + if let Ok(event) = serde_json::from_str(line) { + cx.emit(event); + } + } + fn on_down( + &mut self, + down: &Down, + _recovery: Recovery, + cx: &mut Control<'_, NativeObservation>, + ) { + if !matches!(down, Down::Stopped) { + cx.emit(NativeObservation { + kind: "helper_ended".into(), + ticket: 0, + field: String::new(), + value: None, + start: 0, + length: 0, + composing: false, + reason: Some(format!("helper_ended: {down:?}")), + }); + } + } +} + +/// In-process AppKit lookups. pub(crate) mod appkit { use objc2_app_kit::{ NSApplicationActivationOptions, NSApplicationActivationPolicy, NSRunningApplication, @@ -748,7 +1331,7 @@ mod tests { use crate::native::supervised_child::ChildEvent; use crate::native::supervised_child::tests::FakeSpawner; use crate::native::timer::FakeTimers; - use std::sync::atomic::AtomicUsize; + use std::sync::atomic::{AtomicUsize, Ordering}; fn sh(script: &str) -> HelperResolver { let script = script.to_string(); @@ -801,16 +1384,6 @@ mod tests { Arc::new(|| None) } - fn watch( - monitor: &TextEditMonitor, - text: &str, - pid: Option, - ) -> async_channel::Receiver { - let (report, receiver) = async_channel::bounded(1); - monitor.start_monitoring(text, MONITOR_TIMEOUT, pid, report); - receiver - } - fn counting(values: Vec>, delay: Duration) -> (PidSource, Arc) { let calls = Arc::new(AtomicUsize::new(0)); let counter = Arc::clone(&calls); @@ -822,82 +1395,568 @@ mod tests { (source, calls) } - fn assert_pending_start_cancelled(receiver: &async_channel::Receiver) { - let deadline = Instant::now() + Duration::from_secs(5); - while !receiver.is_closed() { - assert!( - Instant::now() < deadline, - "stale monitoring start was not cancelled" + #[test] + fn prepared_observation_captures_a_correction_in_the_first_second() { + let spawner = Arc::new(FakeSpawner::default()); + let timers = FakeTimers::new(); + let monitor = TextEditMonitor::build(fake_hooks(&spawner), timers.clone()); + let reports = monitor.subscribe(); + let other = monitor.clone(); + let preparing = std::thread::spawn(move || other.prepare_paste("hello world", Some(42))); + wait_for_spawn(&spawner); + let deadline = Instant::now() + Duration::from_secs(2); + while lock(&spawner.child(0)).writes.is_empty() { + assert!(Instant::now() < deadline); + std::thread::sleep(Duration::from_millis(5)); + } + spawner.emit( + 0, + ChildEvent::Stdout( + r#"{"kind":"prepared","ticket":1,"field":"a","value":"","start":0,"length":0}"# + .to_string() + + "\n", + ), + ); + let ticket = preparing.join().unwrap().expect("bound before paste"); + ticket.commit(true); + spawner.emit( + 0, + ChildEvent::Stdout( + r#"{"kind":"value","field":"a","value":"hello World","composing":false}"# + .to_string() + + "\n", + ), + ); + timers.advance(1_000); + let report = std::iter::from_fn(|| reports.try_recv().ok()) + .find(|report| report.edit.is_some()) + .unwrap(); + assert_eq!( + report.edit.unwrap(), + TextEdit { + baseline: "hello world".into(), + edited: "hello World".into(), + pasted_ranges: std::iter::once(0..11).collect(), + } + ); + monitor.stop_monitoring(); + } + + struct ObservationFixture { + monitor: TextEditMonitor, + spawner: Arc, + timers: Arc, + reports: async_channel::Receiver, + } + + impl ObservationFixture { + fn new() -> Self { + let spawner = Arc::new(FakeSpawner::default()); + let timers = FakeTimers::new(); + let monitor = TextEditMonitor::build(fake_hooks(&spawner), timers.clone()); + let reports = monitor.subscribe(); + Self { + monitor, + spawner, + timers, + reports, + } + } + + fn paste(&self, field: &str, pre: &str, text: &str) { + self.prepare(field, Some(pre), text).unwrap().commit(true); + } + + fn prepare(&self, field: &str, pre: Option<&str>, text: &str) -> Option { + self.prepare_at( + field, + pre, + text, + pre.unwrap_or_default().encode_utf16().count(), + 0, + ) + } + + fn prepare_at( + &self, + field: &str, + pre: Option<&str>, + text: &str, + start: usize, + length: usize, + ) -> Option { + let previous_count = self.spawner.count(); + let previous = if previous_count > 0 { + lock(&self.spawner.child(previous_count - 1)).writes.len() + } else { + 0 + }; + let monitor = self.monitor.clone(); + let text = text.to_string(); + let preparing = std::thread::spawn(move || monitor.prepare_paste(&text, Some(42))); + wait_for_spawn(&self.spawner); + let deadline = Instant::now() + Duration::from_secs(2); + let ticket = loop { + let count = self.spawner.count(); + let offset = if count == previous_count { previous } else { 0 }; + if let Some(command) = lock(&self.spawner.child(count - 1)).writes.get(offset) { + let command: serde_json::Value = serde_json::from_str(command).unwrap(); + break command["ticket"].as_u64().unwrap(); + } + assert!(Instant::now() < deadline); + std::thread::sleep(Duration::from_millis(5)); + }; + self.emit( + serde_json::json!({"kind":"prepared", "ticket":ticket, "field":field, + "value":pre, "start":start, "length":length}), ); - std::thread::sleep(Duration::from_millis(10)); + preparing.join().unwrap() + } + + fn emit(&self, event: serde_json::Value) { + self.spawner.emit( + self.spawner.count() - 1, + ChildEvent::Stdout(event.to_string() + "\n"), + ); + } + + fn edits(&self) -> Vec { + std::iter::from_fn(|| self.reports.try_recv().ok()) + .filter(|report| report.outcome == ObservationOutcome::CompletedEdit) + .collect() } + } + + impl Drop for ObservationFixture { + fn drop(&mut self) { + self.monitor.stop_monitoring(); + } + } + + #[test] + fn temporary_emptiness_interrupts_completion_until_replacement_is_stable() { + let fixture = ObservationFixture::new(); + fixture.paste("a", "", "hello world"); + fixture.emit(serde_json::json!({"kind":"value", "field":"a", "value":"hello World"})); + fixture.timers.advance(500); + fixture.emit(serde_json::json!({"kind":"value", "field":"a", "value":""})); + fixture.timers.advance(1_000); + assert!( + fixture.edits().is_empty(), + "select-delete must not finalize the previous candidate" + ); + fixture.emit(serde_json::json!({"kind":"value", "field":"a", "value":"hello WORLD"})); + fixture.timers.advance(1_000); assert_eq!( - receiver.try_recv(), - Err(async_channel::TryRecvError::Closed) + fixture.edits()[0].edit.as_ref().unwrap().edited, + "hello WORLD" ); } #[test] - fn stop_invalidates_a_delayed_start() { - let spawner = Arc::new(FakeSpawner::default()); - let timers = FakeTimers::new(); - let monitor = TextEditMonitor::build(fake_hooks(&spawner), timers.clone()); - let pending = monitor.start_monitoring_after_paste("old", Some(42)); - monitor.stop_monitoring(); - timers.advance(MONITOR_START_DELAY.as_millis() as u64); - assert_pending_start_cancelled(&pending); - assert!(!monitor.is_monitoring()); - assert_eq!(spawner.count(), 0); + fn unreadable_observation_recovers_once_without_losing_its_field() { + let fixture = ObservationFixture::new(); + fixture.paste("a", "", "hello world"); + fixture.edits(); + for _ in 0..2 { + fixture.emit(serde_json::json!({"kind":"unreadable", "field":"a", "reason":"ax_cannot_complete"})); + } + let failure = fixture.reports.try_recv().unwrap(); + assert_eq!(failure.outcome, ObservationOutcome::Unreadable); + assert!(fixture.reports.try_recv().is_err()); + fixture.emit(serde_json::json!({"kind":"value", "field":"a", "value":""})); + let recovered = fixture + .reports + .try_recv() + .expect("even an empty field is a successful read"); + assert_eq!(recovered.outcome, ObservationOutcome::Recovered); + assert_eq!(recovered.session_id, failure.session_id); } #[test] - fn newer_delayed_start_invalidates_the_older_start() { - let spawner = Arc::new(FakeSpawner::default()); - let timers = FakeTimers::new(); - let monitor = TextEditMonitor::build(fake_hooks(&spawner), timers.clone()); - let old = monitor.start_monitoring_after_paste("old", Some(42)); - timers.advance(500); - let new = monitor.start_monitoring_after_paste("new", Some(42)); - timers.advance(500); - assert_pending_start_cancelled(&old); - assert!(!monitor.is_monitoring()); - timers.advance(500); - wait_for_spawn(&spawner); - assert_eq!(spawner.count(), 1); + fn a_new_dictation_preserves_an_unfinished_correction_to_the_earlier_passage() { + let fixture = ObservationFixture::new(); + fixture.paste("a", "", "hello world"); + fixture.emit(serde_json::json!({"kind":"value", "field":"a", "value":"hello World"})); + fixture.timers.advance(500); + fixture.paste("a", "hello World", " again"); + fixture.timers.advance(1_000); + let edits = fixture.edits(); assert_eq!( - lock(&monitor.inner.monitoring) - .session - .as_ref() - .unwrap() - .pasted, - ["new"] + edits.len(), + 1, + "the second dictation must not erase the pending earlier correction" ); - monitor.stop_monitoring(); - assert!(new.recv_blocking().is_ok()); + assert_eq!( + edits[0].edit.as_ref().unwrap(), + &TextEdit { + baseline: "hello world again".into(), + edited: "hello World again".into(), + pasted_ranges: vec![0..11, 11..17], + } + ); + } + + #[test] + fn synchronizing_flush_publishes_an_already_stable_edit_before_returning() { + let fixture = ObservationFixture::new(); + fixture.paste("a", "", "hello world"); + fixture.emit(serde_json::json!({"kind":"value", "field":"a", "value":"hello World"})); + fixture.timers.advance_without_running(1_000); + let previous = lock(&fixture.spawner.child(0)).writes.len(); + let monitor = fixture.monitor.clone(); + let flushing = std::thread::spawn(move || monitor.flush_completed_edits()); + let deadline = Instant::now() + Duration::from_secs(2); + let ticket = loop { + if let Some(command) = lock(&fixture.spawner.child(0)).writes.get(previous) { + let command: serde_json::Value = serde_json::from_str(command).unwrap(); + break command["ticket"].as_u64().unwrap(); + } + assert!(Instant::now() < deadline); + std::thread::sleep(Duration::from_millis(5)); + }; + fixture.emit(serde_json::json!({"kind":"synced", "ticket":ticket})); + flushing.join().unwrap(); + assert_eq!( + fixture.edits().len(), + 1, + "a delayed timer must not hide a completed edit at capture" + ); + } + + #[test] + fn initially_unreadable_field_recovers_without_learning_unknown_old_content() { + let fixture = ObservationFixture::new(); + assert!(fixture.prepare("a", None, "hello world").is_none()); + assert_eq!( + fixture.reports.try_recv().unwrap().outcome, + ObservationOutcome::Unreadable + ); + assert!(fixture.prepare("a", None, "hello world").is_none()); + assert!( + fixture.reports.try_recv().is_err(), + "one unreadable hint per episode" + ); + fixture.emit(serde_json::json!({"kind":"value", "field":"a", "value":"unrelated old draft hello World"})); + assert_eq!( + fixture.reports.try_recv().unwrap().outcome, + ObservationOutcome::Recovered + ); + fixture.timers.advance(1_000); + assert!( + fixture.edits().is_empty(), + "unknown baseline cannot support attribution" + ); + fixture.paste("a", "unrelated old draft hello World", " again"); + fixture.emit(serde_json::json!({"kind":"value", "field":"a", "value":"unrelated old draft hello World AGAIN"})); + fixture.timers.advance(1_000); + let edits = fixture.edits(); + assert_eq!( + edits[0].edit.as_ref().unwrap().pasted_ranges, + std::iter::once(31..37).collect::>() + ); + } + + #[test] + fn fields_keep_independent_completed_revisions_and_flush_an_immediate_send() { + let fixture = ObservationFixture::new(); + fixture.paste("a", "", "hello world"); + fixture.paste("b", "", "second field"); + fixture.emit(serde_json::json!({"kind":"value", "field":"a", "value":"hello World", "composing":true})); + fixture.emit(serde_json::json!({"kind":"value", "field":"b", "value":"second Field"})); + fixture.timers.advance(1_000); + let first = fixture.edits(); + assert_eq!(first.len(), 1); + assert_eq!(first[0].field.as_deref(), Some("b")); + fixture.emit(serde_json::json!({"kind":"value", "field":"a", "value":"hello World", "composing":false})); + fixture.timers.advance(1_000); + let a = fixture.edits().remove(0); + assert_ne!(a.session_id, first[0].session_id); + fixture.emit(serde_json::json!({"kind":"value", "field":"a", "value":"hello world"})); + fixture.timers.advance(1_000); + let restored = fixture.edits().remove(0); + assert_eq!(restored.session_id, a.session_id); + assert_eq!(restored.revision, 2); + assert_eq!( + restored.edit.as_ref().unwrap().baseline, + restored.edit.as_ref().unwrap().edited + ); + fixture.emit(serde_json::json!({"kind":"value", "field":"a", "value":"hello WORLD"})); + fixture.emit(serde_json::json!({"kind":"value", "field":"a", "value":"\n"})); + fixture.emit(serde_json::json!({"kind":"sent", "field":"a"})); + let sent = fixture.edits().remove(0); + assert_eq!(sent.revision, 3); + assert_eq!(sent.edit.unwrap().edited, "hello WORLD"); + fixture.emit( + serde_json::json!({"kind":"value", "field":"a", "value":"unrelated next message"}), + ); + fixture.timers.advance(1_000); + assert!(fixture.edits().is_empty()); + } + + #[test] + fn disabling_observation_invalidates_prepared_tickets_and_reenable_starts_fresh() { + let fixture = ObservationFixture::new(); + let ticket = fixture.prepare("a", Some(""), "hello world").unwrap(); + fixture.monitor.set_enabled(false); + ticket.commit(true); + assert!(fixture.monitor.prepare_paste("ignored", Some(42)).is_none()); + fixture.timers.advance(1_000); + assert!(fixture.edits().is_empty()); + fixture.monitor.set_enabled(true); + fixture.paste("a", "", "new passage"); + fixture.emit(serde_json::json!({"kind":"value", "field":"a", "value":"new Passage"})); + fixture.timers.advance(1_000); + let edits = fixture.edits(); + assert_eq!(edits.len(), 1); + assert_eq!( + edits[0].edit.as_ref().unwrap().pasted_ranges, + std::iter::once(0..11).collect::>() + ); + } + + #[test] + fn canceled_delivery_does_not_create_a_learning_scope() { + let fixture = ObservationFixture::new(); + fixture + .prepare("a", Some("old draft"), "ignored") + .unwrap() + .commit(false); + fixture + .emit(serde_json::json!({"kind":"value", "field":"a", "value":"unrelated correction"})); + fixture.timers.advance(1_000); + assert!(fixture.edits().is_empty()); } #[test] - fn a_delayed_start_cannot_replace_a_newer_immediate_session() { + fn missing_helper_retries_and_reports_recovery_without_another_paste() { let spawner = Arc::new(FakeSpawner::default()); let timers = FakeTimers::new(); - let monitor = TextEditMonitor::build(fake_hooks(&spawner), timers.clone()); - let old = monitor.start_monitoring_after_paste("old", Some(42)); - let new = watch(&monitor, "new", Some(42)); - wait_for_spawn(&spawner); - timers.advance(MONITOR_START_DELAY.as_millis() as u64); - assert_pending_start_cancelled(&old); - assert!(monitor.is_monitoring()); - assert_eq!(spawner.count(), 1); + let available = Arc::new(std::sync::atomic::AtomicBool::new(false)); + let readable = available.clone(); + let mut hooks = fake_hooks(&spawner); + hooks.resolve_helper = Arc::new(move || { + readable.load(Ordering::SeqCst).then(|| HelperCommand { + program: PathBuf::from("/fake/observer"), + args: Vec::new(), + }) + }); + let monitor = TextEditMonitor::build(hooks, timers.clone()); + let reports = monitor.subscribe(); + assert!(monitor.prepare_paste("hello world", Some(42)).is_none()); + assert_eq!( + reports.try_recv().unwrap().diagnostic.as_deref(), + Some("helper_unavailable") + ); + assert!(monitor.prepare_paste("hello world", Some(42)).is_none()); + assert!(reports.try_recv().is_err()); + available.store(true, Ordering::SeqCst); + timers.advance(1_000); + spawner.emit( + 0, + ChildEvent::Stdout("{\"kind\":\"health_recovered\"}\n".into()), + ); assert_eq!( - lock(&monitor.inner.monitoring) - .session - .as_ref() - .unwrap() - .pasted, - ["new"] + reports.try_recv().unwrap().outcome, + ObservationOutcome::Recovered ); + timers.advance(1_000); + assert!(reports.try_recv().is_err()); monitor.stop_monitoring(); - assert!(new.recv_blocking().is_ok()); + } + + #[test] + fn disabling_during_preparation_cancels_the_wait_without_a_stale_scope() { + let fixture = ObservationFixture::new(); + let monitor = fixture.monitor.clone(); + let preparing = std::thread::spawn(move || monitor.prepare_paste("hello world", Some(42))); + wait_for_spawn(&fixture.spawner); + fixture.monitor.set_enabled(false); + assert!(preparing.join().unwrap().is_none()); + fixture.timers.advance(1_000); + assert!(fixture.edits().is_empty()); + } + + #[test] + fn a_correction_near_a_long_editor_tail_keeps_its_original_context() { + let fixture = ObservationFixture::new(); + let context = "existing context 😀 ".repeat(1_024); + fixture.paste("a", &context, "hello world"); + fixture.emit(serde_json::json!({"kind":"value", "field":"a", "value":format!("{context}hello World")})); + fixture.timers.advance(1_000); + let edits = fixture.edits(); + let edit = edits[0].edit.as_ref().unwrap(); + assert_eq!(edit.baseline, format!("{context}hello world")); + assert_eq!(edit.edited, format!("{context}hello World")); + assert_eq!( + edit.pasted_ranges, + std::iter::once(22_528..22_539).collect::>() + ); + } + + #[test] + fn repeated_identical_dictation_retains_each_delivered_region() { + let fixture = ObservationFixture::new(); + fixture.paste("a", "", "hello world "); + fixture.paste("a", "hello world ", "hello world "); + fixture.emit( + serde_json::json!({"kind":"value", "field":"a", "value":"hello World hello world "}), + ); + fixture.timers.advance(1_000); + let report = fixture.edits().remove(0); + let edit = report.edit.unwrap(); + assert_eq!(edit.baseline, "hello world hello world "); + assert_eq!(edit.pasted_ranges, [0..12, 12..24]); + } + + #[test] + fn dictation_before_identical_existing_content_tracks_only_the_delivered_occurrence() { + let fixture = ObservationFixture::new(); + fixture + .prepare_at("a", Some("hello world"), "hello world ", 0, 0) + .unwrap() + .commit(true); + fixture.emit( + serde_json::json!({"kind":"value", "field":"a", "value":"hello World hello world"}), + ); + fixture.timers.advance(1_000); + let edit = fixture.edits().remove(0).edit.unwrap(); + assert_eq!(edit.baseline, "hello world hello world"); + assert_eq!( + edit.pasted_ranges, + std::iter::once(0..12).collect::>() + ); + } + + #[test] + fn dictation_between_multiple_corrections_preserves_original_baseline_and_regions() { + let fixture = ObservationFixture::new(); + fixture.paste("a", "", "use cloud code with Shunade today"); + fixture.emit(serde_json::json!({"kind":"value", "field":"a", "value":"use Claude Code with Sinead today"})); + fixture + .prepare_at( + "a", + Some("use Claude Code with Sinead today"), + "new ", + 21, + 0, + ) + .unwrap() + .commit(true); + fixture.timers.advance(1_000); + let edit = fixture.edits().remove(0).edit.unwrap(); + assert_eq!(edit.baseline, "use cloud code with new Shunade today"); + assert_eq!(edit.edited, "use Claude Code with new Sinead today"); + assert_eq!(edit.pasted_ranges, [0..20, 24..37, 20..24]); + } + + #[test] + fn replacement_dictation_splits_old_regions_and_keeps_unicode_byte_boundaries() { + let fixture = ObservationFixture::new(); + fixture.paste("a", "", "hello world 😀 tail"); + fixture + .prepare_at("a", Some("hello world 😀 tail"), "Claude Code", 6, 5) + .unwrap() + .commit(true); + fixture.emit( + serde_json::json!({"kind":"value", "field":"a", "value":"Hello Claude code 😀 tail"}), + ); + fixture.timers.advance(1_000); + let edit = fixture.edits().remove(0).edit.unwrap(); + assert_eq!(edit.baseline, "hello Claude Code 😀 tail"); + assert_eq!(edit.pasted_ranges, [0..6, 17..27, 6..17]); + } + + #[test] + fn alignment_budget_failure_preserves_prior_learning_without_claiming_the_new_paste() { + let fixture = ObservationFixture::new(); + let original = format!( + "use cloud code {} middle {}", + "a".repeat(1_100), + "b".repeat(1_100) + ); + let corrected = format!( + "use Claude Code {} middle {}", + "a".repeat(1_100), + "b".repeat(1_100) + ); + fixture.paste("a", "", &original); + fixture.emit(serde_json::json!({"kind":"value", "field":"a", "value":corrected})); + fixture.timers.advance(1_000); + let completed = fixture.edits().remove(0); + let replaced = format!( + "use Claude Code {} middle {}", + "c".repeat(1_100), + "d".repeat(1_100) + ); + fixture.emit(serde_json::json!({"kind":"value", "field":"a", "value":replaced})); + fixture + .prepare_at("a", Some(&replaced), "new ", 1_117, 0) + .unwrap() + .commit(true); + let unavailable = fixture.reports.try_recv().unwrap(); + assert_eq!(unavailable.session_id, completed.session_id); + assert_eq!( + unavailable.diagnostic.as_deref(), + Some("text_alignment_budget_or_ambiguity") + ); + fixture.emit( + serde_json::json!({"kind":"value", "field":"a", "value":"unrelated new content"}), + ); + fixture.timers.advance(1_000); + assert!( + fixture.edits().is_empty(), + "uncertain insertion must not revoke or replace the prior completed correction" + ); + fixture.emit(serde_json::json!({"kind":"sent", "field":"a"})); + fixture.paste("a", "", "fresh passage"); + fixture.emit(serde_json::json!({"kind":"value", "field":"a", "value":"fresh Passage"})); + fixture.timers.advance(1_000); + let fresh = fixture.edits().remove(0); + assert_ne!(fresh.session_id, completed.session_id); + assert_eq!( + fresh.edit.unwrap().pasted_ranges, + std::iter::once(0..13).collect::>() + ); + } + + #[test] + fn repeated_character_alignment_does_not_guess_which_original_position_survived() { + let fixture = ObservationFixture::new(); + fixture.paste("a", "", "aa"); + fixture.emit(serde_json::json!({"kind":"value", "field":"a", "value":"aaa"})); + fixture.edits(); + fixture + .prepare_at("a", Some("aaa"), "B", 1, 0) + .unwrap() + .commit(true); + let report = fixture.reports.try_recv().unwrap(); + assert_eq!( + report.outcome, + ObservationOutcome::Unreadable, + "three equally optimal insertion locations cannot establish provenance" + ); + fixture.timers.advance(1_000); + assert!(fixture.edits().is_empty()); + } + + #[test] + fn appending_to_a_long_field_maps_the_physical_end_without_an_alignment_budget() { + let fixture = ObservationFixture::new(); + let context = "context ".repeat(30_000); + let original = format!("start cloud {context} Shunade end"); + let corrected = format!("start Claude {context} Sinead end"); + fixture.paste("a", "", &original); + fixture.emit(serde_json::json!({"kind":"value", "field":"a", "value":corrected})); + fixture.paste("a", &corrected, " again"); + fixture.timers.advance(1_000); + let edit = fixture.edits().remove(0).edit.unwrap(); + assert_eq!(edit.baseline, original.clone() + " again"); + assert_eq!(edit.edited, corrected + " again"); + assert_eq!( + edit.pasted_ranges, + [0..original.len(), original.len()..original.len() + 6] + ); } #[test] @@ -1054,216 +2113,4 @@ mod tests { )); assert_eq!(monitor.can_paste_at_target(Some(42)), Some(false)); } - - #[test] - fn monitoring_without_target_helper_or_for_cached_pid_stops_at_once() { - let monitor = TextEditMonitor::with_hooks(hooks(sh("echo"), Arc::new(|| None))); - let reports = watch(&monitor, "pasted text", None); - assert!(!monitor.is_monitoring()); - assert!(!reports.try_recv().unwrap().readable); - lock(&monitor.inner.unsupported_pids).insert(42); - let reports = watch(&monitor, "pasted text", Some(42)); - assert!(!monitor.is_monitoring()); - assert!(!reports.try_recv().unwrap().readable); - - let monitor = TextEditMonitor::with_hooks(hooks(no_helper(), Arc::new(|| None))); - let reports = watch(&monitor, "pasted text", Some(42)); - assert!(!monitor.is_monitoring()); - assert!(!reports.try_recv().unwrap().readable); - } - - #[test] - fn no_element_with_uniform_no_value_teaches_the_cache() { - let spawner = Arc::new(FakeSpawner::default()); - let monitor = TextEditMonitor::with_hooks(fake_hooks(&spawner)); - let _reports = watch(&monitor, "pasted text", Some(4242)); - wait_for_spawn(&spawner); - assert_eq!(spawner.args(0), ["4242", "90"]); - spawner.emit( - 0, - ChildEvent::Stderr( - "Attempt 5/5: Cannot get focused element for PID 4242 (error: -25212)\n".into(), - ), - ); - spawner.emit(0, ChildEvent::Stdout("NO_ELEMENT\n".into())); - assert!(monitor.is_monitoring(), "monitoring ends with the helper"); - spawner.emit( - 0, - ChildEvent::Exit { - code: Some(1), - signal: None, - }, - ); - assert!(!monitor.is_monitoring()); - assert!(lock(&monitor.inner.unsupported_pids).contains(&4242)); - } - - #[test] - fn persistent_failure_detection() { - assert!(is_persistent_ax_no_value_failure( - "Attempt 1/5: x (error: -25212)\nAttempt 2/5: x (error: -25212)" - )); - assert!(!is_persistent_ax_no_value_failure( - "Attempt 1/5: x (error: -25212)\nGot focused element on attempt 2" - )); - assert!(!is_persistent_ax_no_value_failure( - "Attempt 1/5: x (error: -25204)" - )); - assert!(!is_persistent_ax_no_value_failure("")); - } - - /// A session on a fake helper, with its first lines already delivered. - fn session( - lines: &str, - ) -> ( - TextEditMonitor, - Arc, - Arc, - async_channel::Receiver, - ) { - let timers = FakeTimers::new(); - let spawner = Arc::new(FakeSpawner::default()); - let monitor = TextEditMonitor::build(fake_hooks(&spawner), timers.clone()); - let reports = watch(&monitor, "hello world", Some(42)); - wait_for_spawn(&spawner); - spawner.emit(0, ChildEvent::Stdout(lines.into())); - (monitor, spawner, timers, reports) - } - - fn edit(baseline: &str, edited: &str) -> Option { - Some(TextEdit { - baseline: baseline.into(), - edited: edited.into(), - pasted: vec!["hello world".into()], - }) - } - - #[test] - fn a_session_reports_the_last_text_once_it_ends() { - let (monitor, _spawner, _timers, reports) = - session("BASELINE:hello world\nCHANGED:hello wrld\nCHANGED_B64:aGVsbG8Kd29ybGQ=\n"); - assert!(reports.try_recv().is_err(), "nothing before the end"); - monitor.stop_monitoring(); - assert_eq!( - reports.try_recv().unwrap(), - SessionReport { - app: None, - readable: true, - edit: edit("hello world", "hello\nworld"), - } - ); - } - - #[test] - fn a_session_with_no_edit_reports_none() { - let (monitor, _spawner, _timers, reports) = session("BASELINE:hello world\n"); - monitor.stop_monitoring(); - let report = reports.try_recv().unwrap(); - assert!(report.readable); - assert_eq!(report.edit, None); - } - - #[test] - fn typing_the_text_back_is_no_edit() { - let (monitor, _spawner, _timers, reports) = - session("BASELINE:hello world\nCHANGED:hello\nCHANGED:hello world\n"); - monitor.stop_monitoring(); - assert_eq!(reports.try_recv().unwrap().edit, None); - } - - #[test] - fn the_timeout_ends_a_session() { - let (monitor, _spawner, timers, reports) = - session("BASELINE:hello world\nCHANGED:hello there\n"); - timers.advance(90_000); - assert!(!monitor.is_monitoring()); - assert_eq!( - reports.try_recv().unwrap().edit, - edit("hello world", "hello there") - ); - } - - #[test] - fn emptying_the_field_keeps_the_fix_made_just_before() { - let (monitor, _spawner, _timers, reports) = - session("BASELINE:hello world\nCHANGED:hello there\nCHANGED:\n"); - assert!(!monitor.is_monitoring(), "sending ends the session"); - assert_eq!( - reports.try_recv().unwrap().edit, - edit("hello world", "hello there") - ); - } - - #[test] - fn a_paste_landing_after_the_baseline_becomes_the_baseline() { - let (monitor, _spawner, _timers, reports) = - session("BASELINE:\nCHANGED:hello world\nCHANGED:hello there\n"); - assert!(monitor.is_monitoring()); - monitor.stop_monitoring(); - assert_eq!( - reports.try_recv().unwrap().edit, - edit("hello world", "hello there") - ); - } - - #[test] - fn the_helper_ending_ends_the_session() { - let (monitor, spawner, _timers, reports) = - session("BASELINE:hello world\nCHANGED:hello there\n"); - spawner.emit( - 0, - ChildEvent::Exit { - code: Some(0), - signal: None, - }, - ); - assert!(!monitor.is_monitoring()); - assert!(reports.try_recv().unwrap().edit.is_some()); - } - - #[test] - fn a_new_session_ends_the_last_and_remembers_its_paste() { - let (monitor, spawner, _timers, first) = - session("BASELINE:hello world\nCHANGED:hello there\n"); - let second = watch(&monitor, "next part", Some(42)); - assert!(first.try_recv().unwrap().edit.is_some()); - let deadline = Instant::now() + Duration::from_secs(5); - while spawner.count() < 2 { - assert!(Instant::now() < deadline, "monitor never spawned"); - std::thread::sleep(Duration::from_millis(10)); - } - spawner.emit( - 1, - ChildEvent::Stdout( - "BASELINE:hello there next part\nCHANGED:hello you next part\n".into(), - ), - ); - monitor.stop_monitoring(); - let report = second.try_recv().unwrap(); - assert_eq!(report.edit.unwrap().pasted, ["hello world", "next part"]); - } - - #[test] - fn recent_pastes_are_kept_per_app_and_capped() { - let mut monitoring = Monitoring::default(); - for i in 0..7 { - monitoring.remember_paste(Some(1), &i.to_string()); - } - assert_eq!( - monitoring.remember_paste(Some(1), "7"), - ["3", "4", "5", "6", "7"] - ); - assert_eq!(monitoring.remember_paste(Some(2), "other"), ["other"]); - } - - #[test] - fn payloads_decode_plain_and_base64_lines() { - assert_eq!(payload("CHANGED:a: b", "CHANGED").as_deref(), Some("a: b")); - assert_eq!( - payload("BASELINE_B64:aGVsbG8Kd29ybGQ=", "BASELINE").as_deref(), - Some("hello\nworld") - ); - assert_eq!(payload("CHANGED_B64:not base64!", "CHANGED"), None); - assert_eq!(payload("OTHER:x", "CHANGED"), None); - } } diff --git a/src/pipeline.rs b/src/pipeline.rs index ba06536..b053d2a 100644 --- a/src/pipeline.rs +++ b/src/pipeline.rs @@ -111,12 +111,10 @@ pub fn build_whisper_prompt(input: &PipelineInput) -> (Option, Option bool { - settings.use_cleanup_model - && !settings.cleanup_model.trim().is_empty() - && !settings.cleanup_remote_url.trim().is_empty() + settings.cleanup_enabled() } -pub fn cleanup_system_prompt(input: &PipelineInput) -> String { +pub fn cleanup_system_prompt(input: &PipelineInput, text: &str) -> String { let s = &input.settings; let words = get_prioritized_dictionary_hint_words( &input.dictionary, @@ -135,6 +133,12 @@ pub fn cleanup_system_prompt(input: &PipelineInput) -> String { .known_corrections .iter() .filter(|c| !c.applies_automatically()) + // Reserve the bounded prompt for pairs relevant to this dictation; + // unrelated frequent corrections must not crowd out a newly learned one. + .filter(|c| { + !crate::text::term_match::find_term(text, &c.misheard).is_empty() + || !crate::text::term_match::find_term(text, &c.term).is_empty() + }) .collect(); crate::text::prompts::append_corrections_suffix(&prompt, &judged, &s.ui_language) } @@ -158,7 +162,7 @@ pub async fn clean_up( .as_deref() .filter(|k| !k.trim().is_empty()), s.cleanup_model.trim(), - &cleanup_system_prompt(input), + &cleanup_system_prompt(input, normalized), s.cleanup_disable_thinking, ); match cleanup::process_text(normalized, &config, cancel).await { @@ -480,7 +484,7 @@ mod tests { fn the_cleanup_prompt_lists_only_corrections_that_need_context() { let mut i = input(&["Claude"]); i.known_corrections = corrections(); - let prompt = cleanup_system_prompt(&i); + let prompt = cleanup_system_prompt(&i, "ask cloud"); assert!(prompt.contains("cloud → Claude")); assert!(!prompt.contains("克劳德 →")); } @@ -490,12 +494,32 @@ mod tests { let mut i = input(&[]); i.dictionary = (0..1000).map(|n| format!("filler-term-{n}")).collect(); i.dictionary_priority = i.dictionary.iter().rev().cloned().collect(); - let prompt = cleanup_system_prompt(&i); + let prompt = cleanup_system_prompt(&i, ""); assert!(prompt.contains("filler-term-999")); assert!(!prompt.contains("filler-term-500")); assert!(prompt.len() < 12_000); } + #[test] + fn relevant_new_corrections_are_not_crowded_out_by_frequent_unrelated_pairs() { + let mut i = input(&[]); + i.known_corrections = (0..30) + .map(|n| KnownCorrection { + misheard: format!("word{n}"), + term: format!("Term{n}"), + count: 100, + }) + .collect(); + i.known_corrections.push(KnownCorrection { + misheard: "cloud".into(), + term: "Claude".into(), + count: 1, + }); + let prompt = cleanup_system_prompt(&i, "Ask cloud to review this code."); + assert!(prompt.contains("cloud → Claude")); + assert!(!prompt.contains("word0 → Term0")); + } + #[test] fn cleanup_failure_still_delivers_the_raw_transcript() { let asr = asr("hello there"); diff --git a/src/settings.rs b/src/settings.rs index 2b34307..88f7222 100644 --- a/src/settings.rs +++ b/src/settings.rs @@ -204,6 +204,18 @@ impl Default for Settings { pub const DEFAULT_HOTKEY: &str = "GLOBE"; impl Settings { + /// Whether cleanup is enabled and has a model and server configured. + pub fn cleanup_enabled(&self) -> bool { + self.use_cleanup_model + && !self.cleanup_model.trim().is_empty() + && !self.cleanup_remote_url.trim().is_empty() + } + + /// Whether learning and learned influence are available for dictation. + pub fn auto_learn_enabled(&self) -> bool { + self.auto_learn_corrections && self.cleanup_enabled() + } + pub fn load() -> Self { Self::load_from(&paths::settings_path()) } @@ -266,6 +278,35 @@ impl Settings { mod tests { use super::*; + #[test] + fn auto_learn_requires_configured_cleanup_and_resumes_without_losing_preference() { + let mut settings = Settings::default(); + assert!(!settings.auto_learn_enabled()); + settings.cleanup_model = "cleanup".into(); + assert!(!settings.auto_learn_enabled()); + settings.cleanup_remote_url = "http://localhost:8080/v1".into(); + assert!(settings.auto_learn_enabled()); + + settings.use_cleanup_model = false; + assert!(!settings.auto_learn_enabled()); + assert!(settings.auto_learn_corrections); + settings.use_cleanup_model = true; + assert!(settings.auto_learn_enabled()); + + settings.auto_learn_corrections = false; + assert!(!settings.auto_learn_enabled()); + settings.use_cleanup_model = false; + settings.use_cleanup_model = true; + assert!(!settings.auto_learn_enabled()); + + settings.auto_learn_corrections = true; + settings.cleanup_model = " ".into(); + assert!(!settings.auto_learn_enabled()); + settings.cleanup_model = "cleanup".into(); + settings.cleanup_remote_url = " ".into(); + assert!(!settings.auto_learn_enabled()); + } + #[test] fn legacy_output_settings_migrate_on_load_and_save_only_one_key() { for (paste, keep, expected) in [ diff --git a/src/text/correction_learner.rs b/src/text/correction_learner.rs index e578f79..6482623 100644 --- a/src/text/correction_learner.rs +++ b/src/text/correction_learner.rs @@ -60,8 +60,8 @@ const MAX_PINYIN_DISTANCE: f64 = 0.34; pub struct Correction { pub misheard: String, pub term: String, - /// Too weak on its own: wait for a second sighting before the Dictionary - /// takes the term. + /// Legacy uncertainty signal from extraction. Learning no longer requires + /// a deliberate repeat; application still checks the current context. pub needs_repeat: bool, } @@ -77,7 +77,7 @@ pub struct KnownCorrection { impl KnownCorrection { /// Whether the delivered text is rewritten without asking the cleanup model. pub fn applies_automatically(&self) -> bool { - self.count >= 2 && is_context_free(&self.misheard, &self.term) + self.count >= 1 && is_context_free(&self.misheard, &self.term) } } @@ -89,7 +89,10 @@ fn is_context_free(misheard: &str, term: &str) -> bool { return utf16_len(misheard) >= 4 && has_inner_capital(term); } if is_unspaced_script_only(misheard) { - return misheard.chars().filter(|c| !c.is_whitespace()).count() >= 2; + // Familiar Chinese/Japanese words can express different valid meanings + // with the same reading. Leave those choices to context as well. + return misheard.chars().filter(|c| !c.is_whitespace()).count() >= 2 + && (!is_unspaced_script_only(term) || word_segments(term).len() > 1); } misheard.split_whitespace().count() >= 2 || misheard.chars().any(is_unspaced_script) } @@ -245,6 +248,7 @@ fn tokenize(text: &str) -> Vec { /// Where the pasted texts sit in the field. An app that reformats a paste /// (smart quotes, rich text) leaves no exact copy; a field about that long is /// then taken to be the paste. +#[cfg(test)] fn pasted_regions>(baseline: &str, pasted: &[S]) -> Vec> { let found: Vec> = pasted .iter() @@ -503,12 +507,33 @@ fn classify(misheard: &str, term: &str, term_units: &[Unit], sentence_start: boo return classify_case_fix(term, sentence_start); } if is_unspaced_script_only(term) { - // The segmenter's dictionary already knows a single-unit term (认为, - // 说得), so it is ordinary vocabulary or a grammar fix. Misheard names - // fall outside that dictionary and split into several units (章|珊). - if term_units.len() < 2 { + if utf16_len(term) < 2 { return None; } + let from: Vec = misheard.chars().collect(); + let to: Vec = term.chars().collect(); + let prefix = from.iter().zip(&to).take_while(|(a, b)| a == b).count(); + let suffix = from[prefix..] + .iter() + .rev() + .zip(to[prefix..].iter().rev()) + .take_while(|(a, b)| a == b) + .count(); + if from[prefix..from.len() - suffix] + .iter() + .chain(&to[prefix..to.len() - suffix]) + .all(|c| FUNCTION_CHARS.contains(*c)) + { + return None; + } + // Familiar vocabulary can be misheard too. Require matching sounds + // for these terms so ordinary word choices remain content edits. + if term_units.len() == 1 && is_unspaced_script_only(misheard) { + return fuzzy_pinyin(misheard) + .zip(fuzzy_pinyin(term)) + .filter(|(a, b)| a == b) + .map(|_| false); + } return if is_unspaced_script_only(misheard) { compare_sound(misheard, term) } else { @@ -534,10 +559,21 @@ fn classify(misheard: &str, term: &str, term_units: &[Unit], sentence_start: boo /// Corrections the user made to `pasted` text: the differences between the /// field right after the paste (`baseline`) and its last value (`edited`) /// that fall inside a pasted region. +#[cfg(test)] pub fn extract_corrections>( baseline: &str, edited: &str, pasted: &[S], +) -> Vec { + extract_corrections_in_regions(baseline, edited, &pasted_regions(baseline, pasted)) +} + +/// Extracts corrections only from the byte ranges recorded at delivery, so +/// identical dictated and pre-existing text cannot exchange ownership. +pub fn extract_corrections_in_regions( + baseline: &str, + edited: &str, + regions: &[Range], ) -> Vec { if baseline.is_empty() || edited.is_empty() || baseline == edited { return Vec::new(); @@ -548,9 +584,15 @@ pub fn extract_corrections>( return Vec::new(); } let hunks = diff_units(&base_units, &edited_units); - let regions: Vec> = pasted_regions(baseline, pasted) - .into_iter() - .filter(|region| !is_rewritten(region, &base_units, &hunks)) + let regions: Vec<_> = regions + .iter() + .filter(|range| { + range.start < range.end + && range.end <= baseline.len() + && baseline.is_char_boundary(range.start) + && baseline.is_char_boundary(range.end) + }) + .cloned() .collect(); let mut seen: HashSet<(String, String)> = HashSet::new(); @@ -584,6 +626,26 @@ pub fn extract_corrections>( ) else { continue; }; + if is_rewritten(region, &base_units, &hunks) { + // Replacing a short dictated name or product can replace the whole + // passage. Keep the rewrite guard for clauses and multiple edits. + let short_term = hunks.len() == 1 + && utf16_len(misheard) <= 24 + && utf16_len(term) <= 32 + && (fuzzy_pinyin(misheard) + .zip(fuzzy_pinyin(term)) + .is_some_and(|(a, b)| a == b) + || (!is_unspaced_script_only(misheard) + && misheard.split_whitespace().count() == 1 + && term.split_whitespace().count() == 1) + || term.split_whitespace().all(|word| { + word.chars().next().is_some_and(char::is_uppercase) + || has_inner_capital(word) + })); + if !short_term { + continue; + } + } if seen.insert((misheard.to_lowercase(), term.to_lowercase())) { results.push(Correction { misheard: misheard.to_string(), @@ -599,6 +661,39 @@ pub fn extract_corrections>( mod tests { use super::*; + #[test] + fn explicit_delivery_regions_distinguish_identical_passages() { + let baseline = "Hey Shunade. Hey Shunade."; + let first = 0.."Hey Shunade.".len(); + let learned = extract_corrections_in_regions( + baseline, + "Hey Sinead. Hey Shunade.", + std::slice::from_ref(&first), + ); + assert_eq!(learned.len(), 1); + assert_eq!( + (&*learned[0].misheard, &*learned[0].term), + ("Shunade", "Sinead") + ); + assert!( + extract_corrections_in_regions(baseline, "Hey Shunade. Hey Sinead.", &[first],) + .is_empty() + ); + } + + #[test] + fn ordinary_chinese_homophones_require_context_even_after_repetition() { + let rules = [KnownCorrection { + misheard: "权利".into(), + term: "权力".into(), + count: 9, + }]; + assert_eq!( + apply_known_corrections("我有这个权利。", &rules), + "我有这个权利。" + ); + } + fn corrections(original: &str, edited: &str) -> Vec { extract_corrections(original, edited, &[original]) } @@ -610,7 +705,7 @@ mod tests { .collect() } - /// Terms the Dictionary takes at once. + /// Terms extraction recognizes without an uncertainty signal. fn sure(original: &str, edited: &str) -> Vec { corrections(original, edited) .into_iter() @@ -701,7 +796,7 @@ mod tests { } #[test] - fn lowercase_terms_wait_for_a_second_sighting() { + fn lowercase_terms_are_marked_uncertain() { assert_eq!( corrections("this will affect it", "this will effect it"), [Correction { @@ -756,7 +851,31 @@ mod tests { } #[test] - fn one_character_name_fix_widens_and_waits() { + fn learns_a_misheard_ordinary_chinese_term() { + assert_eq!( + corrections("我们需要检查帐户设置", "我们需要检查账户设置"), + [Correction { + misheard: "帐户".into(), + term: "账户".into(), + needs_repeat: false, + }] + ); + } + + #[test] + fn learns_a_clear_term_replacing_a_short_utterance() { + assert_eq!( + corrections("克劳德扣的", "Claude Code"), + [Correction { + misheard: "克劳德扣的".into(), + term: "Claude Code".into(), + needs_repeat: false, + }] + ); + } + + #[test] + fn one_character_name_fix_widens_the_learned_term() { assert_eq!( corrections("我今天和张山一起吃饭", "我今天和张珊一起吃饭"), [Correction { @@ -797,7 +916,7 @@ mod tests { } #[test] - fn a_capital_alone_needs_a_second_sighting_and_a_mid_sentence_word() { + fn a_capital_alone_is_uncertain_and_needs_a_mid_sentence_word() { assert_eq!( corrections("we use python here", "we use Python here"), [Correction { @@ -943,11 +1062,11 @@ mod tests { } #[test] - fn phrases_and_chinese_apply_automatically_after_a_repeat() { + fn phrases_and_chinese_apply_automatically_after_one_correction() { assert!(known("cloud code", "Claude Code", 2).applies_automatically()); assert!(known("克劳德", "Claude", 2).applies_automatically()); assert!(known("github", "GitHub", 2).applies_automatically()); - assert!(!known("cloud code", "Claude Code", 1).applies_automatically()); + assert!(known("cloud code", "Claude Code", 1).applies_automatically()); } #[test] @@ -957,6 +1076,21 @@ mod tests { assert!(!known("python", "Python", 9).applies_automatically()); } + #[test] + fn one_clear_correction_is_used_in_the_next_dictation() { + assert_eq!( + apply_known_corrections( + "用 cloud code 和克劳德写,推到 github", + &[ + known("cloud code", "Claude Code", 1), + known("克劳德", "Claude", 1), + known("github", "GitHub", 1), + ], + ), + "用 Claude Code 和Claude写,推到 GitHub" + ); + } + #[test] fn applies_longest_mishearing_first_and_ignores_case() { let text = apply_known_corrections( @@ -1019,9 +1153,9 @@ mod tests { "工具 和 工具", ), ( - "甲乙", - vec![known("甲乙", "丙丁", 2), known("丙丁", "戊己", 2)], - "戊己", + "甲A", + vec![known("甲A", "乙B", 2), known("乙B", "丙C", 2)], + "丙C", ), ] { assert!(rules.iter().all(KnownCorrection::applies_automatically)); @@ -1145,11 +1279,11 @@ mod tests { } #[test] - fn unconfirmed_and_context_dependent_corrections_change_nothing() { + fn context_dependent_corrections_stay_with_the_model_after_repeats() { let corrections = [known("克劳德", "Claude", 1), known("cloud", "Claude", 5)]; assert_eq!( apply_known_corrections("克劳德 cloud", &corrections), - "克劳德 cloud" + "Claude cloud" ); } } diff --git a/src/ui/dev_scene.rs b/src/ui/dev_scene.rs index be009b3..edc51f2 100644 --- a/src/ui/dev_scene.rs +++ b/src/ui/dev_scene.rs @@ -5,6 +5,9 @@ //! `settings[:general|language|hotkeys|speech|llms|privacy]`, and the pill states //! `pill:recording`, `pill:processing`, `pill:preparing`, `pill:transition`, //! `pill:cleanup`, `pill:cleanup-transition`, `pill:flow`, `pill:error`, `pill:fallback`. +//! `pill:learning-context`, `pill:learning-unreadable`, `pill:learning-recovered`, +//! `pill:learning-learned`, and `pill:learning-cycle` show auto-learn feedback. +//! `WHISPER_DEV_LEARNING_WITH_PILL=1` keeps the existing recording pill visible. //! `WHISPER_DEV_SCROLL_FRACTION=0.5` or `1` captures the middle or bottom of Settings. use std::time::Duration; @@ -53,6 +56,11 @@ pub fn scene() -> Option { "error" => "error", "no-speech" => "no-speech", "fallback" => "fallback", + "learning-context" => "learning-context", + "learning-unreadable" => "learning-unreadable", + "learning-recovered" => "learning-recovered", + "learning-learned" => "learning-learned", + "learning-cycle" => "learning-cycle", _ => "recording", }), _ => return None, @@ -64,6 +72,12 @@ pub fn scene() -> Option { pub fn pill_head_start() -> Duration { match scene() { Some(Scene::Pill("recording")) => Duration::from_secs(14), + Some(Scene::Pill(state)) + if state.starts_with("learning-") + && std::env::var("WHISPER_DEV_LEARNING_WITH_PILL").as_deref() == Ok("1") => + { + Duration::from_secs(14) + } _ => Duration::ZERO, } } @@ -73,7 +87,37 @@ pub fn apply_pill(cx: &mut App) { let Some(Scene::Pill(state)) = scene() else { return; }; + if state.starts_with("learning-") + && std::env::var("WHISPER_DEV_LEARNING_WITH_PILL").as_deref() == Ok("1") + { + AppState::global(cx).update(cx, |state, cx| { + state.dictation.is_recording = true; + cx.notify(); + }); + } match state { + "learning-context" | "learning-unreadable" | "learning-recovered" | "learning-learned" => { + let notice = learning_notice(state); + Dictation::global(cx).update(cx, |_, cx| cx.emit(notice)); + } + "learning-cycle" => { + cx.spawn(async move |cx| { + loop { + for state in [ + "learning-context", + "learning-unreadable", + "learning-recovered", + ] { + cx.update(|cx| { + Dictation::global(cx) + .update(cx, |_, cx| cx.emit(learning_notice(state))); + }); + cx.background_executor().timer(Duration::from_secs(7)).await; + } + } + }) + .detach(); + } "flow" => { // One whole Dictation on a loop: record, transcribe, polish, // skip, deliver. For recording the pill's motion end to end. @@ -210,3 +254,14 @@ pub fn apply_pill(cx: &mut App) { }), } } + +fn learning_notice(state: &str) -> DictationNotice { + match state { + "learning-context" => { + DictationNotice::ContextDependentCorrection(vec![("cloud".into(), "Claude".into())]) + } + "learning-unreadable" => DictationNotice::LearningUnreadable, + "learning-recovered" => DictationNotice::LearningRecovered, + _ => DictationNotice::LearnedWords(vec!["Claude".into()]), + } +} diff --git a/src/ui/pill.rs b/src/ui/pill.rs index 38c92cb..ab1a1d0 100644 --- a/src/ui/pill.rs +++ b/src/ui/pill.rs @@ -515,6 +515,15 @@ impl PillView { }); } + fn correction_pairs_text(pairs: &[(String, String)]) -> String { + pairs + .iter() + .take(3) + .map(|(misheard, term)| format!("“{misheard} → {term}”")) + .collect::>() + .join(", ") + } + /// Closes the error card by the user's hand or its timer. A recording /// kept for it goes with it, so the words never outlive the offer. fn close_error(&mut self, cx: &mut App) { @@ -627,6 +636,50 @@ impl PillView { Some(words), ); } + DictationNotice::ContextDependentCorrection(pairs) => { + let corrections = Self::correction_pairs_text(&pairs); + self.push_toast( + crate::i18n::t_with( + "app.toasts.learningContext.title", + &[("corrections", &corrections)], + ), + Some(t("app.toasts.learningContext.description")), + ToastKind::LearningContext, + Duration::from_millis(6000), + None, + ); + } + DictationNotice::LearnedCorrection(pairs) => { + let corrections = Self::correction_pairs_text(&pairs); + self.push_toast( + crate::i18n::t_with( + "app.toasts.learningCorrection", + &[("corrections", &corrections)], + ), + None, + ToastKind::Success, + Duration::from_millis(6000), + None, + ); + } + DictationNotice::LearningUnreadable => self.push_toast( + t("app.toasts.learningUnreadable.title"), + Some(t("app.toasts.learningUnreadable.description")), + ToastKind::LearningUnavailable, + Duration::from_millis(6000), + None, + ), + DictationNotice::LearningRecovered => { + self.toasts + .retain(|toast| toast.kind != ToastKind::LearningUnavailable); + self.push_toast( + t("app.toasts.learningRecovered.title"), + Some(t("app.toasts.learningRecovered.description")), + ToastKind::Success, + Duration::from_millis(6000), + None, + ); + } DictationNotice::Delivered => { let resolved_at = Instant::now(); self.resolved_at = Some(resolved_at); diff --git a/src/ui/pill/cards.rs b/src/ui/pill/cards.rs index 3a47858..11c4370 100644 --- a/src/ui/pill/cards.rs +++ b/src/ui/pill/cards.rs @@ -519,6 +519,8 @@ impl PillView { ToastKind::Destructive => ("alert-circle", p.destructive), ToastKind::Success => ("check", p.success), ToastKind::Hint => ("keyboard", p.muted_foreground), + ToastKind::LearningContext => ("clock", p.muted_foreground), + ToastKind::LearningUnavailable => ("alert-circle", p.muted_foreground), }; let undo = toast.undo_words.clone(); stack = stack.child( diff --git a/src/ui/pill/state.rs b/src/ui/pill/state.rs index aac5413..a7defb2 100644 --- a/src/ui/pill/state.rs +++ b/src/ui/pill/state.rs @@ -157,6 +157,8 @@ pub(super) enum ToastKind { Destructive, Success, Hint, + LearningContext, + LearningUnavailable, } pub(super) struct PillToast { diff --git a/src/ui/settings_modal/general.rs b/src/ui/settings_modal/general.rs index 41c7ead..8de7f80 100644 --- a/src/ui/settings_modal/general.rs +++ b/src/ui/settings_modal/general.rs @@ -214,12 +214,18 @@ impl SettingsModal { )) .child(settings_section( t("settings.dictionary.title"), - vec![toggle( + vec![toggle_row( "auto-learn", - "settings.dictionary.autoLearnTitle", - "settings.dictionary.autoLearnDescription", - s.auto_learn_corrections, - |s, v| s.auto_learn_corrections = v, + t("settings.dictionary.autoLearnTitle"), + Some(t(if s.cleanup_enabled() { + "settings.dictionary.autoLearnDescription" + } else { + "settings.dictionary.autoLearnRequiresCleanup" + })), + s.auto_learn_enabled(), + !s.cleanup_enabled(), + |value, _, cx| update_settings(cx, |s| s.auto_learn_corrections = value), + &p, )], &p, )) diff --git a/tests/native/check_text_observation.py b/tests/native/check_text_observation.py new file mode 100644 index 0000000..52d1e5b --- /dev/null +++ b/tests/native/check_text_observation.py @@ -0,0 +1,126 @@ +"""Exercise the real AppKit/AX process boundary using a disposable draft. + +Run on macOS with Accessibility permission already granted: + uv run --no-project tests/native/check_text_observation.py + +The fixture creates its own window and never reads or modifies user documents. +This smoke check does not establish Chromium compatibility or input-method behavior. +""" + +import json +import queue +import subprocess +import tempfile +import threading +import time +from pathlib import Path +from typing import Any + + +def lines(process: subprocess.Popen[str]) -> queue.Queue[str]: + output: queue.Queue[str] = queue.Queue() + + def read() -> None: + assert process.stdout is not None + for line in process.stdout: + output.put(line.rstrip()) + + threading.Thread(target=read, daemon=True).start() + return output + + +def write(process: subprocess.Popen[str], value: str) -> None: + assert process.stdin is not None + process.stdin.write(value + "\n") + process.stdin.flush() + + +def event(output: queue.Queue[str], kind: str, text: str | None = None) -> dict[str, Any]: + deadline = time.monotonic() + 4 + while time.monotonic() < deadline: + value = json.loads(output.get(timeout=4)) + summary = dict(value) + if len(summary.get("value", "")) > 160: + summary["value_length"] = len(summary["value"]) + summary["value_tail"] = summary.pop("value")[-80:] + print(json.dumps(summary, ensure_ascii=False), flush=True) + if value["kind"] == kind and (text is None or value.get("value") == text): + return value + raise AssertionError(kind) + + +def main() -> None: + root = Path(__file__).resolve().parents[2] + with tempfile.TemporaryDirectory(prefix="whisper-ax-smoke-") as directory: + temporary = Path(directory) + fixture_binary = temporary / "fixture" + helper_binary = temporary / "observer" + subprocess.run(["swiftc", str(root / "resources/native/macos-text-monitor.swift"), "-o", str(helper_binary)], check=True) + subprocess.run(["swiftc", str(Path(__file__).with_name("text_observation_fixture.swift")), "-o", str(fixture_binary)], check=True) + check(fixture_binary, helper_binary) + + +def check(fixture_binary: Path, helper_binary: Path) -> None: + fixture = subprocess.Popen([str(fixture_binary)], stdin=subprocess.PIPE, stdout=subprocess.PIPE, text=True) + helper: subprocess.Popen[str] | None = None + try: + fixture_output = lines(fixture) + pid = int(fixture_output.get(timeout=4)) + time.sleep(0.2) + helper = subprocess.Popen([str(helper_binary), "--observe", str(pid)], stdin=subprocess.PIPE, stdout=subprocess.PIPE, text=True) + output = lines(helper) + write(helper, json.dumps({"command": "prepare", "ticket": 1})) + prepared = event(output, "prepared") + assert prepared.get("value") == "", prepared + assert prepared["start"] == 0 + write(fixture, "paste") + assert fixture_output.get(timeout=4) == "DONE:paste" + write(helper, json.dumps({"command": "commit", "ticket": 1})) + pasted = event(output, "value", "hello world") + write(fixture, "correct") + assert fixture_output.get(timeout=4) == "DONE:correct" + corrected = event(output, "value", "hello World") + assert corrected["field"] == prepared["field"] == pasted["field"] + write(fixture, "clear") + assert fixture_output.get(timeout=4) == "DONE:clear" + empty = event(output, "value", "") + assert empty["kind"] != "sent", "deletion without send must remain a replacement" + write(helper, json.dumps({"command":"snapshot", "ticket":2})) + event(output, "synced") + write(fixture, "paste") + assert fixture_output.get(timeout=4) == "DONE:paste" + event(output, "value", "hello world") + write(fixture, "correct") + assert fixture_output.get(timeout=4) == "DONE:correct" + event(output, "value", "hello World") + write(fixture, "send") + assert fixture_output.get(timeout=4) == "DONE:send" + sent = event(output, "sent") + assert sent["field"] == prepared["field"] + write(fixture, "long") + assert fixture_output.get(timeout=4) == "DONE:long" + write(helper, json.dumps({"command": "prepare", "ticket": 3})) + long = event(output, "prepared") + assert long["value"] == "existing " * 2000 + "克劳德" + assert long["start"] == len(long["value"]) + write(helper, json.dumps({"command": "commit", "ticket": 3})) + event(output, "value", long["value"]) + write(fixture, "long-correct") + assert fixture_output.get(timeout=4) == "DONE:long-correct" + event(output, "value", "existing " * 2000 + "Claude") + print("PASS: real AX preparation, delivery, immediate edit, temporary clear, snapshot barrier, accessible Send button, long editor tail") + finally: + if helper is not None and helper.poll() is None: + helper.terminate() + helper.wait(timeout=3) + if fixture.poll() is None: + write(fixture, "quit") + try: + fixture.wait(timeout=3) + except subprocess.TimeoutExpired: + fixture.kill() + fixture.wait(timeout=3) + + +if __name__ == "__main__": + main() diff --git a/tests/native/text_observation.html b/tests/native/text_observation.html new file mode 100644 index 0000000..8250dbf --- /dev/null +++ b/tests/native/text_observation.html @@ -0,0 +1,21 @@ + + + +Whisper text observation fixture + +

Whisper text observation fixture

+

Disposable local inputs for native observation checks. Sending only clears the local field.

+
+
+ + + diff --git a/tests/native/text_observation_fixture.swift b/tests/native/text_observation_fixture.swift new file mode 100644 index 0000000..1c25304 --- /dev/null +++ b/tests/native/text_observation_fixture.swift @@ -0,0 +1,42 @@ +// Standalone AppKit fixture for the read-only AX observation smoke check. +import Cocoa +let app = NSApplication.shared +app.setActivationPolicy(.regular) +let window = NSWindow(contentRect: NSRect(x: 300, y: 300, width: 500, height: 250), styleMask: [.titled, .closable], backing: .buffered, defer: false) +window.title = "Whisper observation verification fixture" +let editor = NSTextView(frame: NSRect(x: 10, y: 50, width: 480, height: 180)) +final class SendButton: NSButton { + override var acceptsFirstResponder: Bool { true } +} +final class Actions: NSObject { + @objc func send() { + editor.setSelectedRange(NSRange(location: 0, length: editor.string.utf16.count)) + editor.insertText("", replacementRange: editor.selectedRange()) + } +} +let actions = Actions() +let send = SendButton(title: "Send", target: actions, action: #selector(Actions.send)) +send.frame = NSRect(x: 10, y: 10, width: 100, height: 30) +window.contentView?.addSubview(editor) +window.contentView?.addSubview(send) +window.makeKeyAndOrderFront(nil) +window.makeFirstResponder(editor) +app.activate(ignoringOtherApps: true) +print(ProcessInfo.processInfo.processIdentifier) +fflush(stdout) +DispatchQueue.global().async { + while let line = readLine() { + DispatchQueue.main.async { + if line == "quit" { app.terminate(nil) } + else if line == "correct" { editor.setSelectedRange(NSRange(location: 6, length: 5)); editor.insertText("World", replacementRange: editor.selectedRange()) } + else if line == "paste" { window.makeFirstResponder(editor); editor.insertText("hello world", replacementRange: editor.selectedRange()) } + else if line == "clear" { editor.setSelectedRange(NSRange(location: 0, length: editor.string.utf16.count)); editor.insertText("", replacementRange: editor.selectedRange()) } + else if line == "send" { window.makeFirstResponder(send); send.performClick(nil) } + else if line == "long" { window.makeFirstResponder(editor); editor.string = String(repeating: "existing ", count: 2000) + "克劳德"; editor.setSelectedRange(NSRange(location: editor.string.utf16.count, length: 0)) } + else if line == "long-correct" { editor.setSelectedRange(NSRange(location: editor.string.utf16.count - 3, length: 3)); editor.insertText("Claude", replacementRange: editor.selectedRange()) } + print("DONE:" + line) + fflush(stdout) + } + } +} +app.run()