Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
14 changes: 14 additions & 0 deletions locales/en.json
Original file line number Diff line number Diff line change
Expand Up @@ -43,6 +43,7 @@
"dictionary": {
"autoLearnTitle": "Auto-learn from corrections",
"autoLearnDescription": "When you fix a transcript in another app, Whisper learns how the word was misheard and corrects it next time",
"autoLearnRequiresCleanup": "Enable and configure text cleanup to use auto-learn. Learned data is kept, and manually added dictionary words stay active.",
"title": "Dictionary"
},
"general": {
Expand Down Expand Up @@ -727,6 +728,19 @@
}
},
"addedToDict": "Added {{words}} to your dictionary",
"learningCorrection": "Learned correction {{corrections}}",
"learningContext": {
"title": "Remembered {{corrections}}",
"description": "This correction depends on context and will be judged in future dictations."
},
"learningUnreadable": {
"title": "Cannot read this input field right now",
"description": "Auto-learn is temporarily unavailable. You can still dictate."
},
"learningRecovered": {
"title": "Auto-learn is available again",
"description": "Corrections in this input field can now be learned."
},
"undo": "Undo",
"cleanupFailed": {
"title": "Cleanup failed",
Expand Down
14 changes: 14 additions & 0 deletions locales/zh-CN.json
Original file line number Diff line number Diff line change
Expand Up @@ -43,6 +43,7 @@
"dictionary": {
"autoLearnTitle": "从纠正中自动学习",
"autoLearnDescription": "你在其他应用里改正转录内容后,Whisper 会记住这个词被误听成什么,下次自动修正",
"autoLearnRequiresCleanup": "开启并配置文本润色后可用。已学内容会保留,手动添加的词典仍可使用。",
"title": "词典"
},
"general": {
Expand Down Expand Up @@ -706,6 +707,19 @@
}
},
"addedToDict": "已将 {{words}} 添加到你的词典",
"learningCorrection": "已记住纠正 {{corrections}}",
"learningContext": {
"title": "已记下 {{corrections}}",
"description": "这处纠正有歧义,后续结合语境使用。"
},
"learningUnreadable": {
"title": "暂时无法读取这个输入框",
"description": "自动学习暂时不可用,口述仍可使用。"
},
"learningRecovered": {
"title": "已恢复自动学习",
"description": "接下来对这个输入框的纠正会继续学习。"
},
"undo": "撤消",
"cleanupFailed": {
"title": "清理失败",
Expand Down
449 changes: 306 additions & 143 deletions resources/native/macos-text-monitor.swift

Large diffs are not rendered by default.

4 changes: 4 additions & 0 deletions src/app.rs
Original file line number Diff line number Diff line change
Expand Up @@ -307,6 +307,9 @@ pub fn run() {
AppState::init(settings, db, cx);
ui::overlays::Overlays::init(cx);
Dictation::init(cx);
crate::native::text_monitor::TextEditMonitor::shared()
.set_enabled(AppState::settings(cx).auto_learn_enabled());
backend::start_learning_listener(cx);
cx.bind_keys([
KeyBinding::new("cmd-k", OpenSearch, Some("ControlPanel")),
KeyBinding::new("cmd-,", ShowSettings, None),
Expand All @@ -318,6 +321,7 @@ pub fn run() {
]);
cx.on_app_quit(|cx| {
AppState::global(cx).update(cx, |state, _| state.invalidate_learning());
crate::native::text_monitor::TextEditMonitor::shared().set_enabled(false);
if let Some(manager) = backend::hotkeys() {
manager.shutdown();
}
Expand Down
48 changes: 31 additions & 17 deletions src/app_state.rs
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,10 @@ use std::sync::atomic::{AtomicU64, Ordering};
use gpui_kit::*;

use crate::audio_storage::{AudioStorage, RetentionCleanupReport};
use crate::db::{Database, DictionaryEntry, DictionarySource, Snippet, TranscriptionItem};
use crate::db::{
CorrectionLearningOutcome, Database, DictionaryEntry, DictionarySource, Snippet,
TranscriptionItem,
};
use crate::history::{HistoryStore, HistoryWrite};
use crate::i18n::{self, Language};
use crate::native::text_monitor::SessionReport;
Expand Down Expand Up @@ -167,8 +170,10 @@ impl AppState {
}
})
.detach();
if before.auto_learn_corrections != self.settings.auto_learn_corrections {
if before.auto_learn_enabled() != self.settings.auto_learn_enabled() {
self.dictionary_revision.fetch_add(1, Ordering::SeqCst);
crate::native::text_monitor::TextEditMonitor::shared()
.set_enabled(self.settings.auto_learn_enabled());
}
if before.theme != self.settings.theme {
theme::apply(self.settings.theme, cx);
Expand Down Expand Up @@ -461,23 +466,27 @@ impl AppState {
.detach();
}

/// Learns from a finished post-paste session. Returns the words added to
/// the Dictionary.
/// Replaces a session's learning contribution with its latest completed edit.
pub fn learn_from_session(
&mut self,
report: SessionReport,
cx: &mut Context<Self>,
) -> Task<Vec<String>> {
) -> Task<CorrectionLearningOutcome> {
let revision = self.dictionary_revision.load(Ordering::SeqCst);
let token = self.dictionary_revision.clone();
cx.spawn(async move |this, cx| {
let SessionReport { edit, .. } = report;
let SessionReport {
edit,
session_id,
revision: session_revision,
..
} = report;
let corrections = if let Some(edit) = edit {
crate::services::runtime::blocking(move || {
crate::text::correction_learner::extract_corrections(
crate::text::correction_learner::extract_corrections_in_regions(
&edit.baseline,
&edit.edited,
&edit.pasted,
&edit.pasted_ranges,
)
})
.await
Expand All @@ -487,27 +496,32 @@ impl AppState {
};
let Ok(Some(db)) = this.update(cx, |this, _| {
(this.dictionary_revision.load(Ordering::SeqCst) == revision
&& this.settings.auto_learn_corrections)
.then(|| this.db.clone())
&& this.settings.auto_learn_enabled())
.then(|| this.db.clone())
}) else {
return Vec::new();
return CorrectionLearningOutcome::default();
};
let (learned, dictionary) = crate::services::runtime::storage(move || {
let learned = db
.record_corrections_if_current(&corrections, &token, revision)
.record_session_corrections_if_current(
&session_id,
session_revision,
&corrections,
&token,
revision,
)
.inspect_err(|err| log::warn!("failed to save learned words: {err}"))
.unwrap_or_default();
let dictionary = (!corrections.is_empty()).then(|| DictionarySnapshot::read(&db));
// An empty replacement can revoke an earlier learned correction.
let dictionary = DictionarySnapshot::read(&db);
(learned, dictionary)
})
.await;
this.update(cx, |this, cx| {
if this.dictionary_revision.load(Ordering::SeqCst) != revision {
return Vec::new();
}
if let Some(dictionary) = dictionary {
this.apply_dictionary_snapshot(dictionary, cx);
return CorrectionLearningOutcome::default();
}
this.apply_dictionary_snapshot(dictionary, cx);
learned
})
.unwrap_or_default()
Expand Down
157 changes: 134 additions & 23 deletions src/backend.rs
Original file line number Diff line number Diff line change
Expand Up @@ -296,14 +296,28 @@ pub fn paste_text(
text: String,
target_pid: Option<i32>,
output_mode: crate::settings::OutputMode,
learn: bool,
) -> impl std::future::Future<Output = bool> + 'static {
let (tx, rx) = futures::channel::oneshot::channel();
std::thread::spawn(move || {
crate::native::text_monitor::TextEditMonitor::shared().stop_monitoring();
let outcome = crate::native::paste::paste_text(
&text,
crate::native::paste::PasteOptions::new(target_pid, output_mode),
);
let options = crate::native::paste::PasteOptions::new(target_pid, output_mode);
let outcome = if learn {
crate::native::paste::Paster::shared().paste_text_observed(
&text,
options,
|text, pid| {
let prepared = crate::native::text_monitor::TextEditMonitor::shared()
.prepare_paste(text, pid);
move |pasted| {
if let Some(prepared) = prepared {
prepared.commit(pasted);
}
}
},
)
} else {
crate::native::paste::paste_text(&text, options)
};
if let Some(failure) = &outcome.failure {
log::warn!("automatic paste failed: {failure:?}");
}
Expand All @@ -312,26 +326,123 @@ pub fn paste_text(
async move { rx.await.unwrap_or(false) }
}

/// Watches the target field after an automatic paste and adds learned
/// corrections to the Dictionary (auto-learn). Resolves to the words added
/// once the session ends.
pub fn monitor_corrections(
text: String,
target_pid: Option<i32>,
cx: &mut gpui_kit::App,
) -> gpui_kit::Task<Vec<String>> {
let reports = crate::native::text_monitor::TextEditMonitor::shared()
.start_monitoring_after_paste(&text, target_pid);
cx.spawn(async move |cx| {
let Ok(report) = reports.recv().await else {
return Vec::new();
};
cx.update(|cx| {
crate::app_state::AppState::global(cx)
.update(cx, |s, cx| s.learn_from_session(report, cx))
/// Consumes completed edits and observation health independently of dictation
/// generations, so later edits to earlier passages are not discarded.
struct LearningBarriers(async_channel::Sender<futures::channel::oneshot::Sender<()>>);
impl gpui_kit::Global for LearningBarriers {}

/// Waits for completed native edits and their queued dictionary writes before
/// taking the next pipeline snapshot. Unfinished composition remains pending.
pub fn flush_learning(cx: &mut gpui_kit::App) -> gpui_kit::Task<()> {
let barriers = cx.global::<LearningBarriers>().0.clone();
cx.spawn(async move |_| {
let _ = crate::services::runtime::blocking(|| {
crate::native::text_monitor::TextEditMonitor::shared().flush_completed_edits();
})
.await
.await;
let (done, completed) = futures::channel::oneshot::channel();
if barriers.send(done).await.is_ok() {
let _ = completed.await;
}
})
}

pub fn start_learning_listener(cx: &mut gpui_kit::App) {
use crate::app_state::AppState;
use crate::dictation::{Dictation, DictationNotice};
use crate::native::text_monitor::{ObservationOutcome, TextEditMonitor};
use futures::FutureExt;

let reports = TextEditMonitor::shared().subscribe();
let (barrier_sender, barriers) = async_channel::unbounded();
cx.set_global(LearningBarriers(barrier_sender));
cx.spawn(async move |cx| {
loop {
let report = reports.recv().fuse();
let barrier = barriers.recv().fuse();
futures::pin_mut!(report, barrier);
let report = futures::select_biased! {
report = report => match report { Ok(report) => report, Err(_) => break },
barrier = barrier => {
if let Ok(barrier) = barrier {
let _ = barrier.send(());
}
continue;
}
};
if let Some(diagnostic) = &report.diagnostic {
log::debug!(
"[auto-learn] outcome={:?} app={:?} field={:?} reason={diagnostic}",
report.outcome,
report.app,
report.field
);
}
if !cx.update(|cx| AppState::settings(cx).auto_learn_enabled()) {
continue;
}
match report.outcome {
ObservationOutcome::Unreadable | ObservationOutcome::Recovered => {
let notice = if report.outcome == ObservationOutcome::Unreadable {
DictationNotice::LearningUnreadable
} else {
DictationNotice::LearningRecovered
};
cx.update(|cx| {
Dictation::global(cx).update(cx, |_, cx| cx.emit(notice));
});
}
ObservationOutcome::CompletedEdit => {
let learning = cx.update(|cx| {
AppState::global(cx)
.update(cx, |state, cx| state.learn_from_session(report, cx))
});
let outcome = learning.await;
cx.update(|cx| {
if !AppState::settings(cx).auto_learn_enabled() {
return;
}
let contextual: std::collections::HashSet<_> = outcome
.context_dependent_terms
.iter()
.map(String::as_str)
.collect();
let clear: Vec<_> = outcome
.added_terms
.iter()
.filter(|term| !contextual.contains(term.as_str()))
.cloned()
.collect();
let existing_word_pairs: Vec<_> = outcome
.clear_correction_pairs
.into_iter()
.filter(|pair| !outcome.added_terms.contains(&pair.term))
.map(|pair| (pair.misheard, pair.term))
.collect();
Dictation::global(cx).update(cx, |_, cx| {
if !clear.is_empty() {
cx.emit(DictationNotice::LearnedWords(clear));
}
if !existing_word_pairs.is_empty() {
cx.emit(DictationNotice::LearnedCorrection(existing_word_pairs));
}
if !outcome.context_dependent_pairs.is_empty() {
cx.emit(DictationNotice::ContextDependentCorrection(
outcome
.context_dependent_pairs
.into_iter()
.map(|pair| (pair.misheard, pair.term))
.collect(),
));
}
});
});
}
ObservationOutcome::Readable => {}
}
}
})
.detach();
}

/// Removes words auto-learn just added (the toast's Undo).
Expand Down
2 changes: 1 addition & 1 deletion src/db.rs
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@ use chrono::{DateTime, Duration, Utc};
use parking_lot::{Mutex, MutexGuard};
use rusqlite::{Connection, OptionalExtension, Row, TransactionBehavior, params};

pub use dictionary::{DictionaryEntry, DictionarySource};
pub use dictionary::{CorrectionLearningOutcome, DictionaryEntry, DictionarySource};
pub use snippets::{MAX_SNIPPET_TRIGGER_LENGTH, Snippet};

/// Shared handle to the database; cheap to share behind an `Arc`.
Expand Down
Loading
Loading