diff --git a/CHANGELOG.md b/CHANGELOG.md index f5562aa..43b95cc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,20 @@ the `mimir-mem` crate, and the on-disk schema move together. ## [Unreleased] ### Added +- **`mimir link --scan --all-projects`, and a `doctor` nudge to run it.** + Grounding only exists if something creates the links, and on a real + 728-memory store 1.4% of memories had any. The cause was a scope + mismatch, not discoverability: 93% of memories are *global*, but a scan + compares them against one project's graph at a time, so scanning where + you happen to be standing leaves the rest unlinked. Measured on that + store — one project took grounding to 2.6%, all 24 took it to **20.7%** + (563 links). `doctor` now says when you have code graphs and have never + scanned, records `last_link_scan`, and goes quiet until the scan is 30 + days stale — a health check that repeats advice you already took is one + people learn to skim. README documents grounding and the scan together, + including the honest ceiling: most memories name no code and are + legitimately ungrounded. + - **`--confidence certain|likely|unsure` separates how sure the author was from how often the memory gets used.** `strength` conflated the two: a guess recalled enough times outranked things that had been checked, with diff --git a/README.md b/README.md index 653387a..c2aa5bf 100644 --- a/README.md +++ b/README.md @@ -205,6 +205,7 @@ mimir graph callers resolve_ref # who calls this? mimir graph impact $(git diff --name-only) # blast radius of a change mimir graph viz --open # interactive graph map (self-contained HTML) mimir link m:ABC123 my_function --rel about # decisions ↔ code +mimir link --scan --all-projects # auto-link memories to symbols they name # code content (function/method bodies, not just signatures, in recall) mimir code add ~/src/myproject --name myproject @@ -224,6 +225,43 @@ mimir export > backup.jsonl # everything, always yours claude mcp add --scope user mimir -- mimir mcp ``` +### Grounding: which memories point at real code + +A memory linked to an indexed symbol makes a claim Mimir can re-check. When +the symbol goes away, the link is *falsified* — not the memory, which is +often still right, but the connection nobody has revisited since the code +moved. That shows up wherever you meet the memory: + +```sh +mimir grounding # 151 grounded, 0 stale, 577 ungrounded +mimir grounding --stale # the ones whose target is gone +``` + +``` +m:QJ18BM [gotcha pr:mimir 08-11 stale-link] retry_with_backoff doubles the delay … +``` + +It is reported, never scored: grounding changes what you can see about a +memory, not which memories come back. Most memories are legitimately +ungrounded — a note about a DNS quirk or a person names no code — so treat +it as a signal on the code-referencing subset, not a number to maximise. + +**Grounding only exists if something creates the links.** `--all-projects` +matters more than it looks: most memories are *global*, and a scan compares +them against the graph of one project at a time, so scanning only where you +happen to be standing leaves the rest unlinked. On a real 728-memory store, +scanning one project moved grounding from 1.4% to 2.6%; scanning all 24 took +it to 20.7%. + +```sh +mimir graph build # in each project you care about +mimir link --scan --all-projects --dry-run # see what it would link +mimir link --scan --all-projects # ~600 links on a mature store +``` + +`mimir doctor` reminds you if you have code graphs and have never scanned, +and goes quiet once you have. + ### The fast path (recommended setup) Three commands get you the lowest-latency configuration Mimir has — measured diff --git a/crates/mimir-cli/src/commands.rs b/crates/mimir-cli/src/commands.rs index f36b7b6..9158f89 100644 --- a/crates/mimir-cli/src/commands.rs +++ b/crates/mimir-cli/src/commands.rs @@ -911,7 +911,7 @@ pub fn doctor(check_only: bool) -> Result<()> { // that Mimir derived by checking rather than by policy: the // memory said it was about a symbol, and the symbol is gone. if let Ok((grounded, stale, ungrounded)) = mimir_core::grounding::tally(&conn) { - if grounded + stale > 0 { + if grounded + stale + ungrounded > 0 { check( "grounding", true, @@ -927,6 +927,14 @@ pub fn doctor(check_only: bool) -> Result<()> { ); } } + // The nudge that grounding actually depends on. Anchors sat at + // zero adoption for months because the only way to set one was + // a command nobody knew to run; this is the same failure, and + // the fix is to say so at the moment someone is already looking + // at the store's health rather than in a README they read once. + if let Some(detail) = link_scan_hint(&conn) { + check("link scan", true, detail, &mut failures); + } } Err(e) => check("db", false, e.to_string(), &mut failures), } @@ -1684,19 +1692,138 @@ pub fn supersede(old: &str, by: &str) -> Result<()> { Ok(()) } -/// Auto-link memories to the code symbols their text literally mentions -/// (current project + global memories vs the current project's graph). +/// `doctor`'s line about auto-linking, or `None` when there is nothing +/// useful to say (no code graphs, or a recent scan). +/// +/// Deliberately quiet once you have scanned: a health check that repeats +/// advice you already took is one people learn to skim, and then it cannot +/// tell them anything. +fn link_scan_hint(conn: &rusqlite::Connection) -> Option { + let graphed: i64 = conn + .query_row( + "SELECT COUNT(DISTINCT project_id) FROM node + WHERE kind='symbol' AND deleted_at IS NULL AND project_id IS NOT NULL", + [], + |r| r.get(0), + ) + .ok()?; + if graphed == 0 { + return None; + } + let last: Option = conn + .query_row( + "SELECT value FROM meta WHERE key = 'last_link_scan'", + [], + |r| r.get::<_, String>(0), + ) + .ok() + .and_then(|v| v.parse().ok()); + let days = last.map(|t| (mimir_core::model::now_unix() - t) / 86_400); + match days { + // Symbols exist and nothing has ever been linked from them. + None => Some(format!( + "{graphed} project(s) have a code graph, never scanned — \ + `mimir link --scan --all-projects` links memories to symbols they name" + )), + Some(d) if d >= 30 => Some(format!( + "last scanned {d}d ago; graphs move — \ + `mimir link --scan --all-projects` to catch up" + )), + Some(_) => None, + } +} + +/// Auto-link memories to the code symbols their text literally mentions. +/// /// Precision-first: only code-shaped names (snake_case, ::path, CamelCase) -/// or backticked mentions link, on word boundaries; ambiguous names that -/// resolve to many symbols are skipped. Idempotent: existing edges are kept. -pub fn link_scan(dry_run: bool) -> Result<()> { +/// on word boundaries; names resolving to more than three symbols are +/// skipped as ambiguous. Idempotent — existing edges are kept. +/// +/// `all_projects` exists because of a mismatch measured on a real store: +/// 93% of memories are *global*, but a scan only ever compares them +/// against the graph of the project you happen to be standing in. The same +/// global Rust gotcha can match a symbol in any project; running only in +/// the current one leaves the rest unlinked. Measured: scanning one project +/// took grounding from 1.4% to 2.6%, scanning four took it to 15.4%. +pub fn link_scan(dry_run: bool, all_projects: bool) -> Result<()> { let mimir = Mimir::open()?; - let proj = mimir - .project_for_cwd(&std::env::current_dir()?)? - .context("not inside a project (the scan links memories to this project's symbols)")?; + let targets: Vec = if all_projects { + // Only projects that actually have a graph; the rest cost a query + // and can match nothing. + let mut stmt = mimir.conn.prepare(&format!( + "SELECT {} FROM node p WHERE p.kind='project' AND p.deleted_at IS NULL + AND EXISTS (SELECT 1 FROM node s WHERE s.kind='symbol' + AND s.project_id=p.id AND s.deleted_at IS NULL) + ORDER BY p.title", + store::NODE_COLS + ))?; + let rows = stmt.query_map([], mimir_core::store::row_to_node)?; + rows.collect::>()? + } else { + vec![mimir.project_for_cwd(&std::env::current_dir()?)?.context( + "not inside a project (the scan links memories to this project's symbols) \ + — or pass --all-projects to scan every project that has a code graph", + )?] + }; + if targets.is_empty() { + println!("no project has a code graph yet — run `mimir graph build` in one first"); + return Ok(()); + } + + // Every edge, once. The inner loop asks "is this pair already linked?" + // per candidate, and re-reading the table per project turned that into + // the dominant cost on a store with a hundred thousand symbols. + let mut existing: std::collections::HashSet<(i64, i64)> = Default::default(); + { + let mut stmt = mimir.conn.prepare("SELECT src, dst FROM edge")?; + let mut rows = stmt.query([])?; + while let Some(r) = rows.next()? { + let (s, d): (i64, i64) = (r.get(0)?, r.get(1)?); + existing.insert((s, d)); + existing.insert((d, s)); + } + } + + let mut total = 0usize; + for proj in &targets { + let created = scan_one_project(&mimir, proj, dry_run, &mut existing)?; + total += created; + if all_projects { + println!( + "{} {created} link(s) in {}", + if dry_run { "would create" } else { "created" }, + proj.title.as_deref().unwrap_or("(unnamed)") + ); + } + } + println!( + "{} {total} link(s) across {} project(s)", + if dry_run { "would create" } else { "created" }, + targets.len() + ); + if !dry_run { + // Mirrors `last_consolidate`, so `doctor` can say how stale this is + // instead of nagging about a scan you ran an hour ago. + mimir.conn.execute( + "INSERT INTO meta (key, value) VALUES ('last_link_scan', ?1) + ON CONFLICT(key) DO UPDATE SET value = excluded.value", + [mimir_core::model::now_unix().to_string()], + )?; + } + Ok(()) +} + +/// One project's pass. Returns links created (or, in a dry run, the number +/// that would be). +fn scan_one_project( + mimir: &Mimir, + proj: &Node, + dry_run: bool, + existing: &mut std::collections::HashSet<(i64, i64)>, +) -> Result { // name → symbol nodes carrying it (bare name from meta, fallback title) - let mut by_name: HashMap> = HashMap::new(); + let mut by_name: HashMap> = HashMap::new(); { let mut stmt = mimir.conn.prepare( "SELECT id, uid, COALESCE(json_extract(meta,'$.name'), title) FROM node @@ -1708,10 +1835,7 @@ pub fn link_scan(dry_run: bool) -> Result<()> { let name: Option = r.get(2)?; if let Some(name) = name { if name.len() >= 4 { - by_name - .entry(name) - .or_default() - .push((id, uid, String::new())); + by_name.entry(name).or_default().push((id, uid)); } } } @@ -1727,17 +1851,6 @@ pub fn link_scan(dry_run: bool) -> Result<()> { rows.collect::>()? }; - let mut existing: std::collections::HashSet<(i64, i64)> = Default::default(); - { - let mut stmt = mimir.conn.prepare("SELECT src, dst FROM edge")?; - let mut rows = stmt.query([])?; - while let Some(r) = rows.next()? { - let (s, d): (i64, i64) = (r.get(0)?, r.get(1)?); - existing.insert((s, d)); - existing.insert((d, s)); - } - } - let mut created = 0usize; for (mid, muid, text) in &memories { for (name, syms) in &by_name { @@ -1747,32 +1860,25 @@ pub fn link_scan(dry_run: bool) -> Result<()> { if !mentions_symbol(text, name) { continue; } - for (sid, suid, _) in syms { + for (sid, suid) in syms { if existing.contains(&(*mid, *sid)) { continue; } - if dry_run { - println!( - "would link m:{} —mentions→ {name} (c:{})", - tail(muid), - tail(suid) - ); - } else { + if !dry_run { store::link(&mimir.conn, *mid, *sid, Rel::Mentions, 1.0)?; - println!("m:{} —mentions→ {name} (c:{})", tail(muid), tail(suid)); } + println!( + "{}m:{} —mentions→ {name} (c:{})", + if dry_run { "would link " } else { "" }, + tail(muid), + tail(suid) + ); existing.insert((*mid, *sid)); created += 1; } } } - println!( - "{} {created} link(s) ({} memories × {} distinct symbol names)", - if dry_run { "would create" } else { "created" }, - memories.len(), - by_name.len(), - ); - Ok(()) + Ok(created) } fn tail(uid: &str) -> &str { @@ -2474,3 +2580,68 @@ mod link_target_tests { assert!(resolve_link_target(&conn, "nope_not_here", Some(1)).is_err()); } } + +#[cfg(test)] +mod link_scan_hint_tests { + use super::link_scan_hint; + use mimir_core::model::{Kind, NewNode}; + use mimir_core::store; + + fn store_with_graph(symbols: bool) -> rusqlite::Connection { + let conn = mimir_core::db::open_in_memory().unwrap(); + if symbols { + let mut p = NewNode::new(Kind::Project); + p.title = Some("probe".into()); + let proj = store::insert_node(&conn, p).unwrap(); + let mut s = NewNode::new(Kind::Symbol); + s.title = Some("retry_with_backoff".into()); + s.project_id = Some(proj.id); + store::insert_node(&conn, s).unwrap(); + } + conn + } + + fn set_last_scan(conn: &rusqlite::Connection, days_ago: i64) { + let at = mimir_core::model::now_unix() - days_ago * 86_400; + conn.execute( + "INSERT INTO meta (key, value) VALUES ('last_link_scan', ?1) + ON CONFLICT(key) DO UPDATE SET value = excluded.value", + [at.to_string()], + ) + .unwrap(); + } + + /// Nothing to link against means nothing to say. Advising a scan on a + /// store with no code graph is advice that cannot be acted on. + #[test] + fn silent_without_any_code_graph() { + assert!(link_scan_hint(&store_with_graph(false)).is_none()); + } + + #[test] + fn speaks_up_when_graphs_exist_and_nothing_was_ever_scanned() { + let hint = link_scan_hint(&store_with_graph(true)).expect("should nudge"); + assert!(hint.contains("never scanned"), "{hint}"); + assert!(hint.contains("--all-projects"), "{hint}"); + } + + /// The property that keeps the nudge worth reading: it stops once you + /// have acted on it. A health check that repeats advice you already + /// took is one people learn to skim past. + #[test] + fn goes_quiet_after_a_recent_scan() { + let conn = store_with_graph(true); + set_last_scan(&conn, 1); + assert!(link_scan_hint(&conn).is_none()); + } + + /// ...but graphs move, so it comes back rather than staying silent for + /// good after one scan a year ago. + #[test] + fn returns_when_the_last_scan_has_gone_stale() { + let conn = store_with_graph(true); + set_last_scan(&conn, 45); + let hint = link_scan_hint(&conn).expect("should nudge again"); + assert!(hint.contains("45d ago"), "{hint}"); + } +} diff --git a/crates/mimir-cli/src/main.rs b/crates/mimir-cli/src/main.rs index 56bd0f2..465f5fc 100644 --- a/crates/mimir-cli/src/main.rs +++ b/crates/mimir-cli/src/main.rs @@ -283,6 +283,12 @@ enum Command { /// With --scan: show what would be linked without writing. #[arg(long)] dry_run: bool, + /// With --scan: scan every project that has a code graph, not just + /// the current one. Global memories (most of them) can match + /// symbols in any project, so scanning one at a time leaves the + /// rest unlinked. + #[arg(long)] + all_projects: bool, }, /// Manage docs collections. Docs { @@ -824,11 +830,13 @@ fn run(cli: Cli) -> anyhow::Result<()> { rel, scan, dry_run, + all_projects, } => match (scan, a, b) { - (true, _, _) => commands::link_scan(dry_run), + (true, _, _) => commands::link_scan(dry_run, all_projects), (false, Some(a), Some(b)) => commands::link(&a, &b, &rel), _ => anyhow::bail!( - "usage: mimir link [--rel REL] | mimir link --scan [--dry-run]" + "usage: mimir link [--rel REL] | \ + mimir link --scan [--all-projects] [--dry-run]" ), }, Command::Docs { cmd } => match cmd {