diff --git a/.claude-plugin/plugin.json b/.claude-plugin/plugin.json index 30c3f70..79f3abd 100644 --- a/.claude-plugin/plugin.json +++ b/.claude-plugin/plugin.json @@ -1,7 +1,7 @@ { "name": "vta-agent-memory", "description": "Durable agent memory stored in your own Verifiable Trust Agent, not in the tool. Save and recall facts across sessions, scoped to a VTA trust context you control and can revoke.", - "version": "0.6.0", + "version": "0.6.1", "keywords": [ "memory", "vta", diff --git a/CLAUDE.md b/CLAUDE.md index f728d15..ac04ad0 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -198,7 +198,8 @@ longer sends, returned the whole context with no prefix at all.) Everything in `confidence`: the rooms schema it comes from says a reader MUST NOT compare it across authors. Keep `basis`/`evidence` as strings/raw values in the record — a typed field would make a newer writer's record undecodable. -- `rank()` is weighted token containment. Deliberately crude; see its doc +- `rank()` is weighted whole-word matching, plurals folded, query stopwords + dropped. Not substring containment: that let "run" match "runner". Deliberately crude; see its doc comment before replacing it. `Store::list` is the single place to change if the upstream task ever grows a diff --git a/Cargo.lock b/Cargo.lock index 69efd24..a5353fc 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5437,7 +5437,7 @@ checksum = "5c3082ca00d5a5ef149bb8b555a72ae84c9c59f7250f013ac822ac2e49b19c64" [[package]] name = "vta-agent-memory" -version = "0.6.0" +version = "0.6.1" dependencies = [ "anyhow", "chrono", diff --git a/Cargo.toml b/Cargo.toml index be8dc16..9a64098 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "vta-agent-memory" -version = "0.6.0" +version = "0.6.1" edition = "2024" rust-version = "1.95.0" description = "Agentic memory for Claude Code, stored in a Verifiable Trust Agent" diff --git a/README.md b/README.md index ba39639..4341edd 100644 --- a/README.md +++ b/README.md @@ -208,7 +208,10 @@ context in key order and narrows by key prefix, but does not search. So: **descriptions, not bodies**. Bodies come back only from `memory_get`. A memory service that pastes every body into the context window has made the session worse. -- Ranking is weighted token containment (name 4, description 2, body 1). Crude +- Ranking is weighted whole-word matching (name 4, description 2, body 1), with + plurals folded to their singular and a short list of English stopwords + dropped from the query — so "run" does not match "runner", and "what is the" + matches nothing. Crude and predictable, chosen over embeddings because the VTA offers no vector index and a scoring function nobody can predict is worse than one everybody can. - Relevance is then weighted by **basis** — what kind of claim the memory is: diff --git a/src/record.rs b/src/record.rs index 933c34a..3f0a073 100644 --- a/src/record.rs +++ b/src/record.rs @@ -635,9 +635,77 @@ pub struct Hit { pub score: u32, } +/// Words that carry no topic, dropped from a query before it is matched. +/// +/// Without this, "what is the deploy for" scored every memory containing +/// "the" or "for" — which is every memory — and a query of nothing but these +/// words returned the context instead of nothing. Kept short and English on +/// purpose: a word missing from here costs some precision, while a content word +/// wrongly in here makes memories unfindable. Negations ("not", "never", "no") +/// are deliberately absent — they change what a memory means. +const STOPWORDS: &[&str] = &[ + "a", "about", "am", "an", "and", "any", "are", "as", "at", "be", "been", "by", "can", "could", + "did", "do", "does", "for", "from", "had", "has", "have", "how", "i", "if", "in", "into", "is", + "it", "its", "me", "my", "of", "on", "or", "our", "should", "so", "than", "that", "the", + "their", "them", "then", "there", "these", "they", "this", "those", "to", "us", "was", "we", + "were", "what", "when", "where", "which", "who", "whom", "why", "will", "with", "would", "you", + "your", +]; + +/// Split `s` into its words: runs of letters and digits, in any script, +/// lowercased and reduced to a singular form (see [`singular`]). +fn tokens(s: &str) -> impl Iterator + '_ { + s.split(|c: char| !c.is_alphanumeric()) + .filter(|t| !t.is_empty()) + .map(|t| singular(&t.to_lowercase())) +} + +/// The set of words in a field, for whole-word matching. +fn words(s: &str) -> std::collections::HashSet { + tokens(s).collect() +} + +/// The terms a query is matched on: its words, less [`STOPWORDS`] and +/// single characters, each once. +fn query_terms(query: &str) -> Vec { + let mut terms: Vec = Vec::new(); + for t in tokens(query) { + if t.chars().count() > 1 && !STOPWORDS.contains(&t.as_str()) && !terms.contains(&t) { + terms.push(t); + } + } + terms +} + +/// A plural reduced to its singular, by rule — so "tests" finds "test" and +/// "policies" finds "policy". +/// +/// Plurals only, and deliberately so. A stemmer that also strips "-ing" and +/// "-ed" has to guess ("named" → "nam"?), and a guess that is wrong makes a +/// memory unfindable in a way nobody can predict; a plural rule is wrong rarely +/// and visibly. Applied to both sides, so a mistake is at least symmetric. +fn singular(w: &str) -> String { + let n = w.chars().count(); + if n > 4 && w.ends_with("ies") { + return format!("{}y", &w[..w.len() - 3]); + } + if n > 4 + && w.ends_with("es") + && ["sses", "xes", "ches", "shes", "zes"] + .iter() + .any(|e| w.ends_with(e)) + { + return w[..w.len() - 2].to_string(); + } + if n > 3 && w.ends_with('s') && !w.ends_with("ss") && !w.ends_with("us") && !w.ends_with("is") { + return w[..w.len() - 1].to_string(); + } + w.to_string() +} + /// Rank `entries` against a free-text query. /// -/// Scoring is deliberately dumb — token containment, weighted by field — and +/// Scoring is deliberately dumb — whole-word matching, weighted by field — and /// that is a decision, not a shortcut. The alternative (embeddings) needs a /// model, a vector index, and somewhere to keep it; the VTA offers none of the /// three, and a scoring function nobody can predict is worse than a crude one @@ -659,21 +727,16 @@ pub struct Hit { /// the user stated survives a limit ahead of what was inferred, and output is /// stable. pub fn rank(entries: Vec, query: &str) -> Vec { - let terms: Vec = query - .split(|c: char| !c.is_ascii_alphanumeric()) - .filter(|t| t.len() > 1) - .map(|t| t.to_ascii_lowercase()) - .collect(); - + let terms = query_terms(query); let mut hits: Vec = entries .into_iter() .map(|entry| { let score = if terms.is_empty() { 0 } else { - let name = entry.record.name.to_ascii_lowercase(); - let desc = entry.record.description.to_ascii_lowercase(); - let body = entry.record.body.to_ascii_lowercase(); + let name = words(&entry.record.name); + let desc = words(&entry.record.description); + let body = words(&entry.record.body); let relevance: u32 = terms .iter() .map(|t| { @@ -951,6 +1014,71 @@ mod tests { assert_eq!(r.full(&key)["evidence"][0]["kind"], "notarised"); } + #[test] + fn terms_match_whole_words_not_fragments() { + // "run" is not in "runner", and "test" is not in "latest". + let hits = rank( + vec![ + entry(MemoryType::Feedback, "Test runner", "executed by CI", ""), + entry(MemoryType::Project, "Latest release", "shipped", ""), + ], + "run", + ); + assert!(hits.iter().all(|h| h.score == 0), "{hits:?}"); + let hits = rank( + vec![entry(MemoryType::Project, "Latest release", "shipped", "")], + "test", + ); + assert_eq!(hits[0].score, 0); + } + + #[test] + fn plurals_match_their_singular_both_ways() { + for (query, text) in [ + ("tests", "how to run the test"), + ("test", "the tests live here"), + ("policy", "retention policies"), + ("box", "inboxes and boxes"), + ("push", "no pushes to main"), + ] { + let hits = rank(vec![entry(MemoryType::User, "x", text, "")], query); + assert!(hits[0].score > 0, "{query:?} should find {text:?}"); + } + } + + #[test] + fn words_ending_in_s_that_are_not_plurals_are_left_alone() { + for w in ["status", "analysis", "process", "bus", "gas"] { + assert_eq!(singular(w), w, "{w}"); + } + } + + #[test] + fn a_query_of_only_stopwords_has_no_terms() { + assert!(query_terms("what is the for").is_empty()); + assert_eq!(query_terms("what is the deploy for"), vec!["deploy"]); + // Negations are content: they change what is meant. + assert_eq!(query_terms("never push"), vec!["never", "push"]); + } + + #[test] + fn non_latin_words_match() { + let hits = rank( + vec![entry( + MemoryType::User, + "Café", + "prefers the café on Straße", + "", + )], + "CAFÉ straße", + ); + assert_eq!( + hits[0].score, + (4 + 2 + 2) * hits[0].entry.weight(), + "café in name and description, straße in description" + ); + } + #[test] fn ranking_is_stable_for_equal_scores() { let a = entry(MemoryType::User, "A", "x", "x"); diff --git a/tests/eval/scenarios/10-minimisation-stopwords.json b/tests/eval/scenarios/10-minimisation-stopwords.json index 7f3a5b9..25cb524 100644 --- a/tests/eval/scenarios/10-minimisation-stopwords.json +++ b/tests/eval/scenarios/10-minimisation-stopwords.json @@ -1,7 +1,7 @@ { "name": "minimisation-stopwords", "dimension": "minimisation", - "description": "A query made only of function words should not match every memory that happens to contain them. Known gap: ranking has no stopword list, so 'the' and 'for' count as terms.", + "description": "A query made only of function words matches nothing, rather than every memory that contains them. Fixed by the query stopword list; this probe was soft (a known gap) until then.", "steps": [ { "save": { @@ -47,8 +47,7 @@ "probe": { "label": "function words only", "query": "what is the for", - "expectEmpty": true, - "soft": true + "expectEmpty": true } }, { diff --git a/tests/eval/scenarios/18-precision-word-boundaries.json b/tests/eval/scenarios/18-precision-word-boundaries.json index debfb61..6310954 100644 --- a/tests/eval/scenarios/18-precision-word-boundaries.json +++ b/tests/eval/scenarios/18-precision-word-boundaries.json @@ -1,7 +1,7 @@ { "name": "precision-word-boundaries", "dimension": "precision", - "description": "Query terms should match words, not fragments of words. Known gap: ranking uses substring containment, so 'run' matches 'runner' and 'test' matches 'latest', and a memory that merely contains the letters can outrank the one that answers.", + "description": "Query terms match words, not fragments of words: 'run' must not match 'runner', nor 'test' match 'latest'. Fixed by whole-word matching; these probes were soft (a known gap) until then.", "steps": [ { "save": { @@ -15,7 +15,7 @@ "save": { "type": "feedback", "name": "Test runner", - "description": "Tests are run by the CI only", + "description": "Tests are executed by the CI only", "body": "" } }, @@ -34,8 +34,7 @@ "relevant": [ "feedback/test-command" ], - "top": "feedback/test-command", - "soft": true + "top": "feedback/test-command" } }, { @@ -44,8 +43,7 @@ "query": "test", "absent": [ "project/latest-release" - ], - "soft": true + ] } } ]