test: memory quality evaluation harness (#35)

.github/workflows/ci.yml

@@ -89,6 +89,16 @@ jobs:
       - name: Test
         run: cargo test --all-features
 
+      # The memory quality evaluation (#35) already ran as part of `cargo test`
+      # above, where a failed hard check fails the build. This runs it again —
+      # it is already built and takes well under a second — to put the scored
+      # report in the job summary, so a change to ranking, provenance or
+      # lifecycle arrives with its numbers.
+      - name: Memory quality evaluation
+        env:
+          EVAL_SUMMARY: "1"
+        run: cargo test --all-features --test eval
+
   workflows:
     name: workflow hygiene
     runs-on: ubuntu-latest

CLAUDE.md

@@ -34,7 +34,9 @@ src/
   lifecycle.rs   status, validUntil/reviewAfter, history keys, standing
   server.rs    the nine MCP tools
 tests/
+  common/                    the in-process fake VTA + signers, shared
   memory_roundtrip.rs        Store end-to-end against an in-process fake VTA
+  eval.rs, eval/scenarios/   memory quality evaluation — scored scenarios
   mcp_starts_without_a_vta.rs  the real binary, over stdio, with no VTA
   hook_contract.rs           the SessionStart hook's contract
 ```
@@ -54,6 +56,14 @@ routing all sit below the loopback point. Those need a live VTA.
 Add a test there — not a unit test — whenever you change what goes on the wire
 or what `Store` does with what comes back.
 
+**Ranking, provenance and lifecycle changes go through the evaluation.**
+`cargo test --test eval` replays the scenarios in `tests/eval/scenarios/` and
+scores them; a failed hard check fails the build, and CI posts the report to the
+job summary. Change a weight and the numbers move — say in the PR how. A fix
+for a listed *known gap* should make its probe hard. A new behaviour gets a
+scenario, and an empty-`query` probe if it is about what must *not* surface:
+with a query, a zero score hides things that the session-start hook would show.
+
 ## Non-negotiables
 
 **Nothing here is transport-aware.** `VtaClient::memory_{put,list,delete}` drive

Cargo.toml

@@ -18,6 +18,14 @@ path = "src/lib.rs"
 name = "vta-agent-memory"
 path = "src/main.rs"
 
+# The memory quality evaluation (#35): scripted scenarios under
+# `tests/eval/scenarios/`, scored, with a markdown report. Its own `main` so it
+# can print that report and fail on a hard check — `cargo test --test eval`.
+[[test]]
+name = "eval"
+path = "tests/eval.rs"
+harness = false
+
 [dependencies]
 # The whole reason this repo is thin: `VtaClient`'s memory methods
 # already drive `vta/memory/*` through the generic Trust-Task dispatcher, which

README.md

@@ -354,6 +354,12 @@ config at all, asserting it completes an MCP handshake, offers all nine tools, a
 returns an actionable error from a tool call. That is a property about process
 lifetime, so it can only be observed by running the thing.
 
+`tests/eval.rs` is the memory quality evaluation: 18 scripted multi-session
+scenarios — precision, staleness, contradiction, minimisation, repairability,
+poisoning — replayed against the real store and scored (precision, recall, MRR,
+repair operations). `cargo test --test eval` prints the report; CI puts it in the
+job summary. See `tests/eval/README.md`.
+
 `tests/memory_roundtrip.rs` runs the real `VtaClient::memory_*` bodies through
 the SDK's in-process loopback transport into a fake keyspace with the VTA's own
 storage semantics — upsert on `(contextId, key)`, whole-context `list`,

src/pnm.rs

@@ -103,7 +103,15 @@ impl PnmConfig {
     ///
     /// `$PNM_CONFIG_DIR` overrides, for a non-standard install.
     pub fn default_dir() -> anyhow::Result<PathBuf> {
-        if let Ok(d) = std::env::var("PNM_CONFIG_DIR")
+        Self::dir_from(std::env::var("PNM_CONFIG_DIR").ok())
+    }
+
+    /// [`default_dir`](Self::default_dir), given the value of
+    /// `$PNM_CONFIG_DIR`. Split out so the override can be tested without
+    /// setting a process-wide environment variable under a parallel test
+    /// runner — which is what made the default-dir test flaky.
+    fn dir_from(override_dir: Option<String>) -> anyhow::Result<PathBuf> {
+        if let Some(d) = override_dir
             && !d.is_empty()
         {
             return Ok(PathBuf::from(d));
@@ -339,11 +347,17 @@ vta_did = "did:key:zVta"
 
     #[test]
     fn the_config_dir_can_be_overridden() {
-        // Safety: single-threaded test, and the var is removed immediately.
-        unsafe { std::env::set_var("PNM_CONFIG_DIR", "/tmp/elsewhere") };
-        let dir = PnmConfig::default_dir().unwrap();
-        unsafe { std::env::remove_var("PNM_CONFIG_DIR") };
-        assert_eq!(dir, PathBuf::from("/tmp/elsewhere"));
+        // Through `dir_from`, not `set_var`: tests run in parallel, and a
+        // process-wide override leaked into the default-dir test above.
+        assert_eq!(
+            PnmConfig::dir_from(Some("/tmp/elsewhere".into())).unwrap(),
+            PathBuf::from("/tmp/elsewhere")
+        );
+        assert_eq!(
+            PnmConfig::dir_from(Some(String::new())).unwrap(),
+            PnmConfig::dir_from(None).unwrap(),
+            "an empty override is no override"
+        );
     }
 
     #[test]

tests/common/mod.rs

@@ -0,0 +1,266 @@
+//! Shared by the integration tests: an in-process stand-in for the VTA's
+//! memory keyspace, and the identities that write to it.
+//!
+//! The SDK's `test-loopback` transport answers the Trust-Task surface in the
+//! same process, so whatever drives a [`Store`] over [`FakeMemoryKeyspace`]
+//! runs the *real* `VtaClient` memory method bodies — the payloads that ship —
+//! against the storage semantics the VTA's own `operations::memory` has at
+//! `vta/memory/*/0.3` and `get/0.2`:
+//!
+//! - `put` upserts on `(contextId, key)` and takes the context's next version;
+//!   `expectedVersion` makes it conditional, and a failed precondition is
+//!   `versionConflict` carrying the current version and value.
+//! - `list` returns the context's live items in ascending key order, narrowed
+//!   by `prefix`, and never another context's.
+//! - `get` answers each key in `items` or `missing`.
+//! - `delete` leaves nothing live and answers `existed`.
+//! - every operation names a `purpose`, held to the purposes granted.
+//!
+//! Used by `memory_roundtrip.rs` and the evaluation harness (`eval.rs`), so the
+//! harness scores the same store the tests pin.
+
+#![allow(dead_code)] // each test target uses a different subset
+
+use std::collections::BTreeMap;
+use std::sync::{Arc, Mutex};
+
+use serde_json::{Value, json};
+use vta_sdk::client::VtaClient;
+use vta_sdk::client::loopback::LoopbackSink;
+use vta_sdk::error::VtaError;
+
+use vta_agent_memory::authorship::{AuthorKind, Signer};
+use vta_agent_memory::store::Store;
+
+/// The memory Trust Tasks, backed by a `BTreeMap` keyed by
+/// `(contextId, key)`, so ordering and the context-isolation property are the
+/// real ones rather than assumed.
+#[derive(Default)]
+pub struct FakeMemoryKeyspace {
+    /// `(contextId, key)` → `(value, version)`.
+    pub entries: Mutex<BTreeMap<(String, String), (String, u64)>>,
+    pub counter: Mutex<u64>,
+    /// Every `(type_uri, payload)` dispatched, so a test can assert on the wire
+    /// shape as well as the outcome.
+    pub seen: Mutex<Vec<(String, Value)>>,
+    /// The purposes the caller's entry lists; `None` is `any`.
+    pub granted: Mutex<Option<Vec<String>>>,
+    /// A write another writer makes right after the next `get` is answered:
+    /// `(context, key, value)`. Lets a test land a save in the window between a
+    /// store's read and its conditional write.
+    pub race: Mutex<Option<(String, String, String)>>,
+}
+
+impl FakeMemoryKeyspace {
+    pub fn recorded(&self) -> Vec<(String, Value)> {
+        self.seen.lock().unwrap().clone()
+    }
+
+    /// Live memories, not counting history copies.
+    pub fn len(&self) -> usize {
+        self.entries
+            .lock()
+            .unwrap()
+            .keys()
+            .filter(|(_, k)| !k.starts_with("archive/"))
+            .count()
+    }
+
+    /// History copies held for `key` in `context`.
+    pub fn history_len(&self, context: &str, key: &str) -> usize {
+        let prefix = format!("archive/{key}/");
+        self.entries
+            .lock()
+            .unwrap()
+            .keys()
+            .filter(|(c, k)| c == context && k.starts_with(&prefix))
+            .count()
+    }
+
+    pub fn record(context_id: &str, key: &str, value: &str, version: u64) -> Value {
+        json!({
+            "contextId": context_id, "key": key, "value": value, "version": version,
+            "valueBytes": value.len(), "deleted": false,
+            "updatedAt": "2026-11-09T00:00:00Z",
+        })
+    }
+}
+
+fn field<'a>(payload: &'a Value, name: &str) -> Result<&'a str, VtaError> {
+    payload
+        .get(name)
+        .and_then(Value::as_str)
+        .ok_or_else(|| VtaError::Validation(format!("payload is missing `{name}`: {payload}")))
+}
+
+impl LoopbackSink for FakeMemoryKeyspace {
+    fn dispatch(&self, type_uri: &str, payload: &Value) -> Result<Value, VtaError> {
+        self.seen
+            .lock()
+            .unwrap()
+            .push((type_uri.to_string(), payload.clone()));
+
+        let context_id = field(payload, "contextId")?;
+        if let Some(granted) = self.granted.lock().unwrap().as_ref() {
+            let purpose = payload.get("purpose").and_then(Value::as_str);
+            let ok = match purpose {
+                Some(p) => granted.iter().any(|g| g == p),
+                None => granted.len() == 1,
+            };
+            if !ok {
+                let op = type_uri.rsplit('/').nth(1).unwrap_or_default().to_string();
+                return Err(VtaError::Rejected {
+                    code: format!("vta/memory/{op}:purposeNotGranted"),
+                    message: "this entry does not grant the purpose".into(),
+                    details: Some(json!({ "purposeNamed": purpose.is_some(), "purpose": purpose })),
+                });
+            }
+        }
+
+        match type_uri {
+            u if u.ends_with("/vta/memory/put/0.3") => {
+                let key = field(payload, "key")?;
+                let value = field(payload, "value")?;
+                // `expectedVersion`: 0 is create-only, n requires version n.
+                // A failed precondition is `versionConflict` with the current
+                // version and value, as the VTA sends it.
+                if let Some(expected) = payload.get("expectedVersion").and_then(Value::as_u64) {
+                    let current = self
+                        .entries
+                        .lock()
+                        .unwrap()
+                        .get(&(context_id.to_string(), key.to_string()))
+                        .cloned();
+                    let reason = match (&current, expected) {
+                        (None, 0) => None,
+                        (Some(_), 0) => Some("recordExists"),
+                        (None, _) => Some("recordAbsent"),
+                        (Some((_, v)), e) if *v != e => Some("versionMismatch"),
+                        _ => None,
+                    };
+                    if let Some(reason) = reason {
+                        let mut details = json!({ "reason": reason });
+                        if let Some((v, ver)) = current {
+                            details["currentVersion"] = json!(ver);
+                            details["currentValue"] = json!(v);
+                        }
+                        return Err(VtaError::Rejected {
+                            code: "vta/memory/put:versionConflict".into(),
+                            message: "precondition failed".into(),
+                            details: Some(details),
+                        });
+                    }
+                }
+                let mut counter = self.counter.lock().unwrap();
+                *counter += 1;
+                let created = self
+                    .entries
+                    .lock()
+                    .unwrap()
+                    .insert(
+                        (context_id.to_string(), key.to_string()),
+                        (value.to_string(), *counter),
+                    )
+                    .is_none();
+                Ok(json!({
+                    "contextId": context_id, "key": key, "version": *counter,
+                    "wasCreated": created, "updatedAt": "2026-11-09T00:00:00Z",
+                    "valueBytes": value.len(),
+                }))
+            }
+            u if u.ends_with("/vta/memory/list/0.3") => {
+                let prefix = payload.get("prefix").and_then(Value::as_str).unwrap_or("");
+                let items: Vec<Value> = self
+                    .entries
+                    .lock()
+                    .unwrap()
+                    .iter()
+                    .filter(|((c, k), _)| c == context_id && k.starts_with(prefix))
+                    .map(|((c, k), (v, ver))| Self::record(c, k, v, *ver))
+                    .collect();
+                Ok(json!({ "items": items, "highWatermark": *self.counter.lock().unwrap() }))
+            }
+            u if u.ends_with("/vta/memory/get/0.2") => {
+                let race = self.race.lock().unwrap().take();
+                let entries = self.entries.lock().unwrap();
+                let (mut items, mut missing) = (Vec::new(), Vec::new());
+                for k in payload["keys"].as_array().into_iter().flatten() {
+                    let k = k.as_str().unwrap_or_default();
+                    match entries.get(&(context_id.to_string(), k.to_string())) {
+                        Some((v, ver)) => items.push(Self::record(context_id, k, v, *ver)),
+                        None => missing.push(json!(k)),
+                    }
+                }
+                drop(entries);
+                if let Some((c, k, v)) = race {
+                    self.plant(&c, &k, v);
+                }
+                Ok(json!({ "items": items, "missing": missing }))
+            }
+            u if u.ends_with("/vta/memory/delete/0.3") => {
+                let key = field(payload, "key")?;
+                let existed = self
+                    .entries
+                    .lock()
+                    .unwrap()
+                    .remove(&(context_id.to_string(), key.to_string()))
+                    .is_some();
+                Ok(json!({ "contextId": context_id, "key": key, "existed": existed }))
+            }
+            other => Err(VtaError::Validation(format!(
+                "the memory store must not dispatch `{other}`"
+            ))),
+        }
+    }
+}
+
+/// A fresh `did:key` signer, as the session store would hold for an agent.
+pub fn signer(kind: AuthorKind) -> Signer {
+    let (did, private) = did_key();
+    Signer::from_did_key(&did, &private, kind).unwrap()
+}
+
+/// A fresh Ed25519 `did:key` and its private key, multibase.
+pub fn did_key() -> (String, String) {
+    let mut seed = [0u8; 32];
+    getrandom::fill(&mut seed).unwrap();
+    let public = ed25519_dalek::SigningKey::from_bytes(&seed)
+        .verifying_key()
+        .to_bytes();
+    let did = format!(
+        "did:key:{}",
+        vta_sdk::did_key::ed25519_multibase_pubkey(&public)
+    );
+    let private = multibase::encode(multibase::Base::Base58Btc, seed);
+    (did, private)
+}
+
+/// A store over `vta` in `context`, signing as a fresh agent.
+pub fn store_on(vta: &Arc<FakeMemoryKeyspace>, context: &str) -> Store {
+    Store::new(VtaClient::loopback(vta.clone()), context).with_signer(signer(AuthorKind::Agent))
+}
+
+pub fn store_with(context: &str) -> (Store, Arc<FakeMemoryKeyspace>) {
+    let vta = Arc::new(FakeMemoryKeyspace::default());
+    (store_on(&vta, context), vta)
+}
+
+impl FakeMemoryKeyspace {
+    /// The stored value at `(context, key)`.
+    pub fn value(&self, context: &str, key: &str) -> String {
+        self.entries.lock().unwrap()[&(context.to_string(), key.to_string())]
+            .0
+            .clone()
+    }
+
+    /// Store `value` at `(context, key)` directly, as another writer with
+    /// access to the context could.
+    pub fn plant(&self, context: &str, key: &str, value: impl Into<String>) {
+        let mut counter = self.counter.lock().unwrap();
+        *counter += 1;
+        self.entries
+            .lock()
+            .unwrap()
+            .insert((context.into(), key.into()), (value.into(), *counter));
+    }
+}