From 740f3b26c029d94c5d4487f805f13173af545d24 Mon Sep 17 00:00:00 2001 From: ajianaz Date: Fri, 16 Oct 2026 11:48:22 +0700 Subject: [PATCH] test(mcp,cli): scope-isolation suites and the coverage document (#1467, part 3) --- CHANGELOG.md | 1 + crates/uteke-cli/tests/scope_isolation.rs | 172 +++++++++++++++++++++ crates/uteke-mcp/Cargo.toml | 1 + crates/uteke-mcp/src/lib.rs | 180 ++++++++++++++++++++++ docs/isolation-suite.md | 45 ++++++ 5 files changed, 399 insertions(+) create mode 100644 crates/uteke-cli/tests/scope_isolation.rs create mode 100644 docs/isolation-suite.md diff --git a/CHANGELOG.md b/CHANGELOG.md index d65886d2..c6756611 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,7 @@ ### Changed - Internal: scope-isolation regression gate, parts 1 and 2 (#1467): the HTTP suite runs 20 scoped read requests through `route` (every recall strategy, search, list, recent, export, graph, room, orphans, tags, stats, doc) and a test fails when a read route is added to the registry without an isolation check or a documented exemption. Part 1: a two-namespace fixture with overlapping vocabulary and a core suite asserting that `recall` (every strategy), unified recall, `search`, `list`, tags, export, room recall and graph views never return a record from another namespace. The fixture ships behind the `test-support` feature of `uteke-core` for the server, MCP and CLI suites that follow. +- Internal: scope-isolation regression gate, part 3 (#1467): the MCP suite over the read-only tools (a test fails when a tool is added without an isolation check or an exemption), a CLI suite that runs the binary against a two-namespace store, and `docs/isolation-suite.md`, which lists what each transport covers and the exemptions. - **`tools/list` for a read-only token lists only the tools it can call (#1446)** - MCP over HTTP advertised every tool to read-only callers, which then failed at call time. A read-only caller now sees exactly the read tools; full-access callers, the stdio transport and `tools/call` are unchanged. Contract: uteke-cloud#203 (approved by the owner). - **`unsafe_code` is now enforced (#1456)** - the workspace declared `unsafe_code = "forbid"` but no crate inherited it, so nothing was enforced. Every crate now has `[lints] workspace = true` and the level is `deny`: production code cannot contain `unsafe`, and only the five test modules that call `std::env::set_var`/`remove_var` (edition 2024) opt out with `#[allow(unsafe_code)]`. `forbid` cannot be used because it cannot be relaxed per module. diff --git a/crates/uteke-cli/tests/scope_isolation.rs b/crates/uteke-cli/tests/scope_isolation.rs new file mode 100644 index 00000000..991fc88d --- /dev/null +++ b/crates/uteke-cli/tests/scope_isolation.rs @@ -0,0 +1,172 @@ +//! Scope-isolation regression gate over the CLI (#1467, part 3). +//! +//! Seeds a store with two namespaces (`alpha`, `beta`) whose memories share one +//! vocabulary, then reads it through every CLI command that takes a scope. Each +//! memory ends in `of `, so output scoped to one namespace must hold +//! its own marker and never the other's, nor the other's project tag. +//! +//! `uteke recall` needs the ONNX runtime, which CI does not have (memories are +//! stored FTS-only here, see `json_output_stays_clean`); the core, HTTP and MCP +//! suites cover vector recall. `docs/isolation-suite.md` lists the coverage. + +use std::path::PathBuf; +use std::process::Command; + +const Q: &str = "quartz meridian orchid ledger"; +const SCOPES: [&str; 2] = ["alpha", "beta"]; + +struct Store { + dir: PathBuf, +} + +impl Store { + fn new(name: &str) -> Self { + let dir = + std::env::temp_dir().join(format!("uteke_isolation_{name}_{}", std::process::id())); + let _ = std::fs::remove_dir_all(&dir); + std::fs::create_dir_all(dir.join("home")).unwrap(); + // Server routing off, so every command runs against the local store. + std::fs::write(dir.join("home/uteke.toml"), "[server]\nenabled = false\n").unwrap(); + Self { dir } + } + + fn run(&self, args: &[&str]) -> String { + let out = Command::new(env!("CARGO_BIN_EXE_uteke")) + .arg("--store") + .arg(self.dir.join("store")) + .args(args) + .env("UTEKE_HOME", self.dir.join("home")) + .env("UTEKE_NO_UPDATE_CHECK", "1") + // No embedder: memories are stored FTS-only, commands still run. + .env("ORT_LIB_PATH", "/nonexistent/uteke-test/libonnxruntime.so") + .env_remove("UTEKE_AUTH_TOKEN") + .output() + .expect("run uteke"); + assert!( + out.status.success(), + "uteke {args:?} failed: {}", + String::from_utf8_lossy(&out.stderr) + ); + String::from_utf8(out.stdout).unwrap() + } + + fn seed(&self) { + for scope in SCOPES { + let tags = format!("project:{scope},shared"); + for i in 0..3 { + let text = format!("{Q} note {i} of {scope}"); + self.run(&[ + "--json", + "remember", + &text, + "--namespace", + scope, + "--tags", + &tags, + ]); + } + let room = format!("room-{scope}"); + let text = format!("{Q} room entry of {scope}"); + self.run(&[ + "--json", + "remember", + &text, + "--namespace", + scope, + "--tags", + &tags, + "--room", + &room, + "--author", + "tester", + ]); + // Unrelated vocabulary: the other memories link to each other, this + // one stays an orphan. + let text = format!("solitary zebra fjord of {scope}"); + self.run(&["--json", "remember", &text, "--namespace", scope]); + } + } +} + +impl Drop for Store { + fn drop(&mut self) { + let _ = std::fs::remove_dir_all(&self.dir); + } +} + +/// (arguments, the output must hold an `of ` marker). +fn surfaces(scope: &str) -> Vec<(Vec, bool)> { + let s = |v: &[&str]| v.iter().map(|x| x.to_string()).collect::>(); + let room = format!("room-{scope}"); + vec![ + ( + s(&["--json", "search", Q, "--namespace", scope, "--limit", "20"]), + true, + ), + ( + s(&["--json", "list", "--namespace", scope, "--limit", "50"]), + true, + ), + ( + s(&[ + "--json", + "list", + "--tag", + "shared", + "--namespace", + scope, + "--limit", + "50", + ]), + true, + ), + (s(&["--json", "export", "-", "--namespace", scope]), true), + (s(&["--json", "room", "recall", &room]), true), + ( + s(&[ + "--json", + "orphans", + "--threshold", + "2", + "--namespace", + scope, + ]), + true, + ), + (s(&["--json", "tags", "list", "--namespace", scope]), false), + (s(&["--json", "stats", "--namespace", scope]), false), + (s(&["--json", "room", "list", "--namespace", scope]), false), + ( + s(&["--json", "contradictions", "list", "--namespace", scope]), + false, + ), + ] +} + +#[test] +fn every_scoped_read_command_returns_only_its_own_namespace() { + let store = Store::new("read"); + store.seed(); + for scope in SCOPES { + let foreign = if scope == "alpha" { "beta" } else { "alpha" }; + for (args, expects_hits) in surfaces(scope) { + let refs: Vec<&str> = args.iter().map(String::as_str).collect(); + let out = store.run(&refs); + let shown: String = out.chars().take(300).collect(); + assert!( + !out.contains(&format!("of {foreign}")), + "uteke {args:?} scoped to {scope} returned {foreign} records: {shown}" + ); + assert!( + !out.contains(&format!("project:{foreign}")), + "uteke {args:?} scoped to {scope} exposed the {foreign} project tag: {shown}" + ); + if expects_hits { + assert!( + out.contains(&format!("of {scope}")), + "uteke {args:?} found no {scope} record, the check would be vacuous: {shown}" + ); + } + } + } +} diff --git a/crates/uteke-mcp/Cargo.toml b/crates/uteke-mcp/Cargo.toml index efa1b854..344f5528 100644 --- a/crates/uteke-mcp/Cargo.toml +++ b/crates/uteke-mcp/Cargo.toml @@ -38,6 +38,7 @@ uuid = "1" [dev-dependencies] uuid = { version = "1", features = ["v4"] } chrono = "0.4" +uteke-core = { path = "../uteke-core", features = ["test-support"] } [lints] workspace = true diff --git a/crates/uteke-mcp/src/lib.rs b/crates/uteke-mcp/src/lib.rs index a6b59dc5..5f63a977 100644 --- a/crates/uteke-mcp/src/lib.rs +++ b/crates/uteke-mcp/src/lib.rs @@ -3968,3 +3968,183 @@ mod read_only_gate_tests { assert_eq!(u.count(None).unwrap(), 1); } } + +/// Scope-isolation regression gate over MCP (#1467, part 3). +/// +/// The two-namespace store of `uteke_core::test_support::seed_two_scopes` +/// (records of `alpha` and `beta` share one vocabulary, each ends in +/// `of `) is read through every read-only tool that takes a scope +/// and must never return the other namespace's records or project tag. +/// `every_read_tool_is_covered_or_exempt` fails when a read-only tool is added +/// without an isolation check or a documented exemption. +#[cfg(test)] +mod isolation_tests { + use super::*; + use serde_json::{Value, json}; + use uteke_core::test_support::{ISOLATION_QUERY as Q, ISOLATION_SCOPES, seed_two_scopes}; + + fn call(u: &Uteke, tool: &str, args: Value) -> String { + let raw = json!({ + "jsonrpc": "2.0", "id": 1, "method": "tools/call", + "params": {"name": tool, "arguments": args} + }) + .to_string(); + handle_jsonrpc_as(u, &raw, Access::ReadOnly).expect("a response") + } + + /// (tool, arguments, the response must hold an `of ` marker). + fn surfaces(scope: &str) -> Vec<(&'static str, Value, bool)> { + let mut v = vec![ + ( + "uteke_search", + json!({"query": Q, "namespace": scope, "limit": 20}), + true, + ), + ("uteke_list", json!({"namespace": scope, "limit": 50}), true), + ( + "uteke_list", + json!({"namespace": scope, "tag": "shared", "limit": 50}), + true, + ), + ("uteke_graph", json!({"namespace": scope}), true), + ( + "uteke_room_recall", + json!({"room_id": format!("room-{scope}"), "query": Q}), + true, + ), + ( + "uteke_room_memories", + json!({"room_id": format!("room-{scope}")}), + true, + ), + ("uteke_tags_list", json!({"namespace": scope}), false), + ("uteke_room_list", json!({"namespace": scope}), false), + ("uteke_stats", json!({"namespace": scope}), false), + ("uteke_contradictions", json!({"namespace": scope}), false), + ]; + for strategy in ["hybrid", "vector", "fts5", "graph", "fusion"] { + for kind in ["memory", "all"] { + v.push(( + "uteke_recall", + json!({"query": Q, "namespace": scope, "limit": 20, "strategy": strategy, "type": kind}), + true, + )); + } + } + v.push(( + "uteke_recall", + json!({"query": Q, "namespace": scope, "pack": true, "budget_chars": 4000}), + true, + )); + v.push(( + "uteke_recall", + json!({"query": Q, "namespace": scope, "explain": true}), + true, + )); + v.push(( + "uteke_recall", + json!({"query": Q, "namespace": scope, "tags": ["shared"]}), + true, + )); + v + } + + #[test] + fn every_scoped_read_tool_returns_only_its_own_namespace() { + let u = seed_two_scopes(); + for scope in ISOLATION_SCOPES { + let foreign = if scope == "alpha" { "beta" } else { "alpha" }; + for (tool, args, expects_hits) in surfaces(scope) { + let text = call(&u, tool, args.clone()); + let what = format!("{tool} {args}"); + let shown: String = text.chars().take(300).collect(); + assert!(!text.contains("\"isError\":true"), "{what}: {shown}"); + assert!( + !text.contains(&format!("of {foreign}")), + "{what} scoped to {scope} returned {foreign} records: {shown}" + ); + assert!( + !text.contains(&format!("project:{foreign}")), + "{what} scoped to {scope} exposed the {foreign} project tag: {shown}" + ); + if expects_hits { + assert!( + text.contains(&format!("of {scope}")), + "{what} found no {scope} record, the check would be vacuous: {shown}" + ); + } + } + } + } + + /// Read-only tools the suite does not scope-check, each with the reason. + const EXEMPT: &[(&str, &str)] = &[ + ("uteke_get", "fetch by ID: IDs are global handles"), + ("uteke_provenance", "by memory ID"), + ("uteke_timeline", "by memory ID"), + ( + "uteke_doc_get", + "by document slug; documents are global (#1271)", + ), + ( + "uteke_doc_list", + "documents are global (#1271), no memory content", + ), + ( + "uteke_doc_search", + "documents are global (#1271), no memory content", + ), + ("uteke_doc_list_rooms", "by document slug"), + ("uteke_room_stats", "by room ID; the room is the scope"), + ("uteke_room_summary", "by room ID; the room is the scope"), + ( + "uteke_room_summary_document", + "by room ID; the room is the scope", + ), + ( + "uteke_room_document", + "dispatch alias of uteke_room_summary_document, never advertised", + ), + ( + "uteke_room_list_documents", + "by room ID; documents are global", + ), + ]; + + /// Every read-only tool must be covered by `surfaces` or exempt with a + /// reason, and a tool that advertises a `namespace` argument can never be + /// exempt: a new retrieval tool cannot skip the isolation suite unnoticed. + #[test] + fn every_read_tool_is_covered_or_exempt() { + let covered: std::collections::BTreeSet<&str> = + surfaces("alpha").iter().map(|(t, _, _)| *t).collect(); + let exempt: std::collections::BTreeSet<&str> = EXEMPT.iter().map(|(t, _)| *t).collect(); + for tool in READ_ONLY_TOOLS { + assert!( + covered.contains(tool) || exempt.contains(tool), + "{tool} is a read-only tool with no isolation check: add it to `surfaces` or to `EXEMPT` with a reason (#1467)" + ); + } + for tool in covered.iter().chain(exempt.iter()) { + assert!( + READ_ONLY_TOOLS.contains(tool), + "{tool} is not a read-only tool" + ); + } + assert!( + covered.is_disjoint(&exempt), + "a tool is both covered and exempt" + ); + let u = seed_two_scopes(); + let list = handle_request(&u, "tools/list", None, Access::ReadOnly).unwrap(); + for t in list["tools"].as_array().unwrap() { + let name = t["name"].as_str().unwrap(); + if t["inputSchema"]["properties"]["namespace"].is_object() { + assert!( + covered.contains(name), + "{name} takes a namespace but is not covered by the isolation suite" + ); + } + } + } +} diff --git a/docs/isolation-suite.md b/docs/isolation-suite.md new file mode 100644 index 00000000..5d526768 --- /dev/null +++ b/docs/isolation-suite.md @@ -0,0 +1,45 @@ +# Scope-isolation suite + +A retrieval surface must never return a record from outside the scope it was asked for (#1467, after the namespace orphans of #1353 and the unified-recall leak of #1270). The suite is a regression gate: it runs in `cargo test`, so a failure fails the `Test` check of a pull request. + +## The fixture + +`uteke_core::test_support::seed_two_scopes()` (behind the hidden `test-support` feature of `uteke-core`, used as a dev-dependency only) builds one store: + +- two namespaces, `alpha` and `beta`, whose memories share the **same vocabulary** (`quartz meridian orchid ledger`), so every surface has hits in both scopes for one query; +- three plain memories per namespace tagged `project:` and `shared`, and one memory per namespace in room `room-`; +- every memory ends in `of `, so a response can be checked by text, whatever its shape; +- one global document, `shared-doc`, with the same vocabulary (documents have no namespace by design, #1271). + +A check passes only if the response scoped to one namespace holds no `of ` record and no `project:` tag. Where hits are expected it first asserts the scope's own marker is present, so a check cannot pass by returning nothing. + +## What is covered + +| Transport | Where | Surfaces | +|---|---|---| +| Core | `crates/uteke-core/src/isolation_tests.rs` | `recall` and `recall_hybrid` (all five strategies), tag filters, unified recall (`All`, `Memory`), `search`, `list`, tag counts, `export`, `count`, scoped room recall, room listing, `graph_data`, `graph_view` | +| HTTP | `crates/uteke-server/src/handlers/isolation_tests.rs` | `/recall` (five strategies × `memory`/`all`, `pack`, `explain`, tags), `/search`, `/list`, `/recent`, `/export`, `/graph`, `/room/memories`, `/room/recall`, `/orphans`, `/tags`, `/room/list`, `/stats`, `/doc/search`, `/doc/list` | +| MCP | `crates/uteke-mcp/src/lib.rs`, `mod isolation_tests` | `uteke_recall` (five strategies × `memory`/`all`, `pack`, `explain`, tags), `uteke_search`, `uteke_list`, `uteke_graph`, `uteke_room_recall`, `uteke_room_memories`, `uteke_tags_list`, `uteke_room_list`, `uteke_stats`, `uteke_contradictions` | +| CLI | `crates/uteke-cli/tests/scope_isolation.rs` | `search`, `list` (with a tag), `export`, `room recall`, `orphans`, `tags list`, `stats`, `room list`, `contradictions list` | + +`uteke recall` is not in the CLI suite: it needs the ONNX runtime, which CI does not have. The same code path is covered by the core, HTTP and MCP suites. + +## Exemptions + +These read surfaces take no scope, and each is listed with its reason in the suite instead of being skipped silently: + +- **Fetch by ID** (`GET /memory`, `/provenance`, `/timeline`, `/edges`, `POST /memory/doc-refs`, `uteke_get`, `uteke_provenance`, `uteke_timeline`): IDs are global handles. +- **By room or slug** (`/room/summary`, `/room/stats`, `/doc/get`, `uteke_room_stats`, ...): the room or the document is the scope; documents are global. +- **No records**: `/health`, `/guide`, `/routes`, `/store`, `/namespaces`. +- **`SearchType::Document`**: the global document view; it returns documents only, never a memory, and `All` and `Memory` never return a document when scoped (pinned in the core suite). + +## Adding a retrieval surface + +1. Add it to `surfaces()` in the suites of the transports that expose it. A surface that exists in one transport must be covered in all of them. +2. If it truly takes no scope, add it to `EXEMPT` with the reason. + +Two tests enforce this: the HTTP suite fails when a read row (a `GET`, or a `POST` that does not write) of `api_registry::ENDPOINTS` is neither covered nor exempt, and the MCP suite fails when a tool of `READ_ONLY_TOOLS` is neither covered nor exempt, or when a tool that advertises a `namespace` argument is not covered. + +## Requiring it + +The suite runs in the `Test` job. To make a failure block merging, `Test` has to be a required status check of the branch protection rule for `develop` (a repository setting).