From cd7ca64df37e9f61bcee2825bba740074e3224ff Mon Sep 17 00:00:00 2001 From: ajianaz Date: Wed, 21 Oct 2026 14:26:11 +0700 Subject: [PATCH] feat(server): redacted on the write responses and 400 secret_detected in strict mode (#1464, part 2b-2b) --- CHANGELOG.md | 1 + crates/uteke-cli/src/commands/maintenance.rs | 17 +- crates/uteke-core/src/import_export.rs | 15 +- crates/uteke-core/src/memory/types.rs | 11 +- crates/uteke-core/src/privacy_write_tests.rs | 13 +- crates/uteke-server/src/context.rs | 27 ++++ .../uteke-server/src/handlers/maintenance.rs | 2 + crates/uteke-server/src/handlers/memory.rs | 10 ++ crates/uteke-server/src/handlers/mod.rs | 16 +- .../src/handlers/privacy_tests.rs | 145 ++++++++++++++++++ crates/uteke-server/src/handlers/rooms.rs | 4 + docs/configuration.md | 8 + 12 files changed, 254 insertions(+), 15 deletions(-) create mode 100644 crates/uteke-server/src/handlers/privacy_tests.rs diff --git a/CHANGELOG.md b/CHANGELOG.md index 2f6484fb..2cc6c5d1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,7 @@ ### Added - **`[privacy]` in `uteke.toml` (#1464, part 2b-2a)** - `mode` (`redact`, the default, `strict` or `off`) and `allowlist` (regexes) now configure the secret filter in the CLI, `uteke-serve` and `uteke-mcp`; `UTEKE_PRIVACY_MODE` overrides the mode. The section is read from the global file only (a project file could otherwise switch the filter off), and a bad value falls back to the default instead of disabling it. +- **`redacted` on the HTTP write responses and `secret_detected` in strict mode (#1464, part 2b-2b)** - `POST /remember` and `POST /room/remember` return `redacted: true` when the privacy filter masked a secret; a strict-mode write is refused with `400 {"error": "secret_detected", "message", "kinds"}` (also on `PUT /memory`); `POST /import` reports `redacted` and `rejected_secret`. Additive; agreed with cloud in uteke-cloud#218. - **`author` on every write path (#1427, phase 2)** - a memory records who wrote it (the agent, user or tool name), with or without a room. `POST /remember`, MCP `uteke_remember` and `uteke remember --author` accept it (free text, trimmed, 1 to 128 characters; an invalid value is a 400, a tool error or a CLI error before anything is stored). A room write stamps the named writer on the memory as well as on the room link; the `anonymous` / `user` fallback of the room link is not an author. `author` is returned on the memory object by recall, search, list and get (omitted when unset), echoed by `POST /remember`, and kept by export and import. Stores migrate to schema v22 on first open (see Internal below): back up a production store first. Filtering by author is not part of this phase. Contract: uteke-cloud#219 (cloud already has the field). ### Fixed diff --git a/crates/uteke-cli/src/commands/maintenance.rs b/crates/uteke-cli/src/commands/maintenance.rs index a8a6d22c..946d0bf5 100644 --- a/crates/uteke-cli/src/commands/maintenance.rs +++ b/crates/uteke-cli/src/commands/maintenance.rs @@ -462,10 +462,7 @@ fn import_with_extraction( if facts.is_empty() { tracing::warn!("Extraction produced no facts from input"); - return Ok(ImportResult { - imported: 0, - skipped: 0, - }); + return Ok(ImportResult::default()); } let mut imported = 0usize; @@ -505,7 +502,11 @@ fn import_with_extraction( } } - Ok(ImportResult { imported, skipped }) + Ok(ImportResult { + imported, + skipped, + ..Default::default() + }) } /// Detect import format from file extension and content. @@ -564,7 +565,11 @@ fn import_text( } } - Ok(ImportResult { imported, skipped }) + Ok(ImportResult { + imported, + skipped, + ..Default::default() + }) } /// Split markdown by headings. Each heading + body becomes a chunk. diff --git a/crates/uteke-core/src/import_export.rs b/crates/uteke-core/src/import_export.rs index bc5e3b08..94c984ac 100644 --- a/crates/uteke-core/src/import_export.rs +++ b/crates/uteke-core/src/import_export.rs @@ -79,6 +79,8 @@ impl crate::Uteke { let mut imported = 0; let mut deduped = 0; + let mut redacted = 0; + let mut rejected_secret = 0; for mut entry in entries { if entry.content.is_empty() { @@ -96,6 +98,9 @@ impl crate::Uteke { Ok(p) => p, Err(e) => { tracing::warn!("Skipping import entry: {e}"); + if matches!(e, Error::SecretDetected { .. }) { + rejected_secret += 1; + } skipped += 1; continue; } @@ -186,6 +191,9 @@ impl crate::Uteke { } self.record_redaction(&id, &redactions); + if !redactions.is_empty() { + redacted += 1; + } // Auto-link cosine edges (consistent with remember path, #1005). // Must run AFTER index.insert() so the new memory is searchable. @@ -209,7 +217,12 @@ impl crate::Uteke { ); } - Ok(ImportResult { imported, skipped }) + Ok(ImportResult { + imported, + skipped, + redacted, + rejected_secret, + }) } /// Parse JSONL input: one self-contained JSON object per line. diff --git a/crates/uteke-core/src/memory/types.rs b/crates/uteke-core/src/memory/types.rs index 06038323..4a70a273 100644 --- a/crates/uteke-core/src/memory/types.rs +++ b/crates/uteke-core/src/memory/types.rs @@ -455,12 +455,19 @@ fn default_created_at() -> chrono::DateTime { } /// Result of an import operation. -#[derive(Debug, Clone, Serialize, Deserialize)] +#[derive(Debug, Clone, Default, Serialize, Deserialize)] pub struct ImportResult { /// Number of memories imported. pub imported: usize, - /// Number of entries skipped (duplicate or invalid). + /// Number of entries skipped (duplicate, invalid or rejected by the + /// privacy filter in strict mode). pub skipped: usize, + /// Imported memories in which the privacy filter masked a secret (#1464). + #[serde(default)] + pub redacted: usize, + /// Entries skipped because strict mode found a secret (also in `skipped`). + #[serde(default)] + pub rejected_secret: usize, } /// A tag with its usage count. diff --git a/crates/uteke-core/src/privacy_write_tests.rs b/crates/uteke-core/src/privacy_write_tests.rs index a8124203..26214282 100644 --- a/crates/uteke-core/src/privacy_write_tests.rs +++ b/crates/uteke-core/src/privacy_write_tests.rs @@ -442,7 +442,12 @@ fn import_masks_rows_and_strict_skips_them() { gh() ); let u = store(PrivacyMode::Redact); - assert_eq!(u.import(&file, None).unwrap().imported, 2); + let r = u.import(&file, None).unwrap(); + assert_eq!( + (r.imported, r.redacted, r.rejected_secret), + (2, 1, 0), + "one of the two rows had a secret masked" + ); let rows = u.list(None, 10, 0, None).unwrap(); assert!(rows.iter().all(|m| !m.content.contains(&gh())), "{rows:?}"); let masked = rows @@ -462,9 +467,9 @@ fn import_masks_rows_and_strict_skips_them() { let s = store(PrivacyMode::Strict); let r = s.import(&file, None).unwrap(); assert_eq!( - (r.imported, r.skipped), - (1, 1), - "the row with a secret is skipped" + (r.imported, r.skipped, r.rejected_secret, r.redacted), + (1, 1, 1, 0), + "the row with a secret is skipped and counted" ); assert_eq!(s.list(None, 10, 0, None).unwrap()[0].content, "plain row"); } diff --git a/crates/uteke-server/src/context.rs b/crates/uteke-server/src/context.rs index f3d514ec..f6ddfeaf 100644 --- a/crates/uteke-server/src/context.rs +++ b/crates/uteke-server/src/context.rs @@ -335,6 +335,33 @@ impl ReqCtx { ) } + /// The privacy filter rejected the write in strict mode (#1464): 400 with + /// the kinds found, never the values. + pub fn secret_detected_response( + &self, + req: &Request, + kinds: &[String], + ) -> Response>> { + let data = serde_json::json!({ + "error": "secret_detected", + "message": format!( + "the content contains a secret ({}); remove it or set [privacy] mode to redact", + kinds.join(", ") + ), + "kinds": kinds, + }) + .to_string(); + let mut headers = self.cors_headers_for(req); + headers.push(json_header()); + Response::new( + StatusCode::from(400), + headers, + Cursor::new(data.into_bytes()), + None, + None, + ) + } + /// Build an OK response with CORS headers specific to a request. pub fn ok_response_for( &self, diff --git a/crates/uteke-server/src/handlers/maintenance.rs b/crates/uteke-server/src/handlers/maintenance.rs index e1866aba..30e9e5e6 100644 --- a/crates/uteke-server/src/handlers/maintenance.rs +++ b/crates/uteke-server/src/handlers/maintenance.rs @@ -302,6 +302,8 @@ pub(super) fn post_import(c: &mut Call) -> Response>> { serde_json::json!({ "imported": result.imported, "skipped": result.skipped, + "redacted": result.redacted, + "rejected_secret": result.rejected_secret, }), import_ns, ), diff --git a/crates/uteke-server/src/handlers/memory.rs b/crates/uteke-server/src/handlers/memory.rs index b9301243..e891f1ca 100644 --- a/crates/uteke-server/src/handlers/memory.rs +++ b/crates/uteke-server/src/handlers/memory.rs @@ -51,6 +51,7 @@ pub(super) fn post_remember(c: &mut Call) -> Response>> { match result { Ok(outcome) => { + let redacted = outcome.redacted(); let id = outcome.id; // Set source provenance after storage (#682) — matches CLI. if req_data.source.is_some() || req_data.source_type.is_some() { @@ -83,6 +84,9 @@ pub(super) fn post_remember(c: &mut Call) -> Response>> { "id": id, "author_type": req_data.author_type.clone().unwrap_or_else(|| "agent".to_string()), "embedding_written": outcome.embedding_written, + // #1464: true when the privacy filter masked a secret, + // so the stored text differs from the request. + "redacted": redacted, "warning": uteke_core::namespace_deprecation::merge_warning( outcome.warning, uteke_core::namespace_deprecation::namespace_write_warning( @@ -96,6 +100,9 @@ pub(super) fn post_remember(c: &mut Call) -> Response>> { } ctx.ok_response_for(req, &body) } + Err(uteke_core::Error::SecretDetected { kinds }) => { + ctx.secret_detected_response(req, &kinds) + } Err(e) => { error!("Internal error: {e}"); ctx.error_response_for(req, 500, "Internal server error") @@ -814,6 +821,9 @@ pub(super) fn put_memory(c: &mut Call) -> Response>> { Ok(false) => { ctx.error_response_for(req, 404, format!("Memory not found: {}", req_data.id)) } + Err(uteke_core::Error::SecretDetected { kinds }) => { + ctx.secret_detected_response(req, &kinds) + } Err(e) => { error!("Update memory error: {e}"); // Propagate validation errors with proper status codes diff --git a/crates/uteke-server/src/handlers/mod.rs b/crates/uteke-server/src/handlers/mod.rs index 680d7106..380ec399 100644 --- a/crates/uteke-server/src/handlers/mod.rs +++ b/crates/uteke-server/src/handlers/mod.rs @@ -35,6 +35,8 @@ mod maintenance; mod mcp; mod memory; mod namespaces; +#[cfg(test)] +mod privacy_tests; mod rooms; mod tags; @@ -2665,11 +2667,21 @@ mod namespace_deprecation_tests { w.is_null() || !w.as_str().unwrap().contains("Namespaces are deprecated"), "{raw}" ); - // Same keys as before the change: warning stays a key, no new ones. + // Same keys as before the change: warning stays a key. `redacted` + // is the one additive key (#1464, agreed with cloud in #218). let v: serde_json::Value = serde_json::from_str(&raw).unwrap(); let mut keys: Vec<_> = v.as_object().unwrap().keys().cloned().collect(); keys.sort(); - assert_eq!(keys, ["author_type", "embedding_written", "id", "warning"]); + assert_eq!( + keys, + [ + "author_type", + "embedding_written", + "id", + "redacted", + "warning" + ] + ); } } diff --git a/crates/uteke-server/src/handlers/privacy_tests.rs b/crates/uteke-server/src/handlers/privacy_tests.rs new file mode 100644 index 00000000..1872dec8 --- /dev/null +++ b/crates/uteke-server/src/handlers/privacy_tests.rs @@ -0,0 +1,145 @@ +//! The privacy filter over HTTP (#1464): `redacted` on the remember responses, +//! and a strict-mode rejection as `400 secret_detected` with the kinds and +//! never the value. + +use super::*; +use serde_json::{Value, json}; +use tiny_http::TestRequest; +use uteke_core::privacy::{PrivacyConfig, PrivacyMode}; + +struct App { + uteke: Mutex, +} + +impl App { + fn new(mode: PrivacyMode) -> Self { + let mut u = Uteke::open_with_backend(":memory:", None).expect("open without embedder"); + u.set_privacy_config(PrivacyConfig { + mode, + allowlist: Vec::new(), + }); + Self { + uteke: Mutex::new(u), + } + } + + fn call(&self, method: Method, url: &str, body: Option) -> (u16, Value, String) { + let mut req: Request = match body { + Some(b) => { + let leaked: &'static str = Box::leak(b.to_string().into_boxed_str()); + TestRequest::new() + .with_method(method) + .with_path(url) + .with_body(leaked) + .into() + } + None => TestRequest::new().with_method(method).with_path(url).into(), + }; + let ctx = ReqCtx { + auth_token_hash: None, + read_only_token_hash: None, + cors_origins: Vec::new(), + recall_config: None, + extraction_config: None, + host_guard: Default::default(), + store_id: String::new(), + }; + let resp = route(&self.uteke, &ctx, &mut req); + let status = resp.status_code().0; + let text = String::from_utf8(resp.into_reader().into_inner()).unwrap(); + ( + status, + serde_json::from_str(&text).unwrap_or(Value::Null), + text, + ) + } + + fn count(&self) -> usize { + let (_, listed, _) = self.call(Method::Post, "/list", Some(json!({"limit": 50}))); + listed.as_array().map_or(0, Vec::len) + } +} + +fn token() -> String { + format!("{}{}", "ghp_", "aB3dE5gH7jK9mN1pQ3rS5tU7vW9xY1zA3bC5") +} + +#[test] +fn remember_reports_redacted_and_stores_the_masked_text() { + let app = App::new(PrivacyMode::Redact); + let (status, body, _) = app.call( + Method::Post, + "/remember", + Some(json!({"content": format!("deploy key {}", token())})), + ); + assert_eq!(status, 200, "{body}"); + assert_eq!(body["redacted"], true); + let id = body["id"].as_str().unwrap(); + let (_, memory, raw) = app.call(Method::Get, &format!("/memory?id={id}"), None); + assert!( + memory["content"] + .as_str() + .unwrap() + .contains("[REDACTED:github-token]") + ); + assert!( + !raw.contains("aB3dE5gH7jK9"), + "the secret must not come back: {raw}" + ); + assert_eq!(memory["metadata"]["redacted"], true); + + let (_, clean, _) = app.call( + Method::Post, + "/remember", + Some(json!({"content": "nothing secret here"})), + ); + assert_eq!(clean["redacted"], false, "{clean}"); +} + +#[test] +fn strict_mode_rejects_with_400_secret_detected_and_stores_nothing() { + let app = App::new(PrivacyMode::Strict); + let (status, body, raw) = app.call( + Method::Post, + "/remember", + Some(json!({"content": format!("deploy key {}", token())})), + ); + assert_eq!(status, 400, "{body}"); + assert_eq!(body["error"], "secret_detected"); + assert_eq!(body["kinds"], json!(["github-token"])); + assert!(body["message"].as_str().unwrap().contains("github-token")); + assert!( + !raw.contains("aB3dE5gH7jK9"), + "the value must not be echoed: {raw}" + ); + assert_eq!(app.count(), 0); + + // Room remember and update answer the same way. + let (status, body, _) = app.call( + Method::Post, + "/room/remember", + Some(json!({"room_id": "r1", "content": format!("key {}", token())})), + ); + assert_eq!((status, &body["error"]), (400, &json!("secret_detected"))); + let (_, ok, _) = app.call(Method::Post, "/remember", Some(json!({"content": "plain"}))); + let id = ok["id"].as_str().unwrap(); + let (status, body, _) = app.call( + Method::Put, + "/memory", + Some(json!({"id": id, "content": format!("now {}", token())})), + ); + assert_eq!((status, &body["error"]), (400, &json!("secret_detected"))); + assert_eq!(app.count(), 1, "only the plain memory exists"); +} + +#[test] +fn room_remember_reports_redacted() { + let app = App::new(PrivacyMode::Redact); + let (status, body, _) = app.call( + Method::Post, + "/room/remember", + Some(json!({"room_id": "r1", "content": format!("key {}", token())})), + ); + assert_eq!(status, 200, "{body}"); + assert_eq!(body["redacted"], true); +} diff --git a/crates/uteke-server/src/handlers/rooms.rs b/crates/uteke-server/src/handlers/rooms.rs index 81370add..daf5506a 100644 --- a/crates/uteke-server/src/handlers/rooms.rs +++ b/crates/uteke-server/src/handlers/rooms.rs @@ -457,6 +457,7 @@ pub(super) fn post_room_remember(c: &mut Call) -> Response>> { "id": outcome.id, "room_id": req_data.room_id, "embedding_written": outcome.embedding_written, + "redacted": outcome.redacted(), "warning": uteke_core::namespace_deprecation::merge_warning( outcome.warning, uteke_core::namespace_deprecation::namespace_write_warning( @@ -466,6 +467,9 @@ pub(super) fn post_room_remember(c: &mut Call) -> Response>> { }), ) } + Err(uteke_core::Error::SecretDetected { kinds }) => { + ctx.secret_detected_response(req, &kinds) + } Err(e) => { error!("room/remember error: {e}"); let status = match e { diff --git a/docs/configuration.md b/docs/configuration.md index ce7e1757..5a4fba12 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -422,6 +422,14 @@ allowlist = ["^dev-"] # regexes; a value matching any of them is kept as it `UTEKE_PRIVACY_MODE` overrides `mode`. An unknown mode or an invalid regex is logged and replaced by the default (`redact`) for that setting; the filter never turns itself off because of a typo. The filter covers the content of `remember`, room remember, `update` and `import`; metadata, tags and `source` are not filtered yet. +Over HTTP, `POST /remember` and `POST /room/remember` return `"redacted": true` when something was masked (the stored text then differs from the request). In `strict` mode a write that contains a secret is refused with `400`: + +```json +{"error": "secret_detected", "message": "...", "kinds": ["github-token"]} +``` + +`PUT /memory` answers the same way when new content has a secret. `POST /import` reports `redacted` (rows masked) and `rejected_secret` (rows skipped in strict mode, also counted in `skipped`). + ## CLI Flag Override CLI flags always take precedence over config file values: