feat(core): filter tags, metadata strings and source, and report masked updates (#1464, part 2c)

CHANGELOG.md

@@ -8,6 +8,7 @@
 
 ### Added
 
+- **The privacy filter now covers tags, metadata strings and `source` (#1464, part 2c)** - a secret in a tag, in any string inside the metadata or in the `source` is masked (or refused in strict mode) on `remember`, room remember, `update`, `import`, `set_source` and room consolidation, and counted in `redactions`. `update_memory_detailed` returns what an update masked; `PUT /memory` returns `redacted` and `uteke_update` reports it. Importing a structural export (`uteke export --full`) restores a backup as it was and is not filtered.
 - **Privacy filter reporting in the CLI and MCP (#1464, part 2b-2c)** - `uteke remember` prints which kinds of secret were masked and its `--json` output gains `redacted`; `uteke import` reports masked and rejected rows; `uteke_remember` over MCP says so in its result, and in strict mode `uteke_remember` and `uteke_update` fail with a tool error that starts with `secret_detected:` and names the kinds, never the value.
 - **`[privacy]` in `uteke.toml` (#1464, part 2b-2a)** - `mode` (`redact`, the default, `strict` or `off`) and `allowlist` (regexes) now configure the secret filter in the CLI, `uteke-serve` and `uteke-mcp`; `UTEKE_PRIVACY_MODE` overrides the mode. The section is read from the global file only (a project file could otherwise switch the filter off), and a bad value falls back to the default instead of disabling it.
 - **`redacted` on the HTTP write responses and `secret_detected` in strict mode (#1464, part 2b-2b)** - `POST /remember` and `POST /room/remember` return `redacted: true` when the privacy filter masked a secret; a strict-mode write is refused with `400 {"error": "secret_detected", "message", "kinds"}` (also on `PUT /memory`); `POST /import` reports `redacted` and `rejected_secret`. Additive; agreed with cloud in uteke-cloud#218.

crates/uteke-cli/src/commands/remember.rs

@@ -96,6 +96,15 @@ pub(crate) fn run(
         })
         .transpose()?;
 
+    // #1464: the source is stored after the memory, so strict mode refuses a
+    // source with a secret before anything is written.
+    if let Some(src) = source {
+        uteke
+            .privacy_config()
+            .mask_text(src)
+            .map_err(|e| format!("Failed to store memory: {e}"))?;
+    }
+
     // Read from stdin when content argument is "-" (#620).
     // Standard Unix convention: pipe content via echo "text" | uteke remember - --tags "test"
     let content = if content == "-" {

crates/uteke-core/src/consolidate.rs

@@ -73,9 +73,14 @@ impl crate::Uteke {
         crate::validate_input(content, tags)?;
         let ns = namespace.unwrap_or(DEFAULT_NAMESPACE);
         // Privacy filter (#1464), before the content is embedded or stored.
-        let prepared = self.privacy_config.prepare(content, metadata)?;
+        let prepared = self
+            .privacy_config
+            .prepare_with_tags(content, tags, metadata)?;
         let content = prepared.content.as_str();
         let metadata = prepared.metadata.clone();
+        let tag_owned = prepared.tags.clone();
+        let tags: Vec<&str> = tag_owned.iter().map(String::as_str).collect();
+        let tags = tags.as_slice();
 
         // Embed first to check for contradictions before persisting
         self.ensure_embedder()?;

crates/uteke-core/src/consolidation_api.rs

@@ -26,6 +26,22 @@ impl ConsolidationStore for Uteke {
     }
 
     fn insert_memory(&self, memory: &mut Memory) -> Result<(), Error> {
+        // Privacy filter (#1464): the consolidated text comes from an LLM and
+        // is stored like any other write. (The embedding was computed from the
+        // text before this point; it does not carry the secret as text.)
+        let tag_refs: Vec<&str> = memory.tags.iter().map(String::as_str).collect();
+        let prepared = self.privacy_config().prepare_with_tags(
+            &memory.content,
+            &tag_refs,
+            Some(memory.metadata.clone()),
+        )?;
+        memory.content = prepared.content;
+        memory.tags = prepared.tags;
+        if let Some(m) = prepared.metadata {
+            memory.metadata = m;
+        }
+        let redactions = prepared.redactions;
+
         let room_id = memory
             .metadata
             .get("room_id")
@@ -34,6 +50,7 @@ impl ConsolidationStore for Uteke {
 
         // Persist (transactional insert, mirrors crud flow).
         self.store().insert(memory)?;
+        self.record_redaction(&memory.id, &redactions);
 
         // Insert into the vector index so recall finds it; roll back the row
         // if indexing fails so we never leave an un-searchable memory behind.
@@ -205,4 +222,31 @@ mod tests {
             "deprecated memory must not appear in room recall"
         );
     }
+
+    #[test]
+    fn a_consolidated_memory_goes_through_the_privacy_filter() {
+        let mut uteke = Uteke::open(":memory:").unwrap();
+        uteke.set_privacy_config(crate::privacy::PrivacyConfig::default());
+        let token = format!("{}{}", "ghp_", "aB3dE5gH7jK9mN1pQ3rS5tU7vW9xY1zA3bC5");
+        let mut m = mem("m1", &format!("the LLM wrote {token}"));
+        m.metadata = serde_json::json!({"summary": format!("see {token}")});
+        ConsolidationStore::insert_memory(&uteke, &mut m).unwrap();
+        let stored = uteke.store.get_by_id("m1").unwrap().unwrap();
+        assert!(
+            !stored.content.contains("aB3dE5gH7jK9"),
+            "{}",
+            stored.content
+        );
+        assert!(!stored.metadata.to_string().contains("aB3dE5gH7jK9"));
+        assert_eq!(stored.metadata["redacted"], true);
+
+        let mut strict = Uteke::open(":memory:").unwrap();
+        strict.set_privacy_config(crate::privacy::PrivacyConfig {
+            mode: crate::privacy::PrivacyMode::Strict,
+            allowlist: Vec::new(),
+        });
+        let mut m = mem("m2", &format!("the LLM wrote {token}"));
+        assert!(ConsolidationStore::insert_memory(&strict, &mut m).is_err());
+        assert!(strict.store.get_by_id("m2").unwrap().is_none());
+    }
 }

crates/uteke-core/src/import_export.rs

@@ -91,10 +91,17 @@ impl crate::Uteke {
             // Privacy filter (#1464), before the content is embedded or stored.
             // Strict mode skips the row (counted as skipped) instead of
             // aborting the whole import.
-            let prepared = match self
-                .privacy_config
-                .prepare(&entry.content, Some(std::mem::take(&mut entry.metadata)))
-            {
+            // The tags and the source are filtered with the content (the
+            // source rides along as the last "tag" and is taken off again).
+            let mut fields: Vec<&str> = entry.tags.iter().map(String::as_str).collect();
+            if let Some(src) = entry.source.as_deref() {
+                fields.push(src);
+            }
+            let prepared = match self.privacy_config.prepare_with_tags(
+                &entry.content,
+                &fields,
+                Some(std::mem::take(&mut entry.metadata)),
+            ) {
                 Ok(p) => p,
                 Err(e) => {
                     tracing::warn!("Skipping import entry: {e}");
@@ -106,6 +113,11 @@ impl crate::Uteke {
                 }
             };
             entry.content = prepared.content;
+            let mut masked_fields = prepared.tags;
+            if entry.source.is_some() {
+                entry.source = masked_fields.pop();
+            }
+            entry.tags = masked_fields;
             if let Some(m) = prepared.metadata {
                 entry.metadata = m;
             }

crates/uteke-core/src/lib.rs

@@ -37,6 +37,8 @@ mod operations;
 pub mod pack_mode;
 pub mod privacy;
 #[cfg(test)]
+mod privacy_field_tests;
+#[cfg(test)]
 mod privacy_write_tests;
 
 pub use forget_request::{
@@ -47,7 +49,7 @@ pub use list_request::{
     LIST_DEFAULT_LIMIT, LIST_HTTP_DEFAULT_LIMIT, LIST_HTTP_MAX_LIMIT, ListInput, ListPageMeta,
     ListPolicy, ListRequest, ListRequestError,
 };
-pub use operations::RememberOutcome;
+pub use operations::{RememberOutcome, UpdateOutcome};
 pub use recall_request::{
     RECALL_DEFAULT_BUDGET_CHARS, RECALL_DEFAULT_LIMIT, RECALL_DEFAULT_MIN_SCORE,
     RECALL_HTTP_MAX_LIMIT, RECALL_STRICT_MIN_SCORE, RecallInput, RecallPolicy, RecallRequest,
@@ -1437,7 +1439,18 @@ impl Uteke {
         source: Option<&str>,
         source_type: &str,
     ) -> Result<bool, Error> {
-        self.store.set_source(id, source, source_type)
+        // #1464: the source text is filtered like the content. It is set after
+        // the memory is stored, so in strict mode the rejection leaves the
+        // memory without a source.
+        let masked = source
+            .map(|s| self.privacy_config.mask_text(s))
+            .transpose()?;
+        let source = masked.as_ref().map(|(text, _)| text.as_str());
+        let found = self.store.set_source(id, source, source_type)?;
+        if let Some((_, redactions)) = &masked {
+            self.record_redaction(id, redactions);
+        }
+        Ok(found)
     }
 
     /// Set the content hash recorded at write time (#1172 Fase 1).

crates/uteke-core/src/operations.rs

@@ -92,6 +92,36 @@ pub struct RememberOutcome {
     pub redactions: Vec<crate::privacy::RedactionCount>,
 }
 
+/// What an update did (#1464): whether the memory was found and updated, and
+/// what the privacy filter masked in the new content, tags or metadata.
+#[derive(Debug, Clone, Default)]
+pub struct UpdateOutcome {
+    pub updated: bool,
+    pub redactions: Vec<crate::privacy::RedactionCount>,
+}
+
+impl UpdateOutcome {
+    /// Whether the privacy filter masked anything in this update.
+    pub fn redacted(&self) -> bool {
+        !self.redactions.is_empty()
+    }
+
+    /// What was masked, for a human (see [`RememberOutcome::redaction_summary`]).
+    pub fn redaction_summary(&self) -> Option<String> {
+        redaction_summary(&self.redactions)
+    }
+}
+
+fn redaction_summary(redactions: &[crate::privacy::RedactionCount]) -> Option<String> {
+    (!redactions.is_empty()).then(|| {
+        redactions
+            .iter()
+            .map(|r| format!("{} x{}", r.kind, r.count))
+            .collect::<Vec<_>>()
+            .join(", ")
+    })
+}
+
 impl RememberOutcome {
     /// Whether the privacy filter masked anything in the stored content.
     pub fn redacted(&self) -> bool {
@@ -101,13 +131,7 @@ impl RememberOutcome {
     /// What was masked, for a human: `github-token x1, jwt x2`. `None` when
     /// nothing was.
     pub fn redaction_summary(&self) -> Option<String> {
-        self.redacted().then(|| {
-            self.redactions
-                .iter()
-                .map(|r| format!("{} x{}", r.kind, r.count))
-                .collect::<Vec<_>>()
-                .join(", ")
-        })
+        redaction_summary(&self.redactions)
     }
 }
 
@@ -289,9 +313,14 @@ impl crate::Uteke {
         })?;
         // Privacy filter (#1464): before the content is embedded, indexed or
         // stored, so a secret never reaches the vector, FTS5 or the row.
-        let prepared = self.privacy_config.prepare(content, metadata)?;
+        let prepared = self
+            .privacy_config
+            .prepare_with_tags(content, tags, metadata)?;
         let content = prepared.content.as_str();
         let metadata = prepared.metadata.clone();
+        let tag_owned = prepared.tags.clone();
+        let tags: Vec<&str> = tag_owned.iter().map(String::as_str).collect();
+        let tags = tags.as_slice();
         // Detect JSON content and use flattened text for embedding
         let content_type = crate::memory::crud::detect_content_type(content);
         let embed_text = if content_type == "json" {
@@ -1865,7 +1894,9 @@ impl crate::Uteke {
     /// Only provided fields are changed. If `content` is changed, the
     /// embedding is regenerated and the vector index is updated.
     /// Returns `Ok(true)` if the memory was found and updated,
-    /// `Ok(false)` if the memory ID doesn't exist.
+    /// `Ok(false)` if the memory ID doesn't exist. Use
+    /// [`update_memory_detailed`](Self::update_memory_detailed) to also learn
+    /// what the privacy filter masked.
     ///
     /// Acceptance criteria:
     /// - Partial update semantics (only provided fields changed)
@@ -1882,6 +1913,24 @@ impl crate::Uteke {
         pinned: Option<bool>,
         memory_type: Option<&str>,
     ) -> Result<bool, Error> {
+        self.update_memory_detailed(id, content, tags, metadata, importance, pinned, memory_type)
+            .map(|o| o.updated)
+    }
+
+    /// [`update_memory`](Self::update_memory) that returns an [`UpdateOutcome`]:
+    /// whether the memory was updated, and what the privacy filter masked in
+    /// the new content, tags or metadata (#1464).
+    #[allow(clippy::too_many_arguments)]
+    pub fn update_memory_detailed(
+        &self,
+        id: &str,
+        content: Option<&str>,
+        tags: Option<&[String]>,
+        metadata: Option<&serde_json::Value>,
+        importance: Option<f64>,
+        pinned: Option<bool>,
+        memory_type: Option<&str>,
+    ) -> Result<UpdateOutcome, Error> {
         // Validate memory exists
         let existing = self
             .store
@@ -1924,10 +1973,37 @@ impl crate::Uteke {
             Some(c) => Some(self.privacy_config.prepare(c, None)?),
             None => None,
         };
-        let redactions = prepared
-            .as_ref()
-            .map(|p| p.redactions.clone())
-            .unwrap_or_default();
+        // The tags and the metadata strings are filtered too; what they hold
+        // counts with the content's redactions.
+        let (masked_metadata, meta_redactions) = match metadata {
+            Some(m) => {
+                let (v, r) = self.privacy_config.mask_metadata(m)?;
+                (Some(v), r)
+            }
+            None => (None, Vec::new()),
+        };
+        let metadata: Option<&serde_json::Value> = masked_metadata.as_ref();
+        let (masked_tags, tag_redactions) = match tags {
+            Some(t) => {
+                let (v, r) = self.privacy_config.mask_tags(t)?;
+                (Some(v), r)
+            }
+            None => (None, Vec::new()),
+        };
+        let tags: Option<&[String]> = masked_tags.as_deref();
+        let mut merged: std::collections::BTreeMap<&'static str, usize> = Default::default();
+        for r in prepared
+            .iter()
+            .flat_map(|p| p.redactions.iter())
+            .chain(&meta_redactions)
+            .chain(&tag_redactions)
+        {
+            *merged.entry(r.kind).or_default() += r.count;
+        }
+        let redactions: Vec<crate::privacy::RedactionCount> = merged
+            .into_iter()
+            .map(|(kind, count)| crate::privacy::RedactionCount { kind, count })
+            .collect();
         let masked_content: Option<String> = prepared.map(|p| p.content);
         let content: Option<&str> = masked_content.as_deref();
         let flagged_metadata: Option<serde_json::Value> = if redactions.is_empty() {
@@ -1993,7 +2069,7 @@ impl crate::Uteke {
             )?;
 
             if !updated {
-                return Ok(false);
+                return Ok(UpdateOutcome::default());
             }
 
             // Update vector index (insert handles dedup by removing old entry)
@@ -2026,7 +2102,7 @@ impl crate::Uteke {
                 now,
             )?;
             if !updated {
-                return Ok(false);
+                return Ok(UpdateOutcome::default());
             }
         }
 
@@ -2035,7 +2111,10 @@ impl crate::Uteke {
         // Invalidate recall cache for the memory's namespace
         self.recall_cache.invalidate_namespace(&existing.namespace);
 
-        Ok(true)
+        Ok(UpdateOutcome {
+            updated: true,
+            redactions,
+        })
     }
 
     /// Recall memories that existed at a specific point in time.