From f9ec379814eaaf52c5b76bc8c1eb1881f7efd321 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sat, 5 Sep 2026 16:57:51 +0900 Subject: [PATCH 01/47] test(zotero): require bound full-text capture admission --- .../tests/full_text_capture_cli.rs | 41 +++++++++++++++++++ 1 file changed, 41 insertions(+) create mode 100644 crates/conceptweave-zotero/tests/full_text_capture_cli.rs diff --git a/crates/conceptweave-zotero/tests/full_text_capture_cli.rs b/crates/conceptweave-zotero/tests/full_text_capture_cli.rs new file mode 100644 index 00000000..f90a4b7c --- /dev/null +++ b/crates/conceptweave-zotero/tests/full_text_capture_cli.rs @@ -0,0 +1,41 @@ +use std::fs::{self, OpenOptions}; +use std::io::Write; +use std::process::Command; + +#[test] +#[cfg(unix)] +fn full_text_capture_rejects_an_unbound_report_before_creating_output() { + use std::os::unix::fs::OpenOptionsExt; + + let report = conceptweave_zotero::classify_snapshot("9.0.6".into(), None, 0, vec![]); + let input_path = std::env::temp_dir().join(format!( + "conceptweave-full-text-unbound-{}.json", + std::process::id() + )); + let output_path = input_path.with_extension("capture.json"); + let mut input_file = OpenOptions::new() + .write(true) + .create_new(true) + .mode(0o600) + .open(&input_path) + .unwrap(); + input_file + .write_all(&serde_json::to_vec(&report).unwrap()) + .unwrap(); + drop(input_file); + + let command = Command::new(env!("CARGO_BIN_EXE_conceptweave-zotero")) + .arg("--capture-full-text") + .arg(&input_path) + .arg(&output_path) + .output() + .unwrap(); + fs::remove_file(input_path).unwrap(); + + assert!(!command.status.success()); + assert!(!output_path.exists()); + assert!(command.stdout.is_empty()); + assert!(String::from_utf8(command.stderr) + .unwrap() + .contains("full-text capture requires a bound Zotero 10+ report")); +} From 9aafff597a405030af71bfdbc4d4f25af205c6de Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sat, 5 Sep 2026 16:58:07 +0900 Subject: [PATCH 02/47] test(zotero): reproduce environment proxy routing of local requests --- crates/conceptweave-zotero/src/lib.rs | 2 + .../src/tests/proxy_isolation.rs | 146 ++++++++++++++++++ 2 files changed, 148 insertions(+) create mode 100644 crates/conceptweave-zotero/src/tests/proxy_isolation.rs diff --git a/crates/conceptweave-zotero/src/lib.rs b/crates/conceptweave-zotero/src/lib.rs index d231c0ab..9c5b2b93 100644 --- a/crates/conceptweave-zotero/src/lib.rs +++ b/crates/conceptweave-zotero/src/lib.rs @@ -3263,6 +3263,8 @@ fn normalize_title(value: &str) -> Option { #[cfg(test)] mod tests { + mod proxy_isolation; + use super::*; use std::io::{Read, Write}; use std::net::TcpListener; diff --git a/crates/conceptweave-zotero/src/tests/proxy_isolation.rs b/crates/conceptweave-zotero/src/tests/proxy_isolation.rs new file mode 100644 index 00000000..0c88167c --- /dev/null +++ b/crates/conceptweave-zotero/src/tests/proxy_isolation.rs @@ -0,0 +1,146 @@ +use super::*; +use std::process::{Command, Stdio}; +use std::time::Instant; + +const CHILD_CASE: &str = "CONCEPTWEAVE_PROXY_ISOLATION_CASE"; + +fn assert_direct_routing(test_case: &str) { + let mut failures = Vec::new(); + for proxy_variable in [ + "HTTP_PROXY", + "http_proxy", + "HTTPS_PROXY", + "https_proxy", + "ALL_PROXY", + "all_proxy", + ] { + let proxy_listener = TcpListener::bind("127.0.0.1:0").unwrap(); + proxy_listener.set_nonblocking(true).unwrap(); + let proxy_url = format!("http://{}", proxy_listener.local_addr().unwrap()); + // Only the child receives synthetic settings. No process-global environment + // mutation or actual local Zotero endpoint is used by this regression. + let mut child = Command::new(std::env::current_exe().unwrap()) + .args(["--exact", "tests::proxy_isolation::proxy_isolation_child"]) + .env_clear() + .env(CHILD_CASE, test_case) + .env(proxy_variable, proxy_url) + .stdout(Stdio::null()) + .stderr(Stdio::null()) + .spawn() + .unwrap(); + let started = Instant::now(); + let mut proxy_connections = 0; + let child_status = loop { + match proxy_listener.accept() { + Ok((mut stream, _)) => { + proxy_connections += 1; + // Fail a misrouted request promptly without reading its body. + let _ = stream.write_all( + b"HTTP/1.1 502 Bad Gateway\r\nContent-Length: 0\r\nConnection: close\r\n\r\n", + ); + } + Err(error) => assert_eq!(error.kind(), std::io::ErrorKind::WouldBlock), + } + if let Some(status) = child.try_wait().unwrap() { + break status; + } + if started.elapsed() > Duration::from_secs(10) { + child.kill().unwrap(); + child.wait().unwrap(); + panic!("isolated {test_case} routing check timed out"); + } + thread::sleep(Duration::from_millis(5)); + }; + if proxy_connections != 0 || !child_status.success() { + failures.push(format!( + "{proxy_variable}: proxy_connections={proxy_connections}, direct_success={}", + child_status.success() + )); + } + } + assert!(failures.is_empty(), "{test_case}: {failures:?}"); +} + +#[test] +fn snapshot_ignores_proxy_environment() { + assert_direct_routing("snapshot"); +} + +#[test] +fn authorization_ignores_proxy_environment() { + assert_direct_routing("authorization"); +} + +#[test] +fn item_reads_and_authenticated_writes_ignore_proxy_environment() { + assert_direct_routing("item"); +} + +#[test] +fn proxy_isolation_child() { + let Ok(test_case) = std::env::var(CHILD_CASE) else { + return; + }; + match test_case.as_str() { + "snapshot" => { + let response = "HTTP/1.1 200 OK\r\nContent-Length: 2\r\nTotal-Results: 0\r\nLast-Modified-Version: 42\r\nX-Zotero-Version: 10.0.0\r\nZotero-API-Version: 3\r\nZotero-Schema-Version: 42\r\nZotero-Server-ID: server-10\r\nConnection: close\r\n\r\n[]"; + let (base, server) = serve(vec![response]); + *TEST_LOCAL_API.lock().unwrap() = Some(base); + let report = read_local_snapshot(); + *TEST_LOCAL_API.lock().unwrap() = None; + let report = report.unwrap(); + assert_eq!(report.library_version, 42); + assert!(report.classified_items.is_empty()); + let requests = server.join().unwrap(); + assert_eq!(requests.len(), 1); + assert!(requests[0].starts_with( + "GET /api/users/0/items?format=json&include=data&limit=100&start=0 HTTP/1.1\r\n" + )); + assert!(requests[0].contains("zotero-api-version: 3\r\n")); + } + "authorization" => { + let response = authorize_response( + "200 OK", + Some("server-10"), + r#"{"key":"0123456789abcdef0123456789abcdef","remember":true}"#, + ); + let (base, server) = serve(vec![Box::leak(response.into_boxed_str())]); + let authorization = Zotero10LocalAuthorization::request_with_base( + "Synthetic Proxy Isolation Test", + "server-10", + base.replace("/api/users/0/items", ""), + ) + .unwrap(); + assert!(authorization.remembered()); + let requests = server.join().unwrap(); + assert_eq!(requests.len(), 1); + assert!(requests[0].starts_with("POST /api/local/authorize HTTP/1.1\r\n")); + assert!(requests[0].contains("zotero-server-id: server-10\r\n")); + } + "item" => { + let responses = [ + library_response("server-10", 42), + item_response("server-10", 7), + library_response("server-10", 42), + write_response("server-10", 43, 43), + ] + .into_iter() + .map(|response| &*Box::leak(response.into_boxed_str())) + .collect(); + let (base, server) = serve(responses); + let adapter = transport(base); + assert_eq!(adapter.get_item("ABCD2345").unwrap().library_version, 42); + assert_eq!( + adapter.write_item(&write_request()).unwrap().item_version, + 43 + ); + let requests = server.join().unwrap(); + assert_eq!(requests.len(), 4); + assert!(requests[1].starts_with("GET /api/users/0/items/ABCD2345?")); + assert!(requests[3].starts_with("POST /api/users/0/items HTTP/1.1\r\n")); + assert!(requests[3].contains("zotero-api-key: 0123456789abcdef0123456789abcdef\r\n")); + assert!(requests[3].contains("if-unmodified-since-version: 42\r\n")); + } + _ => panic!("unknown synthetic routing case"), + } +} From a2848e59c6a7f4ca7858fcec4052c0cf11862313 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sat, 5 Sep 2026 16:58:45 +0900 Subject: [PATCH 03/47] fix(zotero): bypass environment proxies for both local agents --- crates/conceptweave-zotero/src/lib.rs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/crates/conceptweave-zotero/src/lib.rs b/crates/conceptweave-zotero/src/lib.rs index 9c5b2b93..ea96729d 100644 --- a/crates/conceptweave-zotero/src/lib.rs +++ b/crates/conceptweave-zotero/src/lib.rs @@ -773,6 +773,7 @@ impl Zotero10LocalAdapter { fn local_agent() -> ureq::Agent { let config = ureq::Agent::config_builder() + .proxy(None) .timeout_global(Some(Duration::from_secs(30))) .timeout_connect(Some(Duration::from_secs(2))) .timeout_recv_response(Some(Duration::from_secs(10))) @@ -2732,6 +2733,7 @@ struct FetchedPage { /// ureq transport shim is excluded from deterministic coverage. pub fn read_local_snapshot() -> Result { let config = ureq::Agent::config_builder() + .proxy(None) .timeout_global(Some(Duration::from_secs(60))) .timeout_connect(Some(Duration::from_secs(2))) .timeout_recv_response(Some(Duration::from_secs(10))) From 3d4e2b4442aeadba4a9582db1071af186ef22c8f Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sat, 5 Sep 2026 17:00:43 +0900 Subject: [PATCH 04/47] test(zotero): specify full-text content and provenance capture --- .../src/full_text_capture_tests.rs | 114 ++++++++++++++++++ 1 file changed, 114 insertions(+) create mode 100644 crates/conceptweave-zotero/src/full_text_capture_tests.rs diff --git a/crates/conceptweave-zotero/src/full_text_capture_tests.rs b/crates/conceptweave-zotero/src/full_text_capture_tests.rs new file mode 100644 index 00000000..a73423f0 --- /dev/null +++ b/crates/conceptweave-zotero/src/full_text_capture_tests.rs @@ -0,0 +1,114 @@ +use super::*; +use crate::{ZoteroItem, classify_snapshot}; + +fn report_fixture() -> ClassificationReport { + let items: Vec = serde_json::from_value(serde_json::json!([ + {"key":"ABCD2345","version":2,"data":{"itemType":"journalArticle","title":"fixture paper"}}, + {"key":"BCDE3456","version":1,"data":{"itemType":"attachment","parentItem":"ABCD2345"}}, + {"key":"CDEF4567","version":0,"data":{"itemType":"attachment","parentItem":"ABCD2345"}}, + {"key":"DEFG5678","version":2,"data":{"itemType":"book","title":"no attachment fixture"}} + ])).unwrap(); + let mut report = classify_snapshot("10.0.1".into(), Some("fixture-server".into()), 2, items); + report.api_version = Some(3); + report.schema_version = Some(44); + report +} + +fn response_fixture(request_path: &str) -> CapturedResponse { + let (status, version, body) = match request_path { + "items?limit=1" => (200, Some(2), "[]"), + "fulltext?since=0" => (200, None, r#"{"BCDE3456":12403,"CDEF4567":0}"#), + "items/BCDE3456" => (200, Some(1), r#"{"key":"BCDE3456","version":1,"data":{"itemType":"attachment","parentItem":"ABCD2345"}}"#), + "items/CDEF4567" => (200, Some(0), r#"{"key":"CDEF4567","version":0,"data":{"itemType":"attachment","parentItem":"ABCD2345"}}"#), + "items/BCDE3456/fulltext" => (200, Some(12403), r#"{"content":"fixture text 한글","indexedPages":2,"totalPages":2,"providerExtra":{"retained":true}}"#), + "items/CDEF4567/fulltext" => (404, None, "missing"), + _ => panic!("unexpected fixture request"), + }; + CapturedResponse { status, version, body: body.into() } +} + +#[test] +fn capture_retains_exact_text_missing_results_and_full_parent_denominator() { + let report = report_fixture(); + let old_digest = report.snapshot_digest.clone(); + let mut requests = Vec::new(); + let capture = capture_with(&report, 4096, &mut |request_path, _| { + requests.push(request_path.to_owned()); + Ok(response_fixture(request_path)) + }).unwrap(); + assert_eq!(requests.len(), 8); + assert_eq!(requests.first().unwrap(), "items?limit=1"); + assert_eq!(requests.last().unwrap(), "items?limit=1"); + let saved = serde_json::to_value(&capture).unwrap(); + assert_eq!(saved["capture_evidence"]["capture_kind"], "non_atomic_fulltext_sweep_v1"); + assert_eq!(saved["capture_evidence"]["bibliographic_item_count"], 2); + assert_eq!(saved["capture_evidence"]["records"].as_array().unwrap().len(), 2); + assert_eq!(saved["capture_evidence"]["records"][0]["content_response"]["body"], response_fixture("items/BCDE3456/fulltext").body); + assert_eq!(saved["capture_evidence"]["records"][0]["content_response"]["version"], 12403); + assert_eq!(saved["capture_evidence"]["records"][1]["content_response"]["status"], 404); + assert_eq!(report.snapshot_digest, old_digest); + let restored: FullTextCapture = serde_json::from_value(saved.clone()).unwrap(); + verify_full_text_capture(&restored, &report).unwrap(); + let mut changed = saved; + changed["capture_evidence"]["records"][0]["content_response"]["body"] = "changed text".into(); + let changed: FullTextCapture = serde_json::from_value(changed).unwrap(); + assert!(verify_full_text_capture(&changed, &report).is_err()); +} + +#[test] +fn capture_rejects_unbound_reports_before_any_request() { + let mut report = report_fixture(); + report.server_id = None; + let mut calls = 0; + assert!(capture_with(&report, 4096, &mut |_, _| { calls += 1; unreachable!() }).is_err()); + assert_eq!(calls, 0); +} + +#[test] +fn capture_rejects_foreign_manifest_items_and_duplicate_manifest_keys() { + for body in [r#"{"EFGH6789":0}"#, r#"{"BCDE3456":1,"BCDE3456":2}"#] { + let mut calls = 0; + assert!(capture_with(&report_fixture(), 4096, &mut |request_path, _| { + calls += 1; + let mut response = response_fixture(request_path); + if request_path == "fulltext?since=0" { response.body = body.into(); } + Ok(response) + }).is_err()); + assert_eq!(calls, 2); + } +} + +#[test] +fn capture_rejects_parent_version_status_and_bookend_drift() { + for scenario in 0..5 { + let mut manifest_reads = 0; + let result = capture_with(&report_fixture(), 4096, &mut |request_path, _| { + let mut response = response_fixture(request_path); + if request_path == "items/BCDE3456" { + if scenario == 0 { response.body = response.body.replace("ABCD2345", "DEFG5678"); } + if scenario == 1 { response.version = Some(3); } + } + if request_path == "items/BCDE3456/fulltext" { + if scenario == 2 { response.status = 500; } + if scenario == 3 { response.version = Some(12404); } + } + if request_path == "fulltext?since=0" { + manifest_reads += 1; + if scenario == 4 && manifest_reads == 2 { response.body = "{}".into(); } + } + Ok(response) + }); + assert!(result.is_err(), "drift scenario {scenario}"); + } +} + +#[test] +fn capture_checks_total_budget_before_another_request() { + let mut calls = 0; + assert!(capture_with(&report_fixture(), 2, &mut |request_path, limit| { + calls += 1; + assert_eq!(limit, 2); + Ok(response_fixture(request_path)) + }).is_err()); + assert_eq!(calls, 1); +} From 39b7fe8aa3ceae558c7bef3e3ce7de0156a7aae0 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sat, 5 Sep 2026 17:03:13 +0900 Subject: [PATCH 05/47] test(zotero): activate failing full-text capture contracts --- .../src/full_text_capture.rs | 96 +++++++++++++++ .../src/full_text_capture_tests.rs | 116 +++++++++++++----- crates/conceptweave-zotero/src/lib.rs | 5 + 3 files changed, 189 insertions(+), 28 deletions(-) create mode 100644 crates/conceptweave-zotero/src/full_text_capture.rs diff --git a/crates/conceptweave-zotero/src/full_text_capture.rs b/crates/conceptweave-zotero/src/full_text_capture.rs new file mode 100644 index 00000000..b2cd09a6 --- /dev/null +++ b/crates/conceptweave-zotero/src/full_text_capture.rs @@ -0,0 +1,96 @@ +//! Private, content-bound full-text read sweeps, separate from classification approval. + +use crate::ClassificationReport; +use serde::{Deserialize, Serialize}; +use std::fmt; + +/// A bounded full-text observation artifact, not an atomic snapshot or approval. +/// +/// This contains sensitive source text. Store it only through the owner-only CLI +/// boundary. Deserialization establishes shape; call [`verify_full_text_capture`] +/// before using a restored artifact. Neither operation authenticates a provider. +#[derive(Deserialize, Serialize)] +#[serde(deny_unknown_fields)] +pub struct FullTextCapture { + capture_digest: String, + capture_evidence: CaptureEvidence, +} + +#[derive(Deserialize, Serialize)] +#[serde(deny_unknown_fields)] +struct CaptureEvidence { + capture_kind: String, + metadata_report_digest: String, + metadata_snapshot_digest: String, + bibliographic_item_count: usize, + started_unix_ms: u64, + finished_unix_ms: u64, + library_before: CapturedResponse, + manifest_before: CapturedResponse, + records: Vec, + manifest_after: CapturedResponse, + library_after: CapturedResponse, +} + +#[derive(Deserialize, Serialize)] +#[serde(deny_unknown_fields)] +struct CapturedItem { + item_key: String, + metadata_response: CapturedResponse, + content_response: CapturedResponse, +} + +#[derive(Deserialize, Serialize)] +#[serde(deny_unknown_fields)] +struct CapturedResponse { + status: u16, + version: Option, + body: String, +} + +/// A secret-free capture failure that never embeds an item URL or source content. +#[derive(Debug, PartialEq, Eq)] +pub struct FullTextError(&'static str); + +impl fmt::Display for FullTextError { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter.write_str(self.0) + } +} + +impl std::error::Error for FullTextError {} + +/// Reads every full-text manifest entry through the fixed loopback API. +/// +/// The report remains unchanged. Missing content is retained explicitly, while +/// invalid identity, request failure, drift or exhausted budgets fail the sweep. +pub fn read_local_full_text( + _report: &ClassificationReport, +) -> Result { + Err(FullTextError( + "full-text capture requires a bound Zotero 10+ report", + )) +} + +/// Verifies a restored capture's complete content and original report binding. +/// +/// This detects altered artifacts under an unchanged digest. It does not grant +/// approval, establish atomicity, or authenticate a locally replaced digest. +pub fn verify_full_text_capture( + _capture: &FullTextCapture, + _report: &ClassificationReport, +) -> Result<(), FullTextError> { + Err(FullTextError("full-text capture evidence is invalid")) +} + +fn capture_with( + _report: &ClassificationReport, + _max_bytes: u64, + _fetch: &mut dyn FnMut(&str, u64) -> Result, +) -> Result { + Err(FullTextError("full-text capture evidence is invalid")) +} + +#[cfg(test)] +#[path = "full_text_capture_tests.rs"] +mod tests; diff --git a/crates/conceptweave-zotero/src/full_text_capture_tests.rs b/crates/conceptweave-zotero/src/full_text_capture_tests.rs index a73423f0..9ac1c0e0 100644 --- a/crates/conceptweave-zotero/src/full_text_capture_tests.rs +++ b/crates/conceptweave-zotero/src/full_text_capture_tests.rs @@ -7,7 +7,8 @@ fn report_fixture() -> ClassificationReport { {"key":"BCDE3456","version":1,"data":{"itemType":"attachment","parentItem":"ABCD2345"}}, {"key":"CDEF4567","version":0,"data":{"itemType":"attachment","parentItem":"ABCD2345"}}, {"key":"DEFG5678","version":2,"data":{"itemType":"book","title":"no attachment fixture"}} - ])).unwrap(); + ])) + .unwrap(); let mut report = classify_snapshot("10.0.1".into(), Some("fixture-server".into()), 2, items); report.api_version = Some(3); report.schema_version = Some(44); @@ -18,13 +19,29 @@ fn response_fixture(request_path: &str) -> CapturedResponse { let (status, version, body) = match request_path { "items?limit=1" => (200, Some(2), "[]"), "fulltext?since=0" => (200, None, r#"{"BCDE3456":12403,"CDEF4567":0}"#), - "items/BCDE3456" => (200, Some(1), r#"{"key":"BCDE3456","version":1,"data":{"itemType":"attachment","parentItem":"ABCD2345"}}"#), - "items/CDEF4567" => (200, Some(0), r#"{"key":"CDEF4567","version":0,"data":{"itemType":"attachment","parentItem":"ABCD2345"}}"#), - "items/BCDE3456/fulltext" => (200, Some(12403), r#"{"content":"fixture text 한글","indexedPages":2,"totalPages":2,"providerExtra":{"retained":true}}"#), + "items/BCDE3456" => ( + 200, + Some(1), + r#"{"key":"BCDE3456","version":1,"data":{"itemType":"attachment","parentItem":"ABCD2345"}}"#, + ), + "items/CDEF4567" => ( + 200, + Some(0), + r#"{"key":"CDEF4567","version":0,"data":{"itemType":"attachment","parentItem":"ABCD2345"}}"#, + ), + "items/BCDE3456/fulltext" => ( + 200, + Some(12403), + r#"{"content":"fixture text 한글","indexedPages":2,"totalPages":2,"providerExtra":{"retained":true}}"#, + ), "items/CDEF4567/fulltext" => (404, None, "missing"), _ => panic!("unexpected fixture request"), }; - CapturedResponse { status, version, body: body.into() } + CapturedResponse { + status, + version, + body: body.into(), + } } #[test] @@ -35,17 +52,36 @@ fn capture_retains_exact_text_missing_results_and_full_parent_denominator() { let capture = capture_with(&report, 4096, &mut |request_path, _| { requests.push(request_path.to_owned()); Ok(response_fixture(request_path)) - }).unwrap(); + }) + .unwrap(); assert_eq!(requests.len(), 8); assert_eq!(requests.first().unwrap(), "items?limit=1"); assert_eq!(requests.last().unwrap(), "items?limit=1"); let saved = serde_json::to_value(&capture).unwrap(); - assert_eq!(saved["capture_evidence"]["capture_kind"], "non_atomic_fulltext_sweep_v1"); + assert_eq!( + saved["capture_evidence"]["capture_kind"], + "non_atomic_fulltext_sweep_v1" + ); assert_eq!(saved["capture_evidence"]["bibliographic_item_count"], 2); - assert_eq!(saved["capture_evidence"]["records"].as_array().unwrap().len(), 2); - assert_eq!(saved["capture_evidence"]["records"][0]["content_response"]["body"], response_fixture("items/BCDE3456/fulltext").body); - assert_eq!(saved["capture_evidence"]["records"][0]["content_response"]["version"], 12403); - assert_eq!(saved["capture_evidence"]["records"][1]["content_response"]["status"], 404); + assert_eq!( + saved["capture_evidence"]["records"] + .as_array() + .unwrap() + .len(), + 2 + ); + assert_eq!( + saved["capture_evidence"]["records"][0]["content_response"]["body"], + response_fixture("items/BCDE3456/fulltext").body + ); + assert_eq!( + saved["capture_evidence"]["records"][0]["content_response"]["version"], + 12403 + ); + assert_eq!( + saved["capture_evidence"]["records"][1]["content_response"]["status"], + 404 + ); assert_eq!(report.snapshot_digest, old_digest); let restored: FullTextCapture = serde_json::from_value(saved.clone()).unwrap(); verify_full_text_capture(&restored, &report).unwrap(); @@ -60,7 +96,13 @@ fn capture_rejects_unbound_reports_before_any_request() { let mut report = report_fixture(); report.server_id = None; let mut calls = 0; - assert!(capture_with(&report, 4096, &mut |_, _| { calls += 1; unreachable!() }).is_err()); + assert!( + capture_with(&report, 4096, &mut |_, _| { + calls += 1; + unreachable!() + }) + .is_err() + ); assert_eq!(calls, 0); } @@ -68,12 +110,17 @@ fn capture_rejects_unbound_reports_before_any_request() { fn capture_rejects_foreign_manifest_items_and_duplicate_manifest_keys() { for body in [r#"{"EFGH6789":0}"#, r#"{"BCDE3456":1,"BCDE3456":2}"#] { let mut calls = 0; - assert!(capture_with(&report_fixture(), 4096, &mut |request_path, _| { - calls += 1; - let mut response = response_fixture(request_path); - if request_path == "fulltext?since=0" { response.body = body.into(); } - Ok(response) - }).is_err()); + assert!( + capture_with(&report_fixture(), 4096, &mut |request_path, _| { + calls += 1; + let mut response = response_fixture(request_path); + if request_path == "fulltext?since=0" { + response.body = body.into(); + } + Ok(response) + }) + .is_err() + ); assert_eq!(calls, 2); } } @@ -85,16 +132,26 @@ fn capture_rejects_parent_version_status_and_bookend_drift() { let result = capture_with(&report_fixture(), 4096, &mut |request_path, _| { let mut response = response_fixture(request_path); if request_path == "items/BCDE3456" { - if scenario == 0 { response.body = response.body.replace("ABCD2345", "DEFG5678"); } - if scenario == 1 { response.version = Some(3); } + if scenario == 0 { + response.body = response.body.replace("ABCD2345", "DEFG5678"); + } + if scenario == 1 { + response.version = Some(3); + } } if request_path == "items/BCDE3456/fulltext" { - if scenario == 2 { response.status = 500; } - if scenario == 3 { response.version = Some(12404); } + if scenario == 2 { + response.status = 500; + } + if scenario == 3 { + response.version = Some(12404); + } } if request_path == "fulltext?since=0" { manifest_reads += 1; - if scenario == 4 && manifest_reads == 2 { response.body = "{}".into(); } + if scenario == 4 && manifest_reads == 2 { + response.body = "{}".into(); + } } Ok(response) }); @@ -105,10 +162,13 @@ fn capture_rejects_parent_version_status_and_bookend_drift() { #[test] fn capture_checks_total_budget_before_another_request() { let mut calls = 0; - assert!(capture_with(&report_fixture(), 2, &mut |request_path, limit| { - calls += 1; - assert_eq!(limit, 2); - Ok(response_fixture(request_path)) - }).is_err()); + assert!( + capture_with(&report_fixture(), 2, &mut |request_path, limit| { + calls += 1; + assert_eq!(limit, 2); + Ok(response_fixture(request_path)) + }) + .is_err() + ); assert_eq!(calls, 1); } diff --git a/crates/conceptweave-zotero/src/lib.rs b/crates/conceptweave-zotero/src/lib.rs index ea96729d..c790bfa1 100644 --- a/crates/conceptweave-zotero/src/lib.rs +++ b/crates/conceptweave-zotero/src/lib.rs @@ -9,6 +9,11 @@ use std::collections::{BTreeMap, BTreeSet}; use std::fmt; use std::time::Duration; +mod full_text_capture; +pub use full_text_capture::{ + FullTextCapture, FullTextError, read_local_full_text, verify_full_text_capture, +}; + /// Classification rule revision recorded in every report. pub const RULE_REVISION: &str = "ontology-research-v2"; From 5f36ff5ffa942f0a08778ccecf55b816c75aeae4 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sat, 5 Sep 2026 17:15:11 +0900 Subject: [PATCH 06/47] feat(zotero): retain report-bound full-text observations --- .../src/full_text_capture.rs | 327 +++++++++++++++++- crates/conceptweave-zotero/src/main.rs | 32 +- .../tests/full_text_capture_cli.rs | 8 +- 3 files changed, 349 insertions(+), 18 deletions(-) diff --git a/crates/conceptweave-zotero/src/full_text_capture.rs b/crates/conceptweave-zotero/src/full_text_capture.rs index b2cd09a6..6980ec02 100644 --- a/crates/conceptweave-zotero/src/full_text_capture.rs +++ b/crates/conceptweave-zotero/src/full_text_capture.rs @@ -1,8 +1,20 @@ //! Private, content-bound full-text read sweeps, separate from classification approval. -use crate::ClassificationReport; +use crate::{ + ClassificationReport, MAX_PAGE_BYTES, MAX_SNAPSHOT_BYTES, MAX_SNAPSHOT_ITEMS, + SnapshotItemRevision, ZoteroItem, bounded_body_with_limit, build_steward_review_worksheet, + local_agent, validate_item_key, verify_server_id, +}; use serde::{Deserialize, Serialize}; +use sha2::{Digest, Sha256}; +use std::collections::BTreeMap; use std::fmt; +use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH}; + +const CAPTURE_KIND: &str = "non_atomic_fulltext_sweep_v1"; +const INVALID_EVIDENCE: FullTextError = FullTextError("full-text capture evidence is invalid"); +const BUDGET_EXCEEDED: FullTextError = FullTextError("full-text capture budget exceeded"); +const CAPTURE_DEADLINE: Duration = Duration::from_secs(300); /// A bounded full-text observation artifact, not an atomic snapshot or approval. /// @@ -65,11 +77,12 @@ impl std::error::Error for FullTextError {} /// The report remains unchanged. Missing content is retained explicitly, while /// invalid identity, request failure, drift or exhausted budgets fail the sweep. pub fn read_local_full_text( - _report: &ClassificationReport, + report: &ClassificationReport, ) -> Result { - Err(FullTextError( - "full-text capture requires a bound Zotero 10+ report", - )) + let agent = local_agent(); + capture_with(report, MAX_SNAPSHOT_BYTES, &mut |request_path, limit| { + fetch_response(&agent, report, crate::LOCAL_API_ROOT, request_path, limit) + }) } /// Verifies a restored capture's complete content and original report binding. @@ -77,18 +90,308 @@ pub fn read_local_full_text( /// This detects altered artifacts under an unchanged digest. It does not grant /// approval, establish atomicity, or authenticate a locally replaced digest. pub fn verify_full_text_capture( - _capture: &FullTextCapture, - _report: &ClassificationReport, + capture: &FullTextCapture, + report: &ClassificationReport, ) -> Result<(), FullTextError> { - Err(FullTextError("full-text capture evidence is invalid")) + let snapshot = validate_report(report)?; + let evidence = &capture.capture_evidence; + if capture.capture_digest != json_digest(evidence) + || evidence.capture_kind != CAPTURE_KIND + || evidence.metadata_report_digest != json_digest(report) + || evidence.metadata_snapshot_digest != report.snapshot_digest + || evidence.bibliographic_item_count != report.classified_items.len() + || evidence.finished_unix_ms < evidence.started_unix_ms + { + return Err(INVALID_EVIDENCE); + } + validate_library(&evidence.library_before, report)?; + validate_library(&evidence.library_after, report)?; + let manifest = parse_manifest(&evidence.manifest_before, &snapshot)?; + if evidence.manifest_after.status != 200 + || evidence.manifest_after.body != evidence.manifest_before.body + || evidence.records.len() != manifest.len() + { + return Err(INVALID_EVIDENCE); + } + let mut remaining = MAX_SNAPSHOT_BYTES; + for response in [ + &evidence.library_before, + &evidence.manifest_before, + &evidence.manifest_after, + &evidence.library_after, + ] { + account_body(&mut remaining, response)?; + } + for (record, (item_key, version)) in evidence.records.iter().zip(&manifest) { + if &record.item_key != item_key { + return Err(INVALID_EVIDENCE); + } + validate_metadata(&record.metadata_response, snapshot[item_key.as_str()])?; + validate_content(&record.content_response, *version)?; + account_body(&mut remaining, &record.metadata_response)?; + account_body(&mut remaining, &record.content_response)?; + } + Ok(()) } fn capture_with( - _report: &ClassificationReport, - _max_bytes: u64, - _fetch: &mut dyn FnMut(&str, u64) -> Result, + report: &ClassificationReport, + max_bytes: u64, + fetch: &mut dyn FnMut(&str, u64) -> Result, ) -> Result { - Err(FullTextError("full-text capture evidence is invalid")) + let snapshot = validate_report(report)?; + let started_unix_ms = unix_millis(SystemTime::now())?; + let started = Instant::now(); + let mut remaining = max_bytes.min(MAX_SNAPSHOT_BYTES); + let mut read = |request_path: &str| { + check_admission(remaining, started.elapsed())?; + let response = fetch(request_path, remaining.min(MAX_PAGE_BYTES))?; + account_body(&mut remaining, &response)?; + check_deadline(started.elapsed())?; + Ok::<_, FullTextError>(response) + }; + let library_before = read("items?limit=1")?; + validate_library(&library_before, report)?; + let manifest_before = read("fulltext?since=0")?; + let manifest = parse_manifest(&manifest_before, &snapshot)?; + let mut records = Vec::with_capacity(manifest.len()); + for (item_key, version) in manifest { + let metadata_response = read(&format!("items/{item_key}"))?; + validate_metadata(&metadata_response, snapshot[item_key.as_str()])?; + let content_response = read(&format!("items/{item_key}/fulltext"))?; + validate_content(&content_response, version)?; + records.push(CapturedItem { + item_key, + metadata_response, + content_response, + }); + } + let manifest_after = read("fulltext?since=0")?; + let library_after = read("items?limit=1")?; + let capture_evidence = CaptureEvidence { + capture_kind: CAPTURE_KIND.into(), + metadata_report_digest: json_digest(report), + metadata_snapshot_digest: report.snapshot_digest.clone(), + bibliographic_item_count: report.classified_items.len(), + started_unix_ms, + finished_unix_ms: unix_millis(SystemTime::now())?, + library_before, + manifest_before, + records, + manifest_after, + library_after, + }; + let capture = FullTextCapture { + capture_digest: json_digest(&capture_evidence), + capture_evidence, + }; + verify_full_text_capture(&capture, report)?; + check_deadline(started.elapsed())?; + Ok(capture) +} + +fn validate_report( + report: &ClassificationReport, +) -> Result, FullTextError> { + let valid = report.api_version == Some(3) + && report.schema_version.is_some() + && report + .server_id + .as_deref() + .is_some_and(|server| !server.trim().is_empty()) + && report + .zotero_version + .split('.') + .next() + .and_then(|major| major.parse::().ok()) + .is_some_and(|major| major >= 10) + && !report.classified_items.is_empty() + && report.observed_item_count <= MAX_SNAPSHOT_ITEMS + && build_steward_review_worksheet(report).is_ok(); + if !valid { + return Err(FullTextError( + "full-text capture requires a bound Zotero 10+ report", + )); + } + let mut snapshot = BTreeMap::new(); + for item in &report.snapshot_items { + validate_item_key(&item.item_key).map_err(|_| INVALID_EVIDENCE)?; + snapshot.insert(item.item_key.as_str(), item); + } + Ok(snapshot) +} + +fn validate_library( + response: &CapturedResponse, + report: &ClassificationReport, +) -> Result<(), FullTextError> { + if response.status != 200 || response.version != Some(report.library_version) { + return Err(INVALID_EVIDENCE); + } + let _: Vec = + serde_json::from_str(&response.body).map_err(|_| INVALID_EVIDENCE)?; + Ok(()) +} + +fn parse_manifest( + response: &CapturedResponse, + snapshot: &BTreeMap<&str, &SnapshotItemRevision>, +) -> Result, FullTextError> { + struct UniqueManifest; + impl<'de> serde::de::Visitor<'de> for UniqueManifest { + type Value = BTreeMap; + fn expecting(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter.write_str("a unique full-text manifest") + } + fn visit_map>( + self, + mut map: M, + ) -> Result { + let mut entries = BTreeMap::new(); + while let Some((key, value)) = map.next_entry::()? { + if entries.len() == MAX_SNAPSHOT_ITEMS || entries.insert(key, value).is_some() { + return Err(serde::de::Error::custom("invalid manifest entries")); + } + } + Ok(entries) + } + } + if response.status != 200 { + return Err(INVALID_EVIDENCE); + } + let mut deserializer = serde_json::Deserializer::from_str(&response.body); + let manifest = serde::Deserializer::deserialize_map(&mut deserializer, UniqueManifest) + .map_err(|_| INVALID_EVIDENCE)?; + deserializer.end().map_err(|_| INVALID_EVIDENCE)?; + if manifest + .keys() + .any(|key| !snapshot.contains_key(key.as_str())) + { + return Err(INVALID_EVIDENCE); + } + Ok(manifest) +} + +fn validate_metadata( + response: &CapturedResponse, + expected: &SnapshotItemRevision, +) -> Result<(), FullTextError> { + if response.status != 200 || response.version != Some(expected.item_version) { + return Err(INVALID_EVIDENCE); + } + let item: ZoteroItem = serde_json::from_str(&response.body).map_err(|_| INVALID_EVIDENCE)?; + if item.key != expected.item_key + || item.version != expected.item_version + || item.data.item_type != "attachment" + || item.data.parent_item != expected.parent_item_key.as_deref().unwrap_or("") + { + return Err(INVALID_EVIDENCE); + } + Ok(()) +} + +fn validate_content(response: &CapturedResponse, version: u64) -> Result<(), FullTextError> { + match response.status { + 404 => Ok(()), + 200 if response.version == Some(version) => { + #[derive(Deserialize)] + struct ContentProjection { + content: String, + } + let content: ContentProjection = + serde_json::from_str(&response.body).map_err(|_| INVALID_EVIDENCE)?; + let _ = content.content; + Ok(()) + } + _ => Err(INVALID_EVIDENCE), + } +} + +fn account_body(remaining: &mut u64, response: &CapturedResponse) -> Result<(), FullTextError> { + let body_bytes = response.body.len() as u64; + if body_bytes > MAX_PAGE_BYTES { + return Err(BUDGET_EXCEEDED); + } + *remaining = remaining.checked_sub(body_bytes).ok_or(BUDGET_EXCEEDED)?; + Ok(()) +} + +fn check_admission(remaining: u64, elapsed: Duration) -> Result<(), FullTextError> { + if remaining == 0 { + return Err(BUDGET_EXCEEDED); + } + check_deadline(elapsed) +} + +fn check_deadline(elapsed: Duration) -> Result<(), FullTextError> { + if elapsed >= CAPTURE_DEADLINE { + Err(BUDGET_EXCEEDED) + } else { + Ok(()) + } +} + +fn unix_millis(time: SystemTime) -> Result { + time.duration_since(UNIX_EPOCH) + .ok() + .and_then(|value| value.as_millis().try_into().ok()) + .ok_or(INVALID_EVIDENCE) +} + +fn json_digest(value: &impl Serialize) -> String { + let bytes = serde_json::to_vec(value).expect("capture values are JSON-compatible"); + format!("sha256:{:x}", Sha256::digest(bytes)) +} + +fn fetch_response( + agent: &ureq::Agent, + report: &ClassificationReport, + api_root: &str, + request_path: &str, + limit: u64, +) -> Result { + let mut response = agent + .get(&format!("{api_root}/api/users/0/{request_path}")) + .header("Zotero-API-Version", "3") + .header( + "Zotero-Server-ID", + report.server_id.as_deref().ok_or(INVALID_EVIDENCE)?, + ) + .call() + .map_err(|_| FullTextError("full-text local request failed"))?; + let headers = response.headers(); + verify_server_id( + headers, + report.server_id.as_deref().ok_or(INVALID_EVIDENCE)?, + ) + .map_err(|_| INVALID_EVIDENCE)?; + for (header, expected) in [ + ("Zotero-API-Version", "3".to_owned()), + ( + "Zotero-Schema-Version", + report.schema_version.ok_or(INVALID_EVIDENCE)?.to_string(), + ), + ("X-Zotero-Version", report.zotero_version.clone()), + ] { + if headers.get(header).and_then(|value| value.to_str().ok()) != Some(expected.as_str()) { + return Err(INVALID_EVIDENCE); + } + } + let version = headers + .get("Last-Modified-Version") + .map(|value| { + value + .to_str() + .ok() + .and_then(|text| text.parse::().ok()) + .ok_or(INVALID_EVIDENCE) + }) + .transpose()?; + Ok(CapturedResponse { + status: response.status().as_u16(), + version, + body: bounded_body_with_limit(&mut response, limit).map_err(|_| INVALID_EVIDENCE)?, + }) } #[cfg(test)] diff --git a/crates/conceptweave-zotero/src/main.rs b/crates/conceptweave-zotero/src/main.rs index 5912ab5c..20c95d1b 100644 --- a/crates/conceptweave-zotero/src/main.rs +++ b/crates/conceptweave-zotero/src/main.rs @@ -5,7 +5,8 @@ use conceptweave_zotero::{ ClassificationReport, GoldenSetApproval, MAX_REVIEW_BATCH_ITEMS, StewardDecisionPatch, StewardReviewBatch, StewardReviewWorksheet, apply_steward_decision_patch, assess_steward_review_progress, build_steward_review_batch, build_steward_review_worksheet, - decision_patch_from_review_batch, read_local_snapshot, reviewed_golden_set_from_worksheet, + decision_patch_from_review_batch, read_local_full_text, read_local_snapshot, + reviewed_golden_set_from_worksheet, }; use serde::de::DeserializeOwned; use std::collections::BTreeSet; @@ -20,6 +21,10 @@ const MAX_ARTIFACT_BYTES: u64 = 16 * 1024 * 1024; #[derive(Debug, PartialEq, Eq)] enum OutputRequest { Report(String), + FullTextCapture { + report: String, + output: String, + }, Worksheet { report: String, worksheet: String, @@ -63,7 +68,18 @@ where { let mut args = args.into_iter().map(Into::into); let first = args.next().ok_or(USAGE)?; - let request = if first == "--worksheet" { + let request = if first == "--capture-full-text" { + let report = args + .next() + .ok_or("--capture-full-text requires report and output paths")?; + let output = args + .next() + .ok_or("--capture-full-text requires report and output paths")?; + if report == output { + return Err("full-text report and output paths must differ"); + } + OutputRequest::FullTextCapture { report, output } + } else if first == "--worksheet" { let report = args .next() .ok_or("--worksheet requires report and worksheet output paths")?; @@ -357,7 +373,7 @@ fn write_all_and_flush(writer: &mut BufWriter, content: &[u8]) -> io::Resu fn write_private_output_with( path: &Path, content: &[u8], - write: fn(&mut BufWriter, &[u8]) -> io::Result<()>, + write: impl FnOnce(&mut BufWriter, &[u8]) -> io::Result<()>, ) -> io::Result<()> { let file = create_report_file(path)?; let mut writer = BufWriter::new(file); @@ -463,6 +479,16 @@ fn create_report_file_with( /// Reads one Zotero snapshot and writes its sensitive local proposal report. fn main() -> Result<(), Box> { match parse_output_request(env::args().skip(1))? { + OutputRequest::FullTextCapture { report, output } => { + let output = validate_output_path(&output)?; + let (report, _): (ClassificationReport, _) = + read_private_json(&report).map_err(|error| label_input("report", error))?; + let capture = read_local_full_text(&report)?; + write_private_output_with(&output, &[], |writer, _| { + serde_json::to_writer(&mut *writer, &capture).map_err(io::Error::other)?; + writer.flush() + })?; + } OutputRequest::Report(output) => { let output = validate_output_path(&output)?; let report = read_local_snapshot()?; diff --git a/crates/conceptweave-zotero/tests/full_text_capture_cli.rs b/crates/conceptweave-zotero/tests/full_text_capture_cli.rs index f90a4b7c..a0188a87 100644 --- a/crates/conceptweave-zotero/tests/full_text_capture_cli.rs +++ b/crates/conceptweave-zotero/tests/full_text_capture_cli.rs @@ -35,7 +35,9 @@ fn full_text_capture_rejects_an_unbound_report_before_creating_output() { assert!(!command.status.success()); assert!(!output_path.exists()); assert!(command.stdout.is_empty()); - assert!(String::from_utf8(command.stderr) - .unwrap() - .contains("full-text capture requires a bound Zotero 10+ report")); + assert!( + String::from_utf8(command.stderr) + .unwrap() + .contains("full-text capture requires a bound Zotero 10+ report") + ); } From 53efefb9211984d2a1e579ebe750224241065d58 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sat, 5 Sep 2026 17:18:33 +0900 Subject: [PATCH 07/47] test(zotero): exercise full-text replay and admission failures --- .../src/full_text_capture_tests.rs | 198 ++++++++++++++++++ crates/conceptweave-zotero/src/main.rs | 17 ++ 2 files changed, 215 insertions(+) diff --git a/crates/conceptweave-zotero/src/full_text_capture_tests.rs b/crates/conceptweave-zotero/src/full_text_capture_tests.rs index 9ac1c0e0..6117f336 100644 --- a/crates/conceptweave-zotero/src/full_text_capture_tests.rs +++ b/crates/conceptweave-zotero/src/full_text_capture_tests.rs @@ -172,3 +172,201 @@ fn capture_checks_total_budget_before_another_request() { ); assert_eq!(calls, 1); } + +#[test] +fn capture_validates_each_report_admission_condition() { + for scenario in 0..10 { + let mut report = report_fixture(); + match scenario { + 0 => report.api_version = None, + 1 => report.schema_version = None, + 2 => report.server_id = Some(" ".into()), + 3 => report.zotero_version = "9.0.6".into(), + 4 => report.zotero_version = "invalid".into(), + 5 => report.classified_items.clear(), + 6 => report.observed_item_count = MAX_SNAPSHOT_ITEMS + 1, + 7 => report.snapshot_digest.clear(), + 8 => report.snapshot_items[0].item_version += 1, + _ => { + let items = serde_json::from_value(serde_json::json!([ + {"key":"INVALID0","version":2,"data":{"itemType":"book","title":"fixture"}} + ])) + .unwrap(); + report = + classify_snapshot("10.0.1".into(), Some("fixture-server".into()), 2, items); + report.api_version = Some(3); + report.schema_version = Some(44); + } + } + let result = capture_with(&report, 4096, &mut |_, _| { + panic!("admission must precede requests") + }); + assert!(result.is_err(), "admission scenario {scenario}"); + } +} + +#[test] +fn replay_checks_bindings_and_structure_even_when_digest_is_recomputed() { + let report = report_fixture(); + let capture = capture_with(&report, 4096, &mut |request_path, _| { + Ok(response_fixture(request_path)) + }) + .unwrap(); + let saved = serde_json::to_value(capture).unwrap(); + for scenario in 0..15 { + let mut restored: FullTextCapture = serde_json::from_value(saved.clone()).unwrap(); + let evidence = &mut restored.capture_evidence; + match scenario { + 0 => evidence.capture_kind = "snapshot".into(), + 1 => evidence.metadata_report_digest.clear(), + 2 => evidence.metadata_snapshot_digest.clear(), + 3 => evidence.bibliographic_item_count = 1, + 4 => evidence.finished_unix_ms = 0, + 5 => evidence.library_before.status = 500, + 6 => evidence.library_after.version = Some(3), + 7 => evidence.library_after.body = "null".into(), + 8 => evidence.manifest_after.status = 500, + 9 => evidence.manifest_after.body = "{}".into(), + 10 => { + evidence.records.pop(); + } + 11 => evidence.records.swap(0, 1), + 12 => evidence.records[0].metadata_response.body = "{}".into(), + 13 => evidence.records[0].content_response.body = r#"{"content":null}"#.into(), + _ => evidence.records[0].content_response.version = None, + } + restored.capture_digest = json_digest(evidence); + assert!( + verify_full_text_capture(&restored, &report).is_err(), + "replay scenario {scenario}" + ); + } + let mut another_report = report_fixture(); + another_report.schema_version = Some(45); + let restored: FullTextCapture = serde_json::from_value(saved).unwrap(); + assert!(verify_full_text_capture(&restored, &another_report).is_err()); +} + +#[test] +fn malformed_and_oversized_manifests_never_become_partial_captures() { + let report = report_fixture(); + let snapshot = validate_report(&report).unwrap(); + for body in [ + "[]", + "not json", + r#"{"BCDE3456":-1}"#, + r#"{"BCDE3456":1.5}"#, + r#"{"BCDE3456":0} {}"#, + ] { + let response = CapturedResponse { + status: 200, + version: None, + body: body.into(), + }; + assert!(parse_manifest(&response, &snapshot).is_err()); + } + let response = CapturedResponse { + status: 503, + version: None, + body: "{}".into(), + }; + assert!(parse_manifest(&response, &snapshot).is_err()); + let entries: BTreeMap<_, _> = (0..=MAX_SNAPSHOT_ITEMS) + .map(|index| (format!("{index:08}"), 0)) + .collect(); + let response = CapturedResponse { + status: 200, + version: None, + body: serde_json::to_string(&entries).unwrap(), + }; + assert!(parse_manifest(&response, &snapshot).is_err()); + assert!( + capture_with(&report, 4096, &mut |_, _| Err(FullTextError( + "fixture request failed" + ))) + .is_err() + ); +} + +#[test] +fn metadata_requires_attachment_identity_revision_and_parent() { + let report = report_fixture(); + let snapshot = validate_report(&report).unwrap(); + let expected = snapshot["BCDE3456"]; + for scenario in 0..5 { + let mut response = response_fixture("items/BCDE3456"); + let mut body: serde_json::Value = serde_json::from_str(&response.body).unwrap(); + match scenario { + 0 => response.status = 404, + 1 => body["key"] = "CDEF4567".into(), + 2 => body["version"] = 2.into(), + 3 => body["data"]["itemType"] = "note".into(), + _ => body["data"]["parentItem"] = "DEFG5678".into(), + } + response.body = serde_json::to_string(&body).unwrap(); + assert!(validate_metadata(&response, expected).is_err()); + } + let response = CapturedResponse { + status: 200, + version: Some(1), + body: r#"{"key":"BCDE3456","version":1,"data":{"itemType":"attachment"}}"#.into(), + }; + let expected = SnapshotItemRevision { + item_key: "BCDE3456".into(), + item_version: 1, + parent_item_key: None, + }; + validate_metadata(&response, &expected).unwrap(); +} + +#[test] +fn empty_partial_and_unknown_content_counters_are_retained_without_approval() { + for body in [ + r#"{"content":""}"#, + r#"{"content":"text","indexedPages":1,"totalPages":2}"#, + r#"{"content":"text","indexedChars":null,"totalChars":null}"#, + ] { + let capture = capture_with(&report_fixture(), 4096, &mut |request_path, _| { + let mut response = response_fixture(request_path); + if request_path == "items/BCDE3456/fulltext" { + response.body = body.into(); + } + Ok(response) + }) + .unwrap(); + assert_eq!( + capture.capture_evidence.records[0].content_response.body, + body + ); + } +} + +#[test] +fn byte_and_clock_limits_include_exact_boundary_and_overflow_failures() { + let small = response_fixture("items?limit=1"); + assert!(account_body(&mut 1, &small).is_err()); + let mut exact_remaining = 2; + account_body(&mut exact_remaining, &small).unwrap(); + assert_eq!(exact_remaining, 0); + let oversized = CapturedResponse { + status: 404, + version: None, + body: "x".repeat(MAX_PAGE_BYTES as usize + 1), + }; + assert!(account_body(&mut MAX_SNAPSHOT_BYTES.clone(), &oversized).is_err()); + assert!(check_admission(0, Duration::ZERO).is_err()); + assert!(check_admission(1, CAPTURE_DEADLINE).is_err()); + check_admission(1, CAPTURE_DEADLINE - Duration::from_nanos(1)).unwrap(); + assert!(unix_millis(UNIX_EPOCH - Duration::from_secs(1)).is_err()); + assert_eq!(unix_millis(UNIX_EPOCH).unwrap(), 0); + let max_time = UNIX_EPOCH + .checked_add(Duration::from_secs(u64::MAX / 1000 + 1)) + .unwrap(); + assert!(unix_millis(max_time).is_err()); + let message = FullTextError("fixture static error"); + assert_eq!(message.to_string(), "fixture static error"); + assert_eq!( + format!("{message:?}"), + "FullTextError(\"fixture static error\")" + ); +} diff --git a/crates/conceptweave-zotero/src/main.rs b/crates/conceptweave-zotero/src/main.rs index 20c95d1b..a07e6b91 100644 --- a/crates/conceptweave-zotero/src/main.rs +++ b/crates/conceptweave-zotero/src/main.rs @@ -634,6 +634,23 @@ fn main() -> Result<(), Box> { mod tests { use super::*; + #[test] + fn full_text_mode_requires_two_distinct_artifact_paths() { + let report = "/tmp/report.json"; + let output = "/tmp/full-text.json"; + assert_eq!( + parse_output_request(["--capture-full-text", report, output]), + Ok(OutputRequest::FullTextCapture { + report: report.into(), + output: output.into(), + }) + ); + assert!(parse_output_request(["--capture-full-text"]).is_err()); + assert!(parse_output_request(["--capture-full-text", report]).is_err()); + assert!(parse_output_request(["--capture-full-text", report, report]).is_err()); + assert!(parse_output_request(["--capture-full-text", report, output, "extra"]).is_err()); + } + #[test] fn worksheet_mode_is_explicit_and_rejects_ambiguous_arguments() { let report = "/tmp/conceptweave-zotero-report.json"; From bcf787e3bbc246e7bdf17738d7be5d6a58564ddb Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sat, 5 Sep 2026 17:19:31 +0900 Subject: [PATCH 08/47] fix(zotero): expose the capture command in usage --- crates/conceptweave-zotero/src/full_text_capture_tests.rs | 3 ++- crates/conceptweave-zotero/src/main.rs | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/crates/conceptweave-zotero/src/full_text_capture_tests.rs b/crates/conceptweave-zotero/src/full_text_capture_tests.rs index 6117f336..c644d7f7 100644 --- a/crates/conceptweave-zotero/src/full_text_capture_tests.rs +++ b/crates/conceptweave-zotero/src/full_text_capture_tests.rs @@ -353,7 +353,8 @@ fn byte_and_clock_limits_include_exact_boundary_and_overflow_failures() { version: None, body: "x".repeat(MAX_PAGE_BYTES as usize + 1), }; - assert!(account_body(&mut MAX_SNAPSHOT_BYTES.clone(), &oversized).is_err()); + let mut remaining = MAX_SNAPSHOT_BYTES; + assert!(account_body(&mut remaining, &oversized).is_err()); assert!(check_admission(0, Duration::ZERO).is_err()); assert!(check_admission(1, CAPTURE_DEADLINE).is_err()); check_admission(1, CAPTURE_DEADLINE - Duration::from_nanos(1)).unwrap(); diff --git a/crates/conceptweave-zotero/src/main.rs b/crates/conceptweave-zotero/src/main.rs index a07e6b91..58789fab 100644 --- a/crates/conceptweave-zotero/src/main.rs +++ b/crates/conceptweave-zotero/src/main.rs @@ -15,7 +15,7 @@ use std::fs::{self, File, OpenOptions}; use std::io::{self, BufWriter, Read, Write}; use std::path::{Path, PathBuf}; -const USAGE: &str = "usage: conceptweave-zotero /tmp/REPORT.json | --worksheet /tmp/REPORT.json /tmp/WORKSHEET.json | --review-progress /tmp/REPORT.json /tmp/WORKSHEET.json /tmp/PROGRESS.json | --review-batch /tmp/REPORT.json /tmp/CURRENT_WORKSHEET.json LIMIT /tmp/BATCH.json | --apply-review-batch /tmp/REPORT.json /tmp/CURRENT_WORKSHEET.json /tmp/COMPLETED_BATCH.json /tmp/UPDATED_WORKSHEET.json | --apply-decision-patch /tmp/REPORT.json /tmp/CURRENT_WORKSHEET.json /tmp/PATCH.json /tmp/UPDATED_WORKSHEET.json | --finalize /tmp/REPORT.json /tmp/WORKSHEET.json /tmp/APPROVAL.json /tmp/GOLDEN.json"; +const USAGE: &str = "usage: conceptweave-zotero /tmp/REPORT.json | --capture-full-text /tmp/REPORT.json /tmp/CAPTURE.json | --worksheet /tmp/REPORT.json /tmp/WORKSHEET.json | --review-progress /tmp/REPORT.json /tmp/WORKSHEET.json /tmp/PROGRESS.json | --review-batch /tmp/REPORT.json /tmp/CURRENT_WORKSHEET.json LIMIT /tmp/BATCH.json | --apply-review-batch /tmp/REPORT.json /tmp/CURRENT_WORKSHEET.json /tmp/COMPLETED_BATCH.json /tmp/UPDATED_WORKSHEET.json | --apply-decision-patch /tmp/REPORT.json /tmp/CURRENT_WORKSHEET.json /tmp/PATCH.json /tmp/UPDATED_WORKSHEET.json | --finalize /tmp/REPORT.json /tmp/WORKSHEET.json /tmp/APPROVAL.json /tmp/GOLDEN.json"; const MAX_ARTIFACT_BYTES: u64 = 16 * 1024 * 1024; #[derive(Debug, PartialEq, Eq)] From f3a2847c9dca2335ecc4ada0d8f8b9660b91a9bd Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sat, 5 Sep 2026 17:20:30 +0900 Subject: [PATCH 09/47] test(zotero): verify full-text HTTP transport boundaries --- .../src/full_text_capture.rs | 13 +- .../src/full_text_capture_transport_tests.rs | 258 ++++++++++++++++++ 2 files changed, 270 insertions(+), 1 deletion(-) create mode 100644 crates/conceptweave-zotero/src/full_text_capture_transport_tests.rs diff --git a/crates/conceptweave-zotero/src/full_text_capture.rs b/crates/conceptweave-zotero/src/full_text_capture.rs index 6980ec02..e98f82d8 100644 --- a/crates/conceptweave-zotero/src/full_text_capture.rs +++ b/crates/conceptweave-zotero/src/full_text_capture.rs @@ -78,10 +78,17 @@ impl std::error::Error for FullTextError {} /// invalid identity, request failure, drift or exhausted budgets fail the sweep. pub fn read_local_full_text( report: &ClassificationReport, +) -> Result { + read_full_text_from_api(report, crate::LOCAL_API_ROOT) +} + +fn read_full_text_from_api( + report: &ClassificationReport, + api_root: &str, ) -> Result { let agent = local_agent(); capture_with(report, MAX_SNAPSHOT_BYTES, &mut |request_path, limit| { - fetch_response(&agent, report, crate::LOCAL_API_ROOT, request_path, limit) + fetch_response(&agent, report, api_root, request_path, limit) }) } @@ -397,3 +404,7 @@ fn fetch_response( #[cfg(test)] #[path = "full_text_capture_tests.rs"] mod tests; + +#[cfg(test)] +#[path = "full_text_capture_transport_tests.rs"] +mod transport_tests; diff --git a/crates/conceptweave-zotero/src/full_text_capture_transport_tests.rs b/crates/conceptweave-zotero/src/full_text_capture_transport_tests.rs new file mode 100644 index 00000000..14bad21b --- /dev/null +++ b/crates/conceptweave-zotero/src/full_text_capture_transport_tests.rs @@ -0,0 +1,258 @@ +mod tests { + use super::super::*; + use crate::classify_snapshot; + use std::io::{Read, Write}; + use std::net::TcpListener; + use std::thread::{self, JoinHandle}; + + fn report_fixture() -> ClassificationReport { + let items = serde_json::from_value(serde_json::json!([ + {"key":"ABCD2345","version":2,"data":{"itemType":"book","title":"Synthetic ontology paper"}}, + {"key":"BCDE3456","version":1,"data":{"itemType":"attachment","parentItem":"ABCD2345"}} + ])) + .unwrap(); + let mut report = + classify_snapshot("10.0.1".into(), Some("fixture-server".into()), 2, items); + report.api_version = Some(3); + report.schema_version = Some(44); + report + } + + fn wire_response(status: u16, version: Option<&str>, body: &[u8]) -> Vec { + let version = version + .map(|value| format!("Last-Modified-Version: {value}\r\n")) + .unwrap_or_default(); + let mut response = format!( + "HTTP/1.1 {status} Synthetic\r\nZotero-Server-ID: fixture-server\r\nZotero-API-Version: 3\r\nZotero-Schema-Version: 44\r\nX-Zotero-Version: 10.0.1\r\n{version}Content-Length: {}\r\nConnection: close\r\n\r\n", + body.len() + ) + .into_bytes(); + response.extend_from_slice(body); + response + } + + fn serve_responses(responses: Vec>) -> (String, JoinHandle>) { + let listener = TcpListener::bind("127.0.0.1:0").unwrap(); + let api_root = format!("http://{}", listener.local_addr().unwrap()); + let server = thread::spawn(move || { + responses + .into_iter() + .map(|response| { + let (mut stream, _) = listener.accept().unwrap(); + let mut request = Vec::new(); + while !request.windows(4).any(|part| part == b"\r\n\r\n") { + let mut buffer = [0; 4096]; + let length = stream.read(&mut buffer).unwrap(); + assert_ne!(length, 0); + request.extend_from_slice(&buffer[..length]); + } + // Invalid headers or bounded bodies can make the client stop early. + let _ = stream.write_all(&response); + String::from_utf8(request).unwrap() + }) + .collect() + }); + (api_root, server) + } + + fn fetch_wire(response: Vec, limit: u64) -> Result { + let (api_root, server) = serve_responses(vec![response]); + let result = fetch_response( + &local_agent(), + &report_fixture(), + &api_root, + "items/BCDE3456/fulltext", + limit, + ); + let requests = server.join().unwrap(); + assert_eq!(requests.len(), 1); + assert!(requests[0].starts_with("GET /api/users/0/items/BCDE3456/fulltext HTTP/1.1\r\n")); + assert!(requests[0].contains("zotero-api-version: 3\r\n")); + assert!(requests[0].contains("zotero-server-id: fixture-server\r\n")); + assert!(!requests[0].contains("zotero-api-key:")); + result + } + + #[test] + fn response_keeps_exact_status_version_and_text() { + let body = "synthetic text 한글"; + let response = fetch_wire(wire_response(200, Some("0"), body.as_bytes()), 128).unwrap(); + assert_eq!(response.status, 200); + assert_eq!(response.version, Some(0)); + assert_eq!(response.body, body); + for status in [404, 401, 403, 429, 500, 503] { + let response = + fetch_wire(wire_response(status, None, b"synthetic failure"), 128).unwrap(); + assert_eq!(response.status, status); + assert_eq!(response.version, None); + assert_eq!(response.body, "synthetic failure"); + } + } + + #[test] + fn every_response_requires_matching_identity_and_contract_headers() { + let valid = String::from_utf8(wire_response(200, Some("2"), b"[]")).unwrap(); + for (header, value) in [ + ("Zotero-Server-ID", "fixture-server"), + ("Zotero-API-Version", "3"), + ("Zotero-Schema-Version", "44"), + ("X-Zotero-Version", "10.0.1"), + ] { + let original = format!("{header}: {value}\r\n"); + for replacement in [String::new(), format!("{header}: changed\r\n")] { + let response = valid.replace(&original, &replacement).into_bytes(); + assert_eq!( + fetch_wire(response, 128).err(), + Some(INVALID_EVIDENCE), + "{header}" + ); + } + let mut response = valid + .replace(&original, &format!("{header}: invalid-byte\r\n")) + .into_bytes(); + let position = response + .windows(12) + .position(|part| part == b"invalid-byte") + .unwrap(); + response[position] = 0xff; + assert!(fetch_wire(response, 128).is_err(), "{header}"); + } + } + + #[test] + fn malformed_present_versions_are_not_treated_as_missing() { + for version in ["", "invalid", "-1", "18446744073709551616"] { + assert_eq!( + fetch_wire(wire_response(200, Some(version), b"[]"), 128).err(), + Some(INVALID_EVIDENCE) + ); + } + let mut response = wire_response(200, Some("invalid-byte"), b"[]"); + let position = response + .windows(12) + .position(|part| part == b"invalid-byte") + .unwrap(); + response[position] = 0xff; + assert!(fetch_wire(response, 128).is_err()); + } + + #[test] + fn body_limit_and_strict_utf8_are_enforced_on_wire_bytes() { + assert_eq!( + fetch_wire(wire_response(200, Some("2"), b"[]"), 2) + .unwrap() + .body, + "[]" + ); + assert_eq!( + fetch_wire(wire_response(200, Some("2"), b"[]"), 1).err(), + Some(INVALID_EVIDENCE) + ); + assert_eq!( + fetch_wire(wire_response(200, Some("2"), &[0xff]), 128).err(), + Some(INVALID_EVIDENCE) + ); + let response = String::from_utf8(wire_response(200, Some("2"), b"[]")).unwrap(); + let truncated = response.replace("Content-Length: 2", "Content-Length: 3"); + assert_eq!( + fetch_wire(truncated.into_bytes(), 128).err(), + Some(INVALID_EVIDENCE) + ); + let chunked = response + .replace("Content-Length: 2", "Transfer-Encoding: chunked") + .replace("\r\n\r\n[]", "\r\n\r\n2\r\n[]\r\n0\r\n\r\n"); + assert_eq!( + fetch_wire(chunked.into_bytes(), 1).err(), + Some(INVALID_EVIDENCE) + ); + } + + #[test] + fn capture_rejects_non_success_status_without_following_redirects() { + let target = TcpListener::bind("127.0.0.1:0").unwrap(); + target.set_nonblocking(true).unwrap(); + for status in [301, 302, 303, 307, 308, 401, 403, 404, 429, 500, 503] { + let response = String::from_utf8(wire_response(status, Some("2"), b"[]")) + .unwrap() + .replace( + "Content-Length:", + &format!( + "Location: http://{}/redirect-target\r\nContent-Length:", + target.local_addr().unwrap() + ), + ); + let (api_root, server) = serve_responses(vec![response.into_bytes()]); + assert!(read_full_text_from_api(&report_fixture(), &api_root).is_err()); + assert_eq!(server.join().unwrap().len(), 1); + assert_eq!( + target.accept().err().unwrap().kind(), + std::io::ErrorKind::WouldBlock + ); + } + } + + #[test] + fn connection_failure_is_secret_free_and_missing_identity_prevents_network_access() { + let listener = TcpListener::bind("127.0.0.1:0").unwrap(); + listener.set_nonblocking(true).unwrap(); + let api_root = format!("http://{}", listener.local_addr().unwrap()); + let mut report = report_fixture(); + report.server_id = None; + assert_eq!( + fetch_response(&local_agent(), &report, &api_root, "items?limit=1", 128).err(), + Some(INVALID_EVIDENCE) + ); + assert_eq!( + listener.accept().err().unwrap().kind(), + std::io::ErrorKind::WouldBlock + ); + // The public wrapper rejects before its fixed production URL can be called. + assert!(read_local_full_text(&report).is_err()); + drop(listener); + let error = fetch_response( + &local_agent(), + &report_fixture(), + &api_root, + "items/BCDE3456/fulltext", + 128, + ) + .err() + .unwrap(); + assert_eq!(error.to_string(), "full-text local request failed"); + assert!(!format!("{error:?}").contains("BCDE3456")); + } + + #[test] + fn synthetic_http_sweep_uses_the_same_agent_and_capture_path_as_the_public_wrapper() { + let manifest = br#"{"BCDE3456":7}"#; + let metadata = br#"{"key":"BCDE3456","version":1,"data":{"itemType":"attachment","parentItem":"ABCD2345"}}"#; + let content = br#"{"content":"synthetic full text","providerExtra":{"retained":true}}"#; + let (api_root, server) = serve_responses(vec![ + wire_response(200, Some("2"), b"[]"), + wire_response(200, None, manifest), + wire_response(200, Some("1"), metadata), + wire_response(200, Some("7"), content), + wire_response(200, None, manifest), + wire_response(200, Some("2"), b"[]"), + ]); + let report = report_fixture(); + let capture = read_full_text_from_api(&report, &api_root).unwrap(); + verify_full_text_capture(&capture, &report).unwrap(); + assert_eq!(capture.capture_evidence.records.len(), 1); + assert_eq!( + capture.capture_evidence.records[0] + .content_response + .body + .as_bytes(), + content + ); + let requests = server.join().unwrap(); + assert_eq!(requests.len(), 6); + assert!(requests[0].starts_with("GET /api/users/0/items?limit=1 ")); + assert!(requests[1].starts_with("GET /api/users/0/fulltext?since=0 ")); + assert!(requests[2].starts_with("GET /api/users/0/items/BCDE3456 ")); + assert!(requests[3].starts_with("GET /api/users/0/items/BCDE3456/fulltext ")); + assert_eq!(requests[0], requests[5]); + assert_eq!(requests[1], requests[4]); + } +} From f7d3530287ec8f685734a8cc0d1cf033443ca5ab Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sat, 5 Sep 2026 17:21:32 +0900 Subject: [PATCH 10/47] test(zotero): require replay admission before digest allocation --- .../src/full_text_capture_tests.rs | 29 +++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/crates/conceptweave-zotero/src/full_text_capture_tests.rs b/crates/conceptweave-zotero/src/full_text_capture_tests.rs index c644d7f7..cb0c438d 100644 --- a/crates/conceptweave-zotero/src/full_text_capture_tests.rs +++ b/crates/conceptweave-zotero/src/full_text_capture_tests.rs @@ -371,3 +371,32 @@ fn byte_and_clock_limits_include_exact_boundary_and_overflow_failures() { "FullTextError(\"fixture static error\")" ); } + +#[test] +fn replay_applies_byte_admission_before_digest_or_json_work() { + let report = report_fixture(); + let mut capture = capture_with(&report, 4096, &mut |request_path, _| { + Ok(response_fixture(request_path)) + }) + .unwrap(); + capture.capture_evidence.records[1].content_response.body = + "x".repeat(MAX_PAGE_BYTES as usize + 1); + assert_eq!( + verify_full_text_capture(&capture, &report), + Err(BUDGET_EXCEEDED) + ); +} + +#[test] +fn streaming_digest_preserves_the_existing_exact_json_representation() { + let report = report_fixture(); + let capture = capture_with(&report, 4096, &mut |request_path, _| { + Ok(response_fixture(request_path)) + }) + .unwrap(); + let bytes = serde_json::to_vec(&capture.capture_evidence).unwrap(); + assert_eq!( + capture.capture_digest, + format!("sha256:{:x}", Sha256::digest(bytes)) + ); +} From 301d9d56565963b4a4f2babd07da52896cd0a784 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sat, 5 Sep 2026 17:21:56 +0900 Subject: [PATCH 11/47] fix(zotero): bound replay before streaming its content digest --- .../src/full_text_capture.rs | 41 ++++++++++++------- 1 file changed, 26 insertions(+), 15 deletions(-) diff --git a/crates/conceptweave-zotero/src/full_text_capture.rs b/crates/conceptweave-zotero/src/full_text_capture.rs index 6980ec02..c1a7af76 100644 --- a/crates/conceptweave-zotero/src/full_text_capture.rs +++ b/crates/conceptweave-zotero/src/full_text_capture.rs @@ -95,8 +95,26 @@ pub fn verify_full_text_capture( ) -> Result<(), FullTextError> { let snapshot = validate_report(report)?; let evidence = &capture.capture_evidence; - if capture.capture_digest != json_digest(evidence) - || evidence.capture_kind != CAPTURE_KIND + if evidence.records.len() > snapshot.len() { + return Err(INVALID_EVIDENCE); + } + let mut remaining = MAX_SNAPSHOT_BYTES; + for response in [ + &evidence.library_before, + &evidence.manifest_before, + &evidence.manifest_after, + &evidence.library_after, + ] + .into_iter() + .chain( + evidence + .records + .iter() + .flat_map(|record| [&record.metadata_response, &record.content_response]), + ) { + account_body(&mut remaining, response)?; + } + if evidence.capture_kind != CAPTURE_KIND || evidence.metadata_report_digest != json_digest(report) || evidence.metadata_snapshot_digest != report.snapshot_digest || evidence.bibliographic_item_count != report.classified_items.len() @@ -113,23 +131,15 @@ pub fn verify_full_text_capture( { return Err(INVALID_EVIDENCE); } - let mut remaining = MAX_SNAPSHOT_BYTES; - for response in [ - &evidence.library_before, - &evidence.manifest_before, - &evidence.manifest_after, - &evidence.library_after, - ] { - account_body(&mut remaining, response)?; - } for (record, (item_key, version)) in evidence.records.iter().zip(&manifest) { if &record.item_key != item_key { return Err(INVALID_EVIDENCE); } validate_metadata(&record.metadata_response, snapshot[item_key.as_str()])?; validate_content(&record.content_response, *version)?; - account_body(&mut remaining, &record.metadata_response)?; - account_body(&mut remaining, &record.content_response)?; + } + if capture.capture_digest != json_digest(evidence) { + return Err(INVALID_EVIDENCE); } Ok(()) } @@ -339,8 +349,9 @@ fn unix_millis(time: SystemTime) -> Result { } fn json_digest(value: &impl Serialize) -> String { - let bytes = serde_json::to_vec(value).expect("capture values are JSON-compatible"); - format!("sha256:{:x}", Sha256::digest(bytes)) + let mut digest = Sha256::new(); + serde_json::to_writer(&mut digest, value).expect("capture values are JSON-compatible"); + format!("sha256:{:x}", digest.finalize()) } fn fetch_response( From 2d9ec5cca6788605b4515eb384c42b2fc511e4eb Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sat, 5 Sep 2026 17:24:09 +0900 Subject: [PATCH 12/47] test(zotero): reproduce shared exact-body-limit rejection --- .../src/full_text_capture_transport_tests.rs | 55 +++++++++++++++++++ 1 file changed, 55 insertions(+) diff --git a/crates/conceptweave-zotero/src/full_text_capture_transport_tests.rs b/crates/conceptweave-zotero/src/full_text_capture_transport_tests.rs index 14bad21b..6748324b 100644 --- a/crates/conceptweave-zotero/src/full_text_capture_transport_tests.rs +++ b/crates/conceptweave-zotero/src/full_text_capture_transport_tests.rs @@ -222,6 +222,61 @@ mod tests { assert!(!format!("{error:?}").contains("BCDE3456")); } + #[test] + fn authorization_response_at_its_byte_limit_is_not_rejected() { + let mut body = br#"{"key":"0123456789abcdef0123456789abcdef","remember":true}"#.to_vec(); + body.resize(crate::MAX_AUTH_RESPONSE_BYTES as usize, b' '); + let (api_root, server) = serve_responses(vec![wire_response(200, None, &body)]); + let result = crate::Zotero10LocalAuthorization::request_with_base( + "Synthetic exact limit test", + "fixture-server", + api_root, + ); + server.join().unwrap(); + assert!( + result.is_ok(), + "exact-limit authorization JSON must be accepted" + ); + } + + #[test] + fn snapshot_response_at_its_byte_limit_is_not_rejected() { + const CHILD_CASE: &str = "CONCEPTWEAVE_EXACT_SNAPSHOT_LIMIT_CASE"; + if std::env::var_os(CHILD_CASE).is_none() { + let status = std::process::Command::new(std::env::current_exe().unwrap()) + .args([ + "--exact", + "full_text_capture::transport_tests::tests::snapshot_response_at_its_byte_limit_is_not_rejected", + ]) + .env_clear() + .env(CHILD_CASE, "synthetic") + .status() + .unwrap(); + assert!( + status.success(), + "isolated snapshot byte-limit regression failed" + ); + return; + } + let mut body = b"[]".to_vec(); + body.resize(MAX_PAGE_BYTES as usize, b' '); + let mut response = wire_response(200, Some("2"), &body); + let header_end = response + .windows(4) + .position(|part| part == b"\r\n\r\n") + .unwrap(); + response.splice( + header_end..header_end, + b"\r\nTotal-Results: 0".iter().copied(), + ); + let (api_root, server) = serve_responses(vec![response]); + *crate::TEST_LOCAL_API.lock().unwrap() = Some(format!("{api_root}/api/users/0/items")); + let result = crate::read_local_snapshot(); + *crate::TEST_LOCAL_API.lock().unwrap() = None; + server.join().unwrap(); + assert!(result.is_ok(), "exact-limit snapshot JSON must be accepted"); + } + #[test] fn synthetic_http_sweep_uses_the_same_agent_and_capture_path_as_the_public_wrapper() { let manifest = br#"{"BCDE3456":7}"#; From c959505faccbcd2592ef8563bb7a7d3b42d438d4 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sat, 5 Sep 2026 17:24:55 +0900 Subject: [PATCH 13/47] fix(zotero): enforce inclusive response byte limits consistently --- crates/conceptweave-zotero/src/lib.rs | 30 +++++++++++++++++++-------- 1 file changed, 21 insertions(+), 9 deletions(-) diff --git a/crates/conceptweave-zotero/src/lib.rs b/crates/conceptweave-zotero/src/lib.rs index c790bfa1..6a9cfbe1 100644 --- a/crates/conceptweave-zotero/src/lib.rs +++ b/crates/conceptweave-zotero/src/lib.rs @@ -7,6 +7,7 @@ use serde::{Deserialize, Serialize}; use sha2::{Digest, Sha256}; use std::collections::{BTreeMap, BTreeSet}; use std::fmt; +use std::io::Read; use std::time::Duration; mod full_text_capture; @@ -834,12 +835,27 @@ fn bounded_body_with_limit( response: &mut ureq::http::Response, limit: u64, ) -> Result { + read_bounded_response_text(response, limit).map_err(|_| ZoteroTransportError::InvalidResponse) +} + +/// Reads strict UTF-8 with an inclusive byte limit and one byte of overrun evidence. +fn read_bounded_response_text( + response: &mut ureq::http::Response, + limit: u64, +) -> std::io::Result { + let mut body = String::new(); response .body_mut() - .with_config() - .limit(limit) - .read_to_string() - .map_err(|_| ZoteroTransportError::InvalidResponse) + .as_reader() + .take(limit.saturating_add(1)) + .read_to_string(&mut body)?; + if body.len() as u64 > limit { + return Err(std::io::Error::new( + std::io::ErrorKind::InvalidData, + "response body exceeds byte limit", + )); + } + Ok(body) } fn validate_item_key(item_key: &str) -> Result<(), ZoteroTransportError> { @@ -2783,11 +2799,7 @@ fn fetch_local_page(agent: &ureq::Agent, start: usize) -> Result Date: Sat, 5 Sep 2026 17:25:38 +0900 Subject: [PATCH 14/47] refactor(zotero): keep private writes on one tested dispatch path --- crates/conceptweave-zotero/src/main.rs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/crates/conceptweave-zotero/src/main.rs b/crates/conceptweave-zotero/src/main.rs index 58789fab..0a0e55ff 100644 --- a/crates/conceptweave-zotero/src/main.rs +++ b/crates/conceptweave-zotero/src/main.rs @@ -360,7 +360,7 @@ fn label_input(name: &str, error: io::Error) -> io::Error { /// Writes one create-new owner-only artifact and removes a failed partial write. fn write_private_output(path: &Path, content: &[u8]) -> io::Result<()> { - write_private_output_with(path, content, write_all_and_flush) + write_private_output_with(path, content, &mut write_all_and_flush) } /// Writes and flushes the complete serialized artifact. @@ -373,7 +373,7 @@ fn write_all_and_flush(writer: &mut BufWriter, content: &[u8]) -> io::Resu fn write_private_output_with( path: &Path, content: &[u8], - write: impl FnOnce(&mut BufWriter, &[u8]) -> io::Result<()>, + write: &mut dyn FnMut(&mut BufWriter, &[u8]) -> io::Result<()>, ) -> io::Result<()> { let file = create_report_file(path)?; let mut writer = BufWriter::new(file); @@ -484,7 +484,7 @@ fn main() -> Result<(), Box> { let (report, _): (ClassificationReport, _) = read_private_json(&report).map_err(|error| label_input("report", error))?; let capture = read_local_full_text(&report)?; - write_private_output_with(&output, &[], |writer, _| { + write_private_output_with(&output, &[], &mut |writer, _| { serde_json::to_writer(&mut *writer, &capture).map_err(io::Error::other)?; writer.flush() })?; @@ -1044,7 +1044,7 @@ mod tests { fn failed_private_output_is_removed_for_retry() { let output = unique_temp_path("failed-output"); let _ = fs::remove_file(&output); - let error = write_private_output_with(&output, b"content", |_, _| { + let error = write_private_output_with(&output, b"content", &mut |_, _| { Err(io::Error::new( io::ErrorKind::WriteZero, "injected write failure", From eb3d28239c9b496b9a1824971b8e16a61d5d29ad Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sat, 5 Sep 2026 17:26:28 +0900 Subject: [PATCH 15/47] docs(zotero): define private full-text capture and approval boundaries --- AGENTS.md | 1 + ARCHITECTURE.md | 2 ++ CHANGELOG.md | 4 ++++ CLAUDE.md | 2 ++ OPERABILITY.md | 8 ++++++++ README.md | 8 ++++++++ SECURITY.md | 2 ++ TEST_STRATEGY.md | 4 ++++ docs/CONTEXT_MAP.md | 2 ++ docs/PRD.md | 2 +- docs/TRD.md | 8 +++++++- docs/UBIQUITOUS_LANGUAGE.md | 1 + docs/UML.md | 15 ++++++++++++++- docs/adr/0006-zotero-research-intake.md | 6 +++++- 14 files changed, 61 insertions(+), 4 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 67347566..156d85ff 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -16,6 +16,7 @@ ConceptWeave owns automatic, evidence-bound **Semantic Model Engineering**. Do n - No direct cross-service application-table SQL. - New database objects, when introduced, use descriptive two-or-more-word `snake_case` names and 3NF by default. - Preserve source evidence, truth status, and publication state separately. +- Keep Zotero full-text captures separate from metadata reports and approval receipts; restored captures require bounded verification, and local HTTP continuity is not peer authentication. - Published semantic truth is immutable; correction uses supersession/new release. - Public Rust APIs require beginner-readable documentation. - Owned production coverage target is 100% line/function/region/branch where tooling exposes it. diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md index c5dd4993..c1e6721b 100644 --- a/ARCHITECTURE.md +++ b/ARCHITECTURE.md @@ -22,6 +22,8 @@ flowchart LR ## DDD context map +Research Intake's Zotero adapter retains optional full-text observations in a separate private artifact bound to the original metadata report. It remains inside ConceptWeave: acquisition is supporting evidence work, not a publication authority or another research system of record. Provider version counters remain opaque at this Anti-Corruption Layer; downstream classification and review must explicitly adopt new content under fresh evidence bindings. See [ADR 0006](docs/adr/0006-zotero-research-intake.md). + | Context | Type | Owns | Does not own | | --- | --- | --- | --- | | Source Observation | Supporting | immutable observations, parser receipts, evidence locations | source-system business truth | diff --git a/CHANGELOG.md b/CHANGELOG.md index 5d0df103..6936bbfe 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,8 @@ All notable changes to ConceptWeave are documented here. ### Added +- Private, replayable paper-text capture for later research review, preserving unavailable material and leaving earlier reports and approvals unchanged. + - Full-library research-source audit separating available text, incomplete indexing and missing material from reviewed classification; no paper is excluded because its abstract or text is unavailable. - Initial ConceptWeave product, DDD, security, test, and operability baselines. - Rust 1.98.0 `conceptweave-domain` foundation with evidence-bound semantic candidate contracts. @@ -24,6 +26,8 @@ All notable changes to ConceptWeave are documented here. ### Security +- Local research requests bypass environment-configured proxies. This prevents unintended proxy forwarding; local peer authentication remains an explicit release limitation. + - Source receipts bind complete captured metadata and actual classifier inputs; earlier report and review artifacts require regeneration under the versioned digest representation. - Golden-set evaluation rejects changed predictions or evidence under an earlier approval. Proposal-bound approvals must be reissued; aggregate receipts identify the actual evaluated proposal run. - Model-generated semantics remain non-authoritative until deterministic validation and authorized review. diff --git a/CLAUDE.md b/CLAUDE.md index d8db2650..76caecc3 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -5,3 +5,5 @@ Follow `AGENTS.md`, `ARCHITECTURE.md`, accepted ADRs, and the organization maste ConceptWeave's core invariant is: **inference is not authority**. Every generated concept, relation, constraint, dimension, measure, or physical mapping must retain evidence and pass the explicit governance lifecycle before publication. Keep domain logic in bounded domain modules, LLM/provider logic behind ports/adapters, and source/consumer systems independent. Prefer deterministic validation and explicit abstention over plausible unsupported output. + +Zotero source capture must not alter the metadata report or renew its approval. Preserve private-file protections and the full bibliographic denominator, including missing and partial text. diff --git a/OPERABILITY.md b/OPERABILITY.md index b3302610..ce8123d4 100644 --- a/OPERABILITY.md +++ b/OPERABILITY.md @@ -21,3 +21,11 @@ ConceptWeave has no production network service or durable database in the founda - downstream catalog unavailable: publication retains a durable release/outbox receipt and does not lose the governed release. Concrete SLO/RPO/RTO values require measured runtime evidence and are not guessed in the foundation. + +## Local paper-text capture + +The proposed `--capture-full-text` command uses an existing private metadata report and creates a separate owner-only file. Keep the original report: the new file cannot replace it, renew review, or prove that all text came from one atomic snapshot. No Zotero authorization prompt, mutation or model request is part of this command. + +A changed library, missing provider identity, unexpected response, malformed text or exhausted budget rejects the run. Preserve earlier artifacts; do not disable the checks or overwrite an old file to retry. If metadata has changed, capture a new report and start a separately bound review campaign. Otherwise investigate the reported boundary and rerun to a new temp path. An expected missing-text response is retained; an interrupted run does not emit a partial-success capture. A failed write removes the new partial output. + +Allow space for the source text plus JSON escaping overhead. Responses are limited to 8 MiB each and 256 MiB total; the encoded output may be larger. The sweep has a five-minute admission/completion limit and finite local request timeouts. Source text stays in memory until capture completes; hashing and writing stream without a second full encoded copy. The CLI deliberately does not delete prior reports or schedule private-file cleanup. Review retention according to the research library's policy, and never attach these files to a public PR. diff --git a/README.md b/README.md index 829df78f..1a62b6bc 100644 --- a/README.md +++ b/README.md @@ -117,6 +117,14 @@ cargo doc --workspace --no-deps The repository CI also validates the JSON Schema, lock/toolchain freshness, documentation contracts, and coverage expectations defined by the current source. +For the proposed local research intake, preserve available paper text separately from an existing private report: + +```bash +cargo +1.98.0 run --locked -p conceptweave-zotero -- --capture-full-text /tmp/REPORT.json /tmp/CAPTURE.json +``` + +Zotero 10+ must be running. The report must be an unchanged owner-only file from that library; the capture path must be a new file directly in the system temp directory. Missing or partial text stays visible, and the command does not classify papers, approve decisions or modify Zotero. Keep both files private; see [operation and retry limits](OPERABILITY.md). + ## Core contract A semantic candidate is not the same thing as published semantic truth. diff --git a/SECURITY.md b/SECURITY.md index a0a98d12..05be141a 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -6,6 +6,8 @@ All source artifacts, generated candidate payloads, external ontology files, mod Local Zotero classification reports can contain bibliographic titles, tags, matched values, and abstention abstracts. They are sensitive review material, remain outside the repository, and are not publication artifacts. +Full-text captures additionally retain exact paper text and attachment metadata. They use new `0600` local files, static transport errors and no content logging. Both local HTTP agents explicitly disable inherited proxies; redirects remain disabled. Replay limits record and response sizes before parsing/hash work, and unchanged hashes do not authenticate a replaced local artifact or the provider. Full-text source strings remain untrusted data, never instructions. Metadata approval cannot authorize newly captured text. + ## Required controls - source size, type, nesting, archive/decompression, and parser-time bounds; diff --git a/TEST_STRATEGY.md b/TEST_STRATEGY.md index 6d137795..f2b26252 100644 --- a/TEST_STRATEGY.md +++ b/TEST_STRATEGY.md @@ -9,6 +9,10 @@ - lockfile freshness and clean-tree verification; - public Rust documentation with `missing_docs` denied. +## Local research capture regressions + +The full-text suite covers report admission, exact response retention, parent/item revision binding, independent content versions, missing/empty/partial text, duplicate/foreign manifest rejection, bookend drift, byte/deadline boundaries and replay under changed or recomputed digests. Synthetic HTTP tests cover headers, network/redirect failures, strict encoding and response limits without touching the running Zotero library. Proxy isolation uses fresh subprocess environments for all six supported proxy variable spellings across the three existing local transport paths. Synthetic text is only a unit/integration fixture; live aggregate evidence is separately recorded in doctoring and never reported as approved labels. + ## Future product test families ### Source observation diff --git a/docs/CONTEXT_MAP.md b/docs/CONTEXT_MAP.md index 699c5c22..86bc97f4 100644 --- a/docs/CONTEXT_MAP.md +++ b/docs/CONTEXT_MAP.md @@ -8,6 +8,8 @@ - Governance & Publication -> Interoperability: **Published Language**; adapters consume immutable release contracts. - Research Intake -> Governance & Publication: **Anti-Corruption Layer**; Governance verifies complete duplicate and classification-write review sets and returns only opaque authority receipts before Intake emits canonical-key operations or write plans. +Research Intake also owns the optional private Full-Text Capture bound to a metadata report. It preserves provider observations while rejecting mixed-origin counters as a reliable incremental cursor. This is an adapter responsibility, not a new bounded context, shared catalog or approval owner; downstream proposal adoption remains a separate evidence/review transition. + ## External relationships - Zotero Local API -> research evidence intake: **Anti-Corruption Layer into Semantic Discovery**. Zotero remains the bibliographic system of record; ConceptWeave consumes a version-pinned snapshot and emits proposal evidence. Execute-mode metadata changes cross only a caller-owned authenticated adapter after complete preflight; ConceptWeave retains no API key and records verified item-level outcomes and rollback coordinates. Item metadata, attachments, collection/tag truth, and write authority remain in Zotero. No Zotero record becomes semantic authority without ConceptWeave validation/review/publication. diff --git a/docs/PRD.md b/docs/PRD.md index 2067000e..98a7ef40 100644 --- a/docs/PRD.md +++ b/docs/PRD.md @@ -58,7 +58,7 @@ All LLM-backed induction uses `contextual-orchestrator`. Model output is untrust Read one immutable Zotero Local API library-version snapshot and propose exactly one research disposition for every top-level bibliographic item. Each proposal retains the item key/version, exact matched metadata values, rule revision, linked child records, and any model receipt. Weak evidence and evidence that matches multiple specific disposition families must abstain into steward review. A local abstention retains its nonempty abstract exactly once, as matched evidence when applicable or otherwise as review context; decided items omit the review-only copy. Duplicate DOI/title identities are review candidates only: intake never merges, deletes, or silently mutates Zotero records. -Full-text enrichment must distinguish listed attachments, returned nonempty text, complete or partial indexing, and reviewed meaning. Missing abstracts or unavailable text never remove papers from the campaign denominator or prove irrelevance. Newly retrieved text requires its own immutable evidence capture and renewed review of any changed proposal; it cannot silently replace evidence beneath an earlier approval. The [full-text audit](doctoring/zotero_fulltext_contract_audit.md) establishes availability only, not an implemented enrichment or completed classification. +Full-text enrichment must distinguish listed attachments, returned nonempty text, complete or partial indexing, and reviewed meaning. Missing abstracts or unavailable text never remove papers from the campaign denominator or prove irrelevance. Newly retrieved text requires its own immutable evidence capture and renewed review of any changed proposal; it cannot silently replace evidence beneath an earlier approval. The [full-text audit](doctoring/zotero_fulltext_contract_audit.md) establishes availability only. A separate proposed local capture now preserves the observed text for later review, with missing material still visible. Retained text is neither completed classification nor approved meaning. For every connected duplicate component, accept externally verified steward decisions selecting one component-level canonical item. Produce a local-only manifest that binds the decisions to the raw snapshot, its complete item-key/item-version coordinates, and exact duplicate-candidate membership, and records every component source revision plus before, after, and rollback canonical-key mappings. Classification preserves every Zotero source record. diff --git a/docs/TRD.md b/docs/TRD.md index 6c949e87..50430556 100644 --- a/docs/TRD.md +++ b/docs/TRD.md @@ -59,7 +59,13 @@ Evaluation must separate extraction recall, semantic correctness, structural cor ## 11. Zotero research intake -Full-text consumption is not implemented in the current metadata classifier. The [Zotero 10.0.1 audit](doctoring/zotero_fulltext_contract_audit.md) confirms a provider-contract mismatch: full-text list responses omit the documented library-version header and expose versions written by sync, local indexing and local API paths in different version spaces. Treat them as opaque observations, not metadata revisions or reliable incremental cursors. Future enrichment must enumerate from zero, validate API/schema/server and observed attachment/parent membership, bound every response and the whole capture, and bind exact content plus completeness statistics to a separate immutable receipt. A missing, empty or partially indexed response remains explicit. Stable bookend versions/digests do not prove atomicity; no full-text capture may overwrite the earlier report digest or reuse its approval. These are admission requirements, not a claim of a shipped adapter. +Full-text classification is not implemented in the metadata classifier. The [Zotero 10.0.1 audit](doctoring/zotero_fulltext_contract_audit.md) confirms a provider-contract mismatch: full-text list responses omit the documented library-version header and expose versions written by sync, local indexing and local API paths in different version spaces. Treat them as opaque observations, not metadata revisions or reliable incremental cursors. Stable bookend versions/digests do not prove atomicity; no full-text capture may overwrite the earlier report digest or reuse its approval. + +The proposed capture implementation uses `--capture-full-text /tmp/REPORT.json /tmp/CAPTURE.json` and the existing private file boundary. It admits a nonempty validated Zotero 10+/API 3 metadata report with schema and server coordinates before requesting anything. It enumerates `fulltext?since=0`, rejects duplicate or foreign manifest keys, and reads each attachment's current metadata before its full text. Attachment key, item revision and parent must match the report; successful content requires a version matching that manifest entry. Every response pins server/API/schema/application version. Full-text HTTP 404 remains an explicit record; other non-success statuses, malformed content or drift fail the whole command before output creation. + +The artifact records the complete metadata-report digest, metadata snapshot digest, full bibliographic denominator, read interval, manifest/library bookends and ordered exact response bodies, HTTP statuses and observed versions. Raw content JSON preserves empty text, index counters and unknown provider fields without claiming completeness. SHA-256 is streamed over the compact serialized evidence, with `non_atomic_fulltext_sweep_v1` separating it from a metadata snapshot or approval. Replay checks record/body bounds before parsing or digest work, then rechecks the report binding, structure, attachment membership and versions. Deserialization alone is not verification; a replaced digest is not authenticated authority. + +Capture limits are 8 MiB per body, 256 MiB cumulative body bytes, the existing 50,000-item ceiling and a five-minute monotonic admission/completion budget. Request timeouts retain the local adapter's 30-second global / 2-second connect / 10-second response and body bounds; these are not model timeouts. A request already admitted can finish after the sweep deadline, but no late result is accepted. The writer streams JSON into a create-new `0600` temp file and removes a failed partial write. Raw response bytes are bounded separately from serialized file size, which can expand through JSON escaping. The 16 MiB private review-input reader is only used for the metadata report; it is not advertised as a large-capture reader. In-memory replay callers must bound private file deserialization separately. The capture remains a proposed local capability pending protected integration and review, with no change to proposal/decision/approval counts. `conceptweave-zotero` reads only the loopback Local API with at most 100 records per page, an 8 MiB page-body limit, a 50,000-item whole-snapshot limit, a 256 MiB cumulative body limit, redirects disabled, and finite connect/response/body/global timeouts. Every request pins `Zotero-API-Version: 3`; every response must report API version 3, while its schema version is recorded and must remain stable across the snapshot. Before another request is issued, exhausted whole-snapshot budgets fail closed. Before a parsed page is accumulated, checked item-count and byte arithmetic must remain within both the advertised total and the configured whole-snapshot budgets. `Total-Results`, `Last-Modified-Version`, Zotero version, and server identity must remain identical across all pages; contract drift, malformed JSON, an empty intermediate page, duplicate keys, or an oversized response fails the run. diff --git a/docs/UBIQUITOUS_LANGUAGE.md b/docs/UBIQUITOUS_LANGUAGE.md index 71b8c54e..c59a9b74 100644 --- a/docs/UBIQUITOUS_LANGUAGE.md +++ b/docs/UBIQUITOUS_LANGUAGE.md @@ -3,6 +3,7 @@ | Term | Meaning | | --- | --- | | Source Snapshot | Immutable revision of source evidence observed by ConceptWeave. | +| Full-Text Capture | Separate private record of exact text/metadata responses, missing results and read interval, bound to an earlier metadata report; not an atomic Source Snapshot or an Authority Receipt. | | Observation | Deterministically extracted fact from a Source Snapshot. | | Evidence Reference | Stable source identity, digest, and location supporting a candidate. | | Semantic Candidate | Evidence-bound proposal for a concept, relation, constraint, dimension, measure, or physical mapping. | diff --git a/docs/UML.md b/docs/UML.md index 0270adef..28d5ba38 100644 --- a/docs/UML.md +++ b/docs/UML.md @@ -46,14 +46,27 @@ sequenceDiagram participant Zotero as Zotero Local API participant Intake as Research intake participant Report as Local proposal report + participant Capture as Private text capture participant Steward loop bounded pages Intake->>Zotero: read items at one library version - Zotero-->>Intake: items + immutable version headers + Zotero-->>Intake: items + observed version headers end Intake->>Intake: classify or abstain; link children; find duplicate candidates Intake->>Report: write proposals and evidence + opt separate full-text capture requested + Report->>Intake: unchanged private report binding + Intake->>Zotero: library and complete manifest bookend + loop every manifest attachment within budgets + Intake->>Zotero: read current attachment metadata and full text + Zotero-->>Intake: metadata + content or explicit missing response + Intake->>Intake: check identity, parent and independent versions + end + Intake->>Zotero: repeat manifest and library bookend + Intake->>Capture: create new content-bound owner-only artifact + Note over Report,Capture: non-atomic observation; no changed proposal or approval + end Intake->>Report: derive snapshot-bound decision worksheet without bibliographic text Report->>Steward: review dispositions and merge candidates Steward->>Intake: save partially completed worksheet diff --git a/docs/adr/0006-zotero-research-intake.md b/docs/adr/0006-zotero-research-intake.md index b4640c37..94918420 100644 --- a/docs/adr/0006-zotero-research-intake.md +++ b/docs/adr/0006-zotero-research-intake.md @@ -51,7 +51,11 @@ No dedicated utility repository or Zotero mutation path is created. A future Zot In the context of reviewing papers with missing abstracts, facing a full-text API whose observed versions mix sync and local writes and whose list lacks the documented version header, we decided for separately captured, content-bound full-text observations and against treating attachment listings or unchanged version counters as complete snapshot evidence, to preserve review provenance and the full campaign denominator, accepting another capture/verification step and no current claim of atomic full-text enrichment. -The [source-grounded audit](../doctoring/zotero_fulltext_contract_audit.md) attempted all 3,473 listed entries and demonstrated nonempty text for 3,203/3,715 bibliographic items, including 800/1,000 without retained abstracts. It did not persist raw text or classify those papers. Positive consequence: available source material can guide genuine review without inventing relevance or approval. Negative consequence: content remains unbound to the metadata report until a separate immutable capture contract is implemented and reviewed. Neither a guessed incremental cursor nor adding only the provider's missing header repairs the mixed-version semantics. Direct database edits, cloud credential expansion and a new utility owner are rejected; the provider semantics require an upstream fix, while Research Intake retains the consumer admission boundary. Status remains Proposed. +The [source-grounded audit](../doctoring/zotero_fulltext_contract_audit.md) attempted all 3,473 listed entries and demonstrated nonempty text for 3,203/3,715 bibliographic items, including 800/1,000 without retained abstracts. It did not persist raw text or classify those papers. The follow-up capture contract is implemented locally in `5f36ff5` and hardened in `301d9d5`, pending independent review and protected integration. It binds the original report and exact observed responses in one new private file. It does not retrofit text into the earlier report or approve a changed proposal. + +Positive consequence: stewards can retain replayable source material with explicit missing/partial observations. Negative consequences: a capture can contain hundreds of megabytes of sensitive text, needs private-file retention, and cannot promise an atomic provider snapshot. Streamed hashing/writing avoids another capture-sized serialization buffer; body and record limits precede replay parsing and hashing. A five-minute completion/admission bound rejects late capture results without changing any model timeout. The original report and every earlier receipt remain unchanged. + +Hash-only evidence was rejected because the prior sweep discarded its text and could not replay it. A new database, service, repository or dependency was rejected because the existing Rust adapter, serialization/hash libraries and owner-only writer cover this one consumer. Neither a guessed incremental cursor nor adding only the provider's missing header repairs the mixed-version semantics. Direct database edits and cloud credential expansion are rejected; the provider semantics require an upstream fix, while Research Intake retains the consumer admission boundary. Both existing local agents now explicitly disable environment proxies: six proxy-variable cases across metadata reads, authorization and item reads/writes reproduced interception before the fix and direct local routing after it. This prevents proxy leakage but does not resolve same-host HTTP peer authentication. Status remains Proposed. ### 2026-09-05 integrity amendment (Proposed) From d24a74a83c9c77392976ce3f3d47aa92563acda2 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sat, 5 Sep 2026 17:27:00 +0900 Subject: [PATCH 16/47] test(zotero): reject a missing report schema at the transport boundary --- .../src/full_text_capture_transport_tests.rs | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/crates/conceptweave-zotero/src/full_text_capture_transport_tests.rs b/crates/conceptweave-zotero/src/full_text_capture_transport_tests.rs index 6748324b..491c181d 100644 --- a/crates/conceptweave-zotero/src/full_text_capture_transport_tests.rs +++ b/crates/conceptweave-zotero/src/full_text_capture_transport_tests.rs @@ -220,6 +220,15 @@ mod tests { .unwrap(); assert_eq!(error.to_string(), "full-text local request failed"); assert!(!format!("{error:?}").contains("BCDE3456")); + + let (api_root, server) = serve_responses(vec![wire_response(200, Some("2"), b"[]")]); + let mut report = report_fixture(); + report.schema_version = None; + assert_eq!( + fetch_response(&local_agent(), &report, &api_root, "items?limit=1", 128).err(), + Some(INVALID_EVIDENCE) + ); + assert_eq!(server.join().unwrap().len(), 1); } #[test] From 847535deff54e38153bcfa2bcf5dc8106ca86d81 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sat, 5 Sep 2026 17:27:08 +0900 Subject: [PATCH 17/47] test(zotero): reject oversized replay counts and valid-JSON tampering --- .../src/full_text_capture_tests.rs | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/crates/conceptweave-zotero/src/full_text_capture_tests.rs b/crates/conceptweave-zotero/src/full_text_capture_tests.rs index cb0c438d..32901857 100644 --- a/crates/conceptweave-zotero/src/full_text_capture_tests.rs +++ b/crates/conceptweave-zotero/src/full_text_capture_tests.rs @@ -86,7 +86,8 @@ fn capture_retains_exact_text_missing_results_and_full_parent_denominator() { let restored: FullTextCapture = serde_json::from_value(saved.clone()).unwrap(); verify_full_text_capture(&restored, &report).unwrap(); let mut changed = saved; - changed["capture_evidence"]["records"][0]["content_response"]["body"] = "changed text".into(); + changed["capture_evidence"]["records"][0]["content_response"]["body"] = + r#"{"content":"changed text"}"#.into(); let changed: FullTextCapture = serde_json::from_value(changed).unwrap(); assert!(verify_full_text_capture(&changed, &report).is_err()); } @@ -213,7 +214,7 @@ fn replay_checks_bindings_and_structure_even_when_digest_is_recomputed() { }) .unwrap(); let saved = serde_json::to_value(capture).unwrap(); - for scenario in 0..15 { + for scenario in 0..16 { let mut restored: FullTextCapture = serde_json::from_value(saved.clone()).unwrap(); let evidence = &mut restored.capture_evidence; match scenario { @@ -233,7 +234,15 @@ fn replay_checks_bindings_and_structure_even_when_digest_is_recomputed() { 11 => evidence.records.swap(0, 1), 12 => evidence.records[0].metadata_response.body = "{}".into(), 13 => evidence.records[0].content_response.body = r#"{"content":null}"#.into(), - _ => evidence.records[0].content_response.version = None, + 14 => evidence.records[0].content_response.version = None, + _ => { + let serialized = serde_json::to_string(&evidence.records[0]).unwrap(); + for _ in 0..3 { + evidence + .records + .push(serde_json::from_str(&serialized).unwrap()); + } + } } restored.capture_digest = json_digest(evidence); assert!( From 7854b3a575f987c65fefd897a384cb7f222252a1 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sat, 5 Sep 2026 17:30:20 +0900 Subject: [PATCH 18/47] test(zotero): require late and invalid-clock sweeps to fail --- .../src/full_text_capture.rs | 9 +++++++ .../src/full_text_capture_tests.rs | 24 +++++++++++++++++++ 2 files changed, 33 insertions(+) diff --git a/crates/conceptweave-zotero/src/full_text_capture.rs b/crates/conceptweave-zotero/src/full_text_capture.rs index 043336e2..ee61645a 100644 --- a/crates/conceptweave-zotero/src/full_text_capture.rs +++ b/crates/conceptweave-zotero/src/full_text_capture.rs @@ -238,6 +238,15 @@ fn validate_report( Ok(snapshot) } +fn capture_with_clock( + report: &ClassificationReport, + max_bytes: u64, + fetch: &mut dyn FnMut(&str, u64) -> Result, + _observe_time: &mut dyn FnMut() -> (SystemTime, Duration), +) -> Result { + capture_with(report, max_bytes, fetch) +} + fn validate_library( response: &CapturedResponse, report: &ClassificationReport, diff --git a/crates/conceptweave-zotero/src/full_text_capture_tests.rs b/crates/conceptweave-zotero/src/full_text_capture_tests.rs index 32901857..d02311eb 100644 --- a/crates/conceptweave-zotero/src/full_text_capture_tests.rs +++ b/crates/conceptweave-zotero/src/full_text_capture_tests.rs @@ -409,3 +409,27 @@ fn streaming_digest_preserves_the_existing_exact_json_representation() { format!("sha256:{:x}", Sha256::digest(bytes)) ); } + +#[test] +fn capture_rejects_clock_failure_and_late_results_without_real_waiting() { + for failed_poll in [0, 1, 2, 17, 18] { + let mut clock_polls = 0; + let result = capture_with_clock( + &report_fixture(), + 4096, + &mut |request_path, _| Ok(response_fixture(request_path)), + &mut || { + let poll = clock_polls; + clock_polls += 1; + if poll == failed_poll && [0, 17].contains(&poll) { + (UNIX_EPOCH - Duration::from_secs(1), Duration::ZERO) + } else if poll == failed_poll { + (UNIX_EPOCH + Duration::from_secs(100), CAPTURE_DEADLINE) + } else { + (UNIX_EPOCH + Duration::from_secs(100), Duration::ZERO) + } + }, + ); + assert!(result.is_err(), "clock poll {failed_poll}"); + } +} From 2c2226f1d583c3091cc126c96d27d55d1084c0d1 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sat, 5 Sep 2026 17:31:44 +0900 Subject: [PATCH 19/47] fix(zotero): verify clock and request failure propagation --- .../src/full_text_capture.rs | 44 +++++++++---------- .../src/full_text_capture_tests.rs | 33 ++++++++++++++ 2 files changed, 53 insertions(+), 24 deletions(-) diff --git a/crates/conceptweave-zotero/src/full_text_capture.rs b/crates/conceptweave-zotero/src/full_text_capture.rs index ee61645a..c7220849 100644 --- a/crates/conceptweave-zotero/src/full_text_capture.rs +++ b/crates/conceptweave-zotero/src/full_text_capture.rs @@ -156,15 +156,26 @@ fn capture_with( max_bytes: u64, fetch: &mut dyn FnMut(&str, u64) -> Result, ) -> Result { - let snapshot = validate_report(report)?; - let started_unix_ms = unix_millis(SystemTime::now())?; let started = Instant::now(); + capture_with_clock(report, max_bytes, fetch, &mut || { + (SystemTime::now(), started.elapsed()) + }) +} + +fn capture_with_clock( + report: &ClassificationReport, + max_bytes: u64, + fetch: &mut dyn FnMut(&str, u64) -> Result, + observe_time: &mut dyn FnMut() -> (SystemTime, Duration), +) -> Result { + let snapshot = validate_report(report)?; + let started_unix_ms = unix_millis(observe_time().0)?; let mut remaining = max_bytes.min(MAX_SNAPSHOT_BYTES); let mut read = |request_path: &str| { - check_admission(remaining, started.elapsed())?; + check_admission(remaining, observe_time().1)?; let response = fetch(request_path, remaining.min(MAX_PAGE_BYTES))?; account_body(&mut remaining, &response)?; - check_deadline(started.elapsed())?; + check_deadline(observe_time().1)?; Ok::<_, FullTextError>(response) }; let library_before = read("items?limit=1")?; @@ -191,7 +202,7 @@ fn capture_with( metadata_snapshot_digest: report.snapshot_digest.clone(), bibliographic_item_count: report.classified_items.len(), started_unix_ms, - finished_unix_ms: unix_millis(SystemTime::now())?, + finished_unix_ms: unix_millis(observe_time().0)?, library_before, manifest_before, records, @@ -203,7 +214,7 @@ fn capture_with( capture_evidence, }; verify_full_text_capture(&capture, report)?; - check_deadline(started.elapsed())?; + check_deadline(observe_time().1)?; Ok(capture) } @@ -238,15 +249,6 @@ fn validate_report( Ok(snapshot) } -fn capture_with_clock( - report: &ClassificationReport, - max_bytes: u64, - fetch: &mut dyn FnMut(&str, u64) -> Result, - _observe_time: &mut dyn FnMut() -> (SystemTime, Duration), -) -> Result { - capture_with(report, max_bytes, fetch) -} - fn validate_library( response: &CapturedResponse, report: &ClassificationReport, @@ -377,21 +379,15 @@ fn fetch_response( request_path: &str, limit: u64, ) -> Result { + let expected_server = report.server_id.as_deref().ok_or(INVALID_EVIDENCE)?; let mut response = agent .get(&format!("{api_root}/api/users/0/{request_path}")) .header("Zotero-API-Version", "3") - .header( - "Zotero-Server-ID", - report.server_id.as_deref().ok_or(INVALID_EVIDENCE)?, - ) + .header("Zotero-Server-ID", expected_server) .call() .map_err(|_| FullTextError("full-text local request failed"))?; let headers = response.headers(); - verify_server_id( - headers, - report.server_id.as_deref().ok_or(INVALID_EVIDENCE)?, - ) - .map_err(|_| INVALID_EVIDENCE)?; + verify_server_id(headers, expected_server).map_err(|_| INVALID_EVIDENCE)?; for (header, expected) in [ ("Zotero-API-Version", "3".to_owned()), ( diff --git a/crates/conceptweave-zotero/src/full_text_capture_tests.rs b/crates/conceptweave-zotero/src/full_text_capture_tests.rs index d02311eb..d3ce7cdf 100644 --- a/crates/conceptweave-zotero/src/full_text_capture_tests.rs +++ b/crates/conceptweave-zotero/src/full_text_capture_tests.rs @@ -433,3 +433,36 @@ fn capture_rejects_clock_failure_and_late_results_without_real_waiting() { assert!(result.is_err(), "clock poll {failed_poll}"); } } + +#[test] +fn every_failed_request_and_invalid_replay_input_fails_closed() { + let report = report_fixture(); + for failed_request in 0..8 { + let mut requests = 0; + let result = capture_with(&report, 4096, &mut |request_path, _| { + requests += 1; + if requests == failed_request + 1 { + Err(INVALID_EVIDENCE) + } else { + Ok(response_fixture(request_path)) + } + }); + assert!(result.is_err()); + assert_eq!(requests, failed_request + 1); + } + assert!( + capture_with(&report, 1, &mut |request_path, _| Ok(response_fixture( + request_path + ))) + .is_err() + ); + let mut capture = capture_with(&report, 4096, &mut |request_path, _| { + Ok(response_fixture(request_path)) + }) + .unwrap(); + let mut invalid_report = report_fixture(); + invalid_report.api_version = None; + assert!(verify_full_text_capture(&capture, &invalid_report).is_err()); + capture.capture_evidence.manifest_before.body = "null".into(); + assert!(verify_full_text_capture(&capture, &report).is_err()); +} From 733425df01511d894277fb8682e070f3dde03689 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sat, 5 Sep 2026 17:34:27 +0900 Subject: [PATCH 20/47] refactor(zotero): name the shared private output writer type --- crates/conceptweave-zotero/src/main.rs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/crates/conceptweave-zotero/src/main.rs b/crates/conceptweave-zotero/src/main.rs index 0a0e55ff..c42126b8 100644 --- a/crates/conceptweave-zotero/src/main.rs +++ b/crates/conceptweave-zotero/src/main.rs @@ -370,10 +370,13 @@ fn write_all_and_flush(writer: &mut BufWriter, content: &[u8]) -> io::Resu } /// Runs the private-output boundary with an injectable writer for failure testing. +type PrivateOutputWriter<'writer> = + dyn FnMut(&mut BufWriter, &[u8]) -> io::Result<()> + 'writer; + fn write_private_output_with( path: &Path, content: &[u8], - write: &mut dyn FnMut(&mut BufWriter, &[u8]) -> io::Result<()>, + write: &mut PrivateOutputWriter<'_>, ) -> io::Result<()> { let file = create_report_file(path)?; let mut writer = BufWriter::new(file); From 9cc5bef7ec57905a760ef689b6ff032ed7bb9baf Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sat, 5 Sep 2026 17:40:43 +0900 Subject: [PATCH 21/47] docs(zotero): record retained-text KPI and exact verification evidence --- .../zotero_fulltext_capture_evidence.json | 60 +++++++++++++++++++ .../zotero_fulltext_contract_audit.md | 19 ++++++ docs/product-technical-gap-baseline.md | 12 +++- 3 files changed, 89 insertions(+), 2 deletions(-) create mode 100644 docs/doctoring/zotero_fulltext_capture_evidence.json diff --git a/docs/doctoring/zotero_fulltext_capture_evidence.json b/docs/doctoring/zotero_fulltext_capture_evidence.json new file mode 100644 index 00000000..02e71fa2 --- /dev/null +++ b/docs/doctoring/zotero_fulltext_capture_evidence.json @@ -0,0 +1,60 @@ +{ + "observation_kind": "private_non_atomic_fulltext_capture", + "source_commit": "2c2226f1d583c3091cc126c96d27d55d1084c0d1", + "started_at": "2026-09-05T08:32:31.268Z", + "finished_at": "2026-09-05T08:33:00.166Z", + "read_interval_ms": 28898, + "command_elapsed_seconds": 33.12, + "maximum_resident_set_bytes": 283426816, + "peak_memory_footprint_bytes": 332956272, + "provider_version": "10.0.1", + "api_version": 3, + "schema_version": 44, + "library_version_before": 2, + "library_version_after": 2, + "capture_file_bytes": 235602798, + "capture_file_mode": "0600", + "capture_file_links": 1, + "capture_file_sha256": "56d385398c8da559aa597a4e3783d946638855bba19ac808ce81d917bf06f94d", + "capture_digest": "sha256:429d98dc90e172b4f0bb4e3e1c493feb33b61664793d02a53d8183fb76f76a50", + "metadata_report_file_sha256": "bf45248413f433a537fe8fc62c02b93eef3c7e47ff6245f31610e9ba72031d8d", + "metadata_report_digest": "sha256:3c7b83647ed3b567584cea6a784e40c97c67c9c5aac5623fbf15b3a10cfaee52", + "metadata_snapshot_digest": "sha256:0666dbebfb0c5aa99deb5a6dda1fc02d84bc46d08aaaddf25f5526a18eceef6d", + "bibliographic_denominator": 3715, + "manifest_entries": 3473, + "http_request_count": 6950, + "content_response_counts": { "200": 3432, "404": 41 }, + "empty_content_count": 5, + "index_complete_count": 2755, + "index_partial_count": 67, + "index_unknown_count": 610, + "nonempty_text_parent_count": 3203, + "nonempty_complete_index_parent_count": 2561, + "missing_nonempty_text_parent_count": 512, + "total_response_body_bytes": 232366711, + "limits": { + "per_response_bytes": 8388608, + "cumulative_response_bytes": 268435456, + "snapshot_item_count": 50000, + "sweep_admission_completion_ms": 300000, + "request_global_ms": 30000, + "request_connect_ms": 2000, + "request_response_ms": 10000, + "request_body_ms": 10000, + "concurrent_requests": 1 + }, + "verification": { + "production_verifier_before_write": true, + "independent_saved_json_digest_and_parent_validation": true, + "metadata_report_unchanged": true, + "manifest_bookends_equal": true, + "raw_text_retained_privately": true, + "raw_text_committed": false, + "atomic_provider_snapshot": false, + "authenticated_provider": false + }, + "new_text_bound_proposals": 0, + "steward_decisions": 0, + "approved_labels": 0, + "source_mutations": 0 +} diff --git a/docs/doctoring/zotero_fulltext_contract_audit.md b/docs/doctoring/zotero_fulltext_contract_audit.md index d9427f43..3e6263eb 100644 --- a/docs/doctoring/zotero_fulltext_contract_audit.md +++ b/docs/doctoring/zotero_fulltext_contract_audit.md @@ -73,6 +73,25 @@ The provider fix belongs in Zotero: cover upgrade from synced records, local ind For model-assisted proposals, protected `contextual-orchestrator/main@a080297d2546bb61e89520d637cabc202db331ec` documents API use, but its queried GitHub releases/tags returned empty and the queried PyPI project endpoint returned 404. Those checks do not rule out every deployment or registry; they leave a released integration artifact unverified. Do not replace that missing evidence with provider calls, copied owner source, invented review labels or a new utility repository. +## Follow-up: privately retained content, not reclassification + +The [capture evidence](zotero_fulltext_capture_evidence.json) records a separate live run of the proposed Rust command at `2c2226f1d583c3091cc126c96d27d55d1084c0d1`. Unlike the earlier availability audit, this run preserves exact source-response JSON privately. It performed 6,950 sequential requests: two library bookends, two complete manifests and metadata/content reads for all 3,473 manifest entries. The source-read interval was 28,898 ms; total command elapsed time was 33.12 seconds. Maximum resident memory was 283,426,816 bytes and measured peak memory footprint was 332,956,272 bytes. These are one observed local run, not a latency SLO or a production load benchmark. + +The capture retains 3,432 successful content responses, 41 missing responses and five empty content strings. Nonempty text is now privately retained for 3,203/3,715 bibliographic parents; 2,561 have nonempty text with complete index counters under the earlier predicate. The 512 without demonstrated nonempty text remain in the denominator. Counter partitions remain 2,755 complete, 67 partial and 610 unknown. No source text was summarized, classified or treated as instructions during capture. + +Response bodies total 232,366,711 bytes, including attachment metadata absent from the earlier sweep. The encoded file is 235,602,798 bytes, a new single-link `0600` file outside the repository. The production verifier checked report binding, response bounds, structure, parent/item revision and observed content versions before writing. A separate saved-file audit checked permissions, the complete artifact/evidence/report digests, parent mapping, counts and unchanged report hash without printing identities or source text. Synthetic tests independently exercise Rust deserialization/replay; the saved-file audit does not add an authority verifier. The public JSON retains only aggregate counts, times, limits and digests. No private file is published here. + +The original metadata report remains byte-identical. Retained-text coverage improves from zero replayable full-text artifacts to 3,203/3,715 parents; new text-bound proposals, authentic decisions and approved labels remain zero. Follow-up work must present the captured evidence under a new proposal/review binding, preserve missing/partial coverage, and prove a released contextual-orchestrator integration before model assistance. The Zotero version-space defect remains upstream; a consumer capture does not repair it. + +## Consumer transport and replay root-cause repairs + +- Environment proxy inheritance: RED `9aafff5` reproduced forwarding in 18 synthetic subprocess cases covering six environment-variable spellings and metadata/authorization/item paths. GREEN `a2848e5` explicitly disables proxies in both existing agents. No actual key or live write was involved, and same-host HTTP authentication remains unresolved. +- Inclusive body limits: RED `f3a2847` and `2d9ec5c` reproduced rejection at exactly 2 bytes, the 512-byte authorization limit and the 8 MiB metadata limit. Installed `ureq 3.4.0` source `src/body/limit.rs:21–25`, pinned by `Cargo.lock`, errors when its remaining counter reaches zero before checking EOF. GREEN `c959505` uses the standard library's bounded `Read::take(N+1)`, strict UTF-8 and an explicit `length > N` rejection across the common reader and metadata pagination. Above-limit, invalid-encoding, truncated and chunked-response tests still fail closed. This is a caller-side adaptation to the dependency behavior, not an upstream patch. +- Replay resource order: RED `f7d3530` proved an oversized restored response reached digest work first. GREEN `301d9d5` checks record/body limits before parsing or hashing and uses the installed SHA-256 type's standard `Write` implementation with streamed serialization. A regression proves byte-for-byte digest equivalence with the earlier compact JSON representation; no new wrapper or dependency is introduced. +- Time/failure propagation: RED `7854b3a` and GREEN `2c2226f` exercise invalid clock observations, late responses, the completion deadline and every request failure without sleeping or changing the production deadline. Public transport remains fixed loopback; only private seams accept a synthetic endpoint/clock for tests. + +These fixes are integrated into the proposed full-text branch. Earlier open owner/transport PRs must inherit the applicable repairs through ordinary history before promotion; success at this tip does not prove their old heads safe or checked. No predecessor delta was discarded. + ## References Zotero. (2026a, July 29). *Zotero local API*. https://www.zotero.org/support/dev/web_api/v3/local_api diff --git a/docs/product-technical-gap-baseline.md b/docs/product-technical-gap-baseline.md index b2c3e11a..7b868d85 100644 --- a/docs/product-technical-gap-baseline.md +++ b/docs/product-technical-gap-baseline.md @@ -13,7 +13,7 @@ The active roots observed immediately before this baseline refresh are: 1. Foundation PR #1 — exact head `b538470c963e6524ddc0c3f652a46a4fc8265150`, Draft/open. The current Foundation makes Product CI draft-aware while keeping Ready/non-Draft quality requirements intact. Product CI still cannot materialize from protected `main` because that branch does not yet contain `.github/workflows/product.yml`. 2. Product-CI bootstrap PR #35 — exact head `a31ae0c2df920f2794f7ddb456795b04797ab472`, open/non-Draft. It adds the pull-request form of Product CI and removes no-op closed/converted-to-Draft triggers. Scope detection and review admission have executed successfully, while CodeQL, Semgrep, Noema, Strix, Trivy and Scorecard remain queued. The workflow-only diff skips Dependency Review and OSV; these skips do not prove Foundation's dependency-changing checks. No independent approval exists for this head. 3. Client Consumption PR #5 — exact head `fcf36c8a99f015b963c9f812787df127ac2e2f9e`, Draft/open. The current source retains language-neutral semantic-release admission, integrity, compatibility, diff/resolution and supersession validation. Previously valid review findings are source-repaired, but current protected evidence remains independently required. -4. Source Observation PR #6 — exact head `8ed91afcf520efdd53c9103b332d3e277db29a03`, Draft/open. Its independent owner added explicit schema-allowlist count/UTF-8 byte admission and checked overflow rejection after `51a7344c6b159df8daaf2fca6540f7b712f5f8c6`; exact compare and current PR notes were inspected. This does not transfer Zotero tests to that branch. The concrete bounded read-only PostgreSQL adapter remains absent. +4. Source Observation PR #6 — exact head `f8e1c11054fa0716568ee51c51c8dca509964b31`, Draft/open. Its independent owner preserves registry denial before an authorized-request-only adapter boundary. The earlier denial-vocabulary finding is superseded by that structural repair; a counted zero adapter/source/snapshot execution regression with an authorized control was handed back to the existing owner task. This does not transfer Zotero tests to that branch. The concrete bounded read-only PostgreSQL adapter remains absent. 5. Zotero Research Classification root PR #9 — exact head `256076d12dec80997960b1db89bec0809f129c90`, Draft/open. Integrity root #10 is `e7d4e59f1b55b5954c5f8436527bc96e7ef2fb13`; all 23 descendants inherited it through ordinary merges ending at review-batch PR #34 `a359c5b9d1013e84f5832506f5a57aec364e6493`. The local coverage follow-up is `6f27da9` before this documentation commit. The stack remains proposal/review oriented and does not elevate local classifier output to semantic authority. Predecessor reviews/checks never transfer to successor heads. No force-push, destructive rebase, self-approval, fail-open scanner substitution or routine administrator bypass is acceptance evidence. @@ -80,7 +80,15 @@ At `2a75051f0082103511222e278de24b2690fe6bfe`, the [read-only full-text sweep](d Official Zotero 10.0.1 source `36749bd0bd4fdac9ee46c16f7aa7bed094a0851f` confirms that the full-text version storage receives remote sync versions, zero-valued local indexing and local API client versions, while the list omits the documented library-version header. Unchanged bookend manifest bytes and metadata library version 2 therefore do not prove an atomic full-text snapshot or a safe incremental cursor. No source file/database repair, new text-bound proposal, steward decision, approval or Zotero write is claimed. PRD FR-9, TRD and Proposed ADR 0006 now require separately captured content evidence; lifecycle capability metric remains 25 and approved full review remains 0/3,715. -Next: implement and verify that bounded content-capture boundary without reusing old approvals; retain partial/missing text in the denominator; track the upstream version-contract repair. A released contextual-orchestrator integration artifact remains unverified at the audited protected owner head, so source documentation alone does not authorize model-provider bypass. No new utility repository is justified by this one intake seam. +### Privately retained full-text capture + +The new branch implements and verifies the separate content-capture boundary. Its [live evidence](doctoring/zotero_fulltext_capture_evidence.json) is bound to `2c2226f1d583c3091cc126c96d27d55d1084c0d1`: all 3,473 manifest entries retained in a new private file, 3,432 successful and 41 missing content responses, nonempty text for 3,203/3,715 bibliographic parents and complete-counter nonempty text for 2,561. The remaining 512 parents are not excluded. Source-read interval is 28,898 ms, total command time 33.12 seconds, maximum resident memory 283,426,816 bytes and total response bodies 232,366,711 bytes. The single-link `0600` artifact is 235,602,798 bytes; its file digest is `56d385398c8da559aa597a4e3783d946638855bba19ac808ce81d917bf06f94d`. A separate saved-file audit verified content/report digests, parent links and counts. The original metadata report hash remains unchanged. + +Replayable retained-text coverage has progressed from 0 to 3,203/3,715 parents. Lifecycle capability metric remains 25; new text-bound proposals, authentic steward decisions and externally approved labels remain 0. Neither stable provider bookends nor a recomputed local digest authenticates source authority. No Zotero write, model-provider bypass, release or protected merge is claimed. + +Committed regressions repaired inherited environment proxies, exact-byte-limit rejection and replay checks occurring after digest allocation; clock fault injection verifies late/invalid-clock failures without changing the deadline. Final source verification at `733425df01511d894277fb8682e070f3dde03689` passed 173 tests across 37 suites including documentation tests, strict Clippy, formatting, rustdoc with warnings denied, the CI contract and the existing coverage gate. Coverage is 347/347 functions, 3,710/3,710 source-normalized regions and 674/674 source-normalized branch outcomes. Raw LLVM totals remain 4,159/4,255 lines, 6,129/6,274 regions and 603/674 branch outcomes; those are not 100%. The only source delta after the live run is a writer type alias resolving strict Clippy's complexity finding without changing runtime behavior. Hosted checks and independent protected approval remain separate gates. + +Next: propagate applicable transport repairs to the earlier owner stack without losing deltas; present retained text under new proposal/review bindings with partial/missing coverage; continue the 63 remaining repository capability audits and the upstream version-contract repair. A released contextual-orchestrator integration artifact remains unverified at the audited protected owner head, so source documentation alone does not authorize model-provider bypass. No new utility repository is justified by this one intake seam. ### Historical pre-repair Zotero 10 transition From e19d95f42c0f745cb428133ee7c4a15043e76744 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sat, 5 Sep 2026 17:43:43 +0900 Subject: [PATCH 22/47] docs(zotero): link the capture successor and current stack gates --- docs/product-technical-gap-baseline.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/product-technical-gap-baseline.md b/docs/product-technical-gap-baseline.md index 7b868d85..e7c95814 100644 --- a/docs/product-technical-gap-baseline.md +++ b/docs/product-technical-gap-baseline.md @@ -14,7 +14,7 @@ The active roots observed immediately before this baseline refresh are: 2. Product-CI bootstrap PR #35 — exact head `a31ae0c2df920f2794f7ddb456795b04797ab472`, open/non-Draft. It adds the pull-request form of Product CI and removes no-op closed/converted-to-Draft triggers. Scope detection and review admission have executed successfully, while CodeQL, Semgrep, Noema, Strix, Trivy and Scorecard remain queued. The workflow-only diff skips Dependency Review and OSV; these skips do not prove Foundation's dependency-changing checks. No independent approval exists for this head. 3. Client Consumption PR #5 — exact head `fcf36c8a99f015b963c9f812787df127ac2e2f9e`, Draft/open. The current source retains language-neutral semantic-release admission, integrity, compatibility, diff/resolution and supersession validation. Previously valid review findings are source-repaired, but current protected evidence remains independently required. 4. Source Observation PR #6 — exact head `f8e1c11054fa0716568ee51c51c8dca509964b31`, Draft/open. Its independent owner preserves registry denial before an authorized-request-only adapter boundary. The earlier denial-vocabulary finding is superseded by that structural repair; a counted zero adapter/source/snapshot execution regression with an authorized control was handed back to the existing owner task. This does not transfer Zotero tests to that branch. The concrete bounded read-only PostgreSQL adapter remains absent. -5. Zotero Research Classification root PR #9 — exact head `256076d12dec80997960b1db89bec0809f129c90`, Draft/open. Integrity root #10 is `e7d4e59f1b55b5954c5f8436527bc96e7ef2fb13`; all 23 descendants inherited it through ordinary merges ending at review-batch PR #34 `a359c5b9d1013e84f5832506f5a57aec364e6493`. The local coverage follow-up is `6f27da9` before this documentation commit. The stack remains proposal/review oriented and does not elevate local classifier output to semantic authority. +5. Zotero Research Classification root PR #9 — exact head `256076d12dec80997960b1db89bec0809f129c90`, Draft/open. Integrity root #10 is `e7d4e59f1b55b5954c5f8436527bc96e7ef2fb13`; all 23 descendants inherited it through ordinary merges. Review-batch PR #34 is now `2e6448e896e65562ebeee2fd339dec64d9fdf6e5`. [Full-text capture PR #36](https://github.com/ContextualWisdomLab/ConceptWeave/pull/36) is its Draft child, created at `9cc5bef7ec57905a760ef689b6ff032ed7bb9baf`. At that observation it had only a successful CodeRabbit status context and no submitted reviews; this is not independent approval or hosted Product verification. The stack remains proposal/review oriented and does not elevate local classifier output to semantic authority. Predecessor reviews/checks never transfer to successor heads. No force-push, destructive rebase, self-approval, fail-open scanner substitution or routine administrator bypass is acceptance evidence. From 7cfa8d777e76fab8adcfdf483ac634fcd951f6f6 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sat, 5 Sep 2026 18:09:59 +0900 Subject: [PATCH 23/47] docs(research): bind owner audits to source license and release evidence --- docs/PRD.md | 2 +- docs/TRD.md | 2 ++ .../cwl_ontology_capability_inventory.md | 20 ++++++++++++------- .../zotero_fulltext_contract_audit.md | 10 +++++++++- docs/product-technical-gap-baseline.md | 16 ++++++++++----- 5 files changed, 36 insertions(+), 14 deletions(-) diff --git a/docs/PRD.md b/docs/PRD.md index 98a7ef40..518460cc 100644 --- a/docs/PRD.md +++ b/docs/PRD.md @@ -52,7 +52,7 @@ Support stable adapters for `semantic-data-portal`, `LineageWeave`, `context-gra ### FR-8 LLM assistance -All LLM-backed induction uses `contextual-orchestrator`. Model output is untrusted proposal data and may not skip deterministic validation or review. +All LLM-backed induction uses released `contextual-orchestrator` contracts. Model output is untrusted proposal data and may not skip deterministic validation or review. A documented API, Draft release proposal or successful maintenance job does not prove the deployed service is ready; missing release and runtime evidence keeps model assistance unavailable without excluding papers from review. ### FR-9 Research evidence intake diff --git a/docs/TRD.md b/docs/TRD.md index 50430556..2e64d85b 100644 --- a/docs/TRD.md +++ b/docs/TRD.md @@ -41,6 +41,8 @@ The initial Rust and JSON contracts cover candidate kind, truth status, publicat LLM calls go through `contextual-orchestrator`. The application sends bounded evidence/context and receives structured proposals. LLM output is never a database command, publication decision, validation result, or source-system mutation. Deterministic checks must be able to reject the output without another model call. +Before enabling that adapter, verify an immutable owner artifact and schema digest, protected-source provenance, an identified gateway's deployed version and exact-consumer contract evidence. A provider-catalog maintenance deployment is not serving-gateway evidence. The [current owner audit](doctoring/zotero_fulltext_contract_audit.md#released-orchestration-evidence) connects the unfulfilled release gate to existing owner work; copied source, mutable branches and direct-provider calls are not fallback integrations. + ## 7. Standards strategy Stable publication targets use stable recommendations first: RDF 1.1, OWL 2, SKOS, SHACL 1.0, JSON-LD 1.1, and PROV-O as applicable. RDF 1.2 and SHACL 1.2 are tracked as 2026 drafts/candidate work and are not silently treated as final standards. Apache Ossie (incubating; formerly OSI) is tracked as an emerging semantic-model exchange format for metrics, dimensions, relationships, and datasets. diff --git a/docs/doctoring/cwl_ontology_capability_inventory.md b/docs/doctoring/cwl_ontology_capability_inventory.md index 65ee7dca..2816a78d 100644 --- a/docs/doctoring/cwl_ontology_capability_inventory.md +++ b/docs/doctoring/cwl_ontology_capability_inventory.md @@ -4,11 +4,11 @@ Evidence snapshot: 2026-09-05. Status: research inventory, not dependency-adopti ## Scope and evidence limits -The paginated organization metadata census returned 76 repositories, including one archived repository and 11 forks. Name/description screening for ontology, semantic, knowledge graph, schema, RDF, SHACL, provenance and lineage was combined with the existing Context Map; metadata matches alone miss owners whose descriptions do not use those words. This expanded the original eight-candidate audit to 12 selected candidates. The organization product-goal directive then identified DiskSage's actual OWL use, expanding the audit to 13. The other 63 repositories have not received a source-level capability audit, so this is not proof that all relevant implementations have been found. +The paginated organization metadata census returned 76 repositories, including one archived repository and 11 forks. Name/description screening for ontology, semantic, knowledge graph, schema, RDF, SHACL, provenance and lineage was combined with the existing Context Map; metadata matches alone miss owners whose descriptions do not use those words. This expanded the original eight-candidate audit to 12 selected candidates. The organization product-goal directive then identified DiskSage's actual OWL use, expanding the audit to 13. Subsequent naruon sender-ontology and pg-erd-cloud schema-engineering audits bring the selected total to 15. The other 61 repositories have not received a source-level capability audit, so this is not proof that all relevant implementations have been found. -Each row separates responsibility, exact default-head documentation or tree evidence, and published GitHub release evidence. A name, open PR or protected branch alone does not prove a usable released API. Package registries, deployed behavior, release attestations and consumer conformance were not audited here. CalendarWeave and four-pillars metadata also matched, but their descriptions identify domain-specific calendar/calculation consumers, not a demonstrated shared ontology library; this is a screening disposition, not an architectural exclusion. +Each row separates responsibility, exact default-head documentation or tree evidence, and published GitHub release evidence. A name, open PR or protected branch alone does not prove a usable released API. This is not a complete package-registry, deployment, attestation or consumer-conformance audit; bounded follow-up attempts and their limitations are recorded below. CalendarWeave and four-pillars metadata also matched, but their descriptions identify domain-specific calendar/calculation consumers, not a demonstrated shared ontology library; this is a screening disposition, not an architectural exclusion. -GitHub repository, branch, README-at-SHA, complete-tree and release endpoints supplied these observations. All 13 selected default branches reported protected at their recorded observations. `context-graph-contracts` and `enterprise-architecture-core` default to `develop`; their unprotected `main` branches are not the adoption baseline. GitHub detected MIT for semantic-data-portal, TEPP, LineageWeave, fast-mlsirm and disksage; Apache-2.0 for RankWeave, graphify, Veilpick and mhtml-etl-gateway; and no SPDX identifier for the other four. An undetected license is unresolved evidence, not permission to adopt. DeepWiki had no indexed evidence for graphify, Veilpick or disksage, so their exact GitHub source/tree was used instead. +GitHub repository, branch, README-at-SHA, complete-tree and release endpoints supplied these observations. All 15 selected default branches reported protected at their recorded observations. `context-graph-contracts`, `enterprise-architecture-core` and naruon default to `develop`; do not substitute a branch named `main` for the actual default. For the original 13 candidates, GitHub detected MIT for semantic-data-portal, TEPP, LineageWeave, fast-mlsirm and disksage; Apache-2.0 for RankWeave, graphify, Veilpick and mhtml-etl-gateway; and no SPDX identifier for the other four. An undetected license is unresolved evidence, not permission to adopt. The additional naruon audit read an explicitly proprietary license despite public visibility and NOASSERTION metadata; pg-erd-cloud's exact license contains Apache-2.0 without an appended noncommercial restriction. DeepWiki had no indexed evidence for graphify, Veilpick, disksage or pg-erd-cloud, so their exact GitHub source/tree was used instead. ## Owner and maturity evidence @@ -27,10 +27,16 @@ GitHub repository, branch, README-at-SHA, complete-tree and release endpoints su | graphify | Forked code/document knowledge-graph extraction distinguishes extracted and inferred edges; that distinction alone does not implement governed semantic releases. | `v8@ac16a93bd3f31b86c82f7d90687941e6d5c9776d`; [extraction and backend documentation](https://github.com/ContextualWisdomLab/graphify/blob/ac16a93bd3f31b86c82f7d90687941e6d5c9776d/README.md). | No CWL GitHub release returned. This is a fork of Graphify-Labs/graphify; upstream package documentation is not a CWL release or adoption receipt. Evaluate a versioned evidence port without importing its graph as semantic authority or bypassing contextual-orchestrator for model work. | | Veilpick | Repository metadata proposes ontology-guided web acquisition, outside ConceptWeave's browser-acquisition boundary. | `develop@8fd6931092ccc2076b10e9eb23ac99b404a9880e`; [complete source tree](https://github.com/ContextualWisdomLab/Veilpick/tree/8fd6931092ccc2076b10e9eb23ac99b404a9880e) contains only `LICENSE`; README endpoint returned 404. | No GitHub release returned. No implemented library, API or accepted owner contract is evidenced at this head. Keep the metadata claim separate from implementation; do not create a dependency from a description. | | DiskSage (`disksage`) | Implemented filesystem taxonomy and ontology-driven organization; candidate for bounded reuse, not ownership of general semantic-model publication. | `main@0e90f9cebadbd7f59606baaec4ca1d2f178c899a`; [bundled eight-class ontology](https://github.com/ContextualWisdomLab/disksage/blob/0e90f9cebadbd7f59606baaec4ca1d2f178c899a/src-tauri/resources/ontology/default.ttl) and [Rust named-class reasoning subset](https://github.com/ContextualWisdomLab/disksage/blob/0e90f9cebadbd7f59606baaec4ca1d2f178c899a/src-tauri/src/ontology.rs). | No GitHub release returned. Protected source implements subclass/equivalence closure and disjoint-class checks; it is not full OWL reasoning or a released library. [Accepted ADR 0010](https://github.com/ContextualWisdomLab/disksage/blob/0e90f9cebadbd7f59606baaec4ca1d2f178c899a/docs/architecture/adr/0010-rooted-organize-destinations.md) constrains organize destinations. Establish a released owner contract with provenance and locale preservation before reuse; do not copy the private module. | +| naruon | Product-owned email/PIM sender relationships; not the canonical shared ontology generator or a general semantic-publication library. | `develop@042b0c70531b229af3acbd0421a2f23098d848b3`; [relationship API](https://github.com/ContextualWisdomLab/naruon/blob/042b0c70531b229af3acbd0421a2f23098d848b3/backend/api/ontology.py), [implementation](https://github.com/ContextualWisdomLab/naruon/blob/042b0c70531b229af3acbd0421a2f23098d848b3/backend/services/ontology_service.py), and [proprietary LICENSE](https://github.com/ContextualWisdomLab/naruon/blob/042b0c70531b229af3acbd0421a2f23098d848b3/LICENSE). | 37 GitHub releases returned. Latest [v0.14.4](https://github.com/ContextualWisdomLab/naruon/releases/tag/v0.14.4) resolves to `efbdc7bba1bc7b66016d687b63c8f5fa7b78cef4`, with no attached assets and the same proprietary license. It does not satisfy permissive-library adoption. Preserve its product meaning; any future integration requires a licensed, versioned contract and consumer conformance. | +| pg-erd-cloud | Adjacent schema-engineering and ERD collaboration product; potential relational-schema evidence source, not semantic-publication authority. | `main@8dc746920c12988f082e914879d95e13c9693535`; [relational metadata extraction](https://github.com/ContextualWisdomLab/pg-erd-cloud/blob/8dc746920c12988f082e914879d95e13c9693535/backend/app/pg_introspect/introspect.py#L133-L184), [snapshot APIs](https://github.com/ContextualWisdomLab/pg-erd-cloud/blob/8dc746920c12988f082e914879d95e13c9693535/backend/app/api/snapshots.py#L190-L402), and [Apache-2.0 LICENSE](https://github.com/ContextualWisdomLab/pg-erd-cloud/blob/8dc746920c12988f082e914879d95e13c9693535/LICENSE). | No GitHub release or tag returned. Declared 0.1.0 source/package versions are not distribution evidence. Release a bounded source-observation contract and prove exact-consumer compatibility before integration; do not copy its Python implementation into ConceptWeave. | DiskSage demonstrates why metadata screening alone is insufficient. Its [desktop registration](https://github.com/ContextualWisdomLab/disksage/blob/0e90f9cebadbd7f59606baaec4ca1d2f178c899a/src-tauri/src/lib.rs#L122) connects ontology/coherence/inventory/organization commands, while its [catalog adapter](https://github.com/ContextualWisdomLab/disksage/blob/0e90f9cebadbd7f59606baaec4ca1d2f178c899a/src-tauri/src/semantic_catalog.rs) emits a bounded version-1 preview, not catalog publication. The [package](https://github.com/ContextualWisdomLab/disksage/blob/0e90f9cebadbd7f59606baaec4ca1d2f178c899a/src-tauri/Cargo.toml) is `publish=false`. Its parser retains the first label without preserving a locale map, and comments saying no reasoning/first parent only lag the implementation. These are owner cultivation findings, not authorization to extract its product truth or a claim that this audit ran its tests/runtime. Local branch-only deletion/retention additions are excluded from protected-main evidence. -No runtime dependency, repository, service or database was added. The [Context Map](../CONTEXT_MAP.md) and [ADR 0006](../adr/0006-zotero-research-intake.md) still place Research Intake in ConceptWeave. A separate utility owner needs an evidenced independent consumer and deployment contract. +naruon is a source-backed product-domain finding, not an adoption recommendation. Signed-session owner/organization scopes protect relationship reads and source-email capture derives thread provenance server-side. However, the legacy relationship POST still accepts caller-supplied classifications and optional source references. The service uses keyword/domain heuristics and fixed confidence values; this does not establish calibrated inference, universal source binding, DAG conformance or immutable semantic publication. Exact-head [Application CI](https://github.com/ContextualWisdomLab/naruon/actions/runs/33308065094) reports backend/frontend success with no artifacts. Container-package verification was unavailable because the current token lacked `read:packages`; package visibility, version and runtime remain unverified. The local `8d0fc0c7fb36858f2a89a9a6df9f2378716772cf` checkout was not substituted for protected default source. No naruon tests, service or data access were run by this audit. + +pg-erd-cloud returns relational snapshots, diffs, dictionary/export material and inferred relationship proposals. Its [relationship inference](https://github.com/ContextualWisdomLab/pg-erd-cloud/blob/8dc746920c12988f082e914879d95e13c9693535/backend/app/spec/relationship_inference.py) is explicitly a name/type heuristic using identifier suffixes, simple pluralization, same-schema matching and fixed confidence categories. Those categories are not calibrated semantic evidence. Its opt-in [LLM transport](https://github.com/ContextualWisdomLab/pg-erd-cloud/blob/8dc746920c12988f082e914879d95e13c9693535/backend/app/spec/llm.py#L76-L120) accepts configured endpoint/model values and returns chat text; this does not demonstrate a released contextual-orchestrator contract or a structured source-bound proposal receipt. Classic protection's zero approval count does not mean approval-free merging: effective organization rules also require one approval and seven central workflows. No runtime, private database, model call or source adoption was performed. + +No runtime dependency, repository, service or database was added. The [Context Map](../CONTEXT_MAP.md) and [ADR 0006](../adr/0006-zotero-research-intake.md) still place Research Intake in ConceptWeave. A separate utility owner needs an evidenced independent consumer and deployment contract. The separate [orchestration release audit](zotero_fulltext_contract_audit.md#released-orchestration-evidence) tracks the existing model owner; it does not count that support service as another ontology implementation. ## Actual Zotero evidence @@ -48,10 +54,10 @@ The PR #10 findings for [provider metadata lost before hashing](https://github.c | Measure | Observation | Required next evidence | | --- | --- | --- | -| Metadata census / bounded capability audit | 76 metadata records; 13/13 selected candidates have exact default-head documentation/source/tree and release-query evidence | Audit actual contracts/consumers; metadata coverage is not full source-level discovery, and 63 repositories remain unaudited at that depth. | -| GitHub release with resolved source commit | 3/13 selected candidates | Artifact/provenance and consumer conformance; a tag is insufficient adoption evidence. | +| Metadata census / bounded capability audit | 76 metadata records; 15/15 selected candidates have exact default-head documentation/source/tree and release-query evidence | Audit actual contracts/consumers; metadata coverage is not full source-level discovery, and 61 repositories remain unaudited at that depth. | +| GitHub release with resolved source commit | 4/15 selected candidates, including proprietary naruon | License, artifact/provenance and consumer conformance; a release count is not permissive-library or adoption evidence. | | Verified ConceptWeave adoption | 0 demonstrated in this audit | Released owner contract, exact consumer revision and passing contract/runtime evidence. | -| Demonstrated nonempty full-text availability | 3,203/3,715 parents, including 800/1,000 without retained abstracts | Separate immutable content capture, explicit partial/unknown indexing and review; neither an atomic metadata snapshot nor classification progress. | +| Privately retained nonempty full text | 3,203/3,715 parents in the separate [capture evidence](zotero_fulltext_capture_evidence.json), including 800/1,000 without retained abstracts | New content-bound proposals and authentic review with partial/unknown indexing; neither an atomic metadata snapshot nor classification progress. | | Unverified steward decisions | 0/3,715 on the repaired snapshot; first pending batch has 0/25 decisions | Authentic snapshot-bound decisions; batch generation is not review progress. | | Externally approved full review | 0/3,715 | Complete labels and independently verified approval; no sampled denominator or generated labels. | diff --git a/docs/doctoring/zotero_fulltext_contract_audit.md b/docs/doctoring/zotero_fulltext_contract_audit.md index 3e6263eb..de98ce0d 100644 --- a/docs/doctoring/zotero_fulltext_contract_audit.md +++ b/docs/doctoring/zotero_fulltext_contract_audit.md @@ -71,7 +71,15 @@ Research Intake remains in ConceptWeave. Full text needs a separate immutable ca The provider fix belongs in Zotero: cover upgrade from synced records, local indexing/reindexing, sync downloads/uploads, local API content writes, missing/pending content, and list/header consistency with regression tests. Until a released provider contract proves those semantics, a consumer must re-enumerate from zero and treat version fields as opaque observations. A complete capture can claim exactly the bytes observed, never an atomic cross-endpoint snapshot on the evidence available here. No upstream issue or provider patch was published by this audit. -For model-assisted proposals, protected `contextual-orchestrator/main@a080297d2546bb61e89520d637cabc202db331ec` documents API use, but its queried GitHub releases/tags returned empty and the queried PyPI project endpoint returned 404. Those checks do not rule out every deployment or registry; they leave a released integration artifact unverified. Do not replace that missing evidence with provider calls, copied owner source, invented review labels or a new utility repository. +### Released orchestration evidence + +The 2026-09-05 08:58 UTC owner audit verified public MIT source at protected `contextual-orchestrator/main@a080297d2546bb61e89520d637cabc202db331ec`. Paginated GitHub queries returned zero releases and tags; the exact PyPI project and organization container-package name `contextual-orchestrator` returned 404. These observations do not rule out other names, registries or deployments. The [owner changelog](https://github.com/ContextualWisdomLab/contextual-orchestrator/blob/a080297d2546bb61e89520d637cabc202db331ec/CHANGELOG.md#L3-L11) explicitly labels 0.2.0 Unreleased. Documented `/openapi.json` and local `/v1/responses` examples establish source contracts, not a published schema digest or deployed gateway receipt. + +GitHub does contain deployment records: 66 observed records had 57 failure, one queued and eight success states. All eight successes resolve to Provider catalog sync. The latest successful deployment `6276208512` binds `2e414d15ba58f28597751b625a8a2f00fc9fadcf` to [run 33934725405](https://github.com/ContextualWisdomLab/contextual-orchestrator/actions/runs/33934725405); its job refreshes a PostgreSQL-backed provider catalog, stops its containers and supplies no environment URL. An environment named production does not establish a running model gateway. No model request or credential inspection was performed in this audit. + +The existing owner [PR #1030](https://github.com/ContextualWisdomLab/contextual-orchestrator/pull/1030), Draft at `f753f453ce4fc3dbc612bb9bdbb8db4cbfd93c16`, already owns immutable release work under ADR 0129. Its owner confirmed that release artifacts, schema digest and gateway deployed-version evidence are still unproved; this audit requested those results there rather than duplicating release machinery. At the subsequent check, both branch and Git-ref endpoints still returned `a080297d2546bb61e89520d637cabc202db331ec`, while PR #1030's base object returned `2e414d15ba58f28597751b625a8a2f00fc9fadcf`. The PR base observation is not substituted for the default-branch ref. + +Admission requires an immutable owner artifact and schema digest, protected-source provenance, an identified deployed gateway version and exact-consumer contract evidence. Until then, no model-assisted proposal is generated through copied source, a temporary branch or a direct provider. Catalog-sync success, source documentation and a Draft release PR cannot satisfy this gate. Review labels must not be invented to compensate for unavailable model assistance. ## Follow-up: privately retained content, not reclassification diff --git a/docs/product-technical-gap-baseline.md b/docs/product-technical-gap-baseline.md index e7c95814..bf705f77 100644 --- a/docs/product-technical-gap-baseline.md +++ b/docs/product-technical-gap-baseline.md @@ -13,8 +13,8 @@ The active roots observed immediately before this baseline refresh are: 1. Foundation PR #1 — exact head `b538470c963e6524ddc0c3f652a46a4fc8265150`, Draft/open. The current Foundation makes Product CI draft-aware while keeping Ready/non-Draft quality requirements intact. Product CI still cannot materialize from protected `main` because that branch does not yet contain `.github/workflows/product.yml`. 2. Product-CI bootstrap PR #35 — exact head `a31ae0c2df920f2794f7ddb456795b04797ab472`, open/non-Draft. It adds the pull-request form of Product CI and removes no-op closed/converted-to-Draft triggers. Scope detection and review admission have executed successfully, while CodeQL, Semgrep, Noema, Strix, Trivy and Scorecard remain queued. The workflow-only diff skips Dependency Review and OSV; these skips do not prove Foundation's dependency-changing checks. No independent approval exists for this head. 3. Client Consumption PR #5 — exact head `fcf36c8a99f015b963c9f812787df127ac2e2f9e`, Draft/open. The current source retains language-neutral semantic-release admission, integrity, compatibility, diff/resolution and supersession validation. Previously valid review findings are source-repaired, but current protected evidence remains independently required. -4. Source Observation PR #6 — exact head `f8e1c11054fa0716568ee51c51c8dca509964b31`, Draft/open. Its independent owner preserves registry denial before an authorized-request-only adapter boundary. The earlier denial-vocabulary finding is superseded by that structural repair; a counted zero adapter/source/snapshot execution regression with an authorized control was handed back to the existing owner task. This does not transfer Zotero tests to that branch. The concrete bounded read-only PostgreSQL adapter remains absent. -5. Zotero Research Classification root PR #9 — exact head `256076d12dec80997960b1db89bec0809f129c90`, Draft/open. Integrity root #10 is `e7d4e59f1b55b5954c5f8436527bc96e7ef2fb13`; all 23 descendants inherited it through ordinary merges. Review-batch PR #34 is now `2e6448e896e65562ebeee2fd339dec64d9fdf6e5`. [Full-text capture PR #36](https://github.com/ContextualWisdomLab/ConceptWeave/pull/36) is its Draft child, created at `9cc5bef7ec57905a760ef689b6ff032ed7bb9baf`. At that observation it had only a successful CodeRabbit status context and no submitted reviews; this is not independent approval or hosted Product verification. The stack remains proposal/review oriented and does not elevate local classifier output to semantic authority. +4. Source Observation PR #6 — exact head `c362a73403b6bda2cc0e94de913e39f3139d6205`, Draft/open. Its independent owner preserves registry denial before an authorized-request-only adapter boundary. The counted regression now verifies zero adapter/source/snapshot executions on denial and one of each for an authorized control, retaining the existing denial result. This audit checked source and formatting, not that branch's runtime or coverage. The new submitted repair report is COMMENTED, not approval; current-head Actions/check-runs were absent. The concrete bounded read-only PostgreSQL adapter remains absent. +5. Zotero Research Classification root PR #9 — exact head `a2a84884f67dcac6f6892c958d55450aea6d6c88`, Draft/open. A minimal owner backport reproduces and repairs proxy inheritance and exact-byte-limit rejection without bringing later full-text features backward. Integrity root #10 was `e7d4e59f1b55b5954c5f8436527bc96e7ef2fb13`; all 23 descendants inherited that earlier repair through ordinary merges. The transport repair is now being propagated forward. Review-batch PR #34's pre-propagation checkpoint is `2e6448e896e65562ebeee2fd339dec64d9fdf6e5`. [Full-text capture PR #36](https://github.com/ContextualWisdomLab/ConceptWeave/pull/36) is its Draft child at the audited `e19d95f42c0f745cb428133ee7c4a15043e76744`. At that observation it had only a successful CodeRabbit status context, explicitly skipped Draft review and no submitted reviews; this is not independent approval or hosted Product verification. The stack remains proposal/review oriented and does not elevate local classifier output to semantic authority. Predecessor reviews/checks never transfer to successor heads. No force-push, destructive rebase, self-approval, fail-open scanner substitution or routine administrator bypass is acceptance evidence. @@ -43,7 +43,7 @@ Acceptance remains stricter than the RCA. HTTP 200 availability alone is not GRE ## Central control-plane evidence -Protected central source is `.github/main@6d7fbebec8aec31d88a30a36e71ca5b3925d241d` at this snapshot. This is evidence only, not a mutable ConceptWeave dependency. +Protected central source is `.github/main@8aea81323d93e90c79b71d7718de2798919fa1df` at this snapshot. This is evidence only, not a mutable ConceptWeave dependency. Five commits after the prior `6d7fbebec8aec31d88a30a36e71ca5b3925d241d` checkpoint repair admission coverage and remove two echo-only review jobs, alongside governance-documentation updates; this does not renew evidence for already-created consumer runs. - The current central source includes queue/admission and changed-scope/review-runtime repairs already integrated through ordinary protected history. - `.github#1873@41935494aa234eb458f1cc08f006daaa278b9760` is closed/unmerged because repository dependency-graph initialization, not its retry/sleep source delta, was the verified root cause of the observed public-repository 403. @@ -70,7 +70,7 @@ All four new artifacts remain private mode `0600`, outside the repository: The repaired worksheet has 0/3,715 decisions and the first batch has 0/25; externally approved full-review coverage remains 0/3,715. No authorization prompt, approval, Zotero write, record merge/deletion or rollback was performed. The three historical Zotero 9 artifact hashes below were rechecked unchanged. The pre-repair schema-44 artifacts are also preserved. Stronger source binding is not classification correctness, business approval or loopback peer authentication. -The [CWL ontology capability inventory](doctoring/cwl_ontology_capability_inventory.md) now separates a 76-repository metadata census from a 13-candidate exact-default-head capability audit. Three selected candidates have GitHub releases with resolved source commits: RankWeave, mhtml-etl-gateway and fast-mlsirm. Veilpick's protected default tree contains only a license despite its ontology-related description; graphify is an upstream fork without a returned CWL GitHub release. The organization directive exposed DiskSage's implemented Rust named-class ontology subset at protected `main@0e90f9cebadbd7f59606baaec4ca1d2f178c899a`; it owns filesystem taxonomy and organization, not general semantic publication, and has no returned GitHub release. These are maturity observations, not adoption receipts. Source-level discovery remains incomplete for 63 repositories, and actual ConceptWeave adoption remains unproved. No additional utility owner is justified yet. +The [CWL ontology capability inventory](doctoring/cwl_ontology_capability_inventory.md) now separates a 76-repository metadata census from a 15-candidate exact-default-head capability audit, up from 13. Four selected candidates have GitHub releases with resolved source commits: RankWeave, mhtml-etl-gateway, fast-mlsirm and naruon. naruon owns product-specific sender relationships and is explicitly proprietary, so its release does not satisfy permissive-library adoption. pg-erd-cloud implements adjacent relational-schema evidence and heuristic relation proposals under Apache-2.0, with no returned GitHub release or tag. DiskSage owns its implemented Rust filesystem ontology subset, not general semantic publication. Veilpick's protected tree contains only a license; graphify is an upstream fork without a returned CWL release. These are bounded maturity observations, not adoption receipts. Source-level discovery remains incomplete for 61 repositories, and actual ConceptWeave adoption remains unproved. No additional utility owner is justified yet. Next: apply only authentic snapshot-bound steward decisions to the repaired batch, cover all 3,715 items and independently verify full-review approval; continue owner contract discovery and protected Foundation work while external checks/reviews are pending. The current body notes for PRs #11–#33 distinguish inherited source repair from historical head/check claims. No PR was closed, approved, merged or retargeted by this repair. @@ -88,7 +88,13 @@ Replayable retained-text coverage has progressed from 0 to 3,203/3,715 parents. Committed regressions repaired inherited environment proxies, exact-byte-limit rejection and replay checks occurring after digest allocation; clock fault injection verifies late/invalid-clock failures without changing the deadline. Final source verification at `733425df01511d894277fb8682e070f3dde03689` passed 173 tests across 37 suites including documentation tests, strict Clippy, formatting, rustdoc with warnings denied, the CI contract and the existing coverage gate. Coverage is 347/347 functions, 3,710/3,710 source-normalized regions and 674/674 source-normalized branch outcomes. Raw LLVM totals remain 4,159/4,255 lines, 6,129/6,274 regions and 603/674 branch outcomes; those are not 100%. The only source delta after the live run is a writer type alias resolving strict Clippy's complexity finding without changing runtime behavior. Hosted checks and independent protected approval remain separate gates. -Next: propagate applicable transport repairs to the earlier owner stack without losing deltas; present retained text under new proposal/review bindings with partial/missing coverage; continue the 63 remaining repository capability audits and the upstream version-contract repair. A released contextual-orchestrator integration artifact remains unverified at the audited protected owner head, so source documentation alone does not authorize model-provider bypass. No new utility repository is justified by this one intake seam. +Next: propagate applicable transport repairs to the earlier owner stack without losing deltas; present retained text under new proposal/review bindings with partial/missing coverage; continue the 61 remaining repository capability audits and the upstream version-contract repair. A released contextual-orchestrator integration artifact remains unverified at the audited protected owner head, so source documentation alone does not authorize model-provider bypass. No new utility repository is justified by this one intake seam. + +### Canonical transport repair and released-owner audit + +PR #9's committed RED `31b507ae9feaf58688cf62ddcb597a88d2223366` reproduces six environment-proxy routes and rejection of valid JSON exactly at the 8 MiB response bound. GREEN `a2a84884f67dcac6f6892c958d55450aea6d6c88` disables inherited proxies and introduces the same strict UTF-8 inclusive reader at the original metadata owner. Oversized, invalid-UTF-8 and truncated responses remain rejected. Its 38 workspace tests, strict Clippy, formatting and existing coverage gate pass; source-normalized regions are 686/686 and branches 90/90. Raw LLVM functions are 97/97, lines 981/982 and branches 89/90, not raw 100% coverage. Root review independently reran the four transport tests before the non-force push. Subsequent authenticated adapters must reuse this reader at their own introduction points; full-text feature commits are not reverse-merged into the earlier owner. + +The [released-orchestration audit](doctoring/zotero_fulltext_contract_audit.md#released-orchestration-evidence) verified no qualifying artifact or deployed gateway in the inspected channels. Its 66 deployment records include eight successes, all Provider catalog sync, not proof of a serving gateway. Existing CO [PR #1030](https://github.com/ContextualWisdomLab/contextual-orchestrator/pull/1030) owns release work and its active owner confirmed that artifact/schema/deployed-version evidence is still pending. The handoff requests that evidence without duplicate release machinery or provider bypass. New text-bound proposals and approved labels remain zero. ### Historical pre-repair Zotero 10 transition From 3ae0bbf700a4a5cdda5c7ad178d9e820cb3ef549 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sat, 5 Sep 2026 18:15:44 +0900 Subject: [PATCH 24/47] docs(research): keep transient subgroup evidence separate from retained capture --- docs/doctoring/cwl_ontology_capability_inventory.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/doctoring/cwl_ontology_capability_inventory.md b/docs/doctoring/cwl_ontology_capability_inventory.md index 2816a78d..e1556371 100644 --- a/docs/doctoring/cwl_ontology_capability_inventory.md +++ b/docs/doctoring/cwl_ontology_capability_inventory.md @@ -57,7 +57,7 @@ The PR #10 findings for [provider metadata lost before hashing](https://github.c | Metadata census / bounded capability audit | 76 metadata records; 15/15 selected candidates have exact default-head documentation/source/tree and release-query evidence | Audit actual contracts/consumers; metadata coverage is not full source-level discovery, and 61 repositories remain unaudited at that depth. | | GitHub release with resolved source commit | 4/15 selected candidates, including proprietary naruon | License, artifact/provenance and consumer conformance; a release count is not permissive-library or adoption evidence. | | Verified ConceptWeave adoption | 0 demonstrated in this audit | Released owner contract, exact consumer revision and passing contract/runtime evidence. | -| Privately retained nonempty full text | 3,203/3,715 parents in the separate [capture evidence](zotero_fulltext_capture_evidence.json), including 800/1,000 without retained abstracts | New content-bound proposals and authentic review with partial/unknown indexing; neither an atomic metadata snapshot nor classification progress. | +| Privately retained nonempty full text | 3,203/3,715 parents in the separate [capture evidence](zotero_fulltext_capture_evidence.json); 512 remain without demonstrated nonempty text | New content-bound proposals and authentic review with partial/unknown indexing. The earlier sweep's 800/1,000 abstract-missing subgroup is availability evidence, not a remeasured retained-capture subgroup. | | Unverified steward decisions | 0/3,715 on the repaired snapshot; first pending batch has 0/25 decisions | Authentic snapshot-bound decisions; batch generation is not review progress. | | Externally approved full review | 0/3,715 | Complete labels and independently verified approval; no sampled denominator or generated labels. | From 2402604219160c8b401ec922bf3ec2d23759e887 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sat, 5 Sep 2026 18:17:51 +0900 Subject: [PATCH 25/47] docs(zotero): verify fulltext provider gap on current upstream source --- docs/doctoring/zotero_fulltext_contract_audit.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/docs/doctoring/zotero_fulltext_contract_audit.md b/docs/doctoring/zotero_fulltext_contract_audit.md index de98ce0d..13eec307 100644 --- a/docs/doctoring/zotero_fulltext_contract_audit.md +++ b/docs/doctoring/zotero_fulltext_contract_audit.md @@ -65,6 +65,12 @@ The official `10.0.1` tag resolves to `36749bd0bd4fdac9ee46c16f7aa7bed094a0851f` This is a provider contract mismatch, not evidence of a corrupt user library. The manifest bytes and metadata library version were unchanged at the bookends, but those observations do not establish atomicity or rule out a same-version full-text edit. The mixed-origin field must not become a reliable incremental cursor, an item revision or a write precondition. Adding the missing header alone would not fix the version semantics. +### Current upstream recheck + +At 2026-09-05 09:16 UTC, official maintenance `10.0@a5b4b4d20d12cf07af43d928bd66090faed1a655` and development `main@fc17dcd24ad34686cb24e6b3ffb06a6a7a5e0e5d` retained the same Local API, full-text storage and sync-engine blobs as 10.0.1. Their respective blob identities are `6cfbaf8247a5e914c92e8711be46d4431e79923e`, `ca981b95fdbd41fa00927dd606ac5a8fde0e1cb6` and `225b0449b362201ad5d8717510af3478ef15d2b9`. The [current list endpoint](https://github.com/zotero/zotero/blob/fc17dcd24ad34686cb24e6b3ffb06a6a7a5e0e5d/chrome/content/zotero/xpcom/server/server_localAPI.js#L1459-L1487) still returns the stored version without the promised library-version header. No provider repair was verified. + +Bounded official issue/PR searches returned 34 fulltext matches but no dedicated matching defect report. Historical [issue #5002](https://github.com/zotero/zotero/issues/5002) and merged [PR #5004](https://github.com/zotero/zotero/pull/5004) introduce the endpoint; [Full-text v2 draft #5673](https://github.com/zotero/zotero/issues/5673) concerns a richer format, and [semantic-search Draft PR #6012](https://github.com/zotero/zotero/pull/6012) retains the same endpoint blob at its inspected head `19e79625b1c6fbbdd75367aa85b62d5a7080d7f6`. None establishes this cursor repair. Search results are not global absence proof. No upstream issue, patch, installation change or private-library read was made during this recheck. + ## Proposed admission and owner follow-up Research Intake remains in ConceptWeave. Full text needs a separate immutable capture receipt binding server/API/schema observations, attachment and bibliographic-parent identities, content and index-statistics digests, read interval, returned status and partial/unknown coverage. Reusing an old metadata digest or governance receipt for later text is forbidden. An availability sweep may guide retrieval and steward work but cannot make unsupported content authoritative or renew prior approval. From 53e0befb2026bc7eca388f5e88760b1b844ea4ff Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sat, 5 Sep 2026 18:25:05 +0900 Subject: [PATCH 26/47] docs(zotero): record canonical authenticated transport regressions and repairs --- docs/doctoring/zotero_fulltext_contract_audit.md | 4 +++- docs/product-technical-gap-baseline.md | 2 ++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/docs/doctoring/zotero_fulltext_contract_audit.md b/docs/doctoring/zotero_fulltext_contract_audit.md index 13eec307..5c7d2d31 100644 --- a/docs/doctoring/zotero_fulltext_contract_audit.md +++ b/docs/doctoring/zotero_fulltext_contract_audit.md @@ -104,7 +104,9 @@ The original metadata report remains byte-identical. Retained-text coverage impr - Replay resource order: RED `f7d3530` proved an oversized restored response reached digest work first. GREEN `301d9d5` checks record/body limits before parsing or hashing and uses the installed SHA-256 type's standard `Write` implementation with streamed serialization. A regression proves byte-for-byte digest equivalence with the earlier compact JSON representation; no new wrapper or dependency is introduced. - Time/failure propagation: RED `7854b3a` and GREEN `2c2226f` exercise invalid clock observations, late responses, the completion deadline and every request failure without sleeping or changing the production deadline. Public transport remains fixed loopback; only private seams accept a synthetic endpoint/clock for tests. -These fixes are integrated into the proposed full-text branch. Earlier open owner/transport PRs must inherit the applicable repairs through ordinary history before promotion; success at this tip does not prove their old heads safe or checked. No predecessor delta was discarded. +These fixes are integrated into the proposed full-text branch. The earlier owner backports now preserve their own committed regressions: metadata PR #9 RED `31b507a` → GREEN `a2a8488`; authenticated transport PR #17 RED `f83a63d` → production repair `53bd1fe`; extracted authorization PR #18 RED `178e03b` → GREEN `ba1bbd2`. The shared synthetic server also needed request-framing RED `b6b618b` → test-helper repair `7bcb791`: reading only one TCP chunk could miss the POST body and close while the large response was still arriving. This was test infrastructure, not a new production failure or a reason to reduce the response-size case. + +At #17 `7bcb791853ffa794529418ee9de1337fea4e1b15`, 91 workspace tests and 20 repetitions of five focused tests passed. At #18 `ba1bbd203c2f90afbf97ac3d7eab989982e8bc09`, 99 workspace tests and ten repetitions of eight focused tests passed, including exactly 512-byte success/denial and one-byte-over rejection. Strict Clippy and formatting passed at both heads. All requests were synthetic; no actual key, authorization or Zotero write was used. The normal forward cascade remains in progress, and each descendant needs its own verification before promotion. No predecessor delta was discarded. ## References diff --git a/docs/product-technical-gap-baseline.md b/docs/product-technical-gap-baseline.md index bf705f77..fec6dd77 100644 --- a/docs/product-technical-gap-baseline.md +++ b/docs/product-technical-gap-baseline.md @@ -94,6 +94,8 @@ Next: propagate applicable transport repairs to the earlier owner stack without PR #9's committed RED `31b507ae9feaf58688cf62ddcb597a88d2223366` reproduces six environment-proxy routes and rejection of valid JSON exactly at the 8 MiB response bound. GREEN `a2a84884f67dcac6f6892c958d55450aea6d6c88` disables inherited proxies and introduces the same strict UTF-8 inclusive reader at the original metadata owner. Oversized, invalid-UTF-8 and truncated responses remain rejected. Its 38 workspace tests, strict Clippy, formatting and existing coverage gate pass; source-normalized regions are 686/686 and branches 90/90. Raw LLVM functions are 97/97, lines 981/982 and branches 89/90, not raw 100% coverage. Root review independently reran the four transport tests before the non-force push. Subsequent authenticated adapters must reuse this reader at their own introduction points; full-text feature commits are not reverse-merged into the earlier owner. +That repair now reaches authenticated PR #17 `7bcb791853ffa794529418ee9de1337fea4e1b15` and authorization PR #18 `ba1bbd203c2f90afbf97ac3d7eab989982e8bc09`. Their own committed RED cases verify actual synthetic proxy forwarding and inclusive 1 MiB/512-byte boundaries. #17 additionally repairs the existing synthetic server's incomplete POST read after a committed request-framing regression; it does not weaken production validation. Verification passed 91/99 workspace tests respectively, plus focused repetition and strict Clippy/formatting. The root independently inspected shared callers, status/header handling, key secrecy and the conflict resolutions. Downstream propagation remains ordinary merge/push with original-head and repaired-parent ancestry retained. GitHub PR base objects sometimes lag the actual branch ref, so each merge resolves the named base through fresh fetch and `ls-remote`, not that field alone. + The [released-orchestration audit](doctoring/zotero_fulltext_contract_audit.md#released-orchestration-evidence) verified no qualifying artifact or deployed gateway in the inspected channels. Its 66 deployment records include eight successes, all Provider catalog sync, not proof of a serving gateway. Existing CO [PR #1030](https://github.com/ContextualWisdomLab/contextual-orchestrator/pull/1030) owns release work and its active owner confirmed that artifact/schema/deployed-version evidence is still pending. The handoff requests that evidence without duplicate release machinery or provider bypass. New text-bound proposals and approved labels remain zero. ### Historical pre-repair Zotero 10 transition From 895e64cf2aa4f61258399eb61ceac025f47e1c94 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sat, 5 Sep 2026 18:28:49 +0900 Subject: [PATCH 27/47] docs(governance): record base-ref discrepancy without assuming its cause --- docs/product-technical-gap-baseline.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/product-technical-gap-baseline.md b/docs/product-technical-gap-baseline.md index fec6dd77..9eb472c4 100644 --- a/docs/product-technical-gap-baseline.md +++ b/docs/product-technical-gap-baseline.md @@ -94,7 +94,7 @@ Next: propagate applicable transport repairs to the earlier owner stack without PR #9's committed RED `31b507ae9feaf58688cf62ddcb597a88d2223366` reproduces six environment-proxy routes and rejection of valid JSON exactly at the 8 MiB response bound. GREEN `a2a84884f67dcac6f6892c958d55450aea6d6c88` disables inherited proxies and introduces the same strict UTF-8 inclusive reader at the original metadata owner. Oversized, invalid-UTF-8 and truncated responses remain rejected. Its 38 workspace tests, strict Clippy, formatting and existing coverage gate pass; source-normalized regions are 686/686 and branches 90/90. Raw LLVM functions are 97/97, lines 981/982 and branches 89/90, not raw 100% coverage. Root review independently reran the four transport tests before the non-force push. Subsequent authenticated adapters must reuse this reader at their own introduction points; full-text feature commits are not reverse-merged into the earlier owner. -That repair now reaches authenticated PR #17 `7bcb791853ffa794529418ee9de1337fea4e1b15` and authorization PR #18 `ba1bbd203c2f90afbf97ac3d7eab989982e8bc09`. Their own committed RED cases verify actual synthetic proxy forwarding and inclusive 1 MiB/512-byte boundaries. #17 additionally repairs the existing synthetic server's incomplete POST read after a committed request-framing regression; it does not weaken production validation. Verification passed 91/99 workspace tests respectively, plus focused repetition and strict Clippy/formatting. The root independently inspected shared callers, status/header handling, key secrecy and the conflict resolutions. Downstream propagation remains ordinary merge/push with original-head and repaired-parent ancestry retained. GitHub PR base objects sometimes lag the actual branch ref, so each merge resolves the named base through fresh fetch and `ls-remote`, not that field alone. +That repair now reaches authenticated PR #17 `7bcb791853ffa794529418ee9de1337fea4e1b15` and authorization PR #18 `ba1bbd203c2f90afbf97ac3d7eab989982e8bc09`. Their own committed RED cases verify actual synthetic proxy forwarding and inclusive 1 MiB/512-byte boundaries. #17 additionally repairs the existing synthetic server's incomplete POST read after a committed request-framing regression; it does not weaken production validation. Verification passed 91/99 workspace tests respectively, plus focused repetition and strict Clippy/formatting. The root independently inspected shared callers, status/header handling, key secrecy and the conflict resolutions. Downstream propagation remains ordinary merge/push with original-head and repaired-parent ancestry retained. Observed GitHub PR base objects differed from the actual branch ref, so each merge resolves the named base through fresh fetch and `ls-remote`, not that field alone; the API discrepancy's cause was not diagnosed. The [released-orchestration audit](doctoring/zotero_fulltext_contract_audit.md#released-orchestration-evidence) verified no qualifying artifact or deployed gateway in the inspected channels. Its 66 deployment records include eight successes, all Provider catalog sync, not proof of a serving gateway. Existing CO [PR #1030](https://github.com/ContextualWisdomLab/contextual-orchestrator/pull/1030) owns release work and its active owner confirmed that artifact/schema/deployed-version evidence is still pending. The handoff requests that evidence without duplicate release machinery or provider bypass. New text-bound proposals and approved labels remain zero. From a10de5764af14684599b7529c038fd97a476b6f4 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sat, 5 Sep 2026 19:07:21 +0900 Subject: [PATCH 28/47] docs: record canonical transport cascade and exact verification Preserve the failed coverage checkpoint and its canonical deterministic regression, completed ordinary-merge ancestry, raw versus source-normalized coverage and pending central CodeQL handoffs. Classification and approval remain zero; local evidence is not protected acceptance. --- CHANGELOG.md | 4 ++++ docs/doctoring/zotero_fulltext_contract_audit.md | 6 +++++- docs/product-technical-gap-baseline.md | 10 ++++++++-- 3 files changed, 17 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6936bbfe..97b751cd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -32,3 +32,7 @@ All notable changes to ConceptWeave are documented here. - Golden-set evaluation rejects changed predictions or evidence under an earlier approval. Proposal-bound approvals must be reissued; aggregate receipts identify the actual evaluated proposal run. - Model-generated semantics remain non-authoritative until deterministic validation and authorized review. - Unsafe Rust is forbidden in the core domain crate. + +### Fixed + +- Research reads accept valid responses exactly at their documented size limit while still rejecting oversized, incomplete or invalidly encoded responses. diff --git a/docs/doctoring/zotero_fulltext_contract_audit.md b/docs/doctoring/zotero_fulltext_contract_audit.md index 5c7d2d31..526e6775 100644 --- a/docs/doctoring/zotero_fulltext_contract_audit.md +++ b/docs/doctoring/zotero_fulltext_contract_audit.md @@ -106,7 +106,11 @@ The original metadata report remains byte-identical. Retained-text coverage impr These fixes are integrated into the proposed full-text branch. The earlier owner backports now preserve their own committed regressions: metadata PR #9 RED `31b507a` → GREEN `a2a8488`; authenticated transport PR #17 RED `f83a63d` → production repair `53bd1fe`; extracted authorization PR #18 RED `178e03b` → GREEN `ba1bbd2`. The shared synthetic server also needed request-framing RED `b6b618b` → test-helper repair `7bcb791`: reading only one TCP chunk could miss the POST body and close while the large response was still arriving. This was test infrastructure, not a new production failure or a reason to reduce the response-size case. -At #17 `7bcb791853ffa794529418ee9de1337fea4e1b15`, 91 workspace tests and 20 repetitions of five focused tests passed. At #18 `ba1bbd203c2f90afbf97ac3d7eab989982e8bc09`, 99 workspace tests and ten repetitions of eight focused tests passed, including exactly 512-byte success/denial and one-byte-over rejection. Strict Clippy and formatting passed at both heads. All requests were synthetic; no actual key, authorization or Zotero write was used. The normal forward cascade remains in progress, and each descendant needs its own verification before promotion. No predecessor delta was discarded. +At #17 `7bcb791853ffa794529418ee9de1337fea4e1b15`, 91 workspace tests and 20 repetitions of five focused tests passed. At #18 `ba1bbd203c2f90afbf97ac3d7eab989982e8bc09`, 99 workspace tests and ten repetitions of eight focused tests passed, including exactly 512-byte success/denial and one-byte-over rejection. Strict Clippy and formatting passed at both heads. All requests were synthetic; no actual key, authorization or Zotero write was used. + +The later #19 coverage failure, 333/334 normalized branch outcomes, exposed nondeterministic TCP fragmentation in the shared test server. Test-only canonical #17 `b388810be8bceb3a4f81c336708cf1c56a20d057` adds an 8 KiB header and 8 KiB body through the unchanged 4 KiB read buffer. #17 then passes 92 workspace tests and 318/318 normalized branches; ordinary restacks give #18 `21a7ee8f8b4b0988c13bb45aecbb016242c21308` 100 tests and #19 `aa74f8642e9e8c3804996ce443650df29f08bf5f` 101 tests with 334/334 branches. No exclusion or production behavior changed for this regression. + +The final non-force cascade reaches #34 `b0119a57047e7b1fe5ddfbbf4b973de0f15de172`, with 156 workspace tests and its existing coverage gate passing. Root full-text integration `75da75cf01704d9aae47f1e5573e3bbe3fb42bb0` passes 186 workspace tests across 37 unfiltered suites, strict Clippy, formatting, rustdoc, CI contract and the existing coverage gate. It preserves one unchanged shared reader, inherited regression modules and all earlier full-text safeguards. Source-normalized coverage is 3,710/3,710 regions and 674/674 branches; functions are 347/347. Raw LLVM totals remain 4,159/4,255 lines, 6,129/6,274 regions and 603/674 branches. Independent source review found no actionable merge finding, not approval. No predecessor delta was discarded; every head still requires its own protected acceptance evidence. ## References diff --git a/docs/product-technical-gap-baseline.md b/docs/product-technical-gap-baseline.md index 9eb472c4..5821386d 100644 --- a/docs/product-technical-gap-baseline.md +++ b/docs/product-technical-gap-baseline.md @@ -14,7 +14,7 @@ The active roots observed immediately before this baseline refresh are: 2. Product-CI bootstrap PR #35 — exact head `a31ae0c2df920f2794f7ddb456795b04797ab472`, open/non-Draft. It adds the pull-request form of Product CI and removes no-op closed/converted-to-Draft triggers. Scope detection and review admission have executed successfully, while CodeQL, Semgrep, Noema, Strix, Trivy and Scorecard remain queued. The workflow-only diff skips Dependency Review and OSV; these skips do not prove Foundation's dependency-changing checks. No independent approval exists for this head. 3. Client Consumption PR #5 — exact head `fcf36c8a99f015b963c9f812787df127ac2e2f9e`, Draft/open. The current source retains language-neutral semantic-release admission, integrity, compatibility, diff/resolution and supersession validation. Previously valid review findings are source-repaired, but current protected evidence remains independently required. 4. Source Observation PR #6 — exact head `c362a73403b6bda2cc0e94de913e39f3139d6205`, Draft/open. Its independent owner preserves registry denial before an authorized-request-only adapter boundary. The counted regression now verifies zero adapter/source/snapshot executions on denial and one of each for an authorized control, retaining the existing denial result. This audit checked source and formatting, not that branch's runtime or coverage. The new submitted repair report is COMMENTED, not approval; current-head Actions/check-runs were absent. The concrete bounded read-only PostgreSQL adapter remains absent. -5. Zotero Research Classification root PR #9 — exact head `a2a84884f67dcac6f6892c958d55450aea6d6c88`, Draft/open. A minimal owner backport reproduces and repairs proxy inheritance and exact-byte-limit rejection without bringing later full-text features backward. Integrity root #10 was `e7d4e59f1b55b5954c5f8436527bc96e7ef2fb13`; all 23 descendants inherited that earlier repair through ordinary merges. The transport repair is now being propagated forward. Review-batch PR #34's pre-propagation checkpoint is `2e6448e896e65562ebeee2fd339dec64d9fdf6e5`. [Full-text capture PR #36](https://github.com/ContextualWisdomLab/ConceptWeave/pull/36) is its Draft child at the audited `e19d95f42c0f745cb428133ee7c4a15043e76744`. At that observation it had only a successful CodeRabbit status context, explicitly skipped Draft review and no submitted reviews; this is not independent approval or hosted Product verification. The stack remains proposal/review oriented and does not elevate local classifier output to semantic authority. +5. Zotero Research Classification root PR #9 — exact head `a2a84884f67dcac6f6892c958d55450aea6d6c88`, Draft/open. A minimal owner backport reproduces and repairs proxy inheritance and exact-byte-limit rejection without bringing later full-text features backward. Integrity root #10 was `e7d4e59f1b55b5954c5f8436527bc96e7ef2fb13`; all 23 descendants inherited that earlier repair through ordinary merges. The transport cascade now reaches review-batch PR #34 at `b0119a57047e7b1fe5ddfbbf4b973de0f15de172`, preserving its original `2e6448e896e65562ebeee2fd339dec64d9fdf6e5` and every intermediate delta. [Full-text capture PR #36](https://github.com/ContextualWisdomLab/ConceptWeave/pull/36) integrates that parent at locally verified merge `75da75cf01704d9aae47f1e5573e3bbe3fb42bb0`; its subsequent documentation head must be refreshed separately. Before that push, the remote remained Draft at `e19d95f42c0f745cb428133ee7c4a15043e76744`, with only an explicitly skipped CodeRabbit Draft review and no submitted review or hosted Product verification. The stack remains proposal/review oriented and does not elevate local classifier output to semantic authority. Predecessor reviews/checks never transfer to successor heads. No force-push, destructive rebase, self-approval, fail-open scanner substitution or routine administrator bypass is acceptance evidence. @@ -49,6 +49,8 @@ Protected central source is `.github/main@8aea81323d93e90c79b71d7718de2798919fa1 - `.github#1873@41935494aa234eb458f1cc08f006daaa278b9760` is closed/unmerged because repository dependency-graph initialization, not its retry/sleep source delta, was the verified root cause of the observed public-repository 403. - #35 remains a consumer canary for runner admission and applicable workflow security checks. Its workflow-only change skips Dependency Review, so a dependency-changing Foundation run must separately prove that action's success. Already-created runs remain bound to their own central workflow revisions. +At 2026-09-05 09:47 UTC, Foundation #1's two CodeQL failures in [run 33937211620](https://github.com/ContextualWisdomLab/ConceptWeave/actions/runs/33937211620) were verified runner-release handoffs, not observed scan findings. Both dispatches succeeded but their terminal verdicts remained pending. Exact-head successor runs [33958339895](https://github.com/ContextualWisdomLab/.github/actions/runs/33958339895) and [33958340068](https://github.com/ContextualWisdomLab/.github/actions/runs/33958340068) were queued, bound to ConceptWeave `b538470c963e6524ddc0c3f652a46a4fc8265150` and central run source `7fcada597d5b79bdb14445f24322b2c9f6ed4b19`. This audit did not independently reverify that source's branch protection. The originating job promises an exact-job rerun after the terminal verdict; no manual retry, scanner substitution or new repair was justified by the red status alone. + ## Zotero research campaign evidence ### Repaired current snapshot and source verification @@ -94,7 +96,11 @@ Next: propagate applicable transport repairs to the earlier owner stack without PR #9's committed RED `31b507ae9feaf58688cf62ddcb597a88d2223366` reproduces six environment-proxy routes and rejection of valid JSON exactly at the 8 MiB response bound. GREEN `a2a84884f67dcac6f6892c958d55450aea6d6c88` disables inherited proxies and introduces the same strict UTF-8 inclusive reader at the original metadata owner. Oversized, invalid-UTF-8 and truncated responses remain rejected. Its 38 workspace tests, strict Clippy, formatting and existing coverage gate pass; source-normalized regions are 686/686 and branches 90/90. Raw LLVM functions are 97/97, lines 981/982 and branches 89/90, not raw 100% coverage. Root review independently reran the four transport tests before the non-force push. Subsequent authenticated adapters must reuse this reader at their own introduction points; full-text feature commits are not reverse-merged into the earlier owner. -That repair now reaches authenticated PR #17 `7bcb791853ffa794529418ee9de1337fea4e1b15` and authorization PR #18 `ba1bbd203c2f90afbf97ac3d7eab989982e8bc09`. Their own committed RED cases verify actual synthetic proxy forwarding and inclusive 1 MiB/512-byte boundaries. #17 additionally repairs the existing synthetic server's incomplete POST read after a committed request-framing regression; it does not weaken production validation. Verification passed 91/99 workspace tests respectively, plus focused repetition and strict Clippy/formatting. The root independently inspected shared callers, status/header handling, key secrecy and the conflict resolutions. Downstream propagation remains ordinary merge/push with original-head and repaired-parent ancestry retained. Observed GitHub PR base objects differed from the actual branch ref, so each merge resolves the named base through fresh fetch and `ls-remote`, not that field alone; the API discrepancy's cause was not diagnosed. +Authenticated PR #17's production repair `53bd1fe16c43adc5cb0e7a052183e80b8c6c2e25` and authorization PR #18's `ba1bbd203c2f90afbf97ac3d7eab989982e8bc09` preserve committed RED cases for actual synthetic proxy forwarding and inclusive 1 MiB/512-byte boundaries. The existing synthetic server also needed POST-framing repair `7bcb791853ffa794529418ee9de1337fea4e1b15`; 91/99 workspace tests respectively, focused repetition and strict Clippy/formatting passed at those historical heads. A subsequent #19 coverage run correctly failed at 333/334 normalized branch outcomes because TCP fragmentation did not reliably exercise the helper's full-read loop. Canonical test-only #17 `b388810be8bceb3a4f81c336708cf1c56a20d057` sends an 8 KiB header and 8 KiB body through the unchanged 4 KiB buffer. It closes that gap without a new helper or coverage exclusion: #17 passes 92 workspace tests and 318/318 normalized branch outcomes; restacked #18 `21a7ee8f8b4b0988c13bb45aecbb016242c21308` passes 100 tests; #19 `aa74f8642e9e8c3804996ce443650df29f08bf5f` passes 101 tests and 334/334 branch outcomes. + +The ordered non-force cascade completed through #34 `b0119a57047e7b1fe5ddfbbf4b973de0f15de172`: 156 workspace tests, strict Clippy/formatting and the existing coverage gate passed, with 315/315 functions, 3,214/3,214 normalized regions and 598/598 normalized branch outcomes. A fresh REST/named-ref audit found all 24 descendants #10–#13 and #15–#34 open/Draft at their expected heads; original heads, repaired parents and initially prepared local commits remain ancestors. Observed GitHub PR base objects differed from actual branch refs, so each merge resolved the named base through fresh fetch and `ls-remote`, not that field alone; the API discrepancy's cause was not diagnosed. + +Root integration `75da75cf01704d9aae47f1e5573e3bbe3fb42bb0` retains one identical shared reader at its original metadata-owner location, all three transport regression modules and the richer request parser with the EOF guard. Full-text capture, replay, CLI and proxy-isolation source remain unchanged from its first parent. Independent source review found no actionable merge finding; this is not approval. Root verification passed 186 workspace tests across 37 unfiltered suites, including three doctests; the isolated subprocess invocation is not counted twice. Strict Clippy, formatting, rustdoc with warnings denied, CI contract and existing coverage gate passed. Functions are 347/347, source-normalized regions 3,710/3,710 and branch outcomes 674/674. Raw LLVM totals remain 4,159/4,255 lines, 6,129/6,274 regions and 603/674 branches, not 100%. No coverage exclusion, dependency, real Zotero request, credential use, classification decision or approval was added by this integration. The [released-orchestration audit](doctoring/zotero_fulltext_contract_audit.md#released-orchestration-evidence) verified no qualifying artifact or deployed gateway in the inspected channels. Its 66 deployment records include eight successes, all Provider catalog sync, not proof of a serving gateway. Existing CO [PR #1030](https://github.com/ContextualWisdomLab/contextual-orchestrator/pull/1030) owns release work and its active owner confirmed that artifact/schema/deployed-version evidence is still pending. The handoff requests that evidence without duplicate release machinery or provider bypass. New text-bound proposals and approved labels remain zero. From 1e7d23c91116d84a455b6e6e5a6fb00a5e004c04 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sat, 5 Sep 2026 19:07:54 +0900 Subject: [PATCH 29/47] docs: advance next action after completed transport cascade --- docs/product-technical-gap-baseline.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/product-technical-gap-baseline.md b/docs/product-technical-gap-baseline.md index 5821386d..f48a2e2f 100644 --- a/docs/product-technical-gap-baseline.md +++ b/docs/product-technical-gap-baseline.md @@ -90,7 +90,7 @@ Replayable retained-text coverage has progressed from 0 to 3,203/3,715 parents. Committed regressions repaired inherited environment proxies, exact-byte-limit rejection and replay checks occurring after digest allocation; clock fault injection verifies late/invalid-clock failures without changing the deadline. Final source verification at `733425df01511d894277fb8682e070f3dde03689` passed 173 tests across 37 suites including documentation tests, strict Clippy, formatting, rustdoc with warnings denied, the CI contract and the existing coverage gate. Coverage is 347/347 functions, 3,710/3,710 source-normalized regions and 674/674 source-normalized branch outcomes. Raw LLVM totals remain 4,159/4,255 lines, 6,129/6,274 regions and 603/674 branch outcomes; those are not 100%. The only source delta after the live run is a writer type alias resolving strict Clippy's complexity finding without changing runtime behavior. Hosted checks and independent protected approval remain separate gates. -Next: propagate applicable transport repairs to the earlier owner stack without losing deltas; present retained text under new proposal/review bindings with partial/missing coverage; continue the 61 remaining repository capability audits and the upstream version-contract repair. A released contextual-orchestrator integration artifact remains unverified at the audited protected owner head, so source documentation alone does not authorize model-provider bypass. No new utility repository is justified by this one intake seam. +Next: revalidate current-head protected gates after the completed transport cascade below; present retained text under new proposal/review bindings with partial/missing coverage; continue the 61 remaining repository capability audits and the upstream version-contract repair. A released contextual-orchestrator integration artifact remains unverified at the audited protected owner head, so source documentation alone does not authorize model-provider bypass. No new utility repository is justified by this one intake seam. ### Canonical transport repair and released-owner audit From c7052d93c7a954ce4f31bb4c5b52f6913fc19638 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Mon, 7 Sep 2026 01:33:51 +0900 Subject: [PATCH 30/47] test: verify full text source continuity through shared admission --- .../src/full_text_capture_tests.rs | 57 +++++++++++++++++++ crates/conceptweave-zotero/src/main.rs | 21 +++---- 2 files changed, 68 insertions(+), 10 deletions(-) diff --git a/crates/conceptweave-zotero/src/full_text_capture_tests.rs b/crates/conceptweave-zotero/src/full_text_capture_tests.rs index d3ce7cdf..1d45717b 100644 --- a/crates/conceptweave-zotero/src/full_text_capture_tests.rs +++ b/crates/conceptweave-zotero/src/full_text_capture_tests.rs @@ -107,6 +107,63 @@ fn capture_rejects_unbound_reports_before_any_request() { assert_eq!(calls, 0); } +#[test] +fn capture_rejects_incomplete_or_inconsistent_retained_sources_before_fetch() { + for mismatch_parent in [false, true] { + let mut report = report_fixture(); + if mismatch_parent { + report.unclassified_items[0].data.parent_item = "DEFG5678".into(); + } else { + report.unclassified_items.pop(); + } + let mut calls = 0; + assert!( + capture_with(&report, 4096, &mut |_, _| { + calls += 1; + unreachable!() + }) + .is_err() + ); + assert_eq!(calls, 0); + } +} + +#[test] +fn capture_preserves_pending_sources_and_rejects_rebound_report_context() { + let items: Vec = serde_json::from_value(serde_json::json!([ + {"key":"ABCD2345","version":2,"data":{"itemType":"journalArticle","title":"fixture paper"}}, + {"key":"BCDE3456","version":1,"data":{"itemType":"attachment"}} + ])) + .unwrap(); + let mut report = classify_snapshot("10.0.1".into(), Some("fixture-server".into()), 2, items); + report.api_version = Some(3); + report.schema_version = Some(44); + let original_report = serde_json::to_vec(&report).unwrap(); + let capture = capture_with(&report, 4096, &mut |request_path, _| { + let mut response = response_fixture(request_path); + if request_path == "fulltext?since=0" { + response.body = r#"{"BCDE3456":12403}"#.into(); + } else if request_path == "items/BCDE3456" { + response.body = + r#"{"key":"BCDE3456","version":1,"data":{"itemType":"attachment"}}"#.into(); + } + Ok(response) + }) + .unwrap(); + let restored: FullTextCapture = + serde_json::from_slice(&serde_json::to_vec(&capture).unwrap()).unwrap(); + verify_full_text_capture(&restored, &report).unwrap(); + assert_eq!(serde_json::to_vec(&report).unwrap(), original_report); + assert_eq!(report.pending_source_item_keys, vec!["BCDE3456"]); + assert_eq!(capture.capture_evidence.bibliographic_item_count, 1); + assert_eq!(capture.capture_evidence.records.len(), 1); + let snapshot_digest = report.snapshot_digest.clone(); + report.unclassified_items[0].data.title = "changed retained source context".into(); + assert!(build_steward_review_worksheet(&report).is_ok()); + assert_eq!(report.snapshot_digest, snapshot_digest); + assert!(verify_full_text_capture(&restored, &report).is_err()); +} + #[test] fn capture_rejects_foreign_manifest_items_and_duplicate_manifest_keys() { for body in [r#"{"EFGH6789":0}"#, r#"{"BCDE3456":1,"BCDE3456":2}"#] { diff --git a/crates/conceptweave-zotero/src/main.rs b/crates/conceptweave-zotero/src/main.rs index 4491080a..72455129 100644 --- a/crates/conceptweave-zotero/src/main.rs +++ b/crates/conceptweave-zotero/src/main.rs @@ -1235,7 +1235,7 @@ mod tests { let retained = unique_temp_path("write-race-retained"); assert!(!output.exists()); assert!(!retained.exists()); - let error = write_private_output_with(&output, b"content", |_, _| { + let error = write_private_output_with(&output, b"content", &mut |_, _| { let output = unique_temp_path("write-race"); fs::rename(&output, unique_temp_path("write-race-retained"))?; let mut replacement = OpenOptions::new() @@ -1259,15 +1259,16 @@ mod tests { let retained = unique_temp_path("write-buffer-retained"); assert!(!output.exists()); assert!(!retained.exists()); - let error = write_private_output_with(&output, b"buffered content", |writer, content| { - fs::rename( - unique_temp_path("write-buffer"), - unique_temp_path("write-buffer-retained"), - )?; - writer.write_all(content)?; - Err(io::Error::new(io::ErrorKind::WriteZero, "injected failure")) - }) - .unwrap_err(); + let error = + write_private_output_with(&output, b"buffered content", &mut |writer, content| { + fs::rename( + unique_temp_path("write-buffer"), + unique_temp_path("write-buffer-retained"), + )?; + writer.write_all(content)?; + Err(io::Error::new(io::ErrorKind::WriteZero, "injected failure")) + }) + .unwrap_err(); let retained_bytes = fs::read(&retained).unwrap(); fs::remove_file(retained).unwrap(); assert_eq!(error.kind(), io::ErrorKind::WriteZero); From f26ce5beb813b5e5dfc67826526f069660a3b672 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Mon, 7 Sep 2026 01:34:50 +0900 Subject: [PATCH 31/47] docs: trace capture source scope and inherited private failure policy --- docs/adr/0006-zotero-research-intake.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/adr/0006-zotero-research-intake.md b/docs/adr/0006-zotero-research-intake.md index 5e5bff3d..3d3c762b 100644 --- a/docs/adr/0006-zotero-research-intake.md +++ b/docs/adr/0006-zotero-research-intake.md @@ -5,6 +5,8 @@ ## Context +Proposed September 7 capture-source continuity amendment: full-text observations must not silently lose standalone sources when metadata admission is strengthened. Normal merge `707f2f2` retains original PR36 and PR34 `9eb89b8d4e751c34e640261f4883381571c83f25`. The existing capture validator already calls worksheet construction, so the shared inventory and parent-coordinate checks apply before any fetch. We retain this delegation and the existing whole-report digest rather than adding another validator or content hash. Test `c7052d9` rejects omitted/mismatched retained sources before requests, accepts a mixed paper/standalone-attachment capture without clearing pending keys, preserves report bytes, and rejects old capture verification after retained metadata changes. A valid pending capture is preparation, not source resolution or approval. The existing requirement for at least one bibliographic item remains explicit; all-standalone libraries are unsupported here. Streamed output inherits the shared no-unlink/no-drop-flush failure boundary; two synthetic writer callers are adjusted to its mutable callback signature. The consequence is possible private partial output requiring deliberate inspection, not automatic cleanup. Protected adoption, actual research decisions and full-text approval remain separate outstanding work. + Proposed September 7 completed-batch amendment: a saved review view may outlive changed source context or unresolved-source counts. Preserve PR34's whole-view comparison, including the inherited proposal identity and pending-source count, then project its already verified identity into the decision patch. Do not backfill an old batch with current identity, add another hash, or deserialize a batch directly into the smaller patch: each alternative could discard what the steward actually saw. Strict object boundaries remain. Normal merge `bd8f995` retains PR33 `93faf6ab750a99469196cf71567498be83c22a6b` and the original research-source audit; `d1e8bb7` corrects the synthetic patch initializer. Tests `e889ac8` through `cb8f78c` cover missing, blank and changed identity, altered pending count, and stale context against a freshly generated worksheet. Test compilation and fixture-assumption failures are intermediate evidence, not production RED. The cost is explicit regeneration and renewed review after context drift. Neither accepted local decisions nor this conversion authenticate a reviewer, grant full-text provenance, resolve pending sources or authorize Zotero writes. Later consumers must retain this boundary; protected adoption remains outstanding. Proposed September 7 finalization amendment: when carrying completed metadata decisions between saved artifacts, changed report evidence plus freshly supplied receipt coordinates must not refresh a stale worksheet. We choose two checks in the existing converter—nonblank worksheet proposal identity and equality with the recomputed expected worksheet—rather than a second approval mechanism or automatically rebinding old decisions. RED `f02631e` reproduces both stale-content and blank/replaced-binding admission; `d44b9fe` closes them while preserving prior error precedence. The cost is explicit worksheet regeneration/review after changed source context. Self-consistent local artifacts remain unverified: `e90a02b` demonstrates that locally rewritten digests still fail independent original-receipt verification and that pending sources prevent complete evaluation even after local conversion. This decision grants neither full-text nor Zotero write authority. Later worksheet comparators must inherit the binding check, and protected/runtime evidence remains outstanding. From aca72e78af197afbad2337c35bcf0d77e2d010ac Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Mon, 7 Sep 2026 01:35:48 +0900 Subject: [PATCH 32/47] docs: record PR36 source capture verification and open campaign gaps --- docs/product-technical-gap-baseline.md | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/docs/product-technical-gap-baseline.md b/docs/product-technical-gap-baseline.md index 73445b1b..e51c3254 100644 --- a/docs/product-technical-gap-baseline.md +++ b/docs/product-technical-gap-baseline.md @@ -6,6 +6,14 @@ This file records code-current product and technical gaps. Exact PR/check/run co ## September 6 source inventory checkpoint +### September 7 PR36 full-text source continuity verification + +Original PR36 `d3f991dcc1b746afed7c36f315e8937c39390c5e` passed 202 tests/39 suites. Normal merge `707f2f2` preserves its private full-text capture/proxy-isolation delta and PR34 `9eb89b8d4e751c34e640261f4883381571c83f25`. Existing capture admission delegates to worksheet construction, so shared inventory and parent consistency checks are inherited before any request. Existing whole-report hashing includes retained metadata and pending keys. No duplicate validator or hash was added. Initial integration failed compilation because two parent test callbacks required mutable borrows under PR36's writer signature; `c7052d9` repairs those calls and adds source-scope regressions. This is not a new production RED claim. + +Tests reject omitted retained records and inconsistent parent metadata before fetching; a mixed library capture retains a standalone attachment and unchanged report bytes/pending keys; changed retained metadata invalidates an old restored capture despite unchanged raw snapshot identity. Independent bounded source review found no additional defect and documented the existing at-least-one-bibliographic-item restriction. Final source passes 254 tests/39 suites including three doctests, strict Clippy, warnings-denied rustdoc, formatting, CI contract and diff checks. Unchanged coverage passes 374/374 reported functions, 3,688/3,688 normalized regions and 656/656 normalized branches. Raw 4,631/4,703 lines, 7,064/7,185 regions and 608/656 branches are not 100%. Logs `/tmp/conceptweave-pr36-{baseline,integrated,verified,clippy,rustdoc,coverage}.log`; `integrated` is the failed compile run. Proposed ADR0006 `f26ce5b` records alternatives and downstream obligations; TRD/UML merge preserves both source traversal and separate capture. Failed streamed output now inherits no pathname cleanup or implicit buffer retry. + +Root and later consumers still require cascade adoption. Actual decisions and independent approvals remain 0/3,715 plus four unresolved standalone sources. Capture tests are synthetic preparation evidence, not reviewed meaning, authority, peer authentication or an atomic snapshot. Native Visual Inspection was retried but the Mac remains locked; manual unlock is required and no fresh screenshot exists. Keep OPEN Draft; no real Zotero write, hosted GREEN, protected merge or release is claimed. + ### September 7 PR34 completed-view continuity verification Original PR34 `b9060df2cb1ea02314be429932031fc07de1de30` passed 171 tests/37 suites. Normal merge `bd8f995` preserves that delta and PR33 `93faf6ab750a99469196cf71567498be83c22a6b`: whole-view comparison, strict JSON boundaries, research-source audit, pending-source scope and inherited private-file protections all remain. The required patch identity is projected only after whole-view equality. Fixture correction `d1e8bb7` yields 221 integrated tests/37 suites. Tests `e889ac8`, `5a23660` and `cb8f78c` add tamper/missing-field/stale-view coverage; intermediate compilation and fixture-assumption failures are recorded, not claimed as production RED. The final stale-view case holds raw snapshot identity unchanged while changing retained context and regenerating the worksheet. Independent bounded review found no additional defect; it is not GitHub approval. From 2cbac821243f1f3ad647380023ca15a8b99d0d5e Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Mon, 7 Sep 2026 04:47:34 +0900 Subject: [PATCH 33/47] test(zotero): require manifest library version binding --- .../src/full_text_capture_tests.rs | 27 +++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/crates/conceptweave-zotero/src/full_text_capture_tests.rs b/crates/conceptweave-zotero/src/full_text_capture_tests.rs index 1d45717b..9160a450 100644 --- a/crates/conceptweave-zotero/src/full_text_capture_tests.rs +++ b/crates/conceptweave-zotero/src/full_text_capture_tests.rs @@ -217,6 +217,33 @@ fn capture_rejects_parent_version_status_and_bookend_drift() { } } +#[test] +fn manifest_bookends_require_the_bound_library_version_even_after_rehash() { + let report = report_fixture(); + let capture = capture_with(&report, 4096, &mut |request_path, _| { + Ok(response_fixture(request_path)) + }) + .unwrap(); + let saved = serde_json::to_value(capture).unwrap(); + for (field, version) in [ + ("manifest_before", serde_json::Value::Null), + ("manifest_before", serde_json::json!(3)), + ("manifest_after", serde_json::Value::Null), + ("manifest_after", serde_json::json!(3)), + ] { + let mut restored: FullTextCapture = serde_json::from_value(saved.clone()).unwrap(); + let value = serde_json::to_value(&restored).unwrap(); + let mut value = value; + value["capture_evidence"][field]["version"] = version; + restored = serde_json::from_value(value).unwrap(); + restored.capture_digest = json_digest(&restored.capture_evidence); + assert!( + verify_full_text_capture(&restored, &report).is_err(), + "{field} accepted an unbound manifest version" + ); + } +} + #[test] fn capture_checks_total_budget_before_another_request() { let mut calls = 0; From 2c16fea8500b4c5db6ae8ee95c81a39e375edb30 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Mon, 7 Sep 2026 04:48:39 +0900 Subject: [PATCH 34/47] fix(zotero): bind manifest bookends to library version --- .../conceptweave-zotero/src/full_text_capture.rs | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/crates/conceptweave-zotero/src/full_text_capture.rs b/crates/conceptweave-zotero/src/full_text_capture.rs index c7220849..4f6c7ebe 100644 --- a/crates/conceptweave-zotero/src/full_text_capture.rs +++ b/crates/conceptweave-zotero/src/full_text_capture.rs @@ -131,7 +131,9 @@ pub fn verify_full_text_capture( } validate_library(&evidence.library_before, report)?; validate_library(&evidence.library_after, report)?; + validate_manifest_version(&evidence.manifest_before, report.library_version)?; let manifest = parse_manifest(&evidence.manifest_before, &snapshot)?; + validate_manifest_version(&evidence.manifest_after, report.library_version)?; if evidence.manifest_after.status != 200 || evidence.manifest_after.body != evidence.manifest_before.body || evidence.records.len() != manifest.len() @@ -181,6 +183,7 @@ fn capture_with_clock( let library_before = read("items?limit=1")?; validate_library(&library_before, report)?; let manifest_before = read("fulltext?since=0")?; + validate_manifest_version(&manifest_before, report.library_version)?; let manifest = parse_manifest(&manifest_before, &snapshot)?; let mut records = Vec::with_capacity(manifest.len()); for (item_key, version) in manifest { @@ -195,6 +198,7 @@ fn capture_with_clock( }); } let manifest_after = read("fulltext?since=0")?; + validate_manifest_version(&manifest_after, report.library_version)?; let library_after = read("items?limit=1")?; let capture_evidence = CaptureEvidence { capture_kind: CAPTURE_KIND.into(), @@ -261,6 +265,16 @@ fn validate_library( Ok(()) } +fn validate_manifest_version( + response: &CapturedResponse, + library_version: u64, +) -> Result<(), FullTextError> { + if response.status != 200 || response.version != Some(library_version) { + return Err(INVALID_EVIDENCE); + } + Ok(()) +} + fn parse_manifest( response: &CapturedResponse, snapshot: &BTreeMap<&str, &SnapshotItemRevision>, @@ -423,4 +437,4 @@ mod tests; #[cfg(test)] #[path = "full_text_capture_transport_tests.rs"] -mod transport_tests; +mod transport_tests; \ No newline at end of file From 4c1f81c00f22a25f13d71806828ca473da186e61 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Mon, 7 Sep 2026 04:49:51 +0900 Subject: [PATCH 35/47] test(zotero): bind manifest fixture to library version --- crates/conceptweave-zotero/src/full_text_capture_tests.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/conceptweave-zotero/src/full_text_capture_tests.rs b/crates/conceptweave-zotero/src/full_text_capture_tests.rs index 9160a450..6c740360 100644 --- a/crates/conceptweave-zotero/src/full_text_capture_tests.rs +++ b/crates/conceptweave-zotero/src/full_text_capture_tests.rs @@ -18,7 +18,7 @@ fn report_fixture() -> ClassificationReport { fn response_fixture(request_path: &str) -> CapturedResponse { let (status, version, body) = match request_path { "items?limit=1" => (200, Some(2), "[]"), - "fulltext?since=0" => (200, None, r#"{"BCDE3456":12403,"CDEF4567":0}"#), + "fulltext?since=0" => (200, Some(2), r#"{"BCDE3456":12403,"CDEF4567":0}"#), "items/BCDE3456" => ( 200, Some(1), From 9be1ca067c722957941a321a81545234e16f2db5 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Mon, 7 Sep 2026 04:50:42 +0900 Subject: [PATCH 36/47] test(zotero): emit manifest library version on wire --- .../src/full_text_capture_transport_tests.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/crates/conceptweave-zotero/src/full_text_capture_transport_tests.rs b/crates/conceptweave-zotero/src/full_text_capture_transport_tests.rs index 491c181d..bfee1848 100644 --- a/crates/conceptweave-zotero/src/full_text_capture_transport_tests.rs +++ b/crates/conceptweave-zotero/src/full_text_capture_transport_tests.rs @@ -293,10 +293,10 @@ mod tests { let content = br#"{"content":"synthetic full text","providerExtra":{"retained":true}}"#; let (api_root, server) = serve_responses(vec![ wire_response(200, Some("2"), b"[]"), - wire_response(200, None, manifest), + wire_response(200, Some("2"), manifest), wire_response(200, Some("1"), metadata), wire_response(200, Some("7"), content), - wire_response(200, None, manifest), + wire_response(200, Some("2"), manifest), wire_response(200, Some("2"), b"[]"), ]); let report = report_fixture(); From 3ab00417c229aeae59709f8980c79d5339687893 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Mon, 7 Sep 2026 04:52:18 +0900 Subject: [PATCH 37/47] revert(zotero): preserve observed local full-text contract --- .../src/full_text_capture.rs | 16 +--------- .../src/full_text_capture_tests.rs | 29 +------------------ .../src/full_text_capture_transport_tests.rs | 4 +-- 3 files changed, 4 insertions(+), 45 deletions(-) diff --git a/crates/conceptweave-zotero/src/full_text_capture.rs b/crates/conceptweave-zotero/src/full_text_capture.rs index 4f6c7ebe..c7220849 100644 --- a/crates/conceptweave-zotero/src/full_text_capture.rs +++ b/crates/conceptweave-zotero/src/full_text_capture.rs @@ -131,9 +131,7 @@ pub fn verify_full_text_capture( } validate_library(&evidence.library_before, report)?; validate_library(&evidence.library_after, report)?; - validate_manifest_version(&evidence.manifest_before, report.library_version)?; let manifest = parse_manifest(&evidence.manifest_before, &snapshot)?; - validate_manifest_version(&evidence.manifest_after, report.library_version)?; if evidence.manifest_after.status != 200 || evidence.manifest_after.body != evidence.manifest_before.body || evidence.records.len() != manifest.len() @@ -183,7 +181,6 @@ fn capture_with_clock( let library_before = read("items?limit=1")?; validate_library(&library_before, report)?; let manifest_before = read("fulltext?since=0")?; - validate_manifest_version(&manifest_before, report.library_version)?; let manifest = parse_manifest(&manifest_before, &snapshot)?; let mut records = Vec::with_capacity(manifest.len()); for (item_key, version) in manifest { @@ -198,7 +195,6 @@ fn capture_with_clock( }); } let manifest_after = read("fulltext?since=0")?; - validate_manifest_version(&manifest_after, report.library_version)?; let library_after = read("items?limit=1")?; let capture_evidence = CaptureEvidence { capture_kind: CAPTURE_KIND.into(), @@ -265,16 +261,6 @@ fn validate_library( Ok(()) } -fn validate_manifest_version( - response: &CapturedResponse, - library_version: u64, -) -> Result<(), FullTextError> { - if response.status != 200 || response.version != Some(library_version) { - return Err(INVALID_EVIDENCE); - } - Ok(()) -} - fn parse_manifest( response: &CapturedResponse, snapshot: &BTreeMap<&str, &SnapshotItemRevision>, @@ -437,4 +423,4 @@ mod tests; #[cfg(test)] #[path = "full_text_capture_transport_tests.rs"] -mod transport_tests; \ No newline at end of file +mod transport_tests; diff --git a/crates/conceptweave-zotero/src/full_text_capture_tests.rs b/crates/conceptweave-zotero/src/full_text_capture_tests.rs index 6c740360..1d45717b 100644 --- a/crates/conceptweave-zotero/src/full_text_capture_tests.rs +++ b/crates/conceptweave-zotero/src/full_text_capture_tests.rs @@ -18,7 +18,7 @@ fn report_fixture() -> ClassificationReport { fn response_fixture(request_path: &str) -> CapturedResponse { let (status, version, body) = match request_path { "items?limit=1" => (200, Some(2), "[]"), - "fulltext?since=0" => (200, Some(2), r#"{"BCDE3456":12403,"CDEF4567":0}"#), + "fulltext?since=0" => (200, None, r#"{"BCDE3456":12403,"CDEF4567":0}"#), "items/BCDE3456" => ( 200, Some(1), @@ -217,33 +217,6 @@ fn capture_rejects_parent_version_status_and_bookend_drift() { } } -#[test] -fn manifest_bookends_require_the_bound_library_version_even_after_rehash() { - let report = report_fixture(); - let capture = capture_with(&report, 4096, &mut |request_path, _| { - Ok(response_fixture(request_path)) - }) - .unwrap(); - let saved = serde_json::to_value(capture).unwrap(); - for (field, version) in [ - ("manifest_before", serde_json::Value::Null), - ("manifest_before", serde_json::json!(3)), - ("manifest_after", serde_json::Value::Null), - ("manifest_after", serde_json::json!(3)), - ] { - let mut restored: FullTextCapture = serde_json::from_value(saved.clone()).unwrap(); - let value = serde_json::to_value(&restored).unwrap(); - let mut value = value; - value["capture_evidence"][field]["version"] = version; - restored = serde_json::from_value(value).unwrap(); - restored.capture_digest = json_digest(&restored.capture_evidence); - assert!( - verify_full_text_capture(&restored, &report).is_err(), - "{field} accepted an unbound manifest version" - ); - } -} - #[test] fn capture_checks_total_budget_before_another_request() { let mut calls = 0; diff --git a/crates/conceptweave-zotero/src/full_text_capture_transport_tests.rs b/crates/conceptweave-zotero/src/full_text_capture_transport_tests.rs index bfee1848..491c181d 100644 --- a/crates/conceptweave-zotero/src/full_text_capture_transport_tests.rs +++ b/crates/conceptweave-zotero/src/full_text_capture_transport_tests.rs @@ -293,10 +293,10 @@ mod tests { let content = br#"{"content":"synthetic full text","providerExtra":{"retained":true}}"#; let (api_root, server) = serve_responses(vec![ wire_response(200, Some("2"), b"[]"), - wire_response(200, Some("2"), manifest), + wire_response(200, None, manifest), wire_response(200, Some("1"), metadata), wire_response(200, Some("7"), content), - wire_response(200, Some("2"), manifest), + wire_response(200, None, manifest), wire_response(200, Some("2"), b"[]"), ]); let report = report_fixture(); From 4cfff3fc89bef495c16ca1856667b16ae8140ea9 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Mon, 7 Sep 2026 07:56:59 +0900 Subject: [PATCH 38/47] fix(zotero): bound persisted full-text capture size --- .../src/full_text_capture.rs | 124 ++++++++++++++++++ 1 file changed, 124 insertions(+) diff --git a/crates/conceptweave-zotero/src/full_text_capture.rs b/crates/conceptweave-zotero/src/full_text_capture.rs index c7220849..a6c9c390 100644 --- a/crates/conceptweave-zotero/src/full_text_capture.rs +++ b/crates/conceptweave-zotero/src/full_text_capture.rs @@ -9,12 +9,14 @@ use serde::{Deserialize, Serialize}; use sha2::{Digest, Sha256}; use std::collections::BTreeMap; use std::fmt; +use std::io::{self, Write}; use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH}; const CAPTURE_KIND: &str = "non_atomic_fulltext_sweep_v1"; const INVALID_EVIDENCE: FullTextError = FullTextError("full-text capture evidence is invalid"); const BUDGET_EXCEEDED: FullTextError = FullTextError("full-text capture budget exceeded"); const CAPTURE_DEADLINE: Duration = Duration::from_secs(300); +const MAX_PERSISTED_CAPTURE_BYTES: u64 = 512 * 1024 * 1024; /// A bounded full-text observation artifact, not an atomic snapshot or approval. /// @@ -101,6 +103,7 @@ pub fn verify_full_text_capture( report: &ClassificationReport, ) -> Result<(), FullTextError> { let snapshot = validate_report(report)?; + validate_persisted_capture_size(capture, MAX_PERSISTED_CAPTURE_BYTES)?; let evidence = &capture.capture_evidence; if evidence.records.len() > snapshot.len() { return Err(INVALID_EVIDENCE); @@ -366,6 +369,40 @@ fn unix_millis(time: SystemTime) -> Result { .ok_or(INVALID_EVIDENCE) } +struct SizeLimitedWriter { + written: u64, + max_bytes: u64, +} + +impl Write for SizeLimitedWriter { + fn write(&mut self, bytes: &[u8]) -> io::Result { + let length = bytes.len() as u64; + if length > self.max_bytes - self.written { + return Err(io::Error::new( + io::ErrorKind::InvalidData, + "full-text capture exceeds the persisted size limit", + )); + } + self.written += length; + Ok(bytes.len()) + } + + fn flush(&mut self) -> io::Result<()> { + Ok(()) + } +} + +fn validate_persisted_capture_size( + capture: &FullTextCapture, + max_bytes: u64, +) -> Result<(), FullTextError> { + let mut writer = SizeLimitedWriter { + written: 0, + max_bytes, + }; + serde_json::to_writer(&mut writer, capture).map_err(|_| BUDGET_EXCEEDED) +} + fn json_digest(value: &impl Serialize) -> String { let mut digest = Sha256::new(); serde_json::to_writer(&mut digest, value).expect("capture values are JSON-compatible"); @@ -417,6 +454,93 @@ fn fetch_response( }) } +#[cfg(test)] +#[test] +fn persisted_capture_limit_counts_outer_json_escaping_at_the_exact_boundary() { + let content_body = serde_json::to_string(&serde_json::json!({ + "content": "\\".repeat(1850) + })) + .unwrap(); + let manifest_body = r#"{"ABCD2345":1}"#.to_owned(); + let metadata_body = + r#"{"key":"ABCD2345","version":1,"data":{"itemType":"attachment"}}"#.to_owned(); + let capture_evidence = CaptureEvidence { + capture_kind: CAPTURE_KIND.into(), + metadata_report_digest: format!("sha256:{}", "a".repeat(64)), + metadata_snapshot_digest: format!("sha256:{}", "b".repeat(64)), + bibliographic_item_count: 1, + started_unix_ms: 0, + finished_unix_ms: 0, + library_before: CapturedResponse { + status: 200, + version: Some(1), + body: "[]".into(), + }, + manifest_before: CapturedResponse { + status: 200, + version: None, + body: manifest_body.clone(), + }, + records: vec![CapturedItem { + item_key: "ABCD2345".into(), + metadata_response: CapturedResponse { + status: 200, + version: Some(1), + body: metadata_body, + }, + content_response: CapturedResponse { + status: 200, + version: Some(1), + body: content_body, + }, + }], + manifest_after: CapturedResponse { + status: 200, + version: None, + body: manifest_body, + }, + library_after: CapturedResponse { + status: 200, + version: Some(1), + body: "[]".into(), + }, + }; + let capture = FullTextCapture { + capture_digest: json_digest(&capture_evidence), + capture_evidence, + }; + let raw_body_bytes: usize = [ + &capture.capture_evidence.library_before, + &capture.capture_evidence.manifest_before, + &capture.capture_evidence.manifest_after, + &capture.capture_evidence.library_after, + ] + .into_iter() + .chain( + capture + .capture_evidence + .records + .iter() + .flat_map(|record| [&record.metadata_response, &record.content_response]), + ) + .map(|response| response.body.len()) + .sum(); + let serialized_bytes = serde_json::to_vec(&capture).unwrap().len() as u64; + assert!(raw_body_bytes <= 4096); + assert!(serialized_bytes > 8192); + assert_eq!( + validate_persisted_capture_size(&capture, 8192), + Err(BUDGET_EXCEEDED) + ); + validate_persisted_capture_size(&capture, serialized_bytes).unwrap(); + SizeLimitedWriter { + written: 0, + max_bytes: 0, + } + .flush() + .unwrap(); +} + #[cfg(test)] #[path = "full_text_capture_tests.rs"] mod tests; From 4d22595f46ae5aad82c0ad34957fbe5d6c44942a Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Mon, 7 Sep 2026 07:58:37 +0900 Subject: [PATCH 39/47] docs(zotero): document persisted capture and partial-write boundaries --- OPERABILITY.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/OPERABILITY.md b/OPERABILITY.md index ce8123d4..a54af2b1 100644 --- a/OPERABILITY.md +++ b/OPERABILITY.md @@ -26,6 +26,6 @@ Concrete SLO/RPO/RTO values require measured runtime evidence and are not guesse The proposed `--capture-full-text` command uses an existing private metadata report and creates a separate owner-only file. Keep the original report: the new file cannot replace it, renew review, or prove that all text came from one atomic snapshot. No Zotero authorization prompt, mutation or model request is part of this command. -A changed library, missing provider identity, unexpected response, malformed text or exhausted budget rejects the run. Preserve earlier artifacts; do not disable the checks or overwrite an old file to retry. If metadata has changed, capture a new report and start a separately bound review campaign. Otherwise investigate the reported boundary and rerun to a new temp path. An expected missing-text response is retained; an interrupted run does not emit a partial-success capture. A failed write removes the new partial output. +A changed library, missing provider identity, unexpected response, malformed text or exhausted budget rejects the run. Preserve earlier artifacts; do not disable the checks or overwrite an old file to retry. If metadata has changed, capture a new report and start a separately bound review campaign. Otherwise investigate the reported boundary and rerun to a new temp path. An expected missing-text response is retained. Admission failures create no output, while a write or flush failure may leave a private partial file at the create-new path; the writer deliberately does not unlink or retry buffered bytes against a pathname that may have been replaced. -Allow space for the source text plus JSON escaping overhead. Responses are limited to 8 MiB each and 256 MiB total; the encoded output may be larger. The sweep has a five-minute admission/completion limit and finite local request timeouts. Source text stays in memory until capture completes; hashing and writing stream without a second full encoded copy. The CLI deliberately does not delete prior reports or schedule private-file cleanup. Review retention according to the research library's policy, and never attach these files to a public PR. +Responses are limited to 8 MiB each and 256 MiB total. The persisted compact JSON capture has a separate 512 MiB ceiling because nested source JSON can expand when its text is escaped by the outer capture envelope. Capture verification counts the exact serialized representation before a newly acquired capture can be returned, so the CLI does not create an artifact that the bounded restore path is specified to reject solely because of JSON-encoding expansion. This preflight streams into a counting writer and does not allocate a second encoded copy. The sweep also has a five-minute admission/completion limit and finite local request timeouts. Source text stays in memory until capture completes; hashing and final writing stream. The CLI deliberately does not delete prior reports or schedule private-file cleanup. Review retention according to the research library's policy, and never attach these files to a public PR. From e517fc0a2ef268a5f80ff8e8f90ac24c36a170f3 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Mon, 7 Sep 2026 07:58:55 +0900 Subject: [PATCH 40/47] docs(test): cover serialized full-text capture ceiling --- TEST_STRATEGY.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/TEST_STRATEGY.md b/TEST_STRATEGY.md index f2b26252..366b4d4e 100644 --- a/TEST_STRATEGY.md +++ b/TEST_STRATEGY.md @@ -11,7 +11,7 @@ ## Local research capture regressions -The full-text suite covers report admission, exact response retention, parent/item revision binding, independent content versions, missing/empty/partial text, duplicate/foreign manifest rejection, bookend drift, byte/deadline boundaries and replay under changed or recomputed digests. Synthetic HTTP tests cover headers, network/redirect failures, strict encoding and response limits without touching the running Zotero library. Proxy isolation uses fresh subprocess environments for all six supported proxy variable spellings across the three existing local transport paths. Synthetic text is only a unit/integration fixture; live aggregate evidence is separately recorded in doctoring and never reported as approved labels. +The full-text suite covers report admission, exact response retention, parent/item revision binding, independent content versions, missing/empty/partial text, duplicate/foreign manifest rejection, bookend drift, byte/deadline boundaries and replay under changed or recomputed digests. It also distinguishes the 256 MiB aggregate source-body budget from the 512 MiB persisted compact-JSON ceiling: an escape-heavy valid capture fixture stays within a proportionally scaled raw-body budget while its outer JSON grows past the corresponding persisted ceiling, and the size validator rejects that representation while accepting the exact serialized-byte boundary. Synthetic HTTP tests cover headers, network/redirect failures, strict encoding and response limits without touching the running Zotero library. Proxy isolation uses fresh subprocess environments for all six supported proxy variable spellings across the three existing local transport paths. Synthetic text is only a unit/integration fixture; live aggregate evidence is separately recorded in doctoring and never reported as approved labels. ## Future product test families From 9fe952ee1d75b2a221f608e01f83be628315dfd3 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Mon, 7 Sep 2026 10:04:18 +0900 Subject: [PATCH 41/47] test(zotero): exercise restored capture persisted boundary --- .../src/full_text_capture_tests.rs | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/crates/conceptweave-zotero/src/full_text_capture_tests.rs b/crates/conceptweave-zotero/src/full_text_capture_tests.rs index 1d45717b..ca333a1c 100644 --- a/crates/conceptweave-zotero/src/full_text_capture_tests.rs +++ b/crates/conceptweave-zotero/src/full_text_capture_tests.rs @@ -44,6 +44,24 @@ fn response_fixture(request_path: &str) -> CapturedResponse { } } +#[test] +fn restored_capture_verifier_enforces_exact_persisted_byte_boundary() { + let report = report_fixture(); + let capture = capture_with(&report, 4096, &mut |request_path, _| { + Ok(response_fixture(request_path)) + }) + .unwrap(); + let persisted_bytes = serde_json::to_vec(&capture).unwrap().len() as u64; + assert_eq!( + verify_capture_with_persisted_limit(&capture, &report, persisted_bytes), + Ok(()) + ); + assert_eq!( + verify_capture_with_persisted_limit(&capture, &report, persisted_bytes - 1), + Err(BUDGET_EXCEEDED) + ); +} + #[test] fn capture_retains_exact_text_missing_results_and_full_parent_denominator() { let report = report_fixture(); From 308dc31c9804894c06f5e88bf40ab00a4b14929b Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Mon, 7 Sep 2026 10:06:46 +0900 Subject: [PATCH 42/47] fix(zotero): share persisted capture verification boundary --- crates/conceptweave-zotero/src/full_text_capture.rs | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/crates/conceptweave-zotero/src/full_text_capture.rs b/crates/conceptweave-zotero/src/full_text_capture.rs index a6c9c390..c2bc29e4 100644 --- a/crates/conceptweave-zotero/src/full_text_capture.rs +++ b/crates/conceptweave-zotero/src/full_text_capture.rs @@ -101,9 +101,17 @@ fn read_full_text_from_api( pub fn verify_full_text_capture( capture: &FullTextCapture, report: &ClassificationReport, +) -> Result<(), FullTextError> { + verify_capture_with_persisted_limit(capture, report, MAX_PERSISTED_CAPTURE_BYTES) +} + +fn verify_capture_with_persisted_limit( + capture: &FullTextCapture, + report: &ClassificationReport, + max_persisted_bytes: u64, ) -> Result<(), FullTextError> { let snapshot = validate_report(report)?; - validate_persisted_capture_size(capture, MAX_PERSISTED_CAPTURE_BYTES)?; + validate_persisted_capture_size(capture, max_persisted_bytes)?; let evidence = &capture.capture_evidence; if evidence.records.len() > snapshot.len() { return Err(INVALID_EVIDENCE); From 51c2837ed22ea44e4d1359338383c9b83d9af476 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Mon, 7 Sep 2026 10:07:56 +0900 Subject: [PATCH 43/47] docs(test): record shared capture verifier boundary evidence --- TEST_STRATEGY.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/TEST_STRATEGY.md b/TEST_STRATEGY.md index 366b4d4e..a8d2929a 100644 --- a/TEST_STRATEGY.md +++ b/TEST_STRATEGY.md @@ -13,6 +13,8 @@ The full-text suite covers report admission, exact response retention, parent/item revision binding, independent content versions, missing/empty/partial text, duplicate/foreign manifest rejection, bookend drift, byte/deadline boundaries and replay under changed or recomputed digests. It also distinguishes the 256 MiB aggregate source-body budget from the 512 MiB persisted compact-JSON ceiling: an escape-heavy valid capture fixture stays within a proportionally scaled raw-body budget while its outer JSON grows past the corresponding persisted ceiling, and the size validator rejects that representation while accepting the exact serialized-byte boundary. Synthetic HTTP tests cover headers, network/redirect failures, strict encoding and response limits without touching the running Zotero library. Proxy isolation uses fresh subprocess environments for all six supported proxy variable spellings across the three existing local transport paths. Synthetic text is only a unit/integration fixture; live aggregate evidence is separately recorded in doctoring and never reported as approved labels. +The shared restored-capture verifier also has an exact compact-JSON boundary regression: the existing valid report/capture fixture passes at its serialized byte count and returns the budget error with one byte less allowance. The private limit parameter exposes the real verifier error path without allocating a 512 MiB fixture; the public verifier retains its fixed production ceiling and report-before-size validation order. This scaled test is not live-library or approval evidence. + ## Future product test families ### Source observation From 93929381e3a7797c371610b581057c8a5b411bcf Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Mon, 7 Sep 2026 10:14:01 +0900 Subject: [PATCH 44/47] docs(gap): distinguish verifier repair and visual evidence --- docs/product-technical-gap-baseline.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/docs/product-technical-gap-baseline.md b/docs/product-technical-gap-baseline.md index e51c3254..8285d247 100644 --- a/docs/product-technical-gap-baseline.md +++ b/docs/product-technical-gap-baseline.md @@ -6,6 +6,12 @@ This file records code-current product and technical gaps. Exact PR/check/run co ## September 6 source inventory checkpoint +### September 7 persisted-capture verifier follow-up + +Supplier `e517fc0a2ef268a5f80ff8e8f90ac24c36a170f3` already enforces a fixed 512 MiB compact-JSON ceiling; its focused serialization test passes. Downstream PR38 integration `21c3b88` exposed one uncovered verifier error-propagation region, not a demonstrated acceptance defect: 4,463/4,464 normalized regions and 710/710 normalized branches passed. The repair stays in PR36, the earliest owner. Test commit `9fe952e` introduced a private test seam and failed compilation with E0425 before implementation; this is not claimed as a new production behavioral RED. Production `308dc31c9804894c06f5e88bf40ab00a4b14929b` extracts the unchanged verifier body behind a private limit parameter. The public ceiling and report-before-size validation order remain unchanged. Its focused test passes at the exact serialized byte count and rejects a one-byte-smaller allowance. Independent read-only review found no semantic or validation-order regression. Full workspace, strict checks and unchanged coverage are pending in `/tmp/conceptweave-pr36-verifier-{tests,clippy,rustdoc,coverage}.log`; no push or hosted result is claimed for this follow-up. + +Earlier lock-screen statements below describe historical attempts, not a current blocker. The later September 7 PR37 inspection obtained both a native Zotero screenshot and accessibility state: 3,719 displayed items and visibly ellipsized long names, titles and creators. That display count is not the bibliographic denominator. The image remains private, and no click, edit, paper decision, approval or Zotero write accompanied inspection. Actual decisions and independent approvals remain 0/3,715 plus four unresolved standalone sources. + ### September 7 PR36 full-text source continuity verification Original PR36 `d3f991dcc1b746afed7c36f315e8937c39390c5e` passed 202 tests/39 suites. Normal merge `707f2f2` preserves its private full-text capture/proxy-isolation delta and PR34 `9eb89b8d4e751c34e640261f4883381571c83f25`. Existing capture admission delegates to worksheet construction, so shared inventory and parent consistency checks are inherited before any request. Existing whole-report hashing includes retained metadata and pending keys. No duplicate validator or hash was added. Initial integration failed compilation because two parent test callbacks required mutable borrows under PR36's writer signature; `c7052d9` repairs those calls and adds source-scope regressions. This is not a new production RED claim. From c2c155809bfa753cff82ec62883e1994cc8dfc82 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Mon, 7 Sep 2026 10:18:07 +0900 Subject: [PATCH 45/47] docs(trd): specify persisted capture ceiling --- docs/TRD.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/TRD.md b/docs/TRD.md index a8f6628c..458142b3 100644 --- a/docs/TRD.md +++ b/docs/TRD.md @@ -67,7 +67,7 @@ The proposed capture implementation uses `--capture-full-text /tmp/REPORT.json / The artifact records the complete metadata-report digest, metadata snapshot digest, full bibliographic denominator, read interval, manifest/library bookends and ordered exact response bodies, HTTP statuses and observed versions. Raw content JSON preserves empty text, index counters and unknown provider fields without claiming completeness. SHA-256 is streamed over the compact serialized evidence, with `non_atomic_fulltext_sweep_v1` separating it from a metadata snapshot or approval. Replay checks record/body bounds before parsing or digest work, then rechecks the report binding, structure, attachment membership and versions. Deserialization alone is not verification; a replaced digest is not authenticated authority. -Capture limits are 8 MiB per body, 256 MiB cumulative body bytes, the existing 50,000-item ceiling and a five-minute monotonic admission/completion budget. Request timeouts retain the local adapter's 30-second global / 2-second connect / 10-second response and body bounds; these are not model timeouts. A request already admitted can finish after the sweep deadline, but no late result is accepted. The writer streams JSON into a create-new `0600` temp file and retains a failed partial write for deliberate inspection. Raw response bytes are bounded separately from serialized file size, which can expand through JSON escaping. The 16 MiB private review-input reader is only used for the metadata report; it is not advertised as a large-capture reader. In-memory replay callers must bound private file deserialization separately. The capture remains a proposed local capability pending protected integration and review, with no change to proposal/decision/approval counts. +Capture limits are 8 MiB per body, 256 MiB cumulative body bytes, the existing 50,000-item ceiling and a five-minute monotonic admission/completion budget. Request timeouts retain the local adapter's 30-second global / 2-second connect / 10-second response and body bounds; these are not model timeouts. A request already admitted can finish after the sweep deadline, but no late result is accepted. The writer streams JSON into a create-new `0600` temp file and retains a failed partial write for deliberate inspection. Raw response bytes are bounded separately from the fixed 512 MiB compact-JSON ceiling, which includes outer JSON escaping and is checked by the shared capture verifier. The public verifier does not accept a caller-selected limit. The 16 MiB private review-input reader is only used for the metadata report; it is not advertised as a large-capture reader. In-memory replay callers must bound private file deserialization separately. The capture remains a proposed local capability pending protected integration and review, with no change to proposal/decision/approval counts. Execution receipts retain the verified proposal/source binding in every outcome. The authenticated-transport regression composes the executor with an ephemeral loopback HTTP fixture: a failed POST followed by a GET matching the requested From 5b0785444f5b217cba37d4f74a67f0f9bb03fb25 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Mon, 7 Sep 2026 10:20:18 +0900 Subject: [PATCH 46/47] docs(gap): record capture verifier workspace gates --- docs/product-technical-gap-baseline.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/product-technical-gap-baseline.md b/docs/product-technical-gap-baseline.md index 8285d247..8adc4ac7 100644 --- a/docs/product-technical-gap-baseline.md +++ b/docs/product-technical-gap-baseline.md @@ -12,6 +12,8 @@ Supplier `e517fc0a2ef268a5f80ff8e8f90ac24c36a170f3` already enforces a fixed 512 Earlier lock-screen statements below describe historical attempts, not a current blocker. The later September 7 PR37 inspection obtained both a native Zotero screenshot and accessibility state: 3,719 displayed items and visibly ellipsized long names, titles and creators. That display count is not the bibliographic denominator. The image remains private, and no click, edit, paper decision, approval or Zotero write accompanied inspection. Actual decisions and independent approvals remain 0/3,715 plus four unresolved standalone sources. +The `308dc31` follow-up subsequently passed 255 tests in 38 unfiltered suites, including three doctests; a nested filtered subprocess result is excluded rather than counted twice. Strict all-target Clippy, warnings-denied rustdoc, formatting, the CI contract and diff checks also pass. Session `48628` is still executing unchanged pinned coverage; the complete verification chain and push remain pending. Commits through `c2c1558` after production repair only update documentation. + ### September 7 PR36 full-text source continuity verification Original PR36 `d3f991dcc1b746afed7c36f315e8937c39390c5e` passed 202 tests/39 suites. Normal merge `707f2f2` preserves its private full-text capture/proxy-isolation delta and PR34 `9eb89b8d4e751c34e640261f4883381571c83f25`. Existing capture admission delegates to worksheet construction, so shared inventory and parent consistency checks are inherited before any request. Existing whole-report hashing includes retained metadata and pending keys. No duplicate validator or hash was added. Initial integration failed compilation because two parent test callbacks required mutable borrows under PR36's writer signature; `c7052d9` repairs those calls and adds source-scope regressions. This is not a new production RED claim. From c062845da8268089f09d7313dd1fb9604adfa178 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Mon, 7 Sep 2026 10:22:47 +0900 Subject: [PATCH 47/47] docs(gap): record terminal capture verifier coverage --- docs/product-technical-gap-baseline.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/product-technical-gap-baseline.md b/docs/product-technical-gap-baseline.md index 8adc4ac7..31634d3b 100644 --- a/docs/product-technical-gap-baseline.md +++ b/docs/product-technical-gap-baseline.md @@ -14,6 +14,8 @@ Earlier lock-screen statements below describe historical attempts, not a current The `308dc31` follow-up subsequently passed 255 tests in 38 unfiltered suites, including three doctests; a nested filtered subprocess result is excluded rather than counted twice. Strict all-target Clippy, warnings-denied rustdoc, formatting, the CI contract and diff checks also pass. Session `48628` is still executing unchanged pinned coverage; the complete verification chain and push remain pending. Commits through `c2c1558` after production repair only update documentation. +The follow-up verification chain `48628` subsequently completed successfully. Unchanged `nightly-2026-08-20` coverage executed 252 tests/36 unfiltered suites and reports 381/381 functions, 3,788/3,788 normalized regions and 658/658 normalized branches. Raw LLVM reports 4,746/4,818 lines, 7,164/7,285 regions and 610/658 branches, not 100%. Fresh fetch still places the supplier at `e517fc0a2ef268a5f80ff8e8f90ac24c36a170f3`, already an ancestor of this repair; no concurrent delta is discarded. Normal push updates the existing PR36, not protected main. PR37, PR38 and PR39 must inherit the repair in order and verify their combined trees before adoption claims. + ### September 7 PR36 full-text source continuity verification Original PR36 `d3f991dcc1b746afed7c36f315e8937c39390c5e` passed 202 tests/39 suites. Normal merge `707f2f2` preserves its private full-text capture/proxy-isolation delta and PR34 `9eb89b8d4e751c34e640261f4883381571c83f25`. Existing capture admission delegates to worksheet construction, so shared inventory and parent consistency checks are inherited before any request. Existing whole-report hashing includes retained metadata and pending keys. No duplicate validator or hash was added. Initial integration failed compilation because two parent test callbacks required mutable borrows under PR36's writer signature; `c7052d9` repairs those calls and adds source-scope regressions. This is not a new production RED claim.