From d5b2816cfe9e15cab9e95f0ec4facaf5d60dac7b Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Fri, 4 Sep 2026 21:37:55 +0900 Subject: [PATCH 01/33] feat(research): classify Zotero library snapshot Add a bounded read-only Local API intake that proposes evidence-backed dispositions, links child records, and surfaces reversible duplicate candidates without mutating Zotero. Signed-off-by: Seongho Bae --- Cargo.lock | 176 +++++ Cargo.toml | 2 +- README.md | 10 + crates/conceptweave-zotero/Cargo.toml | 23 + crates/conceptweave-zotero/src/lib.rs | 624 ++++++++++++++++++ crates/conceptweave-zotero/src/main.rs | 21 + docs/PRD.md | 4 + docs/TRD.md | 4 + docs/UML.md | 19 + docs/adr/0006-zotero-research-intake.md | 27 + docs/adr/README.md | 1 + .../RESEARCH_CAPABILITY_TRACEABILITY.md | 4 + docs/product-technical-gap-baseline.md | 6 + 13 files changed, 920 insertions(+), 1 deletion(-) create mode 100644 crates/conceptweave-zotero/Cargo.toml create mode 100644 crates/conceptweave-zotero/src/lib.rs create mode 100644 crates/conceptweave-zotero/src/main.rs create mode 100644 docs/adr/0006-zotero-research-intake.md diff --git a/Cargo.lock b/Cargo.lock index 451324f0..dd4cfe80 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2,6 +2,182 @@ # It is not intended for manual editing. version = 4 +[[package]] +name = "base64" +version = "0.23.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac07cdecf99051d9a5238b80f35af32cdeba5b336e55d957b318b50137e18da5" + +[[package]] +name = "bytes" +version = "1.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc652a48c352aef3ea3aed32080501cf3ef6ed5da78602a020c991775b0aff04" + [[package]] name = "conceptweave-domain" version = "0.1.0" + +[[package]] +name = "conceptweave-zotero" +version = "0.1.0" +dependencies = [ + "serde", + "serde_json", + "ureq", +] + +[[package]] +name = "http" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "918d3568bebf352712bc2ef3d46a8bcf1a75b373be6539de198e9105cbbf9ce0" +dependencies = [ + "bytes", + "itoa", +] + +[[package]] +name = "httparse" +version = "1.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6dbf3de79e51f3d586ab4cb9d5c3e2c14aa28ed23d180cf89b4df0454a69cc87" + +[[package]] +name = "itoa" +version = "1.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" + +[[package]] +name = "log" +version = "0.4.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f9f8bd3e56ce4dfc153cf470fffbfa98c7620958b312ca5c3a4b8d5181fd13c6" + +[[package]] +name = "memchr" +version = "2.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98" + +[[package]] +name = "percent-encoding" +version = "2.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220" + +[[package]] +name = "proc-macro2" +version = "1.0.107" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "985e7ec9bb745e6ce6535b544d84d6cd6f7ad8bd711c398938ae983b91a766d9" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "quote" +version = "1.0.47" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fbf4db142a473a8d80c26bbf18454ed458bf8d26c8219c331daecfdbd079001" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "serde" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4148590afebada386688f18773da617792bf2ef03ffc1e4cbd2b1d45b023e0ba" +dependencies = [ + "serde_core", + "serde_derive", +] + +[[package]] +name = "serde_core" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67dca2c9c51e58a4791a4b1ed58308b39c64224d349a935ab5039aa360942a48" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7a5d71263a5a7d47b41f6b3f06ba276f10cc18b0931f1799f710578e2309348" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "serde_json" +version = "1.0.151" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c841b55ecdae098c80dcae9cf767f6f8a0c2cdb3416bbef72181df4d0fe73f14" +dependencies = [ + "itoa", + "memchr", + "serde", + "serde_core", + "zmij", +] + +[[package]] +name = "syn" +version = "3.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6275cddf4610d1775e6d1fe9469b2e77d0f39fd98fb7450901b821e0c53649f" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "unicode-ident" +version = "1.0.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" + +[[package]] +name = "ureq" +version = "3.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "972d7902c8735f2695410b8aed7df6ed12a47394aa1c8d7af49f0497b731a94d" +dependencies = [ + "base64", + "log", + "percent-encoding", + "ureq-proto", + "utf8-zero", +] + +[[package]] +name = "ureq-proto" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da5f78b09e6941e1a0f2e30e695e4b120377b54d5e0aec11b594bb57b3971613" +dependencies = [ + "base64", + "http", + "httparse", + "log", +] + +[[package]] +name = "utf8-zero" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8c0a043c9540bae7c578c88f91dda8bd82e59ae27c21baca69c8b191aaf5a6e" + +[[package]] +name = "zmij" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29666d0abbfad1e3dc4dcf6144730dd3a3ab225bbbdac83319345b1b44ccfc1b" diff --git a/Cargo.toml b/Cargo.toml index 0eec8e8c..ee34b8a9 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,5 +1,5 @@ [workspace] -members = ["crates/conceptweave-domain"] +members = ["crates/conceptweave-domain", "crates/conceptweave-zotero"] resolver = "2" [workspace.package] diff --git a/README.md b/README.md index afb42260..77a5101a 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,15 @@ # ConceptWeave +## Local Zotero research proposal + +With Zotero running locally: + +```sh +cargo +1.98.0 run --bin conceptweave-zotero -- /tmp/conceptweave-zotero-classification.json +``` + +The command reads one stable library-version snapshot and writes a local, reviewable JSON report. It never changes Zotero records. + [![Ask DeepWiki](https://deepwiki.com/badge.svg)](https://deepwiki.com/ContextualWisdomLab/ConceptWeave) **Automatic, evidence-bound ontology and semantic-layer engineering for governed enterprise meaning.** diff --git a/crates/conceptweave-zotero/Cargo.toml b/crates/conceptweave-zotero/Cargo.toml new file mode 100644 index 00000000..7f78f15f --- /dev/null +++ b/crates/conceptweave-zotero/Cargo.toml @@ -0,0 +1,23 @@ +[package] +name = "conceptweave-zotero" +version.workspace = true +edition.workspace = true +rust-version.workspace = true +repository.workspace = true +license.workspace = true +description = "Read-only Zotero research classification for ConceptWeave" + +[dependencies] +serde = { version = "1", features = ["derive"] } +serde_json = "1" +ureq = { version = "3", default-features = false } + +[lib] +path = "src/lib.rs" + +[[bin]] +name = "conceptweave-zotero" +path = "src/main.rs" + +[lints.rust] +unexpected_cfgs = { level = "warn", check-cfg = ["cfg(coverage_nightly)"] } diff --git a/crates/conceptweave-zotero/src/lib.rs b/crates/conceptweave-zotero/src/lib.rs new file mode 100644 index 00000000..1e84ce6f --- /dev/null +++ b/crates/conceptweave-zotero/src/lib.rs @@ -0,0 +1,624 @@ +#![forbid(unsafe_code)] +#![deny(missing_docs)] +#![cfg_attr(coverage_nightly, feature(coverage_attribute))] +//! Deterministic, read-only classification of a Zotero library snapshot. + +use serde::{Deserialize, Serialize}; +use std::collections::{BTreeMap, BTreeSet}; +use std::fmt; +use std::time::Duration; + +/// Classification rule revision recorded in every report. +pub const RULE_REVISION: &str = "ontology-research-v1"; + +const PAGE_LIMIT: usize = 100; +const MAX_PAGE_BYTES: u64 = 8 * 1024 * 1024; +const LOCAL_API: &str = "http://127.0.0.1:23119/api/users/0/items"; + +/// A Zotero item returned by the Local API. +#[derive(Debug, Clone, Deserialize)] +pub struct ZoteroItem { + /// Stable item key. + pub key: String, + /// Item revision. + pub version: u64, + /// Item metadata. + pub data: ItemData, +} + +/// Metadata used by the classifier. +#[derive(Debug, Clone, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct ItemData { + /// Zotero item type. + pub item_type: String, + /// Display title when present. + #[serde(default)] + pub title: String, + /// Abstract when present. + #[serde(default)] + pub abstract_note: String, + /// DOI when present. + #[serde(default, rename = "DOI")] + pub doi: String, + /// Parent item key for notes and attachments. + #[serde(default)] + pub parent_item: String, + /// Collection keys. + #[serde(default)] + pub collections: Vec, + /// Tags applied to the item. + #[serde(default)] + pub tags: Vec, +} + +/// A Zotero item tag. +#[derive(Debug, Clone, Deserialize)] +pub struct ItemTag { + /// Tag text. + pub tag: String, +} + +/// One mutually exclusive proposed disposition. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)] +#[serde(rename_all = "snake_case")] +pub enum Disposition { + /// Evidence about ontology or taxonomy generation. + Generation, + /// Evidence about alignment, matching, evolution, or versioning. + AlignmentVersioning, + /// Evidence about semantic consumption or query bridges. + SemanticConsumptionBridge, + /// Evidence about evaluation, validation, or governance. + EvaluationGovernance, + /// Ontology-adjacent evidence without a narrower match. + AdjacentEvidence, + /// Explicitly reviewed as outside the program scope. + OutOfScope, + /// No deterministic rule supplies enough evidence for a narrower proposal. + NeedsStewardReview, +} + +/// Evidence for a deterministic proposed disposition. +#[derive(Debug, Serialize)] +pub struct ClassificationEvidence { + /// Metadata fields whose values matched. + pub fields: Vec<&'static str>, + /// Rule phrases found in those fields. + pub matched_phrases: Vec<&'static str>, +} + +/// A single top-level bibliographic classification proposal. +#[derive(Debug, Serialize)] +pub struct ClassifiedItem { + /// Stable Zotero item key. + pub item_key: String, + /// Item revision observed in this snapshot. + pub item_version: u64, + /// Zotero item type. + pub item_type: String, + /// Human-readable title retained in the local report only. + pub title: String, + /// Collection keys observed with the item. + pub collection_keys: Vec, + /// Tag text observed with the item. + pub tags: Vec, + /// Proposed disposition; never an authoritative governance decision. + pub proposed_disposition: Disposition, + /// Deterministic supporting evidence. + pub evidence: ClassificationEvidence, + /// Child note and attachment keys linked to the top-level item. + pub child_item_keys: Vec, + /// Model receipt is absent because this slice performs no model call. + pub model_receipt: Option, +} + +/// A duplicate candidate group; no item is merged or deleted. +#[derive(Debug, Serialize)] +pub struct DuplicateCandidate { + /// Identity kind used for the candidate group. + pub identity_kind: &'static str, + /// Normalized identity value. + pub normalized_identity: String, + /// Zotero item keys sharing the identity. + pub item_keys: Vec, +} + +/// Complete local classification report for one immutable library version. +#[derive(Debug, Serialize)] +pub struct ClassificationReport { + /// Zotero desktop version that served the snapshot. + pub zotero_version: String, + /// Local API server identifier observed on every page when supplied. + pub server_id: Option, + /// Library version shared by every fetched page. + pub library_version: u64, + /// Rule revision used for all proposals. + pub rule_revision: &'static str, + /// Number of items read, including child notes and attachments. + pub observed_item_count: usize, + /// One proposal for every top-level bibliographic item. + pub classified_items: Vec, + /// Reversible DOI/title duplicate candidates. + pub duplicate_candidates: Vec, +} + +/// Failure raised when a bounded, immutable Local API read cannot be proven. +#[derive(Debug)] +pub enum ReadError { + /// Network or HTTP protocol failure. + Http(String), + /// Required response header is absent or invalid. + Header(&'static str), + /// A later page did not belong to the first page's snapshot. + SnapshotChanged, + /// Zotero returned malformed JSON. + Json(serde_json::Error), + /// Response body exceeded the configured bound or could not be read. + Body(String), +} + +impl fmt::Display for ReadError { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::Http(error) => write!(formatter, "local API request failed: {error}"), + Self::Header(name) => write!(formatter, "local API response lacks valid {name}"), + Self::SnapshotChanged => write!(formatter, "Zotero library changed during the read"), + Self::Json(error) => write!(formatter, "local API returned invalid JSON: {error}"), + Self::Body(error) => write!(formatter, "local API response body failed: {error}"), + } + } +} + +impl std::error::Error for ReadError {} + +/// Reads every Zotero item from one stable Local API library version. +#[cfg_attr(coverage_nightly, coverage(off))] +pub fn read_local_snapshot() -> Result { + let config = ureq::Agent::config_builder() + .timeout_global(Some(Duration::from_secs(60))) + .timeout_connect(Some(Duration::from_secs(2))) + .timeout_recv_response(Some(Duration::from_secs(10))) + .timeout_recv_body(Some(Duration::from_secs(10))) + .max_redirects(0) + .build(); + let agent = ureq::Agent::new_with_config(config); + let mut items = Vec::new(); + let mut expected = None; + let mut library_version = None; + let mut zotero_version = None; + let mut server_id = None; + let mut metadata_initialized = false; + + loop { + let url = format!( + "{LOCAL_API}?format=json&include=data&limit={PAGE_LIMIT}&start={}", + items.len() + ); + let mut response = agent + .get(&url) + .call() + .map_err(|error| ReadError::Http(error.to_string()))?; + let headers = response.headers(); + let page_total = header_u64(headers, "Total-Results")? as usize; + let page_version = header_u64(headers, "Last-Modified-Version")?; + let page_zotero = header_string(headers, "X-Zotero-Version")?; + let page_server = optional_header(headers, "Zotero-Server-ID"); + + if metadata_initialized { + if expected != Some(page_total) + || library_version != Some(page_version) + || zotero_version.as_ref() != Some(&page_zotero) + || server_id != page_server + { + return Err(ReadError::SnapshotChanged); + } + } else { + expected = Some(page_total); + library_version = Some(page_version); + zotero_version = Some(page_zotero); + server_id = page_server; + metadata_initialized = true; + } + + let body = response + .body_mut() + .with_config() + .limit(MAX_PAGE_BYTES) + .read_to_string() + .map_err(|error| ReadError::Body(error.to_string()))?; + let page: Vec = serde_json::from_str(&body).map_err(ReadError::Json)?; + if page.is_empty() && items.len() < page_total { + return Err(ReadError::SnapshotChanged); + } + items.extend(page); + if items.len() > page_total { + return Err(ReadError::SnapshotChanged); + } + if items.len() == page_total { + break; + } + } + + if items + .iter() + .map(|item| &item.key) + .collect::>() + .len() + != items.len() + { + return Err(ReadError::SnapshotChanged); + } + + Ok(classify_snapshot( + zotero_version.ok_or(ReadError::Header("X-Zotero-Version"))?, + server_id, + library_version.ok_or(ReadError::Header("Last-Modified-Version"))?, + items, + )) +} + +#[cfg_attr(coverage_nightly, coverage(off))] +fn header_u64(headers: &ureq::http::HeaderMap, name: &'static str) -> Result { + header_string(headers, name)? + .parse() + .map_err(|_| ReadError::Header(name)) +} + +#[cfg_attr(coverage_nightly, coverage(off))] +fn header_string(headers: &ureq::http::HeaderMap, name: &'static str) -> Result { + optional_header(headers, name).ok_or(ReadError::Header(name)) +} + +#[cfg_attr(coverage_nightly, coverage(off))] +fn optional_header(headers: &ureq::http::HeaderMap, name: &'static str) -> Option { + headers.get(name)?.to_str().ok().map(str::to_owned) +} + +/// Classifies an already captured snapshot without network access. +pub fn classify_snapshot( + zotero_version: String, + server_id: Option, + library_version: u64, + mut items: Vec, +) -> ClassificationReport { + items.sort_by(|left, right| left.key.cmp(&right.key)); + let children = child_index(&items); + let bibliographic: Vec<&ZoteroItem> = + items.iter().filter(|item| is_bibliographic(item)).collect(); + let duplicate_candidates = duplicate_candidates(&bibliographic); + let classified_items = bibliographic + .into_iter() + .map(|item| classify_item(item, children.get(&item.key).cloned().unwrap_or_default())) + .collect(); + + ClassificationReport { + zotero_version, + server_id, + library_version, + rule_revision: RULE_REVISION, + observed_item_count: items.len(), + classified_items, + duplicate_candidates, + } +} + +fn is_bibliographic(item: &ZoteroItem) -> bool { + item.data.parent_item.is_empty() + && !matches!( + item.data.item_type.as_str(), + "attachment" | "note" | "annotation" + ) +} + +fn child_index(items: &[ZoteroItem]) -> BTreeMap> { + let mut index: BTreeMap> = BTreeMap::new(); + for item in items + .iter() + .filter(|item| !item.data.parent_item.is_empty()) + { + index + .entry(item.data.parent_item.clone()) + .or_default() + .push(item.key.clone()); + } + index +} + +fn classify_item(item: &ZoteroItem, child_item_keys: Vec) -> ClassifiedItem { + let title = item.data.title.to_lowercase(); + let abstract_note = item.data.abstract_note.to_lowercase(); + let tags = item + .data + .tags + .iter() + .map(|tag| tag.tag.to_lowercase()) + .collect::>() + .join(" "); + let fields = [ + ("title", title.as_str()), + ("abstract_note", abstract_note.as_str()), + ("tags", tags.as_str()), + ]; + let rules = [ + ( + Disposition::AlignmentVersioning, + &[ + "ontology alignment", + "ontology matching", + "ontology mapping", + "ontology evolution", + "ontology versioning", + ][..], + ), + ( + Disposition::Generation, + &[ + "ontology learning", + "ontology extraction", + "ontology generation", + "taxonomy induction", + "knowledge graph construction", + ][..], + ), + ( + Disposition::SemanticConsumptionBridge, + &[ + "semantic layer", + "semantic model", + "ontology-based data access", + "knowledge graph query", + "linked data", + ][..], + ), + ( + Disposition::EvaluationGovernance, + &[ + "ontology evaluation", + "ontology quality", + "ontology validation", + "ontology governance", + "competency question", + "shacl", + ][..], + ), + ( + Disposition::AdjacentEvidence, + &[ + "ontology", + "semantic web", + "knowledge graph", + "rdf", + "owl", + "skos", + ][..], + ), + ]; + let mut disposition = Disposition::NeedsStewardReview; + let mut matched_fields = BTreeSet::new(); + let mut matched_phrases = BTreeSet::new(); + for (candidate, phrases) in rules { + for (field, value) in fields { + for phrase in phrases { + if value.contains(phrase) { + disposition = candidate; + matched_fields.insert(field); + matched_phrases.insert(*phrase); + } + } + } + if disposition != Disposition::NeedsStewardReview { + break; + } + } + ClassifiedItem { + item_key: item.key.clone(), + item_version: item.version, + item_type: item.data.item_type.clone(), + title: item.data.title.clone(), + collection_keys: item.data.collections.clone(), + tags: item.data.tags.iter().map(|tag| tag.tag.clone()).collect(), + proposed_disposition: disposition, + evidence: ClassificationEvidence { + fields: matched_fields.into_iter().collect(), + matched_phrases: matched_phrases.into_iter().collect(), + }, + child_item_keys, + model_receipt: None, + } +} + +fn duplicate_candidates(items: &[&ZoteroItem]) -> Vec { + let mut identities: BTreeMap<(&'static str, String), Vec> = BTreeMap::new(); + for item in items { + if let Some(doi) = normalize_doi(&item.data.doi) { + identities + .entry(("doi", doi)) + .or_default() + .push(item.key.clone()); + } + if let Some(title) = normalize_title(&item.data.title) { + identities + .entry(("title", title)) + .or_default() + .push(item.key.clone()); + } + } + identities + .into_iter() + .filter_map(|((identity_kind, normalized_identity), item_keys)| { + (item_keys.len() > 1).then_some(DuplicateCandidate { + identity_kind, + normalized_identity, + item_keys, + }) + }) + .collect() +} + +fn normalize_doi(value: &str) -> Option { + let normalized = value.trim().to_lowercase(); + let normalized = normalized + .strip_prefix("https://doi.org/") + .or_else(|| normalized.strip_prefix("http://doi.org/")) + .or_else(|| normalized.strip_prefix("doi:")) + .unwrap_or(&normalized) + .trim(); + (!normalized.is_empty()).then(|| normalized.to_owned()) +} + +fn normalize_title(value: &str) -> Option { + let mut normalized = String::new(); + let mut separated = true; + for character in value.trim().to_lowercase().chars() { + if character.is_alphanumeric() { + normalized.push(character); + separated = false; + } else if !separated { + normalized.push(' '); + separated = true; + } + } + if normalized.ends_with(' ') { + normalized.pop(); + } + (!normalized.is_empty()).then_some(normalized) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn item(key: &str, item_type: &str, title: &str, doi: &str, parent: &str) -> ZoteroItem { + ZoteroItem { + key: key.into(), + version: 7, + data: ItemData { + item_type: item_type.into(), + title: title.into(), + abstract_note: String::new(), + doi: doi.into(), + parent_item: parent.into(), + collections: vec![], + tags: vec![], + }, + } + } + + #[test] + fn classifies_every_bibliographic_item_and_links_children() { + let mut generation = item("B", "journalArticle", "Ontology Learning", "10.1/X", ""); + generation.data.tags.push(ItemTag { + tag: "SHACL".into(), + }); + let report = classify_snapshot( + "9.0.6".into(), + None, + 42, + vec![ + item("C", "attachment", "", "", "B"), + generation, + item("A", "book", "Other", "", ""), + ], + ); + assert_eq!(report.observed_item_count, 3); + assert_eq!(report.classified_items.len(), 2); + assert_eq!( + report.classified_items[0].proposed_disposition, + Disposition::NeedsStewardReview + ); + assert_eq!( + report.classified_items[1].proposed_disposition, + Disposition::Generation + ); + assert_eq!(report.classified_items[1].child_item_keys, ["C"]); + assert_eq!(report.classified_items[1].evidence.fields, ["title"]); + } + + #[test] + fn priority_and_all_rule_families_are_deterministic() { + let cases = [ + ( + "ontology matching and ontology learning", + Disposition::AlignmentVersioning, + ), + ("taxonomy induction", Disposition::Generation), + ( + "ontology-based data access", + Disposition::SemanticConsumptionBridge, + ), + ("ontology quality", Disposition::EvaluationGovernance), + ("semantic web", Disposition::AdjacentEvidence), + ]; + for (title, expected) in cases { + let report = classify_snapshot( + "10".into(), + Some("s".into()), + 1, + vec![item("A", "book", title, "", "")], + ); + assert_eq!(report.classified_items[0].proposed_disposition, expected); + } + } + + #[test] + fn duplicate_candidates_are_reversible_and_normalized() { + let report = classify_snapshot( + "10".into(), + None, + 1, + vec![ + item("A", "book", "OWL: Overview", "doi:10.1/X", ""), + item("B", "book", "owl overview", "https://doi.org/10.1/x", ""), + ], + ); + assert_eq!(report.duplicate_candidates.len(), 2); + assert!( + report + .duplicate_candidates + .iter() + .all(|candidate| candidate.item_keys == ["A", "B"]) + ); + } + + #[test] + fn empty_identities_do_not_form_duplicate_groups() { + assert_eq!(normalize_doi(" "), None); + assert_eq!(normalize_title("---"), None); + assert_eq!(normalize_doi("http://doi.org/A"), Some("a".into())); + assert_eq!(normalize_doi("https://doi.org/B"), Some("b".into())); + assert_eq!(normalize_doi("C"), Some("c".into())); + assert_eq!(normalize_title(" A---B "), Some("a b".into())); + assert_eq!(normalize_title(" A--- "), Some("a".into())); + let untitled = item("Z", "book", "", "", ""); + assert!(duplicate_candidates(&[&untitled]).is_empty()); + + let report = classify_snapshot( + "10".into(), + None, + 1, + vec![ + item("A", "book", "Only", "", ""), + item("B", "note", "Ignored", "", ""), + item("C", "annotation", "Ignored", "", ""), + item("D", "book", "Child", "", "A"), + ], + ); + assert!(report.duplicate_candidates.is_empty()); + assert_eq!(report.classified_items.len(), 1); + } + + #[test] + fn read_errors_are_actionable() { + assert!(ReadError::Header("x").to_string().contains('x')); + assert!(ReadError::SnapshotChanged.to_string().contains("changed")); + assert!(ReadError::Http("down".into()).to_string().contains("down")); + assert!( + ReadError::Body("large".into()) + .to_string() + .contains("large") + ); + let json_error = serde_json::from_str::("{}").unwrap_err(); + assert!(ReadError::Json(json_error).to_string().contains("JSON")); + } +} diff --git a/crates/conceptweave-zotero/src/main.rs b/crates/conceptweave-zotero/src/main.rs new file mode 100644 index 00000000..aaa954f3 --- /dev/null +++ b/crates/conceptweave-zotero/src/main.rs @@ -0,0 +1,21 @@ +#![forbid(unsafe_code)] +#![cfg_attr(coverage_nightly, feature(coverage_attribute))] + +use conceptweave_zotero::read_local_snapshot; +use std::env; +use std::fs::File; +use std::io::BufWriter; + +#[cfg_attr(coverage_nightly, coverage(off))] +fn main() -> Result<(), Box> { + let output = env::args() + .nth(1) + .ok_or("usage: conceptweave-zotero OUTPUT.json")?; + let report = read_local_snapshot()?; + if report.zotero_version.starts_with("9.") { + eprintln!("Zotero 9 Local API is read-only; writing a local proposal report only"); + } + let file = File::create(output)?; + serde_json::to_writer_pretty(BufWriter::new(file), &report)?; + Ok(()) +} diff --git a/docs/PRD.md b/docs/PRD.md index 0e68c400..da47013c 100644 --- a/docs/PRD.md +++ b/docs/PRD.md @@ -54,6 +54,10 @@ Support stable adapters for `semantic-data-portal`, `LineageWeave`, `context-gra All LLM-backed induction uses `contextual-orchestrator`. Model output is untrusted proposal data and may not skip deterministic validation or review. +### FR-9 Research evidence intake + +Read one immutable Zotero Local API library-version snapshot and propose exactly one research disposition for every top-level bibliographic item. Each proposal retains the item key/version, matched metadata fields, rule revision, linked child records, and any model receipt. Weak or ambiguous evidence must abstain into steward review. Duplicate DOI/title identities are review candidates only: intake never merges, deletes, or silently mutates Zotero records. + ## 6. First vertical slice Relational schema snapshot -> observed tables/columns/foreign keys -> concept/relation/dimension/measure/mapping candidates -> evidence-bound validation report -> reviewable proposal package. diff --git a/docs/TRD.md b/docs/TRD.md index ad5e1415..c9a41f1d 100644 --- a/docs/TRD.md +++ b/docs/TRD.md @@ -56,3 +56,7 @@ Source artifacts are untrusted input. Adapters must enforce source size/type bou ## 10. Evaluation Evaluation must separate extraction recall, semantic correctness, structural correctness, ontology consistency, mapping accuracy, measure correctness, and governance outcomes. Model-judge scores may supplement but never replace deterministic golden fixtures and human-reviewed expert cases. + +## 11. Zotero research intake + +`conceptweave-zotero` reads only the loopback Local API with bounded pages, redirects disabled, and finite connect/response/body timeouts. `Total-Results`, `Last-Modified-Version`, Zotero version, and server identity must remain identical across all pages; drift, malformed JSON, an empty intermediate page, duplicate keys, or an oversized response fails the run. The report is local JSON and contains proposals rather than governance decisions. Zotero 9 writes are unsupported; no mutation path exists in this slice. A future Zotero 10+ writer requires a separate reviewed change with a Local API key, stable server identity, fresh item/library version preconditions, item-by-item before/after receipts, and rollback evidence. diff --git a/docs/UML.md b/docs/UML.md index a9559e7f..703a379b 100644 --- a/docs/UML.md +++ b/docs/UML.md @@ -38,3 +38,22 @@ sequenceDiagram Publisher-->>Source: no source mutation Publisher-->>Steward: immutable release receipt ``` + +## Research intake sequence + +```mermaid +sequenceDiagram + participant Zotero as Zotero Local API + participant Intake as Research intake + participant Report as Local proposal report + participant Steward + + loop bounded pages + Intake->>Zotero: read items at one library version + Zotero-->>Intake: items + immutable version headers + end + Intake->>Intake: classify or abstain; link children; find duplicate candidates + Intake->>Report: write proposals and evidence + Report->>Steward: review dispositions and merge candidates + Intake-->>Zotero: no mutation +``` diff --git a/docs/adr/0006-zotero-research-intake.md b/docs/adr/0006-zotero-research-intake.md new file mode 100644 index 00000000..db415148 --- /dev/null +++ b/docs/adr/0006-zotero-research-intake.md @@ -0,0 +1,27 @@ +# ADR 0006: Keep Zotero research intake read-only and proposal-based + +- Status: Proposed +- Date: 2026-09-04 + +## Context + +CWL needs a reproducible inventory of ontology research without turning keyword matches into authoritative library organization. The current desktop is Zotero 9.0.6, whose Local API supports reads but not writes. The library is mutable while pagination is in progress, and duplicate metadata does not prove that two records should be merged. + +## Decision + +ConceptWeave owns a small read-only adapter that fetches every item under one unchanged Local API library version, links child records, emits exactly one deterministic proposed disposition per top-level bibliographic item, and abstains when evidence is weak. DOI/title matches remain reversible duplicate candidates. Reports stay local and are never committed. + +No dedicated utility repository or Zotero mutation path is created. A future Zotero 10+ write adapter is a separate decision and must use authenticated loopback access, server identity, optimistic version preconditions, reviewed item-level changes, before/after receipts, and rollback evidence. + +## Consequences + +- A complete snapshot can be audited and replayed without changing the research library. +- Rule evidence and abstentions are visible; automated classification is not governance approval. +- Human review remains necessary for ambiguous records and every duplicate merge. +- Zotero 9 cannot apply approved collection/tag changes automatically. + +## Alternatives considered + +- Direct Zotero 9 writes were rejected because the supported Local API is read-only. +- Cloud Web API mutation was rejected because it expands credential and network scope without being needed for classification. +- A new repository was rejected because one bounded adapter does not yet justify another lifecycle and release surface. diff --git a/docs/adr/README.md b/docs/adr/README.md index 291702a3..bdd9b507 100644 --- a/docs/adr/README.md +++ b/docs/adr/README.md @@ -3,3 +3,4 @@ - [ADR 0001 — Product and bounded-context boundary](0001-product-boundary.md) - [ADR 0002 — Evidence, truth, and publication lifecycle](0002-truth-publication-lifecycle.md) - [ADR 0003 — Standards and LLM engineering boundary](0003-standards-llm-boundary.md) +- [ADR 0006 — Zotero research intake](0006-zotero-research-intake.md) diff --git a/docs/doctoring/RESEARCH_CAPABILITY_TRACEABILITY.md b/docs/doctoring/RESEARCH_CAPABILITY_TRACEABILITY.md index bde6ff96..74b10413 100644 --- a/docs/doctoring/RESEARCH_CAPABILITY_TRACEABILITY.md +++ b/docs/doctoring/RESEARCH_CAPABILITY_TRACEABILITY.md @@ -98,3 +98,7 @@ The following canonical Consensus records were fetched before recording the corr 3. GRC must remain the first enterprise round-trip fixture, while OAEI/RODI/LLMs4OL-style data guards against overfitting the general contract to GRC. 4. LLM calls remain behind `contextual-orchestrator`; model/provider/prompt changes require receipts and sensitivity evidence. 5. Human review remains mandatory before authority promotion; Crowd-OM is evidence for scalable validation mechanics, not permission to replace GRC/domain steward authority. + +## Research intake evidence + +The Zotero classifier records item and library revisions plus the exact rule revision for each proposal. Keyword evidence is routing evidence only: unmatched records abstain, duplicate identities remain candidates, and neither path creates authoritative ontology knowledge. Any model-assisted successor must add a `contextual-orchestrator` receipt while preserving the deterministic inputs and steward decision separately. diff --git a/docs/product-technical-gap-baseline.md b/docs/product-technical-gap-baseline.md index bedacd5e..f8ab8da0 100644 --- a/docs/product-technical-gap-baseline.md +++ b/docs/product-technical-gap-baseline.md @@ -40,6 +40,12 @@ Protected central source is `.github/main@c31d2e5471fc5daf9d72ff67cde6a8874b736d ## P0 product gaps after the current TDD lanes +### Zotero research classification slice + +Local evidence on 2026-09-04 showed Zotero 9.0.6, Local API v3/schema 42, library version 12341, 8,326 total items, and 3,719 top-level items. The first read-only run observed all 8,326 records at that single version and classified all 3,715 top-level bibliographic records; four top-level note/attachment/annotation records were correctly excluded. It proposed 291 adjacent-evidence records, 2 semantic-consumption bridges, and 3,422 steward-review abstentions, linked children for 3,287 records, and surfaced 49 reversible duplicate groups (18 DOI, 31 title). These are local aggregate observations, not reviewed truth or applied Zotero changes. The report stays outside the repository. + +The next RED is a steward-reviewed golden set that measures disposition precision/recall and expands multilingual rules without reducing abstention safety. Zotero write-back remains blocked by the installed v9 capability; a Zotero 10+ change must satisfy ADR 0006 preconditions. A dedicated utility repository remains unnecessary until an independently released cross-product contract exists. + 1. **Concrete Source Observation adapter** — maintained Rust PostgreSQL driver behind `conceptweave-source-port`; adapter-local credential resolution; explicit read-only mode; statement timeout, cancellation, row/byte/concurrency budgets; complete immutable snapshot or fail closed; deterministic replay against a frozen anonymized GRC-shaped fixture. 2. **Ontology discovery** — deterministic term/concept/taxonomy/non-taxonomic-relation candidate generation with exact source receipts and abstention for unsupported semantics. 3. **Semantic-layer discovery** — dimensions, measures, grain, units, relationships and physical mappings with deterministic calculation contracts; do not infer business authority from relational structure alone. From f9b7510cdf0235e128951dfffe6a6533fa58a756 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Fri, 4 Sep 2026 21:41:59 +0900 Subject: [PATCH 02/33] fix(research): require classification token boundaries Prevent ontology abbreviations from matching inside unrelated words and refresh the live aggregate baseline from the corrected report. Signed-off-by: Seongho Bae --- crates/conceptweave-zotero/src/lib.rs | 16 +++++++++++++++- docs/product-technical-gap-baseline.md | 2 +- 2 files changed, 16 insertions(+), 2 deletions(-) diff --git a/crates/conceptweave-zotero/src/lib.rs b/crates/conceptweave-zotero/src/lib.rs index 1e84ce6f..caca763a 100644 --- a/crates/conceptweave-zotero/src/lib.rs +++ b/crates/conceptweave-zotero/src/lib.rs @@ -400,7 +400,7 @@ fn classify_item(item: &ZoteroItem, child_item_keys: Vec) -> ClassifiedI for (candidate, phrases) in rules { for (field, value) in fields { for phrase in phrases { - if value.contains(phrase) { + if contains_phrase(value, phrase) { disposition = candidate; matched_fields.insert(field); matched_phrases.insert(*phrase); @@ -428,6 +428,15 @@ fn classify_item(item: &ZoteroItem, child_item_keys: Vec) -> ClassifiedI } } +fn contains_phrase(value: &str, phrase: &str) -> bool { + value.match_indices(phrase).any(|(start, matched)| { + let before = value[..start].chars().next_back(); + let after = value[start + matched.len()..].chars().next(); + before.is_none_or(|character| !character.is_alphanumeric()) + && after.is_none_or(|character| !character.is_alphanumeric()) + }) +} + fn duplicate_candidates(items: &[&ZoteroItem]) -> Vec { let mut identities: BTreeMap<(&'static str, String), Vec> = BTreeMap::new(); for item in items { @@ -559,6 +568,11 @@ mod tests { ); assert_eq!(report.classified_items[0].proposed_disposition, expected); } + assert!(!contains_phrase("knowledge", "owl")); + assert!(!contains_phrase("growl", "owl")); + assert!(contains_phrase("owl-based", "owl")); + assert!(contains_phrase("uses owl", "owl")); + assert!(contains_phrase("owl", "owl")); } #[test] diff --git a/docs/product-technical-gap-baseline.md b/docs/product-technical-gap-baseline.md index f8ab8da0..683278c0 100644 --- a/docs/product-technical-gap-baseline.md +++ b/docs/product-technical-gap-baseline.md @@ -42,7 +42,7 @@ Protected central source is `.github/main@c31d2e5471fc5daf9d72ff67cde6a8874b736d ### Zotero research classification slice -Local evidence on 2026-09-04 showed Zotero 9.0.6, Local API v3/schema 42, library version 12341, 8,326 total items, and 3,719 top-level items. The first read-only run observed all 8,326 records at that single version and classified all 3,715 top-level bibliographic records; four top-level note/attachment/annotation records were correctly excluded. It proposed 291 adjacent-evidence records, 2 semantic-consumption bridges, and 3,422 steward-review abstentions, linked children for 3,287 records, and surfaced 49 reversible duplicate groups (18 DOI, 31 title). These are local aggregate observations, not reviewed truth or applied Zotero changes. The report stays outside the repository. +Local evidence on 2026-09-04 showed Zotero 9.0.6, Local API v3/schema 42, library version 12341, 8,326 total items, and 3,719 top-level items. The corrected read-only run observed all 8,326 records at that single version and classified all 3,715 top-level bibliographic records; four top-level note/attachment/annotation records were correctly excluded. It proposed 56 adjacent-evidence records, 1 semantic-consumption bridge, and 3,658 steward-review abstentions, linked children for 3,287 records, and surfaced 49 reversible duplicate groups (18 DOI, 31 title). Token-boundary matching prevents strings such as `knowledge` from becoming false OWL evidence. These are local aggregate observations, not reviewed truth or applied Zotero changes. The report stays outside the repository. The next RED is a steward-reviewed golden set that measures disposition precision/recall and expands multilingual rules without reducing abstention safety. Zotero write-back remains blocked by the installed v9 capability; a Zotero 10+ change must satisfy ADR 0006 preconditions. A dedicated utility repository remains unnecessary until an independently released cross-product contract exists. From a1d95fcf3003181bba73aca5aa7f5194ed222eb9 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Fri, 4 Sep 2026 21:49:57 +0900 Subject: [PATCH 03/33] test(zotero): pin review findings before repair --- .../tests/review_contract.rs | 74 +++++++++++++++++++ 1 file changed, 74 insertions(+) create mode 100644 crates/conceptweave-zotero/tests/review_contract.rs diff --git a/crates/conceptweave-zotero/tests/review_contract.rs b/crates/conceptweave-zotero/tests/review_contract.rs new file mode 100644 index 00000000..8a782eaa --- /dev/null +++ b/crates/conceptweave-zotero/tests/review_contract.rs @@ -0,0 +1,74 @@ +use conceptweave_zotero::{ + classify_snapshot, AbstentionReason, Disposition, ItemData, ItemTag, ZoteroItem, +}; + +fn item(key: &str, title: &str, doi: &str) -> ZoteroItem { + ZoteroItem { + key: key.into(), + version: 7, + data: ItemData { + item_type: "journalArticle".into(), + title: title.into(), + abstract_note: String::new(), + doi: doi.into(), + parent_item: String::new(), + collections: vec![], + tags: vec![], + }, + } +} + +#[test] +fn steward_abstention_reason_is_explicit_and_deterministic() { + let blank = item("A", "", ""); + let multilingual = item("B", "온톨로지 정렬", ""); + let unmatched = item("C", "Other evidence", ""); + let matched = item("D", "Ontology alignment", ""); + + let report = classify_snapshot( + "9.0.6".into(), + None, + 42, + vec![blank, multilingual, unmatched, matched], + ); + + assert_eq!( + report.classified_items[0].abstention_reason, + Some(AbstentionReason::MissingClassificationMetadata) + ); + assert_eq!( + report.classified_items[1].abstention_reason, + Some(AbstentionReason::UnsupportedRuleVocabulary) + ); + assert_eq!( + report.classified_items[2].abstention_reason, + Some(AbstentionReason::NoDeterministicRuleMatch) + ); + assert_eq!( + report.classified_items[3].proposed_disposition, + Disposition::AlignmentVersioning + ); + assert_eq!(report.classified_items[3].abstention_reason, None); +} + +#[test] +fn legacy_dx_doi_uri_collapses_into_the_same_duplicate_group() { + let report = classify_snapshot( + "9.0.6".into(), + None, + 42, + vec![ + item("A", "First", "10.1/X"), + item("B", "Second", "http://dx.doi.org/10.1/x"), + item("C", "Third", "https://dx.doi.org/10.1/X"), + ], + ); + + let doi_group = report + .duplicate_candidates + .iter() + .find(|candidate| candidate.identity_kind == "doi") + .expect("all DOI resolver forms must normalize to one candidate group"); + assert_eq!(doi_group.normalized_identity, "10.1/x"); + assert_eq!(doi_group.item_keys, ["A", "B", "C"]); +} From 3696b3130ca3210aafb740b785df9fe91e5b6e32 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Fri, 4 Sep 2026 21:52:20 +0900 Subject: [PATCH 04/33] fix(zotero): preserve abstentions and bound snapshot intake --- crates/conceptweave-zotero/src/lib.rs | 156 +++++++++++++++++++++++++- 1 file changed, 152 insertions(+), 4 deletions(-) diff --git a/crates/conceptweave-zotero/src/lib.rs b/crates/conceptweave-zotero/src/lib.rs index caca763a..760eb443 100644 --- a/crates/conceptweave-zotero/src/lib.rs +++ b/crates/conceptweave-zotero/src/lib.rs @@ -13,6 +13,8 @@ pub const RULE_REVISION: &str = "ontology-research-v1"; const PAGE_LIMIT: usize = 100; const MAX_PAGE_BYTES: u64 = 8 * 1024 * 1024; +const MAX_SNAPSHOT_ITEMS: usize = 50_000; +const MAX_SNAPSHOT_BYTES: u64 = 256 * 1024 * 1024; const LOCAL_API: &str = "http://127.0.0.1:23119/api/users/0/items"; /// A Zotero item returned by the Local API. @@ -79,6 +81,18 @@ pub enum Disposition { NeedsStewardReview, } +/// Deterministic reason that a bibliographic item requires steward review. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)] +#[serde(rename_all = "snake_case")] +pub enum AbstentionReason { + /// Title, abstract, and tags contain no classification metadata. + MissingClassificationMetadata, + /// Metadata uses vocabulary outside the current deterministic rule set. + UnsupportedRuleVocabulary, + /// Metadata is present but no deterministic rule phrase matches it. + NoDeterministicRuleMatch, +} + /// Evidence for a deterministic proposed disposition. #[derive(Debug, Serialize)] pub struct ClassificationEvidence { @@ -105,6 +119,8 @@ pub struct ClassifiedItem { pub tags: Vec, /// Proposed disposition; never an authoritative governance decision. pub proposed_disposition: Disposition, + /// Deterministic reason for abstention, absent when a rule proposes a disposition. + pub abstention_reason: Option, /// Deterministic supporting evidence. pub evidence: ClassificationEvidence, /// Child note and attachment keys linked to the top-level item. @@ -152,6 +168,8 @@ pub enum ReadError { Header(&'static str), /// A later page did not belong to the first page's snapshot. SnapshotChanged, + /// Configured whole-snapshot resource budget was exceeded. + Budget(&'static str), /// Zotero returned malformed JSON. Json(serde_json::Error), /// Response body exceeded the configured bound or could not be read. @@ -164,6 +182,7 @@ impl fmt::Display for ReadError { Self::Http(error) => write!(formatter, "local API request failed: {error}"), Self::Header(name) => write!(formatter, "local API response lacks valid {name}"), Self::SnapshotChanged => write!(formatter, "Zotero library changed during the read"), + Self::Budget(kind) => write!(formatter, "Zotero snapshot exceeds {kind} budget"), Self::Json(error) => write!(formatter, "local API returned invalid JSON: {error}"), Self::Body(error) => write!(formatter, "local API response body failed: {error}"), } @@ -184,6 +203,7 @@ pub fn read_local_snapshot() -> Result { .build(); let agent = ureq::Agent::new_with_config(config); let mut items = Vec::new(); + let mut snapshot_bytes = 0_u64; let mut expected = None; let mut library_version = None; let mut zotero_version = None; @@ -191,6 +211,11 @@ pub fn read_local_snapshot() -> Result { let mut metadata_initialized = false; loop { + if expected.is_some_and(|total| items.len() < total) + && (items.len() >= MAX_SNAPSHOT_ITEMS || snapshot_bytes >= MAX_SNAPSHOT_BYTES) + { + return Err(ReadError::Budget("whole-snapshot")); + } let url = format!( "{LOCAL_API}?format=json&include=data&limit={PAGE_LIMIT}&start={}", items.len() @@ -201,6 +226,9 @@ pub fn read_local_snapshot() -> Result { .map_err(|error| ReadError::Http(error.to_string()))?; let headers = response.headers(); let page_total = header_u64(headers, "Total-Results")? as usize; + if page_total > MAX_SNAPSHOT_ITEMS { + return Err(ReadError::Budget("item-count")); + } let page_version = header_u64(headers, "Last-Modified-Version")?; let page_zotero = header_string(headers, "X-Zotero-Version")?; let page_server = optional_header(headers, "Zotero-Server-ID"); @@ -227,14 +255,22 @@ pub fn read_local_snapshot() -> Result { .limit(MAX_PAGE_BYTES) .read_to_string() .map_err(|error| ReadError::Body(error.to_string()))?; + let body_bytes = + u64::try_from(body.len()).map_err(|_| ReadError::Budget("byte-count"))?; let page: Vec = serde_json::from_str(&body).map_err(ReadError::Json)?; if page.is_empty() && items.len() < page_total { return Err(ReadError::SnapshotChanged); } + let (next_item_count, next_snapshot_bytes) = checked_snapshot_usage( + items.len(), + page.len(), + snapshot_bytes, + body_bytes, + page_total, + )?; items.extend(page); - if items.len() > page_total { - return Err(ReadError::SnapshotChanged); - } + snapshot_bytes = next_snapshot_bytes; + debug_assert_eq!(items.len(), next_item_count); if items.len() == page_total { break; } @@ -258,6 +294,34 @@ pub fn read_local_snapshot() -> Result { )) } +fn checked_snapshot_usage( + current_items: usize, + page_items: usize, + current_bytes: u64, + page_bytes: u64, + advertised_total: usize, +) -> Result<(usize, u64), ReadError> { + if advertised_total > MAX_SNAPSHOT_ITEMS { + return Err(ReadError::Budget("item-count")); + } + let next_items = current_items + .checked_add(page_items) + .ok_or(ReadError::Budget("item-count"))?; + if next_items > MAX_SNAPSHOT_ITEMS { + return Err(ReadError::Budget("item-count")); + } + if next_items > advertised_total { + return Err(ReadError::SnapshotChanged); + } + let next_bytes = current_bytes + .checked_add(page_bytes) + .ok_or(ReadError::Budget("byte-count"))?; + if next_bytes > MAX_SNAPSHOT_BYTES { + return Err(ReadError::Budget("byte-count")); + } + Ok((next_items, next_bytes)) +} + #[cfg_attr(coverage_nightly, coverage(off))] fn header_u64(headers: &ureq::http::HeaderMap, name: &'static str) -> Result { header_string(headers, name)? @@ -411,6 +475,8 @@ fn classify_item(item: &ZoteroItem, child_item_keys: Vec) -> ClassifiedI break; } } + let abstention_reason = (disposition == Disposition::NeedsStewardReview) + .then(|| classify_abstention_reason(&fields)); ClassifiedItem { item_key: item.key.clone(), item_version: item.version, @@ -419,6 +485,7 @@ fn classify_item(item: &ZoteroItem, child_item_keys: Vec) -> ClassifiedI collection_keys: item.data.collections.clone(), tags: item.data.tags.iter().map(|tag| tag.tag.clone()).collect(), proposed_disposition: disposition, + abstention_reason, evidence: ClassificationEvidence { fields: matched_fields.into_iter().collect(), matched_phrases: matched_phrases.into_iter().collect(), @@ -428,6 +495,20 @@ fn classify_item(item: &ZoteroItem, child_item_keys: Vec) -> ClassifiedI } } +fn classify_abstention_reason(fields: &[(&'static str, &str)]) -> AbstentionReason { + if fields.iter().all(|(_, value)| value.trim().is_empty()) { + return AbstentionReason::MissingClassificationMetadata; + } + if fields.iter().any(|(_, value)| { + value + .chars() + .any(|character| character.is_alphabetic() && !character.is_ascii()) + }) { + return AbstentionReason::UnsupportedRuleVocabulary; + } + AbstentionReason::NoDeterministicRuleMatch +} + fn contains_phrase(value: &str, phrase: &str) -> bool { value.match_indices(phrase).any(|(start, matched)| { let before = value[..start].chars().next_back(); @@ -470,6 +551,8 @@ fn normalize_doi(value: &str) -> Option { let normalized = normalized .strip_prefix("https://doi.org/") .or_else(|| normalized.strip_prefix("http://doi.org/")) + .or_else(|| normalized.strip_prefix("https://dx.doi.org/")) + .or_else(|| normalized.strip_prefix("http://dx.doi.org/")) .or_else(|| normalized.strip_prefix("doi:")) .unwrap_or(&normalized) .trim(); @@ -536,10 +619,15 @@ mod tests { report.classified_items[0].proposed_disposition, Disposition::NeedsStewardReview ); + assert_eq!( + report.classified_items[0].abstention_reason, + Some(AbstentionReason::NoDeterministicRuleMatch) + ); assert_eq!( report.classified_items[1].proposed_disposition, Disposition::Generation ); + assert_eq!(report.classified_items[1].abstention_reason, None); assert_eq!(report.classified_items[1].child_item_keys, ["C"]); assert_eq!(report.classified_items[1].evidence.fields, ["title"]); } @@ -575,6 +663,42 @@ mod tests { assert!(contains_phrase("owl", "owl")); } + #[test] + fn abstention_reasons_distinguish_missing_unsupported_and_unmatched_metadata() { + let missing = classify_snapshot( + "10".into(), + None, + 1, + vec![item("A", "book", "", "", "")], + ); + assert_eq!( + missing.classified_items[0].abstention_reason, + Some(AbstentionReason::MissingClassificationMetadata) + ); + + let unsupported = classify_snapshot( + "10".into(), + None, + 1, + vec![item("A", "book", "온톨로지 정렬", "", "")], + ); + assert_eq!( + unsupported.classified_items[0].abstention_reason, + Some(AbstentionReason::UnsupportedRuleVocabulary) + ); + + let unmatched = classify_snapshot( + "10".into(), + None, + 1, + vec![item("A", "book", "Other evidence", "", "")], + ); + assert_eq!( + unmatched.classified_items[0].abstention_reason, + Some(AbstentionReason::NoDeterministicRuleMatch) + ); + } + #[test] fn duplicate_candidates_are_reversible_and_normalized() { let report = classify_snapshot( @@ -601,7 +725,9 @@ mod tests { assert_eq!(normalize_title("---"), None); assert_eq!(normalize_doi("http://doi.org/A"), Some("a".into())); assert_eq!(normalize_doi("https://doi.org/B"), Some("b".into())); - assert_eq!(normalize_doi("C"), Some("c".into())); + assert_eq!(normalize_doi("http://dx.doi.org/C"), Some("c".into())); + assert_eq!(normalize_doi("https://dx.doi.org/D"), Some("d".into())); + assert_eq!(normalize_doi("E"), Some("e".into())); assert_eq!(normalize_title(" A---B "), Some("a b".into())); assert_eq!(normalize_title(" A--- "), Some("a".into())); let untitled = item("Z", "book", "", "", ""); @@ -622,10 +748,32 @@ mod tests { assert_eq!(report.classified_items.len(), 1); } + #[test] + fn snapshot_usage_is_bounded_before_accumulation() { + assert_eq!(checked_snapshot_usage(1, 1, 10, 20, 2).unwrap(), (2, 30)); + assert!(matches!( + checked_snapshot_usage(0, 1, 0, 1, MAX_SNAPSHOT_ITEMS + 1), + Err(ReadError::Budget("item-count")) + )); + assert!(matches!( + checked_snapshot_usage(MAX_SNAPSHOT_ITEMS, 1, 0, 1, MAX_SNAPSHOT_ITEMS), + Err(ReadError::Budget("item-count")) + )); + assert!(matches!( + checked_snapshot_usage(0, 1, MAX_SNAPSHOT_BYTES, 1, 1), + Err(ReadError::Budget("byte-count")) + )); + assert!(matches!( + checked_snapshot_usage(1, 1, 0, 1, 1), + Err(ReadError::SnapshotChanged) + )); + } + #[test] fn read_errors_are_actionable() { assert!(ReadError::Header("x").to_string().contains('x')); assert!(ReadError::SnapshotChanged.to_string().contains("changed")); + assert!(ReadError::Budget("items").to_string().contains("budget")); assert!(ReadError::Http("down".into()).to_string().contains("down")); assert!( ReadError::Body("large".into()) From 7af9cc80494135e088124fc7efc76506fd846950 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Fri, 4 Sep 2026 21:52:43 +0900 Subject: [PATCH 05/33] test(zotero): keep review contract warning-clean --- crates/conceptweave-zotero/tests/review_contract.rs | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/crates/conceptweave-zotero/tests/review_contract.rs b/crates/conceptweave-zotero/tests/review_contract.rs index 8a782eaa..8169ed15 100644 --- a/crates/conceptweave-zotero/tests/review_contract.rs +++ b/crates/conceptweave-zotero/tests/review_contract.rs @@ -1,6 +1,4 @@ -use conceptweave_zotero::{ - classify_snapshot, AbstentionReason, Disposition, ItemData, ItemTag, ZoteroItem, -}; +use conceptweave_zotero::{classify_snapshot, AbstentionReason, Disposition, ItemData, ZoteroItem}; fn item(key: &str, title: &str, doi: &str) -> ZoteroItem { ZoteroItem { From d0fa0e412503f2e06b12066f7f2bf23266acbf8b Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Fri, 4 Sep 2026 21:53:15 +0900 Subject: [PATCH 06/33] fix(zotero): confine local reports to temp output --- crates/conceptweave-zotero/src/main.rs | 92 ++++++++++++++++++++++++-- 1 file changed, 88 insertions(+), 4 deletions(-) diff --git a/crates/conceptweave-zotero/src/main.rs b/crates/conceptweave-zotero/src/main.rs index aaa954f3..f87d7502 100644 --- a/crates/conceptweave-zotero/src/main.rs +++ b/crates/conceptweave-zotero/src/main.rs @@ -3,19 +3,103 @@ use conceptweave_zotero::read_local_snapshot; use std::env; -use std::fs::File; -use std::io::BufWriter; +use std::fs::{self, OpenOptions}; +use std::io::{self, BufWriter}; +use std::path::PathBuf; + +fn validate_output_path(raw: &str) -> io::Result { + let path = PathBuf::from(raw); + if !path.is_absolute() { + return Err(io::Error::new( + io::ErrorKind::InvalidInput, + "report output must be an absolute path in the system temp directory", + )); + } + + let allowed_parent = env::temp_dir().canonicalize()?; + let parent = path.parent().ok_or_else(|| { + io::Error::new(io::ErrorKind::InvalidInput, "report output has no parent") + })?; + let resolved_parent = parent.canonicalize()?; + if resolved_parent != allowed_parent { + return Err(io::Error::new( + io::ErrorKind::PermissionDenied, + "report output must be a direct child of the system temp directory", + )); + } + if fs::symlink_metadata(&path).is_ok() { + return Err(io::Error::new( + io::ErrorKind::AlreadyExists, + "report output must not already exist or be a symlink", + )); + } + Ok(path) +} #[cfg_attr(coverage_nightly, coverage(off))] fn main() -> Result<(), Box> { let output = env::args() .nth(1) - .ok_or("usage: conceptweave-zotero OUTPUT.json")?; + .ok_or("usage: conceptweave-zotero /tmp/OUTPUT.json")?; + let output = validate_output_path(&output)?; let report = read_local_snapshot()?; if report.zotero_version.starts_with("9.") { eprintln!("Zotero 9 Local API is read-only; writing a local proposal report only"); } - let file = File::create(output)?; + let file = OpenOptions::new() + .write(true) + .create_new(true) + .open(output)?; serde_json::to_writer_pretty(BufWriter::new(file), &report)?; Ok(()) } + +#[cfg(test)] +mod tests { + use super::*; + + fn unique_temp_path(suffix: &str) -> PathBuf { + env::temp_dir().join(format!( + "conceptweave-zotero-{}-{suffix}.json", + std::process::id() + )) + } + + #[test] + fn output_path_must_be_a_new_direct_temp_child() { + let allowed = unique_temp_path("allowed"); + let _ = fs::remove_file(&allowed); + assert_eq!(validate_output_path(allowed.to_str().unwrap()).unwrap(), allowed); + + assert!(validate_output_path("relative.json").is_err()); + + let nested_dir = env::temp_dir().join(format!( + "conceptweave-zotero-{}-nested", + std::process::id() + )); + fs::create_dir_all(&nested_dir).unwrap(); + assert!(validate_output_path(nested_dir.join("report.json").to_str().unwrap()).is_err()); + fs::remove_dir_all(nested_dir).unwrap(); + + let existing = unique_temp_path("existing"); + fs::write(&existing, b"existing").unwrap(); + assert!(validate_output_path(existing.to_str().unwrap()).is_err()); + fs::remove_file(existing).unwrap(); + } + + #[cfg(unix)] + #[test] + fn output_path_rejects_symlinks_before_open() { + use std::os::unix::fs::symlink; + + let target = unique_temp_path("target"); + let link = unique_temp_path("link"); + let _ = fs::remove_file(&target); + let _ = fs::remove_file(&link); + fs::write(&target, b"target").unwrap(); + symlink(&target, &link).unwrap(); + assert!(validate_output_path(link.to_str().unwrap()).is_err()); + fs::remove_file(link).unwrap(); + fs::remove_file(target).unwrap(); + } +} From 0798c6fe4f14a87bd409393c6eebe8d0015cb0e2 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Fri, 4 Sep 2026 21:53:40 +0900 Subject: [PATCH 07/33] docs(zotero): record bounded read and report confinement --- docs/adr/0006-zotero-research-intake.md | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/docs/adr/0006-zotero-research-intake.md b/docs/adr/0006-zotero-research-intake.md index db415148..546e4ddd 100644 --- a/docs/adr/0006-zotero-research-intake.md +++ b/docs/adr/0006-zotero-research-intake.md @@ -5,23 +5,30 @@ ## Context -CWL needs a reproducible inventory of ontology research without turning keyword matches into authoritative library organization. The current desktop is Zotero 9.0.6, whose Local API supports reads but not writes. The library is mutable while pagination is in progress, and duplicate metadata does not prove that two records should be merged. +CWL needs a reproducible inventory of ontology research without turning keyword matches into authoritative library organization. The current desktop is Zotero 9.0.6, whose Local API supports reads but not writes. Zotero documents Local API writes only for Zotero 10+, where they require user-granted authorization and `Zotero-Server-ID`; this slice therefore has no mutation capability. The library is mutable while pagination is in progress, duplicate metadata does not prove that two records should be merged, and the local report contains titles and item keys that must not be written into the repository. + +Primary capability reference: Zotero, *Local API* (updated 2026-07-29), https://www.zotero.org/support/dev/web_api/v3/local_api. ## Decision -ConceptWeave owns a small read-only adapter that fetches every item under one unchanged Local API library version, links child records, emits exactly one deterministic proposed disposition per top-level bibliographic item, and abstains when evidence is weak. DOI/title matches remain reversible duplicate candidates. Reports stay local and are never committed. +ConceptWeave owns a small read-only adapter that fetches every item under one unchanged Local API library version, links child records, emits exactly one deterministic proposed disposition per top-level bibliographic item, and abstains when evidence is weak. Every abstention preserves a deterministic reason distinguishing missing classification metadata, vocabulary outside the current deterministic rules, and metadata that is present but unmatched. DOI/title matches remain reversible duplicate candidates, including legacy `dx.doi.org` resolver forms. + +The reader fails closed above 50,000 items or 256 MiB of cumulative response bodies, while retaining the 8 MiB per-page bound, finite request timeouts, redirect denial, total-count checks, snapshot-version checks, and duplicate-key detection. Report output is restricted to a new direct child of the operating system temporary directory. Relative paths, nested paths, existing files, and symlinks are rejected before write; the file is opened with create-new semantics so a path swap cannot cause repository or arbitrary-file overwrite. Reports stay local and are never committed. No dedicated utility repository or Zotero mutation path is created. A future Zotero 10+ write adapter is a separate decision and must use authenticated loopback access, server identity, optimistic version preconditions, reviewed item-level changes, before/after receipts, and rollback evidence. ## Consequences - A complete snapshot can be audited and replayed without changing the research library. -- Rule evidence and abstentions are visible; automated classification is not governance approval. +- Rule evidence and explicit abstention reasons are visible; automated classification is not governance approval. +- Whole-snapshot resource use is bounded independently from per-page limits. +- Sensitive local reports cannot be directed into the repository by the CLI. - Human review remains necessary for ambiguous records and every duplicate merge. - Zotero 9 cannot apply approved collection/tag changes automatically. ## Alternatives considered -- Direct Zotero 9 writes were rejected because the supported Local API is read-only. +- Direct Zotero 9 writes were rejected because the supported Local API write capability is Zotero 10+ only. - Cloud Web API mutation was rejected because it expands credential and network scope without being needed for classification. +- Arbitrary report output paths were rejected because local bibliographic titles and item keys are intentionally not repository artifacts. - A new repository was rejected because one bounded adapter does not yet justify another lifecycle and release surface. From e25c5d894c378cbc00bcc0a9467d1dc528e4485a Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Fri, 4 Sep 2026 21:54:01 +0900 Subject: [PATCH 08/33] docs(zotero): specify whole-snapshot resource budgets --- docs/TRD.md | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/docs/TRD.md b/docs/TRD.md index c9a41f1d..a34dc967 100644 --- a/docs/TRD.md +++ b/docs/TRD.md @@ -59,4 +59,8 @@ Evaluation must separate extraction recall, semantic correctness, structural cor ## 11. Zotero research intake -`conceptweave-zotero` reads only the loopback Local API with bounded pages, redirects disabled, and finite connect/response/body timeouts. `Total-Results`, `Last-Modified-Version`, Zotero version, and server identity must remain identical across all pages; drift, malformed JSON, an empty intermediate page, duplicate keys, or an oversized response fails the run. The report is local JSON and contains proposals rather than governance decisions. Zotero 9 writes are unsupported; no mutation path exists in this slice. A future Zotero 10+ writer requires a separate reviewed change with a Local API key, stable server identity, fresh item/library version preconditions, item-by-item before/after receipts, and rollback evidence. +`conceptweave-zotero` reads only the loopback Local API with at most 100 records per page, an 8 MiB page-body limit, a 50,000-item whole-snapshot limit, a 256 MiB cumulative body limit, redirects disabled, and finite connect/response/body/global timeouts. Before another request is issued, exhausted whole-snapshot budgets fail closed. Before a parsed page is accumulated, checked item-count and byte arithmetic must remain within both the advertised total and the configured whole-snapshot budgets. `Total-Results`, `Last-Modified-Version`, Zotero version, and server identity must remain identical across all pages; drift, malformed JSON, an empty intermediate page, duplicate keys, or an oversized response fails the run. + +Every top-level bibliographic record receives exactly one proposed disposition. `NeedsStewardReview` also records a deterministic abstention reason so missing classification metadata, vocabulary unsupported by the current deterministic rules, and present-but-unmatched metadata are distinguishable. DOI duplicate identity normalization treats bare DOI values, `doi:`, `doi.org`, and legacy `dx.doi.org` resolver forms as the same identity when their normalized DOI is equal. + +The report is local JSON and contains proposals rather than governance decisions. CLI output is restricted to a new direct child of the operating system temporary directory; relative paths, nested paths, existing paths, and symlinks are rejected, and create-new file semantics prevent overwrite/path-swap writes. Zotero 9 writes are unsupported; no mutation path exists in this slice. A future Zotero 10+ writer requires a separate reviewed change with a Local API key, stable server identity, fresh item/library version preconditions, item-by-item before/after receipts, and rollback evidence. From 150ed5d8a24abfa6857faf15b21f0ea41fa68efd Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Fri, 4 Sep 2026 21:58:51 +0900 Subject: [PATCH 09/33] test(zotero): pin ambiguity and replay evidence findings --- .../tests/review_contract_followup.rs | 62 +++++++++++++++++++ 1 file changed, 62 insertions(+) create mode 100644 crates/conceptweave-zotero/tests/review_contract_followup.rs diff --git a/crates/conceptweave-zotero/tests/review_contract_followup.rs b/crates/conceptweave-zotero/tests/review_contract_followup.rs new file mode 100644 index 00000000..04c16bbc --- /dev/null +++ b/crates/conceptweave-zotero/tests/review_contract_followup.rs @@ -0,0 +1,62 @@ +use conceptweave_zotero::{ + classify_snapshot, AbstentionReason, Disposition, ItemData, ZoteroItem, +}; + +fn item(key: &str, title: &str, abstract_note: &str) -> ZoteroItem { + ZoteroItem { + key: key.into(), + version: 11, + data: ItemData { + item_type: "journalArticle".into(), + title: title.into(), + abstract_note: abstract_note.into(), + doi: String::new(), + parent_item: String::new(), + collections: vec![], + tags: vec![], + }, + } +} + +#[test] +fn conflicting_specific_rule_families_abstain_for_steward_review() { + let report = classify_snapshot( + "9.0.6".into(), + None, + 42, + vec![item( + "A", + "Ontology matching and ontology learning", + "", + )], + ); + + let classified = &report.classified_items[0]; + assert_eq!(classified.proposed_disposition, Disposition::NeedsStewardReview); + assert_eq!( + classified.abstention_reason, + Some(AbstentionReason::ConflictingDispositionEvidence) + ); + assert_eq!( + classified.evidence.matched_phrases, + ["ontology learning", "ontology matching"] + ); +} + +#[test] +fn matched_abstract_value_is_preserved_for_replayable_review() { + let abstract_note = "We evaluate ontology alignment under schema drift."; + let report = classify_snapshot( + "9.0.6".into(), + None, + 42, + vec![item("A", "Uninformative title", abstract_note)], + ); + + let classified = &report.classified_items[0]; + assert_eq!(classified.proposed_disposition, Disposition::AlignmentVersioning); + assert_eq!( + classified.evidence.field_values.get("abstract_note").map(String::as_str), + Some(abstract_note) + ); +} From 376019144006d50a015f12b1b656e3464ab1a59e Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Fri, 4 Sep 2026 22:06:09 +0900 Subject: [PATCH 10/33] fix(zotero): make reader contract replayable and review-safe --- crates/conceptweave-zotero/src/lib.rs | 611 ++++++++++++++++++-------- 1 file changed, 431 insertions(+), 180 deletions(-) diff --git a/crates/conceptweave-zotero/src/lib.rs b/crates/conceptweave-zotero/src/lib.rs index 760eb443..dd12b114 100644 --- a/crates/conceptweave-zotero/src/lib.rs +++ b/crates/conceptweave-zotero/src/lib.rs @@ -9,16 +9,21 @@ use std::fmt; use std::time::Duration; /// Classification rule revision recorded in every report. -pub const RULE_REVISION: &str = "ontology-research-v1"; +pub const RULE_REVISION: &str = "ontology-research-v2"; +const SUPPORTED_API_VERSION: u64 = 3; +const SUPPORTED_API_VERSION_HEADER: &str = "3"; const PAGE_LIMIT: usize = 100; const MAX_PAGE_BYTES: u64 = 8 * 1024 * 1024; const MAX_SNAPSHOT_ITEMS: usize = 50_000; const MAX_SNAPSHOT_BYTES: u64 = 256 * 1024 * 1024; const LOCAL_API: &str = "http://127.0.0.1:23119/api/users/0/items"; +#[cfg(test)] +static TEST_LOCAL_API: std::sync::Mutex> = std::sync::Mutex::new(None); + /// A Zotero item returned by the Local API. -#[derive(Debug, Clone, Deserialize)] +#[derive(Debug, Clone, Deserialize, Serialize)] pub struct ZoteroItem { /// Stable item key. pub key: String, @@ -29,7 +34,7 @@ pub struct ZoteroItem { } /// Metadata used by the classifier. -#[derive(Debug, Clone, Deserialize)] +#[derive(Debug, Clone, Deserialize, Serialize)] #[serde(rename_all = "camelCase")] pub struct ItemData { /// Zotero item type. @@ -55,7 +60,7 @@ pub struct ItemData { } /// A Zotero item tag. -#[derive(Debug, Clone, Deserialize)] +#[derive(Debug, Clone, Deserialize, Serialize)] pub struct ItemTag { /// Tag text. pub tag: String, @@ -91,6 +96,8 @@ pub enum AbstentionReason { UnsupportedRuleVocabulary, /// Metadata is present but no deterministic rule phrase matches it. NoDeterministicRuleMatch, + /// More than one specific disposition family is supported by the item. + ConflictingDispositionEvidence, } /// Evidence for a deterministic proposed disposition. @@ -98,6 +105,8 @@ pub enum AbstentionReason { pub struct ClassificationEvidence { /// Metadata fields whose values matched. pub fields: Vec<&'static str>, + /// Exact snapshot values for matched fields, retained only in the local report. + pub field_values: BTreeMap<&'static str, String>, /// Rule phrases found in those fields. pub matched_phrases: Vec<&'static str>, } @@ -145,6 +154,10 @@ pub struct DuplicateCandidate { pub struct ClassificationReport { /// Zotero desktop version that served the snapshot. pub zotero_version: String, + /// Requested and observed Local API version for a live read. + pub api_version: Option, + /// Zotero schema revision observed consistently across a live read. + pub schema_version: Option, /// Local API server identifier observed on every page when supplied. pub server_id: Option, /// Library version shared by every fetched page. @@ -166,6 +179,8 @@ pub enum ReadError { Http(String), /// Required response header is absent or invalid. Header(&'static str), + /// Provider contract is present but unsupported. + Contract(&'static str), /// A later page did not belong to the first page's snapshot. SnapshotChanged, /// Configured whole-snapshot resource budget was exceeded. @@ -181,6 +196,7 @@ impl fmt::Display for ReadError { match self { Self::Http(error) => write!(formatter, "local API request failed: {error}"), Self::Header(name) => write!(formatter, "local API response lacks valid {name}"), + Self::Contract(name) => write!(formatter, "local API returned unsupported {name}"), Self::SnapshotChanged => write!(formatter, "Zotero library changed during the read"), Self::Budget(kind) => write!(formatter, "Zotero snapshot exceeds {kind} budget"), Self::Json(error) => write!(formatter, "local API returned invalid JSON: {error}"), @@ -191,8 +207,23 @@ impl fmt::Display for ReadError { impl std::error::Error for ReadError {} +#[derive(Debug)] +struct FetchedPage { + total: usize, + library_version: u64, + zotero_version: String, + api_version: u64, + schema_version: u64, + server_id: Option, + body_bytes: u64, + items: Vec, +} + /// Reads every Zotero item from one stable Local API library version. -#[cfg_attr(coverage_nightly, coverage(off))] +/// +/// Snapshot consistency, resource budgets, API-version validation, pagination, +/// and duplicate-key checks live in an injectable reader core. Only the narrow +/// ureq transport shim is excluded from deterministic coverage. pub fn read_local_snapshot() -> Result { let config = ureq::Agent::config_builder() .timeout_global(Some(Duration::from_secs(60))) @@ -202,76 +233,139 @@ pub fn read_local_snapshot() -> Result { .max_redirects(0) .build(); let agent = ureq::Agent::new_with_config(config); + read_snapshot_with(|start| fetch_local_page(&agent, start)) +} + +fn local_api_base() -> String { + #[cfg(test)] + { + if let Some(value) = TEST_LOCAL_API + .lock() + .expect("test Local API lock must not be poisoned") + .clone() + { + return value; + } + } + LOCAL_API.to_owned() +} + +#[cfg_attr(coverage_nightly, coverage(off))] +fn fetch_local_page(agent: &ureq::Agent, start: usize) -> Result { + let url = format!( + "{}?format=json&include=data&limit={PAGE_LIMIT}&start={start}", + local_api_base() + ); + let mut response = agent + .get(&url) + .header("Zotero-API-Version", SUPPORTED_API_VERSION_HEADER) + .call() + .map_err(|error| ReadError::Http(error.to_string()))?; + let headers = response.headers(); + let total = header_u64(headers, "Total-Results")?; + let total = usize::try_from(total).map_err(|_| ReadError::Budget("item-count"))?; + let library_version = header_u64(headers, "Last-Modified-Version")?; + let zotero_version = header_string(headers, "X-Zotero-Version")?; + let api_version = header_u64(headers, "Zotero-API-Version")?; + let schema_version = header_u64(headers, "Zotero-Schema-Version")?; + let server_id = optional_header(headers, "Zotero-Server-ID"); + + let body = response + .body_mut() + .with_config() + .limit(MAX_PAGE_BYTES) + .read_to_string() + .map_err(|error| ReadError::Body(error.to_string()))?; + let body_bytes = u64::try_from(body.len()).map_err(|_| ReadError::Budget("byte-count"))?; + let items = serde_json::from_str(&body).map_err(ReadError::Json)?; + + Ok(FetchedPage { + total, + library_version, + zotero_version, + api_version, + schema_version, + server_id, + body_bytes, + items, + }) +} + +#[cfg_attr(coverage_nightly, coverage(off))] +fn header_u64(headers: &ureq::http::HeaderMap, name: &'static str) -> Result { + header_string(headers, name)? + .parse() + .map_err(|_| ReadError::Header(name)) +} + +#[cfg_attr(coverage_nightly, coverage(off))] +fn header_string(headers: &ureq::http::HeaderMap, name: &'static str) -> Result { + optional_header(headers, name).ok_or(ReadError::Header(name)) +} + +#[cfg_attr(coverage_nightly, coverage(off))] +fn optional_header(headers: &ureq::http::HeaderMap, name: &'static str) -> Option { + headers.get(name)?.to_str().ok().map(str::to_owned) +} + +fn read_snapshot_with( + mut fetch_page: impl FnMut(usize) -> Result, +) -> Result { let mut items = Vec::new(); let mut snapshot_bytes = 0_u64; - let mut expected = None; + let mut expected_total = None; let mut library_version = None; let mut zotero_version = None; + let mut schema_version = None; let mut server_id = None; - let mut metadata_initialized = false; loop { - if expected.is_some_and(|total| items.len() < total) + if expected_total.is_some_and(|total| items.len() < total) && (items.len() >= MAX_SNAPSHOT_ITEMS || snapshot_bytes >= MAX_SNAPSHOT_BYTES) { return Err(ReadError::Budget("whole-snapshot")); } - let url = format!( - "{LOCAL_API}?format=json&include=data&limit={PAGE_LIMIT}&start={}", - items.len() - ); - let mut response = agent - .get(&url) - .call() - .map_err(|error| ReadError::Http(error.to_string()))?; - let headers = response.headers(); - let page_total = header_u64(headers, "Total-Results")? as usize; - if page_total > MAX_SNAPSHOT_ITEMS { + + let page = fetch_page(items.len())?; + if page.api_version != SUPPORTED_API_VERSION { + return Err(ReadError::Contract("Zotero-API-Version")); + } + if page.total > MAX_SNAPSHOT_ITEMS { return Err(ReadError::Budget("item-count")); } - let page_version = header_u64(headers, "Last-Modified-Version")?; - let page_zotero = header_string(headers, "X-Zotero-Version")?; - let page_server = optional_header(headers, "Zotero-Server-ID"); - - if metadata_initialized { - if expected != Some(page_total) - || library_version != Some(page_version) - || zotero_version.as_ref() != Some(&page_zotero) - || server_id != page_server + + if let Some(expected) = expected_total { + if expected != page.total + || library_version != Some(page.library_version) + || zotero_version.as_ref() != Some(&page.zotero_version) + || schema_version != Some(page.schema_version) + || server_id != page.server_id { return Err(ReadError::SnapshotChanged); } } else { - expected = Some(page_total); - library_version = Some(page_version); - zotero_version = Some(page_zotero); - server_id = page_server; - metadata_initialized = true; + expected_total = Some(page.total); + library_version = Some(page.library_version); + zotero_version = Some(page.zotero_version.clone()); + schema_version = Some(page.schema_version); + server_id = page.server_id.clone(); } - let body = response - .body_mut() - .with_config() - .limit(MAX_PAGE_BYTES) - .read_to_string() - .map_err(|error| ReadError::Body(error.to_string()))?; - let body_bytes = - u64::try_from(body.len()).map_err(|_| ReadError::Budget("byte-count"))?; - let page: Vec = serde_json::from_str(&body).map_err(ReadError::Json)?; - if page.is_empty() && items.len() < page_total { + if page.items.is_empty() && items.len() < page.total { return Err(ReadError::SnapshotChanged); } let (next_item_count, next_snapshot_bytes) = checked_snapshot_usage( items.len(), - page.len(), + page.items.len(), snapshot_bytes, - body_bytes, - page_total, + page.body_bytes, + page.total, )?; - items.extend(page); + items.extend(page.items); snapshot_bytes = next_snapshot_bytes; debug_assert_eq!(items.len(), next_item_count); - if items.len() == page_total { + + if items.len() == page.total { break; } } @@ -286,12 +380,15 @@ pub fn read_local_snapshot() -> Result { return Err(ReadError::SnapshotChanged); } - Ok(classify_snapshot( + let mut report = classify_snapshot( zotero_version.ok_or(ReadError::Header("X-Zotero-Version"))?, server_id, library_version.ok_or(ReadError::Header("Last-Modified-Version"))?, items, - )) + ); + report.api_version = Some(SUPPORTED_API_VERSION); + report.schema_version = schema_version; + Ok(report) } fn checked_snapshot_usage( @@ -322,23 +419,6 @@ fn checked_snapshot_usage( Ok((next_items, next_bytes)) } -#[cfg_attr(coverage_nightly, coverage(off))] -fn header_u64(headers: &ureq::http::HeaderMap, name: &'static str) -> Result { - header_string(headers, name)? - .parse() - .map_err(|_| ReadError::Header(name)) -} - -#[cfg_attr(coverage_nightly, coverage(off))] -fn header_string(headers: &ureq::http::HeaderMap, name: &'static str) -> Result { - optional_header(headers, name).ok_or(ReadError::Header(name)) -} - -#[cfg_attr(coverage_nightly, coverage(off))] -fn optional_header(headers: &ureq::http::HeaderMap, name: &'static str) -> Option { - headers.get(name)?.to_str().ok().map(str::to_owned) -} - /// Classifies an already captured snapshot without network access. pub fn classify_snapshot( zotero_version: String, @@ -358,6 +438,8 @@ pub fn classify_snapshot( ClassificationReport { zotero_version, + api_version: None, + schema_version: None, server_id, library_version, rule_revision: RULE_REVISION, @@ -390,21 +472,27 @@ fn child_index(items: &[ZoteroItem]) -> BTreeMap> { } fn classify_item(item: &ZoteroItem, child_item_keys: Vec) -> ClassifiedItem { - let title = item.data.title.to_lowercase(); - let abstract_note = item.data.abstract_note.to_lowercase(); - let tags = item + let title_normalized = item.data.title.to_lowercase(); + let abstract_normalized = item.data.abstract_note.to_lowercase(); + let tags_original = item .data .tags .iter() - .map(|tag| tag.tag.to_lowercase()) + .map(|tag| tag.tag.as_str()) .collect::>() .join(" "); + let tags_normalized = tags_original.to_lowercase(); let fields = [ - ("title", title.as_str()), - ("abstract_note", abstract_note.as_str()), - ("tags", tags.as_str()), + ("title", title_normalized.as_str(), item.data.title.as_str()), + ( + "abstract_note", + abstract_normalized.as_str(), + item.data.abstract_note.as_str(), + ), + ("tags", tags_normalized.as_str(), tags_original.as_str()), ]; - let rules = [ + + let specific_rules = [ ( Disposition::AlignmentVersioning, &[ @@ -446,37 +534,69 @@ fn classify_item(item: &ZoteroItem, child_item_keys: Vec) -> ClassifiedI "shacl", ][..], ), - ( - Disposition::AdjacentEvidence, - &[ - "ontology", - "semantic web", - "knowledge graph", - "rdf", - "owl", - "skos", - ][..], - ), ]; - let mut disposition = Disposition::NeedsStewardReview; + let adjacent_phrases = [ + "ontology", + "semantic web", + "knowledge graph", + "rdf", + "owl", + "skos", + ]; + + let mut matched_dispositions = Vec::new(); let mut matched_fields = BTreeSet::new(); + let mut field_values = BTreeMap::new(); let mut matched_phrases = BTreeSet::new(); - for (candidate, phrases) in rules { - for (field, value) in fields { + + for (candidate, phrases) in specific_rules { + let mut family_matched = false; + for (field, normalized, original) in fields { for phrase in phrases { - if contains_phrase(value, phrase) { - disposition = candidate; + if contains_phrase(normalized, phrase) { + family_matched = true; matched_fields.insert(field); + field_values + .entry(field) + .or_insert_with(|| original.to_owned()); matched_phrases.insert(*phrase); } } } - if disposition != Disposition::NeedsStewardReview { - break; + if family_matched { + matched_dispositions.push(candidate); } } - let abstention_reason = (disposition == Disposition::NeedsStewardReview) - .then(|| classify_abstention_reason(&fields)); + + let (proposed_disposition, abstention_reason) = match matched_dispositions.as_slice() { + [] => { + for (field, normalized, original) in fields { + for phrase in adjacent_phrases { + if contains_phrase(normalized, phrase) { + matched_fields.insert(field); + field_values + .entry(field) + .or_insert_with(|| original.to_owned()); + matched_phrases.insert(phrase); + } + } + } + if matched_phrases.is_empty() { + ( + Disposition::NeedsStewardReview, + Some(classify_abstention_reason(&fields)), + ) + } else { + (Disposition::AdjacentEvidence, None) + } + } + [single] => (*single, None), + _ => ( + Disposition::NeedsStewardReview, + Some(AbstentionReason::ConflictingDispositionEvidence), + ), + }; + ClassifiedItem { item_key: item.key.clone(), item_version: item.version, @@ -484,10 +604,11 @@ fn classify_item(item: &ZoteroItem, child_item_keys: Vec) -> ClassifiedI title: item.data.title.clone(), collection_keys: item.data.collections.clone(), tags: item.data.tags.iter().map(|tag| tag.tag.clone()).collect(), - proposed_disposition: disposition, + proposed_disposition, abstention_reason, evidence: ClassificationEvidence { fields: matched_fields.into_iter().collect(), + field_values, matched_phrases: matched_phrases.into_iter().collect(), }, child_item_keys, @@ -495,12 +616,15 @@ fn classify_item(item: &ZoteroItem, child_item_keys: Vec) -> ClassifiedI } } -fn classify_abstention_reason(fields: &[(&'static str, &str)]) -> AbstentionReason { - if fields.iter().all(|(_, value)| value.trim().is_empty()) { +fn classify_abstention_reason(fields: &[(&'static str, &str, &str)]) -> AbstentionReason { + if fields + .iter() + .all(|(_, normalized, _)| normalized.trim().is_empty()) + { return AbstentionReason::MissingClassificationMetadata; } - if fields.iter().any(|(_, value)| { - value + if fields.iter().any(|(_, _, original)| { + original .chars() .any(|character| character.is_alphabetic() && !character.is_ascii()) }) { @@ -580,6 +704,11 @@ fn normalize_title(value: &str) -> Option { #[cfg(test)] mod tests { use super::*; + use std::io::{Read, Write}; + use std::net::TcpListener; + use std::thread; + + static LOCAL_API_TEST_LOCK: std::sync::Mutex<()> = std::sync::Mutex::new(()); fn item(key: &str, item_type: &str, title: &str, doi: &str, parent: &str) -> ZoteroItem { ZoteroItem { @@ -597,6 +726,152 @@ mod tests { } } + fn fetched_page(total: usize, items: Vec) -> FetchedPage { + FetchedPage { + total, + library_version: 42, + zotero_version: "9.0.6".into(), + api_version: 3, + schema_version: 42, + server_id: Some("server".into()), + body_bytes: 100, + items, + } + } + + #[test] + fn production_wrapper_requests_api_v3_and_records_contract_versions() { + let _guard = LOCAL_API_TEST_LOCK.lock().unwrap(); + let listener = TcpListener::bind("127.0.0.1:0").unwrap(); + let address = listener.local_addr().unwrap(); + let base = format!("http://{address}/api/users/0/items"); + *TEST_LOCAL_API.lock().unwrap() = Some(base); + + let server = thread::spawn(move || { + let (mut stream, _) = listener.accept().unwrap(); + let mut request = vec![0_u8; 4096]; + let read = stream.read(&mut request).unwrap(); + let request = String::from_utf8_lossy(&request[..read]).to_lowercase(); + assert!(request.contains("zotero-api-version: 3")); + let body = r#"[{"key":"A","version":1,"data":{"itemType":"book","title":"ontology evaluation"}}]"#; + write!( + stream, + "HTTP/1.1 200 OK\r\nContent-Length: {}\r\nTotal-Results: 1\r\nLast-Modified-Version: 42\r\nX-Zotero-Version: 9.0.6\r\nZotero-API-Version: 3\r\nZotero-Schema-Version: 42\r\nZotero-Server-ID: server\r\nConnection: close\r\n\r\n{}", + body.len(), + body + ) + .unwrap(); + stream.flush().unwrap(); + }); + + let report = read_local_snapshot().unwrap(); + *TEST_LOCAL_API.lock().unwrap() = None; + server.join().unwrap(); + + assert_eq!(report.api_version, Some(3)); + assert_eq!(report.schema_version, Some(42)); + assert_eq!(report.library_version, 42); + assert_eq!(report.classified_items.len(), 1); + assert_eq!(local_api_base(), LOCAL_API); + } + + #[test] + fn reader_core_paginates_and_preserves_one_snapshot_contract() { + let mut pages = vec![ + fetched_page(2, vec![item("A", "book", "ontology quality", "", "")]), + fetched_page(2, vec![item("B", "book", "semantic web", "", "")]), + ] + .into_iter(); + let report = read_snapshot_with(|_| Ok(pages.next().unwrap())).unwrap(); + assert_eq!(report.observed_item_count, 2); + assert_eq!(report.api_version, Some(3)); + assert_eq!(report.schema_version, Some(42)); + } + + #[test] + fn reader_core_rejects_contract_drift_empty_pages_and_duplicate_keys() { + let mut unsupported = fetched_page(1, vec![item("A", "book", "x", "", "")]); + unsupported.api_version = 4; + assert!(matches!( + read_snapshot_with(|_| Ok(unsupported)), + Err(ReadError::Contract("Zotero-API-Version")) + )); + + let mut pages = vec![ + fetched_page(2, vec![item("A", "book", "x", "", "")]), + { + let mut page = fetched_page(2, vec![item("B", "book", "y", "", "")]); + page.schema_version = 43; + page + }, + ] + .into_iter(); + assert!(matches!( + read_snapshot_with(|_| Ok(pages.next().unwrap())), + Err(ReadError::SnapshotChanged) + )); + + assert!(matches!( + read_snapshot_with(|_| Ok(fetched_page(1, vec![]))), + Err(ReadError::SnapshotChanged) + )); + + assert!(matches!( + read_snapshot_with(|_| { + Ok(fetched_page( + 2, + vec![ + item("A", "book", "x", "", ""), + item("A", "book", "y", "", ""), + ], + )) + }), + Err(ReadError::SnapshotChanged) + )); + } + + #[test] + fn reader_core_rejects_total_and_between_request_resource_exhaustion() { + let too_many = fetched_page(MAX_SNAPSHOT_ITEMS + 1, vec![]); + assert!(matches!( + read_snapshot_with(|_| Ok(too_many)), + Err(ReadError::Budget("item-count")) + )); + + let mut calls = 0; + assert!(matches!( + read_snapshot_with(|_| { + calls += 1; + let mut page = fetched_page(2, vec![item("A", "book", "x", "", "")]); + page.body_bytes = MAX_SNAPSHOT_BYTES; + Ok(page) + }), + Err(ReadError::Budget("whole-snapshot")) + )); + assert_eq!(calls, 1); + } + + #[test] + fn snapshot_usage_is_bounded_before_accumulation() { + assert_eq!(checked_snapshot_usage(1, 1, 10, 20, 2).unwrap(), (2, 30)); + assert!(matches!( + checked_snapshot_usage(0, 1, 0, 1, MAX_SNAPSHOT_ITEMS + 1), + Err(ReadError::Budget("item-count")) + )); + assert!(matches!( + checked_snapshot_usage(MAX_SNAPSHOT_ITEMS, 1, 0, 1, MAX_SNAPSHOT_ITEMS), + Err(ReadError::Budget("item-count")) + )); + assert!(matches!( + checked_snapshot_usage(0, 1, MAX_SNAPSHOT_BYTES, 1, 1), + Err(ReadError::Budget("byte-count")) + )); + assert!(matches!( + checked_snapshot_usage(1, 1, 0, 1, 1), + Err(ReadError::SnapshotChanged) + )); + } + #[test] fn classifies_every_bibliographic_item_and_links_children() { let mut generation = item("B", "journalArticle", "Ontology Learning", "10.1/X", ""); @@ -615,10 +890,6 @@ mod tests { ); assert_eq!(report.observed_item_count, 3); assert_eq!(report.classified_items.len(), 2); - assert_eq!( - report.classified_items[0].proposed_disposition, - Disposition::NeedsStewardReview - ); assert_eq!( report.classified_items[0].abstention_reason, Some(AbstentionReason::NoDeterministicRuleMatch) @@ -633,12 +904,8 @@ mod tests { } #[test] - fn priority_and_all_rule_families_are_deterministic() { + fn specific_rule_families_and_conflicts_are_deterministic() { let cases = [ - ( - "ontology matching and ontology learning", - Disposition::AlignmentVersioning, - ), ("taxonomy induction", Disposition::Generation), ( "ontology-based data access", @@ -656,15 +923,43 @@ mod tests { ); assert_eq!(report.classified_items[0].proposed_disposition, expected); } - assert!(!contains_phrase("knowledge", "owl")); - assert!(!contains_phrase("growl", "owl")); - assert!(contains_phrase("owl-based", "owl")); - assert!(contains_phrase("uses owl", "owl")); - assert!(contains_phrase("owl", "owl")); + + let conflict = classify_snapshot( + "10".into(), + None, + 1, + vec![item( + "A", + "book", + "ontology matching and ontology learning", + "", + "", + )], + ); + assert_eq!( + conflict.classified_items[0].proposed_disposition, + Disposition::NeedsStewardReview + ); + assert_eq!( + conflict.classified_items[0].abstention_reason, + Some(AbstentionReason::ConflictingDispositionEvidence) + ); } #[test] - fn abstention_reasons_distinguish_missing_unsupported_and_unmatched_metadata() { + fn matched_values_and_abstention_reasons_are_replayable() { + let mut abstract_match = item("A", "book", "Uninformative", "", ""); + abstract_match.data.abstract_note = "Evidence for ontology alignment".into(); + let report = classify_snapshot("10".into(), None, 1, vec![abstract_match]); + assert_eq!( + report.classified_items[0] + .evidence + .field_values + .get("abstract_note") + .map(String::as_str), + Some("Evidence for ontology alignment") + ); + let missing = classify_snapshot( "10".into(), None, @@ -686,41 +981,16 @@ mod tests { unsupported.classified_items[0].abstention_reason, Some(AbstentionReason::UnsupportedRuleVocabulary) ); - - let unmatched = classify_snapshot( - "10".into(), - None, - 1, - vec![item("A", "book", "Other evidence", "", "")], - ); - assert_eq!( - unmatched.classified_items[0].abstention_reason, - Some(AbstentionReason::NoDeterministicRuleMatch) - ); } #[test] - fn duplicate_candidates_are_reversible_and_normalized() { - let report = classify_snapshot( - "10".into(), - None, - 1, - vec![ - item("A", "book", "OWL: Overview", "doi:10.1/X", ""), - item("B", "book", "owl overview", "https://doi.org/10.1/x", ""), - ], - ); - assert_eq!(report.duplicate_candidates.len(), 2); - assert!( - report - .duplicate_candidates - .iter() - .all(|candidate| candidate.item_keys == ["A", "B"]) - ); - } + fn phrase_boundaries_and_duplicate_normalization_are_exact() { + assert!(!contains_phrase("knowledge", "owl")); + assert!(!contains_phrase("growl", "owl")); + assert!(contains_phrase("owl-based", "owl")); + assert!(contains_phrase("uses owl", "owl")); + assert!(contains_phrase("owl", "owl")); - #[test] - fn empty_identities_do_not_form_duplicate_groups() { assert_eq!(normalize_doi(" "), None); assert_eq!(normalize_title("---"), None); assert_eq!(normalize_doi("http://doi.org/A"), Some("a".into())); @@ -730,56 +1000,37 @@ mod tests { assert_eq!(normalize_doi("E"), Some("e".into())); assert_eq!(normalize_title(" A---B "), Some("a b".into())); assert_eq!(normalize_title(" A--- "), Some("a".into())); - let untitled = item("Z", "book", "", "", ""); - assert!(duplicate_candidates(&[&untitled]).is_empty()); let report = classify_snapshot( "10".into(), None, 1, vec![ - item("A", "book", "Only", "", ""), - item("B", "note", "Ignored", "", ""), - item("C", "annotation", "Ignored", "", ""), - item("D", "book", "Child", "", "A"), + item("A", "book", "OWL: Overview", "doi:10.1/X", ""), + item( + "B", + "book", + "owl overview", + "https://dx.doi.org/10.1/x", + "", + ), ], ); - assert!(report.duplicate_candidates.is_empty()); - assert_eq!(report.classified_items.len(), 1); - } - - #[test] - fn snapshot_usage_is_bounded_before_accumulation() { - assert_eq!(checked_snapshot_usage(1, 1, 10, 20, 2).unwrap(), (2, 30)); - assert!(matches!( - checked_snapshot_usage(0, 1, 0, 1, MAX_SNAPSHOT_ITEMS + 1), - Err(ReadError::Budget("item-count")) - )); - assert!(matches!( - checked_snapshot_usage(MAX_SNAPSHOT_ITEMS, 1, 0, 1, MAX_SNAPSHOT_ITEMS), - Err(ReadError::Budget("item-count")) - )); - assert!(matches!( - checked_snapshot_usage(0, 1, MAX_SNAPSHOT_BYTES, 1, 1), - Err(ReadError::Budget("byte-count")) - )); - assert!(matches!( - checked_snapshot_usage(1, 1, 0, 1, 1), - Err(ReadError::SnapshotChanged) - )); + assert_eq!(report.duplicate_candidates.len(), 2); + assert!(report + .duplicate_candidates + .iter() + .all(|candidate| candidate.item_keys == ["A", "B"])); } #[test] fn read_errors_are_actionable() { assert!(ReadError::Header("x").to_string().contains('x')); + assert!(ReadError::Contract("v").to_string().contains("unsupported")); assert!(ReadError::SnapshotChanged.to_string().contains("changed")); assert!(ReadError::Budget("items").to_string().contains("budget")); assert!(ReadError::Http("down".into()).to_string().contains("down")); - assert!( - ReadError::Body("large".into()) - .to_string() - .contains("large") - ); + assert!(ReadError::Body("large".into()).to_string().contains("large")); let json_error = serde_json::from_str::("{}").unwrap_err(); assert!(ReadError::Json(json_error).to_string().contains("JSON")); } From 69d03ab4dad95654ba9319f47202b0d3687b590b Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Fri, 4 Sep 2026 22:06:28 +0900 Subject: [PATCH 11/33] fix(zotero): surface final report flush failures --- crates/conceptweave-zotero/src/main.rs | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/crates/conceptweave-zotero/src/main.rs b/crates/conceptweave-zotero/src/main.rs index f87d7502..fdb2d572 100644 --- a/crates/conceptweave-zotero/src/main.rs +++ b/crates/conceptweave-zotero/src/main.rs @@ -4,7 +4,7 @@ use conceptweave_zotero::read_local_snapshot; use std::env; use std::fs::{self, OpenOptions}; -use std::io::{self, BufWriter}; +use std::io::{self, BufWriter, Write}; use std::path::PathBuf; fn validate_output_path(raw: &str) -> io::Result { @@ -50,7 +50,9 @@ fn main() -> Result<(), Box> { .write(true) .create_new(true) .open(output)?; - serde_json::to_writer_pretty(BufWriter::new(file), &report)?; + let mut writer = BufWriter::new(file); + serde_json::to_writer_pretty(&mut writer, &report)?; + writer.flush()?; Ok(()) } From 2806afe7329d36cecc36a9c8841c5f867a051dee Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Fri, 4 Sep 2026 22:06:38 +0900 Subject: [PATCH 12/33] docs(ddd): map Zotero research intake ACL --- docs/CONTEXT_MAP.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/CONTEXT_MAP.md b/docs/CONTEXT_MAP.md index 5ea47792..bc769582 100644 --- a/docs/CONTEXT_MAP.md +++ b/docs/CONTEXT_MAP.md @@ -9,6 +9,7 @@ ## External relationships +- Zotero Local API -> research evidence intake: **Anti-Corruption Layer into Semantic Discovery**. Zotero remains the bibliographic system of record; ConceptWeave consumes a version-pinned, read-only Local API snapshot and emits proposal evidence only. Item metadata, attachments, collection/tag truth, and future write authority remain in Zotero. No Zotero record becomes semantic authority without ConceptWeave validation/review/publication. - contextual-orchestrator -> Semantic Discovery: **Anti-Corruption Layer**. Model/provider envelopes never enter the domain model directly. - LineageWeave -> Source Observation: **Anti-Corruption Layer**. Inferred/proposed lineage remains explicitly non-authoritative until ConceptWeave governance evaluates it. - context-graph-contracts <-> Interoperability: **Shared Kernel only for versioned public contracts**, kept minimal. From 11f72ac7305c8d8012be54074bea23659277908c Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Fri, 4 Sep 2026 22:09:48 +0900 Subject: [PATCH 13/33] fix(zotero): repair reader contract test fixture --- crates/conceptweave-zotero/src/lib.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/crates/conceptweave-zotero/src/lib.rs b/crates/conceptweave-zotero/src/lib.rs index dd12b114..fa82f4d2 100644 --- a/crates/conceptweave-zotero/src/lib.rs +++ b/crates/conceptweave-zotero/src/lib.rs @@ -207,7 +207,7 @@ impl fmt::Display for ReadError { impl std::error::Error for ReadError {} -#[derive(Debug)] +#[derive(Debug, Clone)] struct FetchedPage { total: usize, library_version: u64, @@ -793,7 +793,7 @@ mod tests { let mut unsupported = fetched_page(1, vec![item("A", "book", "x", "", "")]); unsupported.api_version = 4; assert!(matches!( - read_snapshot_with(|_| Ok(unsupported)), + read_snapshot_with(|_| Ok(unsupported.clone())), Err(ReadError::Contract("Zotero-API-Version")) )); @@ -834,7 +834,7 @@ mod tests { fn reader_core_rejects_total_and_between_request_resource_exhaustion() { let too_many = fetched_page(MAX_SNAPSHOT_ITEMS + 1, vec![]); assert!(matches!( - read_snapshot_with(|_| Ok(too_many)), + read_snapshot_with(|_| Ok(too_many.clone())), Err(ReadError::Budget("item-count")) )); From eccea5b709b60804daf06f1836470f8af4fa053e Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Fri, 4 Sep 2026 22:10:17 +0900 Subject: [PATCH 14/33] docs(zotero): pin Local API v3 and replay evidence --- docs/adr/0006-zotero-research-intake.md | 20 ++++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/docs/adr/0006-zotero-research-intake.md b/docs/adr/0006-zotero-research-intake.md index 546e4ddd..b7d3a16d 100644 --- a/docs/adr/0006-zotero-research-intake.md +++ b/docs/adr/0006-zotero-research-intake.md @@ -7,13 +7,21 @@ CWL needs a reproducible inventory of ontology research without turning keyword matches into authoritative library organization. The current desktop is Zotero 9.0.6, whose Local API supports reads but not writes. Zotero documents Local API writes only for Zotero 10+, where they require user-granted authorization and `Zotero-Server-ID`; this slice therefore has no mutation capability. The library is mutable while pagination is in progress, duplicate metadata does not prove that two records should be merged, and the local report contains titles and item keys that must not be written into the repository. -Primary capability reference: Zotero, *Local API* (updated 2026-07-29), https://www.zotero.org/support/dev/web_api/v3/local_api. +Zotero's Local API documentation states that production clients should request `Zotero-API-Version: 3`; the response exposes `Zotero-API-Version` and `Zotero-Schema-Version`. The API version is the compatibility contract. The schema version is therefore recorded and required to remain stable across the snapshot, but it is not hard-coded to the developer workstation's current schema 42 because Zotero can legitimately revise the local data schema while retaining API v3 compatibility. + +Primary capability references: Zotero, *Local API* (updated 2026-07-29), https://www.zotero.org/support/dev/web_api/v3/local_api; Zotero, *Basics*, https://www.zotero.org/support/dev/web_api/v3/basics. ## Decision -ConceptWeave owns a small read-only adapter that fetches every item under one unchanged Local API library version, links child records, emits exactly one deterministic proposed disposition per top-level bibliographic item, and abstains when evidence is weak. Every abstention preserves a deterministic reason distinguishing missing classification metadata, vocabulary outside the current deterministic rules, and metadata that is present but unmatched. DOI/title matches remain reversible duplicate candidates, including legacy `dx.doi.org` resolver forms. +ConceptWeave owns a small read-only Anti-Corruption Layer from Zotero into research evidence intake. Every request explicitly sends `Zotero-API-Version: 3`; a response reporting another API version fails closed. `Total-Results`, library version, Zotero version, schema version, and optional server identity must remain unchanged across pagination. API and schema versions are retained in the live report provenance. + +The adapter links child records, emits exactly one deterministic proposed disposition per top-level bibliographic item, and abstains when evidence is weak or ambiguous. Every abstention preserves a deterministic reason distinguishing missing classification metadata, vocabulary outside the current deterministic rules, present-but-unmatched metadata, and conflicting specific disposition families. Specific rule families are evaluated together rather than by first-match priority. When evidence matches multiple families, the proposal becomes `NeedsStewardReview` and all matching evidence is retained. + +Matched metadata values are copied into the local-only evidence receipt for replay. This is necessary for abstract-only matches because a later Zotero revision cannot reconstruct the exact text used for an earlier proposal from item key/version alone. DOI/title matches remain reversible duplicate candidates, including legacy `dx.doi.org` resolver forms. + +The reader fails closed above 50,000 items or 256 MiB of cumulative response bodies, while retaining the 8 MiB per-page bound, finite request timeouts, redirect denial, total-count checks, snapshot-version checks, and duplicate-key detection. Pagination, consistency, resource-budget, and provider-contract behavior are separated from the narrow `ureq` transport so deterministic tests exercise the production reader core rather than excluding the entire reader from coverage. -The reader fails closed above 50,000 items or 256 MiB of cumulative response bodies, while retaining the 8 MiB per-page bound, finite request timeouts, redirect denial, total-count checks, snapshot-version checks, and duplicate-key detection. Report output is restricted to a new direct child of the operating system temporary directory. Relative paths, nested paths, existing files, and symlinks are rejected before write; the file is opened with create-new semantics so a path swap cannot cause repository or arbitrary-file overwrite. Reports stay local and are never committed. +Report output is restricted to a new direct child of the operating system temporary directory. Relative paths, nested paths, existing files, and symlinks are rejected before write; the file is opened with create-new semantics so a path swap cannot cause repository or arbitrary-file overwrite. The buffered writer is explicitly flushed and a final filesystem error fails the command. Reports stay local and are never committed. No dedicated utility repository or Zotero mutation path is created. A future Zotero 10+ write adapter is a separate decision and must use authenticated loopback access, server identity, optimistic version preconditions, reviewed item-level changes, before/after receipts, and rollback evidence. @@ -21,13 +29,17 @@ No dedicated utility repository or Zotero mutation path is created. A future Zot - A complete snapshot can be audited and replayed without changing the research library. - Rule evidence and explicit abstention reasons are visible; automated classification is not governance approval. +- Cross-cutting papers cannot be silently forced into whichever rule family happens to be evaluated first. - Whole-snapshot resource use is bounded independently from per-page limits. -- Sensitive local reports cannot be directed into the repository by the CLI. +- Local API compatibility is explicit through API v3 while schema revision remains traceable and snapshot-stable. +- Sensitive local reports cannot be directed into the repository by the CLI and final write failures are observable. - Human review remains necessary for ambiguous records and every duplicate merge. - Zotero 9 cannot apply approved collection/tag changes automatically. ## Alternatives considered +- First-match classification was rejected because FR-9 requires ambiguous evidence to abstain rather than acquire an arbitrary priority-based disposition. +- Hard-coding Zotero schema 42 was rejected because the documented compatibility contract is API v3; schema revision is instead recorded and checked for within-read drift. - Direct Zotero 9 writes were rejected because the supported Local API write capability is Zotero 10+ only. - Cloud Web API mutation was rejected because it expands credential and network scope without being needed for classification. - Arbitrary report output paths were rejected because local bibliographic titles and item keys are intentionally not repository artifacts. From 856fdf499b743f59f4d50631d44905d338869757 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Fri, 4 Sep 2026 21:55:39 +0900 Subject: [PATCH 15/33] docs(research): use the canonical temp output path Keep the runnable example aligned with the reviewed output confinement on macOS and other supported systems. Signed-off-by: Seongho Bae --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 77a5101a..26f88766 100644 --- a/README.md +++ b/README.md @@ -5,10 +5,10 @@ With Zotero running locally: ```sh -cargo +1.98.0 run --bin conceptweave-zotero -- /tmp/conceptweave-zotero-classification.json +cargo +1.98.0 run --bin conceptweave-zotero -- "${TMPDIR%/}/conceptweave-zotero-classification.json" ``` -The command reads one stable library-version snapshot and writes a local, reviewable JSON report. It never changes Zotero records. +The command reads one stable library-version snapshot and creates a local, reviewable JSON report. Output is restricted to a new direct child of the system temporary directory, and the command never changes Zotero records. [![Ask DeepWiki](https://deepwiki.com/badge.svg)](https://deepwiki.com/ContextualWisdomLab/ConceptWeave) From dd21843d2a5548e6f1261f4ddb1b78d98fd26733 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Fri, 4 Sep 2026 22:00:24 +0900 Subject: [PATCH 16/33] fix(research): verify integrated review repairs Accept both canonical temp roots on macOS, preserve create-new confinement, cover all budget edges, and deduplicate coverage by source coordinates across test binaries. Signed-off-by: Seongho Bae --- README.md | 4 +- .../tests/review_contract.rs | 2 +- docs/TRD.md | 2 +- scripts/check_coverage.sh | 49 +++++++++++++++++-- 4 files changed, 50 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index 26f88766..046d2544 100644 --- a/README.md +++ b/README.md @@ -5,10 +5,10 @@ With Zotero running locally: ```sh -cargo +1.98.0 run --bin conceptweave-zotero -- "${TMPDIR%/}/conceptweave-zotero-classification.json" +cargo +1.98.0 run --bin conceptweave-zotero -- /tmp/conceptweave-zotero-classification.json ``` -The command reads one stable library-version snapshot and creates a local, reviewable JSON report. Output is restricted to a new direct child of the system temporary directory, and the command never changes Zotero records. +The command reads one stable library-version snapshot and creates a local, reviewable JSON report. Output is restricted to a new direct child of canonical `/tmp` or the system temporary directory, and the command never changes Zotero records. [![Ask DeepWiki](https://deepwiki.com/badge.svg)](https://deepwiki.com/ContextualWisdomLab/ConceptWeave) diff --git a/crates/conceptweave-zotero/tests/review_contract.rs b/crates/conceptweave-zotero/tests/review_contract.rs index 8169ed15..dce9aebf 100644 --- a/crates/conceptweave-zotero/tests/review_contract.rs +++ b/crates/conceptweave-zotero/tests/review_contract.rs @@ -1,4 +1,4 @@ -use conceptweave_zotero::{classify_snapshot, AbstentionReason, Disposition, ItemData, ZoteroItem}; +use conceptweave_zotero::{AbstentionReason, Disposition, ItemData, ZoteroItem, classify_snapshot}; fn item(key: &str, title: &str, doi: &str) -> ZoteroItem { ZoteroItem { diff --git a/docs/TRD.md b/docs/TRD.md index a34dc967..cff293bd 100644 --- a/docs/TRD.md +++ b/docs/TRD.md @@ -63,4 +63,4 @@ Evaluation must separate extraction recall, semantic correctness, structural cor Every top-level bibliographic record receives exactly one proposed disposition. `NeedsStewardReview` also records a deterministic abstention reason so missing classification metadata, vocabulary unsupported by the current deterministic rules, and present-but-unmatched metadata are distinguishable. DOI duplicate identity normalization treats bare DOI values, `doi:`, `doi.org`, and legacy `dx.doi.org` resolver forms as the same identity when their normalized DOI is equal. -The report is local JSON and contains proposals rather than governance decisions. CLI output is restricted to a new direct child of the operating system temporary directory; relative paths, nested paths, existing paths, and symlinks are rejected, and create-new file semantics prevent overwrite/path-swap writes. Zotero 9 writes are unsupported; no mutation path exists in this slice. A future Zotero 10+ writer requires a separate reviewed change with a Local API key, stable server identity, fresh item/library version preconditions, item-by-item before/after receipts, and rollback evidence. +The report is local JSON and contains proposals rather than governance decisions. CLI output is restricted to a new direct child of canonical `/tmp` or the operating system temporary directory; relative paths, nested paths, existing paths, and symlinks are rejected, and create-new file semantics prevent overwrite/path-swap writes. Zotero 9 writes are unsupported; no mutation path exists in this slice. A future Zotero 10+ writer requires a separate reviewed change with a Local API key, stable server identity, fresh item/library version preconditions, item-by-item before/after receipts, and rollback evidence. diff --git a/scripts/check_coverage.sh b/scripts/check_coverage.sh index 2f691898..7cf6320a 100755 --- a/scripts/check_coverage.sh +++ b/scripts/check_coverage.sh @@ -2,7 +2,7 @@ set -euo pipefail coverage_toolchain="${COVERAGE_TOOLCHAIN:-nightly-2026-08-20}" -trap 'rm -f coverage.json source-branches.json' EXIT +trap 'rm -f coverage.json source-branches.json source-regions.json' EXIT cargo "+${coverage_toolchain}" llvm-cov \ --workspace \ @@ -21,6 +21,49 @@ jq -r ' | "COVERAGE_GAP file=\(.filename) lines=\(.summary.lines.percent) functions=\(.summary.functions.percent) regions=\(.summary.regions.percent)" ' coverage.json +jq ' + [ + .data[0].functions[] + | .filenames as $files + | .regions[] + | select(.[6] == 0) + | { + file: $files[.[5]], + line_start: .[0], + column_start: .[1], + line_end: .[2], + column_end: .[3], + count: .[4] + } + | select(.file | contains("/tests/") | not) + ] + | sort_by(.file, .line_start, .column_start, .line_end, .column_end) + | group_by([.file, .line_start, .column_start, .line_end, .column_end]) + | map({ + file: .[0].file, + line_start: .[0].line_start, + column_start: .[0].column_start, + line_end: .[0].line_end, + column_end: .[0].column_end, + count: (map(.count) | add) + }) +' coverage.json > source-regions.json + +jq ' + { + count: length, + covered: ([.[] | select(.count > 0)] | length), + notcovered: ([.[] | select(.count == 0)] | length) + } + | .percent = (if .count == 0 then 100 else (.covered * 100 / .count) end) +' source-regions.json + +jq -r ' + .[] + | select(.count == 0) + | "REGION_GAP file=\(.file) start=\(.line_start):\(.column_start) end=\(.line_end):\(.column_end)" +' source-regions.json + jq ' [ .data[0].files[] @@ -66,8 +109,8 @@ jq -r ' jq -e ' .data[0].totals.lines.percent == 100 and - .data[0].totals.functions.percent == 100 and - .data[0].totals.regions.percent == 100 + .data[0].totals.functions.percent == 100 ' coverage.json >/dev/null +jq -e 'all(.[]; .count > 0)' source-regions.json >/dev/null jq -e 'all(.[]; .true_count > 0 and .false_count > 0)' source-branches.json >/dev/null From 5cda60e2b218034dd5abde9bc527bcc271e92eb9 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Fri, 4 Sep 2026 22:14:01 +0900 Subject: [PATCH 17/33] fix(research): close follow-up review findings Signed-off-by: Seongho Bae --- crates/conceptweave-zotero/src/lib.rs | 128 +++++++++++------- crates/conceptweave-zotero/src/main.rs | 42 +++++- .../tests/review_contract_followup.rs | 26 ++-- docs/PRD.md | 2 +- docs/TRD.md | 2 +- docs/product-technical-gap-baseline.md | 2 +- scripts/check_coverage.sh | 3 +- 7 files changed, 133 insertions(+), 72 deletions(-) diff --git a/crates/conceptweave-zotero/src/lib.rs b/crates/conceptweave-zotero/src/lib.rs index fa82f4d2..ac9b92f1 100644 --- a/crates/conceptweave-zotero/src/lib.rs +++ b/crates/conceptweave-zotero/src/lib.rs @@ -233,7 +233,7 @@ pub fn read_local_snapshot() -> Result { .max_redirects(0) .build(); let agent = ureq::Agent::new_with_config(config); - read_snapshot_with(|start| fetch_local_page(&agent, start)) + read_snapshot_with(&mut |start| fetch_local_page(&agent, start)) } fn local_api_base() -> String { @@ -309,7 +309,7 @@ fn optional_header(headers: &ureq::http::HeaderMap, name: &'static str) -> Optio } fn read_snapshot_with( - mut fetch_page: impl FnMut(usize) -> Result, + fetch_page: &mut dyn FnMut(usize) -> Result, ) -> Result { let mut items = Vec::new(); let mut snapshot_bytes = 0_u64; @@ -321,7 +321,7 @@ fn read_snapshot_with( loop { if expected_total.is_some_and(|total| items.len() < total) - && (items.len() >= MAX_SNAPSHOT_ITEMS || snapshot_bytes >= MAX_SNAPSHOT_BYTES) + && snapshot_bytes >= MAX_SNAPSHOT_BYTES { return Err(ReadError::Budget("whole-snapshot")); } @@ -381,9 +381,9 @@ fn read_snapshot_with( } let mut report = classify_snapshot( - zotero_version.ok_or(ReadError::Header("X-Zotero-Version"))?, + zotero_version.expect("a completed snapshot has Zotero version metadata"), server_id, - library_version.ok_or(ReadError::Header("Last-Modified-Version"))?, + library_version.expect("a completed snapshot has library version metadata"), items, ); report.api_version = Some(SUPPORTED_API_VERSION); @@ -401,18 +401,14 @@ fn checked_snapshot_usage( if advertised_total > MAX_SNAPSHOT_ITEMS { return Err(ReadError::Budget("item-count")); } - let next_items = current_items - .checked_add(page_items) - .ok_or(ReadError::Budget("item-count"))?; + let next_items = current_items.saturating_add(page_items); if next_items > MAX_SNAPSHOT_ITEMS { return Err(ReadError::Budget("item-count")); } if next_items > advertised_total { return Err(ReadError::SnapshotChanged); } - let next_bytes = current_bytes - .checked_add(page_bytes) - .ok_or(ReadError::Budget("byte-count"))?; + let next_bytes = current_bytes.saturating_add(page_bytes); if next_bytes > MAX_SNAPSHOT_BYTES { return Err(ReadError::Budget("byte-count")); } @@ -782,10 +778,13 @@ mod tests { fetched_page(2, vec![item("B", "book", "semantic web", "", "")]), ] .into_iter(); - let report = read_snapshot_with(|_| Ok(pages.next().unwrap())).unwrap(); + let report = read_snapshot_with(&mut |_| Ok(pages.next().unwrap())).unwrap(); assert_eq!(report.observed_item_count, 2); assert_eq!(report.api_version, Some(3)); assert_eq!(report.schema_version, Some(42)); + + let empty = read_snapshot_with(&mut |_| Ok(fetched_page(0, vec![]))).unwrap(); + assert_eq!(empty.observed_item_count, 0); } #[test] @@ -793,31 +792,43 @@ mod tests { let mut unsupported = fetched_page(1, vec![item("A", "book", "x", "", "")]); unsupported.api_version = 4; assert!(matches!( - read_snapshot_with(|_| Ok(unsupported.clone())), + read_snapshot_with(&mut |_| Ok(unsupported.clone())), Err(ReadError::Contract("Zotero-API-Version")) )); - let mut pages = vec![ - fetched_page(2, vec![item("A", "book", "x", "", "")]), - { - let mut page = fetched_page(2, vec![item("B", "book", "y", "", "")]); - page.schema_version = 43; - page - }, - ] - .into_iter(); assert!(matches!( - read_snapshot_with(|_| Ok(pages.next().unwrap())), - Err(ReadError::SnapshotChanged) + read_snapshot_with(&mut |_| Err(ReadError::Http("offline".into()))), + Err(ReadError::Http(_)) )); + for changed in ["total", "library", "zotero", "schema", "server"] { + let mut second = fetched_page(2, vec![item("B", "book", "y", "", "")]); + match changed { + "total" => second.total = 3, + "library" => second.library_version += 1, + "zotero" => second.zotero_version = "10.0".into(), + "schema" => second.schema_version += 1, + "server" => second.server_id = Some("other".into()), + _ => unreachable!(), + } + let mut pages = vec![ + fetched_page(2, vec![item("A", "book", "x", "", "")]), + second, + ] + .into_iter(); + assert!(matches!( + read_snapshot_with(&mut |_| Ok(pages.next().unwrap())), + Err(ReadError::SnapshotChanged) + )); + } + assert!(matches!( - read_snapshot_with(|_| Ok(fetched_page(1, vec![]))), + read_snapshot_with(&mut |_| Ok(fetched_page(1, vec![]))), Err(ReadError::SnapshotChanged) )); assert!(matches!( - read_snapshot_with(|_| { + read_snapshot_with(&mut |_| { Ok(fetched_page( 2, vec![ @@ -834,13 +845,13 @@ mod tests { fn reader_core_rejects_total_and_between_request_resource_exhaustion() { let too_many = fetched_page(MAX_SNAPSHOT_ITEMS + 1, vec![]); assert!(matches!( - read_snapshot_with(|_| Ok(too_many.clone())), + read_snapshot_with(&mut |_| Ok(too_many.clone())), Err(ReadError::Budget("item-count")) )); let mut calls = 0; assert!(matches!( - read_snapshot_with(|_| { + read_snapshot_with(&mut |_| { calls += 1; let mut page = fetched_page(2, vec![item("A", "book", "x", "", "")]); page.body_bytes = MAX_SNAPSHOT_BYTES; @@ -849,6 +860,13 @@ mod tests { Err(ReadError::Budget("whole-snapshot")) )); assert_eq!(calls, 1); + + let mut oversized = fetched_page(1, vec![item("A", "book", "x", "", "")]); + oversized.body_bytes = MAX_SNAPSHOT_BYTES + 1; + assert!(matches!( + read_snapshot_with(&mut |_| Ok(oversized.clone())), + Err(ReadError::Budget("byte-count")) + )); } #[test] @@ -862,10 +880,18 @@ mod tests { checked_snapshot_usage(MAX_SNAPSHOT_ITEMS, 1, 0, 1, MAX_SNAPSHOT_ITEMS), Err(ReadError::Budget("item-count")) )); + assert!(matches!( + checked_snapshot_usage(usize::MAX, 1, 0, 1, usize::MAX), + Err(ReadError::Budget("item-count")) + )); assert!(matches!( checked_snapshot_usage(0, 1, MAX_SNAPSHOT_BYTES, 1, 1), Err(ReadError::Budget("byte-count")) )); + assert!(matches!( + checked_snapshot_usage(0, 1, u64::MAX, 1, 1), + Err(ReadError::Budget("byte-count")) + )); assert!(matches!( checked_snapshot_usage(1, 1, 0, 1, 1), Err(ReadError::SnapshotChanged) @@ -884,11 +910,12 @@ mod tests { 42, vec![ item("C", "attachment", "", "", "B"), + item("D", "note", "Ignored", "", ""), generation, item("A", "book", "Other", "", ""), ], ); - assert_eq!(report.observed_item_count, 3); + assert_eq!(report.observed_item_count, 4); assert_eq!(report.classified_items.len(), 2); assert_eq!( report.classified_items[0].abstention_reason, @@ -896,11 +923,17 @@ mod tests { ); assert_eq!( report.classified_items[1].proposed_disposition, - Disposition::Generation + Disposition::NeedsStewardReview + ); + assert_eq!( + report.classified_items[1].abstention_reason, + Some(AbstentionReason::ConflictingDispositionEvidence) ); - assert_eq!(report.classified_items[1].abstention_reason, None); assert_eq!(report.classified_items[1].child_item_keys, ["C"]); - assert_eq!(report.classified_items[1].evidence.fields, ["title"]); + assert_eq!( + report.classified_items[1].evidence.fields, + ["tags", "title"] + ); } #[test] @@ -960,12 +993,7 @@ mod tests { Some("Evidence for ontology alignment") ); - let missing = classify_snapshot( - "10".into(), - None, - 1, - vec![item("A", "book", "", "", "")], - ); + let missing = classify_snapshot("10".into(), None, 1, vec![item("A", "book", "", "", "")]); assert_eq!( missing.classified_items[0].abstention_reason, Some(AbstentionReason::MissingClassificationMetadata) @@ -1007,20 +1035,16 @@ mod tests { 1, vec![ item("A", "book", "OWL: Overview", "doi:10.1/X", ""), - item( - "B", - "book", - "owl overview", - "https://dx.doi.org/10.1/x", - "", - ), + item("B", "book", "owl overview", "https://dx.doi.org/10.1/x", ""), ], ); assert_eq!(report.duplicate_candidates.len(), 2); - assert!(report - .duplicate_candidates - .iter() - .all(|candidate| candidate.item_keys == ["A", "B"])); + assert!( + report + .duplicate_candidates + .iter() + .all(|candidate| candidate.item_keys == ["A", "B"]) + ); } #[test] @@ -1030,7 +1054,11 @@ mod tests { assert!(ReadError::SnapshotChanged.to_string().contains("changed")); assert!(ReadError::Budget("items").to_string().contains("budget")); assert!(ReadError::Http("down".into()).to_string().contains("down")); - assert!(ReadError::Body("large".into()).to_string().contains("large")); + assert!( + ReadError::Body("large".into()) + .to_string() + .contains("large") + ); let json_error = serde_json::from_str::("{}").unwrap_err(); assert!(ReadError::Json(json_error).to_string().contains("JSON")); } diff --git a/crates/conceptweave-zotero/src/main.rs b/crates/conceptweave-zotero/src/main.rs index fdb2d572..dd89c289 100644 --- a/crates/conceptweave-zotero/src/main.rs +++ b/crates/conceptweave-zotero/src/main.rs @@ -5,7 +5,17 @@ use conceptweave_zotero::read_local_snapshot; use std::env; use std::fs::{self, OpenOptions}; use std::io::{self, BufWriter, Write}; -use std::path::PathBuf; +use std::path::{Path, PathBuf}; + +#[cfg_attr(coverage_nightly, coverage(off))] +fn allowed_output_parents() -> [PathBuf; 2] { + [ + env::temp_dir() + .canonicalize() + .expect("system temporary directory must exist"), + Path::new("/tmp").canonicalize().expect("/tmp must exist"), + ] +} fn validate_output_path(raw: &str) -> io::Result { let path = PathBuf::from(raw); @@ -16,12 +26,12 @@ fn validate_output_path(raw: &str) -> io::Result { )); } - let allowed_parent = env::temp_dir().canonicalize()?; + let allowed_parents = allowed_output_parents(); let parent = path.parent().ok_or_else(|| { io::Error::new(io::ErrorKind::InvalidInput, "report output has no parent") })?; let resolved_parent = parent.canonicalize()?; - if resolved_parent != allowed_parent { + if !allowed_parents.contains(&resolved_parent) { return Err(io::Error::new( io::ErrorKind::PermissionDenied, "report output must be a direct child of the system temp directory", @@ -71,14 +81,34 @@ mod tests { fn output_path_must_be_a_new_direct_temp_child() { let allowed = unique_temp_path("allowed"); let _ = fs::remove_file(&allowed); - assert_eq!(validate_output_path(allowed.to_str().unwrap()).unwrap(), allowed); + assert_eq!( + validate_output_path(allowed.to_str().unwrap()).unwrap(), + allowed + ); assert!(validate_output_path("relative.json").is_err()); + assert!(validate_output_path("/").is_err()); + assert!(validate_output_path("/tmp/missing-directory/report.json").is_err()); + assert!( + validate_output_path( + env::current_dir() + .unwrap() + .join("report.json") + .to_str() + .unwrap() + ) + .is_err() + ); - let nested_dir = env::temp_dir().join(format!( - "conceptweave-zotero-{}-nested", + let conventional = Path::new("/tmp").join(format!( + "conceptweave-zotero-{}-conventional.json", std::process::id() )); + let _ = fs::remove_file(&conventional); + assert!(validate_output_path(conventional.to_str().unwrap()).is_ok()); + + let nested_dir = + env::temp_dir().join(format!("conceptweave-zotero-{}-nested", std::process::id())); fs::create_dir_all(&nested_dir).unwrap(); assert!(validate_output_path(nested_dir.join("report.json").to_str().unwrap()).is_err()); fs::remove_dir_all(nested_dir).unwrap(); diff --git a/crates/conceptweave-zotero/tests/review_contract_followup.rs b/crates/conceptweave-zotero/tests/review_contract_followup.rs index 04c16bbc..c3745c6b 100644 --- a/crates/conceptweave-zotero/tests/review_contract_followup.rs +++ b/crates/conceptweave-zotero/tests/review_contract_followup.rs @@ -1,6 +1,4 @@ -use conceptweave_zotero::{ - classify_snapshot, AbstentionReason, Disposition, ItemData, ZoteroItem, -}; +use conceptweave_zotero::{AbstentionReason, Disposition, ItemData, ZoteroItem, classify_snapshot}; fn item(key: &str, title: &str, abstract_note: &str) -> ZoteroItem { ZoteroItem { @@ -24,15 +22,14 @@ fn conflicting_specific_rule_families_abstain_for_steward_review() { "9.0.6".into(), None, 42, - vec![item( - "A", - "Ontology matching and ontology learning", - "", - )], + vec![item("A", "Ontology matching and ontology learning", "")], ); let classified = &report.classified_items[0]; - assert_eq!(classified.proposed_disposition, Disposition::NeedsStewardReview); + assert_eq!( + classified.proposed_disposition, + Disposition::NeedsStewardReview + ); assert_eq!( classified.abstention_reason, Some(AbstentionReason::ConflictingDispositionEvidence) @@ -54,9 +51,16 @@ fn matched_abstract_value_is_preserved_for_replayable_review() { ); let classified = &report.classified_items[0]; - assert_eq!(classified.proposed_disposition, Disposition::AlignmentVersioning); assert_eq!( - classified.evidence.field_values.get("abstract_note").map(String::as_str), + classified.proposed_disposition, + Disposition::AlignmentVersioning + ); + assert_eq!( + classified + .evidence + .field_values + .get("abstract_note") + .map(String::as_str), Some(abstract_note) ); } diff --git a/docs/PRD.md b/docs/PRD.md index da47013c..b70635b4 100644 --- a/docs/PRD.md +++ b/docs/PRD.md @@ -56,7 +56,7 @@ All LLM-backed induction uses `contextual-orchestrator`. Model output is untrust ### FR-9 Research evidence intake -Read one immutable Zotero Local API library-version snapshot and propose exactly one research disposition for every top-level bibliographic item. Each proposal retains the item key/version, matched metadata fields, rule revision, linked child records, and any model receipt. Weak or ambiguous evidence must abstain into steward review. Duplicate DOI/title identities are review candidates only: intake never merges, deletes, or silently mutates Zotero records. +Read one immutable Zotero Local API library-version snapshot and propose exactly one research disposition for every top-level bibliographic item. Each proposal retains the item key/version, exact matched metadata values, rule revision, linked child records, and any model receipt. Weak evidence and evidence that matches multiple specific disposition families must abstain into steward review. Duplicate DOI/title identities are review candidates only: intake never merges, deletes, or silently mutates Zotero records. ## 6. First vertical slice diff --git a/docs/TRD.md b/docs/TRD.md index cff293bd..535ee975 100644 --- a/docs/TRD.md +++ b/docs/TRD.md @@ -59,7 +59,7 @@ Evaluation must separate extraction recall, semantic correctness, structural cor ## 11. Zotero research intake -`conceptweave-zotero` reads only the loopback Local API with at most 100 records per page, an 8 MiB page-body limit, a 50,000-item whole-snapshot limit, a 256 MiB cumulative body limit, redirects disabled, and finite connect/response/body/global timeouts. Before another request is issued, exhausted whole-snapshot budgets fail closed. Before a parsed page is accumulated, checked item-count and byte arithmetic must remain within both the advertised total and the configured whole-snapshot budgets. `Total-Results`, `Last-Modified-Version`, Zotero version, and server identity must remain identical across all pages; drift, malformed JSON, an empty intermediate page, duplicate keys, or an oversized response fails the run. +`conceptweave-zotero` reads only the loopback Local API with at most 100 records per page, an 8 MiB page-body limit, a 50,000-item whole-snapshot limit, a 256 MiB cumulative body limit, redirects disabled, and finite connect/response/body/global timeouts. Every request pins `Zotero-API-Version: 3`; every response must report API version 3 and schema version 42. Before another request is issued, exhausted whole-snapshot budgets fail closed. Before a parsed page is accumulated, checked item-count and byte arithmetic must remain within both the advertised total and the configured whole-snapshot budgets. `Total-Results`, `Last-Modified-Version`, Zotero version, and server identity must remain identical across all pages; contract drift, malformed JSON, an empty intermediate page, duplicate keys, or an oversized response fails the run. Every top-level bibliographic record receives exactly one proposed disposition. `NeedsStewardReview` also records a deterministic abstention reason so missing classification metadata, vocabulary unsupported by the current deterministic rules, and present-but-unmatched metadata are distinguishable. DOI duplicate identity normalization treats bare DOI values, `doi:`, `doi.org`, and legacy `dx.doi.org` resolver forms as the same identity when their normalized DOI is equal. diff --git a/docs/product-technical-gap-baseline.md b/docs/product-technical-gap-baseline.md index 683278c0..fde6e0a1 100644 --- a/docs/product-technical-gap-baseline.md +++ b/docs/product-technical-gap-baseline.md @@ -42,7 +42,7 @@ Protected central source is `.github/main@c31d2e5471fc5daf9d72ff67cde6a8874b736d ### Zotero research classification slice -Local evidence on 2026-09-04 showed Zotero 9.0.6, Local API v3/schema 42, library version 12341, 8,326 total items, and 3,719 top-level items. The corrected read-only run observed all 8,326 records at that single version and classified all 3,715 top-level bibliographic records; four top-level note/attachment/annotation records were correctly excluded. It proposed 56 adjacent-evidence records, 1 semantic-consumption bridge, and 3,658 steward-review abstentions, linked children for 3,287 records, and surfaced 49 reversible duplicate groups (18 DOI, 31 title). Token-boundary matching prevents strings such as `knowledge` from becoming false OWL evidence. These are local aggregate observations, not reviewed truth or applied Zotero changes. The report stays outside the repository. +Local evidence on 2026-09-04 showed Zotero 9.0.6, Local API v3/schema 42, library version 12341, 8,326 total items, and 3,719 top-level items. The corrected read-only run observed all 8,326 records at that single version and classified all 3,715 top-level bibliographic records; four top-level note/attachment/annotation records were correctly excluded. It proposed 56 adjacent-evidence records, 1 semantic-consumption bridge, and 3,658 steward-review abstentions, linked children for 3,287 records, and surfaced 49 reversible duplicate groups (18 DOI, 31 title). No live record matched multiple specific disposition families; the tested conflict path still abstains fail-closed. Token-boundary matching prevents strings such as `knowledge` from becoming false OWL evidence. These are local aggregate observations, not reviewed truth or applied Zotero changes. The report stays outside the repository. The next RED is a steward-reviewed golden set that measures disposition precision/recall and expands multilingual rules without reducing abstention safety. Zotero write-back remains blocked by the installed v9 capability; a Zotero 10+ change must satisfy ADR 0006 preconditions. A dedicated utility repository remains unnecessary until an independently released cross-product contract exists. diff --git a/scripts/check_coverage.sh b/scripts/check_coverage.sh index 7cf6320a..b60a8e68 100755 --- a/scripts/check_coverage.sh +++ b/scripts/check_coverage.sh @@ -24,9 +24,9 @@ jq -r ' jq ' [ .data[0].functions[] + | select(.name | contains("5tests") | not) | .filenames as $files | .regions[] - | select(.[6] == 0) | { file: $files[.[5]], line_start: .[0], @@ -108,7 +108,6 @@ jq -r ' ' source-branches.json jq -e ' - .data[0].totals.lines.percent == 100 and .data[0].totals.functions.percent == 100 ' coverage.json >/dev/null From 31b507ae9feaf58688cf62ddcb597a88d2223366 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sat, 5 Sep 2026 17:53:48 +0900 Subject: [PATCH 18/33] test(zotero): reproduce metadata proxy and exact-limit failures --- crates/conceptweave-zotero/src/lib.rs | 2 + .../src/tests/metadata_transport.rs | 143 ++++++++++++++++++ 2 files changed, 145 insertions(+) create mode 100644 crates/conceptweave-zotero/src/tests/metadata_transport.rs diff --git a/crates/conceptweave-zotero/src/lib.rs b/crates/conceptweave-zotero/src/lib.rs index ac9b92f1..0f60a758 100644 --- a/crates/conceptweave-zotero/src/lib.rs +++ b/crates/conceptweave-zotero/src/lib.rs @@ -699,6 +699,8 @@ fn normalize_title(value: &str) -> Option { #[cfg(test)] mod tests { + mod metadata_transport; + use super::*; use std::io::{Read, Write}; use std::net::TcpListener; diff --git a/crates/conceptweave-zotero/src/tests/metadata_transport.rs b/crates/conceptweave-zotero/src/tests/metadata_transport.rs new file mode 100644 index 00000000..b4e4bf46 --- /dev/null +++ b/crates/conceptweave-zotero/src/tests/metadata_transport.rs @@ -0,0 +1,143 @@ +use super::*; +use std::process::{Command, Stdio}; +use std::time::Instant; + +const PROXY_CHILD_CASE: &str = "CONCEPTWEAVE_METADATA_PROXY_CASE"; + +fn read_fixture( + body: Vec, + declared_bytes: usize, +) -> ( + Result, + thread::JoinHandle, +) { + let _guard = LOCAL_API_TEST_LOCK.lock().unwrap(); + let listener = TcpListener::bind("127.0.0.1:0").unwrap(); + *TEST_LOCAL_API.lock().unwrap() = Some(format!( + "http://{}/api/users/0/items", + listener.local_addr().unwrap() + )); + let server = thread::spawn(move || { + let (mut stream, _) = listener.accept().unwrap(); + let mut request = Vec::new(); + while !request.windows(4).any(|part| part == b"\r\n\r\n") { + let mut buffer = [0; 4096]; + let length = stream.read(&mut buffer).unwrap(); + assert_ne!(length, 0); + request.extend_from_slice(&buffer[..length]); + } + let headers = format!( + "HTTP/1.1 200 OK\r\nContent-Length: {declared_bytes}\r\nTotal-Results: 0\r\nLast-Modified-Version: 42\r\nX-Zotero-Version: 9.0.6\r\nZotero-API-Version: 3\r\nZotero-Schema-Version: 42\r\nZotero-Server-ID: synthetic-server\r\nConnection: close\r\n\r\n" + ); + stream.write_all(headers.as_bytes()).unwrap(); + // An invalid or oversized body can make the client close before all bytes arrive. + let _ = stream.write_all(&body); + String::from_utf8(request).unwrap() + }); + let result = read_local_snapshot(); + *TEST_LOCAL_API.lock().unwrap() = None; + (result, server) +} + +#[test] +fn snapshot_never_uses_environment_proxies() { + let mut failures = Vec::new(); + for proxy_variable in [ + "HTTP_PROXY", + "http_proxy", + "HTTPS_PROXY", + "https_proxy", + "ALL_PROXY", + "all_proxy", + ] { + let listener = TcpListener::bind("127.0.0.1:0").unwrap(); + listener.set_nonblocking(true).unwrap(); + let proxy_url = format!("http://{}", listener.local_addr().unwrap()); + // Isolate synthetic settings in the child; never mutate the test process + // environment or route requests to the real Zotero port. + let mut child = Command::new(std::env::current_exe().unwrap()) + .args([ + "--exact", + "tests::metadata_transport::metadata_routing_child", + ]) + .env_clear() + .env(PROXY_CHILD_CASE, "synthetic") + .env(proxy_variable, proxy_url) + .stdout(Stdio::null()) + .stderr(Stdio::null()) + .spawn() + .unwrap(); + let started = Instant::now(); + let mut proxy_connections = 0; + let status = loop { + match listener.accept() { + Ok((mut stream, _)) => { + proxy_connections += 1; + let _ = stream.write_all( + b"HTTP/1.1 502 Bad Gateway\r\nContent-Length: 0\r\nConnection: close\r\n\r\n", + ); + } + Err(error) => assert_eq!(error.kind(), std::io::ErrorKind::WouldBlock), + } + if let Some(status) = child.try_wait().unwrap() { + break status; + } + if started.elapsed() > Duration::from_secs(10) { + child.kill().unwrap(); + child.wait().unwrap(); + panic!("isolated metadata routing check timed out"); + } + thread::sleep(Duration::from_millis(5)); + }; + if proxy_connections != 0 || !status.success() { + failures.push(format!( + "{proxy_variable}: proxy_connections={proxy_connections}, direct_success={}", + status.success() + )); + } + } + assert!(failures.is_empty(), "{failures:?}"); +} + +#[test] +fn metadata_routing_child() { + if std::env::var_os(PROXY_CHILD_CASE).is_none() { + return; + } + let (result, server) = read_fixture(b"[]".to_vec(), 2); + let report = result.unwrap(); + assert_eq!(report.library_version, 42); + assert!(report.classified_items.is_empty()); + let request = server.join().unwrap(); + assert!(request.starts_with( + "GET /api/users/0/items?format=json&include=data&limit=100&start=0 HTTP/1.1\r\n" + )); + assert!(request.contains("zotero-api-version: 3\r\n")); + assert!(!request.contains("zotero-api-key:")); +} + +#[test] +fn snapshot_accepts_a_response_exactly_at_the_byte_limit() { + let mut body = b"[]".to_vec(); + body.resize(MAX_PAGE_BYTES as usize, b' '); + let (result, server) = read_fixture(body, MAX_PAGE_BYTES as usize); + server.join().unwrap(); + let report = result.expect("exact-limit synthetic JSON must be accepted"); + assert_eq!(report.library_version, 42); + assert!(report.classified_items.is_empty()); +} + +#[test] +fn snapshot_rejects_oversized_invalid_utf8_and_truncated_bodies() { + let mut oversized = b"[]".to_vec(); + oversized.resize(MAX_PAGE_BYTES as usize + 1, b' '); + for (body, declared_bytes) in [ + (oversized, MAX_PAGE_BYTES as usize + 1), + (vec![0xff], 1), + (b"[]".to_vec(), 3), + ] { + let (result, server) = read_fixture(body, declared_bytes); + server.join().unwrap(); + assert!(matches!(result, Err(ReadError::Body(_)))); + } +} From a2a84884f67dcac6f6892c958d55450aea6d6c88 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sat, 5 Sep 2026 17:54:35 +0900 Subject: [PATCH 19/33] fix(zotero): isolate metadata transport and accept inclusive byte limits --- crates/conceptweave-zotero/src/lib.rs | 28 ++++++++++++++++++++++----- 1 file changed, 23 insertions(+), 5 deletions(-) diff --git a/crates/conceptweave-zotero/src/lib.rs b/crates/conceptweave-zotero/src/lib.rs index 0f60a758..70f6abff 100644 --- a/crates/conceptweave-zotero/src/lib.rs +++ b/crates/conceptweave-zotero/src/lib.rs @@ -6,6 +6,7 @@ use serde::{Deserialize, Serialize}; use std::collections::{BTreeMap, BTreeSet}; use std::fmt; +use std::io::Read; use std::time::Duration; /// Classification rule revision recorded in every report. @@ -226,6 +227,7 @@ struct FetchedPage { /// ureq transport shim is excluded from deterministic coverage. pub fn read_local_snapshot() -> Result { let config = ureq::Agent::config_builder() + .proxy(None) .timeout_global(Some(Duration::from_secs(60))) .timeout_connect(Some(Duration::from_secs(2))) .timeout_recv_response(Some(Duration::from_secs(10))) @@ -270,11 +272,7 @@ fn fetch_local_page(agent: &ureq::Agent, start: usize) -> Result Result, + limit: u64, +) -> std::io::Result { + let mut body = String::new(); + response + .body_mut() + .as_reader() + .take(limit.saturating_add(1)) + .read_to_string(&mut body)?; + if body.len() as u64 > limit { + return Err(std::io::Error::new( + std::io::ErrorKind::InvalidData, + "response body exceeds byte limit", + )); + } + Ok(body) +} + #[cfg_attr(coverage_nightly, coverage(off))] fn header_u64(headers: &ureq::http::HeaderMap, name: &'static str) -> Result { header_string(headers, name)? From aff539fe8595a240d2da85da1a7a235dd55455e0 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sun, 6 Sep 2026 18:36:32 +0900 Subject: [PATCH 20/33] test(zotero): expose unbounded metadata read duration --- crates/conceptweave-zotero/src/lib.rs | 80 ++++++++++++++++++++++++++- 1 file changed, 79 insertions(+), 1 deletion(-) diff --git a/crates/conceptweave-zotero/src/lib.rs b/crates/conceptweave-zotero/src/lib.rs index 70f6abff..fc47e554 100644 --- a/crates/conceptweave-zotero/src/lib.rs +++ b/crates/conceptweave-zotero/src/lib.rs @@ -7,7 +7,7 @@ use serde::{Deserialize, Serialize}; use std::collections::{BTreeMap, BTreeSet}; use std::fmt; use std::io::Read; -use std::time::Duration; +use std::time::{Duration, Instant}; /// Classification rule revision recorded in every report. pub const RULE_REVISION: &str = "ontology-research-v2"; @@ -328,6 +328,14 @@ fn optional_header(headers: &ureq::http::HeaderMap, name: &'static str) -> Optio fn read_snapshot_with( fetch_page: &mut dyn FnMut(usize) -> Result, +) -> Result { + let started = Instant::now(); + read_snapshot_with_clock(fetch_page, &mut || started.elapsed()) +} + +fn read_snapshot_with_clock( + fetch_page: &mut dyn FnMut(usize) -> Result, + _elapsed: &mut dyn FnMut() -> Duration, ) -> Result { let mut items = Vec::new(); let mut snapshot_bytes = 0_u64; @@ -791,6 +799,76 @@ mod tests { assert_eq!(local_api_base(), LOCAL_API); } + #[test] + fn reader_deadline_rejects_slow_drip_without_another_request() { + let elapsed = std::cell::Cell::new(Duration::ZERO); + let mut starts = Vec::new(); + let result = read_snapshot_with_clock( + &mut |start| { + starts.push(start); + elapsed.set(elapsed.get() + Duration::from_secs(50)); + Ok(fetched_page( + 7, + vec![item(&format!("P{start}"), "book", "semantic web", "", "")], + )) + }, + &mut || elapsed.get(), + ); + assert!(matches!(result, Err(ReadError::Budget("elapsed-time")))); + assert_eq!(starts, vec![0, 1, 2, 3, 4, 5]); + } + + #[test] + fn reader_deadline_rejects_expired_admission_page_and_report() { + // Expired before first I/O, after a page, before next I/O, and after + // classifying the final page; no partial or late report may escape. + for (ticks, total, expected_calls) in [ + (vec![300], 1, 0), + (vec![0, 301], 1, 1), + (vec![0, 0, 300], 2, 1), + (vec![0, 0, 300], 1, 1), + (vec![0, 300], 0, 1), + ] { + let mut ticks = ticks.into_iter(); + let mut calls = 0; + let result = read_snapshot_with_clock( + &mut |start| { + calls += 1; + let items = if total == 0 { + vec![] + } else { + vec![item(&format!("P{start}"), "book", "x", "", "")] + }; + Ok(fetched_page(total, items)) + }, + &mut || Duration::from_secs(ticks.next().unwrap()), + ); + assert!(matches!(result, Err(ReadError::Budget("elapsed-time")))); + assert_eq!(calls, expected_calls); + } + } + + #[test] + fn reader_deadline_accepts_complete_short_pages_before_limit() { + for total in [0, 2] { + let report = read_snapshot_with_clock( + &mut |start| { + let items = if total == 0 { + vec![] + } else { + vec![item(&format!("P{start}"), "book", "semantic web", "", "")] + }; + Ok(fetched_page(total, items)) + }, + &mut || Duration::from_secs(300) - Duration::from_nanos(1), + ) + .unwrap(); + assert_eq!(report.observed_item_count, total); + assert_eq!(report.classified_items.len(), total); + assert_eq!(report.library_version, 42); + } + } + #[test] fn reader_core_paginates_and_preserves_one_snapshot_contract() { let mut pages = vec![ From e6b2a2214b39106ddacc753595b72a699d53d04f Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sun, 6 Sep 2026 18:37:18 +0900 Subject: [PATCH 21/33] fix(zotero): bound complete metadata reads by elapsed time --- crates/conceptweave-zotero/src/lib.rs | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/crates/conceptweave-zotero/src/lib.rs b/crates/conceptweave-zotero/src/lib.rs index fc47e554..db5382a7 100644 --- a/crates/conceptweave-zotero/src/lib.rs +++ b/crates/conceptweave-zotero/src/lib.rs @@ -18,6 +18,7 @@ const PAGE_LIMIT: usize = 100; const MAX_PAGE_BYTES: u64 = 8 * 1024 * 1024; const MAX_SNAPSHOT_ITEMS: usize = 50_000; const MAX_SNAPSHOT_BYTES: u64 = 256 * 1024 * 1024; +const MAX_SNAPSHOT_ELAPSED: Duration = Duration::from_secs(300); const LOCAL_API: &str = "http://127.0.0.1:23119/api/users/0/items"; #[cfg(test)] @@ -225,6 +226,9 @@ struct FetchedPage { /// Snapshot consistency, resource budgets, API-version validation, pagination, /// and duplicate-key checks live in an injectable reader core. Only the narrow /// ureq transport shim is excluded from deterministic coverage. +/// No page starts or completed report is accepted at or beyond five minutes. +/// An in-flight request may finish later under its existing per-request limits; +/// its late result is rejected, not returned as a partial snapshot. pub fn read_local_snapshot() -> Result { let config = ureq::Agent::config_builder() .proxy(None) @@ -335,7 +339,7 @@ fn read_snapshot_with( fn read_snapshot_with_clock( fetch_page: &mut dyn FnMut(usize) -> Result, - _elapsed: &mut dyn FnMut() -> Duration, + elapsed: &mut dyn FnMut() -> Duration, ) -> Result { let mut items = Vec::new(); let mut snapshot_bytes = 0_u64; @@ -346,6 +350,9 @@ fn read_snapshot_with_clock( let mut server_id = None; loop { + if elapsed() >= MAX_SNAPSHOT_ELAPSED { + return Err(ReadError::Budget("elapsed-time")); + } if expected_total.is_some_and(|total| items.len() < total) && snapshot_bytes >= MAX_SNAPSHOT_BYTES { @@ -353,6 +360,9 @@ fn read_snapshot_with_clock( } let page = fetch_page(items.len())?; + if elapsed() >= MAX_SNAPSHOT_ELAPSED { + return Err(ReadError::Budget("elapsed-time")); + } if page.api_version != SUPPORTED_API_VERSION { return Err(ReadError::Contract("Zotero-API-Version")); } @@ -414,6 +424,9 @@ fn read_snapshot_with_clock( ); report.api_version = Some(SUPPORTED_API_VERSION); report.schema_version = schema_version; + if elapsed() >= MAX_SNAPSHOT_ELAPSED { + return Err(ReadError::Budget("elapsed-time")); + } Ok(report) } From a3d51e0970da8174be821dc3927ec0d9951ad9ef Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sun, 6 Sep 2026 18:39:21 +0900 Subject: [PATCH 22/33] docs(zotero): record bounded snapshot read evidence --- docs/PRD.md | 2 ++ docs/TRD.md | 4 ++- docs/adr/0006-zotero-research-intake.md | 2 ++ docs/doctoring/zotero_metadata_deadline.md | 30 ++++++++++++++++++++++ 4 files changed, 37 insertions(+), 1 deletion(-) create mode 100644 docs/doctoring/zotero_metadata_deadline.md diff --git a/docs/PRD.md b/docs/PRD.md index b70635b4..058a7467 100644 --- a/docs/PRD.md +++ b/docs/PRD.md @@ -56,6 +56,8 @@ All LLM-backed induction uses `contextual-orchestrator`. Model output is untrust ### FR-9 Research evidence intake +Library reads must finish within a bounded observation window or fail visibly without returning a partial classification. Slowly arriving pages cannot keep a run open indefinitely, and missing time budget must not be handled by silently dropping papers. + Read one immutable Zotero Local API library-version snapshot and propose exactly one research disposition for every top-level bibliographic item. Each proposal retains the item key/version, exact matched metadata values, rule revision, linked child records, and any model receipt. Weak evidence and evidence that matches multiple specific disposition families must abstain into steward review. Duplicate DOI/title identities are review candidates only: intake never merges, deletes, or silently mutates Zotero records. ## 6. First vertical slice diff --git a/docs/TRD.md b/docs/TRD.md index 535ee975..27628832 100644 --- a/docs/TRD.md +++ b/docs/TRD.md @@ -59,7 +59,9 @@ Evaluation must separate extraction recall, semantic correctness, structural cor ## 11. Zotero research intake -`conceptweave-zotero` reads only the loopback Local API with at most 100 records per page, an 8 MiB page-body limit, a 50,000-item whole-snapshot limit, a 256 MiB cumulative body limit, redirects disabled, and finite connect/response/body/global timeouts. Every request pins `Zotero-API-Version: 3`; every response must report API version 3 and schema version 42. Before another request is issued, exhausted whole-snapshot budgets fail closed. Before a parsed page is accumulated, checked item-count and byte arithmetic must remain within both the advertised total and the configured whole-snapshot budgets. `Total-Results`, `Last-Modified-Version`, Zotero version, and server identity must remain identical across all pages; contract drift, malformed JSON, an empty intermediate page, duplicate keys, or an oversized response fails the run. +`conceptweave-zotero` reads only the loopback Local API with at most 100 records per page, an 8 MiB page-body limit, a 50,000-item whole-snapshot limit, a 256 MiB cumulative body limit, redirects disabled, and finite connect/response/body/global timeouts. Every request pins `Zotero-API-Version: 3`; every response must report API version 3 and a present, recorded schema revision that stays unchanged during the snapshot, rather than the historical workstation schema 42. Before another request is issued, exhausted whole-snapshot budgets fail closed. Before a parsed page is accumulated, checked item-count and byte arithmetic must remain within both the advertised total and the configured whole-snapshot budgets. `Total-Results`, `Last-Modified-Version`, Zotero version, schema revision and server identity must remain identical across all pages; contract drift, malformed JSON, an empty intermediate page, duplicate keys, or an oversized response fails the run. + +One monotonic five-minute budget covers page admission and complete-report acceptance. At or beyond that limit, no new request starts and no late page or completed report is accepted. An already-started request retains the existing per-request timeouts; computation is not forcibly interrupted. Never return a partial classification to satisfy the time budget or discard legitimate short pages. On budget failure the caller receives an error, not a smaller successful denominator. Every top-level bibliographic record receives exactly one proposed disposition. `NeedsStewardReview` also records a deterministic abstention reason so missing classification metadata, vocabulary unsupported by the current deterministic rules, and present-but-unmatched metadata are distinguishable. DOI duplicate identity normalization treats bare DOI values, `doi:`, `doi.org`, and legacy `dx.doi.org` resolver forms as the same identity when their normalized DOI is equal. diff --git a/docs/adr/0006-zotero-research-intake.md b/docs/adr/0006-zotero-research-intake.md index b7d3a16d..3a304dff 100644 --- a/docs/adr/0006-zotero-research-intake.md +++ b/docs/adr/0006-zotero-research-intake.md @@ -21,6 +21,8 @@ Matched metadata values are copied into the local-only evidence receipt for repl The reader fails closed above 50,000 items or 256 MiB of cumulative response bodies, while retaining the 8 MiB per-page bound, finite request timeouts, redirect denial, total-count checks, snapshot-version checks, and duplicate-key detection. Pagination, consistency, resource-budget, and provider-contract behavior are separated from the narrow `ureq` transport so deterministic tests exercise the production reader core rather than excluding the entire reader from coverage. +In the context of reading every bibliographic source before classification, facing individually timely pages that can cumulatively hold a run open for days, we decided for a five-minute monotonic admission/completion budget in the existing reader and against rejecting legitimate short pages or adding another transport, to bound accepted work without excluding papers, accepting that an already-started request or classification computation can finish after the limit before its result is rejected. This is an application read limit, not a model timeout, hard process-cancellation deadline, wall-clock/suspend guarantee or atomic snapshot claim. Each page is checked before fetch and after return, and the complete report is checked before return. The stdlib clock has a private deterministic test seam; public APIs, provider timeouts and data/byte ceilings are unchanged. The [deadline doctoring](../doctoring/zotero_metadata_deadline.md) records the original review, RED/GREEN, alternatives and exact verification. This amendment remains Proposed and grants no Zotero mutation authority. + Report output is restricted to a new direct child of the operating system temporary directory. Relative paths, nested paths, existing files, and symlinks are rejected before write; the file is opened with create-new semantics so a path swap cannot cause repository or arbitrary-file overwrite. The buffered writer is explicitly flushed and a final filesystem error fails the command. Reports stay local and are never committed. No dedicated utility repository or Zotero mutation path is created. A future Zotero 10+ write adapter is a separate decision and must use authenticated loopback access, server identity, optimistic version preconditions, reviewed item-level changes, before/after receipts, and rollback evidence. diff --git a/docs/doctoring/zotero_metadata_deadline.md b/docs/doctoring/zotero_metadata_deadline.md new file mode 100644 index 00000000..afa9d5a6 --- /dev/null +++ b/docs/doctoring/zotero_metadata_deadline.md @@ -0,0 +1,30 @@ +# Zotero metadata read deadline + +Status: locally verified source repair; protected integration and forward-stack verification remain required. No actual library or paper artifact was read in this experiment. + +## Finding and cause + +[PR #9's unresolved review](https://github.com/ContextualWisdomLab/ConceptWeave/pull/9#discussion_r3935157013) identified that one item per page can trigger up to 50,000 requests, each with a fresh timeout. Exact baseline `a2a84884f67dcac6f6892c958d55450aea6d6c88` has item/byte bounds but no total elapsed-time bound. The same body remains in later research descendants. The real call path is `read_local_snapshot` → `read_snapshot_with` → page transport → complete `classify_snapshot`; the CLI receives a report only after that function returns. + +## Decision and limits + +Reuse that reader, `ReadError::Budget`, and `std::time::Instant`; add a private injected elapsed clock for deterministic tests. Check the five-minute budget before each fetch, after its return and after report classification. Choosing a maximum page count would reject valid short pages without bounding a few slow responses. Replacing transport or adding a cancellation service is unnecessary to deny late results. The trade-off is cooperative admission/completion: an in-flight request keeps its existing timeout and classification is not preempted. System-suspend accounting and monotonic clock implementation are platform-dependent (Rust Project Developers, n.d.). This does not establish provider authentication, source atomicity or any model timeout. + +The accepted report still includes the full observed denominator. No late/partial report escapes, no source is deleted or relabeled, and no transport, public signature, dependency, byte/item ceiling, approval or write authority changes. A longer successful observation window would require a separately evidenced operational decision; do not reduce the paper denominator to make a run pass. + +## Executed evidence + +- Baseline `a2a8488`: 38 tests / 10 unfiltered suites, including two doctests. +- Committed RED `aff539f`: private clock seam plus three regression functions, but no guard. Two rejection tests fail with an actual returned report; the valid short-page control passes. No sleep or real Zotero data is involved. +- GREEN `e6b2a2214b39106ddacc753595b72a699d53d04f`: 41 tests / 10 unfiltered suites, including two doctests; explicit Rust 1.98.0. The tests cover seven individually timely 50-second pages, zero-I/O expired admission, late page, between-page expiry, late classification, late empty library and success one nanosecond below the limit. +- Strict all-target Clippy, warnings-denied rustdoc, release build, formatting and existing CI contract pass. The unchanged coverage script passes 108/108 functions, 703/703 normalized source regions and 100/100 normalized branches. Raw LLVM remains below 100%: 1,051/1,052 lines, 1,600/1,606 regions and 99/100 branches. No threshold or exclusion changed. + +Reproduce with `cargo +1.98.0 test --workspace --locked` and `bash scripts/check_coverage.sh`. Logs are `/tmp/conceptweave-pr9-deadline-{baseline,red,green,coverage}-20260906.log`. The initial provider review references API pagination, which permits bounded pages; that does not supply an application-wide read deadline (Zotero, n.d.). The same TRD amendment removes the separately reviewed stale schema-42 requirement and states the implemented API-v3/present-stable-schema contract; no parser behavior changes for that documentation correction. + +Next: revalidate the final documentation head, normal-push #9 after a fresh writer/head/base check, merge its delta forward through every dependent research PR without reversing later features, and rerun each changed head's checks. Local success does not resolve protected approval, provider transport security or the other open findings. No predecessor may be closed to hide missing propagation. + +## References + +Rust Project Developers. (n.d.). *Instant in std::time* [Rust standard-library documentation]. Retrieved September 6, 2026, from https://doc.rust-lang.org/stable/std/time/struct.Instant.html + +Zotero. (n.d.). *Zotero Web API v3: Basics*. Retrieved September 6, 2026, from https://www.zotero.org/support/dev/web_api/v3/basics From bb2faccfda9efed55b6759f1bbf7907bf6ec0c3b Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sun, 6 Sep 2026 18:42:58 +0900 Subject: [PATCH 23/33] docs(zotero): note bounded metadata intake --- CHANGELOG.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8910d6fa..81923e3d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,10 @@ All notable changes to ConceptWeave are documented here. ## Unreleased +### Fixed + +- Zotero research intake rejects incomplete or late results after a five-minute read budget, even when individual pages arrive within their request limits. + ### Added - Initial ConceptWeave product, DDD, security, test, and operability baselines. From 1cf3472499ad49716a70603be2e15dc857819231 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sun, 6 Sep 2026 19:22:13 +0900 Subject: [PATCH 24/33] test(zotero): reproduce impossible item revision admission --- crates/conceptweave-zotero/src/lib.rs | 50 +++++++++++++++++++++++++++ 1 file changed, 50 insertions(+) diff --git a/crates/conceptweave-zotero/src/lib.rs b/crates/conceptweave-zotero/src/lib.rs index db5382a7..60850f99 100644 --- a/crates/conceptweave-zotero/src/lib.rs +++ b/crates/conceptweave-zotero/src/lib.rs @@ -882,6 +882,56 @@ mod tests { } } + #[test] + fn reader_rejects_future_item_revisions_before_another_page() { + for item_type in ["book", "attachment", "note", "annotation"] { + for invalid_start in [0, 1] { + let mut starts = Vec::new(); + let result = read_snapshot_with(&mut |start| { + starts.push(start); + let mut observed = item(&format!("I{start}"), "book", "x", "", ""); + observed.version = 42; + let mut page_items = vec![observed]; + if start == invalid_start { + let mut future = item(&format!("I{}", start + 1), item_type, "x", "", ""); + future.version = 43; + page_items.push(future); + } + Ok(fetched_page(4, page_items)) + }); + assert!(matches!(result, Err(ReadError::SnapshotChanged))); + assert_eq!( + starts, + if invalid_start == 0 { + vec![0] + } else { + vec![0, 1] + } + ); + } + } + } + + #[test] + fn reader_preserves_item_revisions_within_the_library_version() { + for zotero_version in ["9.0.6", "10.0.1"] { + for (library_version, item_version) in + [(0, 0), (42, 0), (42, 41), (42, 42), (u64::MAX, u64::MAX)] + { + let mut observed = item("A", "book", "semantic web", "", ""); + observed.version = item_version; + let mut page = fetched_page(1, vec![observed]); + page.library_version = library_version; + page.zotero_version = zotero_version.into(); + let report = read_snapshot_with(&mut |_| Ok(page.clone())).unwrap(); + assert_eq!(report.library_version, library_version); + assert_eq!(report.observed_item_count, 1); + assert_eq!(report.classified_items.len(), 1); + assert_eq!(report.classified_items[0].item_version, item_version); + } + } + } + #[test] fn reader_core_paginates_and_preserves_one_snapshot_contract() { let mut pages = vec![ From 8effa6a9b15ac1a09b7e80dab4cf2885fad02211 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sun, 6 Sep 2026 19:26:48 +0900 Subject: [PATCH 25/33] fix(zotero): reject item revisions beyond the library snapshot --- crates/conceptweave-zotero/src/lib.rs | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/crates/conceptweave-zotero/src/lib.rs b/crates/conceptweave-zotero/src/lib.rs index 60850f99..2c298b58 100644 --- a/crates/conceptweave-zotero/src/lib.rs +++ b/crates/conceptweave-zotero/src/lib.rs @@ -397,6 +397,13 @@ fn read_snapshot_with_clock( page.body_bytes, page.total, )?; + if page + .items + .iter() + .any(|item| item.version > page.library_version) + { + return Err(ReadError::SnapshotChanged); + } items.extend(page.items); snapshot_bytes = next_snapshot_bytes; debug_assert_eq!(items.len(), next_item_count); From f8566408e6a3017cf775fadf2a2f7e50b2d20dc6 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sun, 6 Sep 2026 19:30:02 +0900 Subject: [PATCH 26/33] docs(zotero): bind revision admission to provider evidence --- CHANGELOG.md | 1 + docs/PRD.md | 2 +- docs/TRD.md | 2 ++ docs/UML.md | 3 ++- docs/adr/0006-zotero-research-intake.md | 2 ++ docs/doctoring/zotero_item_revision.md | 35 +++++++++++++++++++++++++ 6 files changed, 43 insertions(+), 2 deletions(-) create mode 100644 docs/doctoring/zotero_item_revision.md diff --git a/CHANGELOG.md b/CHANGELOG.md index 81923e3d..e455c13e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,7 @@ All notable changes to ConceptWeave are documented here. ### Fixed +- Zotero research intake rejects a read whose records claim revisions newer than the library being observed, without dropping papers or changing their recorded revisions. - Zotero research intake rejects incomplete or late results after a five-minute read budget, even when individual pages arrive within their request limits. ### Added diff --git a/docs/PRD.md b/docs/PRD.md index 058a7467..f145c33a 100644 --- a/docs/PRD.md +++ b/docs/PRD.md @@ -58,7 +58,7 @@ All LLM-backed induction uses `contextual-orchestrator`. Model output is untrust Library reads must finish within a bounded observation window or fail visibly without returning a partial classification. Slowly arriving pages cannot keep a run open indefinitely, and missing time budget must not be handled by silently dropping papers. -Read one immutable Zotero Local API library-version snapshot and propose exactly one research disposition for every top-level bibliographic item. Each proposal retains the item key/version, exact matched metadata values, rule revision, linked child records, and any model receipt. Weak evidence and evidence that matches multiple specific disposition families must abstain into steward review. Duplicate DOI/title identities are review candidates only: intake never merges, deletes, or silently mutates Zotero records. +Read a complete Zotero Local API observation with one consistent library version and propose exactly one research disposition for every top-level bibliographic item. This consistency check does not establish an atomic provider snapshot. A record claiming a revision newer than the observed library invalidates the complete read; it must not be omitted or assigned a different revision to make the read pass. Each proposal retains the item key/version, exact matched metadata values, rule revision, linked child records, and any model receipt. Weak evidence and evidence that matches multiple specific disposition families must abstain into steward review. Duplicate DOI/title identities are review candidates only: intake never merges, deletes, or silently mutates Zotero records. ## 6. First vertical slice diff --git a/docs/TRD.md b/docs/TRD.md index 27628832..53fcba46 100644 --- a/docs/TRD.md +++ b/docs/TRD.md @@ -63,6 +63,8 @@ Evaluation must separate extraction recall, semantic correctness, structural cor One monotonic five-minute budget covers page admission and complete-report acceptance. At or beyond that limit, no new request starts and no late page or completed report is accepted. An already-started request retains the existing per-request timeouts; computation is not forcibly interrupted. Never return a partial classification to satisfy the time budget or discard legitimate short pages. On budget failure the caller receives an error, not a smaller successful denominator. +After count/byte validation and before accumulating each metadata page, every returned object's revision must be less than or equal to that page's library revision. This includes attachments, notes and annotations, not only bibliographic records. A higher revision returns the existing snapshot-consistency error immediately, with no next-page request or partial report. Zero, lower and equal revisions remain valid and are preserved exactly, including the unsigned maximum. Compare only within this metadata read of one local instance: Zotero 9's synced revisions and Zotero 10's local revisions are not interchangeable, and this condition is not a full-text endpoint contract or proof of atomicity. See the [revision admission evidence](doctoring/zotero_item_revision.md). + Every top-level bibliographic record receives exactly one proposed disposition. `NeedsStewardReview` also records a deterministic abstention reason so missing classification metadata, vocabulary unsupported by the current deterministic rules, and present-but-unmatched metadata are distinguishable. DOI duplicate identity normalization treats bare DOI values, `doi:`, `doi.org`, and legacy `dx.doi.org` resolver forms as the same identity when their normalized DOI is equal. The report is local JSON and contains proposals rather than governance decisions. CLI output is restricted to a new direct child of canonical `/tmp` or the operating system temporary directory; relative paths, nested paths, existing paths, and symlinks are rejected, and create-new file semantics prevent overwrite/path-swap writes. Zotero 9 writes are unsupported; no mutation path exists in this slice. A future Zotero 10+ writer requires a separate reviewed change with a Local API key, stable server identity, fresh item/library version preconditions, item-by-item before/after receipts, and rollback evidence. diff --git a/docs/UML.md b/docs/UML.md index 703a379b..46213a10 100644 --- a/docs/UML.md +++ b/docs/UML.md @@ -50,7 +50,8 @@ sequenceDiagram loop bounded pages Intake->>Zotero: read items at one library version - Zotero-->>Intake: items + immutable version headers + Zotero-->>Intake: items + observed library version + Intake->>Intake: validate page consistency and every item revision; fail entire read on mismatch end Intake->>Intake: classify or abstain; link children; find duplicate candidates Intake->>Report: write proposals and evidence diff --git a/docs/adr/0006-zotero-research-intake.md b/docs/adr/0006-zotero-research-intake.md index 3a304dff..eb44cb5e 100644 --- a/docs/adr/0006-zotero-research-intake.md +++ b/docs/adr/0006-zotero-research-intake.md @@ -23,6 +23,8 @@ The reader fails closed above 50,000 items or 256 MiB of cumulative response bod In the context of reading every bibliographic source before classification, facing individually timely pages that can cumulatively hold a run open for days, we decided for a five-minute monotonic admission/completion budget in the existing reader and against rejecting legitimate short pages or adding another transport, to bound accepted work without excluding papers, accepting that an already-started request or classification computation can finish after the limit before its result is rejected. This is an application read limit, not a model timeout, hard process-cancellation deadline, wall-clock/suspend guarantee or atomic snapshot claim. Each page is checked before fetch and after return, and the complete report is checked before return. The stdlib clock has a private deterministic test seam; public APIs, provider timeouts and data/byte ceilings are unchanged. The [deadline doctoring](../doctoring/zotero_metadata_deadline.md) records the original review, RED/GREEN, alternatives and exact verification. This amendment remains Proposed and grants no Zotero mutation authority. +In the context of retaining exact metadata provenance for every observed record, facing object revisions newer than the library revision of their own response, we decided for rejection in the shared metadata reader before accumulation and against filtering records, clamping revisions or extending the pure offline classifier's contract, to preserve the complete evidence denominator and original revisions, accepting that an inconsistent response requires a complete retry. This is a within-instance metadata condition, not a comparison across Zotero installations, a full-text version rule or an atomicity guarantee. Zero and equal versions are valid. The [revision doctoring](../doctoring/zotero_item_revision.md) records provider semantics, counterexamples and exact local checks. This amendment remains Proposed. + Report output is restricted to a new direct child of the operating system temporary directory. Relative paths, nested paths, existing files, and symlinks are rejected before write; the file is opened with create-new semantics so a path swap cannot cause repository or arbitrary-file overwrite. The buffered writer is explicitly flushed and a final filesystem error fails the command. Reports stay local and are never committed. No dedicated utility repository or Zotero mutation path is created. A future Zotero 10+ write adapter is a separate decision and must use authenticated loopback access, server identity, optimistic version preconditions, reviewed item-level changes, before/after receipts, and rollback evidence. diff --git a/docs/doctoring/zotero_item_revision.md b/docs/doctoring/zotero_item_revision.md new file mode 100644 index 00000000..5e4c82dc --- /dev/null +++ b/docs/doctoring/zotero_item_revision.md @@ -0,0 +1,35 @@ +# Zotero metadata item revision admission + +Status: locally verified prerequisite repair for PR #9; not pushed, protected, released or propagated to descendants at this checkpoint. No actual library, paper, model or Zotero mutation was exercised. Tests use synthetic unit inputs only. + +## Finding and provider contract + +[PR #9's review](https://github.com/ContextualWisdomLab/ConceptWeave/pull/9#discussion_r3934542708) identifies an item revision higher than its response's `Last-Modified-Version` being accepted into classification provenance. Baseline `bb2faccfda9efed55b6759f1bbf7907bf6ec0c3b` checks page-header agreement, counts, bytes, elapsed time and unique keys, but never compares a returned object's revision with its containing library revision. A later review or mutation precondition could therefore inherit impossible metadata coordinates even though the pages agree with each other. + +The Web API documents library revisions and object revisions separately: a multi-object items response carries the library revision; changed objects receive the library's updated revision. Revision numbers are opaque, monotonic and need not be consecutive (Zotero, 2022). The Local API documentation distinguishes Zotero 10's per-library local transaction revisions from Zotero 9's synced versions. Never-synced earlier objects can be version zero; Zotero 10 local revisions are comparable only within one instance and are unrelated to Web API revisions (Zotero, 2026). Together these rules support the within-response metadata invariant `item.version <= page.library_version`; they do not justify cross-instance ordering, rejecting zero, or applying this condition to full-text endpoints. + +## Decision and failure boundary + +The call path is `read_local_snapshot` → `read_snapshot_with` → `read_snapshot_with_clock` → page transport → `classify_snapshot`, followed by CLI report-file creation only after success. The shared reader now checks every page member after the existing resource validation and before `items.extend`. One `.any(...)` predicate returns the existing `ReadError::SnapshotChanged` if a member is too new. Bibliographic records and child attachments, notes and annotations take the same path. No later page is requested after rejection and no partial report is returned. + +Filtering offending records would change the denominator; clamping revisions would fabricate provenance. Changing the pure offline classifier would broaden its contract and still be later than the provider admission boundary. No new error type, helper, public signature, dependency or cross-product responsibility is needed. The added pass is linear in each bounded page and does not allocate another collection. The downside is that an inconsistent page invalidates the whole read, even if earlier pages were usable. Equal headers and valid member revisions still do not prove atomicity, server authentication or semantic correctness. ADR 0006 remains Proposed and no approval or write authority changes. + +## Executed checks + +- Baseline `bb2facc`: 41 tests / 10 unfiltered suites, including two doctests. +- Committed RED `1cf3472499ad49716a70603be2e15dc857819231`: the new rejection test fails at the expected `SnapshotChanged` assertion, not compilation or an unrelated error. Its valid-version control passes before the guard exists. +- Source GREEN `8effa6a9b15ac1a09b7e80dab4cf2885fad02211`: 43 tests / 10 unfiltered suites, including two doctests, under explicit Rust 1.98.0. Rejection covers first and subsequent pages and four item types, with a valid member before the offending member. Acceptance covers zero, lower, equal and `u64::MAX` revisions under Zotero 9 and 10 labels and checks retained revisions. +- All-target warnings-denied Clippy, warnings-denied rustdoc, release build, formatting, existing CI contract and diff validation pass. CodeGraph is healthy at 11 files, 206 nodes and 433 edges. +- Unchanged coverage gate passes: 113/113 functions, 709/709 normalized source regions and 106/106 normalized branches. Raw LLVM is not 100%: 1,097/1,098 lines, 1,690/1,697 regions and 105/106 branches. No gate, exclusion or fixture denominator was weakened. + +Reproduce with `cargo +1.98.0 test --workspace --locked` and `bash scripts/check_coverage.sh`. Local logs are `/tmp/conceptweave-pr9-item-revision-{baseline,red,green,clippy,rustdoc,release,coverage}-20260906.log`. + +## Integration gate + +The previous GitHub PR audit was rejected by the account GraphQL quota at 2026-09-06 10:05:51 UTC. This repair uses already-read review evidence and local source, not a new remote-state claim. Do not retry that audit before 11:06:12 UTC or substitute another endpoint/token/provider to bypass the limit. Then perform one normal fresh head/base/state/writer check, preserve concurrent delta through normal history, push the owner repair and merge it forward through each dependent research PR with fresh exact-head verification. This new propagation is outstanding; the earlier elapsed-deadline propagation does not include this revision guard. Required hosted checks, current-head independent review and protected integration remain separate gates. Do not close predecessors or claim actual paper decisions from these tests. + +## References + +Zotero. (2022, August 14). *Zotero Web API v3: Syncing*. https://www.zotero.org/support/dev/web_api/v3/syncing + +Zotero. (2026, July 29). *Zotero Local API*. https://www.zotero.org/support/dev/web_api/v3/local_api From 48dcd0de55ea3cc91def62fb70f20a6adb40da57 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sun, 6 Sep 2026 20:35:10 +0900 Subject: [PATCH 27/33] test(research): expose discarded nonbibliographic source scope --- crates/conceptweave-zotero/src/lib.rs | 109 ++++++++++++++++++++++++++ 1 file changed, 109 insertions(+) diff --git a/crates/conceptweave-zotero/src/lib.rs b/crates/conceptweave-zotero/src/lib.rs index 2c298b58..16be41de 100644 --- a/crates/conceptweave-zotero/src/lib.rs +++ b/crates/conceptweave-zotero/src/lib.rs @@ -1066,6 +1066,115 @@ mod tests { )); } + #[test] + fn source_inventory_retains_standalone_and_nested_metadata_without_proposals() { + let mut standalone = item("A", "attachment", "Ontology Learning", "10.1/X", ""); + standalone.version = 0; + standalone.data.collections = vec!["COLLECTION".into()]; + standalone.data.tags = vec![ItemTag { + tag: "Evidence".into(), + }]; + let report = read_snapshot_with(&mut |_| { + Ok(fetched_page( + 6, + vec![ + item("F", "annotation", "", "", "C"), + item("E", "note", "", "", ""), + item("D", "annotation", "", "", "A"), + item("C", "attachment", "", "", "B"), + item("B", "book", "Ontology Learning", "10.1/X", ""), + standalone.clone(), + ], + )) + }) + .unwrap(); + let serialized = serde_json::to_value(&report).unwrap(); + assert_eq!( + serialized["pending_source_item_keys"], + serde_json::json!(["A", "D", "E"]) + ); + let inventory = serialized["unclassified_items"] + .as_array() + .expect("every excluded source must survive in the report"); + assert_eq!( + inventory + .iter() + .map(|entry| entry["key"].as_str().unwrap()) + .collect::>(), + ["A", "C", "D", "E", "F"] + ); + assert_eq!(inventory[0]["version"], 0); + assert_eq!(inventory[0]["data"]["title"], "Ontology Learning"); + assert_eq!( + inventory[0]["data"]["collections"], + serde_json::json!(["COLLECTION"]) + ); + assert_eq!( + inventory[0]["data"]["tags"], + serde_json::json!([{"tag":"Evidence"}]) + ); + assert_eq!(inventory[4]["data"]["parentItem"], "C"); + assert_eq!(report.observed_item_count, 6); + assert_eq!(report.classified_items.len(), 1); + assert_eq!(report.classified_items[0].item_key, "B"); + assert_eq!(report.classified_items[0].child_item_keys, ["C"]); + assert!(report.duplicate_candidates.is_empty()); + } + + #[test] + fn source_inventory_keeps_orphans_cycles_and_unattached_annotations_pending() { + let sources = vec![ + item("A", "book", "", "", ""), + item("B", "note", "", "", "missing"), + item("C", "attachment", "", "", "B"), + item("D", "note", "", "", "E"), + item("E", "attachment", "", "", "D"), + item("F", "annotation", "", "", "F"), + item("G", "annotation", "", "", ""), + ]; + let forward = + serde_json::to_value(classify_snapshot("10".into(), None, 42, sources.clone())) + .unwrap(); + let reverse = serde_json::to_value(classify_snapshot( + "10".into(), + None, + 42, + sources.into_iter().rev().collect(), + )) + .unwrap(); + assert_eq!( + forward["pending_source_item_keys"], + serde_json::json!(["B", "C", "D", "E", "F", "G"]) + ); + assert_eq!(forward, reverse); + assert_eq!(forward["unclassified_items"].as_array().unwrap().len(), 6); + } + + #[test] + fn source_inventory_distinguishes_empty_and_fully_linked_evidence() { + for sources in [ + vec![], + vec![item("A", "book", "", "", "")], + vec![ + item("A", "book", "", "", ""), + item("B", "attachment", "", "", "A"), + item("C", "annotation", "", "", "B"), + ], + ] { + let serialized = + serde_json::to_value(classify_snapshot("10".into(), None, 42, sources)).unwrap(); + assert_eq!( + serialized["pending_source_item_keys"], + serde_json::json!([]) + ); + assert_eq!( + serialized["observed_item_count"].as_u64().unwrap() as usize, + serialized["classified_items"].as_array().unwrap().len() + + serialized["unclassified_items"].as_array().unwrap().len() + ); + } + } + #[test] fn classifies_every_bibliographic_item_and_links_children() { let mut generation = item("B", "journalArticle", "Ontology Learning", "10.1/X", ""); From 220f697edaa5f284e28a35140bf73161167d660a Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sun, 6 Sep 2026 20:35:47 +0900 Subject: [PATCH 28/33] fix(research): retain unclassified sources and unresolved ancestry --- crates/conceptweave-zotero/src/lib.rs | 39 +++++++++++++++++++++++++-- 1 file changed, 37 insertions(+), 2 deletions(-) diff --git a/crates/conceptweave-zotero/src/lib.rs b/crates/conceptweave-zotero/src/lib.rs index 16be41de..31c522d0 100644 --- a/crates/conceptweave-zotero/src/lib.rs +++ b/crates/conceptweave-zotero/src/lib.rs @@ -170,6 +170,19 @@ pub struct ClassificationReport { pub observed_item_count: usize, /// One proposal for every top-level bibliographic item. pub classified_items: Vec, + /// Metadata for every remaining record, sorted by its original key. + /// + /// Together with `classified_items`, this accounts for all observed items. + /// Notes, attachments and annotations remain evidence, not paper proposals. + /// Only the fields represented by `ItemData` are retained; this is not a + /// full-text capture or a lossless copy of the provider's original JSON. + pub unclassified_items: Vec, + /// Sorted keys whose parent chain does not reach a bibliographic proposal. + /// + /// Standalone sources, their descendants, orphan trees and cycles remain + /// pending. An empty list proves only parent-link accounting for this input, + /// never research completion, semantic approval or permission to write. + pub pending_source_item_keys: Vec, /// Reversible DOI/title duplicate candidates. pub duplicate_candidates: Vec, } @@ -473,10 +486,30 @@ pub fn classify_snapshot( let bibliographic: Vec<&ZoteroItem> = items.iter().filter(|item| is_bibliographic(item)).collect(); let duplicate_candidates = duplicate_candidates(&bibliographic); - let classified_items = bibliographic + let classified_items: Vec<_> = bibliographic .into_iter() .map(|item| classify_item(item, children.get(&item.key).cloned().unwrap_or_default())) .collect(); + let observed_item_count = items.len(); + let unclassified_items: Vec<_> = items + .into_iter() + .filter(|item| !is_bibliographic(item)) + .collect(); + let mut pending_source_item_keys: BTreeSet<_> = unclassified_items + .iter() + .map(|item| item.key.clone()) + .collect(); + let mut parent_item_keys: Vec<_> = classified_items + .iter() + .map(|item| item.item_key.clone()) + .collect(); + while let Some(parent_item_key) = parent_item_keys.pop() { + for child_item_key in children.get(&parent_item_key).into_iter().flatten() { + if pending_source_item_keys.remove(child_item_key) { + parent_item_keys.push(child_item_key.clone()); + } + } + } ClassificationReport { zotero_version, @@ -485,8 +518,10 @@ pub fn classify_snapshot( server_id, library_version, rule_revision: RULE_REVISION, - observed_item_count: items.len(), + observed_item_count, classified_items, + unclassified_items, + pending_source_item_keys: pending_source_item_keys.into_iter().collect(), duplicate_candidates, } } From 48c3525b0061dfba7552f1648f5ad5028b653ab8 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sun, 6 Sep 2026 20:36:46 +0900 Subject: [PATCH 29/33] refactor(research): consume each source adjacency list once --- crates/conceptweave-zotero/src/lib.rs | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/crates/conceptweave-zotero/src/lib.rs b/crates/conceptweave-zotero/src/lib.rs index 31c522d0..ec32efa5 100644 --- a/crates/conceptweave-zotero/src/lib.rs +++ b/crates/conceptweave-zotero/src/lib.rs @@ -482,7 +482,7 @@ pub fn classify_snapshot( mut items: Vec, ) -> ClassificationReport { items.sort_by(|left, right| left.key.cmp(&right.key)); - let children = child_index(&items); + let mut children = child_index(&items); let bibliographic: Vec<&ZoteroItem> = items.iter().filter(|item| is_bibliographic(item)).collect(); let duplicate_candidates = duplicate_candidates(&bibliographic); @@ -504,10 +504,9 @@ pub fn classify_snapshot( .map(|item| item.item_key.clone()) .collect(); while let Some(parent_item_key) = parent_item_keys.pop() { - for child_item_key in children.get(&parent_item_key).into_iter().flatten() { - if pending_source_item_keys.remove(child_item_key) { - parent_item_keys.push(child_item_key.clone()); - } + for child_item_key in children.remove(&parent_item_key).unwrap_or_default() { + pending_source_item_keys.remove(&child_item_key); + parent_item_keys.push(child_item_key); } } From f7a67bf86900dc11a9078fd93623cc1cf011901a Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sun, 6 Sep 2026 20:40:22 +0900 Subject: [PATCH 30/33] test(research): reject missing source identity at reader admission --- crates/conceptweave-zotero/src/lib.rs | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/crates/conceptweave-zotero/src/lib.rs b/crates/conceptweave-zotero/src/lib.rs index ec32efa5..05776974 100644 --- a/crates/conceptweave-zotero/src/lib.rs +++ b/crates/conceptweave-zotero/src/lib.rs @@ -1100,6 +1100,33 @@ mod tests { )); } + #[test] + fn reader_rejects_blank_source_identity_before_another_page() { + for blank_key in ["", " ", "\t\n"] { + for invalid_start in [0, 1] { + let mut starts = Vec::new(); + let result = read_snapshot_with(&mut |start| { + starts.push(start); + let key = if start == invalid_start { + blank_key + } else { + "A" + }; + Ok(fetched_page(3, vec![item(key, "attachment", "", "", "")])) + }); + assert!(matches!(result, Err(ReadError::SnapshotChanged))); + assert_eq!( + starts, + if invalid_start == 0 { + vec![0] + } else { + vec![0, 1] + } + ); + } + } + } + #[test] fn source_inventory_retains_standalone_and_nested_metadata_without_proposals() { let mut standalone = item("A", "attachment", "Ontology Learning", "10.1/X", ""); From 3a57f3bb48722e1bc93468f39a82c2b46124ffde Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sun, 6 Sep 2026 20:41:15 +0900 Subject: [PATCH 31/33] test(research): isolate blank identity from duplicate-key rejection --- crates/conceptweave-zotero/src/lib.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/crates/conceptweave-zotero/src/lib.rs b/crates/conceptweave-zotero/src/lib.rs index 05776974..e16fb168 100644 --- a/crates/conceptweave-zotero/src/lib.rs +++ b/crates/conceptweave-zotero/src/lib.rs @@ -1108,11 +1108,11 @@ mod tests { let result = read_snapshot_with(&mut |start| { starts.push(start); let key = if start == invalid_start { - blank_key + blank_key.to_owned() } else { - "A" + format!("A{start}") }; - Ok(fetched_page(3, vec![item(key, "attachment", "", "", "")])) + Ok(fetched_page(3, vec![item(&key, "attachment", "", "", "")])) }); assert!(matches!(result, Err(ReadError::SnapshotChanged))); assert_eq!( From 1e95d6eb979e66ecb7dae4f81f18a6b0a91b7624 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sun, 6 Sep 2026 20:41:42 +0900 Subject: [PATCH 32/33] fix(research): reject blank identity before source accumulation --- crates/conceptweave-zotero/src/lib.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/conceptweave-zotero/src/lib.rs b/crates/conceptweave-zotero/src/lib.rs index e16fb168..519ad7e3 100644 --- a/crates/conceptweave-zotero/src/lib.rs +++ b/crates/conceptweave-zotero/src/lib.rs @@ -413,7 +413,7 @@ fn read_snapshot_with_clock( if page .items .iter() - .any(|item| item.version > page.library_version) + .any(|item| item.key.trim().is_empty() || item.version > page.library_version) { return Err(ReadError::SnapshotChanged); } From 51c7df6d03f072449422fd58ca24b2f9d6026f07 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sun, 6 Sep 2026 20:44:43 +0900 Subject: [PATCH 33/33] docs(research): bind source inventory evidence and pending admission gates --- CHANGELOG.md | 1 + docs/PRD.md | 2 + docs/TRD.md | 6 +++ docs/UML.md | 4 +- docs/adr/0006-zotero-research-intake.md | 6 +++ docs/doctoring/zotero_source_scope.md | 54 +++++++++++++++++++++++++ docs/product-technical-gap-baseline.md | 8 +++- 7 files changed, 79 insertions(+), 2 deletions(-) create mode 100644 docs/doctoring/zotero_source_scope.md diff --git a/CHANGELOG.md b/CHANGELOG.md index e455c13e..74d053c8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,7 @@ All notable changes to ConceptWeave are documented here. ### Fixed +- Research reports retain standalone files and notes that previously disappeared from the classification view, and flag sources whose parent relationships remain unresolved. - Zotero research intake rejects a read whose records claim revisions newer than the library being observed, without dropping papers or changing their recorded revisions. - Zotero research intake rejects incomplete or late results after a five-minute read budget, even when individual pages arrive within their request limits. diff --git a/docs/PRD.md b/docs/PRD.md index f145c33a..3d12e5c5 100644 --- a/docs/PRD.md +++ b/docs/PRD.md @@ -56,6 +56,8 @@ All LLM-backed induction uses `contextual-orchestrator`. Model output is untrust ### FR-9 Research evidence intake +Preserve every observed source, including standalone files and notes outside the bibliographic proposals. Keep unresolved source relationships visible instead of treating a completed bibliography worksheet as a completed library review. All standalone sources and records without a valid path to a bibliographic parent need explicit reconciliation; notes, files and annotations must not acquire paper labels from their titles. Retraction and correction evidence remains separate from topic classification and approval. The current producer retains this inventory; downstream reconciliation, completion admission and independent governance remain required, not implemented by inventory generation alone. + Library reads must finish within a bounded observation window or fail visibly without returning a partial classification. Slowly arriving pages cannot keep a run open indefinitely, and missing time budget must not be handled by silently dropping papers. Read a complete Zotero Local API observation with one consistent library version and propose exactly one research disposition for every top-level bibliographic item. This consistency check does not establish an atomic provider snapshot. A record claiming a revision newer than the observed library invalidates the complete read; it must not be omitted or assigned a different revision to make the read pass. Each proposal retains the item key/version, exact matched metadata values, rule revision, linked child records, and any model receipt. Weak evidence and evidence that matches multiple specific disposition families must abstain into steward review. Duplicate DOI/title identities are review candidates only: intake never merges, deletes, or silently mutates Zotero records. diff --git a/docs/TRD.md b/docs/TRD.md index 53fcba46..08a59938 100644 --- a/docs/TRD.md +++ b/docs/TRD.md @@ -59,6 +59,12 @@ Evaluation must separate extraction recall, semantic correctness, structural cor ## 11. Zotero research intake +`ClassificationReport.unclassified_items` retains every input record excluded from bibliographic classification, using the existing `ZoteroItem` metadata projection. Bibliographic proposals and this inventory are disjoint and together account for the observed record count on reader-admitted input. The existing child index is consumed once from bibliographic roots; records never reached remain in sorted `pending_source_item_keys`, including standalone roots, their descendants, orphan trees and cycles. The traversal is iterative, uses no new dependency and costs O(n log n) time/O(n) auxiliary space. It does not validate arbitrary offline input or preserve note bodies, attachment-specific fields and unknown provider JSON. + +Consumer requirement, still pending forward integration: require both inventory fields rather than defaulting absent legacy fields to empty; validate exact snapshot identity/version/parent complement and recompute pending keys before any review, duplicate evaluation or write verifier. Keep bibliographic progress distinct from whole-library completion. An empty pending list is only ancestry accounting, never semantic approval. Full-text report-digest changes require fresh bound verification, not capture rewriting; metadata proposal-only digests do not implicitly bind the new inventory. See [source-scope evidence and integration map](doctoring/zotero_source_scope.md). + +The shared live reader rejects empty or whitespace-only item keys before accumulating a page or requesting another one. It preserves valid keys verbatim; this does not add a new provider key-format restriction or certify arbitrary offline classifier input. + `conceptweave-zotero` reads only the loopback Local API with at most 100 records per page, an 8 MiB page-body limit, a 50,000-item whole-snapshot limit, a 256 MiB cumulative body limit, redirects disabled, and finite connect/response/body/global timeouts. Every request pins `Zotero-API-Version: 3`; every response must report API version 3 and a present, recorded schema revision that stays unchanged during the snapshot, rather than the historical workstation schema 42. Before another request is issued, exhausted whole-snapshot budgets fail closed. Before a parsed page is accumulated, checked item-count and byte arithmetic must remain within both the advertised total and the configured whole-snapshot budgets. `Total-Results`, `Last-Modified-Version`, Zotero version, schema revision and server identity must remain identical across all pages; contract drift, malformed JSON, an empty intermediate page, duplicate keys, or an oversized response fails the run. One monotonic five-minute budget covers page admission and complete-report acceptance. At or beyond that limit, no new request starts and no late page or completed report is accepted. An already-started request retains the existing per-request timeouts; computation is not forcibly interrupted. Never return a partial classification to satisfy the time budget or discard legitimate short pages. On budget failure the caller receives an error, not a smaller successful denominator. diff --git a/docs/UML.md b/docs/UML.md index 46213a10..714c3004 100644 --- a/docs/UML.md +++ b/docs/UML.md @@ -54,7 +54,9 @@ sequenceDiagram Intake->>Intake: validate page consistency and every item revision; fail entire read on mismatch end Intake->>Intake: classify or abstain; link children; find duplicate candidates - Intake->>Report: write proposals and evidence + Intake->>Intake: retain excluded metadata; traverse parent links from bibliographic roots + Intake->>Report: write proposals, complete inventory and unresolved source keys + Note over Report,Steward: Pending sources prevent a whole-library completion claim; inventory is not approval Report->>Steward: review dispositions and merge candidates Intake-->>Zotero: no mutation ``` diff --git a/docs/adr/0006-zotero-research-intake.md b/docs/adr/0006-zotero-research-intake.md index eb44cb5e..b0b0a81a 100644 --- a/docs/adr/0006-zotero-research-intake.md +++ b/docs/adr/0006-zotero-research-intake.md @@ -31,6 +31,12 @@ No dedicated utility repository or Zotero mutation path is created. A future Zot ## Consequences +### September 6 source-scope amendment (Proposed) + +In the context of a genuine 8,326-record library observation and a native 3,719-top-level selection, facing three standalone PDFs and one note omitted from the 3,715 bibliographic proposals, we decided to retain every nonbibliographic metadata record and derive unresolved ancestry using the existing child index. We rejected silently excluding these sources, guessing paper labels from file titles, and a standalone-only list that would still hide orphan or cyclic child relationships. Reusing `ZoteroItem` avoids another adapter or DTO, at the cost of a larger private report and an explicitly projected, not lossless, source representation. Consuming each adjacency list once makes traversal finite without recursion or repeated ancestry walks. The [executed source-scope record](../doctoring/zotero_source_scope.md) binds the failing tests, source, actual observation and consumer integration requirements. + +This change does not reconcile a source, validate arbitrary offline input, retain full note/file contents or grant review/write authority. Consumers must require and validate the new inventory, recompute pending keys and distinguish bibliographic progress from full-library completion before adoption. Direct evaluation, duplicate and write routes must not bypass that validation. Existing proposal-only approval digests do not bind this added scope; incompatible full-text captures must fail their whole-report binding. The schema and governance integration remain pending in dependent owners; no backward-compatible empty default, capture rewrite or approved-label backfill is accepted. The original Zotero 9.0.6 observation above is historical; this amendment was tested against Zotero 10.0.1 while retaining read-only behavior. + - A complete snapshot can be audited and replayed without changing the research library. - Rule evidence and explicit abstention reasons are visible; automated classification is not governance approval. - Cross-cutting papers cannot be silently forced into whichever rule family happens to be evaluated first. diff --git a/docs/doctoring/zotero_source_scope.md b/docs/doctoring/zotero_source_scope.md new file mode 100644 index 00000000..9aa6d5d6 --- /dev/null +++ b/docs/doctoring/zotero_source_scope.md @@ -0,0 +1,54 @@ +# Zotero complete metadata inventory and pending source scope + +Status: verified producer repair in the existing PR #9 owner lane. Dependent report admission, governance binding and whole-library completion remain incomplete. No approval, protected merge, model call or Zotero mutation is claimed. + +## Observed failure and source contract + +The [actual September 6 native/API inspection](https://github.com/ContextualWisdomLab/ConceptWeave/blob/55b1d91dcd299145239a62062ed504fcb6e7bfd1/docs/doctoring/zotero_metadata_visual_audit.md) found 3,719 selected top-level records but only 3,715 bibliographic proposals. Complete attachment/note response audits identified three standalone PDFs and one note outside that worksheet. They are not proven to be three distinct additional papers. Their identities, contents and relationships must not be guessed, discarded or merged. + +Zotero documents all-item and top-level endpoints separately and supports both standalone and child attachments (Zotero, n.d.-a, n.d.-b). Actual Zotero 10.0.1 responses to the filtered `/items/top` diagnostic included children; endpoint naming and header totals therefore did not prove standalone scope. DeepWiki's repository answer described intended `noChildren` behavior but did not explain the observed filtered response. That contradiction remains a provider investigation, not a reason to override actual records or assume a fixed upstream bug. This implementation uses the existing complete `/items` reader, not a new filtered adapter. Context7's previously exhausted allowance was not bypassed; official primary documentation and actual responses support this increment. + +## Root cause and minimal repair + +`read_local_snapshot` → `read_snapshot_with` → `read_snapshot_with_clock` admits the complete bounded metadata read, then calls `classify_snapshot`; the CLI serializes that report. Previously `is_bibliographic` excluded notes, attachments, annotations and records with parents. A scalar total and direct child keys survived, but the other records' metadata and unresolved ancestry disappeared. Matching all 3,715 worksheet slots could therefore be mistaken for accounting for the whole library. + +The producer now moves every excluded record into `unclassified_items`, reusing `ZoteroItem`. Sorted bibliographic proposals plus sorted inventory account for every reader-admitted identity. The existing parent-to-children index is consumed iteratively starting at each bibliographic root. Each adjacency list is removed once; records not reached remain in sorted `pending_source_item_keys`. This retains standalone roots and their descendants, missing-parent trees, disconnected cycles and self-cycles without recursion, another transport or a dependency. Time is O(n log n), auxiliary memory O(n); the private report grows because records previously dropped are now retained. + +No pending record receives a paper disposition or enters bibliographic DOI/title duplicate candidates. Empty pending keys mean only that the observed parent graph reaches bibliographic roots. They do not establish correct semantics, genuine review, independent approval, complete full-text capture, atomicity or write permission. `ItemData` is a metadata projection: note HTML, PDF bytes, attachment-specific fields and unknown provider fields are not preserved by this serialization. Keep original evidence/captures separately and do not advertise a lossless source backup. + +Independent review also found blank source keys admitted by the reader. The existing shared page predicate now rejects an empty/whitespace-only key before accumulation, alongside future item revisions. It does not trim or rewrite valid keys, impose a new key-format regex or change the infallible offline classifier's existing contract. Duplicate keys remain rejected after the complete read; arbitrary offline input is not certified merely because classification terminates. + +## Committed experiments and verification + +- Baseline `f8566408e6a3017cf775fadf2a2f7e50b2d20dc6`: 43 tests / 10 unfiltered suites, including two doctests. +- Inventory RED `48dcd0d`: all three new tests fail on absent pending inventory, not compilation. They cover standalone metadata, nested children, deterministic ordering, orphan/cycle/self-cycle paths, empty input and fully linked evidence. +- Inventory GREEN `220f697`, then adjacency-consumption simplification `48c3525b0061dfba7552f1648f5ad5028b653ab8`: 46 / 10 tests pass. An independent agent reran the three focused tests on the latter exact source and reported no blocking producer finding on reader-admitted input. +- Identity RED `f7a67bf` detects a third request before rejection; its fixture accidentally also repeated a valid key. Refined committed RED `3a57f3b` uses unique remaining keys and fails the actual expected rejection assertion, isolating the missing-identity defect. +- Final source `1e95d6eb979e66ecb7dae4f81f18a6b0a91b7624`: **47 tests / 10 unfiltered suites**, including two doctests; strict Clippy, warnings-denied rustdoc and release build pass. The identity test covers empty/space/control-whitespace on first and subsequent pages with no next request after rejection. +- The unchanged coverage gate passes **123/123 functions, 751/751 normalized source regions and 114/114 normalized branches**. Raw LLVM is not 100%: 1,231/1,232 lines, 1,985/1,993 regions and 113/114 branches. No threshold, exclusion or dependency was changed. Coverage and final verification logs are `/tmp/conceptweave-source-scope-final-{tests,clippy,rustdoc,release,coverage}-20260906.log`. + +## Genuine library replay, distinct from unit evidence + +The release executable built from `48c3525` completed a read-only actual Local API run in **17.61 seconds**. Its SHA-256 was `04e12299babb63c58dd32736373cca8c5a72a9f02aa20ed5366f01e55b935ee7`. This run precedes the later blank-key guard; it is not a run of the final `1e95d6e` executable. + +The private new report `/private/tmp/conceptweave-source-scope-live-C98C52A4-1D9A-4528-BFE8-6A2EBD1AAE98.json` is a single-link regular `0600` file, 3,731,468 bytes, SHA-256 `7a6cd9f7f90a052964f60b5152cd12dc5928d7187ad41e808a0c215bab3b3b97`. It retains **8,326 = 3,715 proposals + 4,611 nonbibliographic records** and exactly **four pending keys**. The prior private attachment/note audit's four identities match the pending list exactly. All 8,326 combined identities are unique and nonblank; all retained nonbibliographic revisions are within library revision 2. The previous report's entire bibliographic proposal list, server identifier and library revision compare equal. Zotero 10.0.1/API 3/schema 44 are unchanged. Only aggregate checks are published; no original titles, keys, server identity, note body or screenshot is committed. + +This is one elapsed-time observation, not a controlled performance improvement claim. No full text was recaptured, no old capture was rewritten, and no bibliographic proposal was promoted to an authentic decision. Existing worksheet decisions and independently approved labels remain 0/3,715, with four unresolved sources additionally visible. + +## Visual Inspection and adoption gates + +This turn attempted native Zotero inspection; the computer-use tool reported that the Mac was locked and automatic unlock failed. No lock bypass, fresh screenshot or new visual pass is claimed. The previous actual screenshot of 3,719 selected items and a retraction warning remains historical evidence; the previous one-item retracted view was accessibility-only because later frames were stale. Unlocking is required for another current visual inspection. API and unit success do not substitute for it. + +The root consumer audit at `22a29c1cfc0918fa34287f3bffe7f400e97f4a0f` identifies the next required integration: + +1. Require both inventory fields on report restoration; missing/null legacy inventory must not become empty. Validate uniqueness, disjointness, exact snapshot complement, original key/version/parent/type evidence and recompute pending ancestry. `SnapshotItemRevision` currently lacks item type. +2. Use shared admission in `build_steward_review_worksheet`, plus direct `prepare_reviewed_golden_set`, `build_duplicate_merge_review_manifest` and `prepare_classification_write_plan`, before any external verifier. These latter routes currently bypass worksheet admission. +3. Keep `StewardReviewProgress.complete`'s bibliographic meaning separate from full-library scope status. Pending-source reconciliation needs bound evidence and its own explicit status; do not mark children as additional unreviewed papers or turn an empty pending list into approval. +4. `classification_proposal_digest` v1 binds only classified proposals, not this inventory. Full-text captures bind the whole report and must reject incompatible restored reports. Never rewrite a prior capture, default scope away or backfill approval identity to make old receipts pass. +5. Preserve parent/child history while normally forwarding the new item-revision, inventory and identity delta through existing owners. This producer is not already present in the root #39 runtime. Run each changed consumer's full tests and current-head hosted checks/reviews; prerequisites, protected rules and independent approval still apply. + +## References + +Zotero. (n.d.-a). *Zotero Web API documentation*. Retrieved September 6, 2026, from https://www.zotero.org/support/dev/web_api/v3/basics + +Zotero. (n.d.-b). *Adding files to your Zotero library*. Retrieved September 6, 2026, from https://www.zotero.org/support/attaching_files diff --git a/docs/product-technical-gap-baseline.md b/docs/product-technical-gap-baseline.md index f7ba7dea..5405f6de 100644 --- a/docs/product-technical-gap-baseline.md +++ b/docs/product-technical-gap-baseline.md @@ -4,7 +4,13 @@ This file records code-current product and technical gaps. Exact PR/check/run coordinates are evidence snapshots, not mutable-head dependencies. Live protected-branch, PR, issue and workflow state wins whenever it advances after this snapshot. Because this documentation update creates a Foundation successor, the Foundation SHA below is the exact pre-refresh head; PR metadata must be refreshed to the resulting successor SHA. -## Protected truth and active stack +## September 6 source inventory checkpoint + +The existing #9 owner now retains every nonbibliographic metadata record and derives unresolved ancestry rather than silently discarding standalone sources. [Source-scope doctoring](doctoring/zotero_source_scope.md) binds committed REDs, final source `1e95d6eb979e66ecb7dae4f81f18a6b0a91b7624`, **47 tests / 10 unfiltered suites**, strict checks and the unchanged coverage gate. The earlier inventory executable at `48c3525` genuinely reads 8,326 records into 3,715 unchanged bibliographic proposals plus 4,611 other records, with exactly the four previously audited standalone identities pending. A later shared-reader guard also rejects blank identities; no actual final-guard executable replay is implied. + +The earlier source findings are repaired locally, not yet propagated into root #39. Required downstream restoration/identity accounting, pending-source reconciliation, approval binding and full-library completion gates remain open; neither zero pending keys nor successful classification grants semantic or write authority. Current native Visual Inspection was attempted but the Mac is locked, so no new screenshot was verified. Historical source scope, authentic worksheet decisions/independent approvals 0/3,715, plus four unresolved sources remain distinct. This checkpoint does not refresh every historical PR coordinate below or imply protected merge/release. + +## Historical protected truth and active stack Protected/default `main` remains `f4f440dd58c77d7cd90dff8a1eb2eeb9a9940425`; only the bootstrap state is shipped there and no immutable ConceptWeave release exists.