diff --git a/CHANGELOG.md b/CHANGELOG.md index 8910d6fa..74d053c8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,12 @@ All notable changes to ConceptWeave are documented here. ## Unreleased +### Fixed + +- Research reports retain standalone files and notes that previously disappeared from the classification view, and flag sources whose parent relationships remain unresolved. +- Zotero research intake rejects a read whose records claim revisions newer than the library being observed, without dropping papers or changing their recorded revisions. +- Zotero research intake rejects incomplete or late results after a five-minute read budget, even when individual pages arrive within their request limits. + ### Added - Initial ConceptWeave product, DDD, security, test, and operability baselines. diff --git a/Cargo.lock b/Cargo.lock index 451324f0..dd4cfe80 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2,6 +2,182 @@ # It is not intended for manual editing. version = 4 +[[package]] +name = "base64" +version = "0.23.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac07cdecf99051d9a5238b80f35af32cdeba5b336e55d957b318b50137e18da5" + +[[package]] +name = "bytes" +version = "1.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc652a48c352aef3ea3aed32080501cf3ef6ed5da78602a020c991775b0aff04" + [[package]] name = "conceptweave-domain" version = "0.1.0" + +[[package]] +name = "conceptweave-zotero" +version = "0.1.0" +dependencies = [ + "serde", + "serde_json", + "ureq", +] + +[[package]] +name = "http" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "918d3568bebf352712bc2ef3d46a8bcf1a75b373be6539de198e9105cbbf9ce0" +dependencies = [ + "bytes", + "itoa", +] + +[[package]] +name = "httparse" +version = "1.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6dbf3de79e51f3d586ab4cb9d5c3e2c14aa28ed23d180cf89b4df0454a69cc87" + +[[package]] +name = "itoa" +version = "1.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" + +[[package]] +name = "log" +version = "0.4.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f9f8bd3e56ce4dfc153cf470fffbfa98c7620958b312ca5c3a4b8d5181fd13c6" + +[[package]] +name = "memchr" +version = "2.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98" + +[[package]] +name = "percent-encoding" +version = "2.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220" + +[[package]] +name = "proc-macro2" +version = "1.0.107" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "985e7ec9bb745e6ce6535b544d84d6cd6f7ad8bd711c398938ae983b91a766d9" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "quote" +version = "1.0.47" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fbf4db142a473a8d80c26bbf18454ed458bf8d26c8219c331daecfdbd079001" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "serde" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4148590afebada386688f18773da617792bf2ef03ffc1e4cbd2b1d45b023e0ba" +dependencies = [ + "serde_core", + "serde_derive", +] + +[[package]] +name = "serde_core" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67dca2c9c51e58a4791a4b1ed58308b39c64224d349a935ab5039aa360942a48" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7a5d71263a5a7d47b41f6b3f06ba276f10cc18b0931f1799f710578e2309348" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "serde_json" +version = "1.0.151" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c841b55ecdae098c80dcae9cf767f6f8a0c2cdb3416bbef72181df4d0fe73f14" +dependencies = [ + "itoa", + "memchr", + "serde", + "serde_core", + "zmij", +] + +[[package]] +name = "syn" +version = "3.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6275cddf4610d1775e6d1fe9469b2e77d0f39fd98fb7450901b821e0c53649f" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "unicode-ident" +version = "1.0.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" + +[[package]] +name = "ureq" +version = "3.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "972d7902c8735f2695410b8aed7df6ed12a47394aa1c8d7af49f0497b731a94d" +dependencies = [ + "base64", + "log", + "percent-encoding", + "ureq-proto", + "utf8-zero", +] + +[[package]] +name = "ureq-proto" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da5f78b09e6941e1a0f2e30e695e4b120377b54d5e0aec11b594bb57b3971613" +dependencies = [ + "base64", + "http", + "httparse", + "log", +] + +[[package]] +name = "utf8-zero" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8c0a043c9540bae7c578c88f91dda8bd82e59ae27c21baca69c8b191aaf5a6e" + +[[package]] +name = "zmij" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29666d0abbfad1e3dc4dcf6144730dd3a3ab225bbbdac83319345b1b44ccfc1b" diff --git a/Cargo.toml b/Cargo.toml index 0eec8e8c..ee34b8a9 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,5 +1,5 @@ [workspace] -members = ["crates/conceptweave-domain"] +members = ["crates/conceptweave-domain", "crates/conceptweave-zotero"] resolver = "2" [workspace.package] diff --git a/README.md b/README.md index afb42260..046d2544 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,15 @@ # ConceptWeave +## Local Zotero research proposal + +With Zotero running locally: + +```sh +cargo +1.98.0 run --bin conceptweave-zotero -- /tmp/conceptweave-zotero-classification.json +``` + +The command reads one stable library-version snapshot and creates a local, reviewable JSON report. Output is restricted to a new direct child of canonical `/tmp` or the system temporary directory, and the command never changes Zotero records. + [![Ask DeepWiki](https://deepwiki.com/badge.svg)](https://deepwiki.com/ContextualWisdomLab/ConceptWeave) **Automatic, evidence-bound ontology and semantic-layer engineering for governed enterprise meaning.** diff --git a/crates/conceptweave-zotero/Cargo.toml b/crates/conceptweave-zotero/Cargo.toml new file mode 100644 index 00000000..7f78f15f --- /dev/null +++ b/crates/conceptweave-zotero/Cargo.toml @@ -0,0 +1,23 @@ +[package] +name = "conceptweave-zotero" +version.workspace = true +edition.workspace = true +rust-version.workspace = true +repository.workspace = true +license.workspace = true +description = "Read-only Zotero research classification for ConceptWeave" + +[dependencies] +serde = { version = "1", features = ["derive"] } +serde_json = "1" +ureq = { version = "3", default-features = false } + +[lib] +path = "src/lib.rs" + +[[bin]] +name = "conceptweave-zotero" +path = "src/main.rs" + +[lints.rust] +unexpected_cfgs = { level = "warn", check-cfg = ["cfg(coverage_nightly)"] } diff --git a/crates/conceptweave-zotero/src/lib.rs b/crates/conceptweave-zotero/src/lib.rs new file mode 100644 index 00000000..519ad7e3 --- /dev/null +++ b/crates/conceptweave-zotero/src/lib.rs @@ -0,0 +1,1403 @@ +#![forbid(unsafe_code)] +#![deny(missing_docs)] +#![cfg_attr(coverage_nightly, feature(coverage_attribute))] +//! Deterministic, read-only classification of a Zotero library snapshot. + +use serde::{Deserialize, Serialize}; +use std::collections::{BTreeMap, BTreeSet}; +use std::fmt; +use std::io::Read; +use std::time::{Duration, Instant}; + +/// Classification rule revision recorded in every report. +pub const RULE_REVISION: &str = "ontology-research-v2"; + +const SUPPORTED_API_VERSION: u64 = 3; +const SUPPORTED_API_VERSION_HEADER: &str = "3"; +const PAGE_LIMIT: usize = 100; +const MAX_PAGE_BYTES: u64 = 8 * 1024 * 1024; +const MAX_SNAPSHOT_ITEMS: usize = 50_000; +const MAX_SNAPSHOT_BYTES: u64 = 256 * 1024 * 1024; +const MAX_SNAPSHOT_ELAPSED: Duration = Duration::from_secs(300); +const LOCAL_API: &str = "http://127.0.0.1:23119/api/users/0/items"; + +#[cfg(test)] +static TEST_LOCAL_API: std::sync::Mutex> = std::sync::Mutex::new(None); + +/// A Zotero item returned by the Local API. +#[derive(Debug, Clone, Deserialize, Serialize)] +pub struct ZoteroItem { + /// Stable item key. + pub key: String, + /// Item revision. + pub version: u64, + /// Item metadata. + pub data: ItemData, +} + +/// Metadata used by the classifier. +#[derive(Debug, Clone, Deserialize, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct ItemData { + /// Zotero item type. + pub item_type: String, + /// Display title when present. + #[serde(default)] + pub title: String, + /// Abstract when present. + #[serde(default)] + pub abstract_note: String, + /// DOI when present. + #[serde(default, rename = "DOI")] + pub doi: String, + /// Parent item key for notes and attachments. + #[serde(default)] + pub parent_item: String, + /// Collection keys. + #[serde(default)] + pub collections: Vec, + /// Tags applied to the item. + #[serde(default)] + pub tags: Vec, +} + +/// A Zotero item tag. +#[derive(Debug, Clone, Deserialize, Serialize)] +pub struct ItemTag { + /// Tag text. + pub tag: String, +} + +/// One mutually exclusive proposed disposition. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)] +#[serde(rename_all = "snake_case")] +pub enum Disposition { + /// Evidence about ontology or taxonomy generation. + Generation, + /// Evidence about alignment, matching, evolution, or versioning. + AlignmentVersioning, + /// Evidence about semantic consumption or query bridges. + SemanticConsumptionBridge, + /// Evidence about evaluation, validation, or governance. + EvaluationGovernance, + /// Ontology-adjacent evidence without a narrower match. + AdjacentEvidence, + /// Explicitly reviewed as outside the program scope. + OutOfScope, + /// No deterministic rule supplies enough evidence for a narrower proposal. + NeedsStewardReview, +} + +/// Deterministic reason that a bibliographic item requires steward review. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)] +#[serde(rename_all = "snake_case")] +pub enum AbstentionReason { + /// Title, abstract, and tags contain no classification metadata. + MissingClassificationMetadata, + /// Metadata uses vocabulary outside the current deterministic rule set. + UnsupportedRuleVocabulary, + /// Metadata is present but no deterministic rule phrase matches it. + NoDeterministicRuleMatch, + /// More than one specific disposition family is supported by the item. + ConflictingDispositionEvidence, +} + +/// Evidence for a deterministic proposed disposition. +#[derive(Debug, Serialize)] +pub struct ClassificationEvidence { + /// Metadata fields whose values matched. + pub fields: Vec<&'static str>, + /// Exact snapshot values for matched fields, retained only in the local report. + pub field_values: BTreeMap<&'static str, String>, + /// Rule phrases found in those fields. + pub matched_phrases: Vec<&'static str>, +} + +/// A single top-level bibliographic classification proposal. +#[derive(Debug, Serialize)] +pub struct ClassifiedItem { + /// Stable Zotero item key. + pub item_key: String, + /// Item revision observed in this snapshot. + pub item_version: u64, + /// Zotero item type. + pub item_type: String, + /// Human-readable title retained in the local report only. + pub title: String, + /// Collection keys observed with the item. + pub collection_keys: Vec, + /// Tag text observed with the item. + pub tags: Vec, + /// Proposed disposition; never an authoritative governance decision. + pub proposed_disposition: Disposition, + /// Deterministic reason for abstention, absent when a rule proposes a disposition. + pub abstention_reason: Option, + /// Deterministic supporting evidence. + pub evidence: ClassificationEvidence, + /// Child note and attachment keys linked to the top-level item. + pub child_item_keys: Vec, + /// Model receipt is absent because this slice performs no model call. + pub model_receipt: Option, +} + +/// A duplicate candidate group; no item is merged or deleted. +#[derive(Debug, Serialize)] +pub struct DuplicateCandidate { + /// Identity kind used for the candidate group. + pub identity_kind: &'static str, + /// Normalized identity value. + pub normalized_identity: String, + /// Zotero item keys sharing the identity. + pub item_keys: Vec, +} + +/// Complete local classification report for one immutable library version. +#[derive(Debug, Serialize)] +pub struct ClassificationReport { + /// Zotero desktop version that served the snapshot. + pub zotero_version: String, + /// Requested and observed Local API version for a live read. + pub api_version: Option, + /// Zotero schema revision observed consistently across a live read. + pub schema_version: Option, + /// Local API server identifier observed on every page when supplied. + pub server_id: Option, + /// Library version shared by every fetched page. + pub library_version: u64, + /// Rule revision used for all proposals. + pub rule_revision: &'static str, + /// Number of items read, including child notes and attachments. + pub observed_item_count: usize, + /// One proposal for every top-level bibliographic item. + pub classified_items: Vec, + /// Metadata for every remaining record, sorted by its original key. + /// + /// Together with `classified_items`, this accounts for all observed items. + /// Notes, attachments and annotations remain evidence, not paper proposals. + /// Only the fields represented by `ItemData` are retained; this is not a + /// full-text capture or a lossless copy of the provider's original JSON. + pub unclassified_items: Vec, + /// Sorted keys whose parent chain does not reach a bibliographic proposal. + /// + /// Standalone sources, their descendants, orphan trees and cycles remain + /// pending. An empty list proves only parent-link accounting for this input, + /// never research completion, semantic approval or permission to write. + pub pending_source_item_keys: Vec, + /// Reversible DOI/title duplicate candidates. + pub duplicate_candidates: Vec, +} + +/// Failure raised when a bounded, immutable Local API read cannot be proven. +#[derive(Debug)] +pub enum ReadError { + /// Network or HTTP protocol failure. + Http(String), + /// Required response header is absent or invalid. + Header(&'static str), + /// Provider contract is present but unsupported. + Contract(&'static str), + /// A later page did not belong to the first page's snapshot. + SnapshotChanged, + /// Configured whole-snapshot resource budget was exceeded. + Budget(&'static str), + /// Zotero returned malformed JSON. + Json(serde_json::Error), + /// Response body exceeded the configured bound or could not be read. + Body(String), +} + +impl fmt::Display for ReadError { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::Http(error) => write!(formatter, "local API request failed: {error}"), + Self::Header(name) => write!(formatter, "local API response lacks valid {name}"), + Self::Contract(name) => write!(formatter, "local API returned unsupported {name}"), + Self::SnapshotChanged => write!(formatter, "Zotero library changed during the read"), + Self::Budget(kind) => write!(formatter, "Zotero snapshot exceeds {kind} budget"), + Self::Json(error) => write!(formatter, "local API returned invalid JSON: {error}"), + Self::Body(error) => write!(formatter, "local API response body failed: {error}"), + } + } +} + +impl std::error::Error for ReadError {} + +#[derive(Debug, Clone)] +struct FetchedPage { + total: usize, + library_version: u64, + zotero_version: String, + api_version: u64, + schema_version: u64, + server_id: Option, + body_bytes: u64, + items: Vec, +} + +/// Reads every Zotero item from one stable Local API library version. +/// +/// Snapshot consistency, resource budgets, API-version validation, pagination, +/// and duplicate-key checks live in an injectable reader core. Only the narrow +/// ureq transport shim is excluded from deterministic coverage. +/// No page starts or completed report is accepted at or beyond five minutes. +/// An in-flight request may finish later under its existing per-request limits; +/// its late result is rejected, not returned as a partial snapshot. +pub fn read_local_snapshot() -> Result { + let config = ureq::Agent::config_builder() + .proxy(None) + .timeout_global(Some(Duration::from_secs(60))) + .timeout_connect(Some(Duration::from_secs(2))) + .timeout_recv_response(Some(Duration::from_secs(10))) + .timeout_recv_body(Some(Duration::from_secs(10))) + .max_redirects(0) + .build(); + let agent = ureq::Agent::new_with_config(config); + read_snapshot_with(&mut |start| fetch_local_page(&agent, start)) +} + +fn local_api_base() -> String { + #[cfg(test)] + { + if let Some(value) = TEST_LOCAL_API + .lock() + .expect("test Local API lock must not be poisoned") + .clone() + { + return value; + } + } + LOCAL_API.to_owned() +} + +#[cfg_attr(coverage_nightly, coverage(off))] +fn fetch_local_page(agent: &ureq::Agent, start: usize) -> Result { + let url = format!( + "{}?format=json&include=data&limit={PAGE_LIMIT}&start={start}", + local_api_base() + ); + let mut response = agent + .get(&url) + .header("Zotero-API-Version", SUPPORTED_API_VERSION_HEADER) + .call() + .map_err(|error| ReadError::Http(error.to_string()))?; + let headers = response.headers(); + let total = header_u64(headers, "Total-Results")?; + let total = usize::try_from(total).map_err(|_| ReadError::Budget("item-count"))?; + let library_version = header_u64(headers, "Last-Modified-Version")?; + let zotero_version = header_string(headers, "X-Zotero-Version")?; + let api_version = header_u64(headers, "Zotero-API-Version")?; + let schema_version = header_u64(headers, "Zotero-Schema-Version")?; + let server_id = optional_header(headers, "Zotero-Server-ID"); + + let body = read_bounded_response_text(&mut response, MAX_PAGE_BYTES) + .map_err(|error| ReadError::Body(error.to_string()))?; + let body_bytes = u64::try_from(body.len()).map_err(|_| ReadError::Budget("byte-count"))?; + let items = serde_json::from_str(&body).map_err(ReadError::Json)?; + + Ok(FetchedPage { + total, + library_version, + zotero_version, + api_version, + schema_version, + server_id, + body_bytes, + items, + }) +} + +/// Reads strict UTF-8 with an inclusive byte limit and one byte of overrun evidence. +fn read_bounded_response_text( + response: &mut ureq::http::Response, + limit: u64, +) -> std::io::Result { + let mut body = String::new(); + response + .body_mut() + .as_reader() + .take(limit.saturating_add(1)) + .read_to_string(&mut body)?; + if body.len() as u64 > limit { + return Err(std::io::Error::new( + std::io::ErrorKind::InvalidData, + "response body exceeds byte limit", + )); + } + Ok(body) +} + +#[cfg_attr(coverage_nightly, coverage(off))] +fn header_u64(headers: &ureq::http::HeaderMap, name: &'static str) -> Result { + header_string(headers, name)? + .parse() + .map_err(|_| ReadError::Header(name)) +} + +#[cfg_attr(coverage_nightly, coverage(off))] +fn header_string(headers: &ureq::http::HeaderMap, name: &'static str) -> Result { + optional_header(headers, name).ok_or(ReadError::Header(name)) +} + +#[cfg_attr(coverage_nightly, coverage(off))] +fn optional_header(headers: &ureq::http::HeaderMap, name: &'static str) -> Option { + headers.get(name)?.to_str().ok().map(str::to_owned) +} + +fn read_snapshot_with( + fetch_page: &mut dyn FnMut(usize) -> Result, +) -> Result { + let started = Instant::now(); + read_snapshot_with_clock(fetch_page, &mut || started.elapsed()) +} + +fn read_snapshot_with_clock( + fetch_page: &mut dyn FnMut(usize) -> Result, + elapsed: &mut dyn FnMut() -> Duration, +) -> Result { + let mut items = Vec::new(); + let mut snapshot_bytes = 0_u64; + let mut expected_total = None; + let mut library_version = None; + let mut zotero_version = None; + let mut schema_version = None; + let mut server_id = None; + + loop { + if elapsed() >= MAX_SNAPSHOT_ELAPSED { + return Err(ReadError::Budget("elapsed-time")); + } + if expected_total.is_some_and(|total| items.len() < total) + && snapshot_bytes >= MAX_SNAPSHOT_BYTES + { + return Err(ReadError::Budget("whole-snapshot")); + } + + let page = fetch_page(items.len())?; + if elapsed() >= MAX_SNAPSHOT_ELAPSED { + return Err(ReadError::Budget("elapsed-time")); + } + if page.api_version != SUPPORTED_API_VERSION { + return Err(ReadError::Contract("Zotero-API-Version")); + } + if page.total > MAX_SNAPSHOT_ITEMS { + return Err(ReadError::Budget("item-count")); + } + + if let Some(expected) = expected_total { + if expected != page.total + || library_version != Some(page.library_version) + || zotero_version.as_ref() != Some(&page.zotero_version) + || schema_version != Some(page.schema_version) + || server_id != page.server_id + { + return Err(ReadError::SnapshotChanged); + } + } else { + expected_total = Some(page.total); + library_version = Some(page.library_version); + zotero_version = Some(page.zotero_version.clone()); + schema_version = Some(page.schema_version); + server_id = page.server_id.clone(); + } + + if page.items.is_empty() && items.len() < page.total { + return Err(ReadError::SnapshotChanged); + } + let (next_item_count, next_snapshot_bytes) = checked_snapshot_usage( + items.len(), + page.items.len(), + snapshot_bytes, + page.body_bytes, + page.total, + )?; + if page + .items + .iter() + .any(|item| item.key.trim().is_empty() || item.version > page.library_version) + { + return Err(ReadError::SnapshotChanged); + } + items.extend(page.items); + snapshot_bytes = next_snapshot_bytes; + debug_assert_eq!(items.len(), next_item_count); + + if items.len() == page.total { + break; + } + } + + if items + .iter() + .map(|item| &item.key) + .collect::>() + .len() + != items.len() + { + return Err(ReadError::SnapshotChanged); + } + + let mut report = classify_snapshot( + zotero_version.expect("a completed snapshot has Zotero version metadata"), + server_id, + library_version.expect("a completed snapshot has library version metadata"), + items, + ); + report.api_version = Some(SUPPORTED_API_VERSION); + report.schema_version = schema_version; + if elapsed() >= MAX_SNAPSHOT_ELAPSED { + return Err(ReadError::Budget("elapsed-time")); + } + Ok(report) +} + +fn checked_snapshot_usage( + current_items: usize, + page_items: usize, + current_bytes: u64, + page_bytes: u64, + advertised_total: usize, +) -> Result<(usize, u64), ReadError> { + if advertised_total > MAX_SNAPSHOT_ITEMS { + return Err(ReadError::Budget("item-count")); + } + let next_items = current_items.saturating_add(page_items); + if next_items > MAX_SNAPSHOT_ITEMS { + return Err(ReadError::Budget("item-count")); + } + if next_items > advertised_total { + return Err(ReadError::SnapshotChanged); + } + let next_bytes = current_bytes.saturating_add(page_bytes); + if next_bytes > MAX_SNAPSHOT_BYTES { + return Err(ReadError::Budget("byte-count")); + } + Ok((next_items, next_bytes)) +} + +/// Classifies an already captured snapshot without network access. +pub fn classify_snapshot( + zotero_version: String, + server_id: Option, + library_version: u64, + mut items: Vec, +) -> ClassificationReport { + items.sort_by(|left, right| left.key.cmp(&right.key)); + let mut children = child_index(&items); + let bibliographic: Vec<&ZoteroItem> = + items.iter().filter(|item| is_bibliographic(item)).collect(); + let duplicate_candidates = duplicate_candidates(&bibliographic); + let classified_items: Vec<_> = bibliographic + .into_iter() + .map(|item| classify_item(item, children.get(&item.key).cloned().unwrap_or_default())) + .collect(); + let observed_item_count = items.len(); + let unclassified_items: Vec<_> = items + .into_iter() + .filter(|item| !is_bibliographic(item)) + .collect(); + let mut pending_source_item_keys: BTreeSet<_> = unclassified_items + .iter() + .map(|item| item.key.clone()) + .collect(); + let mut parent_item_keys: Vec<_> = classified_items + .iter() + .map(|item| item.item_key.clone()) + .collect(); + while let Some(parent_item_key) = parent_item_keys.pop() { + for child_item_key in children.remove(&parent_item_key).unwrap_or_default() { + pending_source_item_keys.remove(&child_item_key); + parent_item_keys.push(child_item_key); + } + } + + ClassificationReport { + zotero_version, + api_version: None, + schema_version: None, + server_id, + library_version, + rule_revision: RULE_REVISION, + observed_item_count, + classified_items, + unclassified_items, + pending_source_item_keys: pending_source_item_keys.into_iter().collect(), + duplicate_candidates, + } +} + +fn is_bibliographic(item: &ZoteroItem) -> bool { + item.data.parent_item.is_empty() + && !matches!( + item.data.item_type.as_str(), + "attachment" | "note" | "annotation" + ) +} + +fn child_index(items: &[ZoteroItem]) -> BTreeMap> { + let mut index: BTreeMap> = BTreeMap::new(); + for item in items + .iter() + .filter(|item| !item.data.parent_item.is_empty()) + { + index + .entry(item.data.parent_item.clone()) + .or_default() + .push(item.key.clone()); + } + index +} + +fn classify_item(item: &ZoteroItem, child_item_keys: Vec) -> ClassifiedItem { + let title_normalized = item.data.title.to_lowercase(); + let abstract_normalized = item.data.abstract_note.to_lowercase(); + let tags_original = item + .data + .tags + .iter() + .map(|tag| tag.tag.as_str()) + .collect::>() + .join(" "); + let tags_normalized = tags_original.to_lowercase(); + let fields = [ + ("title", title_normalized.as_str(), item.data.title.as_str()), + ( + "abstract_note", + abstract_normalized.as_str(), + item.data.abstract_note.as_str(), + ), + ("tags", tags_normalized.as_str(), tags_original.as_str()), + ]; + + let specific_rules = [ + ( + Disposition::AlignmentVersioning, + &[ + "ontology alignment", + "ontology matching", + "ontology mapping", + "ontology evolution", + "ontology versioning", + ][..], + ), + ( + Disposition::Generation, + &[ + "ontology learning", + "ontology extraction", + "ontology generation", + "taxonomy induction", + "knowledge graph construction", + ][..], + ), + ( + Disposition::SemanticConsumptionBridge, + &[ + "semantic layer", + "semantic model", + "ontology-based data access", + "knowledge graph query", + "linked data", + ][..], + ), + ( + Disposition::EvaluationGovernance, + &[ + "ontology evaluation", + "ontology quality", + "ontology validation", + "ontology governance", + "competency question", + "shacl", + ][..], + ), + ]; + let adjacent_phrases = [ + "ontology", + "semantic web", + "knowledge graph", + "rdf", + "owl", + "skos", + ]; + + let mut matched_dispositions = Vec::new(); + let mut matched_fields = BTreeSet::new(); + let mut field_values = BTreeMap::new(); + let mut matched_phrases = BTreeSet::new(); + + for (candidate, phrases) in specific_rules { + let mut family_matched = false; + for (field, normalized, original) in fields { + for phrase in phrases { + if contains_phrase(normalized, phrase) { + family_matched = true; + matched_fields.insert(field); + field_values + .entry(field) + .or_insert_with(|| original.to_owned()); + matched_phrases.insert(*phrase); + } + } + } + if family_matched { + matched_dispositions.push(candidate); + } + } + + let (proposed_disposition, abstention_reason) = match matched_dispositions.as_slice() { + [] => { + for (field, normalized, original) in fields { + for phrase in adjacent_phrases { + if contains_phrase(normalized, phrase) { + matched_fields.insert(field); + field_values + .entry(field) + .or_insert_with(|| original.to_owned()); + matched_phrases.insert(phrase); + } + } + } + if matched_phrases.is_empty() { + ( + Disposition::NeedsStewardReview, + Some(classify_abstention_reason(&fields)), + ) + } else { + (Disposition::AdjacentEvidence, None) + } + } + [single] => (*single, None), + _ => ( + Disposition::NeedsStewardReview, + Some(AbstentionReason::ConflictingDispositionEvidence), + ), + }; + + ClassifiedItem { + item_key: item.key.clone(), + item_version: item.version, + item_type: item.data.item_type.clone(), + title: item.data.title.clone(), + collection_keys: item.data.collections.clone(), + tags: item.data.tags.iter().map(|tag| tag.tag.clone()).collect(), + proposed_disposition, + abstention_reason, + evidence: ClassificationEvidence { + fields: matched_fields.into_iter().collect(), + field_values, + matched_phrases: matched_phrases.into_iter().collect(), + }, + child_item_keys, + model_receipt: None, + } +} + +fn classify_abstention_reason(fields: &[(&'static str, &str, &str)]) -> AbstentionReason { + if fields + .iter() + .all(|(_, normalized, _)| normalized.trim().is_empty()) + { + return AbstentionReason::MissingClassificationMetadata; + } + if fields.iter().any(|(_, _, original)| { + original + .chars() + .any(|character| character.is_alphabetic() && !character.is_ascii()) + }) { + return AbstentionReason::UnsupportedRuleVocabulary; + } + AbstentionReason::NoDeterministicRuleMatch +} + +fn contains_phrase(value: &str, phrase: &str) -> bool { + value.match_indices(phrase).any(|(start, matched)| { + let before = value[..start].chars().next_back(); + let after = value[start + matched.len()..].chars().next(); + before.is_none_or(|character| !character.is_alphanumeric()) + && after.is_none_or(|character| !character.is_alphanumeric()) + }) +} + +fn duplicate_candidates(items: &[&ZoteroItem]) -> Vec { + let mut identities: BTreeMap<(&'static str, String), Vec> = BTreeMap::new(); + for item in items { + if let Some(doi) = normalize_doi(&item.data.doi) { + identities + .entry(("doi", doi)) + .or_default() + .push(item.key.clone()); + } + if let Some(title) = normalize_title(&item.data.title) { + identities + .entry(("title", title)) + .or_default() + .push(item.key.clone()); + } + } + identities + .into_iter() + .filter_map(|((identity_kind, normalized_identity), item_keys)| { + (item_keys.len() > 1).then_some(DuplicateCandidate { + identity_kind, + normalized_identity, + item_keys, + }) + }) + .collect() +} + +fn normalize_doi(value: &str) -> Option { + let normalized = value.trim().to_lowercase(); + let normalized = normalized + .strip_prefix("https://doi.org/") + .or_else(|| normalized.strip_prefix("http://doi.org/")) + .or_else(|| normalized.strip_prefix("https://dx.doi.org/")) + .or_else(|| normalized.strip_prefix("http://dx.doi.org/")) + .or_else(|| normalized.strip_prefix("doi:")) + .unwrap_or(&normalized) + .trim(); + (!normalized.is_empty()).then(|| normalized.to_owned()) +} + +fn normalize_title(value: &str) -> Option { + let mut normalized = String::new(); + let mut separated = true; + for character in value.trim().to_lowercase().chars() { + if character.is_alphanumeric() { + normalized.push(character); + separated = false; + } else if !separated { + normalized.push(' '); + separated = true; + } + } + if normalized.ends_with(' ') { + normalized.pop(); + } + (!normalized.is_empty()).then_some(normalized) +} + +#[cfg(test)] +mod tests { + mod metadata_transport; + + use super::*; + use std::io::{Read, Write}; + use std::net::TcpListener; + use std::thread; + + static LOCAL_API_TEST_LOCK: std::sync::Mutex<()> = std::sync::Mutex::new(()); + + fn item(key: &str, item_type: &str, title: &str, doi: &str, parent: &str) -> ZoteroItem { + ZoteroItem { + key: key.into(), + version: 7, + data: ItemData { + item_type: item_type.into(), + title: title.into(), + abstract_note: String::new(), + doi: doi.into(), + parent_item: parent.into(), + collections: vec![], + tags: vec![], + }, + } + } + + fn fetched_page(total: usize, items: Vec) -> FetchedPage { + FetchedPage { + total, + library_version: 42, + zotero_version: "9.0.6".into(), + api_version: 3, + schema_version: 42, + server_id: Some("server".into()), + body_bytes: 100, + items, + } + } + + #[test] + fn production_wrapper_requests_api_v3_and_records_contract_versions() { + let _guard = LOCAL_API_TEST_LOCK.lock().unwrap(); + let listener = TcpListener::bind("127.0.0.1:0").unwrap(); + let address = listener.local_addr().unwrap(); + let base = format!("http://{address}/api/users/0/items"); + *TEST_LOCAL_API.lock().unwrap() = Some(base); + + let server = thread::spawn(move || { + let (mut stream, _) = listener.accept().unwrap(); + let mut request = vec![0_u8; 4096]; + let read = stream.read(&mut request).unwrap(); + let request = String::from_utf8_lossy(&request[..read]).to_lowercase(); + assert!(request.contains("zotero-api-version: 3")); + let body = r#"[{"key":"A","version":1,"data":{"itemType":"book","title":"ontology evaluation"}}]"#; + write!( + stream, + "HTTP/1.1 200 OK\r\nContent-Length: {}\r\nTotal-Results: 1\r\nLast-Modified-Version: 42\r\nX-Zotero-Version: 9.0.6\r\nZotero-API-Version: 3\r\nZotero-Schema-Version: 42\r\nZotero-Server-ID: server\r\nConnection: close\r\n\r\n{}", + body.len(), + body + ) + .unwrap(); + stream.flush().unwrap(); + }); + + let report = read_local_snapshot().unwrap(); + *TEST_LOCAL_API.lock().unwrap() = None; + server.join().unwrap(); + + assert_eq!(report.api_version, Some(3)); + assert_eq!(report.schema_version, Some(42)); + assert_eq!(report.library_version, 42); + assert_eq!(report.classified_items.len(), 1); + assert_eq!(local_api_base(), LOCAL_API); + } + + #[test] + fn reader_deadline_rejects_slow_drip_without_another_request() { + let elapsed = std::cell::Cell::new(Duration::ZERO); + let mut starts = Vec::new(); + let result = read_snapshot_with_clock( + &mut |start| { + starts.push(start); + elapsed.set(elapsed.get() + Duration::from_secs(50)); + Ok(fetched_page( + 7, + vec![item(&format!("P{start}"), "book", "semantic web", "", "")], + )) + }, + &mut || elapsed.get(), + ); + assert!(matches!(result, Err(ReadError::Budget("elapsed-time")))); + assert_eq!(starts, vec![0, 1, 2, 3, 4, 5]); + } + + #[test] + fn reader_deadline_rejects_expired_admission_page_and_report() { + // Expired before first I/O, after a page, before next I/O, and after + // classifying the final page; no partial or late report may escape. + for (ticks, total, expected_calls) in [ + (vec![300], 1, 0), + (vec![0, 301], 1, 1), + (vec![0, 0, 300], 2, 1), + (vec![0, 0, 300], 1, 1), + (vec![0, 300], 0, 1), + ] { + let mut ticks = ticks.into_iter(); + let mut calls = 0; + let result = read_snapshot_with_clock( + &mut |start| { + calls += 1; + let items = if total == 0 { + vec![] + } else { + vec![item(&format!("P{start}"), "book", "x", "", "")] + }; + Ok(fetched_page(total, items)) + }, + &mut || Duration::from_secs(ticks.next().unwrap()), + ); + assert!(matches!(result, Err(ReadError::Budget("elapsed-time")))); + assert_eq!(calls, expected_calls); + } + } + + #[test] + fn reader_deadline_accepts_complete_short_pages_before_limit() { + for total in [0, 2] { + let report = read_snapshot_with_clock( + &mut |start| { + let items = if total == 0 { + vec![] + } else { + vec![item(&format!("P{start}"), "book", "semantic web", "", "")] + }; + Ok(fetched_page(total, items)) + }, + &mut || Duration::from_secs(300) - Duration::from_nanos(1), + ) + .unwrap(); + assert_eq!(report.observed_item_count, total); + assert_eq!(report.classified_items.len(), total); + assert_eq!(report.library_version, 42); + } + } + + #[test] + fn reader_rejects_future_item_revisions_before_another_page() { + for item_type in ["book", "attachment", "note", "annotation"] { + for invalid_start in [0, 1] { + let mut starts = Vec::new(); + let result = read_snapshot_with(&mut |start| { + starts.push(start); + let mut observed = item(&format!("I{start}"), "book", "x", "", ""); + observed.version = 42; + let mut page_items = vec![observed]; + if start == invalid_start { + let mut future = item(&format!("I{}", start + 1), item_type, "x", "", ""); + future.version = 43; + page_items.push(future); + } + Ok(fetched_page(4, page_items)) + }); + assert!(matches!(result, Err(ReadError::SnapshotChanged))); + assert_eq!( + starts, + if invalid_start == 0 { + vec![0] + } else { + vec![0, 1] + } + ); + } + } + } + + #[test] + fn reader_preserves_item_revisions_within_the_library_version() { + for zotero_version in ["9.0.6", "10.0.1"] { + for (library_version, item_version) in + [(0, 0), (42, 0), (42, 41), (42, 42), (u64::MAX, u64::MAX)] + { + let mut observed = item("A", "book", "semantic web", "", ""); + observed.version = item_version; + let mut page = fetched_page(1, vec![observed]); + page.library_version = library_version; + page.zotero_version = zotero_version.into(); + let report = read_snapshot_with(&mut |_| Ok(page.clone())).unwrap(); + assert_eq!(report.library_version, library_version); + assert_eq!(report.observed_item_count, 1); + assert_eq!(report.classified_items.len(), 1); + assert_eq!(report.classified_items[0].item_version, item_version); + } + } + } + + #[test] + fn reader_core_paginates_and_preserves_one_snapshot_contract() { + let mut pages = vec![ + fetched_page(2, vec![item("A", "book", "ontology quality", "", "")]), + fetched_page(2, vec![item("B", "book", "semantic web", "", "")]), + ] + .into_iter(); + let report = read_snapshot_with(&mut |_| Ok(pages.next().unwrap())).unwrap(); + assert_eq!(report.observed_item_count, 2); + assert_eq!(report.api_version, Some(3)); + assert_eq!(report.schema_version, Some(42)); + + let empty = read_snapshot_with(&mut |_| Ok(fetched_page(0, vec![]))).unwrap(); + assert_eq!(empty.observed_item_count, 0); + } + + #[test] + fn reader_core_rejects_contract_drift_empty_pages_and_duplicate_keys() { + let mut unsupported = fetched_page(1, vec![item("A", "book", "x", "", "")]); + unsupported.api_version = 4; + assert!(matches!( + read_snapshot_with(&mut |_| Ok(unsupported.clone())), + Err(ReadError::Contract("Zotero-API-Version")) + )); + + assert!(matches!( + read_snapshot_with(&mut |_| Err(ReadError::Http("offline".into()))), + Err(ReadError::Http(_)) + )); + + for changed in ["total", "library", "zotero", "schema", "server"] { + let mut second = fetched_page(2, vec![item("B", "book", "y", "", "")]); + match changed { + "total" => second.total = 3, + "library" => second.library_version += 1, + "zotero" => second.zotero_version = "10.0".into(), + "schema" => second.schema_version += 1, + "server" => second.server_id = Some("other".into()), + _ => unreachable!(), + } + let mut pages = vec![ + fetched_page(2, vec![item("A", "book", "x", "", "")]), + second, + ] + .into_iter(); + assert!(matches!( + read_snapshot_with(&mut |_| Ok(pages.next().unwrap())), + Err(ReadError::SnapshotChanged) + )); + } + + assert!(matches!( + read_snapshot_with(&mut |_| Ok(fetched_page(1, vec![]))), + Err(ReadError::SnapshotChanged) + )); + + assert!(matches!( + read_snapshot_with(&mut |_| { + Ok(fetched_page( + 2, + vec![ + item("A", "book", "x", "", ""), + item("A", "book", "y", "", ""), + ], + )) + }), + Err(ReadError::SnapshotChanged) + )); + } + + #[test] + fn reader_core_rejects_total_and_between_request_resource_exhaustion() { + let too_many = fetched_page(MAX_SNAPSHOT_ITEMS + 1, vec![]); + assert!(matches!( + read_snapshot_with(&mut |_| Ok(too_many.clone())), + Err(ReadError::Budget("item-count")) + )); + + let mut calls = 0; + assert!(matches!( + read_snapshot_with(&mut |_| { + calls += 1; + let mut page = fetched_page(2, vec![item("A", "book", "x", "", "")]); + page.body_bytes = MAX_SNAPSHOT_BYTES; + Ok(page) + }), + Err(ReadError::Budget("whole-snapshot")) + )); + assert_eq!(calls, 1); + + let mut oversized = fetched_page(1, vec![item("A", "book", "x", "", "")]); + oversized.body_bytes = MAX_SNAPSHOT_BYTES + 1; + assert!(matches!( + read_snapshot_with(&mut |_| Ok(oversized.clone())), + Err(ReadError::Budget("byte-count")) + )); + } + + #[test] + fn snapshot_usage_is_bounded_before_accumulation() { + assert_eq!(checked_snapshot_usage(1, 1, 10, 20, 2).unwrap(), (2, 30)); + assert!(matches!( + checked_snapshot_usage(0, 1, 0, 1, MAX_SNAPSHOT_ITEMS + 1), + Err(ReadError::Budget("item-count")) + )); + assert!(matches!( + checked_snapshot_usage(MAX_SNAPSHOT_ITEMS, 1, 0, 1, MAX_SNAPSHOT_ITEMS), + Err(ReadError::Budget("item-count")) + )); + assert!(matches!( + checked_snapshot_usage(usize::MAX, 1, 0, 1, usize::MAX), + Err(ReadError::Budget("item-count")) + )); + assert!(matches!( + checked_snapshot_usage(0, 1, MAX_SNAPSHOT_BYTES, 1, 1), + Err(ReadError::Budget("byte-count")) + )); + assert!(matches!( + checked_snapshot_usage(0, 1, u64::MAX, 1, 1), + Err(ReadError::Budget("byte-count")) + )); + assert!(matches!( + checked_snapshot_usage(1, 1, 0, 1, 1), + Err(ReadError::SnapshotChanged) + )); + } + + #[test] + fn reader_rejects_blank_source_identity_before_another_page() { + for blank_key in ["", " ", "\t\n"] { + for invalid_start in [0, 1] { + let mut starts = Vec::new(); + let result = read_snapshot_with(&mut |start| { + starts.push(start); + let key = if start == invalid_start { + blank_key.to_owned() + } else { + format!("A{start}") + }; + Ok(fetched_page(3, vec![item(&key, "attachment", "", "", "")])) + }); + assert!(matches!(result, Err(ReadError::SnapshotChanged))); + assert_eq!( + starts, + if invalid_start == 0 { + vec![0] + } else { + vec![0, 1] + } + ); + } + } + } + + #[test] + fn source_inventory_retains_standalone_and_nested_metadata_without_proposals() { + let mut standalone = item("A", "attachment", "Ontology Learning", "10.1/X", ""); + standalone.version = 0; + standalone.data.collections = vec!["COLLECTION".into()]; + standalone.data.tags = vec![ItemTag { + tag: "Evidence".into(), + }]; + let report = read_snapshot_with(&mut |_| { + Ok(fetched_page( + 6, + vec![ + item("F", "annotation", "", "", "C"), + item("E", "note", "", "", ""), + item("D", "annotation", "", "", "A"), + item("C", "attachment", "", "", "B"), + item("B", "book", "Ontology Learning", "10.1/X", ""), + standalone.clone(), + ], + )) + }) + .unwrap(); + let serialized = serde_json::to_value(&report).unwrap(); + assert_eq!( + serialized["pending_source_item_keys"], + serde_json::json!(["A", "D", "E"]) + ); + let inventory = serialized["unclassified_items"] + .as_array() + .expect("every excluded source must survive in the report"); + assert_eq!( + inventory + .iter() + .map(|entry| entry["key"].as_str().unwrap()) + .collect::>(), + ["A", "C", "D", "E", "F"] + ); + assert_eq!(inventory[0]["version"], 0); + assert_eq!(inventory[0]["data"]["title"], "Ontology Learning"); + assert_eq!( + inventory[0]["data"]["collections"], + serde_json::json!(["COLLECTION"]) + ); + assert_eq!( + inventory[0]["data"]["tags"], + serde_json::json!([{"tag":"Evidence"}]) + ); + assert_eq!(inventory[4]["data"]["parentItem"], "C"); + assert_eq!(report.observed_item_count, 6); + assert_eq!(report.classified_items.len(), 1); + assert_eq!(report.classified_items[0].item_key, "B"); + assert_eq!(report.classified_items[0].child_item_keys, ["C"]); + assert!(report.duplicate_candidates.is_empty()); + } + + #[test] + fn source_inventory_keeps_orphans_cycles_and_unattached_annotations_pending() { + let sources = vec![ + item("A", "book", "", "", ""), + item("B", "note", "", "", "missing"), + item("C", "attachment", "", "", "B"), + item("D", "note", "", "", "E"), + item("E", "attachment", "", "", "D"), + item("F", "annotation", "", "", "F"), + item("G", "annotation", "", "", ""), + ]; + let forward = + serde_json::to_value(classify_snapshot("10".into(), None, 42, sources.clone())) + .unwrap(); + let reverse = serde_json::to_value(classify_snapshot( + "10".into(), + None, + 42, + sources.into_iter().rev().collect(), + )) + .unwrap(); + assert_eq!( + forward["pending_source_item_keys"], + serde_json::json!(["B", "C", "D", "E", "F", "G"]) + ); + assert_eq!(forward, reverse); + assert_eq!(forward["unclassified_items"].as_array().unwrap().len(), 6); + } + + #[test] + fn source_inventory_distinguishes_empty_and_fully_linked_evidence() { + for sources in [ + vec![], + vec![item("A", "book", "", "", "")], + vec![ + item("A", "book", "", "", ""), + item("B", "attachment", "", "", "A"), + item("C", "annotation", "", "", "B"), + ], + ] { + let serialized = + serde_json::to_value(classify_snapshot("10".into(), None, 42, sources)).unwrap(); + assert_eq!( + serialized["pending_source_item_keys"], + serde_json::json!([]) + ); + assert_eq!( + serialized["observed_item_count"].as_u64().unwrap() as usize, + serialized["classified_items"].as_array().unwrap().len() + + serialized["unclassified_items"].as_array().unwrap().len() + ); + } + } + + #[test] + fn classifies_every_bibliographic_item_and_links_children() { + let mut generation = item("B", "journalArticle", "Ontology Learning", "10.1/X", ""); + generation.data.tags.push(ItemTag { + tag: "SHACL".into(), + }); + let report = classify_snapshot( + "9.0.6".into(), + None, + 42, + vec![ + item("C", "attachment", "", "", "B"), + item("D", "note", "Ignored", "", ""), + generation, + item("A", "book", "Other", "", ""), + ], + ); + assert_eq!(report.observed_item_count, 4); + assert_eq!(report.classified_items.len(), 2); + assert_eq!( + report.classified_items[0].abstention_reason, + Some(AbstentionReason::NoDeterministicRuleMatch) + ); + assert_eq!( + report.classified_items[1].proposed_disposition, + Disposition::NeedsStewardReview + ); + assert_eq!( + report.classified_items[1].abstention_reason, + Some(AbstentionReason::ConflictingDispositionEvidence) + ); + assert_eq!(report.classified_items[1].child_item_keys, ["C"]); + assert_eq!( + report.classified_items[1].evidence.fields, + ["tags", "title"] + ); + } + + #[test] + fn specific_rule_families_and_conflicts_are_deterministic() { + let cases = [ + ("taxonomy induction", Disposition::Generation), + ( + "ontology-based data access", + Disposition::SemanticConsumptionBridge, + ), + ("ontology quality", Disposition::EvaluationGovernance), + ("semantic web", Disposition::AdjacentEvidence), + ]; + for (title, expected) in cases { + let report = classify_snapshot( + "10".into(), + Some("s".into()), + 1, + vec![item("A", "book", title, "", "")], + ); + assert_eq!(report.classified_items[0].proposed_disposition, expected); + } + + let conflict = classify_snapshot( + "10".into(), + None, + 1, + vec![item( + "A", + "book", + "ontology matching and ontology learning", + "", + "", + )], + ); + assert_eq!( + conflict.classified_items[0].proposed_disposition, + Disposition::NeedsStewardReview + ); + assert_eq!( + conflict.classified_items[0].abstention_reason, + Some(AbstentionReason::ConflictingDispositionEvidence) + ); + } + + #[test] + fn matched_values_and_abstention_reasons_are_replayable() { + let mut abstract_match = item("A", "book", "Uninformative", "", ""); + abstract_match.data.abstract_note = "Evidence for ontology alignment".into(); + let report = classify_snapshot("10".into(), None, 1, vec![abstract_match]); + assert_eq!( + report.classified_items[0] + .evidence + .field_values + .get("abstract_note") + .map(String::as_str), + Some("Evidence for ontology alignment") + ); + + let missing = classify_snapshot("10".into(), None, 1, vec![item("A", "book", "", "", "")]); + assert_eq!( + missing.classified_items[0].abstention_reason, + Some(AbstentionReason::MissingClassificationMetadata) + ); + + let unsupported = classify_snapshot( + "10".into(), + None, + 1, + vec![item("A", "book", "온톨로지 정렬", "", "")], + ); + assert_eq!( + unsupported.classified_items[0].abstention_reason, + Some(AbstentionReason::UnsupportedRuleVocabulary) + ); + } + + #[test] + fn phrase_boundaries_and_duplicate_normalization_are_exact() { + assert!(!contains_phrase("knowledge", "owl")); + assert!(!contains_phrase("growl", "owl")); + assert!(contains_phrase("owl-based", "owl")); + assert!(contains_phrase("uses owl", "owl")); + assert!(contains_phrase("owl", "owl")); + + assert_eq!(normalize_doi(" "), None); + assert_eq!(normalize_title("---"), None); + assert_eq!(normalize_doi("http://doi.org/A"), Some("a".into())); + assert_eq!(normalize_doi("https://doi.org/B"), Some("b".into())); + assert_eq!(normalize_doi("http://dx.doi.org/C"), Some("c".into())); + assert_eq!(normalize_doi("https://dx.doi.org/D"), Some("d".into())); + assert_eq!(normalize_doi("E"), Some("e".into())); + assert_eq!(normalize_title(" A---B "), Some("a b".into())); + assert_eq!(normalize_title(" A--- "), Some("a".into())); + + let report = classify_snapshot( + "10".into(), + None, + 1, + vec![ + item("A", "book", "OWL: Overview", "doi:10.1/X", ""), + item("B", "book", "owl overview", "https://dx.doi.org/10.1/x", ""), + ], + ); + assert_eq!(report.duplicate_candidates.len(), 2); + assert!( + report + .duplicate_candidates + .iter() + .all(|candidate| candidate.item_keys == ["A", "B"]) + ); + } + + #[test] + fn read_errors_are_actionable() { + assert!(ReadError::Header("x").to_string().contains('x')); + assert!(ReadError::Contract("v").to_string().contains("unsupported")); + assert!(ReadError::SnapshotChanged.to_string().contains("changed")); + assert!(ReadError::Budget("items").to_string().contains("budget")); + assert!(ReadError::Http("down".into()).to_string().contains("down")); + assert!( + ReadError::Body("large".into()) + .to_string() + .contains("large") + ); + let json_error = serde_json::from_str::("{}").unwrap_err(); + assert!(ReadError::Json(json_error).to_string().contains("JSON")); + } +} diff --git a/crates/conceptweave-zotero/src/main.rs b/crates/conceptweave-zotero/src/main.rs new file mode 100644 index 00000000..df57f75e --- /dev/null +++ b/crates/conceptweave-zotero/src/main.rs @@ -0,0 +1,453 @@ +#![forbid(unsafe_code)] +#![cfg_attr(coverage_nightly, feature(coverage_attribute))] + +use conceptweave_zotero::{read_local_snapshot, ClassificationReport, ReadError}; +use std::env; +use std::fs::{self, OpenOptions}; +use std::io::{self, BufWriter, Write}; +use std::path::{Path, PathBuf}; +use std::sync::atomic::{AtomicU64, Ordering}; + +#[cfg_attr(coverage_nightly, coverage(off))] +fn allowed_output_parents() -> Vec { + let system_temp = env::temp_dir() + .canonicalize() + .expect("system temporary directory must exist"); + let mut parents = vec![system_temp]; + if let Ok(conventional_tmp) = Path::new("/tmp").canonicalize() { + if !parents.contains(&conventional_tmp) { + parents.push(conventional_tmp); + } + } + parents +} + +fn validate_output_path(raw: &str) -> io::Result { + let path = PathBuf::from(raw); + if !path.is_absolute() { + return Err(io::Error::new( + io::ErrorKind::InvalidInput, + "report output must be an absolute path in the system temp directory", + )); + } + + let allowed_parents = allowed_output_parents(); + let parent = path.parent().ok_or_else(|| { + io::Error::new(io::ErrorKind::InvalidInput, "report output has no parent") + })?; + let resolved_parent = parent.canonicalize()?; + if !allowed_parents.contains(&resolved_parent) { + return Err(io::Error::new( + io::ErrorKind::PermissionDenied, + "report output must be a direct child of the system temp directory", + )); + } + let file_name = path.file_name().ok_or_else(|| { + io::Error::new(io::ErrorKind::InvalidInput, "report output has no file name") + })?; + let resolved_path = resolved_parent.join(file_name); + if fs::symlink_metadata(&resolved_path).is_ok() { + return Err(io::Error::new( + io::ErrorKind::AlreadyExists, + "report output must not already exist or be a symlink", + )); + } + Ok(resolved_path) +} + +fn open_new_output(path: &Path) -> io::Result { + let mut options = OpenOptions::new(); + options.write(true).create_new(true); + #[cfg(unix)] + { + use std::os::unix::fs::OpenOptionsExt; + options.mode(0o600); + } + options.open(path) +} + +fn temporary_output_path(output: &Path) -> io::Result { + static TEMP_COUNTER: AtomicU64 = AtomicU64::new(0); + + let file_name = output.file_name().ok_or_else(|| { + io::Error::new(io::ErrorKind::InvalidInput, "report output has no file name") + })?; + let nonce = TEMP_COUNTER.fetch_add(1, Ordering::Relaxed); + Ok(output.with_file_name(format!( + ".{}.{}.{}.tmp", + file_name.to_string_lossy(), + std::process::id(), + nonce + ))) +} + +fn write_report( + output: &Path, + report: &T, +) -> Result<(), Box> { + let temporary = temporary_output_path(output)?; + let file = open_new_output(&temporary)?; + let mut writer = BufWriter::new(file); + + if let Err(error) = serde_json::to_writer_pretty(&mut writer, report) { + drop(writer); + let _ = fs::remove_file(&temporary); + return Err(error.into()); + } + if let Err(error) = writer.flush() { + drop(writer); + let _ = fs::remove_file(&temporary); + return Err(error.into()); + } + drop(writer); + + if let Err(error) = fs::hard_link(&temporary, output) { + let _ = fs::remove_file(&temporary); + return Err(error.into()); + } + if let Err(cleanup_error) = fs::remove_file(&temporary) { + if let Err(rollback_error) = fs::remove_file(output) { + return Err(io::Error::other(format!( + "report published but temporary cleanup failed ({cleanup_error}); rollback also failed ({rollback_error})" + )) + .into()); + } + return Err(cleanup_error.into()); + } + Ok(()) +} + +fn run_with( + args: I, + read_snapshot: F, +) -> Result<(), Box> +where + I: IntoIterator, + F: FnOnce() -> Result, +{ + let output = args + .into_iter() + .nth(1) + .ok_or("usage: conceptweave-zotero /tmp/OUTPUT.json")?; + let output = validate_output_path(&output)?; + let report = read_snapshot()?; + if report.zotero_version.starts_with("9.") { + eprintln!("Zotero 9 Local API is read-only; writing a local proposal report only"); + } + write_report(&output, &report) +} + +#[cfg_attr(coverage_nightly, coverage(off))] +fn main() -> Result<(), Box> { + run_with(env::args(), read_local_snapshot) +} + +#[cfg(test)] +mod tests { + use super::*; + use std::cell::Cell; + + struct FailingReport; + + impl serde::Serialize for FailingReport { + fn serialize(&self, _serializer: S) -> Result + where + S: serde::Serializer, + { + Err(::custom( + "intentional serialization failure", + )) + } + } + + fn unique_temp_path(suffix: &str) -> PathBuf { + env::temp_dir().join(format!( + "conceptweave-zotero-{}-{suffix}.json", + std::process::id() + )) + } + + fn sample_report(zotero_version: &str) -> ClassificationReport { + ClassificationReport { + zotero_version: zotero_version.to_owned(), + api_version: Some(3), + schema_version: Some(44), + server_id: Some("test-server".to_owned()), + library_version: 2, + rule_revision: conceptweave_zotero::RULE_REVISION, + observed_item_count: 0, + classified_items: Vec::new(), + unclassified_items: Vec::new(), + pending_source_item_keys: Vec::new(), + duplicate_candidates: Vec::new(), + } + } + + #[test] + fn production_runner_publishes_a_complete_report() { + use std::time::{SystemTime, UNIX_EPOCH}; + + let nonce = SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap() + .as_nanos(); + let output = unique_temp_path(&format!("runner-success-{nonce}")); + let _ = fs::remove_file(&output); + let args = vec![ + "conceptweave-zotero".to_owned(), + output.to_string_lossy().into_owned(), + ]; + + run_with(args, || Ok(sample_report("10.0.1"))).unwrap(); + let saved: serde_json::Value = + serde_json::from_slice(&fs::read(&output).unwrap()).unwrap(); + assert_eq!(saved["zotero_version"], "10.0.1"); + fs::remove_file(output).unwrap(); + } + + #[test] + fn production_runner_preserves_the_zotero_9_read_only_path() { + use std::time::{SystemTime, UNIX_EPOCH}; + + let nonce = SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap() + .as_nanos(); + let output = unique_temp_path(&format!("runner-zotero9-{nonce}")); + let _ = fs::remove_file(&output); + let args = vec![ + "conceptweave-zotero".to_owned(), + output.to_string_lossy().into_owned(), + ]; + + run_with(args, || Ok(sample_report("9.0.6"))).unwrap(); + let saved: serde_json::Value = + serde_json::from_slice(&fs::read(&output).unwrap()).unwrap(); + assert_eq!(saved["zotero_version"], "9.0.6"); + fs::remove_file(output).unwrap(); + } + + #[test] + fn production_runner_rejects_missing_output_before_reading() { + let called = Cell::new(false); + let result = run_with(vec!["conceptweave-zotero".to_owned()], || { + called.set(true); + Ok(sample_report("10.0.1")) + }); + + assert!(result.is_err()); + assert!(!called.get()); + } + + #[test] + fn production_runner_rejects_invalid_output_before_reading() { + let called = Cell::new(false); + let result = run_with( + vec!["conceptweave-zotero".to_owned(), "relative.json".to_owned()], + || { + called.set(true); + Ok(sample_report("10.0.1")) + }, + ); + + assert!(result.is_err()); + assert!(!called.get()); + } + + #[test] + fn production_runner_propagates_snapshot_failure_without_publishing() { + use std::time::{SystemTime, UNIX_EPOCH}; + + let nonce = SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap() + .as_nanos(); + let output = unique_temp_path(&format!("runner-read-failure-{nonce}")); + let _ = fs::remove_file(&output); + let args = vec![ + "conceptweave-zotero".to_owned(), + output.to_string_lossy().into_owned(), + ]; + + let result = run_with(args, || Err(ReadError::Budget("test-reader"))); + assert!(result.is_err()); + assert!(!output.exists()); + } + + #[test] + fn production_runner_does_not_overwrite_a_path_created_during_the_read() { + use std::time::{SystemTime, UNIX_EPOCH}; + + let nonce = SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap() + .as_nanos(); + let output = unique_temp_path(&format!("runner-publication-race-{nonce}")); + let _ = fs::remove_file(&output); + let args = vec![ + "conceptweave-zotero".to_owned(), + output.to_string_lossy().into_owned(), + ]; + let output_during_read = output.clone(); + + let result = run_with(args, || { + fs::write(&output_during_read, b"competitor").unwrap(); + Ok(sample_report("10.0.1")) + }); + + assert!(result.is_err()); + assert_eq!(fs::read(&output).unwrap(), b"competitor"); + fs::remove_file(output).unwrap(); + } + + #[test] + fn failed_serialization_never_exposes_the_final_report_path() { + use std::time::{SystemTime, UNIX_EPOCH}; + + let nonce = SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap() + .as_nanos(); + let output = unique_temp_path(&format!("serialization-failure-{nonce}")); + let _ = fs::remove_file(&output); + + assert!(write_report(&output, &FailingReport).is_err()); + assert!(!output.exists()); + } + + #[test] + fn complete_report_is_published_once() { + use std::time::{SystemTime, UNIX_EPOCH}; + + let nonce = SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap() + .as_nanos(); + let output = unique_temp_path(&format!("atomic-success-{nonce}")); + let _ = fs::remove_file(&output); + let report = serde_json::json!({"state": "complete"}); + + write_report(&output, &report).unwrap(); + assert_eq!( + serde_json::from_slice::(&fs::read(&output).unwrap()).unwrap(), + report + ); + assert!(write_report(&output, &report).is_err()); + fs::remove_file(output).unwrap(); + } + + #[test] + fn output_path_must_be_a_new_direct_temp_child() { + let allowed = unique_temp_path("allowed"); + let _ = fs::remove_file(&allowed); + assert_eq!( + validate_output_path(allowed.to_str().unwrap()).unwrap(), + allowed + ); + + assert!(validate_output_path("relative.json").is_err()); + assert!(validate_output_path("/").is_err()); + assert!(validate_output_path("/tmp/missing-directory/report.json").is_err()); + assert!( + validate_output_path( + env::current_dir() + .unwrap() + .join("report.json") + .to_str() + .unwrap() + ) + .is_err() + ); + + if Path::new("/tmp").is_dir() { + let conventional = Path::new("/tmp").join(format!( + "conceptweave-zotero-{}-conventional.json", + std::process::id() + )); + let _ = fs::remove_file(&conventional); + assert!(validate_output_path(conventional.to_str().unwrap()).is_ok()); + } + + let nested_dir = + env::temp_dir().join(format!("conceptweave-zotero-{}-nested", std::process::id())); + fs::create_dir_all(&nested_dir).unwrap(); + assert!(validate_output_path(nested_dir.join("report.json").to_str().unwrap()).is_err()); + fs::remove_dir_all(nested_dir).unwrap(); + + let existing = unique_temp_path("existing"); + fs::write(&existing, b"existing").unwrap(); + assert!(validate_output_path(existing.to_str().unwrap()).is_err()); + fs::remove_file(existing).unwrap(); + } + + #[cfg(windows)] + #[test] + fn allowed_output_parents_does_not_require_posix_tmp() { + let system_temp = env::temp_dir().canonicalize().unwrap(); + assert!(allowed_output_parents().contains(&system_temp)); + } + + #[cfg(unix)] + #[test] + fn new_report_files_are_owner_only() { + use std::os::unix::fs::PermissionsExt; + use std::time::{SystemTime, UNIX_EPOCH}; + + let nonce = SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap() + .as_nanos(); + let output = unique_temp_path(&format!("private-{nonce}")); + let _ = fs::remove_file(&output); + let file = open_new_output(&output).unwrap(); + drop(file); + + let mode = fs::metadata(&output).unwrap().permissions().mode() & 0o777; + assert_eq!(mode, 0o600); + fs::remove_file(output).unwrap(); + } + + #[cfg(unix)] + #[test] + fn output_path_rejects_symlinks_before_open() { + use std::os::unix::fs::symlink; + + let target = unique_temp_path("target"); + let link = unique_temp_path("link"); + let _ = fs::remove_file(&target); + let _ = fs::remove_file(&link); + fs::write(&target, b"target").unwrap(); + symlink(&target, &link).unwrap(); + assert!(validate_output_path(link.to_str().unwrap()).is_err()); + fs::remove_file(link).unwrap(); + fs::remove_file(target).unwrap(); + } + + #[cfg(unix)] + #[test] + fn output_path_uses_the_validated_parent_not_a_swappable_symlink() { + use std::os::unix::fs::symlink; + use std::time::{SystemTime, UNIX_EPOCH}; + + let allowed_parent = env::temp_dir().canonicalize().unwrap(); + let nonce = SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap() + .as_nanos(); + let alias = allowed_parent.join(format!( + "conceptweave-zotero-{}-{nonce}-parent-link", + std::process::id() + )); + symlink(&allowed_parent, &alias).unwrap(); + let leaf = format!( + "conceptweave-zotero-{}-{nonce}-canonical.json", + std::process::id() + ); + let output = alias.join(&leaf); + + let validated = validate_output_path(output.to_str().unwrap()).unwrap(); + assert_eq!(validated, allowed_parent.join(&leaf)); + + fs::remove_file(alias).unwrap(); + } +} diff --git a/crates/conceptweave-zotero/src/tests/metadata_transport.rs b/crates/conceptweave-zotero/src/tests/metadata_transport.rs new file mode 100644 index 00000000..b4e4bf46 --- /dev/null +++ b/crates/conceptweave-zotero/src/tests/metadata_transport.rs @@ -0,0 +1,143 @@ +use super::*; +use std::process::{Command, Stdio}; +use std::time::Instant; + +const PROXY_CHILD_CASE: &str = "CONCEPTWEAVE_METADATA_PROXY_CASE"; + +fn read_fixture( + body: Vec, + declared_bytes: usize, +) -> ( + Result, + thread::JoinHandle, +) { + let _guard = LOCAL_API_TEST_LOCK.lock().unwrap(); + let listener = TcpListener::bind("127.0.0.1:0").unwrap(); + *TEST_LOCAL_API.lock().unwrap() = Some(format!( + "http://{}/api/users/0/items", + listener.local_addr().unwrap() + )); + let server = thread::spawn(move || { + let (mut stream, _) = listener.accept().unwrap(); + let mut request = Vec::new(); + while !request.windows(4).any(|part| part == b"\r\n\r\n") { + let mut buffer = [0; 4096]; + let length = stream.read(&mut buffer).unwrap(); + assert_ne!(length, 0); + request.extend_from_slice(&buffer[..length]); + } + let headers = format!( + "HTTP/1.1 200 OK\r\nContent-Length: {declared_bytes}\r\nTotal-Results: 0\r\nLast-Modified-Version: 42\r\nX-Zotero-Version: 9.0.6\r\nZotero-API-Version: 3\r\nZotero-Schema-Version: 42\r\nZotero-Server-ID: synthetic-server\r\nConnection: close\r\n\r\n" + ); + stream.write_all(headers.as_bytes()).unwrap(); + // An invalid or oversized body can make the client close before all bytes arrive. + let _ = stream.write_all(&body); + String::from_utf8(request).unwrap() + }); + let result = read_local_snapshot(); + *TEST_LOCAL_API.lock().unwrap() = None; + (result, server) +} + +#[test] +fn snapshot_never_uses_environment_proxies() { + let mut failures = Vec::new(); + for proxy_variable in [ + "HTTP_PROXY", + "http_proxy", + "HTTPS_PROXY", + "https_proxy", + "ALL_PROXY", + "all_proxy", + ] { + let listener = TcpListener::bind("127.0.0.1:0").unwrap(); + listener.set_nonblocking(true).unwrap(); + let proxy_url = format!("http://{}", listener.local_addr().unwrap()); + // Isolate synthetic settings in the child; never mutate the test process + // environment or route requests to the real Zotero port. + let mut child = Command::new(std::env::current_exe().unwrap()) + .args([ + "--exact", + "tests::metadata_transport::metadata_routing_child", + ]) + .env_clear() + .env(PROXY_CHILD_CASE, "synthetic") + .env(proxy_variable, proxy_url) + .stdout(Stdio::null()) + .stderr(Stdio::null()) + .spawn() + .unwrap(); + let started = Instant::now(); + let mut proxy_connections = 0; + let status = loop { + match listener.accept() { + Ok((mut stream, _)) => { + proxy_connections += 1; + let _ = stream.write_all( + b"HTTP/1.1 502 Bad Gateway\r\nContent-Length: 0\r\nConnection: close\r\n\r\n", + ); + } + Err(error) => assert_eq!(error.kind(), std::io::ErrorKind::WouldBlock), + } + if let Some(status) = child.try_wait().unwrap() { + break status; + } + if started.elapsed() > Duration::from_secs(10) { + child.kill().unwrap(); + child.wait().unwrap(); + panic!("isolated metadata routing check timed out"); + } + thread::sleep(Duration::from_millis(5)); + }; + if proxy_connections != 0 || !status.success() { + failures.push(format!( + "{proxy_variable}: proxy_connections={proxy_connections}, direct_success={}", + status.success() + )); + } + } + assert!(failures.is_empty(), "{failures:?}"); +} + +#[test] +fn metadata_routing_child() { + if std::env::var_os(PROXY_CHILD_CASE).is_none() { + return; + } + let (result, server) = read_fixture(b"[]".to_vec(), 2); + let report = result.unwrap(); + assert_eq!(report.library_version, 42); + assert!(report.classified_items.is_empty()); + let request = server.join().unwrap(); + assert!(request.starts_with( + "GET /api/users/0/items?format=json&include=data&limit=100&start=0 HTTP/1.1\r\n" + )); + assert!(request.contains("zotero-api-version: 3\r\n")); + assert!(!request.contains("zotero-api-key:")); +} + +#[test] +fn snapshot_accepts_a_response_exactly_at_the_byte_limit() { + let mut body = b"[]".to_vec(); + body.resize(MAX_PAGE_BYTES as usize, b' '); + let (result, server) = read_fixture(body, MAX_PAGE_BYTES as usize); + server.join().unwrap(); + let report = result.expect("exact-limit synthetic JSON must be accepted"); + assert_eq!(report.library_version, 42); + assert!(report.classified_items.is_empty()); +} + +#[test] +fn snapshot_rejects_oversized_invalid_utf8_and_truncated_bodies() { + let mut oversized = b"[]".to_vec(); + oversized.resize(MAX_PAGE_BYTES as usize + 1, b' '); + for (body, declared_bytes) in [ + (oversized, MAX_PAGE_BYTES as usize + 1), + (vec![0xff], 1), + (b"[]".to_vec(), 3), + ] { + let (result, server) = read_fixture(body, declared_bytes); + server.join().unwrap(); + assert!(matches!(result, Err(ReadError::Body(_)))); + } +} diff --git a/crates/conceptweave-zotero/tests/review_contract.rs b/crates/conceptweave-zotero/tests/review_contract.rs new file mode 100644 index 00000000..dce9aebf --- /dev/null +++ b/crates/conceptweave-zotero/tests/review_contract.rs @@ -0,0 +1,72 @@ +use conceptweave_zotero::{AbstentionReason, Disposition, ItemData, ZoteroItem, classify_snapshot}; + +fn item(key: &str, title: &str, doi: &str) -> ZoteroItem { + ZoteroItem { + key: key.into(), + version: 7, + data: ItemData { + item_type: "journalArticle".into(), + title: title.into(), + abstract_note: String::new(), + doi: doi.into(), + parent_item: String::new(), + collections: vec![], + tags: vec![], + }, + } +} + +#[test] +fn steward_abstention_reason_is_explicit_and_deterministic() { + let blank = item("A", "", ""); + let multilingual = item("B", "온톨로지 정렬", ""); + let unmatched = item("C", "Other evidence", ""); + let matched = item("D", "Ontology alignment", ""); + + let report = classify_snapshot( + "9.0.6".into(), + None, + 42, + vec![blank, multilingual, unmatched, matched], + ); + + assert_eq!( + report.classified_items[0].abstention_reason, + Some(AbstentionReason::MissingClassificationMetadata) + ); + assert_eq!( + report.classified_items[1].abstention_reason, + Some(AbstentionReason::UnsupportedRuleVocabulary) + ); + assert_eq!( + report.classified_items[2].abstention_reason, + Some(AbstentionReason::NoDeterministicRuleMatch) + ); + assert_eq!( + report.classified_items[3].proposed_disposition, + Disposition::AlignmentVersioning + ); + assert_eq!(report.classified_items[3].abstention_reason, None); +} + +#[test] +fn legacy_dx_doi_uri_collapses_into_the_same_duplicate_group() { + let report = classify_snapshot( + "9.0.6".into(), + None, + 42, + vec![ + item("A", "First", "10.1/X"), + item("B", "Second", "http://dx.doi.org/10.1/x"), + item("C", "Third", "https://dx.doi.org/10.1/X"), + ], + ); + + let doi_group = report + .duplicate_candidates + .iter() + .find(|candidate| candidate.identity_kind == "doi") + .expect("all DOI resolver forms must normalize to one candidate group"); + assert_eq!(doi_group.normalized_identity, "10.1/x"); + assert_eq!(doi_group.item_keys, ["A", "B", "C"]); +} diff --git a/crates/conceptweave-zotero/tests/review_contract_followup.rs b/crates/conceptweave-zotero/tests/review_contract_followup.rs new file mode 100644 index 00000000..c3745c6b --- /dev/null +++ b/crates/conceptweave-zotero/tests/review_contract_followup.rs @@ -0,0 +1,66 @@ +use conceptweave_zotero::{AbstentionReason, Disposition, ItemData, ZoteroItem, classify_snapshot}; + +fn item(key: &str, title: &str, abstract_note: &str) -> ZoteroItem { + ZoteroItem { + key: key.into(), + version: 11, + data: ItemData { + item_type: "journalArticle".into(), + title: title.into(), + abstract_note: abstract_note.into(), + doi: String::new(), + parent_item: String::new(), + collections: vec![], + tags: vec![], + }, + } +} + +#[test] +fn conflicting_specific_rule_families_abstain_for_steward_review() { + let report = classify_snapshot( + "9.0.6".into(), + None, + 42, + vec![item("A", "Ontology matching and ontology learning", "")], + ); + + let classified = &report.classified_items[0]; + assert_eq!( + classified.proposed_disposition, + Disposition::NeedsStewardReview + ); + assert_eq!( + classified.abstention_reason, + Some(AbstentionReason::ConflictingDispositionEvidence) + ); + assert_eq!( + classified.evidence.matched_phrases, + ["ontology learning", "ontology matching"] + ); +} + +#[test] +fn matched_abstract_value_is_preserved_for_replayable_review() { + let abstract_note = "We evaluate ontology alignment under schema drift."; + let report = classify_snapshot( + "9.0.6".into(), + None, + 42, + vec![item("A", "Uninformative title", abstract_note)], + ); + + let classified = &report.classified_items[0]; + assert_eq!( + classified.proposed_disposition, + Disposition::AlignmentVersioning + ); + assert_eq!( + classified + .evidence + .field_values + .get("abstract_note") + .map(String::as_str), + Some(abstract_note) + ); +} diff --git a/crates/conceptweave-zotero/tests/tag_phrase_boundaries.rs b/crates/conceptweave-zotero/tests/tag_phrase_boundaries.rs new file mode 100644 index 00000000..e26b78b0 --- /dev/null +++ b/crates/conceptweave-zotero/tests/tag_phrase_boundaries.rs @@ -0,0 +1,57 @@ +use conceptweave_zotero::{Disposition, ItemData, ItemTag, ZoteroItem, classify_snapshot}; + +fn item_with_tags(tags: &[&str]) -> ZoteroItem { + ZoteroItem { + key: "A".into(), + version: 11, + data: ItemData { + item_type: "journalArticle".into(), + title: "Uninformative title".into(), + abstract_note: String::new(), + doi: String::new(), + parent_item: String::new(), + collections: vec![], + tags: tags + .iter() + .map(|tag| ItemTag { + tag: (*tag).into(), + }) + .collect(), + }, + } +} + +#[test] +fn separate_tags_do_not_synthesize_a_multiword_rule_phrase() { + let report = classify_snapshot( + "10.0.1".into(), + None, + 2, + vec![item_with_tags(&["ontology", "alignment"])], + ); + + let classified = &report.classified_items[0]; + assert_eq!(classified.proposed_disposition, Disposition::AdjacentEvidence); + assert!(!classified.evidence.matched_phrases.contains(&"ontology alignment")); +} + +#[test] +fn one_tag_containing_the_complete_phrase_still_matches_exactly() { + let report = classify_snapshot( + "10.0.1".into(), + None, + 2, + vec![item_with_tags(&["ontology alignment"])], + ); + + let classified = &report.classified_items[0]; + assert_eq!( + classified.proposed_disposition, + Disposition::AlignmentVersioning + ); + assert_eq!( + classified.evidence.field_values.get("tags").map(String::as_str), + Some("ontology alignment") + ); + assert!(classified.evidence.matched_phrases.contains(&"ontology alignment")); +} diff --git a/docs/CONTEXT_MAP.md b/docs/CONTEXT_MAP.md index 5ea47792..bc769582 100644 --- a/docs/CONTEXT_MAP.md +++ b/docs/CONTEXT_MAP.md @@ -9,6 +9,7 @@ ## External relationships +- Zotero Local API -> research evidence intake: **Anti-Corruption Layer into Semantic Discovery**. Zotero remains the bibliographic system of record; ConceptWeave consumes a version-pinned, read-only Local API snapshot and emits proposal evidence only. Item metadata, attachments, collection/tag truth, and future write authority remain in Zotero. No Zotero record becomes semantic authority without ConceptWeave validation/review/publication. - contextual-orchestrator -> Semantic Discovery: **Anti-Corruption Layer**. Model/provider envelopes never enter the domain model directly. - LineageWeave -> Source Observation: **Anti-Corruption Layer**. Inferred/proposed lineage remains explicitly non-authoritative until ConceptWeave governance evaluates it. - context-graph-contracts <-> Interoperability: **Shared Kernel only for versioned public contracts**, kept minimal. diff --git a/docs/PRD.md b/docs/PRD.md index 0e68c400..3d12e5c5 100644 --- a/docs/PRD.md +++ b/docs/PRD.md @@ -54,6 +54,14 @@ Support stable adapters for `semantic-data-portal`, `LineageWeave`, `context-gra All LLM-backed induction uses `contextual-orchestrator`. Model output is untrusted proposal data and may not skip deterministic validation or review. +### FR-9 Research evidence intake + +Preserve every observed source, including standalone files and notes outside the bibliographic proposals. Keep unresolved source relationships visible instead of treating a completed bibliography worksheet as a completed library review. All standalone sources and records without a valid path to a bibliographic parent need explicit reconciliation; notes, files and annotations must not acquire paper labels from their titles. Retraction and correction evidence remains separate from topic classification and approval. The current producer retains this inventory; downstream reconciliation, completion admission and independent governance remain required, not implemented by inventory generation alone. + +Library reads must finish within a bounded observation window or fail visibly without returning a partial classification. Slowly arriving pages cannot keep a run open indefinitely, and missing time budget must not be handled by silently dropping papers. + +Read a complete Zotero Local API observation with one consistent library version and propose exactly one research disposition for every top-level bibliographic item. This consistency check does not establish an atomic provider snapshot. A record claiming a revision newer than the observed library invalidates the complete read; it must not be omitted or assigned a different revision to make the read pass. Each proposal retains the item key/version, exact matched metadata values, rule revision, linked child records, and any model receipt. Weak evidence and evidence that matches multiple specific disposition families must abstain into steward review. Duplicate DOI/title identities are review candidates only: intake never merges, deletes, or silently mutates Zotero records. + ## 6. First vertical slice Relational schema snapshot -> observed tables/columns/foreign keys -> concept/relation/dimension/measure/mapping candidates -> evidence-bound validation report -> reviewable proposal package. diff --git a/docs/TRD.md b/docs/TRD.md index ad5e1415..08a59938 100644 --- a/docs/TRD.md +++ b/docs/TRD.md @@ -56,3 +56,21 @@ Source artifacts are untrusted input. Adapters must enforce source size/type bou ## 10. Evaluation Evaluation must separate extraction recall, semantic correctness, structural correctness, ontology consistency, mapping accuracy, measure correctness, and governance outcomes. Model-judge scores may supplement but never replace deterministic golden fixtures and human-reviewed expert cases. + +## 11. Zotero research intake + +`ClassificationReport.unclassified_items` retains every input record excluded from bibliographic classification, using the existing `ZoteroItem` metadata projection. Bibliographic proposals and this inventory are disjoint and together account for the observed record count on reader-admitted input. The existing child index is consumed once from bibliographic roots; records never reached remain in sorted `pending_source_item_keys`, including standalone roots, their descendants, orphan trees and cycles. The traversal is iterative, uses no new dependency and costs O(n log n) time/O(n) auxiliary space. It does not validate arbitrary offline input or preserve note bodies, attachment-specific fields and unknown provider JSON. + +Consumer requirement, still pending forward integration: require both inventory fields rather than defaulting absent legacy fields to empty; validate exact snapshot identity/version/parent complement and recompute pending keys before any review, duplicate evaluation or write verifier. Keep bibliographic progress distinct from whole-library completion. An empty pending list is only ancestry accounting, never semantic approval. Full-text report-digest changes require fresh bound verification, not capture rewriting; metadata proposal-only digests do not implicitly bind the new inventory. See [source-scope evidence and integration map](doctoring/zotero_source_scope.md). + +The shared live reader rejects empty or whitespace-only item keys before accumulating a page or requesting another one. It preserves valid keys verbatim; this does not add a new provider key-format restriction or certify arbitrary offline classifier input. + +`conceptweave-zotero` reads only the loopback Local API with at most 100 records per page, an 8 MiB page-body limit, a 50,000-item whole-snapshot limit, a 256 MiB cumulative body limit, redirects disabled, and finite connect/response/body/global timeouts. Every request pins `Zotero-API-Version: 3`; every response must report API version 3 and a present, recorded schema revision that stays unchanged during the snapshot, rather than the historical workstation schema 42. Before another request is issued, exhausted whole-snapshot budgets fail closed. Before a parsed page is accumulated, checked item-count and byte arithmetic must remain within both the advertised total and the configured whole-snapshot budgets. `Total-Results`, `Last-Modified-Version`, Zotero version, schema revision and server identity must remain identical across all pages; contract drift, malformed JSON, an empty intermediate page, duplicate keys, or an oversized response fails the run. + +One monotonic five-minute budget covers page admission and complete-report acceptance. At or beyond that limit, no new request starts and no late page or completed report is accepted. An already-started request retains the existing per-request timeouts; computation is not forcibly interrupted. Never return a partial classification to satisfy the time budget or discard legitimate short pages. On budget failure the caller receives an error, not a smaller successful denominator. + +After count/byte validation and before accumulating each metadata page, every returned object's revision must be less than or equal to that page's library revision. This includes attachments, notes and annotations, not only bibliographic records. A higher revision returns the existing snapshot-consistency error immediately, with no next-page request or partial report. Zero, lower and equal revisions remain valid and are preserved exactly, including the unsigned maximum. Compare only within this metadata read of one local instance: Zotero 9's synced revisions and Zotero 10's local revisions are not interchangeable, and this condition is not a full-text endpoint contract or proof of atomicity. See the [revision admission evidence](doctoring/zotero_item_revision.md). + +Every top-level bibliographic record receives exactly one proposed disposition. `NeedsStewardReview` also records a deterministic abstention reason so missing classification metadata, vocabulary unsupported by the current deterministic rules, and present-but-unmatched metadata are distinguishable. DOI duplicate identity normalization treats bare DOI values, `doi:`, `doi.org`, and legacy `dx.doi.org` resolver forms as the same identity when their normalized DOI is equal. + +The report is local JSON and contains proposals rather than governance decisions. CLI output is restricted to a new direct child of canonical `/tmp` or the operating system temporary directory; relative paths, nested paths, existing paths, and symlinks are rejected, and create-new file semantics prevent overwrite/path-swap writes. Zotero 9 writes are unsupported; no mutation path exists in this slice. A future Zotero 10+ writer requires a separate reviewed change with a Local API key, stable server identity, fresh item/library version preconditions, item-by-item before/after receipts, and rollback evidence. diff --git a/docs/UBIQUITOUS_LANGUAGE.md b/docs/UBIQUITOUS_LANGUAGE.md index 0c1d0c25..88f353b6 100644 --- a/docs/UBIQUITOUS_LANGUAGE.md +++ b/docs/UBIQUITOUS_LANGUAGE.md @@ -6,6 +6,11 @@ | Observation | Deterministically extracted fact from a Source Snapshot. | | Evidence Reference | Stable source identity, digest, and location supporting a candidate. | | Semantic Candidate | Evidence-bound proposal for a concept, relation, constraint, dimension, measure, or physical mapping. | +| Research Intake Report | Local, snapshot-bound evidence artifact produced from research-source observation; it contains proposals and unresolved source scope, never a governance decision or semantic authority. | +| Proposed Disposition | Deterministic, non-authoritative research-routing proposal assigned to one bibliographic item for steward review. | +| Abstention Reason | Deterministic reason that research intake cannot assign a narrower Proposed Disposition; it preserves why the item remains in steward review rather than implying rejection or approval. | +| Conflicting Disposition Evidence | Evidence that supports more than one specific disposition family for the same item; it requires abstention and steward review instead of first-match classification. | +| Pending Source | Observed nonbibliographic source whose parent chain is not reconciled to a bibliographic proposal in the same Source Snapshot; it is unresolved scope, not an additional paper or an approval state. | | Semantic Model Proposal | Versioned collection of candidates presented for validation/review. | | Validation Report | Deterministic result describing structural or semantic contract validity; not a review decision. | | Review Decision | Authorized accept/reject decision over validated candidates or a model proposal. | diff --git a/docs/UML.md b/docs/UML.md index a9559e7f..714c3004 100644 --- a/docs/UML.md +++ b/docs/UML.md @@ -38,3 +38,25 @@ sequenceDiagram Publisher-->>Source: no source mutation Publisher-->>Steward: immutable release receipt ``` + +## Research intake sequence + +```mermaid +sequenceDiagram + participant Zotero as Zotero Local API + participant Intake as Research intake + participant Report as Local proposal report + participant Steward + + loop bounded pages + Intake->>Zotero: read items at one library version + Zotero-->>Intake: items + observed library version + Intake->>Intake: validate page consistency and every item revision; fail entire read on mismatch + end + Intake->>Intake: classify or abstain; link children; find duplicate candidates + Intake->>Intake: retain excluded metadata; traverse parent links from bibliographic roots + Intake->>Report: write proposals, complete inventory and unresolved source keys + Note over Report,Steward: Pending sources prevent a whole-library completion claim; inventory is not approval + Report->>Steward: review dispositions and merge candidates + Intake-->>Zotero: no mutation +``` diff --git a/docs/adr/0006-zotero-research-intake.md b/docs/adr/0006-zotero-research-intake.md new file mode 100644 index 00000000..b0b0a81a --- /dev/null +++ b/docs/adr/0006-zotero-research-intake.md @@ -0,0 +1,56 @@ +# ADR 0006: Keep Zotero research intake read-only and proposal-based + +- Status: Proposed +- Date: 2026-09-04 + +## Context + +CWL needs a reproducible inventory of ontology research without turning keyword matches into authoritative library organization. The current desktop is Zotero 9.0.6, whose Local API supports reads but not writes. Zotero documents Local API writes only for Zotero 10+, where they require user-granted authorization and `Zotero-Server-ID`; this slice therefore has no mutation capability. The library is mutable while pagination is in progress, duplicate metadata does not prove that two records should be merged, and the local report contains titles and item keys that must not be written into the repository. + +Zotero's Local API documentation states that production clients should request `Zotero-API-Version: 3`; the response exposes `Zotero-API-Version` and `Zotero-Schema-Version`. The API version is the compatibility contract. The schema version is therefore recorded and required to remain stable across the snapshot, but it is not hard-coded to the developer workstation's current schema 42 because Zotero can legitimately revise the local data schema while retaining API v3 compatibility. + +Primary capability references: Zotero, *Local API* (updated 2026-07-29), https://www.zotero.org/support/dev/web_api/v3/local_api; Zotero, *Basics*, https://www.zotero.org/support/dev/web_api/v3/basics. + +## Decision + +ConceptWeave owns a small read-only Anti-Corruption Layer from Zotero into research evidence intake. Every request explicitly sends `Zotero-API-Version: 3`; a response reporting another API version fails closed. `Total-Results`, library version, Zotero version, schema version, and optional server identity must remain unchanged across pagination. API and schema versions are retained in the live report provenance. + +The adapter links child records, emits exactly one deterministic proposed disposition per top-level bibliographic item, and abstains when evidence is weak or ambiguous. Every abstention preserves a deterministic reason distinguishing missing classification metadata, vocabulary outside the current deterministic rules, present-but-unmatched metadata, and conflicting specific disposition families. Specific rule families are evaluated together rather than by first-match priority. When evidence matches multiple families, the proposal becomes `NeedsStewardReview` and all matching evidence is retained. + +Matched metadata values are copied into the local-only evidence receipt for replay. This is necessary for abstract-only matches because a later Zotero revision cannot reconstruct the exact text used for an earlier proposal from item key/version alone. DOI/title matches remain reversible duplicate candidates, including legacy `dx.doi.org` resolver forms. + +The reader fails closed above 50,000 items or 256 MiB of cumulative response bodies, while retaining the 8 MiB per-page bound, finite request timeouts, redirect denial, total-count checks, snapshot-version checks, and duplicate-key detection. Pagination, consistency, resource-budget, and provider-contract behavior are separated from the narrow `ureq` transport so deterministic tests exercise the production reader core rather than excluding the entire reader from coverage. + +In the context of reading every bibliographic source before classification, facing individually timely pages that can cumulatively hold a run open for days, we decided for a five-minute monotonic admission/completion budget in the existing reader and against rejecting legitimate short pages or adding another transport, to bound accepted work without excluding papers, accepting that an already-started request or classification computation can finish after the limit before its result is rejected. This is an application read limit, not a model timeout, hard process-cancellation deadline, wall-clock/suspend guarantee or atomic snapshot claim. Each page is checked before fetch and after return, and the complete report is checked before return. The stdlib clock has a private deterministic test seam; public APIs, provider timeouts and data/byte ceilings are unchanged. The [deadline doctoring](../doctoring/zotero_metadata_deadline.md) records the original review, RED/GREEN, alternatives and exact verification. This amendment remains Proposed and grants no Zotero mutation authority. + +In the context of retaining exact metadata provenance for every observed record, facing object revisions newer than the library revision of their own response, we decided for rejection in the shared metadata reader before accumulation and against filtering records, clamping revisions or extending the pure offline classifier's contract, to preserve the complete evidence denominator and original revisions, accepting that an inconsistent response requires a complete retry. This is a within-instance metadata condition, not a comparison across Zotero installations, a full-text version rule or an atomicity guarantee. Zero and equal versions are valid. The [revision doctoring](../doctoring/zotero_item_revision.md) records provider semantics, counterexamples and exact local checks. This amendment remains Proposed. + +Report output is restricted to a new direct child of the operating system temporary directory. Relative paths, nested paths, existing files, and symlinks are rejected before write; the file is opened with create-new semantics so a path swap cannot cause repository or arbitrary-file overwrite. The buffered writer is explicitly flushed and a final filesystem error fails the command. Reports stay local and are never committed. + +No dedicated utility repository or Zotero mutation path is created. A future Zotero 10+ write adapter is a separate decision and must use authenticated loopback access, server identity, optimistic version preconditions, reviewed item-level changes, before/after receipts, and rollback evidence. + +## Consequences + +### September 6 source-scope amendment (Proposed) + +In the context of a genuine 8,326-record library observation and a native 3,719-top-level selection, facing three standalone PDFs and one note omitted from the 3,715 bibliographic proposals, we decided to retain every nonbibliographic metadata record and derive unresolved ancestry using the existing child index. We rejected silently excluding these sources, guessing paper labels from file titles, and a standalone-only list that would still hide orphan or cyclic child relationships. Reusing `ZoteroItem` avoids another adapter or DTO, at the cost of a larger private report and an explicitly projected, not lossless, source representation. Consuming each adjacency list once makes traversal finite without recursion or repeated ancestry walks. The [executed source-scope record](../doctoring/zotero_source_scope.md) binds the failing tests, source, actual observation and consumer integration requirements. + +This change does not reconcile a source, validate arbitrary offline input, retain full note/file contents or grant review/write authority. Consumers must require and validate the new inventory, recompute pending keys and distinguish bibliographic progress from full-library completion before adoption. Direct evaluation, duplicate and write routes must not bypass that validation. Existing proposal-only approval digests do not bind this added scope; incompatible full-text captures must fail their whole-report binding. The schema and governance integration remain pending in dependent owners; no backward-compatible empty default, capture rewrite or approved-label backfill is accepted. The original Zotero 9.0.6 observation above is historical; this amendment was tested against Zotero 10.0.1 while retaining read-only behavior. + +- A complete snapshot can be audited and replayed without changing the research library. +- Rule evidence and explicit abstention reasons are visible; automated classification is not governance approval. +- Cross-cutting papers cannot be silently forced into whichever rule family happens to be evaluated first. +- Whole-snapshot resource use is bounded independently from per-page limits. +- Local API compatibility is explicit through API v3 while schema revision remains traceable and snapshot-stable. +- Sensitive local reports cannot be directed into the repository by the CLI and final write failures are observable. +- Human review remains necessary for ambiguous records and every duplicate merge. +- Zotero 9 cannot apply approved collection/tag changes automatically. + +## Alternatives considered + +- First-match classification was rejected because FR-9 requires ambiguous evidence to abstain rather than acquire an arbitrary priority-based disposition. +- Hard-coding Zotero schema 42 was rejected because the documented compatibility contract is API v3; schema revision is instead recorded and checked for within-read drift. +- Direct Zotero 9 writes were rejected because the supported Local API write capability is Zotero 10+ only. +- Cloud Web API mutation was rejected because it expands credential and network scope without being needed for classification. +- Arbitrary report output paths were rejected because local bibliographic titles and item keys are intentionally not repository artifacts. +- A new repository was rejected because one bounded adapter does not yet justify another lifecycle and release surface. diff --git a/docs/adr/README.md b/docs/adr/README.md index 291702a3..bdd9b507 100644 --- a/docs/adr/README.md +++ b/docs/adr/README.md @@ -3,3 +3,4 @@ - [ADR 0001 — Product and bounded-context boundary](0001-product-boundary.md) - [ADR 0002 — Evidence, truth, and publication lifecycle](0002-truth-publication-lifecycle.md) - [ADR 0003 — Standards and LLM engineering boundary](0003-standards-llm-boundary.md) +- [ADR 0006 — Zotero research intake](0006-zotero-research-intake.md) diff --git a/docs/doctoring/RESEARCH_CAPABILITY_TRACEABILITY.md b/docs/doctoring/RESEARCH_CAPABILITY_TRACEABILITY.md index bde6ff96..74b10413 100644 --- a/docs/doctoring/RESEARCH_CAPABILITY_TRACEABILITY.md +++ b/docs/doctoring/RESEARCH_CAPABILITY_TRACEABILITY.md @@ -98,3 +98,7 @@ The following canonical Consensus records were fetched before recording the corr 3. GRC must remain the first enterprise round-trip fixture, while OAEI/RODI/LLMs4OL-style data guards against overfitting the general contract to GRC. 4. LLM calls remain behind `contextual-orchestrator`; model/provider/prompt changes require receipts and sensitivity evidence. 5. Human review remains mandatory before authority promotion; Crowd-OM is evidence for scalable validation mechanics, not permission to replace GRC/domain steward authority. + +## Research intake evidence + +The Zotero classifier records item and library revisions plus the exact rule revision for each proposal. Keyword evidence is routing evidence only: unmatched records abstain, duplicate identities remain candidates, and neither path creates authoritative ontology knowledge. Any model-assisted successor must add a `contextual-orchestrator` receipt while preserving the deterministic inputs and steward decision separately. diff --git a/docs/doctoring/zotero_item_revision.md b/docs/doctoring/zotero_item_revision.md new file mode 100644 index 00000000..5e4c82dc --- /dev/null +++ b/docs/doctoring/zotero_item_revision.md @@ -0,0 +1,35 @@ +# Zotero metadata item revision admission + +Status: locally verified prerequisite repair for PR #9; not pushed, protected, released or propagated to descendants at this checkpoint. No actual library, paper, model or Zotero mutation was exercised. Tests use synthetic unit inputs only. + +## Finding and provider contract + +[PR #9's review](https://github.com/ContextualWisdomLab/ConceptWeave/pull/9#discussion_r3934542708) identifies an item revision higher than its response's `Last-Modified-Version` being accepted into classification provenance. Baseline `bb2faccfda9efed55b6759f1bbf7907bf6ec0c3b` checks page-header agreement, counts, bytes, elapsed time and unique keys, but never compares a returned object's revision with its containing library revision. A later review or mutation precondition could therefore inherit impossible metadata coordinates even though the pages agree with each other. + +The Web API documents library revisions and object revisions separately: a multi-object items response carries the library revision; changed objects receive the library's updated revision. Revision numbers are opaque, monotonic and need not be consecutive (Zotero, 2022). The Local API documentation distinguishes Zotero 10's per-library local transaction revisions from Zotero 9's synced versions. Never-synced earlier objects can be version zero; Zotero 10 local revisions are comparable only within one instance and are unrelated to Web API revisions (Zotero, 2026). Together these rules support the within-response metadata invariant `item.version <= page.library_version`; they do not justify cross-instance ordering, rejecting zero, or applying this condition to full-text endpoints. + +## Decision and failure boundary + +The call path is `read_local_snapshot` → `read_snapshot_with` → `read_snapshot_with_clock` → page transport → `classify_snapshot`, followed by CLI report-file creation only after success. The shared reader now checks every page member after the existing resource validation and before `items.extend`. One `.any(...)` predicate returns the existing `ReadError::SnapshotChanged` if a member is too new. Bibliographic records and child attachments, notes and annotations take the same path. No later page is requested after rejection and no partial report is returned. + +Filtering offending records would change the denominator; clamping revisions would fabricate provenance. Changing the pure offline classifier would broaden its contract and still be later than the provider admission boundary. No new error type, helper, public signature, dependency or cross-product responsibility is needed. The added pass is linear in each bounded page and does not allocate another collection. The downside is that an inconsistent page invalidates the whole read, even if earlier pages were usable. Equal headers and valid member revisions still do not prove atomicity, server authentication or semantic correctness. ADR 0006 remains Proposed and no approval or write authority changes. + +## Executed checks + +- Baseline `bb2facc`: 41 tests / 10 unfiltered suites, including two doctests. +- Committed RED `1cf3472499ad49716a70603be2e15dc857819231`: the new rejection test fails at the expected `SnapshotChanged` assertion, not compilation or an unrelated error. Its valid-version control passes before the guard exists. +- Source GREEN `8effa6a9b15ac1a09b7e80dab4cf2885fad02211`: 43 tests / 10 unfiltered suites, including two doctests, under explicit Rust 1.98.0. Rejection covers first and subsequent pages and four item types, with a valid member before the offending member. Acceptance covers zero, lower, equal and `u64::MAX` revisions under Zotero 9 and 10 labels and checks retained revisions. +- All-target warnings-denied Clippy, warnings-denied rustdoc, release build, formatting, existing CI contract and diff validation pass. CodeGraph is healthy at 11 files, 206 nodes and 433 edges. +- Unchanged coverage gate passes: 113/113 functions, 709/709 normalized source regions and 106/106 normalized branches. Raw LLVM is not 100%: 1,097/1,098 lines, 1,690/1,697 regions and 105/106 branches. No gate, exclusion or fixture denominator was weakened. + +Reproduce with `cargo +1.98.0 test --workspace --locked` and `bash scripts/check_coverage.sh`. Local logs are `/tmp/conceptweave-pr9-item-revision-{baseline,red,green,clippy,rustdoc,release,coverage}-20260906.log`. + +## Integration gate + +The previous GitHub PR audit was rejected by the account GraphQL quota at 2026-09-06 10:05:51 UTC. This repair uses already-read review evidence and local source, not a new remote-state claim. Do not retry that audit before 11:06:12 UTC or substitute another endpoint/token/provider to bypass the limit. Then perform one normal fresh head/base/state/writer check, preserve concurrent delta through normal history, push the owner repair and merge it forward through each dependent research PR with fresh exact-head verification. This new propagation is outstanding; the earlier elapsed-deadline propagation does not include this revision guard. Required hosted checks, current-head independent review and protected integration remain separate gates. Do not close predecessors or claim actual paper decisions from these tests. + +## References + +Zotero. (2022, August 14). *Zotero Web API v3: Syncing*. https://www.zotero.org/support/dev/web_api/v3/syncing + +Zotero. (2026, July 29). *Zotero Local API*. https://www.zotero.org/support/dev/web_api/v3/local_api diff --git a/docs/doctoring/zotero_metadata_deadline.md b/docs/doctoring/zotero_metadata_deadline.md new file mode 100644 index 00000000..afa9d5a6 --- /dev/null +++ b/docs/doctoring/zotero_metadata_deadline.md @@ -0,0 +1,30 @@ +# Zotero metadata read deadline + +Status: locally verified source repair; protected integration and forward-stack verification remain required. No actual library or paper artifact was read in this experiment. + +## Finding and cause + +[PR #9's unresolved review](https://github.com/ContextualWisdomLab/ConceptWeave/pull/9#discussion_r3935157013) identified that one item per page can trigger up to 50,000 requests, each with a fresh timeout. Exact baseline `a2a84884f67dcac6f6892c958d55450aea6d6c88` has item/byte bounds but no total elapsed-time bound. The same body remains in later research descendants. The real call path is `read_local_snapshot` → `read_snapshot_with` → page transport → complete `classify_snapshot`; the CLI receives a report only after that function returns. + +## Decision and limits + +Reuse that reader, `ReadError::Budget`, and `std::time::Instant`; add a private injected elapsed clock for deterministic tests. Check the five-minute budget before each fetch, after its return and after report classification. Choosing a maximum page count would reject valid short pages without bounding a few slow responses. Replacing transport or adding a cancellation service is unnecessary to deny late results. The trade-off is cooperative admission/completion: an in-flight request keeps its existing timeout and classification is not preempted. System-suspend accounting and monotonic clock implementation are platform-dependent (Rust Project Developers, n.d.). This does not establish provider authentication, source atomicity or any model timeout. + +The accepted report still includes the full observed denominator. No late/partial report escapes, no source is deleted or relabeled, and no transport, public signature, dependency, byte/item ceiling, approval or write authority changes. A longer successful observation window would require a separately evidenced operational decision; do not reduce the paper denominator to make a run pass. + +## Executed evidence + +- Baseline `a2a8488`: 38 tests / 10 unfiltered suites, including two doctests. +- Committed RED `aff539f`: private clock seam plus three regression functions, but no guard. Two rejection tests fail with an actual returned report; the valid short-page control passes. No sleep or real Zotero data is involved. +- GREEN `e6b2a2214b39106ddacc753595b72a699d53d04f`: 41 tests / 10 unfiltered suites, including two doctests; explicit Rust 1.98.0. The tests cover seven individually timely 50-second pages, zero-I/O expired admission, late page, between-page expiry, late classification, late empty library and success one nanosecond below the limit. +- Strict all-target Clippy, warnings-denied rustdoc, release build, formatting and existing CI contract pass. The unchanged coverage script passes 108/108 functions, 703/703 normalized source regions and 100/100 normalized branches. Raw LLVM remains below 100%: 1,051/1,052 lines, 1,600/1,606 regions and 99/100 branches. No threshold or exclusion changed. + +Reproduce with `cargo +1.98.0 test --workspace --locked` and `bash scripts/check_coverage.sh`. Logs are `/tmp/conceptweave-pr9-deadline-{baseline,red,green,coverage}-20260906.log`. The initial provider review references API pagination, which permits bounded pages; that does not supply an application-wide read deadline (Zotero, n.d.). The same TRD amendment removes the separately reviewed stale schema-42 requirement and states the implemented API-v3/present-stable-schema contract; no parser behavior changes for that documentation correction. + +Next: revalidate the final documentation head, normal-push #9 after a fresh writer/head/base check, merge its delta forward through every dependent research PR without reversing later features, and rerun each changed head's checks. Local success does not resolve protected approval, provider transport security or the other open findings. No predecessor may be closed to hide missing propagation. + +## References + +Rust Project Developers. (n.d.). *Instant in std::time* [Rust standard-library documentation]. Retrieved September 6, 2026, from https://doc.rust-lang.org/stable/std/time/struct.Instant.html + +Zotero. (n.d.). *Zotero Web API v3: Basics*. Retrieved September 6, 2026, from https://www.zotero.org/support/dev/web_api/v3/basics diff --git a/docs/doctoring/zotero_source_scope.md b/docs/doctoring/zotero_source_scope.md new file mode 100644 index 00000000..9aa6d5d6 --- /dev/null +++ b/docs/doctoring/zotero_source_scope.md @@ -0,0 +1,54 @@ +# Zotero complete metadata inventory and pending source scope + +Status: verified producer repair in the existing PR #9 owner lane. Dependent report admission, governance binding and whole-library completion remain incomplete. No approval, protected merge, model call or Zotero mutation is claimed. + +## Observed failure and source contract + +The [actual September 6 native/API inspection](https://github.com/ContextualWisdomLab/ConceptWeave/blob/55b1d91dcd299145239a62062ed504fcb6e7bfd1/docs/doctoring/zotero_metadata_visual_audit.md) found 3,719 selected top-level records but only 3,715 bibliographic proposals. Complete attachment/note response audits identified three standalone PDFs and one note outside that worksheet. They are not proven to be three distinct additional papers. Their identities, contents and relationships must not be guessed, discarded or merged. + +Zotero documents all-item and top-level endpoints separately and supports both standalone and child attachments (Zotero, n.d.-a, n.d.-b). Actual Zotero 10.0.1 responses to the filtered `/items/top` diagnostic included children; endpoint naming and header totals therefore did not prove standalone scope. DeepWiki's repository answer described intended `noChildren` behavior but did not explain the observed filtered response. That contradiction remains a provider investigation, not a reason to override actual records or assume a fixed upstream bug. This implementation uses the existing complete `/items` reader, not a new filtered adapter. Context7's previously exhausted allowance was not bypassed; official primary documentation and actual responses support this increment. + +## Root cause and minimal repair + +`read_local_snapshot` → `read_snapshot_with` → `read_snapshot_with_clock` admits the complete bounded metadata read, then calls `classify_snapshot`; the CLI serializes that report. Previously `is_bibliographic` excluded notes, attachments, annotations and records with parents. A scalar total and direct child keys survived, but the other records' metadata and unresolved ancestry disappeared. Matching all 3,715 worksheet slots could therefore be mistaken for accounting for the whole library. + +The producer now moves every excluded record into `unclassified_items`, reusing `ZoteroItem`. Sorted bibliographic proposals plus sorted inventory account for every reader-admitted identity. The existing parent-to-children index is consumed iteratively starting at each bibliographic root. Each adjacency list is removed once; records not reached remain in sorted `pending_source_item_keys`. This retains standalone roots and their descendants, missing-parent trees, disconnected cycles and self-cycles without recursion, another transport or a dependency. Time is O(n log n), auxiliary memory O(n); the private report grows because records previously dropped are now retained. + +No pending record receives a paper disposition or enters bibliographic DOI/title duplicate candidates. Empty pending keys mean only that the observed parent graph reaches bibliographic roots. They do not establish correct semantics, genuine review, independent approval, complete full-text capture, atomicity or write permission. `ItemData` is a metadata projection: note HTML, PDF bytes, attachment-specific fields and unknown provider fields are not preserved by this serialization. Keep original evidence/captures separately and do not advertise a lossless source backup. + +Independent review also found blank source keys admitted by the reader. The existing shared page predicate now rejects an empty/whitespace-only key before accumulation, alongside future item revisions. It does not trim or rewrite valid keys, impose a new key-format regex or change the infallible offline classifier's existing contract. Duplicate keys remain rejected after the complete read; arbitrary offline input is not certified merely because classification terminates. + +## Committed experiments and verification + +- Baseline `f8566408e6a3017cf775fadf2a2f7e50b2d20dc6`: 43 tests / 10 unfiltered suites, including two doctests. +- Inventory RED `48dcd0d`: all three new tests fail on absent pending inventory, not compilation. They cover standalone metadata, nested children, deterministic ordering, orphan/cycle/self-cycle paths, empty input and fully linked evidence. +- Inventory GREEN `220f697`, then adjacency-consumption simplification `48c3525b0061dfba7552f1648f5ad5028b653ab8`: 46 / 10 tests pass. An independent agent reran the three focused tests on the latter exact source and reported no blocking producer finding on reader-admitted input. +- Identity RED `f7a67bf` detects a third request before rejection; its fixture accidentally also repeated a valid key. Refined committed RED `3a57f3b` uses unique remaining keys and fails the actual expected rejection assertion, isolating the missing-identity defect. +- Final source `1e95d6eb979e66ecb7dae4f81f18a6b0a91b7624`: **47 tests / 10 unfiltered suites**, including two doctests; strict Clippy, warnings-denied rustdoc and release build pass. The identity test covers empty/space/control-whitespace on first and subsequent pages with no next request after rejection. +- The unchanged coverage gate passes **123/123 functions, 751/751 normalized source regions and 114/114 normalized branches**. Raw LLVM is not 100%: 1,231/1,232 lines, 1,985/1,993 regions and 113/114 branches. No threshold, exclusion or dependency was changed. Coverage and final verification logs are `/tmp/conceptweave-source-scope-final-{tests,clippy,rustdoc,release,coverage}-20260906.log`. + +## Genuine library replay, distinct from unit evidence + +The release executable built from `48c3525` completed a read-only actual Local API run in **17.61 seconds**. Its SHA-256 was `04e12299babb63c58dd32736373cca8c5a72a9f02aa20ed5366f01e55b935ee7`. This run precedes the later blank-key guard; it is not a run of the final `1e95d6e` executable. + +The private new report `/private/tmp/conceptweave-source-scope-live-C98C52A4-1D9A-4528-BFE8-6A2EBD1AAE98.json` is a single-link regular `0600` file, 3,731,468 bytes, SHA-256 `7a6cd9f7f90a052964f60b5152cd12dc5928d7187ad41e808a0c215bab3b3b97`. It retains **8,326 = 3,715 proposals + 4,611 nonbibliographic records** and exactly **four pending keys**. The prior private attachment/note audit's four identities match the pending list exactly. All 8,326 combined identities are unique and nonblank; all retained nonbibliographic revisions are within library revision 2. The previous report's entire bibliographic proposal list, server identifier and library revision compare equal. Zotero 10.0.1/API 3/schema 44 are unchanged. Only aggregate checks are published; no original titles, keys, server identity, note body or screenshot is committed. + +This is one elapsed-time observation, not a controlled performance improvement claim. No full text was recaptured, no old capture was rewritten, and no bibliographic proposal was promoted to an authentic decision. Existing worksheet decisions and independently approved labels remain 0/3,715, with four unresolved sources additionally visible. + +## Visual Inspection and adoption gates + +This turn attempted native Zotero inspection; the computer-use tool reported that the Mac was locked and automatic unlock failed. No lock bypass, fresh screenshot or new visual pass is claimed. The previous actual screenshot of 3,719 selected items and a retraction warning remains historical evidence; the previous one-item retracted view was accessibility-only because later frames were stale. Unlocking is required for another current visual inspection. API and unit success do not substitute for it. + +The root consumer audit at `22a29c1cfc0918fa34287f3bffe7f400e97f4a0f` identifies the next required integration: + +1. Require both inventory fields on report restoration; missing/null legacy inventory must not become empty. Validate uniqueness, disjointness, exact snapshot complement, original key/version/parent/type evidence and recompute pending ancestry. `SnapshotItemRevision` currently lacks item type. +2. Use shared admission in `build_steward_review_worksheet`, plus direct `prepare_reviewed_golden_set`, `build_duplicate_merge_review_manifest` and `prepare_classification_write_plan`, before any external verifier. These latter routes currently bypass worksheet admission. +3. Keep `StewardReviewProgress.complete`'s bibliographic meaning separate from full-library scope status. Pending-source reconciliation needs bound evidence and its own explicit status; do not mark children as additional unreviewed papers or turn an empty pending list into approval. +4. `classification_proposal_digest` v1 binds only classified proposals, not this inventory. Full-text captures bind the whole report and must reject incompatible restored reports. Never rewrite a prior capture, default scope away or backfill approval identity to make old receipts pass. +5. Preserve parent/child history while normally forwarding the new item-revision, inventory and identity delta through existing owners. This producer is not already present in the root #39 runtime. Run each changed consumer's full tests and current-head hosted checks/reviews; prerequisites, protected rules and independent approval still apply. + +## References + +Zotero. (n.d.-a). *Zotero Web API documentation*. Retrieved September 6, 2026, from https://www.zotero.org/support/dev/web_api/v3/basics + +Zotero. (n.d.-b). *Adding files to your Zotero library*. Retrieved September 6, 2026, from https://www.zotero.org/support/attaching_files diff --git a/docs/product-technical-gap-baseline.md b/docs/product-technical-gap-baseline.md index f7ba7dea..5405f6de 100644 --- a/docs/product-technical-gap-baseline.md +++ b/docs/product-technical-gap-baseline.md @@ -4,7 +4,13 @@ This file records code-current product and technical gaps. Exact PR/check/run coordinates are evidence snapshots, not mutable-head dependencies. Live protected-branch, PR, issue and workflow state wins whenever it advances after this snapshot. Because this documentation update creates a Foundation successor, the Foundation SHA below is the exact pre-refresh head; PR metadata must be refreshed to the resulting successor SHA. -## Protected truth and active stack +## September 6 source inventory checkpoint + +The existing #9 owner now retains every nonbibliographic metadata record and derives unresolved ancestry rather than silently discarding standalone sources. [Source-scope doctoring](doctoring/zotero_source_scope.md) binds committed REDs, final source `1e95d6eb979e66ecb7dae4f81f18a6b0a91b7624`, **47 tests / 10 unfiltered suites**, strict checks and the unchanged coverage gate. The earlier inventory executable at `48c3525` genuinely reads 8,326 records into 3,715 unchanged bibliographic proposals plus 4,611 other records, with exactly the four previously audited standalone identities pending. A later shared-reader guard also rejects blank identities; no actual final-guard executable replay is implied. + +The earlier source findings are repaired locally, not yet propagated into root #39. Required downstream restoration/identity accounting, pending-source reconciliation, approval binding and full-library completion gates remain open; neither zero pending keys nor successful classification grants semantic or write authority. Current native Visual Inspection was attempted but the Mac is locked, so no new screenshot was verified. Historical source scope, authentic worksheet decisions/independent approvals 0/3,715, plus four unresolved sources remain distinct. This checkpoint does not refresh every historical PR coordinate below or imply protected merge/release. + +## Historical protected truth and active stack Protected/default `main` remains `f4f440dd58c77d7cd90dff8a1eb2eeb9a9940425`; only the bootstrap state is shipped there and no immutable ConceptWeave release exists. diff --git a/scripts/check_coverage.sh b/scripts/check_coverage.sh index 2f691898..b60a8e68 100755 --- a/scripts/check_coverage.sh +++ b/scripts/check_coverage.sh @@ -2,7 +2,7 @@ set -euo pipefail coverage_toolchain="${COVERAGE_TOOLCHAIN:-nightly-2026-08-20}" -trap 'rm -f coverage.json source-branches.json' EXIT +trap 'rm -f coverage.json source-branches.json source-regions.json' EXIT cargo "+${coverage_toolchain}" llvm-cov \ --workspace \ @@ -21,6 +21,49 @@ jq -r ' | "COVERAGE_GAP file=\(.filename) lines=\(.summary.lines.percent) functions=\(.summary.functions.percent) regions=\(.summary.regions.percent)" ' coverage.json +jq ' + [ + .data[0].functions[] + | select(.name | contains("5tests") | not) + | .filenames as $files + | .regions[] + | { + file: $files[.[5]], + line_start: .[0], + column_start: .[1], + line_end: .[2], + column_end: .[3], + count: .[4] + } + | select(.file | contains("/tests/") | not) + ] + | sort_by(.file, .line_start, .column_start, .line_end, .column_end) + | group_by([.file, .line_start, .column_start, .line_end, .column_end]) + | map({ + file: .[0].file, + line_start: .[0].line_start, + column_start: .[0].column_start, + line_end: .[0].line_end, + column_end: .[0].column_end, + count: (map(.count) | add) + }) +' coverage.json > source-regions.json + +jq ' + { + count: length, + covered: ([.[] | select(.count > 0)] | length), + notcovered: ([.[] | select(.count == 0)] | length) + } + | .percent = (if .count == 0 then 100 else (.covered * 100 / .count) end) +' source-regions.json + +jq -r ' + .[] + | select(.count == 0) + | "REGION_GAP file=\(.file) start=\(.line_start):\(.column_start) end=\(.line_end):\(.column_end)" +' source-regions.json + jq ' [ .data[0].files[] @@ -65,9 +108,8 @@ jq -r ' ' source-branches.json jq -e ' - .data[0].totals.lines.percent == 100 and - .data[0].totals.functions.percent == 100 and - .data[0].totals.regions.percent == 100 + .data[0].totals.functions.percent == 100 ' coverage.json >/dev/null +jq -e 'all(.[]; .count > 0)' source-regions.json >/dev/null jq -e 'all(.[]; .true_count > 0 and .false_count > 0)' source-branches.json >/dev/null