From 432421812587ebe1da1dab46e82698fd9129189e Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Fri, 4 Sep 2026 23:15:16 +0900 Subject: [PATCH 01/19] experiment: add reversible duplicate review manifest Signed-off-by: Seongho Bae --- crates/conceptweave-zotero/src/lib.rs | 205 ++++++++++++++++++ .../tests/duplicate_merge_review_manifest.rs | 179 +++++++++++++++ docs/PRD.md | 2 + docs/TRD.md | 2 + docs/UML.md | 3 + docs/adr/0006-zotero-research-intake.md | 2 + docs/product-technical-gap-baseline.md | 2 +- 7 files changed, 394 insertions(+), 1 deletion(-) create mode 100644 crates/conceptweave-zotero/tests/duplicate_merge_review_manifest.rs diff --git a/crates/conceptweave-zotero/src/lib.rs b/crates/conceptweave-zotero/src/lib.rs index d7a96328..3afe9680 100644 --- a/crates/conceptweave-zotero/src/lib.rs +++ b/crates/conceptweave-zotero/src/lib.rs @@ -150,6 +150,104 @@ pub struct DuplicateCandidate { pub item_keys: Vec, } +/// One steward decision selecting the canonical identity for a duplicate cluster. +#[derive(Debug, Clone, PartialEq, Eq, Deserialize, Serialize)] +pub struct DuplicateMergeDecision { + /// Candidate identity kind from the classification report. + pub identity_kind: String, + /// Candidate normalized identity from the classification report. + pub normalized_identity: String, + /// Existing Zotero item retained as the canonical reference. + pub retained_item_key: String, +} + +/// Snapshot-bound duplicate decisions verified by the caller's governance boundary. +#[derive(Debug, Clone, PartialEq, Eq, Deserialize, Serialize)] +pub struct ReviewedDuplicateMergeSet { + /// Opaque steward review receipt. + pub review_id: String, + /// Opaque governance authority receipt; contains no person identity. + pub authority_receipt: String, + /// Exact Zotero library revision reviewed by the steward. + pub library_version: u64, + /// Exact classifier rule revision reviewed by the steward. + pub rule_revision: String, + /// Exact raw-snapshot digest reviewed by the steward. + pub snapshot_digest: String, + /// Exactly one decision for every duplicate candidate. + pub decisions: Vec, +} + +/// One reversible canonical-key mapping; Zotero source records remain unchanged. +#[derive(Debug, Clone, PartialEq, Eq, Serialize)] +pub struct DuplicateMergeOperation { + /// Candidate identity kind. + pub identity_kind: String, + /// Candidate normalized identity retained only in the local manifest. + pub normalized_identity: String, + /// Steward-selected canonical Zotero item key. + pub retained_item_key: String, + /// Exact source revisions participating in the decision. + pub source_items: Vec, + /// Mapping before the reviewed canonicalization; every item maps to itself. + pub before_canonical_keys: BTreeMap, + /// Reviewed mapping after canonicalization; every item maps to the retained key. + pub after_canonical_keys: BTreeMap, + /// Exact inverse plan restoring the pre-review mapping. + pub rollback_canonical_keys: BTreeMap, +} + +/// Aggregate of reviewed, reversible duplicate identity operations. +#[derive(Debug, Clone, PartialEq, Eq, Serialize)] +pub struct DuplicateMergeReviewManifest { + /// Opaque steward review receipt. + pub review_id: String, + /// Opaque governance authority receipt. + pub authority_receipt: String, + /// Exact Zotero library revision reviewed by the steward. + pub library_version: u64, + /// Exact classifier rule revision reviewed by the steward. + pub rule_revision: String, + /// Exact raw-snapshot digest shared with the reviewed decisions. + pub snapshot_digest: String, + /// Deterministically ordered canonical-key operations. + pub operations: Vec, + /// Classification never deletes or mutates Zotero source records. + pub source_records_preserved: bool, +} + +/// A fail-closed duplicate review contract violation. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum DuplicateReviewError { + /// Required review metadata or a decision is missing. + InvalidReview, + /// The review belongs to another raw snapshot. + SnapshotMismatch, + /// A decision does not identify a report candidate. + UnknownCandidate, + /// More than one decision targets the same candidate. + DuplicateDecision, + /// The retained key is not a member of the candidate cluster. + InvalidRetainedItem, + /// The caller's governance boundary rejected the complete review set. + UnverifiedApproval, +} + +impl fmt::Display for DuplicateReviewError { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter.write_str(match self { + Self::InvalidReview => "duplicate review metadata or decisions are invalid", + Self::SnapshotMismatch => "duplicate review does not match the report snapshot", + Self::UnknownCandidate => "duplicate review contains an unknown candidate", + Self::DuplicateDecision => "duplicate review repeats a candidate decision", + Self::InvalidRetainedItem => "retained item is absent from its duplicate candidate", + Self::UnverifiedApproval => "duplicate review approval is unverified", + }) + } +} + +impl std::error::Error for DuplicateReviewError {} + /// Complete local classification report for one immutable library version. #[derive(Debug, Serialize)] pub struct ClassificationReport { @@ -425,6 +523,113 @@ where }) } +/// Builds a local-only reversible canonical-key manifest from steward-reviewed decisions. +pub fn build_duplicate_merge_review_manifest( + report: &ClassificationReport, + reviewed: &ReviewedDuplicateMergeSet, + verify_review: F, +) -> Result +where + F: FnOnce(&ReviewedDuplicateMergeSet) -> bool, +{ + if reviewed.review_id.trim().is_empty() + || reviewed.authority_receipt.trim().is_empty() + || reviewed.snapshot_digest.trim().is_empty() + || reviewed.rule_revision.trim().is_empty() + || reviewed.decisions.len() != report.duplicate_candidates.len() + { + return Err(DuplicateReviewError::InvalidReview); + } + if reviewed.snapshot_digest != report.snapshot_digest + || reviewed.library_version != report.library_version + || reviewed.rule_revision != report.rule_revision + { + return Err(DuplicateReviewError::SnapshotMismatch); + } + if !verify_review(reviewed) { + return Err(DuplicateReviewError::UnverifiedApproval); + } + + let item_revisions = report + .snapshot_items + .iter() + .map(|item| (item.item_key.as_str(), item.item_version)) + .collect::>(); + let candidates = report + .duplicate_candidates + .iter() + .map(|candidate| { + ( + ( + candidate.identity_kind, + candidate.normalized_identity.as_str(), + ), + candidate, + ) + }) + .collect::>(); + let mut seen_candidates = BTreeSet::new(); + let mut operations = Vec::with_capacity(reviewed.decisions.len()); + + for decision in &reviewed.decisions { + let candidate_key = ( + decision.identity_kind.as_str(), + decision.normalized_identity.as_str(), + ); + if !seen_candidates.insert(candidate_key) { + return Err(DuplicateReviewError::DuplicateDecision); + } + let candidate = candidates + .get(&candidate_key) + .ok_or(DuplicateReviewError::UnknownCandidate)?; + if !candidate.item_keys.contains(&decision.retained_item_key) { + return Err(DuplicateReviewError::InvalidRetainedItem); + } + + let source_items = candidate + .item_keys + .iter() + .map(|item_key| SnapshotItemRevision { + item_key: item_key.clone(), + item_version: item_revisions[item_key.as_str()], + }) + .collect::>(); + let before_canonical_keys = candidate + .item_keys + .iter() + .map(|item_key| (item_key.clone(), item_key.clone())) + .collect::>(); + let after_canonical_keys = candidate + .item_keys + .iter() + .map(|item_key| (item_key.clone(), decision.retained_item_key.clone())) + .collect::>(); + operations.push(DuplicateMergeOperation { + identity_kind: decision.identity_kind.clone(), + normalized_identity: decision.normalized_identity.clone(), + retained_item_key: decision.retained_item_key.clone(), + source_items, + rollback_canonical_keys: before_canonical_keys.clone(), + before_canonical_keys, + after_canonical_keys, + }); + } + + operations.sort_by(|left, right| { + (&left.identity_kind, &left.normalized_identity) + .cmp(&(&right.identity_kind, &right.normalized_identity)) + }); + Ok(DuplicateMergeReviewManifest { + review_id: reviewed.review_id.clone(), + authority_receipt: reviewed.authority_receipt.clone(), + library_version: reviewed.library_version, + rule_revision: reviewed.rule_revision.clone(), + snapshot_digest: reviewed.snapshot_digest.clone(), + operations, + source_records_preserved: true, + }) +} + /// Failure raised when a bounded, immutable Local API read cannot be proven. #[derive(Debug)] pub enum ReadError { diff --git a/crates/conceptweave-zotero/tests/duplicate_merge_review_manifest.rs b/crates/conceptweave-zotero/tests/duplicate_merge_review_manifest.rs new file mode 100644 index 00000000..0fad4eea --- /dev/null +++ b/crates/conceptweave-zotero/tests/duplicate_merge_review_manifest.rs @@ -0,0 +1,179 @@ +use conceptweave_zotero::{ + Disposition, DuplicateMergeDecision, DuplicateReviewError, ItemData, ReviewedDuplicateMergeSet, + ZoteroItem, build_duplicate_merge_review_manifest, classify_snapshot, +}; + +fn item(key: &str, version: u64, title: &str, doi: &str) -> ZoteroItem { + ZoteroItem { + key: key.into(), + version, + data: ItemData { + item_type: "journalArticle".into(), + title: title.into(), + abstract_note: String::new(), + doi: doi.into(), + parent_item: String::new(), + collections: vec![], + tags: vec![], + }, + } +} + +fn report() -> conceptweave_zotero::ClassificationReport { + classify_snapshot( + "9.0.6".into(), + None, + 42, + vec![ + item("A", 7, "Ontology Learning", "10.1000/example"), + item( + "B", + 9, + "Ontology Learning Copy", + "https://doi.org/10.1000/example", + ), + item("C", 3, "Shared Ontology Title", ""), + item("D", 4, "Shared Ontology Title", ""), + ], + ) +} + +fn reviewed(report: &conceptweave_zotero::ClassificationReport) -> ReviewedDuplicateMergeSet { + ReviewedDuplicateMergeSet { + review_id: "review-duplicate-1".into(), + authority_receipt: "authority-receipt-1".into(), + library_version: report.library_version, + rule_revision: report.rule_revision.into(), + snapshot_digest: report.snapshot_digest.clone(), + decisions: vec![ + DuplicateMergeDecision { + identity_kind: "title".into(), + normalized_identity: "shared ontology title".into(), + retained_item_key: "C".into(), + }, + DuplicateMergeDecision { + identity_kind: "doi".into(), + normalized_identity: "10.1000/example".into(), + retained_item_key: "A".into(), + }, + ], + } +} + +#[test] +fn reviewed_duplicate_decision_has_exact_before_after_and_rollback_mappings() { + let report = report(); + let manifest = build_duplicate_merge_review_manifest(&report, &reviewed(&report), |_| true) + .expect("synthetic reviewed duplicate must produce a manifest"); + + assert!(manifest.source_records_preserved); + assert_eq!(manifest.authority_receipt, "authority-receipt-1"); + assert_eq!(manifest.library_version, report.library_version); + assert_eq!(manifest.rule_revision, report.rule_revision); + assert_eq!(manifest.operations.len(), 2); + assert_eq!(manifest.operations[0].identity_kind, "doi"); + assert_eq!(manifest.operations[1].identity_kind, "title"); + let operation = manifest + .operations + .iter() + .find(|operation| operation.identity_kind == "doi") + .expect("DOI duplicate operation must exist"); + assert_eq!(operation.retained_item_key, "A"); + assert_eq!(operation.source_items[0].item_version, 7); + assert_eq!(operation.source_items[1].item_version, 9); + assert_eq!(operation.before_canonical_keys["B"], "B"); + assert_eq!(operation.after_canonical_keys["B"], "A"); + assert_eq!( + operation.rollback_canonical_keys, + operation.before_canonical_keys + ); + assert_eq!( + report.classified_items[0].proposed_disposition, + Disposition::Generation + ); + let serialized = serde_json::to_value(&manifest).expect("manifest must serialize"); + assert_eq!( + serialized["operations"][0]["rollback_canonical_keys"], + serialized["operations"][0]["before_canonical_keys"] + ); +} + +#[test] +fn duplicate_review_contract_fails_closed() { + let report = report(); + let mut review = reviewed(&report); + + assert_eq!( + build_duplicate_merge_review_manifest(&report, &review, |_| false), + Err(DuplicateReviewError::UnverifiedApproval) + ); + review.snapshot_digest = "sha256:stale".into(); + assert_eq!( + build_duplicate_merge_review_manifest(&report, &review, |_| true), + Err(DuplicateReviewError::SnapshotMismatch) + ); + review = reviewed(&report); + review.decisions[1].retained_item_key = "missing".into(); + assert_eq!( + build_duplicate_merge_review_manifest(&report, &review, |_| true), + Err(DuplicateReviewError::InvalidRetainedItem) + ); + review = reviewed(&report); + review.decisions[1].normalized_identity = "missing".into(); + assert_eq!( + build_duplicate_merge_review_manifest(&report, &review, |_| true), + Err(DuplicateReviewError::UnknownCandidate) + ); + review = reviewed(&report); + review.decisions[1] = review.decisions[0].clone(); + assert_eq!( + build_duplicate_merge_review_manifest(&report, &review, |_| true), + Err(DuplicateReviewError::DuplicateDecision) + ); + review.decisions.clear(); + assert_eq!( + build_duplicate_merge_review_manifest(&report, &review, |_| true), + Err(DuplicateReviewError::InvalidReview) + ); + review = reviewed(&report); + review.authority_receipt.clear(); + assert_eq!( + build_duplicate_merge_review_manifest(&report, &review, |_| true), + Err(DuplicateReviewError::InvalidReview) + ); + for clear_field in [ + |review: &mut ReviewedDuplicateMergeSet| review.review_id.clear(), + |review: &mut ReviewedDuplicateMergeSet| review.snapshot_digest.clear(), + |review: &mut ReviewedDuplicateMergeSet| review.rule_revision.clear(), + ] { + review = reviewed(&report); + clear_field(&mut review); + assert_eq!( + build_duplicate_merge_review_manifest(&report, &review, |_| true), + Err(DuplicateReviewError::InvalidReview) + ); + } + review = reviewed(&report); + review.library_version += 1; + assert_eq!( + build_duplicate_merge_review_manifest(&report, &review, |_| true), + Err(DuplicateReviewError::SnapshotMismatch) + ); + review = reviewed(&report); + review.rule_revision = "stale-rule".into(); + assert_eq!( + build_duplicate_merge_review_manifest(&report, &review, |_| true), + Err(DuplicateReviewError::SnapshotMismatch) + ); + + for (error, fragment) in [ + (DuplicateReviewError::InvalidReview, "invalid"), + (DuplicateReviewError::SnapshotMismatch, "snapshot"), + (DuplicateReviewError::UnknownCandidate, "unknown"), + (DuplicateReviewError::DuplicateDecision, "repeats"), + (DuplicateReviewError::InvalidRetainedItem, "absent"), + (DuplicateReviewError::UnverifiedApproval, "unverified"), + ] { + assert!(error.to_string().contains(fragment)); + } +} diff --git a/docs/PRD.md b/docs/PRD.md index 40de5db4..e1fba297 100644 --- a/docs/PRD.md +++ b/docs/PRD.md @@ -58,6 +58,8 @@ All LLM-backed induction uses `contextual-orchestrator`. Model output is untrust Read one immutable Zotero Local API library-version snapshot and propose exactly one research disposition for every top-level bibliographic item. Each proposal retains the item key/version, exact matched metadata values, rule revision, linked child records, and any model receipt. Weak evidence and evidence that matches multiple specific disposition families must abstain into steward review. Duplicate DOI/title identities are review candidates only: intake never merges, deletes, or silently mutates Zotero records. +For every detected duplicate cluster, accept exactly one externally verified steward decision selecting an existing canonical item. Produce a local-only manifest that binds the decision to the raw snapshot and records source revisions plus before, after, and rollback canonical-key mappings. Classification preserves every Zotero source record. + Evaluate classifier quality only against a steward-reviewed local golden set whose governance receipt is externally verified and bound to the canonical SHA-256 digest of the complete Zotero classification report plus its item-key/item-version coordinates. Abstention is a prediction outcome, never an approved truth label. Evaluation emits the verified library revision, rule revision, opaque snapshot digest, and aggregate counts for exact matches, abstentions, and per-disposition true-positive/predicted/expected totals; it must not copy Zotero keys, reviewer identity, or bibliographic text into the result. Every successful classification report includes aggregate evidence for snapshot coverage, proposal coverage, provenance completeness, abstentions, duplicate candidates, disposition totals, and zero unreported failures. diff --git a/docs/TRD.md b/docs/TRD.md index 5990ae08..27e9d720 100644 --- a/docs/TRD.md +++ b/docs/TRD.md @@ -63,6 +63,8 @@ Evaluation must separate extraction recall, semantic correctness, structural cor Every top-level bibliographic record receives exactly one proposed disposition. `NeedsStewardReview` also records a deterministic abstention reason so missing classification metadata, vocabulary unsupported by the current deterministic rules, and present-but-unmatched metadata are distinguishable. DOI duplicate identity normalization treats bare DOI values, `doi:`, `doi.org`, and legacy `dx.doi.org` resolver forms as the same identity when their normalized DOI is equal. +Duplicate review is independent of subject classification. A reviewed decision set must match the exact raw-snapshot digest, cover every duplicate candidate exactly once, select a retained key already in that cluster, and pass an external governance verifier. Its manifest records item revisions, identity mappings before and after canonicalization, and the exact rollback mapping. These mappings affect only downstream identity resolution; Zotero records are neither mutated nor deleted. + Golden-set evaluation accepts only a governance receipt verified with the complete reviewed set by a caller-owned authorization boundary. Its library version, rule revision, canonical SHA-256 content digest, and every observed parent/child item-key/item-version identity must bind the classification report. The digest covers every raw Zotero item in canonical key order. Blank, duplicate, unknown, stale, content-mismatched, label-mismatched, or abstention-as-truth inputs fail closed. The output retains the verified library version, rule revision, and opaque snapshot digest, but contains no item keys, reviewer identity, or bibliographic text. Production authorization remains Keyverse/governance-owned; this crate passes the complete reviewed labels to that boundary instead of minting authority. A successful classification report carries an `audit_summary` whose snapshot, bibliographic, proposed-disposition, provenance-complete, abstention, duplicate-candidate, failure, and per-disposition counts are derived from the same in-memory immutable snapshot. Zotero item version zero remains a valid observed coordinate for never-synced Zotero 9 records; provenance completeness rejects a missing item key rather than inventing a positive-only version invariant. Reader failures return an error instead of a partial report; therefore a returned report records `failure_count=0` rather than hiding partial failures. diff --git a/docs/UML.md b/docs/UML.md index 703a379b..05f375d3 100644 --- a/docs/UML.md +++ b/docs/UML.md @@ -55,5 +55,8 @@ sequenceDiagram Intake->>Intake: classify or abstain; link children; find duplicate candidates Intake->>Report: write proposals and evidence Report->>Steward: review dispositions and merge candidates + Steward->>Intake: verified canonical-item decisions + Intake->>Report: before/after/rollback identity manifest + Report-->>Steward: reversible local mapping; source records preserved Intake-->>Zotero: no mutation ``` diff --git a/docs/adr/0006-zotero-research-intake.md b/docs/adr/0006-zotero-research-intake.md index eb474eb7..a407d96b 100644 --- a/docs/adr/0006-zotero-research-intake.md +++ b/docs/adr/0006-zotero-research-intake.md @@ -19,6 +19,8 @@ The adapter links child records, emits exactly one deterministic proposed dispos Matched metadata values are copied into the local-only evidence receipt for replay. This is necessary for abstract-only matches because a later Zotero revision cannot reconstruct the exact text used for an earlier proposal from item key/version alone. DOI/title matches remain reversible duplicate candidates, including legacy `dx.doi.org` resolver forms. +Duplicate candidates become canonical references only through a snapshot-bound, externally verified steward decision. The resulting local manifest retains source revisions and complete before/after/rollback key mappings. It changes downstream identity resolution only; classification does not merge, delete, or mutate Zotero source records. + Classifier quality is measured only against local steward-reviewed labels whose complete reviewed set is verified outside this crate and bound to the exact library version, rule revision, canonical SHA-256 raw-snapshot digest, and every observed parent/child item-key/item-version coordinate. `NeedsStewardReview` is an abstention prediction and cannot be approved truth. Evaluation returns the verified revisions and opaque digest with aggregate integer evidence; Zotero keys, reviewer identity, and bibliographic text are omitted. Missing, stale, content- or label-mismatched, unverified, unknown, duplicate, or invalid review identities fail closed. Every successful report includes an aggregate audit summary computed from the same captured snapshot. Zotero 9 item version zero is preserved as a valid never-synced source coordinate, not treated as missing provenance. Partial reads never produce a report, so successful output explicitly records zero failures alongside snapshot, proposal, provenance, abstention, duplicate, and per-disposition totals. diff --git a/docs/product-technical-gap-baseline.md b/docs/product-technical-gap-baseline.md index d0e89c97..cb2e680a 100644 --- a/docs/product-technical-gap-baseline.md +++ b/docs/product-technical-gap-baseline.md @@ -44,7 +44,7 @@ Protected central source is `.github/main@c31d2e5471fc5daf9d72ff67cde6a8874b736d Local evidence on 2026-09-04 showed Zotero 9.0.6, Local API v3/schema 42, library version 12341, 8,326 total items, and 3,719 top-level items. The corrected read-only run observed all 8,326 records at that single version and classified all 3,715 top-level bibliographic records; four top-level note/attachment/annotation records were correctly excluded. It proposed 56 adjacent-evidence records, 1 semantic-consumption bridge, and 3,658 steward-review abstentions, linked children for 3,287 records, and surfaced 49 reversible duplicate groups (18 DOI, 31 title). No live record matched multiple specific disposition families; the tested conflict path still abstains fail-closed. Token-boundary matching prevents strings such as `knowledge` from becoming false OWL evidence. These are local aggregate observations, not reviewed truth or applied Zotero changes. The report stays outside the repository. -The golden-set evaluation contract now records aggregate precision/recall numerators and denominators, requires an externally verified governance receipt bound to the complete item-key/item-version snapshot, rejects abstention as expected truth, and retains verified revisions plus an opaque snapshot digest so detached metrics remain attributable. Item and reviewer identities stay out of its output. Successful classification reports also carry same-snapshot aggregate coverage, provenance, abstention, duplicate, disposition, and failure evidence. Synthetic fixtures verify both contracts; no real precision/recall claim exists until a steward supplies reviewed local labels and the production authorization adapter verifies the receipt. Multilingual rule expansion remains a later evidence-driven change and must not reduce abstention safety. Zotero write-back remains blocked by the installed v9 capability; a Zotero 10+ change must satisfy ADR 0006 preconditions. A dedicated utility repository remains unnecessary until an independently released cross-product contract exists. +The golden-set evaluation contract now records aggregate precision/recall numerators and denominators, requires an externally verified governance receipt bound to the complete item-key/item-version snapshot, rejects abstention as expected truth, and retains verified revisions plus an opaque snapshot digest so detached metrics remain attributable. Item and reviewer identities stay out of its output. Successful classification reports also carry same-snapshot aggregate coverage, provenance, abstention, duplicate, disposition, and failure evidence. Duplicate clusters now produce a snapshot-bound local review manifest only after external steward verification; exact source revisions and before/after/rollback canonical mappings are retained while Zotero records remain unchanged. Synthetic fixtures verify these contracts; no real precision/recall or duplicate merge claim exists until a steward supplies reviewed local decisions and the production authorization adapter verifies them. Multilingual rule expansion remains a later evidence-driven change and must not reduce abstention safety. Zotero write-back remains blocked by the installed v9 capability; a Zotero 10+ change must satisfy ADR 0006 preconditions. A dedicated utility repository remains unnecessary until an independently released cross-product contract exists. 1. **Concrete Source Observation adapter** — maintained Rust PostgreSQL driver behind `conceptweave-source-port`; adapter-local credential resolution; explicit read-only mode; statement timeout, cancellation, row/byte/concurrency budgets; complete immutable snapshot or fail closed; deterministic replay against a frozen anonymized GRC-shaped fixture. 2. **Ontology discovery** — deterministic term/concept/taxonomy/non-taxonomic-relation candidate generation with exact source receipts and abstention for unsupported semantics. From 667cbe6859212269396252e86e0068ca2b930c5f Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Fri, 4 Sep 2026 23:34:02 +0900 Subject: [PATCH 02/19] test(zotero): lock duplicate review integrity findings --- .../tests/duplicate_merge_review_manifest.rs | 103 ++++++++++++++++++ 1 file changed, 103 insertions(+) diff --git a/crates/conceptweave-zotero/tests/duplicate_merge_review_manifest.rs b/crates/conceptweave-zotero/tests/duplicate_merge_review_manifest.rs index 0fad4eea..01a62cbb 100644 --- a/crates/conceptweave-zotero/tests/duplicate_merge_review_manifest.rs +++ b/crates/conceptweave-zotero/tests/duplicate_merge_review_manifest.rs @@ -177,3 +177,106 @@ fn duplicate_review_contract_fails_closed() { assert!(error.to_string().contains(fragment)); } } + +#[test] +fn overlapping_duplicate_groups_require_one_consistent_canonical_choice() { + let report = classify_snapshot( + "9.0.6".into(), + None, + 42, + vec![ + item("A", 7, "Same Identity", "10.1000/overlap"), + item("B", 9, "Same Identity", "10.1000/overlap"), + ], + ); + assert_eq!(report.duplicate_candidates.len(), 2); + + let reviewed = ReviewedDuplicateMergeSet { + review_id: "review-overlap".into(), + authority_receipt: "authority-overlap".into(), + library_version: report.library_version, + rule_revision: report.rule_revision.into(), + snapshot_digest: report.snapshot_digest.clone(), + decisions: vec![ + DuplicateMergeDecision { + identity_kind: "doi".into(), + normalized_identity: "10.1000/overlap".into(), + retained_item_key: "A".into(), + }, + DuplicateMergeDecision { + identity_kind: "title".into(), + normalized_identity: "same identity".into(), + retained_item_key: "B".into(), + }, + ], + }; + + assert_eq!( + build_duplicate_merge_review_manifest(&report, &reviewed, |_| true), + Err(DuplicateReviewError::InvalidReview), + "overlapping duplicate groups cannot emit conflicting A->B and B->A canonical mappings" + ); +} + +#[test] +fn duplicate_review_rejects_ambiguous_snapshot_key_revisions() { + let report = classify_snapshot( + "9.0.6".into(), + None, + 42, + vec![ + item("DUPKEY", 7, "Same Identity", "10.1000/duplicate-key"), + item("DUPKEY", 9, "Same Identity", "10.1000/duplicate-key"), + ], + ); + assert_eq!(report.snapshot_items.len(), 2); + assert!(!report.duplicate_candidates.is_empty()); + + let reviewed = ReviewedDuplicateMergeSet { + review_id: "review-duplicate-key".into(), + authority_receipt: "authority-duplicate-key".into(), + library_version: report.library_version, + rule_revision: report.rule_revision.into(), + snapshot_digest: report.snapshot_digest.clone(), + decisions: report + .duplicate_candidates + .iter() + .map(|candidate| DuplicateMergeDecision { + identity_kind: candidate.identity_kind.into(), + normalized_identity: candidate.normalized_identity.clone(), + retained_item_key: "DUPKEY".into(), + }) + .collect(), + }; + + assert_eq!( + build_duplicate_merge_review_manifest(&report, &reviewed, |_| true), + Err(DuplicateReviewError::InvalidReview), + "duplicate Zotero keys must fail before a BTreeMap can collapse distinct observed revisions" + ); +} + +#[test] +fn duplicate_review_domain_language_and_governance_handoff_are_documented() { + let ubiquitous_language = include_str!("../../../docs/UBIQUITOUS_LANGUAGE.md"); + for term in [ + "Reviewed Duplicate Merge Set", + "Authority Receipt", + "Canonical-Key Operation", + ] { + assert!( + ubiquitous_language.contains(term), + "UBIQUITOUS_LANGUAGE.md must define `{term}`" + ); + } + + let context_map = include_str!("../../../docs/CONTEXT_MAP.md"); + assert!( + context_map.contains("Research Intake -> Governance & Publication"), + "CONTEXT_MAP.md must name the Research Intake to Governance & Publication verification handoff" + ); + assert!( + context_map.contains("Anti-Corruption Layer"), + "the duplicate-review governance handoff must preserve an explicit ACL boundary" + ); +} From f7ef6504df64753bf81354e216a5eab49c7a4bf0 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Fri, 4 Sep 2026 23:37:15 +0900 Subject: [PATCH 03/19] fix(zotero): enforce duplicate review integrity --- crates/conceptweave-zotero/src/lib.rs | 19 +++++++++++++++++++ docs/CONTEXT_MAP.md | 1 + docs/UBIQUITOUS_LANGUAGE.md | 3 +++ 3 files changed, 23 insertions(+) diff --git a/crates/conceptweave-zotero/src/lib.rs b/crates/conceptweave-zotero/src/lib.rs index dac871cf..c87ba49d 100644 --- a/crates/conceptweave-zotero/src/lib.rs +++ b/crates/conceptweave-zotero/src/lib.rs @@ -550,6 +550,16 @@ where return Err(DuplicateReviewError::UnverifiedApproval); } + if report + .snapshot_items + .iter() + .map(|item| item.item_key.as_str()) + .collect::>() + .len() + != report.snapshot_items.len() + { + return Err(DuplicateReviewError::InvalidReview); + } let item_revisions = report .snapshot_items .iter() @@ -569,6 +579,7 @@ where }) .collect::>(); let mut seen_candidates = BTreeSet::new(); + let mut canonical_choices = BTreeMap::<&str, &str>::new(); let mut operations = Vec::with_capacity(reviewed.decisions.len()); for decision in &reviewed.decisions { @@ -585,6 +596,14 @@ where if !candidate.item_keys.contains(&decision.retained_item_key) { return Err(DuplicateReviewError::InvalidRetainedItem); } + for item_key in &candidate.item_keys { + if canonical_choices + .insert(item_key, decision.retained_item_key.as_str()) + .is_some_and(|retained_key| retained_key != decision.retained_item_key) + { + return Err(DuplicateReviewError::InvalidReview); + } + } let source_items = candidate .item_keys diff --git a/docs/CONTEXT_MAP.md b/docs/CONTEXT_MAP.md index bc769582..5da32fef 100644 --- a/docs/CONTEXT_MAP.md +++ b/docs/CONTEXT_MAP.md @@ -6,6 +6,7 @@ - Semantic Discovery -> Model Validation: **Conformist to published candidate contract**; validation must not rewrite discovery evidence. - Model Validation -> Governance & Publication: **Customer/Supplier**; governance consumes deterministic validation receipts. - Governance & Publication -> Interoperability: **Published Language**; adapters consume immutable release contracts. +- Research Intake -> Governance & Publication: **Anti-Corruption Layer**; Governance verifies the complete reviewed duplicate set and returns only an opaque authority receipt before Intake emits canonical-key operations. ## External relationships diff --git a/docs/UBIQUITOUS_LANGUAGE.md b/docs/UBIQUITOUS_LANGUAGE.md index 0c1d0c25..24f3d68e 100644 --- a/docs/UBIQUITOUS_LANGUAGE.md +++ b/docs/UBIQUITOUS_LANGUAGE.md @@ -16,3 +16,6 @@ | Dimension | Governed categorical or temporal axis used to group/filter analytical facts. | | Measure | Governed calculation with explicit expression, grain, units, null semantics, and evidence. | | Semantic Steward | Authorized reviewer responsible for accepting or rejecting semantic meaning. | +| Reviewed Duplicate Merge Set | Complete steward decisions selecting one consistent canonical item across every overlapping duplicate group in a snapshot. | +| Authority Receipt | Opaque proof checked by the Governance & Publication boundary; it contains no reviewer identity or credential. | +| Canonical-Key Operation | Reversible local mapping from each duplicate source key to one retained key, with the exact rollback mapping. | From 43c8a5283ae203e2f849540e64588435ab4b4053 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Fri, 4 Sep 2026 23:42:59 +0900 Subject: [PATCH 04/19] test(zotero): cover transitive duplicate components --- .../duplicate_merge_review_components.rs | 101 ++++++++++++++++++ 1 file changed, 101 insertions(+) create mode 100644 crates/conceptweave-zotero/tests/duplicate_merge_review_components.rs diff --git a/crates/conceptweave-zotero/tests/duplicate_merge_review_components.rs b/crates/conceptweave-zotero/tests/duplicate_merge_review_components.rs new file mode 100644 index 00000000..f9c7689d --- /dev/null +++ b/crates/conceptweave-zotero/tests/duplicate_merge_review_components.rs @@ -0,0 +1,101 @@ +use conceptweave_zotero::{ + DuplicateMergeDecision, DuplicateReviewError, ItemData, ReviewedDuplicateMergeSet, ZoteroItem, + build_duplicate_merge_review_manifest, classify_snapshot, +}; + +fn item(key: &str, version: u64, title: &str, doi: &str) -> ZoteroItem { + ZoteroItem { + key: key.into(), + version, + data: ItemData { + item_type: "journalArticle".into(), + title: title.into(), + abstract_note: String::new(), + doi: doi.into(), + parent_item: String::new(), + collections: vec![], + tags: vec![], + }, + } +} + +#[test] +fn transitive_duplicate_component_accepts_one_component_level_canonical_key() { + let report = classify_snapshot( + "9.0.6".into(), + None, + 42, + vec![ + item("A", 1, "Alpha", "10.1000/ab"), + item("B", 2, "Bridge", "10.1000/ab"), + item("C", 3, "Bridge", "10.1000/cd"), + item("D", 4, "Delta", "10.1000/cd"), + ], + ); + assert_eq!(report.duplicate_candidates.len(), 3); + + let reviewed = ReviewedDuplicateMergeSet { + review_id: "review-transitive-component".into(), + authority_receipt: "authority-transitive-component".into(), + library_version: report.library_version, + rule_revision: report.rule_revision.into(), + snapshot_digest: report.snapshot_digest.clone(), + decisions: report + .duplicate_candidates + .iter() + .map(|candidate| DuplicateMergeDecision { + identity_kind: candidate.identity_kind.into(), + normalized_identity: candidate.normalized_identity.clone(), + retained_item_key: "A".into(), + }) + .collect(), + }; + + let manifest = build_duplicate_merge_review_manifest(&report, &reviewed, |_| true) + .expect("one steward-selected canonical key must be valid across a transitive duplicate component"); + + assert_eq!(manifest.operations.len(), 3); + for operation in manifest.operations { + assert_eq!(operation.retained_item_key, "A"); + assert!( + operation + .after_canonical_keys + .values() + .all(|canonical_key| canonical_key == "A") + ); + } +} + +#[test] +fn duplicate_review_rejects_blank_snapshot_item_identity() { + let report = classify_snapshot( + "9.0.6".into(), + None, + 42, + vec![ + item("", 1, "Blank identity", "10.1000/blank-key"), + item("B", 2, "Other identity", "10.1000/blank-key"), + ], + ); + assert_eq!(report.duplicate_candidates.len(), 1); + + let candidate = &report.duplicate_candidates[0]; + let reviewed = ReviewedDuplicateMergeSet { + review_id: "review-blank-key".into(), + authority_receipt: "authority-blank-key".into(), + library_version: report.library_version, + rule_revision: report.rule_revision.into(), + snapshot_digest: report.snapshot_digest.clone(), + decisions: vec![DuplicateMergeDecision { + identity_kind: candidate.identity_kind.into(), + normalized_identity: candidate.normalized_identity.clone(), + retained_item_key: "B".into(), + }], + }; + + assert_eq!( + build_duplicate_merge_review_manifest(&report, &reviewed, |_| true), + Err(DuplicateReviewError::InvalidReview), + "blank Zotero keys are not stable provenance identities and must fail closed before manifest materialization" + ); +} From e4a9cd066a7bb2f254ba2485eae8f77f02279070 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Fri, 4 Sep 2026 23:46:39 +0900 Subject: [PATCH 05/19] fix(zotero): resolve transitive duplicate components --- crates/conceptweave-zotero/src/lib.rs | 68 ++++++++++++++----- .../duplicate_merge_review_components.rs | 5 +- 2 files changed, 54 insertions(+), 19 deletions(-) diff --git a/crates/conceptweave-zotero/src/lib.rs b/crates/conceptweave-zotero/src/lib.rs index c87ba49d..320fee24 100644 --- a/crates/conceptweave-zotero/src/lib.rs +++ b/crates/conceptweave-zotero/src/lib.rs @@ -553,10 +553,14 @@ where if report .snapshot_items .iter() - .map(|item| item.item_key.as_str()) - .collect::>() - .len() - != report.snapshot_items.len() + .any(|item| item.item_key.trim().is_empty()) + || report + .snapshot_items + .iter() + .map(|item| item.item_key.as_str()) + .collect::>() + .len() + != report.snapshot_items.len() { return Err(DuplicateReviewError::InvalidReview); } @@ -579,7 +583,6 @@ where }) .collect::>(); let mut seen_candidates = BTreeSet::new(); - let mut canonical_choices = BTreeMap::<&str, &str>::new(); let mut operations = Vec::with_capacity(reviewed.decisions.len()); for decision in &reviewed.decisions { @@ -593,26 +596,57 @@ where let candidate = candidates .get(&candidate_key) .ok_or(DuplicateReviewError::UnknownCandidate)?; - if !candidate.item_keys.contains(&decision.retained_item_key) { + // ponytail: quadratic component expansion is enough for a steward-sized review; + // replace with union-find only if measured duplicate sets become large. + let mut component_keys = candidate.item_keys.iter().collect::>(); + loop { + let previous_len = component_keys.len(); + for related in &report.duplicate_candidates { + if related + .item_keys + .iter() + .any(|item_key| component_keys.contains(item_key)) + { + component_keys.extend(&related.item_keys); + } + } + if component_keys.len() == previous_len { + break; + } + } + if !component_keys.contains(&decision.retained_item_key) { return Err(DuplicateReviewError::InvalidRetainedItem); } - for item_key in &candidate.item_keys { - if canonical_choices - .insert(item_key, decision.retained_item_key.as_str()) - .is_some_and(|retained_key| retained_key != decision.retained_item_key) - { - return Err(DuplicateReviewError::InvalidReview); - } + if reviewed.decisions.iter().any(|related_decision| { + candidates + .get(&( + related_decision.identity_kind.as_str(), + related_decision.normalized_identity.as_str(), + )) + .is_some_and(|related_candidate| { + related_candidate + .item_keys + .iter() + .any(|item_key| component_keys.contains(item_key)) + && related_decision.retained_item_key != decision.retained_item_key + }) + }) { + return Err(DuplicateReviewError::InvalidReview); } let source_items = candidate .item_keys .iter() - .map(|item_key| SnapshotItemRevision { - item_key: item_key.clone(), - item_version: item_revisions[item_key.as_str()], + .map(|item_key| { + item_revisions + .get(item_key.as_str()) + .map(|item_version| SnapshotItemRevision { + item_key: item_key.clone(), + item_version: *item_version, + }) + .ok_or(DuplicateReviewError::InvalidReview) }) - .collect::>(); + .collect::, _>>()?; let before_canonical_keys = candidate .item_keys .iter() diff --git a/crates/conceptweave-zotero/tests/duplicate_merge_review_components.rs b/crates/conceptweave-zotero/tests/duplicate_merge_review_components.rs index f9c7689d..cfbcf447 100644 --- a/crates/conceptweave-zotero/tests/duplicate_merge_review_components.rs +++ b/crates/conceptweave-zotero/tests/duplicate_merge_review_components.rs @@ -51,8 +51,9 @@ fn transitive_duplicate_component_accepts_one_component_level_canonical_key() { .collect(), }; - let manifest = build_duplicate_merge_review_manifest(&report, &reviewed, |_| true) - .expect("one steward-selected canonical key must be valid across a transitive duplicate component"); + let manifest = build_duplicate_merge_review_manifest(&report, &reviewed, |_| true).expect( + "one steward-selected canonical key must be valid across a transitive duplicate component", + ); assert_eq!(manifest.operations.len(), 3); for operation in manifest.operations { From 7fb7bfa5dc0f09c868a727c5df4ba206d38a5c76 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Fri, 4 Sep 2026 23:47:54 +0900 Subject: [PATCH 06/19] test(zotero): cover missing duplicate revision --- .../tests/duplicate_merge_review_components.rs | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/crates/conceptweave-zotero/tests/duplicate_merge_review_components.rs b/crates/conceptweave-zotero/tests/duplicate_merge_review_components.rs index cfbcf447..a927dfbe 100644 --- a/crates/conceptweave-zotero/tests/duplicate_merge_review_components.rs +++ b/crates/conceptweave-zotero/tests/duplicate_merge_review_components.rs @@ -69,7 +69,7 @@ fn transitive_duplicate_component_accepts_one_component_level_canonical_key() { #[test] fn duplicate_review_rejects_blank_snapshot_item_identity() { - let report = classify_snapshot( + let mut report = classify_snapshot( "9.0.6".into(), None, 42, @@ -99,4 +99,11 @@ fn duplicate_review_rejects_blank_snapshot_item_identity() { Err(DuplicateReviewError::InvalidReview), "blank Zotero keys are not stable provenance identities and must fail closed before manifest materialization" ); + + report.snapshot_items.remove(0); + assert_eq!( + build_duplicate_merge_review_manifest(&report, &reviewed, |_| true), + Err(DuplicateReviewError::InvalidReview), + "every duplicate candidate must resolve to an immutable snapshot revision" + ); } From 89a3ffe005e702511bd4e731de9304c13fd054df Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Fri, 4 Sep 2026 23:56:14 +0900 Subject: [PATCH 07/19] test(zotero): bind duplicate review coordinates --- crates/conceptweave-zotero/src/lib.rs | 2 + .../duplicate_merge_review_components.rs | 40 +++++++++++++++++++ .../tests/duplicate_merge_review_manifest.rs | 3 ++ 3 files changed, 45 insertions(+) diff --git a/crates/conceptweave-zotero/src/lib.rs b/crates/conceptweave-zotero/src/lib.rs index 320fee24..83cad623 100644 --- a/crates/conceptweave-zotero/src/lib.rs +++ b/crates/conceptweave-zotero/src/lib.rs @@ -174,6 +174,8 @@ pub struct ReviewedDuplicateMergeSet { pub rule_revision: String, /// Exact raw-snapshot digest reviewed by the steward. pub snapshot_digest: String, + /// Exact item-key/item-version coordinates reviewed by the steward. + pub snapshot_items: Vec, /// Exactly one decision for every duplicate candidate. pub decisions: Vec, } diff --git a/crates/conceptweave-zotero/tests/duplicate_merge_review_components.rs b/crates/conceptweave-zotero/tests/duplicate_merge_review_components.rs index a927dfbe..99420adf 100644 --- a/crates/conceptweave-zotero/tests/duplicate_merge_review_components.rs +++ b/crates/conceptweave-zotero/tests/duplicate_merge_review_components.rs @@ -40,6 +40,7 @@ fn transitive_duplicate_component_accepts_one_component_level_canonical_key() { library_version: report.library_version, rule_revision: report.rule_revision.into(), snapshot_digest: report.snapshot_digest.clone(), + snapshot_items: report.snapshot_items.clone(), decisions: report .duplicate_candidates .iter() @@ -58,6 +59,8 @@ fn transitive_duplicate_component_accepts_one_component_level_canonical_key() { assert_eq!(manifest.operations.len(), 3); for operation in manifest.operations { assert_eq!(operation.retained_item_key, "A"); + assert_eq!(operation.source_items.len(), 4); + assert!(operation.source_items.iter().any(|item| item.item_key == "A")); assert!( operation .after_canonical_keys @@ -87,6 +90,7 @@ fn duplicate_review_rejects_blank_snapshot_item_identity() { library_version: report.library_version, rule_revision: report.rule_revision.into(), snapshot_digest: report.snapshot_digest.clone(), + snapshot_items: report.snapshot_items.clone(), decisions: vec![DuplicateMergeDecision { identity_kind: candidate.identity_kind.into(), normalized_identity: candidate.normalized_identity.clone(), @@ -107,3 +111,39 @@ fn duplicate_review_rejects_blank_snapshot_item_identity() { "every duplicate candidate must resolve to an immutable snapshot revision" ); } + +#[test] +fn duplicate_review_binds_every_item_revision_to_the_reviewed_snapshot() { + let mut report = classify_snapshot( + "9.0.6".into(), + None, + 42, + vec![ + item("A", 1, "Same title", "10.1000/same"), + item("B", 2, "Same title", "10.1000/same"), + ], + ); + let reviewed = ReviewedDuplicateMergeSet { + review_id: "review-revisions".into(), + authority_receipt: "authority-revisions".into(), + library_version: report.library_version, + rule_revision: report.rule_revision.into(), + snapshot_digest: report.snapshot_digest.clone(), + snapshot_items: report.snapshot_items.clone(), + decisions: report + .duplicate_candidates + .iter() + .map(|candidate| DuplicateMergeDecision { + identity_kind: candidate.identity_kind.into(), + normalized_identity: candidate.normalized_identity.clone(), + retained_item_key: "A".into(), + }) + .collect(), + }; + report.snapshot_items[0].item_version += 1; + + assert_eq!( + build_duplicate_merge_review_manifest(&report, &reviewed, |_| true), + Err(DuplicateReviewError::SnapshotMismatch) + ); +} diff --git a/crates/conceptweave-zotero/tests/duplicate_merge_review_manifest.rs b/crates/conceptweave-zotero/tests/duplicate_merge_review_manifest.rs index 01a62cbb..6e06423d 100644 --- a/crates/conceptweave-zotero/tests/duplicate_merge_review_manifest.rs +++ b/crates/conceptweave-zotero/tests/duplicate_merge_review_manifest.rs @@ -45,6 +45,7 @@ fn reviewed(report: &conceptweave_zotero::ClassificationReport) -> ReviewedDupli library_version: report.library_version, rule_revision: report.rule_revision.into(), snapshot_digest: report.snapshot_digest.clone(), + snapshot_items: report.snapshot_items.clone(), decisions: vec![ DuplicateMergeDecision { identity_kind: "title".into(), @@ -197,6 +198,7 @@ fn overlapping_duplicate_groups_require_one_consistent_canonical_choice() { library_version: report.library_version, rule_revision: report.rule_revision.into(), snapshot_digest: report.snapshot_digest.clone(), + snapshot_items: report.snapshot_items.clone(), decisions: vec![ DuplicateMergeDecision { identity_kind: "doi".into(), @@ -238,6 +240,7 @@ fn duplicate_review_rejects_ambiguous_snapshot_key_revisions() { library_version: report.library_version, rule_revision: report.rule_revision.into(), snapshot_digest: report.snapshot_digest.clone(), + snapshot_items: report.snapshot_items.clone(), decisions: report .duplicate_candidates .iter() From c32d3f08619f631f5dea5ed68b4fcc91a092a577 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Fri, 4 Sep 2026 23:56:59 +0900 Subject: [PATCH 08/19] fix(zotero): preserve component review evidence --- crates/conceptweave-zotero/src/lib.rs | 16 +++++++--------- .../tests/duplicate_merge_review_components.rs | 11 ++++++++--- 2 files changed, 15 insertions(+), 12 deletions(-) diff --git a/crates/conceptweave-zotero/src/lib.rs b/crates/conceptweave-zotero/src/lib.rs index 83cad623..31a013bc 100644 --- a/crates/conceptweave-zotero/src/lib.rs +++ b/crates/conceptweave-zotero/src/lib.rs @@ -545,6 +545,7 @@ where if reviewed.snapshot_digest != report.snapshot_digest || reviewed.library_version != report.library_version || reviewed.rule_revision != report.rule_revision + || reviewed.snapshot_items != report.snapshot_items { return Err(DuplicateReviewError::SnapshotMismatch); } @@ -636,28 +637,25 @@ where return Err(DuplicateReviewError::InvalidReview); } - let source_items = candidate - .item_keys + let source_items = component_keys .iter() .map(|item_key| { item_revisions .get(item_key.as_str()) .map(|item_version| SnapshotItemRevision { - item_key: item_key.clone(), + item_key: (*item_key).clone(), item_version: *item_version, }) .ok_or(DuplicateReviewError::InvalidReview) }) .collect::, _>>()?; - let before_canonical_keys = candidate - .item_keys + let before_canonical_keys = component_keys .iter() - .map(|item_key| (item_key.clone(), item_key.clone())) + .map(|item_key| ((*item_key).clone(), (*item_key).clone())) .collect::>(); - let after_canonical_keys = candidate - .item_keys + let after_canonical_keys = component_keys .iter() - .map(|item_key| (item_key.clone(), decision.retained_item_key.clone())) + .map(|item_key| ((*item_key).clone(), decision.retained_item_key.clone())) .collect::>(); operations.push(DuplicateMergeOperation { identity_kind: decision.identity_kind.clone(), diff --git a/crates/conceptweave-zotero/tests/duplicate_merge_review_components.rs b/crates/conceptweave-zotero/tests/duplicate_merge_review_components.rs index 99420adf..65ecd094 100644 --- a/crates/conceptweave-zotero/tests/duplicate_merge_review_components.rs +++ b/crates/conceptweave-zotero/tests/duplicate_merge_review_components.rs @@ -60,7 +60,12 @@ fn transitive_duplicate_component_accepts_one_component_level_canonical_key() { for operation in manifest.operations { assert_eq!(operation.retained_item_key, "A"); assert_eq!(operation.source_items.len(), 4); - assert!(operation.source_items.iter().any(|item| item.item_key == "A")); + assert!( + operation + .source_items + .iter() + .any(|item| item.item_key == "A") + ); assert!( operation .after_canonical_keys @@ -107,8 +112,8 @@ fn duplicate_review_rejects_blank_snapshot_item_identity() { report.snapshot_items.remove(0); assert_eq!( build_duplicate_merge_review_manifest(&report, &reviewed, |_| true), - Err(DuplicateReviewError::InvalidReview), - "every duplicate candidate must resolve to an immutable snapshot revision" + Err(DuplicateReviewError::SnapshotMismatch), + "every duplicate candidate must remain bound to its reviewed snapshot revision" ); } From 6df2967212973800d94d6717598b010c1d465b64 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Fri, 4 Sep 2026 23:58:11 +0900 Subject: [PATCH 09/19] docs(zotero): define connected duplicate evidence --- docs/PRD.md | 2 +- docs/TRD.md | 2 +- docs/UBIQUITOUS_LANGUAGE.md | 2 +- docs/adr/0006-zotero-research-intake.md | 2 +- docs/product-technical-gap-baseline.md | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/docs/PRD.md b/docs/PRD.md index e1fba297..89c5ef7c 100644 --- a/docs/PRD.md +++ b/docs/PRD.md @@ -58,7 +58,7 @@ All LLM-backed induction uses `contextual-orchestrator`. Model output is untrust Read one immutable Zotero Local API library-version snapshot and propose exactly one research disposition for every top-level bibliographic item. Each proposal retains the item key/version, exact matched metadata values, rule revision, linked child records, and any model receipt. Weak evidence and evidence that matches multiple specific disposition families must abstain into steward review. Duplicate DOI/title identities are review candidates only: intake never merges, deletes, or silently mutates Zotero records. -For every detected duplicate cluster, accept exactly one externally verified steward decision selecting an existing canonical item. Produce a local-only manifest that binds the decision to the raw snapshot and records source revisions plus before, after, and rollback canonical-key mappings. Classification preserves every Zotero source record. +For every connected duplicate component, accept externally verified steward decisions selecting one component-level canonical item. Produce a local-only manifest that binds the decisions to the raw snapshot and its complete item-key/item-version coordinates, and records every component source revision plus before, after, and rollback canonical-key mappings. Classification preserves every Zotero source record. Evaluate classifier quality only against a steward-reviewed local golden set whose governance receipt is externally verified and bound to the canonical SHA-256 digest of the complete Zotero classification report plus its item-key/item-version coordinates. Abstention is a prediction outcome, never an approved truth label. Evaluation emits the verified library revision, rule revision, opaque snapshot digest, and aggregate counts for exact matches, abstentions, and per-disposition true-positive/predicted/expected totals; it must not copy Zotero keys, reviewer identity, or bibliographic text into the result. Every successful classification report includes aggregate evidence for snapshot coverage, proposal coverage, provenance completeness, abstentions, duplicate candidates, disposition totals, and zero unreported failures. diff --git a/docs/TRD.md b/docs/TRD.md index 27e9d720..f58f0fc3 100644 --- a/docs/TRD.md +++ b/docs/TRD.md @@ -63,7 +63,7 @@ Evaluation must separate extraction recall, semantic correctness, structural cor Every top-level bibliographic record receives exactly one proposed disposition. `NeedsStewardReview` also records a deterministic abstention reason so missing classification metadata, vocabulary unsupported by the current deterministic rules, and present-but-unmatched metadata are distinguishable. DOI duplicate identity normalization treats bare DOI values, `doi:`, `doi.org`, and legacy `dx.doi.org` resolver forms as the same identity when their normalized DOI is equal. -Duplicate review is independent of subject classification. A reviewed decision set must match the exact raw-snapshot digest, cover every duplicate candidate exactly once, select a retained key already in that cluster, and pass an external governance verifier. Its manifest records item revisions, identity mappings before and after canonicalization, and the exact rollback mapping. These mappings affect only downstream identity resolution; Zotero records are neither mutated nor deleted. +Duplicate review is independent of subject classification. A reviewed decision set must match the exact raw-snapshot digest and complete item-key/item-version coordinates, cover every duplicate candidate exactly once, select one retained key from the connected duplicate component, and pass an external governance verifier. Every operation records all component item revisions, identity mappings before and after canonicalization, and the exact rollback mapping. These mappings affect only downstream identity resolution; Zotero records are neither mutated nor deleted. Golden-set evaluation accepts only a governance receipt verified with the complete reviewed set by a caller-owned authorization boundary. Its library version, rule revision, canonical SHA-256 content digest, and every observed parent/child item-key/item-version identity must bind the classification report. The digest covers every raw Zotero item in canonical key order. Blank, duplicate, unknown, stale, content-mismatched, label-mismatched, or abstention-as-truth inputs fail closed. The output retains the verified library version, rule revision, and opaque snapshot digest, but contains no item keys, reviewer identity, or bibliographic text. Production authorization remains Keyverse/governance-owned; this crate passes the complete reviewed labels to that boundary instead of minting authority. A successful classification report carries an `audit_summary` whose snapshot, bibliographic, proposed-disposition, provenance-complete, abstention, duplicate-candidate, failure, and per-disposition counts are derived from the same in-memory immutable snapshot. Zotero item version zero remains a valid observed coordinate for never-synced Zotero 9 records; provenance completeness rejects a missing item key rather than inventing a positive-only version invariant. Reader failures return an error instead of a partial report; therefore a returned report records `failure_count=0` rather than hiding partial failures. diff --git a/docs/UBIQUITOUS_LANGUAGE.md b/docs/UBIQUITOUS_LANGUAGE.md index 24f3d68e..566cb2fe 100644 --- a/docs/UBIQUITOUS_LANGUAGE.md +++ b/docs/UBIQUITOUS_LANGUAGE.md @@ -18,4 +18,4 @@ | Semantic Steward | Authorized reviewer responsible for accepting or rejecting semantic meaning. | | Reviewed Duplicate Merge Set | Complete steward decisions selecting one consistent canonical item across every overlapping duplicate group in a snapshot. | | Authority Receipt | Opaque proof checked by the Governance & Publication boundary; it contains no reviewer identity or credential. | -| Canonical-Key Operation | Reversible local mapping from each duplicate source key to one retained key, with the exact rollback mapping. | +| Canonical-Key Operation | Reversible local mapping from every source in a connected duplicate component to one retained key, with complete reviewed revisions and the exact rollback mapping. | diff --git a/docs/adr/0006-zotero-research-intake.md b/docs/adr/0006-zotero-research-intake.md index a407d96b..407a9d64 100644 --- a/docs/adr/0006-zotero-research-intake.md +++ b/docs/adr/0006-zotero-research-intake.md @@ -19,7 +19,7 @@ The adapter links child records, emits exactly one deterministic proposed dispos Matched metadata values are copied into the local-only evidence receipt for replay. This is necessary for abstract-only matches because a later Zotero revision cannot reconstruct the exact text used for an earlier proposal from item key/version alone. DOI/title matches remain reversible duplicate candidates, including legacy `dx.doi.org` resolver forms. -Duplicate candidates become canonical references only through a snapshot-bound, externally verified steward decision. The resulting local manifest retains source revisions and complete before/after/rollback key mappings. It changes downstream identity resolution only; classification does not merge, delete, or mutate Zotero source records. +Duplicate candidates become canonical references only through externally verified steward decisions bound to the raw digest and complete item-key/item-version snapshot. Overlapping candidates form one connected component and must select one component-level canonical item. Every resulting operation retains all component source revisions and complete before/after/rollback key mappings. It changes downstream identity resolution only; classification does not merge, delete, or mutate Zotero source records. Classifier quality is measured only against local steward-reviewed labels whose complete reviewed set is verified outside this crate and bound to the exact library version, rule revision, canonical SHA-256 raw-snapshot digest, and every observed parent/child item-key/item-version coordinate. `NeedsStewardReview` is an abstention prediction and cannot be approved truth. Evaluation returns the verified revisions and opaque digest with aggregate integer evidence; Zotero keys, reviewer identity, and bibliographic text are omitted. Missing, stale, content- or label-mismatched, unverified, unknown, duplicate, or invalid review identities fail closed. Every successful report includes an aggregate audit summary computed from the same captured snapshot. Zotero 9 item version zero is preserved as a valid never-synced source coordinate, not treated as missing provenance. Partial reads never produce a report, so successful output explicitly records zero failures alongside snapshot, proposal, provenance, abstention, duplicate, and per-disposition totals. diff --git a/docs/product-technical-gap-baseline.md b/docs/product-technical-gap-baseline.md index cb2e680a..efe12c43 100644 --- a/docs/product-technical-gap-baseline.md +++ b/docs/product-technical-gap-baseline.md @@ -44,7 +44,7 @@ Protected central source is `.github/main@c31d2e5471fc5daf9d72ff67cde6a8874b736d Local evidence on 2026-09-04 showed Zotero 9.0.6, Local API v3/schema 42, library version 12341, 8,326 total items, and 3,719 top-level items. The corrected read-only run observed all 8,326 records at that single version and classified all 3,715 top-level bibliographic records; four top-level note/attachment/annotation records were correctly excluded. It proposed 56 adjacent-evidence records, 1 semantic-consumption bridge, and 3,658 steward-review abstentions, linked children for 3,287 records, and surfaced 49 reversible duplicate groups (18 DOI, 31 title). No live record matched multiple specific disposition families; the tested conflict path still abstains fail-closed. Token-boundary matching prevents strings such as `knowledge` from becoming false OWL evidence. These are local aggregate observations, not reviewed truth or applied Zotero changes. The report stays outside the repository. -The golden-set evaluation contract now records aggregate precision/recall numerators and denominators, requires an externally verified governance receipt bound to the complete item-key/item-version snapshot, rejects abstention as expected truth, and retains verified revisions plus an opaque snapshot digest so detached metrics remain attributable. Item and reviewer identities stay out of its output. Successful classification reports also carry same-snapshot aggregate coverage, provenance, abstention, duplicate, disposition, and failure evidence. Duplicate clusters now produce a snapshot-bound local review manifest only after external steward verification; exact source revisions and before/after/rollback canonical mappings are retained while Zotero records remain unchanged. Synthetic fixtures verify these contracts; no real precision/recall or duplicate merge claim exists until a steward supplies reviewed local decisions and the production authorization adapter verifies them. Multilingual rule expansion remains a later evidence-driven change and must not reduce abstention safety. Zotero write-back remains blocked by the installed v9 capability; a Zotero 10+ change must satisfy ADR 0006 preconditions. A dedicated utility repository remains unnecessary until an independently released cross-product contract exists. +The golden-set evaluation contract now records aggregate precision/recall numerators and denominators, requires an externally verified governance receipt bound to the complete item-key/item-version snapshot, rejects abstention as expected truth, and retains verified revisions plus an opaque snapshot digest so detached metrics remain attributable. Item and reviewer identities stay out of its output. Successful classification reports also carry same-snapshot aggregate coverage, provenance, abstention, duplicate, disposition, and failure evidence. Connected duplicate components now produce a snapshot-bound local review manifest only after external steward verification; every operation retains all component source revisions and before/after/rollback canonical mappings while Zotero records remain unchanged. Synthetic fixtures verify these contracts; no real precision/recall or duplicate merge claim exists until a steward supplies reviewed local decisions and the production authorization adapter verifies them. Multilingual rule expansion remains a later evidence-driven change and must not reduce abstention safety. Zotero write-back remains blocked by the installed v9 capability; a Zotero 10+ change must satisfy ADR 0006 preconditions. A dedicated utility repository remains unnecessary until an independently released cross-product contract exists. 1. **Concrete Source Observation adapter** — maintained Rust PostgreSQL driver behind `conceptweave-source-port`; adapter-local credential resolution; explicit read-only mode; statement timeout, cancellation, row/byte/concurrency budgets; complete immutable snapshot or fail closed; deterministic replay against a frozen anonymized GRC-shaped fixture. 2. **Ontology discovery** — deterministic term/concept/taxonomy/non-taxonomic-relation candidate generation with exact source receipts and abstention for unsupported semantics. From e77a2d232ca77a7bf8d474141d33f48da451415b Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Fri, 4 Sep 2026 23:59:04 +0900 Subject: [PATCH 10/19] test(zotero): reject detached duplicate candidates --- .../tests/duplicate_merge_review_components.rs | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/crates/conceptweave-zotero/tests/duplicate_merge_review_components.rs b/crates/conceptweave-zotero/tests/duplicate_merge_review_components.rs index 65ecd094..f5c27010 100644 --- a/crates/conceptweave-zotero/tests/duplicate_merge_review_components.rs +++ b/crates/conceptweave-zotero/tests/duplicate_merge_review_components.rs @@ -128,7 +128,7 @@ fn duplicate_review_binds_every_item_revision_to_the_reviewed_snapshot() { item("B", 2, "Same title", "10.1000/same"), ], ); - let reviewed = ReviewedDuplicateMergeSet { + let mut reviewed = ReviewedDuplicateMergeSet { review_id: "review-revisions".into(), authority_receipt: "authority-revisions".into(), library_version: report.library_version, @@ -151,4 +151,16 @@ fn duplicate_review_binds_every_item_revision_to_the_reviewed_snapshot() { build_duplicate_merge_review_manifest(&report, &reviewed, |_| true), Err(DuplicateReviewError::SnapshotMismatch) ); + + report.snapshot_items[0].item_version -= 1; + for decision in &mut reviewed.decisions { + decision.retained_item_key = "B".into(); + } + for candidate in &mut report.duplicate_candidates { + candidate.item_keys[0] = "missing".into(); + } + assert_eq!( + build_duplicate_merge_review_manifest(&report, &reviewed, |_| true), + Err(DuplicateReviewError::InvalidReview) + ); } From eba815ba10fd5f7a3a8bc323a7b159f68759099b Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sat, 5 Sep 2026 00:08:05 +0900 Subject: [PATCH 11/19] fix(zotero): bind duplicate candidate membership --- crates/conceptweave-zotero/src/lib.rs | 5 ++++- .../tests/duplicate_merge_review_components.rs | 18 ++++++++++++++++++ .../tests/duplicate_merge_review_manifest.rs | 3 +++ 3 files changed, 25 insertions(+), 1 deletion(-) diff --git a/crates/conceptweave-zotero/src/lib.rs b/crates/conceptweave-zotero/src/lib.rs index 31a013bc..5c92eb71 100644 --- a/crates/conceptweave-zotero/src/lib.rs +++ b/crates/conceptweave-zotero/src/lib.rs @@ -140,7 +140,7 @@ pub struct ClassifiedItem { } /// A duplicate candidate group; no item is merged or deleted. -#[derive(Debug, Serialize)] +#[derive(Debug, Clone, PartialEq, Eq, Deserialize, Serialize)] pub struct DuplicateCandidate { /// Identity kind used for the candidate group. pub identity_kind: &'static str, @@ -176,6 +176,8 @@ pub struct ReviewedDuplicateMergeSet { pub snapshot_digest: String, /// Exact item-key/item-version coordinates reviewed by the steward. pub snapshot_items: Vec, + /// Exact duplicate membership reviewed by the steward. + pub duplicate_candidates: Vec, /// Exactly one decision for every duplicate candidate. pub decisions: Vec, } @@ -546,6 +548,7 @@ where || reviewed.library_version != report.library_version || reviewed.rule_revision != report.rule_revision || reviewed.snapshot_items != report.snapshot_items + || reviewed.duplicate_candidates != report.duplicate_candidates { return Err(DuplicateReviewError::SnapshotMismatch); } diff --git a/crates/conceptweave-zotero/tests/duplicate_merge_review_components.rs b/crates/conceptweave-zotero/tests/duplicate_merge_review_components.rs index f5c27010..e2e399c7 100644 --- a/crates/conceptweave-zotero/tests/duplicate_merge_review_components.rs +++ b/crates/conceptweave-zotero/tests/duplicate_merge_review_components.rs @@ -41,6 +41,7 @@ fn transitive_duplicate_component_accepts_one_component_level_canonical_key() { rule_revision: report.rule_revision.into(), snapshot_digest: report.snapshot_digest.clone(), snapshot_items: report.snapshot_items.clone(), + duplicate_candidates: report.duplicate_candidates.clone(), decisions: report .duplicate_candidates .iter() @@ -96,6 +97,7 @@ fn duplicate_review_rejects_blank_snapshot_item_identity() { rule_revision: report.rule_revision.into(), snapshot_digest: report.snapshot_digest.clone(), snapshot_items: report.snapshot_items.clone(), + duplicate_candidates: report.duplicate_candidates.clone(), decisions: vec![DuplicateMergeDecision { identity_kind: candidate.identity_kind.into(), normalized_identity: candidate.normalized_identity.clone(), @@ -126,6 +128,7 @@ fn duplicate_review_binds_every_item_revision_to_the_reviewed_snapshot() { vec![ item("A", 1, "Same title", "10.1000/same"), item("B", 2, "Same title", "10.1000/same"), + item("C", 3, "Unrelated title", "10.1000/unrelated"), ], ); let mut reviewed = ReviewedDuplicateMergeSet { @@ -135,6 +138,7 @@ fn duplicate_review_binds_every_item_revision_to_the_reviewed_snapshot() { rule_revision: report.rule_revision.into(), snapshot_digest: report.snapshot_digest.clone(), snapshot_items: report.snapshot_items.clone(), + duplicate_candidates: report.duplicate_candidates.clone(), decisions: report .duplicate_candidates .iter() @@ -157,7 +161,21 @@ fn duplicate_review_binds_every_item_revision_to_the_reviewed_snapshot() { decision.retained_item_key = "B".into(); } for candidate in &mut report.duplicate_candidates { + candidate.item_keys[0] = "C".into(); + } + assert_eq!( + build_duplicate_merge_review_manifest(&report, &reviewed, |_| true), + Err(DuplicateReviewError::SnapshotMismatch) + ); + + reviewed.duplicate_candidates = report.duplicate_candidates.clone(); + for (candidate, reviewed_candidate) in report + .duplicate_candidates + .iter_mut() + .zip(&mut reviewed.duplicate_candidates) + { candidate.item_keys[0] = "missing".into(); + reviewed_candidate.item_keys[0] = "missing".into(); } assert_eq!( build_duplicate_merge_review_manifest(&report, &reviewed, |_| true), diff --git a/crates/conceptweave-zotero/tests/duplicate_merge_review_manifest.rs b/crates/conceptweave-zotero/tests/duplicate_merge_review_manifest.rs index 6e06423d..cf1f1fdc 100644 --- a/crates/conceptweave-zotero/tests/duplicate_merge_review_manifest.rs +++ b/crates/conceptweave-zotero/tests/duplicate_merge_review_manifest.rs @@ -46,6 +46,7 @@ fn reviewed(report: &conceptweave_zotero::ClassificationReport) -> ReviewedDupli rule_revision: report.rule_revision.into(), snapshot_digest: report.snapshot_digest.clone(), snapshot_items: report.snapshot_items.clone(), + duplicate_candidates: report.duplicate_candidates.clone(), decisions: vec![ DuplicateMergeDecision { identity_kind: "title".into(), @@ -199,6 +200,7 @@ fn overlapping_duplicate_groups_require_one_consistent_canonical_choice() { rule_revision: report.rule_revision.into(), snapshot_digest: report.snapshot_digest.clone(), snapshot_items: report.snapshot_items.clone(), + duplicate_candidates: report.duplicate_candidates.clone(), decisions: vec![ DuplicateMergeDecision { identity_kind: "doi".into(), @@ -241,6 +243,7 @@ fn duplicate_review_rejects_ambiguous_snapshot_key_revisions() { rule_revision: report.rule_revision.into(), snapshot_digest: report.snapshot_digest.clone(), snapshot_items: report.snapshot_items.clone(), + duplicate_candidates: report.duplicate_candidates.clone(), decisions: report .duplicate_candidates .iter() From 5bf58aa985716f72b23b2f280103dd13731debea Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sat, 5 Sep 2026 00:08:58 +0900 Subject: [PATCH 12/19] fix(zotero): serialize reviewed duplicate identities --- crates/conceptweave-zotero/src/lib.rs | 6 +++--- .../tests/duplicate_merge_review_components.rs | 6 +++--- .../tests/duplicate_merge_review_manifest.rs | 2 +- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/crates/conceptweave-zotero/src/lib.rs b/crates/conceptweave-zotero/src/lib.rs index 5c92eb71..a52035ac 100644 --- a/crates/conceptweave-zotero/src/lib.rs +++ b/crates/conceptweave-zotero/src/lib.rs @@ -143,7 +143,7 @@ pub struct ClassifiedItem { #[derive(Debug, Clone, PartialEq, Eq, Deserialize, Serialize)] pub struct DuplicateCandidate { /// Identity kind used for the candidate group. - pub identity_kind: &'static str, + pub identity_kind: String, /// Normalized identity value. pub normalized_identity: String, /// Zotero item keys sharing the identity. @@ -581,7 +581,7 @@ where .map(|candidate| { ( ( - candidate.identity_kind, + candidate.identity_kind.as_str(), candidate.normalized_identity.as_str(), ), candidate, @@ -1215,7 +1215,7 @@ fn duplicate_candidates(items: &[&ZoteroItem]) -> Vec { .into_iter() .filter_map(|((identity_kind, normalized_identity), item_keys)| { (item_keys.len() > 1).then_some(DuplicateCandidate { - identity_kind, + identity_kind: identity_kind.into(), normalized_identity, item_keys, }) diff --git a/crates/conceptweave-zotero/tests/duplicate_merge_review_components.rs b/crates/conceptweave-zotero/tests/duplicate_merge_review_components.rs index e2e399c7..4892fb01 100644 --- a/crates/conceptweave-zotero/tests/duplicate_merge_review_components.rs +++ b/crates/conceptweave-zotero/tests/duplicate_merge_review_components.rs @@ -46,7 +46,7 @@ fn transitive_duplicate_component_accepts_one_component_level_canonical_key() { .duplicate_candidates .iter() .map(|candidate| DuplicateMergeDecision { - identity_kind: candidate.identity_kind.into(), + identity_kind: candidate.identity_kind.clone(), normalized_identity: candidate.normalized_identity.clone(), retained_item_key: "A".into(), }) @@ -99,7 +99,7 @@ fn duplicate_review_rejects_blank_snapshot_item_identity() { snapshot_items: report.snapshot_items.clone(), duplicate_candidates: report.duplicate_candidates.clone(), decisions: vec![DuplicateMergeDecision { - identity_kind: candidate.identity_kind.into(), + identity_kind: candidate.identity_kind.clone(), normalized_identity: candidate.normalized_identity.clone(), retained_item_key: "B".into(), }], @@ -143,7 +143,7 @@ fn duplicate_review_binds_every_item_revision_to_the_reviewed_snapshot() { .duplicate_candidates .iter() .map(|candidate| DuplicateMergeDecision { - identity_kind: candidate.identity_kind.into(), + identity_kind: candidate.identity_kind.clone(), normalized_identity: candidate.normalized_identity.clone(), retained_item_key: "A".into(), }) diff --git a/crates/conceptweave-zotero/tests/duplicate_merge_review_manifest.rs b/crates/conceptweave-zotero/tests/duplicate_merge_review_manifest.rs index cf1f1fdc..868590f3 100644 --- a/crates/conceptweave-zotero/tests/duplicate_merge_review_manifest.rs +++ b/crates/conceptweave-zotero/tests/duplicate_merge_review_manifest.rs @@ -248,7 +248,7 @@ fn duplicate_review_rejects_ambiguous_snapshot_key_revisions() { .duplicate_candidates .iter() .map(|candidate| DuplicateMergeDecision { - identity_kind: candidate.identity_kind.into(), + identity_kind: candidate.identity_kind.clone(), normalized_identity: candidate.normalized_identity.clone(), retained_item_key: "DUPKEY".into(), }) From 2d503ab69f8832f9efd01751f22c09beb86aff49 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sat, 5 Sep 2026 00:09:27 +0900 Subject: [PATCH 13/19] docs(zotero): bind reviewed duplicate membership --- docs/PRD.md | 2 +- docs/adr/0006-zotero-research-intake.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/PRD.md b/docs/PRD.md index 89c5ef7c..822679b4 100644 --- a/docs/PRD.md +++ b/docs/PRD.md @@ -58,7 +58,7 @@ All LLM-backed induction uses `contextual-orchestrator`. Model output is untrust Read one immutable Zotero Local API library-version snapshot and propose exactly one research disposition for every top-level bibliographic item. Each proposal retains the item key/version, exact matched metadata values, rule revision, linked child records, and any model receipt. Weak evidence and evidence that matches multiple specific disposition families must abstain into steward review. Duplicate DOI/title identities are review candidates only: intake never merges, deletes, or silently mutates Zotero records. -For every connected duplicate component, accept externally verified steward decisions selecting one component-level canonical item. Produce a local-only manifest that binds the decisions to the raw snapshot and its complete item-key/item-version coordinates, and records every component source revision plus before, after, and rollback canonical-key mappings. Classification preserves every Zotero source record. +For every connected duplicate component, accept externally verified steward decisions selecting one component-level canonical item. Produce a local-only manifest that binds the decisions to the raw snapshot, its complete item-key/item-version coordinates, and exact duplicate-candidate membership, and records every component source revision plus before, after, and rollback canonical-key mappings. Classification preserves every Zotero source record. Evaluate classifier quality only against a steward-reviewed local golden set whose governance receipt is externally verified and bound to the canonical SHA-256 digest of the complete Zotero classification report plus its item-key/item-version coordinates. Abstention is a prediction outcome, never an approved truth label. Evaluation emits the verified library revision, rule revision, opaque snapshot digest, and aggregate counts for exact matches, abstentions, and per-disposition true-positive/predicted/expected totals; it must not copy Zotero keys, reviewer identity, or bibliographic text into the result. Every successful classification report includes aggregate evidence for snapshot coverage, proposal coverage, provenance completeness, abstentions, duplicate candidates, disposition totals, and zero unreported failures. diff --git a/docs/adr/0006-zotero-research-intake.md b/docs/adr/0006-zotero-research-intake.md index 407a9d64..cbba8863 100644 --- a/docs/adr/0006-zotero-research-intake.md +++ b/docs/adr/0006-zotero-research-intake.md @@ -19,7 +19,7 @@ The adapter links child records, emits exactly one deterministic proposed dispos Matched metadata values are copied into the local-only evidence receipt for replay. This is necessary for abstract-only matches because a later Zotero revision cannot reconstruct the exact text used for an earlier proposal from item key/version alone. DOI/title matches remain reversible duplicate candidates, including legacy `dx.doi.org` resolver forms. -Duplicate candidates become canonical references only through externally verified steward decisions bound to the raw digest and complete item-key/item-version snapshot. Overlapping candidates form one connected component and must select one component-level canonical item. Every resulting operation retains all component source revisions and complete before/after/rollback key mappings. It changes downstream identity resolution only; classification does not merge, delete, or mutate Zotero source records. +Duplicate candidates become canonical references only through externally verified steward decisions bound to the raw digest, complete item-key/item-version snapshot, and exact candidate membership. Overlapping candidates form one connected component and must select one component-level canonical item. Every resulting operation retains all component source revisions and complete before/after/rollback key mappings. It changes downstream identity resolution only; classification does not merge, delete, or mutate Zotero source records. Classifier quality is measured only against local steward-reviewed labels whose complete reviewed set is verified outside this crate and bound to the exact library version, rule revision, canonical SHA-256 raw-snapshot digest, and every observed parent/child item-key/item-version coordinate. `NeedsStewardReview` is an abstention prediction and cannot be approved truth. Evaluation returns the verified revisions and opaque digest with aggregate integer evidence; Zotero keys, reviewer identity, and bibliographic text are omitted. Missing, stale, content- or label-mismatched, unverified, unknown, duplicate, or invalid review identities fail closed. Every successful report includes an aggregate audit summary computed from the same captured snapshot. Zotero 9 item version zero is preserved as a valid never-synced source coordinate, not treated as missing provenance. Partial reads never produce a report, so successful output explicitly records zero failures alongside snapshot, proposal, provenance, abstention, duplicate, and per-disposition totals. From 4656d6b0514973db8aed595fb25d87a08e6192e6 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sun, 6 Sep 2026 21:45:48 +0900 Subject: [PATCH 14/19] test: expose duplicate review source scope bypass --- .../tests/duplicate_merge_review_manifest.rs | 55 +++++++++++++++++++ 1 file changed, 55 insertions(+) diff --git a/crates/conceptweave-zotero/tests/duplicate_merge_review_manifest.rs b/crates/conceptweave-zotero/tests/duplicate_merge_review_manifest.rs index e7f8c71c..5454013b 100644 --- a/crates/conceptweave-zotero/tests/duplicate_merge_review_manifest.rs +++ b/crates/conceptweave-zotero/tests/duplicate_merge_review_manifest.rs @@ -35,6 +35,11 @@ fn report() -> conceptweave_zotero::ClassificationReport { ), item("C", 3, "Shared Ontology Title", ""), item("D", 4, "Shared Ontology Title", ""), + { + let mut note = item("NOTE", 0, "Standalone source", ""); + note.data.item_type = "note".into(); + note + }, ], ) } @@ -101,6 +106,56 @@ fn reviewed_duplicate_decision_has_exact_before_after_and_rollback_mappings() { ); } +#[test] +fn duplicate_scope_and_decisions_fail_before_governance() { + for mutation in 0..7 { + let mut report = report(); + let mut review = reviewed(&report); + match mutation { + 0 => report.observed_item_count += 1, + 1 => report.unclassified_items.clear(), + 2 => report.pending_source_item_keys.clear(), + 3 => report.audit_summary.failure_count = 1, + 4 => review.decisions[0].retained_item_key = "missing".into(), + 5 => review.decisions[0].normalized_identity = "missing".into(), + _ => review.decisions[0] = review.decisions[1].clone(), + } + let called = std::cell::Cell::new(false); + assert!( + build_duplicate_merge_review_manifest(&report, &review, |_| { + called.set(true); + true + }) + .is_err(), + "invalid scope or decision {mutation} accepted" + ); + assert!( + !called.get(), + "invalid scope or decision {mutation} reached governance" + ); + } +} + +#[test] +fn retained_source_change_invalidates_duplicate_approval() { + let mut report = report(); + let review = reviewed(&report); + report.unclassified_items[0].data.title = "Changed standalone evidence".into(); + assert_eq!( + build_duplicate_merge_review_manifest(&report, &review, |_| panic!( + "changed scope reached governance" + )), + Err(DuplicateReviewError::SnapshotMismatch) + ); +} + +#[test] +fn duplicate_receipt_requires_explicit_source_scope_binding() { + let mut legacy = serde_json::to_value(reviewed(&report())).unwrap(); + legacy.as_object_mut().unwrap().remove("proposal_digest"); + assert!(serde_json::from_value::(legacy).is_err()); +} + #[test] fn duplicate_review_contract_fails_closed() { let report = report(); From a27363ac5158ef76aa7c5f4494ff34d6e71d3453 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sun, 6 Sep 2026 21:47:21 +0900 Subject: [PATCH 15/19] fix: bind duplicate review to complete validated source scope Reuse shared report admission; exact candidate membership remains independently bound. Remove subsumed key checks and move governance after all local decisions. Require v2 proposal identity on receipt and manifest, with no legacy default or approval issuance. --- crates/conceptweave-zotero/src/lib.rs | 34 +++++++++---------- .../duplicate_merge_review_components.rs | 3 ++ .../tests/duplicate_merge_review_manifest.rs | 3 ++ 3 files changed, 22 insertions(+), 18 deletions(-) diff --git a/crates/conceptweave-zotero/src/lib.rs b/crates/conceptweave-zotero/src/lib.rs index 1947d554..1d93bd59 100644 --- a/crates/conceptweave-zotero/src/lib.rs +++ b/crates/conceptweave-zotero/src/lib.rs @@ -209,6 +209,8 @@ pub struct ReviewedDuplicateMergeSet { pub rule_revision: String, /// Exact raw-snapshot digest reviewed by the steward. pub snapshot_digest: String, + /// Required v2 identity of proposals, retained metadata and pending sources. + pub proposal_digest: String, /// Exact item-key/item-version coordinates reviewed by the steward. pub snapshot_items: Vec, /// Exact duplicate membership reviewed by the steward. @@ -249,6 +251,8 @@ pub struct DuplicateMergeReviewManifest { pub rule_revision: String, /// Exact raw-snapshot digest shared with the reviewed decisions. pub snapshot_digest: String, + /// Verified identity of the proposals and complete retained source scope. + pub proposal_digest: String, /// Deterministically ordered canonical-key operations. pub operations: Vec, /// Classification never deletes or mutates Zotero source records. @@ -687,6 +691,11 @@ where } /// Builds a local-only reversible canonical-key manifest from steward-reviewed decisions. +/// +/// Validates source scope, receipt bindings and all operations before contacting +/// governance. The verifier must authenticate the complete independently issued +/// review, including candidate membership and the v2 proposal digest. Legacy +/// receipts require reapproval; digest recomputation alone grants no authority. pub fn build_duplicate_merge_review_manifest( report: &ClassificationReport, reviewed: &ReviewedDuplicateMergeSet, @@ -695,15 +704,18 @@ pub fn build_duplicate_merge_review_manifest( where F: FnOnce(&ReviewedDuplicateMergeSet) -> bool, { + validate_classification_report(report).map_err(|_| DuplicateReviewError::InvalidReview)?; if reviewed.review_id.trim().is_empty() || reviewed.authority_receipt.trim().is_empty() || reviewed.snapshot_digest.trim().is_empty() + || reviewed.proposal_digest.trim().is_empty() || reviewed.rule_revision.trim().is_empty() || reviewed.decisions.len() != report.duplicate_candidates.len() { return Err(DuplicateReviewError::InvalidReview); } if reviewed.snapshot_digest != report.snapshot_digest + || reviewed.proposal_digest != classification_proposal_digest(report) || reviewed.library_version != report.library_version || reviewed.rule_revision != report.rule_revision || reviewed.snapshot_items != report.snapshot_items @@ -711,24 +723,6 @@ where { return Err(DuplicateReviewError::SnapshotMismatch); } - if !verify_review(reviewed) { - return Err(DuplicateReviewError::UnverifiedApproval); - } - - if report - .snapshot_items - .iter() - .any(|item| item.item_key.trim().is_empty()) - || report - .snapshot_items - .iter() - .map(|item| item.item_key.as_str()) - .collect::>() - .len() - != report.snapshot_items.len() - { - return Err(DuplicateReviewError::InvalidReview); - } let item_revisions = report .snapshot_items .iter() @@ -830,6 +824,9 @@ where }); } + if !verify_review(reviewed) { + return Err(DuplicateReviewError::UnverifiedApproval); + } operations.sort_by(|left, right| { (&left.identity_kind, &left.normalized_identity) .cmp(&(&right.identity_kind, &right.normalized_identity)) @@ -840,6 +837,7 @@ where library_version: reviewed.library_version, rule_revision: reviewed.rule_revision.clone(), snapshot_digest: reviewed.snapshot_digest.clone(), + proposal_digest: reviewed.proposal_digest.clone(), operations, source_records_preserved: true, }) diff --git a/crates/conceptweave-zotero/tests/duplicate_merge_review_components.rs b/crates/conceptweave-zotero/tests/duplicate_merge_review_components.rs index 74a04906..21f95876 100644 --- a/crates/conceptweave-zotero/tests/duplicate_merge_review_components.rs +++ b/crates/conceptweave-zotero/tests/duplicate_merge_review_components.rs @@ -41,6 +41,7 @@ fn transitive_duplicate_component_accepts_one_component_level_canonical_key() { library_version: report.library_version, rule_revision: report.rule_revision.into(), snapshot_digest: report.snapshot_digest.clone(), + proposal_digest: conceptweave_zotero::classification_proposal_digest(&report), snapshot_items: report.snapshot_items.clone(), duplicate_candidates: report.duplicate_candidates.clone(), decisions: report @@ -97,6 +98,7 @@ fn duplicate_review_rejects_blank_snapshot_item_identity() { library_version: report.library_version, rule_revision: report.rule_revision.into(), snapshot_digest: report.snapshot_digest.clone(), + proposal_digest: conceptweave_zotero::classification_proposal_digest(&report), snapshot_items: report.snapshot_items.clone(), duplicate_candidates: report.duplicate_candidates.clone(), decisions: vec![DuplicateMergeDecision { @@ -138,6 +140,7 @@ fn duplicate_review_binds_every_item_revision_to_the_reviewed_snapshot() { library_version: report.library_version, rule_revision: report.rule_revision.into(), snapshot_digest: report.snapshot_digest.clone(), + proposal_digest: conceptweave_zotero::classification_proposal_digest(&report), snapshot_items: report.snapshot_items.clone(), duplicate_candidates: report.duplicate_candidates.clone(), decisions: report diff --git a/crates/conceptweave-zotero/tests/duplicate_merge_review_manifest.rs b/crates/conceptweave-zotero/tests/duplicate_merge_review_manifest.rs index 5454013b..61ab54d6 100644 --- a/crates/conceptweave-zotero/tests/duplicate_merge_review_manifest.rs +++ b/crates/conceptweave-zotero/tests/duplicate_merge_review_manifest.rs @@ -51,6 +51,7 @@ fn reviewed(report: &conceptweave_zotero::ClassificationReport) -> ReviewedDupli library_version: report.library_version, rule_revision: report.rule_revision.into(), snapshot_digest: report.snapshot_digest.clone(), + proposal_digest: conceptweave_zotero::classification_proposal_digest(&report), snapshot_items: report.snapshot_items.clone(), duplicate_candidates: report.duplicate_candidates.clone(), decisions: vec![ @@ -255,6 +256,7 @@ fn overlapping_duplicate_groups_require_one_consistent_canonical_choice() { library_version: report.library_version, rule_revision: report.rule_revision.into(), snapshot_digest: report.snapshot_digest.clone(), + proposal_digest: conceptweave_zotero::classification_proposal_digest(&report), snapshot_items: report.snapshot_items.clone(), duplicate_candidates: report.duplicate_candidates.clone(), decisions: vec![ @@ -298,6 +300,7 @@ fn duplicate_review_rejects_ambiguous_snapshot_key_revisions() { library_version: report.library_version, rule_revision: report.rule_revision.into(), snapshot_digest: report.snapshot_digest.clone(), + proposal_digest: conceptweave_zotero::classification_proposal_digest(&report), snapshot_items: report.snapshot_items.clone(), duplicate_candidates: report.duplicate_candidates.clone(), decisions: report From fc0465e57b52d4c6842f8a07ceebe8eccfa543f9 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sun, 6 Sep 2026 21:48:21 +0900 Subject: [PATCH 16/19] fix: retain duplicate snapshot mismatch error precedence Receipt/source comparison remains before structural admission, and both remain before independent governance. Preserve existing exact error contracts without weakening newly added source and decision checks. --- crates/conceptweave-zotero/src/lib.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/conceptweave-zotero/src/lib.rs b/crates/conceptweave-zotero/src/lib.rs index 1d93bd59..67c81568 100644 --- a/crates/conceptweave-zotero/src/lib.rs +++ b/crates/conceptweave-zotero/src/lib.rs @@ -704,7 +704,6 @@ pub fn build_duplicate_merge_review_manifest( where F: FnOnce(&ReviewedDuplicateMergeSet) -> bool, { - validate_classification_report(report).map_err(|_| DuplicateReviewError::InvalidReview)?; if reviewed.review_id.trim().is_empty() || reviewed.authority_receipt.trim().is_empty() || reviewed.snapshot_digest.trim().is_empty() @@ -723,6 +722,7 @@ where { return Err(DuplicateReviewError::SnapshotMismatch); } + validate_classification_report(report).map_err(|_| DuplicateReviewError::InvalidReview)?; let item_revisions = report .snapshot_items .iter() From 0c825d978e51248f9efaaabe09ee37fdf73a921b Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sun, 6 Sep 2026 21:48:52 +0900 Subject: [PATCH 17/19] test: preserve independent authority for rewritten duplicate scope --- .../tests/duplicate_merge_review_manifest.rs | 30 +++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/crates/conceptweave-zotero/tests/duplicate_merge_review_manifest.rs b/crates/conceptweave-zotero/tests/duplicate_merge_review_manifest.rs index 61ab54d6..66273533 100644 --- a/crates/conceptweave-zotero/tests/duplicate_merge_review_manifest.rs +++ b/crates/conceptweave-zotero/tests/duplicate_merge_review_manifest.rs @@ -79,6 +79,10 @@ fn reviewed_duplicate_decision_has_exact_before_after_and_rollback_mappings() { assert_eq!(manifest.authority_receipt, "authority-receipt-1"); assert_eq!(manifest.library_version, report.library_version); assert_eq!(manifest.rule_revision, report.rule_revision); + assert_eq!( + manifest.proposal_digest, + conceptweave_zotero::classification_proposal_digest(&report) + ); assert_eq!(manifest.operations.len(), 2); assert_eq!(manifest.operations[0].identity_kind, "doi"); assert_eq!(manifest.operations[1].identity_kind, "title"); @@ -157,6 +161,32 @@ fn duplicate_receipt_requires_explicit_source_scope_binding() { assert!(serde_json::from_value::(legacy).is_err()); } +#[test] +fn rewritten_duplicate_scope_receipt_cannot_reuse_independent_approval() { + let mut report = report(); + let mut review = reviewed(&report); + let original_review = review.clone(); + report.unclassified_items[0].data.title = "Changed retained evidence".into(); + review.proposal_digest = conceptweave_zotero::classification_proposal_digest(&report); + let called = std::cell::Cell::new(0); + assert_eq!( + build_duplicate_merge_review_manifest(&report, &review, |candidate| { + called.set(called.get() + 1); + candidate == &original_review + }), + Err(DuplicateReviewError::UnverifiedApproval) + ); + assert_eq!(called.get(), 1); + + review.proposal_digest.clear(); + assert_eq!( + build_duplicate_merge_review_manifest(&report, &review, |_| panic!( + "blank binding reached governance" + )), + Err(DuplicateReviewError::InvalidReview) + ); +} + #[test] fn duplicate_review_contract_fails_closed() { let report = report(); From 5fff9d05c418890513cb8f03faf006031e3fa205 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sun, 6 Sep 2026 21:50:56 +0900 Subject: [PATCH 18/19] test: remove redundant borrowed report reference --- .../tests/duplicate_merge_review_manifest.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/conceptweave-zotero/tests/duplicate_merge_review_manifest.rs b/crates/conceptweave-zotero/tests/duplicate_merge_review_manifest.rs index 66273533..ed085596 100644 --- a/crates/conceptweave-zotero/tests/duplicate_merge_review_manifest.rs +++ b/crates/conceptweave-zotero/tests/duplicate_merge_review_manifest.rs @@ -51,7 +51,7 @@ fn reviewed(report: &conceptweave_zotero::ClassificationReport) -> ReviewedDupli library_version: report.library_version, rule_revision: report.rule_revision.into(), snapshot_digest: report.snapshot_digest.clone(), - proposal_digest: conceptweave_zotero::classification_proposal_digest(&report), + proposal_digest: conceptweave_zotero::classification_proposal_digest(report), snapshot_items: report.snapshot_items.clone(), duplicate_candidates: report.duplicate_candidates.clone(), decisions: vec![ From 3d2c2529af3c57c10d7e31e806b8e39dbf524efe Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sun, 6 Sep 2026 21:51:36 +0900 Subject: [PATCH 19/19] docs: bind duplicate source scope to reviewed evidence --- CHANGELOG.md | 1 + docs/CONTEXT_MAP.md | 2 +- docs/PRD.md | 2 +- docs/TRD.md | 2 +- docs/UBIQUITOUS_LANGUAGE.md | 2 +- docs/adr/0006-zotero-research-intake.md | 10 ++++++++++ docs/product-technical-gap-baseline.md | 2 ++ 7 files changed, 17 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5c625695..7796fdf6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,7 @@ All notable changes to ConceptWeave are documented here. ### Fixed +- Duplicate review rejects incomplete source inventories and changed supporting evidence before approval, while retaining reversible identity mappings. - Research evaluation rejects incomplete source inventories and invalidates prior approvals when retained source metadata changes. - Research reports retain standalone files and notes that previously disappeared from the classification view, and flag sources whose parent relationships remain unresolved. - Zotero research intake rejects a read whose records claim revisions newer than the library being observed, without dropping papers or changing their recorded revisions. diff --git a/docs/CONTEXT_MAP.md b/docs/CONTEXT_MAP.md index 5da32fef..e7c9cce1 100644 --- a/docs/CONTEXT_MAP.md +++ b/docs/CONTEXT_MAP.md @@ -6,7 +6,7 @@ - Semantic Discovery -> Model Validation: **Conformist to published candidate contract**; validation must not rewrite discovery evidence. - Model Validation -> Governance & Publication: **Customer/Supplier**; governance consumes deterministic validation receipts. - Governance & Publication -> Interoperability: **Published Language**; adapters consume immutable release contracts. -- Research Intake -> Governance & Publication: **Anti-Corruption Layer**; Governance verifies the complete reviewed duplicate set and returns only an opaque authority receipt before Intake emits canonical-key operations. +- Research Intake -> Governance & Publication: **Anti-Corruption Layer**; Intake validates source inventory, audit, receipt bindings and all duplicate operations before Governance verifies the complete independently issued review, including candidate membership and retained-source identity. The opaque authority receipt does not authorize source mutation. ## External relationships diff --git a/docs/PRD.md b/docs/PRD.md index c3c0fa11..d4e7f4e0 100644 --- a/docs/PRD.md +++ b/docs/PRD.md @@ -64,7 +64,7 @@ Library reads must finish within a bounded observation window or fail visibly wi Read a complete Zotero Local API observation with one consistent library version and propose exactly one research disposition for every top-level bibliographic item. This consistency check does not establish an atomic provider snapshot. A record claiming a revision newer than the observed library invalidates the complete read; it must not be omitted or assigned a different revision to make the read pass. Each proposal retains the item key/version, exact matched metadata values, rule revision, linked child records, and any model receipt. Weak evidence and evidence that matches multiple specific disposition families must abstain into steward review. Duplicate DOI/title identities are review candidates only: intake never merges, deletes, or silently mutates Zotero records. -For every connected duplicate component, accept externally verified steward decisions selecting one component-level canonical item. Produce a local-only manifest that binds the decisions to the raw snapshot, its complete item-key/item-version coordinates, and exact duplicate-candidate membership, and records every component source revision plus before, after, and rollback canonical-key mappings. Classification preserves every Zotero source record. +For every connected duplicate component, accept externally verified steward decisions selecting one component-level canonical item. Produce a local-only manifest that binds decisions to the raw snapshot, complete item revisions, exact duplicate membership, current proposals and retained source metadata. Reject missing or inconsistent source inventory and invalid decisions before requesting approval. Changed retained evidence requires fresh independent approval, even when duplicate members are unchanged. Record every component source revision plus before, after, and rollback canonical-key mappings. Classification preserves every Zotero source record. Evaluate classifier quality only against a steward-reviewed local golden set whose governance receipt is externally verified and binds both the complete source/classifier-input snapshot and every current proposal field, in addition to the item-key/item-version coordinates. Same-version changes to unmodeled provider metadata, absent/default fields, classifier inputs, predictions or supporting evidence must invalidate the corresponding binding. Evaluation recomputes proposal identity before contacting governance; a locally changed digest cannot renew an approval. Legacy unbound approvals require reissuance, never automatic backfill. Abstention is a prediction outcome, never an approved truth label. Evaluation emits the verified library revision, rule revision, opaque snapshot and proposal digests, and aggregate counts for exact matches, abstentions, and per-disposition true-positive/predicted/expected totals; it must not copy Zotero keys, reviewer identity, or bibliographic text into the result. Every successful classification report includes aggregate evidence for snapshot coverage, proposal coverage, provenance completeness, abstentions, duplicate candidates, disposition totals, and zero unreported failures. diff --git a/docs/TRD.md b/docs/TRD.md index 912bfb0b..04d4db58 100644 --- a/docs/TRD.md +++ b/docs/TRD.md @@ -73,7 +73,7 @@ After count/byte validation and before accumulating each metadata page, every re Every top-level bibliographic record receives exactly one proposed disposition. `NeedsStewardReview` also records a deterministic abstention reason so missing classification metadata, vocabulary unsupported by the current deterministic rules, and present-but-unmatched metadata are distinguishable. DOI duplicate identity normalization treats bare DOI values, `doi:`, `doi.org`, and legacy `dx.doi.org` resolver forms as the same identity when their normalized DOI is equal. -Duplicate review is independent of subject classification. A reviewed decision set must match the exact raw-snapshot digest and complete item-key/item-version coordinates, cover every duplicate candidate exactly once, select one retained key from the connected duplicate component, and pass an external governance verifier. Every operation records all component item revisions, identity mappings before and after canonicalization, and the exact rollback mapping. These mappings affect only downstream identity resolution; Zotero records are neither mutated nor deleted. +Duplicate decisions are independent of subject labels but share the complete source-evidence admission boundary. A reviewed set must match the raw-snapshot digest, item revisions, exact candidate membership and required v2 `proposal_digest`. Receipt comparison retains `SnapshotMismatch` precedence; shared structural/audit admission and all component/decision checks then run before the external verifier. The verifier authenticates the entire independently issued set, not a locally recomputed digest. Missing scope bindings fail deserialization and blank bindings fail admission; no legacy default or automatic reapproval exists. The manifest retains the verified proposal digest. Every operation records all component item revisions, before/after identity maps and exact rollback. Zotero records remain unchanged. Golden-set evaluation accepts only a governance receipt verified with the complete reviewed set by a caller-owned authorization boundary. Its library version, rule revision, canonical SHA-256 content digest, and every observed parent/child item-key/item-version identity must bind the classification report. The snapshot digest covers every raw Zotero item in canonical key order. A separate required `proposal_digest` binds every field of every proposed item, including predictions, supporting evidence, and proposals outside a reviewed sample. `classification_proposal_digest` computes SHA-256 over compact JSON containing the `conceptweave-classification-proposals-v1` domain marker and proposal records sorted by item key and revision. It uses the current records, not a report's self-declared digest or a second stored source snapshot. Governance must issue and independently verify both digests together with the labels; a locally recomputed replacement digest cannot renew an old approval. Legacy approvals missing this field fail closed and require reissuance, not automatic backfill. diff --git a/docs/UBIQUITOUS_LANGUAGE.md b/docs/UBIQUITOUS_LANGUAGE.md index 566cb2fe..281497eb 100644 --- a/docs/UBIQUITOUS_LANGUAGE.md +++ b/docs/UBIQUITOUS_LANGUAGE.md @@ -16,6 +16,6 @@ | Dimension | Governed categorical or temporal axis used to group/filter analytical facts. | | Measure | Governed calculation with explicit expression, grain, units, null semantics, and evidence. | | Semantic Steward | Authorized reviewer responsible for accepting or rejecting semantic meaning. | -| Reviewed Duplicate Merge Set | Complete steward decisions selecting one consistent canonical item across every overlapping duplicate group in a snapshot. | +| Reviewed Duplicate Merge Set | Independently verified decisions selecting one canonical item per connected duplicate component, bound to exact source revisions, candidate membership, proposals and retained source scope. | | Authority Receipt | Opaque proof checked by the Governance & Publication boundary; it contains no reviewer identity or credential. | | Canonical-Key Operation | Reversible local mapping from every source in a connected duplicate component to one retained key, with complete reviewed revisions and the exact rollback mapping. | diff --git a/docs/adr/0006-zotero-research-intake.md b/docs/adr/0006-zotero-research-intake.md index b523ce78..22584a05 100644 --- a/docs/adr/0006-zotero-research-intake.md +++ b/docs/adr/0006-zotero-research-intake.md @@ -75,6 +75,16 @@ GREEN source `f6735b585022aac1c8ceff86c150d9b64fd77ec2` passed 68 tests across 1 ## Alternatives considered +### September 6 duplicate source-scope admission (Proposed) + +PR #12 already binds exact candidate membership and complete item revisions in `ReviewedDuplicateMergeSet`; the prior audit-owner concern about unbound duplicate authority therefore does not describe this consumer. Its real remaining gap was that retained metadata and inventory were absent from its receipt, and external verification preceded local decision checks. RED `4656d6b` reproduced missing legacy scope binding, malformed inventory accepted, and altered standalone evidence reaching governance. + +We reused the existing shared report validator and v2 proposal digest. Duplicate receipts and resulting manifests require `proposal_digest`; original candidate membership binding remains separate and unchanged. Missing bindings fail deserialization, blank bindings fail admission, and locally rewritten bindings still require independently issued approval of the complete set. All component and decision checks precede the external verifier. No source mutation, new authority issuer, digest algorithm or dependency was added. Snapshot mismatch takes precedence over structural errors to preserve existing caller contracts; `fc0465e` restores this after the first implementation exposed two exact-error regressions. + +Rejected alternatives were another source validator, copying the source snapshot into each operation, mixing derived counters into source identity, or defaulting legacy receipts to a recomputed digest. Each duplicates responsibility or weakens review evidence. The cost is explicit reapproval of older duplicate receipts and rejection after changed retained evidence, even when candidate membership stays constant. `0c825d9` proves that rewriting the digest does not reuse approval and the manifest retains the verified binding. Subsequent restored-report/worksheet/write owners must adopt the required fields without granting authority by downcasting full-text evidence. + +The isolated baseline at `a4a7c2d` passed 68 tests/17 suites; normal integration `b758991` preserved it and parent `6dff8c2`, passing 83/17. Current runtime `fc0465e` plus tests `0c825d9` passed 87/17 including two doctests. Independent duplicate tests passed 12/12 with no actionable finding. Unchanged pinned coverage passed 153/153 functions, 1,294/1,294 normalized owned regions and 220/220 normalized branches; raw 1,649/1,658 lines, 2,516/2,536 regions and 200/220 branches remain below 100%. Rustdoc/release passed; strict Clippy found one test-only redundant borrow, corrected in `5fff9d0` without suppression. Hosted checks, protected merge, release and genuine reviewed reclassification remain unproven. + ### September 6 derived audit repair (Proposed) Live PR #11 findings [3934799129](https://github.com/ContextualWisdomLab/ConceptWeave/pull/11#discussion_r3934799129) and [3934994550](https://github.com/ContextualWisdomLab/ConceptWeave/pull/11#discussion_r3934994550) were rechecked after source-scope integration. Source identity must not absorb derived audit fields, but accepting arbitrary audit values alongside a verified report can still misstate completeness. RED `ebdd852` proved both forged audit reaching governance and duplicate source keys counted as complete provenance. diff --git a/docs/product-technical-gap-baseline.md b/docs/product-technical-gap-baseline.md index 49eb6b43..0533d51b 100644 --- a/docs/product-technical-gap-baseline.md +++ b/docs/product-technical-gap-baseline.md @@ -56,6 +56,8 @@ Protected central source is `.github/main@b5efbc2762e472e4a380b0503b1f050f76fbb0 ## September 6 source-scope admission checkpoint +PR #12 successor runtime `fc0465e` now uses the shared inventory/audit validator and required v2 scope binding before independent duplicate governance. The existing exact candidate-membership receipt already covers duplicate selection; it is not an unprotected authority gap in this consumer. RED `4656d6b` reproduced three retained-source admission bypasses, now rejected. Current local result: 87 tests/17 suites, independent duplicate review 12/12, pinned coverage 153/153 functions, 1,294/1,294 normalized regions and 220/220 normalized branches. Raw coverage remains 1,649/1,658 lines, 2,516/2,536 regions, 200/220 branches. See the Proposed ADR 0006 amendment for history, compatibility and remaining gates. Later restored-report/worksheet/write integration is still required; current root PR #39 has not adopted these changes. + PR #11 local successor checkpoint `23178a9768aa216692d77918d56357ae1269535c` normally inherits #10 `fdf8b8d70c05bcb76c55cb6336c9bf31b5e42ce4` while preserving prior #11 `1dc032598b41a35d52c09d8690c871e07365d7e3`. The stable isolated baseline passed 60 tests/15 suites; an earlier run contaminated by merge timing is invalid evidence. RED `ebdd852` reproduced forged derived audit and ambiguous provenance totals. Extracted audit computation now counts unique parent/child identities and is recomputed before governance. Final local tests passed 75/15 suites including two doctests; independent integrity review passed 17 tests with no new regression. Strict Clippy passed at unchanged runtime `935e035`. Pinned coverage is still running; no inherited coverage claim or remote PR update is made here. The pinned coverage run subsequently finished with exit 0: 140/140 functions, 1,101/1,101 normalized owned regions, 182/182 normalized branches. Raw lines 1,502/1,505, regions 2,332/2,343 and branches 177/182 remain below 100%. The coverage script and exclusions are unchanged; the earlier pending sentence records chronology, not current execution state.