From e7d3243fed8955a61ec94f0e044b5dbf8ca12c35 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Fri, 4 Sep 2026 22:25:40 +0900 Subject: [PATCH 01/28] feat(research): add steward golden-set evaluation Signed-off-by: Seongho Bae --- crates/conceptweave-zotero/src/lib.rs | 150 ++++++++++++++++- .../tests/golden_set_evaluation.rs | 156 ++++++++++++++++++ docs/PRD.md | 2 + docs/TRD.md | 4 +- docs/adr/0006-zotero-research-intake.md | 2 + docs/product-technical-gap-baseline.md | 2 +- 6 files changed, 313 insertions(+), 3 deletions(-) create mode 100644 crates/conceptweave-zotero/tests/golden_set_evaluation.rs diff --git a/crates/conceptweave-zotero/src/lib.rs b/crates/conceptweave-zotero/src/lib.rs index ac9b92f1..0e9f239b 100644 --- a/crates/conceptweave-zotero/src/lib.rs +++ b/crates/conceptweave-zotero/src/lib.rs @@ -67,7 +67,7 @@ pub struct ItemTag { } /// One mutually exclusive proposed disposition. -#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)] +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Deserialize, Serialize)] #[serde(rename_all = "snake_case")] pub enum Disposition { /// Evidence about ontology or taxonomy generation. @@ -172,6 +172,154 @@ pub struct ClassificationReport { pub duplicate_candidates: Vec, } +/// One steward-reviewed expected disposition in a local golden set. +#[derive(Debug, Clone, PartialEq, Eq, Deserialize, Serialize)] +pub struct GoldenLabel { + /// Zotero item key used only to join the local report and local review set. + pub item_key: String, + /// Steward-approved disposition used as evaluation truth. + pub expected_disposition: Disposition, +} + +impl GoldenLabel { + /// Creates a local golden label. + pub fn new(item_key: impl Into, expected_disposition: Disposition) -> Self { + Self { + item_key: item_key.into(), + expected_disposition, + } + } +} + +/// Version-bound steward labels that remain outside the repository. +#[derive(Debug, Clone, PartialEq, Eq, Deserialize, Serialize)] +pub struct ReviewedGoldenSet { + /// Opaque review receipt identifier. + pub review_id: String, + /// Zotero library version reviewed by the steward. + pub library_version: u64, + /// Classifier rule revision whose proposals were reviewed. + pub rule_revision: String, + /// Item-level expected dispositions. + pub labels: Vec, +} + +/// Integer evidence from which precision and recall can be calculated exactly. +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize)] +pub struct DispositionEvaluation { + /// Correct predictions for this disposition. + pub true_positive: usize, + /// All classifier predictions for reviewed items in this disposition. + pub predicted: usize, + /// All steward labels expecting this disposition. + pub expected: usize, +} + +/// Aggregate-only evaluation result; item keys and bibliographic text are omitted. +#[derive(Debug, Clone, PartialEq, Eq, Serialize)] +pub struct GoldenSetEvaluation { + /// Opaque review receipt identifier. + pub review_id: String, + /// Number of steward-reviewed items. + pub reviewed_count: usize, + /// Number of exact disposition matches. + pub correct_count: usize, + /// Number of reviewed items on which the classifier abstained. + pub abstention_count: usize, + /// Precision/recall numerators and denominators per observed disposition. + pub by_disposition: BTreeMap, +} + +/// A fail-closed golden-set contract violation. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum EvaluationError { + /// Review receipt, labels, or revisions are missing or incompatible. + InvalidReview, + /// The golden set was reviewed against another library or rule revision. + SnapshotMismatch, + /// A reviewed key is absent from the classification report. + UnknownItem, + /// A reviewed key occurs more than once. + DuplicateItem, +} + +impl fmt::Display for EvaluationError { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter.write_str(match self { + Self::InvalidReview => "golden-set review metadata or labels are invalid", + Self::SnapshotMismatch => "golden set does not match the report snapshot", + Self::UnknownItem => "golden set contains an item absent from the report", + Self::DuplicateItem => "golden set contains a duplicate item", + }) + } +} + +impl std::error::Error for EvaluationError {} + +/// Evaluates reviewed labels without copying item identities into the result. +pub fn evaluate_reviewed_golden_set( + report: &ClassificationReport, + golden: &ReviewedGoldenSet, +) -> Result { + if golden.review_id.trim().is_empty() + || golden.labels.is_empty() + || golden.rule_revision.trim().is_empty() + { + return Err(EvaluationError::InvalidReview); + } + if golden.library_version != report.library_version + || golden.rule_revision != report.rule_revision + { + return Err(EvaluationError::SnapshotMismatch); + } + + let classified = report + .classified_items + .iter() + .map(|item| (item.item_key.as_str(), item.proposed_disposition)) + .collect::>(); + if classified.len() != report.classified_items.len() { + return Err(EvaluationError::InvalidReview); + } + let mut seen = BTreeSet::new(); + let mut correct_count = 0; + let mut abstention_count = 0; + let mut by_disposition = BTreeMap::::new(); + + for label in &golden.labels { + if label.item_key.trim().is_empty() { + return Err(EvaluationError::InvalidReview); + } + if !seen.insert(label.item_key.as_str()) { + return Err(EvaluationError::DuplicateItem); + } + let predicted = classified + .get(label.item_key.as_str()) + .copied() + .ok_or(EvaluationError::UnknownItem)?; + by_disposition.entry(predicted).or_default().predicted += 1; + by_disposition + .entry(label.expected_disposition) + .or_default() + .expected += 1; + if predicted == label.expected_disposition { + correct_count += 1; + by_disposition.entry(predicted).or_default().true_positive += 1; + } + if predicted == Disposition::NeedsStewardReview { + abstention_count += 1; + } + } + + Ok(GoldenSetEvaluation { + review_id: golden.review_id.clone(), + reviewed_count: golden.labels.len(), + correct_count, + abstention_count, + by_disposition, + }) +} + /// Failure raised when a bounded, immutable Local API read cannot be proven. #[derive(Debug)] pub enum ReadError { diff --git a/crates/conceptweave-zotero/tests/golden_set_evaluation.rs b/crates/conceptweave-zotero/tests/golden_set_evaluation.rs new file mode 100644 index 00000000..70ce1709 --- /dev/null +++ b/crates/conceptweave-zotero/tests/golden_set_evaluation.rs @@ -0,0 +1,156 @@ +use conceptweave_zotero::{ + Disposition, EvaluationError, GoldenLabel, ItemData, ReviewedGoldenSet, ZoteroItem, + classify_snapshot, evaluate_reviewed_golden_set, +}; + +fn item(key: &str, title: &str) -> ZoteroItem { + ZoteroItem { + key: key.into(), + version: 1, + data: ItemData { + item_type: "book".into(), + title: title.into(), + abstract_note: String::new(), + doi: String::new(), + parent_item: String::new(), + collections: vec![], + tags: vec![], + }, + } +} + +fn report() -> conceptweave_zotero::ClassificationReport { + classify_snapshot( + "9.0.6".into(), + None, + 42, + vec![ + item("A", "ontology learning"), + item("B", "ontology evaluation"), + item("C", "unmatched"), + ], + ) +} + +fn golden(labels: Vec) -> ReviewedGoldenSet { + ReviewedGoldenSet { + review_id: "synthetic-review-1".into(), + library_version: 42, + rule_revision: "ontology-research-v2".into(), + labels, + } +} + +#[test] +fn reviewed_golden_set_reports_count_based_precision_and_recall_evidence() { + let report = report(); + let evaluation = evaluate_reviewed_golden_set( + &report, + &golden(vec![ + GoldenLabel::new("A", Disposition::Generation), + GoldenLabel::new("B", Disposition::AlignmentVersioning), + GoldenLabel::new("C", Disposition::Generation), + ]), + ) + .unwrap(); + + assert_eq!(evaluation.review_id, "synthetic-review-1"); + assert_eq!(evaluation.reviewed_count, 3); + assert_eq!(evaluation.correct_count, 1); + assert_eq!(evaluation.abstention_count, 1); + let generation = &evaluation.by_disposition[&Disposition::Generation]; + assert_eq!( + ( + generation.true_positive, + generation.predicted, + generation.expected + ), + (1, 1, 2) + ); + let serialized = serde_json::to_value(&evaluation).unwrap(); + assert!(serialized.get("labels").is_none()); + assert!(serialized.get("item_key").is_none()); +} + +#[test] +fn reviewed_golden_set_rejects_stale_unknown_and_duplicate_labels() { + let report = report(); + + assert_eq!( + evaluate_reviewed_golden_set(&report, &golden(vec![])), + Err(EvaluationError::InvalidReview) + ); + let mut blank = golden(vec![GoldenLabel::new(" ", Disposition::Generation)]); + blank.review_id.clear(); + assert_eq!( + evaluate_reviewed_golden_set(&report, &blank), + Err(EvaluationError::InvalidReview) + ); + blank.review_id = "synthetic-review-1".into(); + assert_eq!( + evaluate_reviewed_golden_set(&report, &blank), + Err(EvaluationError::InvalidReview) + ); + let mut missing_revision = golden(vec![GoldenLabel::new("A", Disposition::Generation)]); + missing_revision.rule_revision.clear(); + assert_eq!( + evaluate_reviewed_golden_set(&report, &missing_revision), + Err(EvaluationError::InvalidReview) + ); + + let mut stale = golden(vec![GoldenLabel::new("A", Disposition::Generation)]); + stale.library_version += 1; + assert_eq!( + evaluate_reviewed_golden_set(&report, &stale), + Err(EvaluationError::SnapshotMismatch) + ); + stale.library_version = report.library_version; + stale.rule_revision = "older-rules".into(); + assert_eq!( + evaluate_reviewed_golden_set(&report, &stale), + Err(EvaluationError::SnapshotMismatch) + ); + + assert_eq!( + evaluate_reviewed_golden_set( + &report, + &golden(vec![GoldenLabel::new("missing", Disposition::Generation)]) + ), + Err(EvaluationError::UnknownItem) + ); + assert_eq!( + evaluate_reviewed_golden_set( + &report, + &golden(vec![ + GoldenLabel::new("A", Disposition::Generation), + GoldenLabel::new("A", Disposition::Generation), + ]) + ), + Err(EvaluationError::DuplicateItem) + ); + + let duplicate_report = classify_snapshot( + "9.0.6".into(), + None, + 42, + vec![ + item("A", "ontology learning"), + item("A", "ontology evaluation"), + ], + ); + assert_eq!( + evaluate_reviewed_golden_set( + &duplicate_report, + &golden(vec![GoldenLabel::new("A", Disposition::Generation)]) + ), + Err(EvaluationError::InvalidReview) + ); + for (error, fragment) in [ + (EvaluationError::InvalidReview, "invalid"), + (EvaluationError::SnapshotMismatch, "snapshot"), + (EvaluationError::UnknownItem, "absent"), + (EvaluationError::DuplicateItem, "duplicate"), + ] { + assert!(error.to_string().contains(fragment)); + } +} diff --git a/docs/PRD.md b/docs/PRD.md index b70635b4..a31916ee 100644 --- a/docs/PRD.md +++ b/docs/PRD.md @@ -58,6 +58,8 @@ All LLM-backed induction uses `contextual-orchestrator`. Model output is untrust Read one immutable Zotero Local API library-version snapshot and propose exactly one research disposition for every top-level bibliographic item. Each proposal retains the item key/version, exact matched metadata values, rule revision, linked child records, and any model receipt. Weak evidence and evidence that matches multiple specific disposition families must abstain into steward review. Duplicate DOI/title identities are review candidates only: intake never merges, deletes, or silently mutates Zotero records. +Evaluate classifier quality only against a version-bound, steward-reviewed local golden set. Evaluation emits aggregate counts for exact matches, abstentions, and per-disposition true-positive/predicted/expected totals; it must not copy Zotero keys or bibliographic text into the result. + ## 6. First vertical slice Relational schema snapshot -> observed tables/columns/foreign keys -> concept/relation/dimension/measure/mapping candidates -> evidence-bound validation report -> reviewable proposal package. diff --git a/docs/TRD.md b/docs/TRD.md index 535ee975..40f32201 100644 --- a/docs/TRD.md +++ b/docs/TRD.md @@ -59,8 +59,10 @@ Evaluation must separate extraction recall, semantic correctness, structural cor ## 11. Zotero research intake -`conceptweave-zotero` reads only the loopback Local API with at most 100 records per page, an 8 MiB page-body limit, a 50,000-item whole-snapshot limit, a 256 MiB cumulative body limit, redirects disabled, and finite connect/response/body/global timeouts. Every request pins `Zotero-API-Version: 3`; every response must report API version 3 and schema version 42. Before another request is issued, exhausted whole-snapshot budgets fail closed. Before a parsed page is accumulated, checked item-count and byte arithmetic must remain within both the advertised total and the configured whole-snapshot budgets. `Total-Results`, `Last-Modified-Version`, Zotero version, and server identity must remain identical across all pages; contract drift, malformed JSON, an empty intermediate page, duplicate keys, or an oversized response fails the run. +`conceptweave-zotero` reads only the loopback Local API with at most 100 records per page, an 8 MiB page-body limit, a 50,000-item whole-snapshot limit, a 256 MiB cumulative body limit, redirects disabled, and finite connect/response/body/global timeouts. Every request pins `Zotero-API-Version: 3`; every response must report API version 3, while its schema version is recorded and must remain stable across the snapshot. Before another request is issued, exhausted whole-snapshot budgets fail closed. Before a parsed page is accumulated, checked item-count and byte arithmetic must remain within both the advertised total and the configured whole-snapshot budgets. `Total-Results`, `Last-Modified-Version`, Zotero version, and server identity must remain identical across all pages; contract drift, malformed JSON, an empty intermediate page, duplicate keys, or an oversized response fails the run. Every top-level bibliographic record receives exactly one proposed disposition. `NeedsStewardReview` also records a deterministic abstention reason so missing classification metadata, vocabulary unsupported by the current deterministic rules, and present-but-unmatched metadata are distinguishable. DOI duplicate identity normalization treats bare DOI values, `doi:`, `doi.org`, and legacy `dx.doi.org` resolver forms as the same identity when their normalized DOI is equal. +Golden-set evaluation accepts only a non-empty review receipt whose library version and rule revision exactly match the classification report. Blank, duplicate, or unknown item keys and duplicate report identities fail closed. The output contains no item keys or bibliographic text; per-disposition `true_positive`, `predicted`, and `expected` counts retain exact precision/recall numerators and denominators without floating-point rounding. + The report is local JSON and contains proposals rather than governance decisions. CLI output is restricted to a new direct child of canonical `/tmp` or the operating system temporary directory; relative paths, nested paths, existing paths, and symlinks are rejected, and create-new file semantics prevent overwrite/path-swap writes. Zotero 9 writes are unsupported; no mutation path exists in this slice. A future Zotero 10+ writer requires a separate reviewed change with a Local API key, stable server identity, fresh item/library version preconditions, item-by-item before/after receipts, and rollback evidence. diff --git a/docs/adr/0006-zotero-research-intake.md b/docs/adr/0006-zotero-research-intake.md index b7d3a16d..18aeba36 100644 --- a/docs/adr/0006-zotero-research-intake.md +++ b/docs/adr/0006-zotero-research-intake.md @@ -19,6 +19,8 @@ The adapter links child records, emits exactly one deterministic proposed dispos Matched metadata values are copied into the local-only evidence receipt for replay. This is necessary for abstract-only matches because a later Zotero revision cannot reconstruct the exact text used for an earlier proposal from item key/version alone. DOI/title matches remain reversible duplicate candidates, including legacy `dx.doi.org` resolver forms. +Classifier quality is measured only against local steward-reviewed labels bound to the same library and rule revisions. Evaluation returns aggregate integer evidence for precision, recall, exact matches, and abstentions; Zotero keys and bibliographic text are deliberately omitted from evaluation output. Missing, stale, unknown, or duplicate review identities fail closed. + The reader fails closed above 50,000 items or 256 MiB of cumulative response bodies, while retaining the 8 MiB per-page bound, finite request timeouts, redirect denial, total-count checks, snapshot-version checks, and duplicate-key detection. Pagination, consistency, resource-budget, and provider-contract behavior are separated from the narrow `ureq` transport so deterministic tests exercise the production reader core rather than excluding the entire reader from coverage. Report output is restricted to a new direct child of the operating system temporary directory. Relative paths, nested paths, existing files, and symlinks are rejected before write; the file is opened with create-new semantics so a path swap cannot cause repository or arbitrary-file overwrite. The buffered writer is explicitly flushed and a final filesystem error fails the command. Reports stay local and are never committed. diff --git a/docs/product-technical-gap-baseline.md b/docs/product-technical-gap-baseline.md index fde6e0a1..a0165ff9 100644 --- a/docs/product-technical-gap-baseline.md +++ b/docs/product-technical-gap-baseline.md @@ -44,7 +44,7 @@ Protected central source is `.github/main@c31d2e5471fc5daf9d72ff67cde6a8874b736d Local evidence on 2026-09-04 showed Zotero 9.0.6, Local API v3/schema 42, library version 12341, 8,326 total items, and 3,719 top-level items. The corrected read-only run observed all 8,326 records at that single version and classified all 3,715 top-level bibliographic records; four top-level note/attachment/annotation records were correctly excluded. It proposed 56 adjacent-evidence records, 1 semantic-consumption bridge, and 3,658 steward-review abstentions, linked children for 3,287 records, and surfaced 49 reversible duplicate groups (18 DOI, 31 title). No live record matched multiple specific disposition families; the tested conflict path still abstains fail-closed. Token-boundary matching prevents strings such as `knowledge` from becoming false OWL evidence. These are local aggregate observations, not reviewed truth or applied Zotero changes. The report stays outside the repository. -The next RED is a steward-reviewed golden set that measures disposition precision/recall and expands multilingual rules without reducing abstention safety. Zotero write-back remains blocked by the installed v9 capability; a Zotero 10+ change must satisfy ADR 0006 preconditions. A dedicated utility repository remains unnecessary until an independently released cross-product contract exists. +The golden-set evaluation contract now records aggregate precision/recall numerators and denominators, rejects stale or malformed review sets, and omits item identities from its output. Synthetic fixtures verify the contract; no real precision/recall claim exists until a steward supplies reviewed local labels. Multilingual rule expansion remains a later evidence-driven change and must not reduce abstention safety. Zotero write-back remains blocked by the installed v9 capability; a Zotero 10+ change must satisfy ADR 0006 preconditions. A dedicated utility repository remains unnecessary until an independently released cross-product contract exists. 1. **Concrete Source Observation adapter** — maintained Rust PostgreSQL driver behind `conceptweave-source-port`; adapter-local credential resolution; explicit read-only mode; statement timeout, cancellation, row/byte/concurrency budgets; complete immutable snapshot or fail closed; deterministic replay against a frozen anonymized GRC-shaped fixture. 2. **Ontology discovery** — deterministic term/concept/taxonomy/non-taxonomic-relation candidate generation with exact source receipts and abstention for unsupported semantics. From 981771f1bf9f828c7a6f1585d4ec5ff06735bbbf Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Fri, 4 Sep 2026 22:43:29 +0900 Subject: [PATCH 02/28] experiment: bind golden evaluation to verified snapshot Signed-off-by: Seongho Bae --- crates/conceptweave-zotero/src/lib.rs | 84 +++++++++++++--- .../tests/golden_set_evaluation.rs | 97 +++++++++++++++---- docs/PRD.md | 2 +- docs/TRD.md | 2 +- docs/adr/0006-zotero-research-intake.md | 2 +- docs/product-technical-gap-baseline.md | 2 +- 6 files changed, 151 insertions(+), 38 deletions(-) diff --git a/crates/conceptweave-zotero/src/lib.rs b/crates/conceptweave-zotero/src/lib.rs index 0e9f239b..deec767b 100644 --- a/crates/conceptweave-zotero/src/lib.rs +++ b/crates/conceptweave-zotero/src/lib.rs @@ -194,14 +194,34 @@ impl GoldenLabel { /// Version-bound steward labels that remain outside the repository. #[derive(Debug, Clone, PartialEq, Eq, Deserialize, Serialize)] pub struct ReviewedGoldenSet { - /// Opaque review receipt identifier. - pub review_id: String, + /// Approval receipt verified by the caller's governance boundary. + pub approval: GoldenSetApproval, + /// Item-level expected dispositions. + pub labels: Vec, +} + +/// One item revision in the exact reviewed classification snapshot. +#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Deserialize, Serialize)] +pub struct SnapshotItemRevision { + /// Stable Zotero item key. + pub item_key: String, + /// Item revision observed during review. + pub item_version: u64, +} + +/// Governance receipt binding a steward approval to one exact classifier input. +#[derive(Debug, Clone, PartialEq, Eq, Deserialize, Serialize)] +pub struct GoldenSetApproval { + /// Opaque receipt identifier. + pub receipt_id: String, + /// Stable reviewer subject understood by the governance verifier. + pub reviewer_subject: String, /// Zotero library version reviewed by the steward. pub library_version: u64, /// Classifier rule revision whose proposals were reviewed. pub rule_revision: String, - /// Item-level expected dispositions. - pub labels: Vec, + /// Complete sorted item-revision identity of the reviewed report. + pub snapshot_items: Vec, } /// Integer evidence from which precision and recall can be calculated exactly. @@ -237,6 +257,10 @@ pub enum EvaluationError { InvalidReview, /// The golden set was reviewed against another library or rule revision. SnapshotMismatch, + /// The caller's governance boundary did not verify the approval receipt. + UnverifiedApproval, + /// Abstention cannot be used as steward-approved semantic truth. + InvalidExpectedDisposition, /// A reviewed key is absent from the classification report. UnknownItem, /// A reviewed key occurs more than once. @@ -248,6 +272,10 @@ impl fmt::Display for EvaluationError { formatter.write_str(match self { Self::InvalidReview => "golden-set review metadata or labels are invalid", Self::SnapshotMismatch => "golden set does not match the report snapshot", + Self::UnverifiedApproval => "golden-set approval receipt is unverified", + Self::InvalidExpectedDisposition => { + "steward truth cannot use the classifier abstention disposition" + } Self::UnknownItem => "golden set contains an item absent from the report", Self::DuplicateItem => "golden set contains a duplicate item", }) @@ -257,18 +285,46 @@ impl fmt::Display for EvaluationError { impl std::error::Error for EvaluationError {} /// Evaluates reviewed labels without copying item identities into the result. -pub fn evaluate_reviewed_golden_set( +pub fn evaluate_reviewed_golden_set( report: &ClassificationReport, golden: &ReviewedGoldenSet, -) -> Result { - if golden.review_id.trim().is_empty() + verify_approval: F, +) -> Result +where + F: FnOnce(&GoldenSetApproval) -> bool, +{ + if golden.approval.receipt_id.trim().is_empty() + || golden.approval.reviewer_subject.trim().is_empty() || golden.labels.is_empty() - || golden.rule_revision.trim().is_empty() + || golden.approval.rule_revision.trim().is_empty() { return Err(EvaluationError::InvalidReview); } - if golden.library_version != report.library_version - || golden.rule_revision != report.rule_revision + if !verify_approval(&golden.approval) { + return Err(EvaluationError::UnverifiedApproval); + } + let report_snapshot = report + .classified_items + .iter() + .map(|item| SnapshotItemRevision { + item_key: item.item_key.clone(), + item_version: item.item_version, + }) + .collect::>(); + let approved_snapshot = golden + .approval + .snapshot_items + .iter() + .cloned() + .collect::>(); + if report_snapshot.len() != report.classified_items.len() + || approved_snapshot.len() != golden.approval.snapshot_items.len() + { + return Err(EvaluationError::InvalidReview); + } + if golden.approval.library_version != report.library_version + || golden.approval.rule_revision != report.rule_revision + || approved_snapshot != report_snapshot { return Err(EvaluationError::SnapshotMismatch); } @@ -278,9 +334,6 @@ pub fn evaluate_reviewed_golden_set( .iter() .map(|item| (item.item_key.as_str(), item.proposed_disposition)) .collect::>(); - if classified.len() != report.classified_items.len() { - return Err(EvaluationError::InvalidReview); - } let mut seen = BTreeSet::new(); let mut correct_count = 0; let mut abstention_count = 0; @@ -290,6 +343,9 @@ pub fn evaluate_reviewed_golden_set( if label.item_key.trim().is_empty() { return Err(EvaluationError::InvalidReview); } + if label.expected_disposition == Disposition::NeedsStewardReview { + return Err(EvaluationError::InvalidExpectedDisposition); + } if !seen.insert(label.item_key.as_str()) { return Err(EvaluationError::DuplicateItem); } @@ -312,7 +368,7 @@ pub fn evaluate_reviewed_golden_set( } Ok(GoldenSetEvaluation { - review_id: golden.review_id.clone(), + review_id: golden.approval.receipt_id.clone(), reviewed_count: golden.labels.len(), correct_count, abstention_count, diff --git a/crates/conceptweave-zotero/tests/golden_set_evaluation.rs b/crates/conceptweave-zotero/tests/golden_set_evaluation.rs index 70ce1709..26539c09 100644 --- a/crates/conceptweave-zotero/tests/golden_set_evaluation.rs +++ b/crates/conceptweave-zotero/tests/golden_set_evaluation.rs @@ -1,6 +1,6 @@ use conceptweave_zotero::{ - Disposition, EvaluationError, GoldenLabel, ItemData, ReviewedGoldenSet, ZoteroItem, - classify_snapshot, evaluate_reviewed_golden_set, + Disposition, EvaluationError, GoldenLabel, GoldenSetApproval, ItemData, ReviewedGoldenSet, + SnapshotItemRevision, ZoteroItem, classify_snapshot, evaluate_reviewed_golden_set, }; fn item(key: &str, title: &str) -> ZoteroItem { @@ -34,13 +34,27 @@ fn report() -> conceptweave_zotero::ClassificationReport { fn golden(labels: Vec) -> ReviewedGoldenSet { ReviewedGoldenSet { - review_id: "synthetic-review-1".into(), - library_version: 42, - rule_revision: "ontology-research-v2".into(), + approval: GoldenSetApproval { + receipt_id: "synthetic-review-1".into(), + reviewer_subject: "synthetic-steward".into(), + library_version: 42, + rule_revision: "ontology-research-v2".into(), + snapshot_items: ["A", "B", "C"] + .into_iter() + .map(|item_key| SnapshotItemRevision { + item_key: item_key.into(), + item_version: 1, + }) + .collect(), + }, labels, } } +fn verify_synthetic_approval(approval: &GoldenSetApproval) -> bool { + approval.receipt_id == "synthetic-review-1" && approval.reviewer_subject == "synthetic-steward" +} + #[test] fn reviewed_golden_set_reports_count_based_precision_and_recall_evidence() { let report = report(); @@ -51,6 +65,7 @@ fn reviewed_golden_set_reports_count_based_precision_and_recall_evidence() { GoldenLabel::new("B", Disposition::AlignmentVersioning), GoldenLabel::new("C", Disposition::Generation), ]), + verify_synthetic_approval, ) .unwrap(); @@ -77,44 +92,82 @@ fn reviewed_golden_set_rejects_stale_unknown_and_duplicate_labels() { let report = report(); assert_eq!( - evaluate_reviewed_golden_set(&report, &golden(vec![])), + evaluate_reviewed_golden_set(&report, &golden(vec![]), verify_synthetic_approval), Err(EvaluationError::InvalidReview) ); let mut blank = golden(vec![GoldenLabel::new(" ", Disposition::Generation)]); - blank.review_id.clear(); + blank.approval.receipt_id.clear(); + assert_eq!( + evaluate_reviewed_golden_set(&report, &blank, verify_synthetic_approval), + Err(EvaluationError::InvalidReview) + ); + blank.approval.receipt_id = "synthetic-review-1".into(); assert_eq!( - evaluate_reviewed_golden_set(&report, &blank), + evaluate_reviewed_golden_set(&report, &blank, verify_synthetic_approval), Err(EvaluationError::InvalidReview) ); - blank.review_id = "synthetic-review-1".into(); + blank.approval.reviewer_subject.clear(); assert_eq!( - evaluate_reviewed_golden_set(&report, &blank), + evaluate_reviewed_golden_set(&report, &blank, verify_synthetic_approval), Err(EvaluationError::InvalidReview) ); let mut missing_revision = golden(vec![GoldenLabel::new("A", Disposition::Generation)]); - missing_revision.rule_revision.clear(); + missing_revision.approval.rule_revision.clear(); assert_eq!( - evaluate_reviewed_golden_set(&report, &missing_revision), + evaluate_reviewed_golden_set(&report, &missing_revision, verify_synthetic_approval), Err(EvaluationError::InvalidReview) ); let mut stale = golden(vec![GoldenLabel::new("A", Disposition::Generation)]); - stale.library_version += 1; + stale.approval.library_version += 1; assert_eq!( - evaluate_reviewed_golden_set(&report, &stale), + evaluate_reviewed_golden_set(&report, &stale, verify_synthetic_approval), Err(EvaluationError::SnapshotMismatch) ); - stale.library_version = report.library_version; - stale.rule_revision = "older-rules".into(); + stale.approval.library_version = report.library_version; + stale.approval.rule_revision = "older-rules".into(); assert_eq!( - evaluate_reviewed_golden_set(&report, &stale), + evaluate_reviewed_golden_set(&report, &stale, verify_synthetic_approval), Err(EvaluationError::SnapshotMismatch) ); + stale.approval.rule_revision = report.rule_revision.into(); + stale.approval.snapshot_items[0].item_version += 1; + assert_eq!( + evaluate_reviewed_golden_set(&report, &stale, verify_synthetic_approval), + Err(EvaluationError::SnapshotMismatch) + ); + let mut duplicate_snapshot = golden(vec![GoldenLabel::new("A", Disposition::Generation)]); + duplicate_snapshot + .approval + .snapshot_items + .push(duplicate_snapshot.approval.snapshot_items[0].clone()); + assert_eq!( + evaluate_reviewed_golden_set(&report, &duplicate_snapshot, verify_synthetic_approval), + Err(EvaluationError::InvalidReview) + ); + + assert_eq!( + evaluate_reviewed_golden_set( + &report, + &golden(vec![GoldenLabel::new("A", Disposition::Generation)]), + |_| false + ), + Err(EvaluationError::UnverifiedApproval) + ); + assert_eq!( + evaluate_reviewed_golden_set( + &report, + &golden(vec![GoldenLabel::new("A", Disposition::NeedsStewardReview)]), + verify_synthetic_approval, + ), + Err(EvaluationError::InvalidExpectedDisposition) + ); assert_eq!( evaluate_reviewed_golden_set( &report, - &golden(vec![GoldenLabel::new("missing", Disposition::Generation)]) + &golden(vec![GoldenLabel::new("missing", Disposition::Generation)]), + verify_synthetic_approval, ), Err(EvaluationError::UnknownItem) ); @@ -124,7 +177,8 @@ fn reviewed_golden_set_rejects_stale_unknown_and_duplicate_labels() { &golden(vec![ GoldenLabel::new("A", Disposition::Generation), GoldenLabel::new("A", Disposition::Generation), - ]) + ]), + verify_synthetic_approval, ), Err(EvaluationError::DuplicateItem) ); @@ -141,13 +195,16 @@ fn reviewed_golden_set_rejects_stale_unknown_and_duplicate_labels() { assert_eq!( evaluate_reviewed_golden_set( &duplicate_report, - &golden(vec![GoldenLabel::new("A", Disposition::Generation)]) + &golden(vec![GoldenLabel::new("A", Disposition::Generation)]), + verify_synthetic_approval, ), Err(EvaluationError::InvalidReview) ); for (error, fragment) in [ (EvaluationError::InvalidReview, "invalid"), (EvaluationError::SnapshotMismatch, "snapshot"), + (EvaluationError::UnverifiedApproval, "unverified"), + (EvaluationError::InvalidExpectedDisposition, "abstention"), (EvaluationError::UnknownItem, "absent"), (EvaluationError::DuplicateItem, "duplicate"), ] { diff --git a/docs/PRD.md b/docs/PRD.md index a31916ee..ef97ce8b 100644 --- a/docs/PRD.md +++ b/docs/PRD.md @@ -58,7 +58,7 @@ All LLM-backed induction uses `contextual-orchestrator`. Model output is untrust Read one immutable Zotero Local API library-version snapshot and propose exactly one research disposition for every top-level bibliographic item. Each proposal retains the item key/version, exact matched metadata values, rule revision, linked child records, and any model receipt. Weak evidence and evidence that matches multiple specific disposition families must abstain into steward review. Duplicate DOI/title identities are review candidates only: intake never merges, deletes, or silently mutates Zotero records. -Evaluate classifier quality only against a version-bound, steward-reviewed local golden set. Evaluation emits aggregate counts for exact matches, abstentions, and per-disposition true-positive/predicted/expected totals; it must not copy Zotero keys or bibliographic text into the result. +Evaluate classifier quality only against a steward-reviewed local golden set whose governance receipt is externally verified and bound to the complete Zotero item-key/item-version snapshot. Abstention is a prediction outcome, never an approved truth label. Evaluation emits aggregate counts for exact matches, abstentions, and per-disposition true-positive/predicted/expected totals; it must not copy Zotero keys, reviewer identity, or bibliographic text into the result. ## 6. First vertical slice diff --git a/docs/TRD.md b/docs/TRD.md index 40f32201..91b75603 100644 --- a/docs/TRD.md +++ b/docs/TRD.md @@ -63,6 +63,6 @@ Evaluation must separate extraction recall, semantic correctness, structural cor Every top-level bibliographic record receives exactly one proposed disposition. `NeedsStewardReview` also records a deterministic abstention reason so missing classification metadata, vocabulary unsupported by the current deterministic rules, and present-but-unmatched metadata are distinguishable. DOI duplicate identity normalization treats bare DOI values, `doi:`, `doi.org`, and legacy `dx.doi.org` resolver forms as the same identity when their normalized DOI is equal. -Golden-set evaluation accepts only a non-empty review receipt whose library version and rule revision exactly match the classification report. Blank, duplicate, or unknown item keys and duplicate report identities fail closed. The output contains no item keys or bibliographic text; per-disposition `true_positive`, `predicted`, and `expected` counts retain exact precision/recall numerators and denominators without floating-point rounding. +Golden-set evaluation accepts only a governance receipt verified by a caller-owned authorization boundary. Its library version, rule revision, and complete sorted item-key/item-version identity must exactly match the classification report. Blank, duplicate, unknown, stale, or abstention-as-truth labels fail closed. The output contains no item keys, reviewer identity, or bibliographic text; per-disposition `true_positive`, `predicted`, and `expected` counts retain exact precision/recall numerators and denominators without floating-point rounding. Production authorization remains Keyverse/governance-owned; this crate validates the binding and does not mint or self-verify authority. The report is local JSON and contains proposals rather than governance decisions. CLI output is restricted to a new direct child of canonical `/tmp` or the operating system temporary directory; relative paths, nested paths, existing paths, and symlinks are rejected, and create-new file semantics prevent overwrite/path-swap writes. Zotero 9 writes are unsupported; no mutation path exists in this slice. A future Zotero 10+ writer requires a separate reviewed change with a Local API key, stable server identity, fresh item/library version preconditions, item-by-item before/after receipts, and rollback evidence. diff --git a/docs/adr/0006-zotero-research-intake.md b/docs/adr/0006-zotero-research-intake.md index 18aeba36..c582155a 100644 --- a/docs/adr/0006-zotero-research-intake.md +++ b/docs/adr/0006-zotero-research-intake.md @@ -19,7 +19,7 @@ The adapter links child records, emits exactly one deterministic proposed dispos Matched metadata values are copied into the local-only evidence receipt for replay. This is necessary for abstract-only matches because a later Zotero revision cannot reconstruct the exact text used for an earlier proposal from item key/version alone. DOI/title matches remain reversible duplicate candidates, including legacy `dx.doi.org` resolver forms. -Classifier quality is measured only against local steward-reviewed labels bound to the same library and rule revisions. Evaluation returns aggregate integer evidence for precision, recall, exact matches, and abstentions; Zotero keys and bibliographic text are deliberately omitted from evaluation output. Missing, stale, unknown, or duplicate review identities fail closed. +Classifier quality is measured only against local steward-reviewed labels whose governance receipt is verified outside this crate and bound to the exact library version, rule revision, and complete item-key/item-version snapshot. `NeedsStewardReview` is an abstention prediction and cannot be approved truth. Evaluation returns aggregate integer evidence for precision, recall, exact matches, and abstentions; Zotero keys, reviewer identity, and bibliographic text are deliberately omitted from evaluation output. Missing, stale, unverified, unknown, duplicate, or invalid review identities fail closed. The reader fails closed above 50,000 items or 256 MiB of cumulative response bodies, while retaining the 8 MiB per-page bound, finite request timeouts, redirect denial, total-count checks, snapshot-version checks, and duplicate-key detection. Pagination, consistency, resource-budget, and provider-contract behavior are separated from the narrow `ureq` transport so deterministic tests exercise the production reader core rather than excluding the entire reader from coverage. diff --git a/docs/product-technical-gap-baseline.md b/docs/product-technical-gap-baseline.md index a0165ff9..885c31b8 100644 --- a/docs/product-technical-gap-baseline.md +++ b/docs/product-technical-gap-baseline.md @@ -44,7 +44,7 @@ Protected central source is `.github/main@c31d2e5471fc5daf9d72ff67cde6a8874b736d Local evidence on 2026-09-04 showed Zotero 9.0.6, Local API v3/schema 42, library version 12341, 8,326 total items, and 3,719 top-level items. The corrected read-only run observed all 8,326 records at that single version and classified all 3,715 top-level bibliographic records; four top-level note/attachment/annotation records were correctly excluded. It proposed 56 adjacent-evidence records, 1 semantic-consumption bridge, and 3,658 steward-review abstentions, linked children for 3,287 records, and surfaced 49 reversible duplicate groups (18 DOI, 31 title). No live record matched multiple specific disposition families; the tested conflict path still abstains fail-closed. Token-boundary matching prevents strings such as `knowledge` from becoming false OWL evidence. These are local aggregate observations, not reviewed truth or applied Zotero changes. The report stays outside the repository. -The golden-set evaluation contract now records aggregate precision/recall numerators and denominators, rejects stale or malformed review sets, and omits item identities from its output. Synthetic fixtures verify the contract; no real precision/recall claim exists until a steward supplies reviewed local labels. Multilingual rule expansion remains a later evidence-driven change and must not reduce abstention safety. Zotero write-back remains blocked by the installed v9 capability; a Zotero 10+ change must satisfy ADR 0006 preconditions. A dedicated utility repository remains unnecessary until an independently released cross-product contract exists. +The golden-set evaluation contract now records aggregate precision/recall numerators and denominators, requires an externally verified governance receipt bound to the complete item-key/item-version snapshot, rejects abstention as expected truth, and omits item and reviewer identities from its output. Synthetic fixtures verify the contract; no real precision/recall claim exists until a steward supplies reviewed local labels and the production authorization adapter verifies the receipt. Multilingual rule expansion remains a later evidence-driven change and must not reduce abstention safety. Zotero write-back remains blocked by the installed v9 capability; a Zotero 10+ change must satisfy ADR 0006 preconditions. A dedicated utility repository remains unnecessary until an independently released cross-product contract exists. 1. **Concrete Source Observation adapter** — maintained Rust PostgreSQL driver behind `conceptweave-source-port`; adapter-local credential resolution; explicit read-only mode; statement timeout, cancellation, row/byte/concurrency budgets; complete immutable snapshot or fail closed; deterministic replay against a frozen anonymized GRC-shaped fixture. 2. **Ontology discovery** — deterministic term/concept/taxonomy/non-taxonomic-relation candidate generation with exact source receipts and abstention for unsupported semantics. From 399794e5ae8274bc13601e98b26ef6e53a502901 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Fri, 4 Sep 2026 22:48:44 +0900 Subject: [PATCH 03/28] fix: retain golden snapshot evaluation identity Signed-off-by: Seongho Bae --- crates/conceptweave-zotero/src/lib.rs | 12 ++++++++++++ .../tests/golden_set_evaluation.rs | 11 +++++++++++ docs/PRD.md | 2 +- docs/TRD.md | 2 +- docs/adr/0006-zotero-research-intake.md | 2 +- docs/product-technical-gap-baseline.md | 2 +- 6 files changed, 27 insertions(+), 4 deletions(-) diff --git a/crates/conceptweave-zotero/src/lib.rs b/crates/conceptweave-zotero/src/lib.rs index deec767b..53ea83e1 100644 --- a/crates/conceptweave-zotero/src/lib.rs +++ b/crates/conceptweave-zotero/src/lib.rs @@ -220,6 +220,8 @@ pub struct GoldenSetApproval { pub library_version: u64, /// Classifier rule revision whose proposals were reviewed. pub rule_revision: String, + /// Immutable digest over the approved snapshot, verified by the caller. + pub snapshot_digest: String, /// Complete sorted item-revision identity of the reviewed report. pub snapshot_items: Vec, } @@ -240,6 +242,12 @@ pub struct DispositionEvaluation { pub struct GoldenSetEvaluation { /// Opaque review receipt identifier. pub review_id: String, + /// Zotero library revision bound to the verified receipt. + pub library_version: u64, + /// Classifier revision bound to the verified receipt. + pub rule_revision: String, + /// Opaque immutable snapshot digest from the verified receipt. + pub snapshot_digest: String, /// Number of steward-reviewed items. pub reviewed_count: usize, /// Number of exact disposition matches. @@ -297,6 +305,7 @@ where || golden.approval.reviewer_subject.trim().is_empty() || golden.labels.is_empty() || golden.approval.rule_revision.trim().is_empty() + || golden.approval.snapshot_digest.trim().is_empty() { return Err(EvaluationError::InvalidReview); } @@ -369,6 +378,9 @@ where Ok(GoldenSetEvaluation { review_id: golden.approval.receipt_id.clone(), + library_version: golden.approval.library_version, + rule_revision: golden.approval.rule_revision.clone(), + snapshot_digest: golden.approval.snapshot_digest.clone(), reviewed_count: golden.labels.len(), correct_count, abstention_count, diff --git a/crates/conceptweave-zotero/tests/golden_set_evaluation.rs b/crates/conceptweave-zotero/tests/golden_set_evaluation.rs index 26539c09..e8f94e43 100644 --- a/crates/conceptweave-zotero/tests/golden_set_evaluation.rs +++ b/crates/conceptweave-zotero/tests/golden_set_evaluation.rs @@ -39,6 +39,7 @@ fn golden(labels: Vec) -> ReviewedGoldenSet { reviewer_subject: "synthetic-steward".into(), library_version: 42, rule_revision: "ontology-research-v2".into(), + snapshot_digest: "sha256:synthetic-snapshot".into(), snapshot_items: ["A", "B", "C"] .into_iter() .map(|item_key| SnapshotItemRevision { @@ -70,6 +71,9 @@ fn reviewed_golden_set_reports_count_based_precision_and_recall_evidence() { .unwrap(); assert_eq!(evaluation.review_id, "synthetic-review-1"); + assert_eq!(evaluation.library_version, 42); + assert_eq!(evaluation.rule_revision, "ontology-research-v2"); + assert_eq!(evaluation.snapshot_digest, "sha256:synthetic-snapshot"); assert_eq!(evaluation.reviewed_count, 3); assert_eq!(evaluation.correct_count, 1); assert_eq!(evaluation.abstention_count, 1); @@ -85,6 +89,7 @@ fn reviewed_golden_set_reports_count_based_precision_and_recall_evidence() { let serialized = serde_json::to_value(&evaluation).unwrap(); assert!(serialized.get("labels").is_none()); assert!(serialized.get("item_key").is_none()); + assert!(serialized.get("reviewer_subject").is_none()); } #[test] @@ -117,6 +122,12 @@ fn reviewed_golden_set_rejects_stale_unknown_and_duplicate_labels() { evaluate_reviewed_golden_set(&report, &missing_revision, verify_synthetic_approval), Err(EvaluationError::InvalidReview) ); + missing_revision.approval.rule_revision = "ontology-research-v2".into(); + missing_revision.approval.snapshot_digest.clear(); + assert_eq!( + evaluate_reviewed_golden_set(&report, &missing_revision, verify_synthetic_approval), + Err(EvaluationError::InvalidReview) + ); let mut stale = golden(vec![GoldenLabel::new("A", Disposition::Generation)]); stale.approval.library_version += 1; diff --git a/docs/PRD.md b/docs/PRD.md index ef97ce8b..e0557754 100644 --- a/docs/PRD.md +++ b/docs/PRD.md @@ -58,7 +58,7 @@ All LLM-backed induction uses `contextual-orchestrator`. Model output is untrust Read one immutable Zotero Local API library-version snapshot and propose exactly one research disposition for every top-level bibliographic item. Each proposal retains the item key/version, exact matched metadata values, rule revision, linked child records, and any model receipt. Weak evidence and evidence that matches multiple specific disposition families must abstain into steward review. Duplicate DOI/title identities are review candidates only: intake never merges, deletes, or silently mutates Zotero records. -Evaluate classifier quality only against a steward-reviewed local golden set whose governance receipt is externally verified and bound to the complete Zotero item-key/item-version snapshot. Abstention is a prediction outcome, never an approved truth label. Evaluation emits aggregate counts for exact matches, abstentions, and per-disposition true-positive/predicted/expected totals; it must not copy Zotero keys, reviewer identity, or bibliographic text into the result. +Evaluate classifier quality only against a steward-reviewed local golden set whose governance receipt is externally verified and bound to the complete Zotero item-key/item-version snapshot. Abstention is a prediction outcome, never an approved truth label. Evaluation emits the verified library revision, rule revision, opaque snapshot digest, and aggregate counts for exact matches, abstentions, and per-disposition true-positive/predicted/expected totals; it must not copy Zotero keys, reviewer identity, or bibliographic text into the result. ## 6. First vertical slice diff --git a/docs/TRD.md b/docs/TRD.md index 91b75603..103eb92a 100644 --- a/docs/TRD.md +++ b/docs/TRD.md @@ -63,6 +63,6 @@ Evaluation must separate extraction recall, semantic correctness, structural cor Every top-level bibliographic record receives exactly one proposed disposition. `NeedsStewardReview` also records a deterministic abstention reason so missing classification metadata, vocabulary unsupported by the current deterministic rules, and present-but-unmatched metadata are distinguishable. DOI duplicate identity normalization treats bare DOI values, `doi:`, `doi.org`, and legacy `dx.doi.org` resolver forms as the same identity when their normalized DOI is equal. -Golden-set evaluation accepts only a governance receipt verified by a caller-owned authorization boundary. Its library version, rule revision, and complete sorted item-key/item-version identity must exactly match the classification report. Blank, duplicate, unknown, stale, or abstention-as-truth labels fail closed. The output contains no item keys, reviewer identity, or bibliographic text; per-disposition `true_positive`, `predicted`, and `expected` counts retain exact precision/recall numerators and denominators without floating-point rounding. Production authorization remains Keyverse/governance-owned; this crate validates the binding and does not mint or self-verify authority. +Golden-set evaluation accepts only a governance receipt verified by a caller-owned authorization boundary. Its library version, rule revision, immutable snapshot digest, and complete sorted item-key/item-version identity must bind the classification report. Blank, duplicate, unknown, stale, or abstention-as-truth labels fail closed. The output retains the verified library revision, rule revision, and opaque snapshot digest so stored metrics remain attributable, but contains no item keys, reviewer identity, or bibliographic text; per-disposition `true_positive`, `predicted`, and `expected` counts retain exact precision/recall numerators and denominators without floating-point rounding. Production authorization remains Keyverse/governance-owned; this crate validates the item-revision binding and does not mint or self-verify authority. The report is local JSON and contains proposals rather than governance decisions. CLI output is restricted to a new direct child of canonical `/tmp` or the operating system temporary directory; relative paths, nested paths, existing paths, and symlinks are rejected, and create-new file semantics prevent overwrite/path-swap writes. Zotero 9 writes are unsupported; no mutation path exists in this slice. A future Zotero 10+ writer requires a separate reviewed change with a Local API key, stable server identity, fresh item/library version preconditions, item-by-item before/after receipts, and rollback evidence. diff --git a/docs/adr/0006-zotero-research-intake.md b/docs/adr/0006-zotero-research-intake.md index c582155a..a1161f0d 100644 --- a/docs/adr/0006-zotero-research-intake.md +++ b/docs/adr/0006-zotero-research-intake.md @@ -19,7 +19,7 @@ The adapter links child records, emits exactly one deterministic proposed dispos Matched metadata values are copied into the local-only evidence receipt for replay. This is necessary for abstract-only matches because a later Zotero revision cannot reconstruct the exact text used for an earlier proposal from item key/version alone. DOI/title matches remain reversible duplicate candidates, including legacy `dx.doi.org` resolver forms. -Classifier quality is measured only against local steward-reviewed labels whose governance receipt is verified outside this crate and bound to the exact library version, rule revision, and complete item-key/item-version snapshot. `NeedsStewardReview` is an abstention prediction and cannot be approved truth. Evaluation returns aggregate integer evidence for precision, recall, exact matches, and abstentions; Zotero keys, reviewer identity, and bibliographic text are deliberately omitted from evaluation output. Missing, stale, unverified, unknown, duplicate, or invalid review identities fail closed. +Classifier quality is measured only against local steward-reviewed labels whose governance receipt is verified outside this crate and bound to the exact library version, rule revision, immutable snapshot digest, and complete item-key/item-version snapshot. `NeedsStewardReview` is an abstention prediction and cannot be approved truth. Evaluation returns the verified revisions and opaque digest with aggregate integer evidence for precision, recall, exact matches, and abstentions; Zotero keys, reviewer identity, and bibliographic text are deliberately omitted from evaluation output. Missing, stale, unverified, unknown, duplicate, or invalid review identities fail closed. The reader fails closed above 50,000 items or 256 MiB of cumulative response bodies, while retaining the 8 MiB per-page bound, finite request timeouts, redirect denial, total-count checks, snapshot-version checks, and duplicate-key detection. Pagination, consistency, resource-budget, and provider-contract behavior are separated from the narrow `ureq` transport so deterministic tests exercise the production reader core rather than excluding the entire reader from coverage. diff --git a/docs/product-technical-gap-baseline.md b/docs/product-technical-gap-baseline.md index 885c31b8..e71f075a 100644 --- a/docs/product-technical-gap-baseline.md +++ b/docs/product-technical-gap-baseline.md @@ -44,7 +44,7 @@ Protected central source is `.github/main@c31d2e5471fc5daf9d72ff67cde6a8874b736d Local evidence on 2026-09-04 showed Zotero 9.0.6, Local API v3/schema 42, library version 12341, 8,326 total items, and 3,719 top-level items. The corrected read-only run observed all 8,326 records at that single version and classified all 3,715 top-level bibliographic records; four top-level note/attachment/annotation records were correctly excluded. It proposed 56 adjacent-evidence records, 1 semantic-consumption bridge, and 3,658 steward-review abstentions, linked children for 3,287 records, and surfaced 49 reversible duplicate groups (18 DOI, 31 title). No live record matched multiple specific disposition families; the tested conflict path still abstains fail-closed. Token-boundary matching prevents strings such as `knowledge` from becoming false OWL evidence. These are local aggregate observations, not reviewed truth or applied Zotero changes. The report stays outside the repository. -The golden-set evaluation contract now records aggregate precision/recall numerators and denominators, requires an externally verified governance receipt bound to the complete item-key/item-version snapshot, rejects abstention as expected truth, and omits item and reviewer identities from its output. Synthetic fixtures verify the contract; no real precision/recall claim exists until a steward supplies reviewed local labels and the production authorization adapter verifies the receipt. Multilingual rule expansion remains a later evidence-driven change and must not reduce abstention safety. Zotero write-back remains blocked by the installed v9 capability; a Zotero 10+ change must satisfy ADR 0006 preconditions. A dedicated utility repository remains unnecessary until an independently released cross-product contract exists. +The golden-set evaluation contract now records aggregate precision/recall numerators and denominators, requires an externally verified governance receipt bound to the complete item-key/item-version snapshot, rejects abstention as expected truth, and retains verified revisions plus an opaque snapshot digest so detached metrics remain attributable. Item and reviewer identities stay out of its output. Synthetic fixtures verify the contract; no real precision/recall claim exists until a steward supplies reviewed local labels and the production authorization adapter verifies the receipt. Multilingual rule expansion remains a later evidence-driven change and must not reduce abstention safety. Zotero write-back remains blocked by the installed v9 capability; a Zotero 10+ change must satisfy ADR 0006 preconditions. A dedicated utility repository remains unnecessary until an independently released cross-product contract exists. 1. **Concrete Source Observation adapter** — maintained Rust PostgreSQL driver behind `conceptweave-source-port`; adapter-local credential resolution; explicit read-only mode; statement timeout, cancellation, row/byte/concurrency budgets; complete immutable snapshot or fail closed; deterministic replay against a frozen anonymized GRC-shaped fixture. 2. **Ontology discovery** — deterministic term/concept/taxonomy/non-taxonomic-relation candidate generation with exact source receipts and abstention for unsupported semantics. From d899d8a06da439a3129e7f7d8a4e7f37608ec139 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Fri, 4 Sep 2026 22:51:13 +0900 Subject: [PATCH 04/28] test(research): require content-bound Zotero snapshot identity --- .../tests/snapshot_content_binding.rs | 50 +++++++++++++++++++ 1 file changed, 50 insertions(+) create mode 100644 crates/conceptweave-zotero/tests/snapshot_content_binding.rs diff --git a/crates/conceptweave-zotero/tests/snapshot_content_binding.rs b/crates/conceptweave-zotero/tests/snapshot_content_binding.rs new file mode 100644 index 00000000..7c4c72a9 --- /dev/null +++ b/crates/conceptweave-zotero/tests/snapshot_content_binding.rs @@ -0,0 +1,50 @@ +use conceptweave_zotero::{ + Disposition, EvaluationError, GoldenLabel, GoldenSetApproval, ItemData, ReviewedGoldenSet, + SnapshotItemRevision, ZoteroItem, classify_snapshot, evaluate_reviewed_golden_set, +}; + +fn item(title: &str) -> ZoteroItem { + ZoteroItem { + key: "A".into(), + version: 1, + data: ItemData { + item_type: "book".into(), + title: title.into(), + abstract_note: String::new(), + doi: String::new(), + parent_item: String::new(), + collections: vec![], + tags: vec![], + }, + } +} + +#[test] +fn golden_approval_rejects_same_revision_coordinates_with_changed_snapshot_content() { + let changed_report = classify_snapshot( + "9.0.6".into(), + None, + 42, + vec![item("ontology evaluation")], + ); + let golden = ReviewedGoldenSet { + approval: GoldenSetApproval { + receipt_id: "review-original-snapshot".into(), + reviewer_subject: "synthetic-steward".into(), + library_version: 42, + rule_revision: "ontology-research-v2".into(), + snapshot_digest: "sha256:approved-original-content".into(), + snapshot_items: vec![SnapshotItemRevision { + item_key: "A".into(), + item_version: 1, + }], + }, + labels: vec![GoldenLabel::new("A", Disposition::Generation)], + }; + + assert_eq!( + evaluate_reviewed_golden_set(&changed_report, &golden, |_| true), + Err(EvaluationError::SnapshotMismatch), + "item key/version coordinates alone cannot bind a Zotero 9 local snapshot whose content changed without a synced-version change" + ); +} From 4949c7b907eb5299bb795b43b793f434ad0cbd88 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Fri, 4 Sep 2026 22:54:50 +0900 Subject: [PATCH 05/28] fix: bind golden approval to report content Signed-off-by: Seongho Bae --- Cargo.lock | 84 +++++++++++++++++++ crates/conceptweave-zotero/Cargo.toml | 1 + crates/conceptweave-zotero/src/lib.rs | 24 ++++++ .../tests/golden_set_evaluation.rs | 7 +- .../tests/snapshot_content_binding.rs | 8 +- docs/PRD.md | 2 +- docs/TRD.md | 2 +- docs/adr/0006-zotero-research-intake.md | 2 +- 8 files changed, 118 insertions(+), 12 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index dd4cfe80..ad534722 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -8,12 +8,27 @@ version = "0.23.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ac07cdecf99051d9a5238b80f35af32cdeba5b336e55d957b318b50137e18da5" +[[package]] +name = "block-buffer" +version = "0.10.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71" +dependencies = [ + "generic-array", +] + [[package]] name = "bytes" version = "1.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fc652a48c352aef3ea3aed32080501cf3ef6ed5da78602a020c991775b0aff04" +[[package]] +name = "cfg-if" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" + [[package]] name = "conceptweave-domain" version = "0.1.0" @@ -24,9 +39,49 @@ version = "0.1.0" dependencies = [ "serde", "serde_json", + "sha2", "ureq", ] +[[package]] +name = "cpufeatures" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "59ed5838eebb26a2bb2e58f6d5b5316989ae9d08bab10e0e6d103e656d1b0280" +dependencies = [ + "libc", +] + +[[package]] +name = "crypto-common" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78c8292055d1c1df0cce5d180393dc8cce0abec0a7102adb6c7b1eef6016d60a" +dependencies = [ + "generic-array", + "typenum", +] + +[[package]] +name = "digest" +version = "0.10.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" +dependencies = [ + "block-buffer", + "crypto-common", +] + +[[package]] +name = "generic-array" +version = "0.14.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a" +dependencies = [ + "typenum", + "version_check", +] + [[package]] name = "http" version = "1.5.0" @@ -49,6 +104,12 @@ version = "1.0.18" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" +[[package]] +name = "libc" +version = "0.2.189" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3eaf3ede3fee6db1a4c2ee091bf8a8b4dccdc6d17f656fb07896ee72867612f2" + [[package]] name = "log" version = "0.4.34" @@ -128,6 +189,17 @@ dependencies = [ "zmij", ] +[[package]] +name = "sha2" +version = "0.10.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283" +dependencies = [ + "cfg-if", + "cpufeatures", + "digest", +] + [[package]] name = "syn" version = "3.0.4" @@ -139,6 +211,12 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "typenum" +version = "1.20.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6f5e870be6c3b371b77fe0ee0bafb859fa4964b4404c27de1d380043c4dda20" + [[package]] name = "unicode-ident" version = "1.0.24" @@ -176,6 +254,12 @@ version = "0.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b8c0a043c9540bae7c578c88f91dda8bd82e59ae27c21baca69c8b191aaf5a6e" +[[package]] +name = "version_check" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" + [[package]] name = "zmij" version = "1.0.23" diff --git a/crates/conceptweave-zotero/Cargo.toml b/crates/conceptweave-zotero/Cargo.toml index 7f78f15f..3f492bbe 100644 --- a/crates/conceptweave-zotero/Cargo.toml +++ b/crates/conceptweave-zotero/Cargo.toml @@ -10,6 +10,7 @@ description = "Read-only Zotero research classification for ConceptWeave" [dependencies] serde = { version = "1", features = ["derive"] } serde_json = "1" +sha2 = "0.10" ureq = { version = "3", default-features = false } [lib] diff --git a/crates/conceptweave-zotero/src/lib.rs b/crates/conceptweave-zotero/src/lib.rs index 53ea83e1..a5cea28a 100644 --- a/crates/conceptweave-zotero/src/lib.rs +++ b/crates/conceptweave-zotero/src/lib.rs @@ -4,6 +4,7 @@ //! Deterministic, read-only classification of a Zotero library snapshot. use serde::{Deserialize, Serialize}; +use sha2::{Digest, Sha256}; use std::collections::{BTreeMap, BTreeSet}; use std::fmt; use std::time::Duration; @@ -226,6 +227,28 @@ pub struct GoldenSetApproval { pub snapshot_items: Vec, } +/// Computes the canonical content identity verified by a golden-set approval. +pub fn classification_snapshot_digest(report: &ClassificationReport) -> String { + let classified_items = report + .classified_items + .iter() + .map(|item| (item.item_key.as_str(), item)) + .collect::>(); + let canonical_snapshot = serde_json::to_vec(&( + &report.zotero_version, + report.api_version, + report.schema_version, + &report.server_id, + report.library_version, + report.rule_revision, + report.observed_item_count, + classified_items, + &report.duplicate_candidates, + )) + .expect("classification reports contain only JSON-compatible values"); + format!("sha256:{:x}", Sha256::digest(canonical_snapshot)) +} + /// Integer evidence from which precision and recall can be calculated exactly. #[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize)] pub struct DispositionEvaluation { @@ -333,6 +356,7 @@ where } if golden.approval.library_version != report.library_version || golden.approval.rule_revision != report.rule_revision + || golden.approval.snapshot_digest != classification_snapshot_digest(report) || approved_snapshot != report_snapshot { return Err(EvaluationError::SnapshotMismatch); diff --git a/crates/conceptweave-zotero/tests/golden_set_evaluation.rs b/crates/conceptweave-zotero/tests/golden_set_evaluation.rs index e8f94e43..49dbe064 100644 --- a/crates/conceptweave-zotero/tests/golden_set_evaluation.rs +++ b/crates/conceptweave-zotero/tests/golden_set_evaluation.rs @@ -1,6 +1,7 @@ use conceptweave_zotero::{ Disposition, EvaluationError, GoldenLabel, GoldenSetApproval, ItemData, ReviewedGoldenSet, - SnapshotItemRevision, ZoteroItem, classify_snapshot, evaluate_reviewed_golden_set, + SnapshotItemRevision, ZoteroItem, classification_snapshot_digest, classify_snapshot, + evaluate_reviewed_golden_set, }; fn item(key: &str, title: &str) -> ZoteroItem { @@ -39,7 +40,7 @@ fn golden(labels: Vec) -> ReviewedGoldenSet { reviewer_subject: "synthetic-steward".into(), library_version: 42, rule_revision: "ontology-research-v2".into(), - snapshot_digest: "sha256:synthetic-snapshot".into(), + snapshot_digest: classification_snapshot_digest(&report()), snapshot_items: ["A", "B", "C"] .into_iter() .map(|item_key| SnapshotItemRevision { @@ -73,7 +74,7 @@ fn reviewed_golden_set_reports_count_based_precision_and_recall_evidence() { assert_eq!(evaluation.review_id, "synthetic-review-1"); assert_eq!(evaluation.library_version, 42); assert_eq!(evaluation.rule_revision, "ontology-research-v2"); - assert_eq!(evaluation.snapshot_digest, "sha256:synthetic-snapshot"); + assert!(evaluation.snapshot_digest.starts_with("sha256:")); assert_eq!(evaluation.reviewed_count, 3); assert_eq!(evaluation.correct_count, 1); assert_eq!(evaluation.abstention_count, 1); diff --git a/crates/conceptweave-zotero/tests/snapshot_content_binding.rs b/crates/conceptweave-zotero/tests/snapshot_content_binding.rs index 7c4c72a9..efe76a1f 100644 --- a/crates/conceptweave-zotero/tests/snapshot_content_binding.rs +++ b/crates/conceptweave-zotero/tests/snapshot_content_binding.rs @@ -21,12 +21,8 @@ fn item(title: &str) -> ZoteroItem { #[test] fn golden_approval_rejects_same_revision_coordinates_with_changed_snapshot_content() { - let changed_report = classify_snapshot( - "9.0.6".into(), - None, - 42, - vec![item("ontology evaluation")], - ); + let changed_report = + classify_snapshot("9.0.6".into(), None, 42, vec![item("ontology evaluation")]); let golden = ReviewedGoldenSet { approval: GoldenSetApproval { receipt_id: "review-original-snapshot".into(), diff --git a/docs/PRD.md b/docs/PRD.md index e0557754..7864e4ed 100644 --- a/docs/PRD.md +++ b/docs/PRD.md @@ -58,7 +58,7 @@ All LLM-backed induction uses `contextual-orchestrator`. Model output is untrust Read one immutable Zotero Local API library-version snapshot and propose exactly one research disposition for every top-level bibliographic item. Each proposal retains the item key/version, exact matched metadata values, rule revision, linked child records, and any model receipt. Weak evidence and evidence that matches multiple specific disposition families must abstain into steward review. Duplicate DOI/title identities are review candidates only: intake never merges, deletes, or silently mutates Zotero records. -Evaluate classifier quality only against a steward-reviewed local golden set whose governance receipt is externally verified and bound to the complete Zotero item-key/item-version snapshot. Abstention is a prediction outcome, never an approved truth label. Evaluation emits the verified library revision, rule revision, opaque snapshot digest, and aggregate counts for exact matches, abstentions, and per-disposition true-positive/predicted/expected totals; it must not copy Zotero keys, reviewer identity, or bibliographic text into the result. +Evaluate classifier quality only against a steward-reviewed local golden set whose governance receipt is externally verified and bound to the canonical SHA-256 digest of the complete Zotero classification report plus its item-key/item-version coordinates. Abstention is a prediction outcome, never an approved truth label. Evaluation emits the verified library revision, rule revision, opaque snapshot digest, and aggregate counts for exact matches, abstentions, and per-disposition true-positive/predicted/expected totals; it must not copy Zotero keys, reviewer identity, or bibliographic text into the result. ## 6. First vertical slice diff --git a/docs/TRD.md b/docs/TRD.md index 103eb92a..ef8aa3dd 100644 --- a/docs/TRD.md +++ b/docs/TRD.md @@ -63,6 +63,6 @@ Evaluation must separate extraction recall, semantic correctness, structural cor Every top-level bibliographic record receives exactly one proposed disposition. `NeedsStewardReview` also records a deterministic abstention reason so missing classification metadata, vocabulary unsupported by the current deterministic rules, and present-but-unmatched metadata are distinguishable. DOI duplicate identity normalization treats bare DOI values, `doi:`, `doi.org`, and legacy `dx.doi.org` resolver forms as the same identity when their normalized DOI is equal. -Golden-set evaluation accepts only a governance receipt verified by a caller-owned authorization boundary. Its library version, rule revision, immutable snapshot digest, and complete sorted item-key/item-version identity must bind the classification report. Blank, duplicate, unknown, stale, or abstention-as-truth labels fail closed. The output retains the verified library revision, rule revision, and opaque snapshot digest so stored metrics remain attributable, but contains no item keys, reviewer identity, or bibliographic text; per-disposition `true_positive`, `predicted`, and `expected` counts retain exact precision/recall numerators and denominators without floating-point rounding. Production authorization remains Keyverse/governance-owned; this crate validates the item-revision binding and does not mint or self-verify authority. +Golden-set evaluation accepts only a governance receipt verified by a caller-owned authorization boundary. Its library version, rule revision, canonical SHA-256 content digest, and complete sorted item-key/item-version identity must bind the classification report. The digest covers report contract versions, source identity, observed count, classified item content and evidence in key order, and duplicate candidates. Blank, duplicate, unknown, stale, content-mismatched, or abstention-as-truth labels fail closed. The output retains the verified library revision, rule revision, and opaque snapshot digest so stored metrics remain attributable, but contains no item keys, reviewer identity, or bibliographic text; per-disposition `true_positive`, `predicted`, and `expected` counts retain exact precision/recall numerators and denominators without floating-point rounding. Production authorization remains Keyverse/governance-owned; this crate validates the content binding and does not mint or self-verify authority. The report is local JSON and contains proposals rather than governance decisions. CLI output is restricted to a new direct child of canonical `/tmp` or the operating system temporary directory; relative paths, nested paths, existing paths, and symlinks are rejected, and create-new file semantics prevent overwrite/path-swap writes. Zotero 9 writes are unsupported; no mutation path exists in this slice. A future Zotero 10+ writer requires a separate reviewed change with a Local API key, stable server identity, fresh item/library version preconditions, item-by-item before/after receipts, and rollback evidence. diff --git a/docs/adr/0006-zotero-research-intake.md b/docs/adr/0006-zotero-research-intake.md index a1161f0d..22d1db06 100644 --- a/docs/adr/0006-zotero-research-intake.md +++ b/docs/adr/0006-zotero-research-intake.md @@ -19,7 +19,7 @@ The adapter links child records, emits exactly one deterministic proposed dispos Matched metadata values are copied into the local-only evidence receipt for replay. This is necessary for abstract-only matches because a later Zotero revision cannot reconstruct the exact text used for an earlier proposal from item key/version alone. DOI/title matches remain reversible duplicate candidates, including legacy `dx.doi.org` resolver forms. -Classifier quality is measured only against local steward-reviewed labels whose governance receipt is verified outside this crate and bound to the exact library version, rule revision, immutable snapshot digest, and complete item-key/item-version snapshot. `NeedsStewardReview` is an abstention prediction and cannot be approved truth. Evaluation returns the verified revisions and opaque digest with aggregate integer evidence for precision, recall, exact matches, and abstentions; Zotero keys, reviewer identity, and bibliographic text are deliberately omitted from evaluation output. Missing, stale, unverified, unknown, duplicate, or invalid review identities fail closed. +Classifier quality is measured only against local steward-reviewed labels whose governance receipt is verified outside this crate and bound to the exact library version, rule revision, canonical SHA-256 report-content digest, and complete item-key/item-version snapshot. `NeedsStewardReview` is an abstention prediction and cannot be approved truth. Evaluation returns the verified revisions and opaque digest with aggregate integer evidence for precision, recall, exact matches, and abstentions; Zotero keys, reviewer identity, and bibliographic text are deliberately omitted from evaluation output. Missing, stale, content-mismatched, unverified, unknown, duplicate, or invalid review identities fail closed. The reader fails closed above 50,000 items or 256 MiB of cumulative response bodies, while retaining the 8 MiB per-page bound, finite request timeouts, redirect denial, total-count checks, snapshot-version checks, and duplicate-key detection. Pagination, consistency, resource-budget, and provider-contract behavior are separated from the narrow `ureq` transport so deterministic tests exercise the production reader core rather than excluding the entire reader from coverage. From 8c939d0c35ae43445940049de0234d4e5f890d66 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Fri, 4 Sep 2026 23:01:31 +0900 Subject: [PATCH 06/28] test(research): close golden-set integrity gaps --- .../tests/golden_set_integrity_contract.rs | 148 ++++++++++++++++++ 1 file changed, 148 insertions(+) create mode 100644 crates/conceptweave-zotero/tests/golden_set_integrity_contract.rs diff --git a/crates/conceptweave-zotero/tests/golden_set_integrity_contract.rs b/crates/conceptweave-zotero/tests/golden_set_integrity_contract.rs new file mode 100644 index 00000000..8bc6d504 --- /dev/null +++ b/crates/conceptweave-zotero/tests/golden_set_integrity_contract.rs @@ -0,0 +1,148 @@ +use conceptweave_zotero::{ + Disposition, EvaluationError, GoldenLabel, GoldenSetApproval, ItemData, ReviewedGoldenSet, + SnapshotItemRevision, ZoteroItem, classification_snapshot_digest, classify_snapshot, + evaluate_reviewed_golden_set, +}; + +fn bibliographic(key: &str, version: u64, title: &str) -> ZoteroItem { + ZoteroItem { + key: key.into(), + version, + data: ItemData { + item_type: "book".into(), + title: title.into(), + abstract_note: String::new(), + doi: String::new(), + parent_item: String::new(), + collections: vec![], + tags: vec![], + }, + } +} + +fn child_note(key: &str, version: u64, parent_item: &str) -> ZoteroItem { + ZoteroItem { + key: key.into(), + version, + data: ItemData { + item_type: "note".into(), + title: String::new(), + abstract_note: String::new(), + doi: String::new(), + parent_item: parent_item.into(), + collections: vec![], + tags: vec![], + }, + } +} + +fn approval( + report: &conceptweave_zotero::ClassificationReport, + snapshot_items: Vec, +) -> GoldenSetApproval { + GoldenSetApproval { + receipt_id: "approved-review".into(), + reviewer_subject: "synthetic-steward".into(), + library_version: report.library_version, + rule_revision: report.rule_revision.into(), + snapshot_digest: classification_snapshot_digest(report), + snapshot_items, + } +} + +#[test] +fn reviewed_snapshot_binding_includes_linked_child_revisions() { + let report = classify_snapshot( + "9.0.6".into(), + None, + 42, + vec![ + bibliographic("PARENT", 7, "ontology learning"), + child_note("NOTE1", 3, "PARENT"), + ], + ); + let golden = ReviewedGoldenSet { + approval: approval( + &report, + vec![ + SnapshotItemRevision { + item_key: "PARENT".into(), + item_version: 7, + }, + SnapshotItemRevision { + item_key: "NOTE1".into(), + item_version: 3, + }, + ], + ), + labels: vec![GoldenLabel::new("PARENT", Disposition::Generation)], + }; + + assert!( + evaluate_reviewed_golden_set(&report, &golden, |_| true).is_ok(), + "an approval for the complete observed Zotero snapshot must include linked child revisions even though only bibliographic parents receive dispositions" + ); +} + +#[test] +fn verified_snapshot_receipt_cannot_authorize_mutated_steward_labels() { + let report = classify_snapshot( + "9.0.6".into(), + None, + 42, + vec![bibliographic("A", 1, "ontology learning")], + ); + let golden = ReviewedGoldenSet { + approval: approval( + &report, + vec![SnapshotItemRevision { + item_key: "A".into(), + item_version: 1, + }], + ), + labels: vec![GoldenLabel::new("A", Disposition::AlignmentVersioning)], + }; + + assert_eq!( + evaluate_reviewed_golden_set(&report, &golden, |receipt| { + receipt.receipt_id == "approved-review" + }), + Err(EvaluationError::UnverifiedApproval), + "approval verification must bind the reviewed labels as well as the snapshot receipt" + ); +} + +#[test] +fn duplicate_zotero_keys_fail_closed_even_when_item_revisions_differ() { + let report = classify_snapshot( + "9.0.6".into(), + None, + 42, + vec![ + bibliographic("A", 1, "ontology learning"), + bibliographic("A", 2, "ontology learning"), + ], + ); + let golden = ReviewedGoldenSet { + approval: approval( + &report, + vec![ + SnapshotItemRevision { + item_key: "A".into(), + item_version: 1, + }, + SnapshotItemRevision { + item_key: "A".into(), + item_version: 2, + }, + ], + ), + labels: vec![GoldenLabel::new("A", Disposition::Generation)], + }; + + assert_eq!( + evaluate_reviewed_golden_set(&report, &golden, |_| true), + Err(EvaluationError::InvalidReview), + "Zotero item keys are identities; duplicate keys cannot become distinct records merely because their revision counters differ" + ); +} From d5ccd73ae3b23154305d2ec51d24a00c71f23f6d Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Fri, 4 Sep 2026 23:05:41 +0900 Subject: [PATCH 07/28] fix: bind golden review to complete snapshot Signed-off-by: Seongho Bae --- crates/conceptweave-zotero/src/lib.rs | 54 ++++++++++--------- .../tests/golden_set_evaluation.rs | 5 +- .../tests/golden_set_integrity_contract.rs | 5 +- docs/TRD.md | 2 +- docs/adr/0006-zotero-research-intake.md | 2 +- 5 files changed, 36 insertions(+), 32 deletions(-) diff --git a/crates/conceptweave-zotero/src/lib.rs b/crates/conceptweave-zotero/src/lib.rs index a5cea28a..30b05237 100644 --- a/crates/conceptweave-zotero/src/lib.rs +++ b/crates/conceptweave-zotero/src/lib.rs @@ -167,6 +167,10 @@ pub struct ClassificationReport { pub rule_revision: &'static str, /// Number of items read, including child notes and attachments. pub observed_item_count: usize, + /// Complete item-revision identity of every observed record. + pub snapshot_items: Vec, + /// Canonical SHA-256 digest of every observed raw Zotero item. + pub snapshot_digest: String, /// One proposal for every top-level bibliographic item. pub classified_items: Vec, /// Reversible DOI/title duplicate candidates. @@ -229,24 +233,7 @@ pub struct GoldenSetApproval { /// Computes the canonical content identity verified by a golden-set approval. pub fn classification_snapshot_digest(report: &ClassificationReport) -> String { - let classified_items = report - .classified_items - .iter() - .map(|item| (item.item_key.as_str(), item)) - .collect::>(); - let canonical_snapshot = serde_json::to_vec(&( - &report.zotero_version, - report.api_version, - report.schema_version, - &report.server_id, - report.library_version, - report.rule_revision, - report.observed_item_count, - classified_items, - &report.duplicate_candidates, - )) - .expect("classification reports contain only JSON-compatible values"); - format!("sha256:{:x}", Sha256::digest(canonical_snapshot)) + report.snapshot_digest.clone() } /// Integer evidence from which precision and recall can be calculated exactly. @@ -322,7 +309,7 @@ pub fn evaluate_reviewed_golden_set( verify_approval: F, ) -> Result where - F: FnOnce(&GoldenSetApproval) -> bool, + F: FnOnce(&ReviewedGoldenSet) -> bool, { if golden.approval.receipt_id.trim().is_empty() || golden.approval.reviewer_subject.trim().is_empty() @@ -332,16 +319,18 @@ where { return Err(EvaluationError::InvalidReview); } - if !verify_approval(&golden.approval) { + if !verify_approval(golden) { return Err(EvaluationError::UnverifiedApproval); } let report_snapshot = report - .classified_items + .snapshot_items .iter() - .map(|item| SnapshotItemRevision { - item_key: item.item_key.clone(), - item_version: item.item_version, - }) + .cloned() + .collect::>(); + let report_keys = report + .snapshot_items + .iter() + .map(|item| item.item_key.as_str()) .collect::>(); let approved_snapshot = golden .approval @@ -349,7 +338,8 @@ where .iter() .cloned() .collect::>(); - if report_snapshot.len() != report.classified_items.len() + if report_snapshot.len() != report.snapshot_items.len() + || report_keys.len() != report.snapshot_items.len() || approved_snapshot.len() != golden.approval.snapshot_items.len() { return Err(EvaluationError::InvalidReview); @@ -663,6 +653,16 @@ pub fn classify_snapshot( mut items: Vec, ) -> ClassificationReport { items.sort_by(|left, right| left.key.cmp(&right.key)); + let snapshot_items = items + .iter() + .map(|item| SnapshotItemRevision { + item_key: item.key.clone(), + item_version: item.version, + }) + .collect(); + let snapshot_bytes = serde_json::to_vec(&items) + .expect("Zotero snapshot items contain only JSON-compatible values"); + let snapshot_digest = format!("sha256:{:x}", Sha256::digest(snapshot_bytes)); let children = child_index(&items); let bibliographic: Vec<&ZoteroItem> = items.iter().filter(|item| is_bibliographic(item)).collect(); @@ -680,6 +680,8 @@ pub fn classify_snapshot( library_version, rule_revision: RULE_REVISION, observed_item_count: items.len(), + snapshot_items, + snapshot_digest, classified_items, duplicate_candidates, } diff --git a/crates/conceptweave-zotero/tests/golden_set_evaluation.rs b/crates/conceptweave-zotero/tests/golden_set_evaluation.rs index 49dbe064..5acbc4a9 100644 --- a/crates/conceptweave-zotero/tests/golden_set_evaluation.rs +++ b/crates/conceptweave-zotero/tests/golden_set_evaluation.rs @@ -53,8 +53,9 @@ fn golden(labels: Vec) -> ReviewedGoldenSet { } } -fn verify_synthetic_approval(approval: &GoldenSetApproval) -> bool { - approval.receipt_id == "synthetic-review-1" && approval.reviewer_subject == "synthetic-steward" +fn verify_synthetic_approval(golden: &ReviewedGoldenSet) -> bool { + golden.approval.receipt_id == "synthetic-review-1" + && golden.approval.reviewer_subject == "synthetic-steward" } #[test] diff --git a/crates/conceptweave-zotero/tests/golden_set_integrity_contract.rs b/crates/conceptweave-zotero/tests/golden_set_integrity_contract.rs index 8bc6d504..dc0615a0 100644 --- a/crates/conceptweave-zotero/tests/golden_set_integrity_contract.rs +++ b/crates/conceptweave-zotero/tests/golden_set_integrity_contract.rs @@ -104,8 +104,9 @@ fn verified_snapshot_receipt_cannot_authorize_mutated_steward_labels() { }; assert_eq!( - evaluate_reviewed_golden_set(&report, &golden, |receipt| { - receipt.receipt_id == "approved-review" + evaluate_reviewed_golden_set(&report, &golden, |reviewed_set| { + reviewed_set.approval.receipt_id == "approved-review" + && reviewed_set.labels == vec![GoldenLabel::new("A", Disposition::Generation)] }), Err(EvaluationError::UnverifiedApproval), "approval verification must bind the reviewed labels as well as the snapshot receipt" diff --git a/docs/TRD.md b/docs/TRD.md index ef8aa3dd..2e641cc8 100644 --- a/docs/TRD.md +++ b/docs/TRD.md @@ -63,6 +63,6 @@ Evaluation must separate extraction recall, semantic correctness, structural cor Every top-level bibliographic record receives exactly one proposed disposition. `NeedsStewardReview` also records a deterministic abstention reason so missing classification metadata, vocabulary unsupported by the current deterministic rules, and present-but-unmatched metadata are distinguishable. DOI duplicate identity normalization treats bare DOI values, `doi:`, `doi.org`, and legacy `dx.doi.org` resolver forms as the same identity when their normalized DOI is equal. -Golden-set evaluation accepts only a governance receipt verified by a caller-owned authorization boundary. Its library version, rule revision, canonical SHA-256 content digest, and complete sorted item-key/item-version identity must bind the classification report. The digest covers report contract versions, source identity, observed count, classified item content and evidence in key order, and duplicate candidates. Blank, duplicate, unknown, stale, content-mismatched, or abstention-as-truth labels fail closed. The output retains the verified library revision, rule revision, and opaque snapshot digest so stored metrics remain attributable, but contains no item keys, reviewer identity, or bibliographic text; per-disposition `true_positive`, `predicted`, and `expected` counts retain exact precision/recall numerators and denominators without floating-point rounding. Production authorization remains Keyverse/governance-owned; this crate validates the content binding and does not mint or self-verify authority. +Golden-set evaluation accepts only a governance receipt verified with the complete reviewed set by a caller-owned authorization boundary. Its library version, rule revision, canonical SHA-256 content digest, and every observed parent/child item-key/item-version identity must bind the classification report. The digest covers every raw Zotero item in canonical key order. Blank, duplicate, unknown, stale, content-mismatched, label-mismatched, or abstention-as-truth inputs fail closed. The output retains the verified library version, rule revision, and opaque snapshot digest, but contains no item keys, reviewer identity, or bibliographic text. Production authorization remains Keyverse/governance-owned; this crate passes the complete reviewed labels to that boundary instead of minting authority. The report is local JSON and contains proposals rather than governance decisions. CLI output is restricted to a new direct child of canonical `/tmp` or the operating system temporary directory; relative paths, nested paths, existing paths, and symlinks are rejected, and create-new file semantics prevent overwrite/path-swap writes. Zotero 9 writes are unsupported; no mutation path exists in this slice. A future Zotero 10+ writer requires a separate reviewed change with a Local API key, stable server identity, fresh item/library version preconditions, item-by-item before/after receipts, and rollback evidence. diff --git a/docs/adr/0006-zotero-research-intake.md b/docs/adr/0006-zotero-research-intake.md index 22d1db06..dca5b407 100644 --- a/docs/adr/0006-zotero-research-intake.md +++ b/docs/adr/0006-zotero-research-intake.md @@ -19,7 +19,7 @@ The adapter links child records, emits exactly one deterministic proposed dispos Matched metadata values are copied into the local-only evidence receipt for replay. This is necessary for abstract-only matches because a later Zotero revision cannot reconstruct the exact text used for an earlier proposal from item key/version alone. DOI/title matches remain reversible duplicate candidates, including legacy `dx.doi.org` resolver forms. -Classifier quality is measured only against local steward-reviewed labels whose governance receipt is verified outside this crate and bound to the exact library version, rule revision, canonical SHA-256 report-content digest, and complete item-key/item-version snapshot. `NeedsStewardReview` is an abstention prediction and cannot be approved truth. Evaluation returns the verified revisions and opaque digest with aggregate integer evidence for precision, recall, exact matches, and abstentions; Zotero keys, reviewer identity, and bibliographic text are deliberately omitted from evaluation output. Missing, stale, content-mismatched, unverified, unknown, duplicate, or invalid review identities fail closed. +Classifier quality is measured only against local steward-reviewed labels whose complete reviewed set is verified outside this crate and bound to the exact library version, rule revision, canonical SHA-256 raw-snapshot digest, and every observed parent/child item-key/item-version coordinate. `NeedsStewardReview` is an abstention prediction and cannot be approved truth. Evaluation returns the verified revisions and opaque digest with aggregate integer evidence; Zotero keys, reviewer identity, and bibliographic text are omitted. Missing, stale, content- or label-mismatched, unverified, unknown, duplicate, or invalid review identities fail closed. The reader fails closed above 50,000 items or 256 MiB of cumulative response bodies, while retaining the 8 MiB per-page bound, finite request timeouts, redirect denial, total-count checks, snapshot-version checks, and duplicate-key detection. Pagination, consistency, resource-budget, and provider-contract behavior are separated from the narrow `ureq` transport so deterministic tests exercise the production reader core rather than excluding the entire reader from coverage. From 2988d2bef3e02d91d2458f2faf0fc62fda1af5f4 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sat, 5 Sep 2026 15:34:23 +0900 Subject: [PATCH 08/28] test(zotero): bind snapshots to unmodeled provider metadata --- .../tests/raw_provider_snapshot_binding.rs | 52 +++++++++++++++++++ 1 file changed, 52 insertions(+) create mode 100644 crates/conceptweave-zotero/tests/raw_provider_snapshot_binding.rs diff --git a/crates/conceptweave-zotero/tests/raw_provider_snapshot_binding.rs b/crates/conceptweave-zotero/tests/raw_provider_snapshot_binding.rs new file mode 100644 index 00000000..312a711b --- /dev/null +++ b/crates/conceptweave-zotero/tests/raw_provider_snapshot_binding.rs @@ -0,0 +1,52 @@ +use conceptweave_zotero::{ZoteroItem, classify_snapshot}; +use serde_json::{Value, json}; + +fn snapshot_digest(raw_item: Value) -> String { + let item: ZoteroItem = serde_json::from_value(raw_item).unwrap(); + classify_snapshot("9.0.6".into(), None, 42, vec![item]).snapshot_digest +} + +#[test] +fn snapshot_digest_binds_unmodeled_provider_metadata_at_every_item_level() { + let original = json!({ + "key": "SYNTH001", + "version": 7, + "meta": {"parsedDate": "2025-01-01"}, + "data": { + "itemType": "journalArticle", + "title": "Ontology learning", + "date": "2025-01-01", + "creators": [{"creatorType": "author", "name": "Synthetic Author"}], + "tags": [{"tag": "ontology", "type": 0}] + } + }); + let original_digest = snapshot_digest(original.clone()); + + for (pointer, replacement) in [ + ("/meta/parsedDate", json!("2026-01-01")), + ("/data/date", json!("2026-01-01")), + ("/data/creators/0/name", json!("Other Synthetic Author")), + ("/data/tags/0/type", json!(1)), + ] { + let mut changed = original.clone(); + *changed.pointer_mut(pointer).unwrap() = replacement; + assert_ne!( + snapshot_digest(changed), + original_digest, + "same-revision content change at {pointer} must invalidate the snapshot receipt" + ); + } +} + +#[test] +fn snapshot_digest_is_independent_of_provider_object_field_order() { + let ordered: Value = serde_json::from_str( + r#"{"key":"SYNTH001","version":7,"meta":{"a":1,"b":{"c":2,"d":3}},"data":{"itemType":"book","date":"2025","creators":[{"name":"Synthetic Author","creatorType":"author"}]}}"#, + ) + .unwrap(); + let reordered: Value = serde_json::from_str( + r#"{"data":{"creators":[{"creatorType":"author","name":"Synthetic Author"}],"date":"2025","itemType":"book"},"meta":{"b":{"d":3,"c":2},"a":1},"version":7,"key":"SYNTH001"}"#, + ) + .unwrap(); + assert_eq!(snapshot_digest(ordered), snapshot_digest(reordered)); +} From d1fef07aa7f011b8fac69777bbe35a0f8e4f141f Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sat, 5 Sep 2026 15:35:19 +0900 Subject: [PATCH 09/28] fix(zotero): retain unmodeled provider metadata in snapshot digests --- crates/conceptweave-zotero/src/lib.rs | 12 ++++++++++++ .../tests/golden_set_evaluation.rs | 2 ++ .../tests/golden_set_integrity_contract.rs | 4 ++++ crates/conceptweave-zotero/tests/review_contract.rs | 2 ++ .../tests/review_contract_followup.rs | 2 ++ .../tests/snapshot_content_binding.rs | 2 ++ 6 files changed, 24 insertions(+) diff --git a/crates/conceptweave-zotero/src/lib.rs b/crates/conceptweave-zotero/src/lib.rs index 30b05237..a39e96a6 100644 --- a/crates/conceptweave-zotero/src/lib.rs +++ b/crates/conceptweave-zotero/src/lib.rs @@ -32,6 +32,9 @@ pub struct ZoteroItem { pub version: u64, /// Item metadata. pub data: ItemData, + /// Unmodeled provider fields retained in the canonical snapshot digest. + #[serde(flatten)] + pub additional_fields: BTreeMap, } /// Metadata used by the classifier. @@ -58,6 +61,9 @@ pub struct ItemData { /// Tags applied to the item. #[serde(default)] pub tags: Vec, + /// Complete provider metadata not needed by the classifier, retained for content binding. + #[serde(flatten)] + pub additional_fields: BTreeMap, } /// A Zotero item tag. @@ -65,6 +71,9 @@ pub struct ItemData { pub struct ItemTag { /// Tag text. pub tag: String, + /// Provider tag metadata retained even when classification uses only the text. + #[serde(flatten)] + pub additional_fields: BTreeMap, } /// One mutually exclusive proposed disposition. @@ -960,7 +969,9 @@ mod tests { parent_item: parent.into(), collections: vec![], tags: vec![], + additional_fields: BTreeMap::new(), }, + additional_fields: BTreeMap::new(), } } @@ -1145,6 +1156,7 @@ mod tests { let mut generation = item("B", "journalArticle", "Ontology Learning", "10.1/X", ""); generation.data.tags.push(ItemTag { tag: "SHACL".into(), + additional_fields: BTreeMap::new(), }); let report = classify_snapshot( "9.0.6".into(), diff --git a/crates/conceptweave-zotero/tests/golden_set_evaluation.rs b/crates/conceptweave-zotero/tests/golden_set_evaluation.rs index 5acbc4a9..ceead4d0 100644 --- a/crates/conceptweave-zotero/tests/golden_set_evaluation.rs +++ b/crates/conceptweave-zotero/tests/golden_set_evaluation.rs @@ -16,7 +16,9 @@ fn item(key: &str, title: &str) -> ZoteroItem { parent_item: String::new(), collections: vec![], tags: vec![], + additional_fields: Default::default(), }, + additional_fields: Default::default(), } } diff --git a/crates/conceptweave-zotero/tests/golden_set_integrity_contract.rs b/crates/conceptweave-zotero/tests/golden_set_integrity_contract.rs index dc0615a0..566a0289 100644 --- a/crates/conceptweave-zotero/tests/golden_set_integrity_contract.rs +++ b/crates/conceptweave-zotero/tests/golden_set_integrity_contract.rs @@ -16,7 +16,9 @@ fn bibliographic(key: &str, version: u64, title: &str) -> ZoteroItem { parent_item: String::new(), collections: vec![], tags: vec![], + additional_fields: Default::default(), }, + additional_fields: Default::default(), } } @@ -32,7 +34,9 @@ fn child_note(key: &str, version: u64, parent_item: &str) -> ZoteroItem { parent_item: parent_item.into(), collections: vec![], tags: vec![], + additional_fields: Default::default(), }, + additional_fields: Default::default(), } } diff --git a/crates/conceptweave-zotero/tests/review_contract.rs b/crates/conceptweave-zotero/tests/review_contract.rs index dce9aebf..31b43016 100644 --- a/crates/conceptweave-zotero/tests/review_contract.rs +++ b/crates/conceptweave-zotero/tests/review_contract.rs @@ -12,7 +12,9 @@ fn item(key: &str, title: &str, doi: &str) -> ZoteroItem { parent_item: String::new(), collections: vec![], tags: vec![], + additional_fields: Default::default(), }, + additional_fields: Default::default(), } } diff --git a/crates/conceptweave-zotero/tests/review_contract_followup.rs b/crates/conceptweave-zotero/tests/review_contract_followup.rs index c3745c6b..7b5eafd0 100644 --- a/crates/conceptweave-zotero/tests/review_contract_followup.rs +++ b/crates/conceptweave-zotero/tests/review_contract_followup.rs @@ -12,7 +12,9 @@ fn item(key: &str, title: &str, abstract_note: &str) -> ZoteroItem { parent_item: String::new(), collections: vec![], tags: vec![], + additional_fields: Default::default(), }, + additional_fields: Default::default(), } } diff --git a/crates/conceptweave-zotero/tests/snapshot_content_binding.rs b/crates/conceptweave-zotero/tests/snapshot_content_binding.rs index efe76a1f..6d22861a 100644 --- a/crates/conceptweave-zotero/tests/snapshot_content_binding.rs +++ b/crates/conceptweave-zotero/tests/snapshot_content_binding.rs @@ -15,7 +15,9 @@ fn item(title: &str) -> ZoteroItem { parent_item: String::new(), collections: vec![], tags: vec![], + additional_fields: Default::default(), }, + additional_fields: Default::default(), } } From 0034eb5ea7c8507855a5cc5aa4ea910463781824 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sat, 5 Sep 2026 15:38:32 +0900 Subject: [PATCH 10/28] test(zotero): preserve raw metadata field presence in snapshot binding --- .../tests/raw_provider_snapshot_binding.rs | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/crates/conceptweave-zotero/tests/raw_provider_snapshot_binding.rs b/crates/conceptweave-zotero/tests/raw_provider_snapshot_binding.rs index 312a711b..11f4e1d9 100644 --- a/crates/conceptweave-zotero/tests/raw_provider_snapshot_binding.rs +++ b/crates/conceptweave-zotero/tests/raw_provider_snapshot_binding.rs @@ -50,3 +50,21 @@ fn snapshot_digest_is_independent_of_provider_object_field_order() { .unwrap(); assert_eq!(snapshot_digest(ordered), snapshot_digest(reordered)); } + +#[test] +fn snapshot_digest_preserves_omitted_versus_explicit_default_metadata() { + let omitted = json!({"key": "SYNTH001", "version": 7, "data": {"itemType": "book"}}); + let omitted_digest = snapshot_digest(omitted.clone()); + for (field_name, explicit_default) in [ + ("title", json!("")), + ("abstractNote", json!("")), + ("DOI", json!("")), + ("parentItem", json!("")), + ("collections", json!([])), + ("tags", json!([])), + ] { + let mut explicit = omitted.clone(); + explicit["data"][field_name] = explicit_default; + assert_ne!(snapshot_digest(explicit), omitted_digest, "{field_name}"); + } +} From a1e7456814436a366fdd6fb41eb8e1cca067108e Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sat, 5 Sep 2026 15:38:32 +0900 Subject: [PATCH 11/28] test(research): reproduce golden prediction approval reuse Signed-off-by: Seongho Bae --- .../tests/golden_set_integrity_contract.rs | 33 +++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/crates/conceptweave-zotero/tests/golden_set_integrity_contract.rs b/crates/conceptweave-zotero/tests/golden_set_integrity_contract.rs index dc0615a0..b46d1ecf 100644 --- a/crates/conceptweave-zotero/tests/golden_set_integrity_contract.rs +++ b/crates/conceptweave-zotero/tests/golden_set_integrity_contract.rs @@ -147,3 +147,36 @@ fn duplicate_zotero_keys_fail_closed_even_when_item_revisions_differ() { "Zotero item keys are identities; duplicate keys cannot become distinct records merely because their revision counters differ" ); } + +#[test] +fn approved_snapshot_cannot_authorize_a_prediction_changed_to_match_the_label() { + let mut report = classify_snapshot( + "9.0.6".into(), + None, + 42, + vec![bibliographic("A", 1, "ontology learning")], + ); + let golden = ReviewedGoldenSet { + approval: approval(&report, report.snapshot_items.clone()), + labels: vec![GoldenLabel::new("A", Disposition::AlignmentVersioning)], + }; + let approved_golden = golden.clone(); + assert_eq!( + evaluate_reviewed_golden_set(&report, &golden, |candidate| candidate == &approved_golden) + .unwrap() + .correct_count, + 0 + ); + + report.classified_items[0].proposed_disposition = Disposition::AlignmentVersioning; + let verifier_called = std::cell::Cell::new(false); + assert_eq!( + evaluate_reviewed_golden_set(&report, &golden, |candidate| { + verifier_called.set(true); + candidate == &approved_golden + }), + Err(EvaluationError::SnapshotMismatch), + "a source receipt cannot authorize changed predictions under unchanged source coordinates" + ); + assert!(!verifier_called.get()); +} From 552d3c70457cf8ebb6259296736870cc8da750e8 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sat, 5 Sep 2026 15:39:23 +0900 Subject: [PATCH 12/28] fix(zotero): hash complete captured JSON before metadata projection --- crates/conceptweave-zotero/src/lib.rs | 57 ++++++++++++++----- .../tests/golden_set_evaluation.rs | 3 +- .../tests/golden_set_integrity_contract.rs | 6 +- .../tests/review_contract.rs | 3 +- .../tests/review_contract_followup.rs | 3 +- .../tests/snapshot_content_binding.rs | 3 +- 6 files changed, 49 insertions(+), 26 deletions(-) diff --git a/crates/conceptweave-zotero/src/lib.rs b/crates/conceptweave-zotero/src/lib.rs index a39e96a6..4cab7422 100644 --- a/crates/conceptweave-zotero/src/lib.rs +++ b/crates/conceptweave-zotero/src/lib.rs @@ -24,7 +24,7 @@ const LOCAL_API: &str = "http://127.0.0.1:23119/api/users/0/items"; static TEST_LOCAL_API: std::sync::Mutex> = std::sync::Mutex::new(None); /// A Zotero item returned by the Local API. -#[derive(Debug, Clone, Deserialize, Serialize)] +#[derive(Debug, Clone, Serialize)] pub struct ZoteroItem { /// Stable item key. pub key: String, @@ -32,9 +32,37 @@ pub struct ZoteroItem { pub version: u64, /// Item metadata. pub data: ItemData, - /// Unmodeled provider fields retained in the canonical snapshot digest. - #[serde(flatten)] - pub additional_fields: BTreeMap, + /// Complete original JSON object, captured automatically during deserialization. + /// + /// Offline callers constructing synthetic typed items use `None`; their typed + /// representation is hashed instead. Provider records retain omitted fields, + /// unknown metadata, nested objects, and array order exactly as observed. + #[serde(skip)] + pub source_record: Option, +} + +impl<'de> Deserialize<'de> for ZoteroItem { + fn deserialize(deserializer: D) -> Result + where + D: serde::Deserializer<'de>, + { + #[derive(Deserialize)] + struct ItemProjection { + key: String, + version: u64, + data: ItemData, + } + + let source_record = serde_json::Value::deserialize(deserializer)?; + let projection = + ItemProjection::deserialize(&source_record).map_err(serde::de::Error::custom)?; + Ok(Self { + key: projection.key, + version: projection.version, + data: projection.data, + source_record: Some(source_record), + }) + } } /// Metadata used by the classifier. @@ -61,9 +89,6 @@ pub struct ItemData { /// Tags applied to the item. #[serde(default)] pub tags: Vec, - /// Complete provider metadata not needed by the classifier, retained for content binding. - #[serde(flatten)] - pub additional_fields: BTreeMap, } /// A Zotero item tag. @@ -71,9 +96,6 @@ pub struct ItemData { pub struct ItemTag { /// Tag text. pub tag: String, - /// Provider tag metadata retained even when classification uses only the text. - #[serde(flatten)] - pub additional_fields: BTreeMap, } /// One mutually exclusive proposed disposition. @@ -669,7 +691,16 @@ pub fn classify_snapshot( item_version: item.version, }) .collect(); - let snapshot_bytes = serde_json::to_vec(&items) + let snapshot_records: Vec<_> = items + .iter() + .map(|item| { + item.source_record.clone().unwrap_or_else(|| { + serde_json::to_value(item) + .expect("synthetic Zotero items contain only JSON-compatible values") + }) + }) + .collect(); + let snapshot_bytes = serde_json::to_vec(&snapshot_records) .expect("Zotero snapshot items contain only JSON-compatible values"); let snapshot_digest = format!("sha256:{:x}", Sha256::digest(snapshot_bytes)); let children = child_index(&items); @@ -969,9 +1000,8 @@ mod tests { parent_item: parent.into(), collections: vec![], tags: vec![], - additional_fields: BTreeMap::new(), }, - additional_fields: BTreeMap::new(), + source_record: None, } } @@ -1156,7 +1186,6 @@ mod tests { let mut generation = item("B", "journalArticle", "Ontology Learning", "10.1/X", ""); generation.data.tags.push(ItemTag { tag: "SHACL".into(), - additional_fields: BTreeMap::new(), }); let report = classify_snapshot( "9.0.6".into(), diff --git a/crates/conceptweave-zotero/tests/golden_set_evaluation.rs b/crates/conceptweave-zotero/tests/golden_set_evaluation.rs index ceead4d0..d49dc490 100644 --- a/crates/conceptweave-zotero/tests/golden_set_evaluation.rs +++ b/crates/conceptweave-zotero/tests/golden_set_evaluation.rs @@ -16,9 +16,8 @@ fn item(key: &str, title: &str) -> ZoteroItem { parent_item: String::new(), collections: vec![], tags: vec![], - additional_fields: Default::default(), }, - additional_fields: Default::default(), + source_record: None, } } diff --git a/crates/conceptweave-zotero/tests/golden_set_integrity_contract.rs b/crates/conceptweave-zotero/tests/golden_set_integrity_contract.rs index 566a0289..b4c63293 100644 --- a/crates/conceptweave-zotero/tests/golden_set_integrity_contract.rs +++ b/crates/conceptweave-zotero/tests/golden_set_integrity_contract.rs @@ -16,9 +16,8 @@ fn bibliographic(key: &str, version: u64, title: &str) -> ZoteroItem { parent_item: String::new(), collections: vec![], tags: vec![], - additional_fields: Default::default(), }, - additional_fields: Default::default(), + source_record: None, } } @@ -34,9 +33,8 @@ fn child_note(key: &str, version: u64, parent_item: &str) -> ZoteroItem { parent_item: parent_item.into(), collections: vec![], tags: vec![], - additional_fields: Default::default(), }, - additional_fields: Default::default(), + source_record: None, } } diff --git a/crates/conceptweave-zotero/tests/review_contract.rs b/crates/conceptweave-zotero/tests/review_contract.rs index 31b43016..e0890264 100644 --- a/crates/conceptweave-zotero/tests/review_contract.rs +++ b/crates/conceptweave-zotero/tests/review_contract.rs @@ -12,9 +12,8 @@ fn item(key: &str, title: &str, doi: &str) -> ZoteroItem { parent_item: String::new(), collections: vec![], tags: vec![], - additional_fields: Default::default(), }, - additional_fields: Default::default(), + source_record: None, } } diff --git a/crates/conceptweave-zotero/tests/review_contract_followup.rs b/crates/conceptweave-zotero/tests/review_contract_followup.rs index 7b5eafd0..d176f2de 100644 --- a/crates/conceptweave-zotero/tests/review_contract_followup.rs +++ b/crates/conceptweave-zotero/tests/review_contract_followup.rs @@ -12,9 +12,8 @@ fn item(key: &str, title: &str, abstract_note: &str) -> ZoteroItem { parent_item: String::new(), collections: vec![], tags: vec![], - additional_fields: Default::default(), }, - additional_fields: Default::default(), + source_record: None, } } diff --git a/crates/conceptweave-zotero/tests/snapshot_content_binding.rs b/crates/conceptweave-zotero/tests/snapshot_content_binding.rs index 6d22861a..ad6bd3e2 100644 --- a/crates/conceptweave-zotero/tests/snapshot_content_binding.rs +++ b/crates/conceptweave-zotero/tests/snapshot_content_binding.rs @@ -15,9 +15,8 @@ fn item(title: &str) -> ZoteroItem { parent_item: String::new(), collections: vec![], tags: vec![], - additional_fields: Default::default(), }, - additional_fields: Default::default(), + source_record: None, } } From 77640ef7df401b288a5744384abb6f6e305f56de Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sat, 5 Sep 2026 15:39:48 +0900 Subject: [PATCH 13/28] fix(research)!: bind golden approvals to evaluated proposals Require a separately verified proposal digest and recompute it from complete sorted proposal records before governance verification. BREAKING CHANGE: GoldenSetApproval requires proposal_digest; unbound receipts must be reissued by governance. Signed-off-by: Seongho Bae --- crates/conceptweave-zotero/src/lib.rs | 50 +++++++++++++++++-- .../tests/golden_set_evaluation.rs | 5 +- .../tests/golden_set_integrity_contract.rs | 5 +- .../tests/snapshot_content_binding.rs | 4 +- 4 files changed, 55 insertions(+), 9 deletions(-) diff --git a/crates/conceptweave-zotero/src/lib.rs b/crates/conceptweave-zotero/src/lib.rs index 30b05237..d053fc9b 100644 --- a/crates/conceptweave-zotero/src/lib.rs +++ b/crates/conceptweave-zotero/src/lib.rs @@ -214,7 +214,7 @@ pub struct SnapshotItemRevision { pub item_version: u64, } -/// Governance receipt binding a steward approval to one exact classifier input. +/// Governance receipt binding a steward approval to exact input and proposals. #[derive(Debug, Clone, PartialEq, Eq, Deserialize, Serialize)] pub struct GoldenSetApproval { /// Opaque receipt identifier. @@ -227,6 +227,8 @@ pub struct GoldenSetApproval { pub rule_revision: String, /// Immutable digest over the approved snapshot, verified by the caller. pub snapshot_digest: String, + /// Digest of the actual proposal records reviewed and verified by the caller. + pub proposal_digest: String, /// Complete sorted item-revision identity of the reviewed report. pub snapshot_items: Vec, } @@ -236,6 +238,22 @@ pub fn classification_snapshot_digest(report: &ClassificationReport) -> String { report.snapshot_digest.clone() } +/// Computes the versioned SHA-256 identity of the report's current proposals. +/// +/// Every proposal field is covered, including its prediction, evidence, and item +/// revision. Records are sorted by item key and revision so page ordering does +/// not change their identity. No second source snapshot is stored. Governance +/// must bind this value when issuing an approval; recomputing it alone grants no +/// authority. Evaluation recomputes it rather than trusting report metadata. +pub fn classification_proposal_digest(report: &ClassificationReport) -> String { + let mut proposals = report.classified_items.iter().collect::>(); + proposals.sort_by_key(|item| (&item.item_key, item.item_version)); + let proposal_bytes = + serde_json::to_vec(&("conceptweave-classification-proposals-v1", proposals)) + .expect("classification proposal records contain only JSON-serializable values"); + format!("sha256:{:x}", Sha256::digest(proposal_bytes)) +} + /// Integer evidence from which precision and recall can be calculated exactly. #[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize)] pub struct DispositionEvaluation { @@ -258,6 +276,8 @@ pub struct GoldenSetEvaluation { pub rule_revision: String, /// Opaque immutable snapshot digest from the verified receipt. pub snapshot_digest: String, + /// Opaque digest binding the exact proposal records used for these counts. + pub proposal_digest: String, /// Number of steward-reviewed items. pub reviewed_count: usize, /// Number of exact disposition matches. @@ -303,6 +323,11 @@ impl fmt::Display for EvaluationError { impl std::error::Error for EvaluationError {} /// Evaluates reviewed labels without copying item identities into the result. +/// +/// Structural, source, proposal, and label validation run before governance is +/// contacted. The verifier must authenticate the complete reviewed set against +/// an independently issued receipt, including both digests and every label; +/// accepting a self-declared receipt identifier or digest is not verification. pub fn evaluate_reviewed_golden_set( report: &ClassificationReport, golden: &ReviewedGoldenSet, @@ -316,12 +341,10 @@ where || golden.labels.is_empty() || golden.approval.rule_revision.trim().is_empty() || golden.approval.snapshot_digest.trim().is_empty() + || golden.approval.proposal_digest.trim().is_empty() { return Err(EvaluationError::InvalidReview); } - if !verify_approval(golden) { - return Err(EvaluationError::UnverifiedApproval); - } let report_snapshot = report .snapshot_items .iter() @@ -357,6 +380,20 @@ where .iter() .map(|item| (item.item_key.as_str(), item.proposed_disposition)) .collect::>(); + if classified.len() != report.classified_items.len() + || report.classified_items.iter().any(|item| { + item.item_key.trim().is_empty() + || !report_snapshot.contains(&SnapshotItemRevision { + item_key: item.item_key.clone(), + item_version: item.item_version, + }) + }) + { + return Err(EvaluationError::InvalidReview); + } + if golden.approval.proposal_digest != classification_proposal_digest(report) { + return Err(EvaluationError::SnapshotMismatch); + } let mut seen = BTreeSet::new(); let mut correct_count = 0; let mut abstention_count = 0; @@ -390,11 +427,16 @@ where } } + if !verify_approval(golden) { + return Err(EvaluationError::UnverifiedApproval); + } + Ok(GoldenSetEvaluation { review_id: golden.approval.receipt_id.clone(), library_version: golden.approval.library_version, rule_revision: golden.approval.rule_revision.clone(), snapshot_digest: golden.approval.snapshot_digest.clone(), + proposal_digest: golden.approval.proposal_digest.clone(), reviewed_count: golden.labels.len(), correct_count, abstention_count, diff --git a/crates/conceptweave-zotero/tests/golden_set_evaluation.rs b/crates/conceptweave-zotero/tests/golden_set_evaluation.rs index 5acbc4a9..1b8b5568 100644 --- a/crates/conceptweave-zotero/tests/golden_set_evaluation.rs +++ b/crates/conceptweave-zotero/tests/golden_set_evaluation.rs @@ -1,7 +1,7 @@ use conceptweave_zotero::{ Disposition, EvaluationError, GoldenLabel, GoldenSetApproval, ItemData, ReviewedGoldenSet, - SnapshotItemRevision, ZoteroItem, classification_snapshot_digest, classify_snapshot, - evaluate_reviewed_golden_set, + SnapshotItemRevision, ZoteroItem, classification_proposal_digest, + classification_snapshot_digest, classify_snapshot, evaluate_reviewed_golden_set, }; fn item(key: &str, title: &str) -> ZoteroItem { @@ -41,6 +41,7 @@ fn golden(labels: Vec) -> ReviewedGoldenSet { library_version: 42, rule_revision: "ontology-research-v2".into(), snapshot_digest: classification_snapshot_digest(&report()), + proposal_digest: classification_proposal_digest(&report()), snapshot_items: ["A", "B", "C"] .into_iter() .map(|item_key| SnapshotItemRevision { diff --git a/crates/conceptweave-zotero/tests/golden_set_integrity_contract.rs b/crates/conceptweave-zotero/tests/golden_set_integrity_contract.rs index b46d1ecf..55022ac9 100644 --- a/crates/conceptweave-zotero/tests/golden_set_integrity_contract.rs +++ b/crates/conceptweave-zotero/tests/golden_set_integrity_contract.rs @@ -1,7 +1,7 @@ use conceptweave_zotero::{ Disposition, EvaluationError, GoldenLabel, GoldenSetApproval, ItemData, ReviewedGoldenSet, - SnapshotItemRevision, ZoteroItem, classification_snapshot_digest, classify_snapshot, - evaluate_reviewed_golden_set, + SnapshotItemRevision, ZoteroItem, classification_proposal_digest, + classification_snapshot_digest, classify_snapshot, evaluate_reviewed_golden_set, }; fn bibliographic(key: &str, version: u64, title: &str) -> ZoteroItem { @@ -46,6 +46,7 @@ fn approval( library_version: report.library_version, rule_revision: report.rule_revision.into(), snapshot_digest: classification_snapshot_digest(report), + proposal_digest: classification_proposal_digest(report), snapshot_items, } } diff --git a/crates/conceptweave-zotero/tests/snapshot_content_binding.rs b/crates/conceptweave-zotero/tests/snapshot_content_binding.rs index efe76a1f..bd21cff5 100644 --- a/crates/conceptweave-zotero/tests/snapshot_content_binding.rs +++ b/crates/conceptweave-zotero/tests/snapshot_content_binding.rs @@ -1,6 +1,7 @@ use conceptweave_zotero::{ Disposition, EvaluationError, GoldenLabel, GoldenSetApproval, ItemData, ReviewedGoldenSet, - SnapshotItemRevision, ZoteroItem, classify_snapshot, evaluate_reviewed_golden_set, + SnapshotItemRevision, ZoteroItem, classification_proposal_digest, classify_snapshot, + evaluate_reviewed_golden_set, }; fn item(title: &str) -> ZoteroItem { @@ -30,6 +31,7 @@ fn golden_approval_rejects_same_revision_coordinates_with_changed_snapshot_conte library_version: 42, rule_revision: "ontology-research-v2".into(), snapshot_digest: "sha256:approved-original-content".into(), + proposal_digest: classification_proposal_digest(&changed_report), snapshot_items: vec![SnapshotItemRevision { item_key: "A".into(), item_version: 1, From e624186da7adc3cfa0fd88cba75fd4da66b974b6 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sat, 5 Sep 2026 15:41:08 +0900 Subject: [PATCH 14/28] test(zotero): bind mutated classifier inputs alongside captured source --- .../tests/raw_provider_snapshot_binding.rs | 29 +++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/crates/conceptweave-zotero/tests/raw_provider_snapshot_binding.rs b/crates/conceptweave-zotero/tests/raw_provider_snapshot_binding.rs index 11f4e1d9..f3432485 100644 --- a/crates/conceptweave-zotero/tests/raw_provider_snapshot_binding.rs +++ b/crates/conceptweave-zotero/tests/raw_provider_snapshot_binding.rs @@ -68,3 +68,32 @@ fn snapshot_digest_preserves_omitted_versus_explicit_default_metadata() { assert_ne!(snapshot_digest(explicit), omitted_digest, "{field_name}"); } } + +#[test] +fn snapshot_digest_binds_actual_classifier_inputs_after_provider_decode() { + let original: ZoteroItem = serde_json::from_value(json!({ + "key": "SYNTH001", "version": 7, + "data": {"itemType": "book", "title": "Ontology learning"} + })) + .unwrap(); + let original_digest = + classify_snapshot("9.0.6".into(), None, 42, vec![original.clone()]).snapshot_digest; + let mut changed = original; + changed.data.title = "Ontology alignment".into(); + assert_ne!( + classify_snapshot("9.0.6".into(), None, 42, vec![changed]).snapshot_digest, + original_digest, + "changed classifier input cannot retain the original source receipt" + ); +} + +#[test] +fn source_capture_preserves_provider_shape_validation() { + for invalid in [ + json!(null), + json!({"key": 7, "version": 7, "data": {"itemType": "book"}}), + json!({"key": "SYNTH001", "version": 7, "data": {}}), + ] { + assert!(serde_json::from_value::(invalid).is_err()); + } +} From 56a2fc4fd45be3cbabb42d1d982ea987ae71079c Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sat, 5 Sep 2026 15:41:27 +0900 Subject: [PATCH 15/28] fix(zotero): bind source JSON and classifier projection in one digest --- crates/conceptweave-zotero/src/lib.rs | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/crates/conceptweave-zotero/src/lib.rs b/crates/conceptweave-zotero/src/lib.rs index 4cab7422..f80142dd 100644 --- a/crates/conceptweave-zotero/src/lib.rs +++ b/crates/conceptweave-zotero/src/lib.rs @@ -34,8 +34,9 @@ pub struct ZoteroItem { pub data: ItemData, /// Complete original JSON object, captured automatically during deserialization. /// - /// Offline callers constructing synthetic typed items use `None`; their typed - /// representation is hashed instead. Provider records retain omitted fields, + /// Offline callers constructing synthetic typed items use `None`. The digest + /// binds this source value together with the actual typed classifier input, + /// so later projection changes also invalidate the receipt. It retains omitted fields, /// unknown metadata, nested objects, and array order exactly as observed. #[serde(skip)] pub source_record: Option, @@ -693,12 +694,7 @@ pub fn classify_snapshot( .collect(); let snapshot_records: Vec<_> = items .iter() - .map(|item| { - item.source_record.clone().unwrap_or_else(|| { - serde_json::to_value(item) - .expect("synthetic Zotero items contain only JSON-compatible values") - }) - }) + .map(|item| (&item.source_record, item)) .collect(); let snapshot_bytes = serde_json::to_vec(&snapshot_records) .expect("Zotero snapshot items contain only JSON-compatible values"); From 52290d5fd8a0d35e65dffe8cc4ad7090e4cca275 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sat, 5 Sep 2026 15:42:22 +0900 Subject: [PATCH 16/28] test(research): verify proposal receipt forgery and validation boundaries Signed-off-by: Seongho Bae --- CHANGELOG.md | 1 + .../tests/golden_set_integrity_contract.rs | 138 ++++++++++++++++++ docs/TRD.md | 4 +- 3 files changed, 142 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8910d6fa..cc75ecd5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,5 +14,6 @@ All notable changes to ConceptWeave are documented here. ### Security +- Golden-set evaluation rejects changed predictions or evidence under an earlier approval. Proposal-bound approvals must be reissued; aggregate receipts identify the actual evaluated proposal run. - Model-generated semantics remain non-authoritative until deterministic validation and authorized review. - Unsafe Rust is forbidden in the core domain crate. diff --git a/crates/conceptweave-zotero/tests/golden_set_integrity_contract.rs b/crates/conceptweave-zotero/tests/golden_set_integrity_contract.rs index 55022ac9..59641847 100644 --- a/crates/conceptweave-zotero/tests/golden_set_integrity_contract.rs +++ b/crates/conceptweave-zotero/tests/golden_set_integrity_contract.rs @@ -181,3 +181,141 @@ fn approved_snapshot_cannot_authorize_a_prediction_changed_to_match_the_label() ); assert!(!verifier_called.get()); } + +#[test] +fn rewriting_the_proposal_digest_cannot_reuse_an_independent_approval() { + let mut report = classify_snapshot( + "9.0.6".into(), + None, + 42, + vec![bibliographic("A", 1, "ontology learning")], + ); + let mut golden = ReviewedGoldenSet { + approval: approval(&report, report.snapshot_items.clone()), + labels: vec![GoldenLabel::new("A", Disposition::AlignmentVersioning)], + }; + let approved_golden = golden.clone(); + report.classified_items[0].proposed_disposition = Disposition::AlignmentVersioning; + golden.approval.proposal_digest = classification_proposal_digest(&report); + let imported_golden = + serde_json::from_slice::(&serde_json::to_vec(&golden).unwrap()).unwrap(); + assert_eq!( + evaluate_reviewed_golden_set(&report, &imported_golden, |candidate| { + candidate == &approved_golden + }), + Err(EvaluationError::UnverifiedApproval) + ); +} + +#[test] +fn approval_binds_unreviewed_proposals_and_supporting_evidence_but_not_record_order() { + let mut report = classify_snapshot( + "9.0.6".into(), + None, + 42, + vec![ + bibliographic("A", 1, "ontology learning"), + bibliographic("B", 1, "ontology evaluation"), + ], + ); + let golden = ReviewedGoldenSet { + approval: approval(&report, report.snapshot_items.clone()), + labels: vec![GoldenLabel::new("A", Disposition::Generation)], + }; + report.classified_items.reverse(); + let evaluation = + evaluate_reviewed_golden_set(&report, &golden, |candidate| candidate == &golden).unwrap(); + assert_eq!(evaluation.correct_count, 1); + assert_eq!(evaluation.proposal_digest, golden.approval.proposal_digest); + + // B is outside the reviewed sample, but belongs to the approved proposal run. + report.classified_items[0].evidence.field_values.clear(); + assert_eq!( + evaluate_reviewed_golden_set(&report, &golden, |_| { + panic!("changed proposal evidence must fail before governance") + }), + Err(EvaluationError::SnapshotMismatch) + ); +} + +#[test] +fn missing_proposal_binding_and_invalid_labels_fail_before_governance() { + let report = classify_snapshot( + "9.0.6".into(), + None, + 42, + vec![bibliographic("A", 1, "ontology learning")], + ); + let mut golden = ReviewedGoldenSet { + approval: approval(&report, report.snapshot_items.clone()), + labels: vec![GoldenLabel::new("A", Disposition::Generation)], + }; + let mut legacy_json = serde_json::to_value(&golden).unwrap(); + legacy_json["approval"] + .as_object_mut() + .unwrap() + .remove("proposal_digest"); + assert!(serde_json::from_value::(legacy_json).is_err()); + + golden.approval.proposal_digest.clear(); + assert_eq!( + evaluate_reviewed_golden_set(&report, &golden, |_| { + panic!("unbound proposals must fail before governance") + }), + Err(EvaluationError::InvalidReview) + ); + golden.approval.proposal_digest = classification_proposal_digest(&report); + for (labels, expected_error) in [ + ( + vec![GoldenLabel::new(" ", Disposition::Generation)], + EvaluationError::InvalidReview, + ), + ( + vec![GoldenLabel::new("A", Disposition::NeedsStewardReview)], + EvaluationError::InvalidExpectedDisposition, + ), + ( + vec![GoldenLabel::new("absent", Disposition::Generation)], + EvaluationError::UnknownItem, + ), + ( + vec![GoldenLabel::new("A", Disposition::Generation); 2], + EvaluationError::DuplicateItem, + ), + ] { + golden.labels = labels; + assert_eq!( + evaluate_reviewed_golden_set(&report, &golden, |_| { + panic!("invalid labels must fail before governance") + }), + Err(expected_error) + ); + } +} + +#[test] +fn malformed_proposal_identities_fail_before_governance() { + for (replacement_key, replacement_version) in [("A", 1), ("B", 9), (" ", 1)] { + let mut report = classify_snapshot( + "9.0.6".into(), + None, + 42, + vec![ + bibliographic("A", 1, "ontology learning"), + bibliographic("B", 1, "ontology evaluation"), + ], + ); + let golden = ReviewedGoldenSet { + approval: approval(&report, report.snapshot_items.clone()), + labels: vec![GoldenLabel::new("A", Disposition::Generation)], + }; + report.classified_items[1].item_key = replacement_key.into(); + report.classified_items[1].item_version = replacement_version; + assert_eq!( + evaluate_reviewed_golden_set(&report, &golden, |_| { + panic!("malformed proposals must fail before governance") + }), + Err(EvaluationError::InvalidReview) + ); + } +} diff --git a/docs/TRD.md b/docs/TRD.md index 2e641cc8..9cd969f7 100644 --- a/docs/TRD.md +++ b/docs/TRD.md @@ -63,6 +63,8 @@ Evaluation must separate extraction recall, semantic correctness, structural cor Every top-level bibliographic record receives exactly one proposed disposition. `NeedsStewardReview` also records a deterministic abstention reason so missing classification metadata, vocabulary unsupported by the current deterministic rules, and present-but-unmatched metadata are distinguishable. DOI duplicate identity normalization treats bare DOI values, `doi:`, `doi.org`, and legacy `dx.doi.org` resolver forms as the same identity when their normalized DOI is equal. -Golden-set evaluation accepts only a governance receipt verified with the complete reviewed set by a caller-owned authorization boundary. Its library version, rule revision, canonical SHA-256 content digest, and every observed parent/child item-key/item-version identity must bind the classification report. The digest covers every raw Zotero item in canonical key order. Blank, duplicate, unknown, stale, content-mismatched, label-mismatched, or abstention-as-truth inputs fail closed. The output retains the verified library version, rule revision, and opaque snapshot digest, but contains no item keys, reviewer identity, or bibliographic text. Production authorization remains Keyverse/governance-owned; this crate passes the complete reviewed labels to that boundary instead of minting authority. +Golden-set evaluation accepts only a governance receipt verified with the complete reviewed set by a caller-owned authorization boundary. Its library version, rule revision, canonical SHA-256 content digest, and every observed parent/child item-key/item-version identity must bind the classification report. The snapshot digest covers every raw Zotero item in canonical key order. A separate required `proposal_digest` binds every field of every proposed item, including predictions, supporting evidence, and proposals outside a reviewed sample. `classification_proposal_digest` computes SHA-256 over compact JSON containing the `conceptweave-classification-proposals-v1` domain marker and proposal records sorted by item key and revision. It uses the current records, not a report's self-declared digest or a second stored source snapshot. Governance must issue and independently verify both digests together with the labels; a locally recomputed replacement digest cannot renew an old approval. Legacy approvals missing this field fail closed and require reissuance, not automatic backfill. + +Structural, source, proposal, and label checks precede the external verifier. Blank, duplicate, unknown, stale, content-mismatched, prediction-mismatched, label-mismatched, or abstention-as-truth inputs fail closed. The aggregate result retains the verified library version, rule revision, and opaque snapshot/proposal digests, but contains no item keys, reviewer identity, or bibliographic text. Production authorization remains Keyverse/governance-owned; this crate passes the complete reviewed labels and approval bindings to that boundary instead of minting authority. The report is local JSON and contains proposals rather than governance decisions. CLI output is restricted to a new direct child of canonical `/tmp` or the operating system temporary directory; relative paths, nested paths, existing paths, and symlinks are rejected, and create-new file semantics prevent overwrite/path-swap writes. Zotero 9 writes are unsupported; no mutation path exists in this slice. A future Zotero 10+ writer requires a separate reviewed change with a Local API key, stable server identity, fresh item/library version preconditions, item-by-item before/after receipts, and rollback evidence. From 9d45aa2322db3cefa8122aa2fa26d9187d0344ea Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sat, 5 Sep 2026 15:42:24 +0900 Subject: [PATCH 17/28] docs(zotero): specify complete source and projection content binding --- docs/TRD.md | 2 ++ docs/doctoring/REFERENCES.md | 8 ++++++++ 2 files changed, 10 insertions(+) diff --git a/docs/TRD.md b/docs/TRD.md index 2e641cc8..6fb414f3 100644 --- a/docs/TRD.md +++ b/docs/TRD.md @@ -65,4 +65,6 @@ Every top-level bibliographic record receives exactly one proposed disposition. Golden-set evaluation accepts only a governance receipt verified with the complete reviewed set by a caller-owned authorization boundary. Its library version, rule revision, canonical SHA-256 content digest, and every observed parent/child item-key/item-version identity must bind the classification report. The digest covers every raw Zotero item in canonical key order. Blank, duplicate, unknown, stale, content-mismatched, label-mismatched, or abstention-as-truth inputs fail closed. The output retains the verified library version, rule revision, and opaque snapshot digest, but contains no item keys, reviewer identity, or bibliographic text. Production authorization remains Keyverse/governance-owned; this crate passes the complete reviewed labels to that boundary instead of minting authority. +Provider deserialization captures each complete JSON object before projecting metadata. Snapshot hashing uses key-ordered pairs of that canonical source JSON and the actual typed classifier input. Unknown nested fields, array order, and omitted-versus-explicit default fields remain bound; changing a typed input after decoding also changes the digest. Synthetic offline typed items have no captured provider object and bind an explicit absent-source value alongside their typed input. Earlier reduced-content digests remain historical evidence and cannot establish this complete-content contract. + The report is local JSON and contains proposals rather than governance decisions. CLI output is restricted to a new direct child of canonical `/tmp` or the operating system temporary directory; relative paths, nested paths, existing paths, and symlinks are rejected, and create-new file semantics prevent overwrite/path-swap writes. Zotero 9 writes are unsupported; no mutation path exists in this slice. A future Zotero 10+ writer requires a separate reviewed change with a Local API key, stable server identity, fresh item/library version preconditions, item-by-item before/after receipts, and rollback evidence. diff --git a/docs/doctoring/REFERENCES.md b/docs/doctoring/REFERENCES.md index 2cbca9b0..84c5e056 100644 --- a/docs/doctoring/REFERENCES.md +++ b/docs/doctoring/REFERENCES.md @@ -98,3 +98,11 @@ Li, J., Garijo, D., & Poveda-Villalón, M. (2026). Large language models for ont - OAEI-LLM/OAEI-LLM-T add LLM-specific hallucination categories to matching evaluation. GRC remains the enterprise round-trip fixture rather than the sole benchmark. - Modular ontology engineering and explicit source provenance are preferred over one opaque prompt that attempts to generate an entire enterprise semantic layer in a single step. - Human review remains mandatory before authority promotion. Scalable validation research may inform review mechanics but cannot replace domain-owner/steward authority. + +## Provider snapshot content binding + +Serde. (n.d.). *Container attributes*. Retrieved September 5, 2026, from https://serde.rs/container-attrs.html + +Serde. (n.d.). *Struct flattening*. Retrieved September 5, 2026, from https://serde.rs/attr-flatten.html + +The derived JSON reader ignores unknown fields by default. Flattened maps retain those fields but still normalize omitted modeled fields through defaults. Therefore `ZoteroItem::deserialize` captures the original JSON object, and `classify_snapshot` hashes it together with the actual classifier projection. The regression suite `raw_provider_snapshot_binding.rs` verifies unknown top-level/data/tag fields, omitted default fields, object-order stability, post-decode input mutation, and invalid provider shapes. This repairs the raw-content finding on PR #10 without treating it as external review approval or protected release evidence. From ea942b0f4800dec9e51d444c6499730efc31523a Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sat, 5 Sep 2026 15:43:44 +0900 Subject: [PATCH 18/28] test(zotero): require versioned snapshot digest domain separation --- .../tests/raw_provider_snapshot_binding.rs | 22 +++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/crates/conceptweave-zotero/tests/raw_provider_snapshot_binding.rs b/crates/conceptweave-zotero/tests/raw_provider_snapshot_binding.rs index f3432485..f9fe4115 100644 --- a/crates/conceptweave-zotero/tests/raw_provider_snapshot_binding.rs +++ b/crates/conceptweave-zotero/tests/raw_provider_snapshot_binding.rs @@ -1,5 +1,6 @@ use conceptweave_zotero::{ZoteroItem, classify_snapshot}; use serde_json::{Value, json}; +use sha2::{Digest, Sha256}; fn snapshot_digest(raw_item: Value) -> String { let item: ZoteroItem = serde_json::from_value(raw_item).unwrap(); @@ -96,4 +97,25 @@ fn source_capture_preserves_provider_shape_validation() { ] { assert!(serde_json::from_value::(invalid).is_err()); } + assert!( + serde_json::from_str::( + r#"{"key":"SYNTH001","version":7,"data":{"itemType":false}}"# + ) + .is_err() + ); +} + +#[test] +fn snapshot_digest_has_versioned_domain_separation() { + let item: ZoteroItem = serde_json::from_value(json!({ + "key": "SYNTH001", "version": 7, "data": {"itemType": "book"} + })) + .unwrap(); + let unmarked_content = serde_json::to_vec(&[(&item.source_record, &item)]).unwrap(); + let unmarked_digest = format!("sha256:{:x}", Sha256::digest(unmarked_content)); + assert_ne!( + classify_snapshot("9.0.6".into(), None, 42, vec![item]).snapshot_digest, + unmarked_digest, + "snapshot receipts must be separated from unversioned content hashes" + ); } From a5aa47f9b44438737ba369dbf32634e41a68fdf6 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sat, 5 Sep 2026 15:44:17 +0900 Subject: [PATCH 19/28] fix(zotero): version the complete snapshot digest envelope --- crates/conceptweave-zotero/src/lib.rs | 3 ++- docs/TRD.md | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/crates/conceptweave-zotero/src/lib.rs b/crates/conceptweave-zotero/src/lib.rs index f80142dd..2f48c4e9 100644 --- a/crates/conceptweave-zotero/src/lib.rs +++ b/crates/conceptweave-zotero/src/lib.rs @@ -13,6 +13,7 @@ use std::time::Duration; pub const RULE_REVISION: &str = "ontology-research-v2"; const SUPPORTED_API_VERSION: u64 = 3; +const SNAPSHOT_DIGEST_DOMAIN: &str = "conceptweave-zotero-snapshot-v2"; const SUPPORTED_API_VERSION_HEADER: &str = "3"; const PAGE_LIMIT: usize = 100; const MAX_PAGE_BYTES: u64 = 8 * 1024 * 1024; @@ -696,7 +697,7 @@ pub fn classify_snapshot( .iter() .map(|item| (&item.source_record, item)) .collect(); - let snapshot_bytes = serde_json::to_vec(&snapshot_records) + let snapshot_bytes = serde_json::to_vec(&(SNAPSHOT_DIGEST_DOMAIN, snapshot_records)) .expect("Zotero snapshot items contain only JSON-compatible values"); let snapshot_digest = format!("sha256:{:x}", Sha256::digest(snapshot_bytes)); let children = child_index(&items); diff --git a/docs/TRD.md b/docs/TRD.md index 6fb414f3..15653a5b 100644 --- a/docs/TRD.md +++ b/docs/TRD.md @@ -65,6 +65,6 @@ Every top-level bibliographic record receives exactly one proposed disposition. Golden-set evaluation accepts only a governance receipt verified with the complete reviewed set by a caller-owned authorization boundary. Its library version, rule revision, canonical SHA-256 content digest, and every observed parent/child item-key/item-version identity must bind the classification report. The digest covers every raw Zotero item in canonical key order. Blank, duplicate, unknown, stale, content-mismatched, label-mismatched, or abstention-as-truth inputs fail closed. The output retains the verified library version, rule revision, and opaque snapshot digest, but contains no item keys, reviewer identity, or bibliographic text. Production authorization remains Keyverse/governance-owned; this crate passes the complete reviewed labels to that boundary instead of minting authority. -Provider deserialization captures each complete JSON object before projecting metadata. Snapshot hashing uses key-ordered pairs of that canonical source JSON and the actual typed classifier input. Unknown nested fields, array order, and omitted-versus-explicit default fields remain bound; changing a typed input after decoding also changes the digest. Synthetic offline typed items have no captured provider object and bind an explicit absent-source value alongside their typed input. Earlier reduced-content digests remain historical evidence and cannot establish this complete-content contract. +Provider deserialization captures each complete JSON object before projecting metadata. Snapshot hashing serializes the domain marker `conceptweave-zotero-snapshot-v2` followed by key-ordered pairs of that canonical source JSON and the actual typed classifier input. Unknown nested fields, array order, and omitted-versus-explicit default fields remain bound; changing a typed input after decoding also changes the digest. Synthetic offline typed items have no captured provider object and bind an explicit absent-source value alongside their typed input. Earlier reduced-content digests remain historical evidence and cannot establish this complete-content contract; regenerate the report and review artifacts and obtain fresh approval before any release or approved write. The report is local JSON and contains proposals rather than governance decisions. CLI output is restricted to a new direct child of canonical `/tmp` or the operating system temporary directory; relative paths, nested paths, existing paths, and symlinks are rejected, and create-new file semantics prevent overwrite/path-swap writes. Zotero 9 writes are unsupported; no mutation path exists in this slice. A future Zotero 10+ writer requires a separate reviewed change with a Local API key, stable server identity, fresh item/library version preconditions, item-by-item before/after receipts, and rollback evidence. From 2665f58189ffe1b372d22b2c4941dfa0c08765b4 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sat, 5 Sep 2026 15:44:18 +0900 Subject: [PATCH 20/28] test(zotero): cover truncated provider JSON capture failure --- .../conceptweave-zotero/tests/raw_provider_snapshot_binding.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/crates/conceptweave-zotero/tests/raw_provider_snapshot_binding.rs b/crates/conceptweave-zotero/tests/raw_provider_snapshot_binding.rs index f9fe4115..feae5618 100644 --- a/crates/conceptweave-zotero/tests/raw_provider_snapshot_binding.rs +++ b/crates/conceptweave-zotero/tests/raw_provider_snapshot_binding.rs @@ -90,6 +90,7 @@ fn snapshot_digest_binds_actual_classifier_inputs_after_provider_decode() { #[test] fn source_capture_preserves_provider_shape_validation() { + assert!(serde_json::from_str::("{").is_err()); for invalid in [ json!(null), json!({"key": 7, "version": 7, "data": {"itemType": "book"}}), From 0dbc35a2d8ce3df24c3324413d4ae38232648970 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sat, 5 Sep 2026 15:46:53 +0900 Subject: [PATCH 21/28] docs(research): record evidence binding repair decisions --- CHANGELOG.md | 1 + docs/adr/0006-zotero-research-intake.md | 12 ++++++++++++ 2 files changed, 13 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index cc75ecd5..92166b32 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,6 +14,7 @@ All notable changes to ConceptWeave are documented here. ### Security +- Source receipts bind complete captured metadata and actual classifier inputs; earlier report and review artifacts require regeneration under the versioned digest representation. - Golden-set evaluation rejects changed predictions or evidence under an earlier approval. Proposal-bound approvals must be reissued; aggregate receipts identify the actual evaluated proposal run. - Model-generated semantics remain non-authoritative until deterministic validation and authorized review. - Unsafe Rust is forbidden in the core domain crate. diff --git a/docs/adr/0006-zotero-research-intake.md b/docs/adr/0006-zotero-research-intake.md index dca5b407..287ee385 100644 --- a/docs/adr/0006-zotero-research-intake.md +++ b/docs/adr/0006-zotero-research-intake.md @@ -29,6 +29,18 @@ No dedicated utility repository or Zotero mutation path is created. A future Zot ## Consequences +### 2026-09-05 integrity amendment (Proposed) + +In the context of replaying a Zotero research classification against a steward's approved labels, facing source fields lost during projection and predictions mutable after review, we decided for separate source-and-input and proposal-content digests verified with the complete reviewed set, and against typed-only source hashing or a report's self-declared cached proposal identity, to preserve the exact evidence used for evaluation, accepting a receipt-format break, report regeneration and fresh governance approval. + +The source identity uses `conceptweave-zotero-snapshot-v2` over item-key-ordered pairs of complete captured provider JSON and actual typed classifier input. Capturing only unknown flattened fields was rejected after the omitted-title versus explicit-empty-title regression showed another collision. Hashing only captured JSON was also rejected: mutating a decoded public title would otherwise change classification without changing source identity. The pair binds both representations without another cloned source snapshot; JSON object order is canonicalized while field presence, nested metadata and array order remain meaningful. + +The approval additionally requires `proposal_digest`, computed from all current proposal records under `conceptweave-classification-proposals-v1`, sorted by item key and revision. Every proposal field is bound, including evidence and records outside a reviewed sample. Evaluation recomputes this value before invoking the external verifier. Changing both the prediction and the submitted digest cannot renew an independently issued approval; the governance verifier must authenticate the complete reviewed set, not merely accept a receipt identifier. Local checks remain before this authority boundary. + +Positive consequence: same-version metadata edits, changed classifier inputs and altered evaluated predictions invalidate the relevant binding. Negative consequence: prior source digests and approval formats are incompatible; no automatic backfill or transfer of approval is allowed. Captured source and typed input also consume memory until classification finishes. Private Rust fields alone were not selected as the approval solution because they cannot authenticate a deserialized report; storing another full source snapshot was unnecessary for proposal binding. No new authority service, external dependency, or repository is introduced. + +The regressions in `raw_provider_snapshot_binding.rs`, `golden_set_integrity_contract.rs` and `snapshot_content_binding.rs` exercise the two existing PR #10 findings, including failed intermediate designs. Source repair is distinct from hosted exact-head checks, independent review, protected integration and externally approved live classification. The dependent review/finalization stack must adopt the required digest and reject old approval JSON before promotion. ADR status remains Proposed. + - A complete snapshot can be audited and replayed without changing the research library. - Rule evidence and explicit abstention reasons are visible; automated classification is not governance approval. - Cross-cutting papers cannot be silently forced into whichever rule family happens to be evaluated first. From e7d4e59f1b55b5954c5f8436527bc96e7ef2fb13 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sat, 5 Sep 2026 15:47:33 +0900 Subject: [PATCH 22/28] docs(research): align quality requirements with dual evidence binding --- docs/PRD.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/PRD.md b/docs/PRD.md index 7864e4ed..05bb8a72 100644 --- a/docs/PRD.md +++ b/docs/PRD.md @@ -58,7 +58,7 @@ All LLM-backed induction uses `contextual-orchestrator`. Model output is untrust Read one immutable Zotero Local API library-version snapshot and propose exactly one research disposition for every top-level bibliographic item. Each proposal retains the item key/version, exact matched metadata values, rule revision, linked child records, and any model receipt. Weak evidence and evidence that matches multiple specific disposition families must abstain into steward review. Duplicate DOI/title identities are review candidates only: intake never merges, deletes, or silently mutates Zotero records. -Evaluate classifier quality only against a steward-reviewed local golden set whose governance receipt is externally verified and bound to the canonical SHA-256 digest of the complete Zotero classification report plus its item-key/item-version coordinates. Abstention is a prediction outcome, never an approved truth label. Evaluation emits the verified library revision, rule revision, opaque snapshot digest, and aggregate counts for exact matches, abstentions, and per-disposition true-positive/predicted/expected totals; it must not copy Zotero keys, reviewer identity, or bibliographic text into the result. +Evaluate classifier quality only against a steward-reviewed local golden set whose governance receipt is externally verified and binds both the complete source/classifier-input snapshot and every current proposal field, in addition to the item-key/item-version coordinates. Same-version changes to unmodeled provider metadata, absent/default fields, classifier inputs, predictions or supporting evidence must invalidate the corresponding binding. Evaluation recomputes proposal identity before contacting governance; a locally changed digest cannot renew an approval. Legacy unbound approvals require reissuance, never automatic backfill. Abstention is a prediction outcome, never an approved truth label. Evaluation emits the verified library revision, rule revision, opaque snapshot and proposal digests, and aggregate counts for exact matches, abstentions, and per-disposition true-positive/predicted/expected totals; it must not copy Zotero keys, reviewer identity, or bibliographic text into the result. ## 6. First vertical slice From 3f2cf556537fe412d310314af22cad2ecac4ca31 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sun, 6 Sep 2026 21:00:19 +0900 Subject: [PATCH 23/28] test(research): expose scope validation and approval binding gaps --- .../tests/golden_set_integrity_contract.rs | 170 ++++++++++++++++++ 1 file changed, 170 insertions(+) diff --git a/crates/conceptweave-zotero/tests/golden_set_integrity_contract.rs b/crates/conceptweave-zotero/tests/golden_set_integrity_contract.rs index 66885b8c..515cb515 100644 --- a/crates/conceptweave-zotero/tests/golden_set_integrity_contract.rs +++ b/crates/conceptweave-zotero/tests/golden_set_integrity_contract.rs @@ -321,3 +321,173 @@ fn malformed_proposal_identities_fail_before_governance() { ); } } + +fn scope_report() -> conceptweave_zotero::ClassificationReport { + classify_snapshot( + "10.0.1".into(), + None, + 42, + vec![ + bibliographic("A", 1, "ontology learning"), + child_note("C", 1, "A"), + child_note("S", 0, ""), + child_note("T", 1, ""), + ], + ) +} + +fn scope_golden(report: &conceptweave_zotero::ClassificationReport) -> ReviewedGoldenSet { + ReviewedGoldenSet { + approval: approval(report, report.snapshot_items.clone()), + labels: vec![GoldenLabel::new("A", Disposition::Generation)], + } +} + +#[test] +fn malformed_source_scope_fails_before_approval_even_with_recomputed_receipt() { + for mutation in [ + "count", + "snapshot_count", + "missing", + "duplicate", + "overlap", + "unknown", + "revision", + "blank", + "top_level_book", + "blank_type", + "proposal_type", + "proposal_blank_type", + "future_snapshot", + "pending_missing", + "pending_extra", + "pending_duplicate", + "child_missing", + "child_extra", + "child_duplicate", + "parent_changed", + ] { + let mut report = scope_report(); + match mutation { + "count" => report.observed_item_count += 1, + "snapshot_count" => { + report.snapshot_items.pop(); + } + "missing" => { + report.unclassified_items.pop(); + } + "duplicate" => report + .unclassified_items + .push(report.unclassified_items[0].clone()), + "overlap" => report.unclassified_items[0].key = "A".into(), + "unknown" => report.unclassified_items[0].key = "unknown".into(), + "revision" => report.unclassified_items[0].version += 1, + "blank" => report.unclassified_items[0].key = " ".into(), + "top_level_book" => report.unclassified_items[1].data.item_type = "book".into(), + "blank_type" => report.unclassified_items[0].data.item_type.clear(), + "proposal_type" => report.classified_items[0].item_type = "attachment".into(), + "proposal_blank_type" => report.classified_items[0].item_type.clear(), + "future_snapshot" => { + report.snapshot_items[0].item_version = 43; + report.classified_items[0].item_version = 43; + } + "pending_missing" => report.pending_source_item_keys.clear(), + "pending_extra" => report.pending_source_item_keys.push("A".into()), + "pending_duplicate" => report.pending_source_item_keys.push("S".into()), + "child_missing" => report.classified_items[0].child_item_keys.clear(), + "child_extra" => report.classified_items[0].child_item_keys.push("S".into()), + "child_duplicate" => report.classified_items[0].child_item_keys.push("C".into()), + "parent_changed" => report.unclassified_items[0].data.parent_item = "missing".into(), + _ => unreachable!(), + } + let golden = scope_golden(&report); + let calls = std::cell::Cell::new(0); + assert_eq!( + evaluate_reviewed_golden_set(&report, &golden, |_| { + calls.set(calls.get() + 1); + true + }), + Err(EvaluationError::InvalidReview), + "mutation {mutation}" + ); + assert_eq!(calls.get(), 0, "mutation {mutation}"); + } +} + +#[test] +fn source_metadata_mutations_invalidate_the_original_approval_before_verification() { + for mutation in [ + "title", + "abstract", + "doi", + "tags", + "collections", + "type", + "parent", + ] { + let mut report = scope_report(); + let golden = scope_golden(&report); + let source = &mut report.unclassified_items[1]; + match mutation { + "title" => source.data.title = "changed evidence".into(), + "abstract" => source.data.abstract_note = "changed evidence".into(), + "doi" => source.data.doi = "10.1/changed".into(), + "tags" => source.data.tags.push(conceptweave_zotero::ItemTag { + tag: "changed".into(), + }), + "collections" => source.data.collections.push("changed".into()), + "type" => source.data.item_type = "attachment".into(), + "parent" => { + source.data.parent_item = "C".into(); + report.pending_source_item_keys = vec!["T".into()]; + } + _ => unreachable!(), + } + let calls = std::cell::Cell::new(0); + assert_eq!( + evaluate_reviewed_golden_set(&report, &golden, |_| { + calls.set(calls.get() + 1); + true + }), + Err(EvaluationError::SnapshotMismatch), + "mutation {mutation}" + ); + assert_eq!(calls.get(), 0); + } +} + +#[test] +fn rewritten_source_scope_receipt_still_requires_independent_approval() { + let mut report = scope_report(); + let mut golden = scope_golden(&report); + let approved = golden.clone(); + report.unclassified_items[1].data.title = "changed evidence".into(); + golden.approval.proposal_digest = classification_proposal_digest(&report); + let calls = std::cell::Cell::new(0); + assert_eq!( + evaluate_reviewed_golden_set(&report, &golden, |candidate| { + calls.set(calls.get() + 1); + candidate == &approved + }), + Err(EvaluationError::UnverifiedApproval) + ); + assert_eq!(calls.get(), 1); +} + +#[test] +fn valid_pending_source_scope_is_order_independent_and_not_additional_paper_labels() { + let mut report = scope_report(); + let golden = scope_golden(&report); + report.unclassified_items.reverse(); + report.pending_source_item_keys.reverse(); + report.snapshot_items.reverse(); + let result = + evaluate_reviewed_golden_set(&report, &golden, |candidate| candidate == &golden).unwrap(); + assert_eq!(result.reviewed_count, 1); + assert_eq!(result.correct_count, 1); + assert_eq!( + classification_proposal_digest(&report), + golden.approval.proposal_digest + ); + assert_eq!(report.pending_source_item_keys.len(), 2); +} From f6735b585022aac1c8ceff86c150d9b64fd77ec2 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sun, 6 Sep 2026 21:06:37 +0900 Subject: [PATCH 24/28] fix: bind reviewed approval to complete source scope Preserve both parent deltas. Validate report partitions before governance, reuse bounded pending traversal, and require v2 approval binding for unclassified metadata. No source writes or approval issuance. --- crates/conceptweave-zotero/src/lib.rs | 130 ++++++++++++++++++++++---- 1 file changed, 110 insertions(+), 20 deletions(-) diff --git a/crates/conceptweave-zotero/src/lib.rs b/crates/conceptweave-zotero/src/lib.rs index 14c17c2c..b2e3a966 100644 --- a/crates/conceptweave-zotero/src/lib.rs +++ b/crates/conceptweave-zotero/src/lib.rs @@ -289,16 +289,25 @@ pub fn classification_snapshot_digest(report: &ClassificationReport) -> String { /// Computes the versioned SHA-256 identity of the report's current proposals. /// /// Every proposal field is covered, including its prediction, evidence, and item -/// revision. Records are sorted by item key and revision so page ordering does +/// revision, plus unclassified metadata and pending source identities. This is +/// not a full-text backup. Records are sorted by item key and revision so ordering does /// not change their identity. No second source snapshot is stored. Governance /// must bind this value when issuing an approval; recomputing it alone grants no /// authority. Evaluation recomputes it rather than trusting report metadata. pub fn classification_proposal_digest(report: &ClassificationReport) -> String { let mut proposals = report.classified_items.iter().collect::>(); proposals.sort_by_key(|item| (&item.item_key, item.item_version)); - let proposal_bytes = - serde_json::to_vec(&("conceptweave-classification-proposals-v1", proposals)) - .expect("classification proposal records contain only JSON-serializable values"); + let mut source_items = report.unclassified_items.iter().collect::>(); + source_items.sort_by_key(|item| (&item.key, item.version)); + let mut pending_keys = report.pending_source_item_keys.iter().collect::>(); + pending_keys.sort(); + let proposal_bytes = serde_json::to_vec(&( + "conceptweave-classification-proposals-v2", + proposals, + source_items, + pending_keys, + )) + .expect("classification proposal records contain only JSON-serializable values"); format!("sha256:{:x}", Sha256::digest(proposal_bytes)) } @@ -370,6 +379,75 @@ impl fmt::Display for EvaluationError { impl std::error::Error for EvaluationError {} +/// Checks that every observed item belongs to exactly one report partition. +/// +/// Child links and unresolved source keys are recomputed from preserved metadata. +/// Orphans and disconnected cycles remain valid pending evidence. This checks +/// internal consistency, not source authenticity or independent approval. +pub fn validate_classification_report( + report: &ClassificationReport, +) -> Result<(), EvaluationError> { + let invalid = EvaluationError::InvalidReview; + if report.observed_item_count != report.snapshot_items.len() + || report + .classified_items + .len() + .checked_add(report.unclassified_items.len()) + != Some(report.observed_item_count) + { + return Err(invalid); + } + let mut remaining_items = BTreeMap::new(); + for item in &report.snapshot_items { + if item.item_key.trim().is_empty() + || item.item_version > report.library_version + || remaining_items + .insert(item.item_key.as_str(), item.item_version) + .is_some() + { + return Err(invalid); + } + } + let children = child_index(&report.unclassified_items); + for item in &report.classified_items { + let mut reported_children = item.child_item_keys.clone(); + reported_children.sort(); + let mut actual_children = children.get(&item.item_key).cloned().unwrap_or_default(); + actual_children.sort(); + if item.item_type.trim().is_empty() + || matches!( + item.item_type.as_str(), + "attachment" | "note" | "annotation" + ) + || remaining_items.remove(item.item_key.as_str()) != Some(item.item_version) + || reported_children != actual_children + { + return Err(invalid); + } + } + for item in &report.unclassified_items { + if item.data.item_type.trim().is_empty() + || is_bibliographic(item) + || remaining_items.remove(item.key.as_str()) != Some(item.version) + { + return Err(invalid); + } + } + let mut reported_pending = report.pending_source_item_keys.clone(); + reported_pending.sort(); + if !remaining_items.is_empty() + || reported_pending + != pending_source_keys( + &report.classified_items, + &report.unclassified_items, + children, + ) + { + return Err(invalid); + } + Ok(()) +} + /// Evaluates reviewed labels without copying item identities into the result. /// /// Structural, source, proposal, and label validation run before governance is @@ -384,6 +462,7 @@ pub fn evaluate_reviewed_golden_set( where F: FnOnce(&ReviewedGoldenSet) -> bool, { + validate_classification_report(report)?; if golden.approval.receipt_id.trim().is_empty() || golden.approval.reviewer_subject.trim().is_empty() || golden.labels.is_empty() @@ -801,7 +880,7 @@ pub fn classify_snapshot( let snapshot_bytes = serde_json::to_vec(&(SNAPSHOT_DIGEST_DOMAIN, snapshot_records)) .expect("Zotero snapshot items contain only JSON-compatible values"); let snapshot_digest = format!("sha256:{:x}", Sha256::digest(snapshot_bytes)); - let mut children = child_index(&items); + let children = child_index(&items); let bibliographic: Vec<&ZoteroItem> = items.iter().filter(|item| is_bibliographic(item)).collect(); let duplicate_candidates = duplicate_candidates(&bibliographic); @@ -814,20 +893,8 @@ pub fn classify_snapshot( .into_iter() .filter(|item| !is_bibliographic(item)) .collect(); - let mut pending_source_item_keys: BTreeSet<_> = unclassified_items - .iter() - .map(|item| item.key.clone()) - .collect(); - let mut parent_item_keys: Vec<_> = classified_items - .iter() - .map(|item| item.item_key.clone()) - .collect(); - while let Some(parent_item_key) = parent_item_keys.pop() { - for child_item_key in children.remove(&parent_item_key).unwrap_or_default() { - pending_source_item_keys.remove(&child_item_key); - parent_item_keys.push(child_item_key); - } - } + let pending_source_item_keys = + pending_source_keys(&classified_items, &unclassified_items, children); ClassificationReport { zotero_version, @@ -841,11 +908,34 @@ pub fn classify_snapshot( snapshot_digest, classified_items, unclassified_items, - pending_source_item_keys: pending_source_item_keys.into_iter().collect(), + pending_source_item_keys, duplicate_candidates, } } +fn pending_source_keys( + classified_items: &[ClassifiedItem], + unclassified_items: &[ZoteroItem], + mut children: BTreeMap>, +) -> Vec { + let mut pending_source_item_keys: BTreeSet<_> = unclassified_items + .iter() + .map(|item| item.key.clone()) + .collect(); + let mut parent_item_keys: Vec<_> = classified_items + .iter() + .map(|item| item.item_key.clone()) + .collect(); + while let Some(parent_item_key) = parent_item_keys.pop() { + for child_item_key in children.remove(&parent_item_key).unwrap_or_default() { + pending_source_item_keys.remove(&child_item_key); + parent_item_keys.push(child_item_key); + } + } + + pending_source_item_keys.into_iter().collect() +} + fn is_bibliographic(item: &ZoteroItem) -> bool { item.data.parent_item.is_empty() && !matches!( From c61555c8e3ae0fe682373ce90454751303983974 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sun, 6 Sep 2026 21:09:28 +0900 Subject: [PATCH 25/28] docs: record source scope admission evidence and remaining gates --- CHANGELOG.md | 1 + docs/adr/0006-zotero-research-intake.md | 10 ++++++++++ docs/product-technical-gap-baseline.md | 6 ++++++ 3 files changed, 17 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6213394a..5c625695 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,7 @@ All notable changes to ConceptWeave are documented here. ### Fixed +- Research evaluation rejects incomplete source inventories and invalidates prior approvals when retained source metadata changes. - Research reports retain standalone files and notes that previously disappeared from the classification view, and flag sources whose parent relationships remain unresolved. - Zotero research intake rejects a read whose records claim revisions newer than the library being observed, without dropping papers or changing their recorded revisions. - Zotero research intake rejects incomplete or late results after a five-minute read budget, even when individual pages arrive within their request limits. diff --git a/docs/adr/0006-zotero-research-intake.md b/docs/adr/0006-zotero-research-intake.md index 023bd45f..efcb8996 100644 --- a/docs/adr/0006-zotero-research-intake.md +++ b/docs/adr/0006-zotero-research-intake.md @@ -60,6 +60,16 @@ This change does not reconcile a source, validate arbitrary offline input, retai - Human review remains necessary for ambiguous records and every duplicate merge. - Zotero 9 cannot apply approved collection/tag changes automatically. +### September 6 source-scope admission amendment (Proposed) + +PR #10 is the earliest evaluation owner. Its previous head `4bb633305b04a1dd4c4ce526806c9469bcb79fd3` was normally merged with producer `51c7df6d03f072449422fd58ca24b2f9d6026f07`, preserving raw-input binding and all inventory fields. RED `3f2cf55` reproduced three authority failures: omitted scope accepted with a recomputed receipt, changed unclassified metadata accepted under the original receipt, and a rewritten receipt reusing independent approval. + +We selected one public structural validator before evaluation governance, reusing the producer's bounded ancestry traversal. Counts, unique snapshot identities/revisions, disjoint and complete partitions, record types, direct children and recomputed pending identities must agree. Orphans and disconnected cycles remain pending evidence, not extra bibliographic labels. This supersedes the earlier proposal-only approval digest with `conceptweave-classification-proposals-v2`, covering sorted proposals, projected unclassified metadata and pending identities. Existing v1 approvals require independent reapproval; recomputation is not migration authority. + +Guards only in later worksheet commands leave direct evaluation exposed. Another copied traversal or utility repository is unnecessary. The cost is an incompatible approval digest and another bounded validation pass. Snapshot coordinates here contain key/version only: internal consistency and independently approved scope binding do not prove original parent/type authenticity. Projected metadata is not a note/PDF backup. Later restoration, worksheet, duplicate and write owners must adopt the validator and required fields without weakening full-text capture bindings. + +GREEN source `f6735b585022aac1c8ceff86c150d9b64fd77ec2` passed 68 tests across 14 unfiltered workspace suites (including two doctests), up from 64 after parent integration and 58 before it. Strict Clippy passed. Full coverage, independent source review, descendant integration, hosted exact-head checks and protected merge remain separate gates. No genuine decision, approval or Zotero write was issued. Visual inspection was attempted again, but the Mac was locked; no fresh screenshot verification is claimed. + ## Alternatives considered - First-match classification was rejected because FR-9 requires ambiguous evidence to abstain rather than acquire an arbitrary priority-based disposition. diff --git a/docs/product-technical-gap-baseline.md b/docs/product-technical-gap-baseline.md index 5405f6de..fe5917b0 100644 --- a/docs/product-technical-gap-baseline.md +++ b/docs/product-technical-gap-baseline.md @@ -54,6 +54,12 @@ Protected central source is `.github/main@b5efbc2762e472e4a380b0503b1f050f76fbb0 - `.github#1873@41935494aa234eb458f1cc08f006daaa278b9760` is closed/unmerged because repository dependency-graph initialization, not its retry/sleep source delta, was the verified root cause of the observed public-repository 403. - #35 remains an exact consumer canary for current runner admission and Dependency Review behavior. Its central workflows are queued, so no protected recovery or dependency-review success is inferred from repository settings alone. +## September 6 source-scope admission checkpoint + +PR #10 source `f6735b585022aac1c8ceff86c150d9b64fd77ec2` repairs evaluation admission and independently approved scope binding after normal integration of producer `51c7df6d03f072449422fd58ca24b2f9d6026f07`. RED `3f2cf55` failed three new integrity tests; GREEN passed 68 tests/14 unfiltered suites and strict Clippy. See the Proposed [ADR 0006 amendment](adr/0006-zotero-research-intake.md). This local result does not establish hosted GREEN, protection-compliant merge, release, downstream adoption or full coverage. + +Remaining work: mandatory adoption by restoration, worksheet, duplicate and write consumers without empty defaults or weakened full-text binding. Genuine reviewed decisions and approvals remain 0/3,715 bibliographic proposals, with four additional standalone sources unresolved. Native visual inspection remains unverified at this checkpoint because the Mac was locked. No UI change or new utility repository was needed. + ## P0 product gaps 1. **Concrete Source Observation adapter** — maintained Rust PostgreSQL driver behind `conceptweave-source-port`; adapter-local registry/credential resolution; explicit read-only session/transaction; exact schema allowlist; total operation and statement deadlines; cancellation plus row/byte/concurrency budgets; complete immutable snapshot or fail closed; source-disappearance handling; deterministic replay against a frozen anonymized GRC-shaped fixture. From d1344c718bc477679f545f03e8e01ea6a9b88a13 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sun, 6 Sep 2026 21:14:13 +0900 Subject: [PATCH 26/28] test: cover legacy approval and unresolved source scope --- .../tests/golden_set_integrity_contract.rs | 51 +++++++++++++++++++ 1 file changed, 51 insertions(+) diff --git a/crates/conceptweave-zotero/tests/golden_set_integrity_contract.rs b/crates/conceptweave-zotero/tests/golden_set_integrity_contract.rs index 515cb515..7ffa4f32 100644 --- a/crates/conceptweave-zotero/tests/golden_set_integrity_contract.rs +++ b/crates/conceptweave-zotero/tests/golden_set_integrity_contract.rs @@ -2,7 +2,58 @@ use conceptweave_zotero::{ Disposition, EvaluationError, GoldenLabel, GoldenSetApproval, ItemData, ReviewedGoldenSet, SnapshotItemRevision, ZoteroItem, classification_proposal_digest, classification_snapshot_digest, classify_snapshot, evaluate_reviewed_golden_set, + validate_classification_report, }; +use sha2::{Digest, Sha256}; + +#[test] +fn legacy_proposal_receipt_is_rejected_without_calling_governance() { + let report = scope_report(); + let mut golden = scope_golden(&report); + let mut proposals = report.classified_items.iter().collect::>(); + proposals.sort_by_key(|item| (&item.item_key, item.item_version)); + let old_bytes = + serde_json::to_vec(&("conceptweave-classification-proposals-v1", proposals)).unwrap(); + golden.approval.proposal_digest = format!("sha256:{:x}", Sha256::digest(old_bytes)); + assert_eq!( + evaluate_reviewed_golden_set(&report, &golden, |_| panic!( + "legacy receipt reached governance" + )), + Err(EvaluationError::SnapshotMismatch) + ); +} + +#[test] +fn empty_and_unresolved_scope_remain_valid_without_becoming_reviewed_papers() { + for (items, pending) in [ + (vec![], vec![]), + (vec![child_note("S", 0, "")], vec!["S"]), + (vec![child_note("C", 1, "MISSING")], vec!["C"]), + ( + vec![child_note("C", 1, "D"), child_note("D", 1, "C")], + vec!["C", "D"], + ), + (vec![child_note("C", 1, "C")], vec!["C"]), + ] { + let report = classify_snapshot("10.0.1".into(), None, 42, items); + assert_eq!(validate_classification_report(&report), Ok(())); + assert_eq!(report.pending_source_item_keys, pending); + assert!(report.classified_items.is_empty()); + } +} + +#[test] +fn blank_snapshot_identity_is_rejected_before_governance() { + let mut report = scope_report(); + report.snapshot_items[0].item_key = " \t\n".into(); + let golden = scope_golden(&report); + assert_eq!( + evaluate_reviewed_golden_set(&report, &golden, |_| panic!( + "invalid identity reached governance" + )), + Err(EvaluationError::InvalidReview) + ); +} fn bibliographic(key: &str, version: u64, title: &str) -> ZoteroItem { ZoteroItem { From 8ccb0d5b3d7705786b6c40c3bcf5a10ff32046d9 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sun, 6 Sep 2026 21:15:11 +0900 Subject: [PATCH 27/28] refactor: remove identity checks subsumed by shared admission The entry validator already checks unique snapshot and proposal identities. Equal partition count plus successful unique coordinate consumption proves no leftover source. Retain approved receipt checks and all malformed-report regressions; remove no authority boundary. --- crates/conceptweave-zotero/src/lib.rs | 35 ++++++--------------------- 1 file changed, 8 insertions(+), 27 deletions(-) diff --git a/crates/conceptweave-zotero/src/lib.rs b/crates/conceptweave-zotero/src/lib.rs index b2e3a966..9c17a7d6 100644 --- a/crates/conceptweave-zotero/src/lib.rs +++ b/crates/conceptweave-zotero/src/lib.rs @@ -435,13 +435,13 @@ pub fn validate_classification_report( } let mut reported_pending = report.pending_source_item_keys.clone(); reported_pending.sort(); - if !remaining_items.is_empty() - || reported_pending - != pending_source_keys( - &report.classified_items, - &report.unclassified_items, - children, - ) + // Equal partition size and one successful removal per record prove completeness. + if reported_pending + != pending_source_keys( + &report.classified_items, + &report.unclassified_items, + children, + ) { return Err(invalid); } @@ -477,21 +477,13 @@ where .iter() .cloned() .collect::>(); - let report_keys = report - .snapshot_items - .iter() - .map(|item| item.item_key.as_str()) - .collect::>(); let approved_snapshot = golden .approval .snapshot_items .iter() .cloned() .collect::>(); - if report_snapshot.len() != report.snapshot_items.len() - || report_keys.len() != report.snapshot_items.len() - || approved_snapshot.len() != golden.approval.snapshot_items.len() - { + if approved_snapshot.len() != golden.approval.snapshot_items.len() { return Err(EvaluationError::InvalidReview); } if golden.approval.library_version != report.library_version @@ -507,17 +499,6 @@ where .iter() .map(|item| (item.item_key.as_str(), item.proposed_disposition)) .collect::>(); - if classified.len() != report.classified_items.len() - || report.classified_items.iter().any(|item| { - item.item_key.trim().is_empty() - || !report_snapshot.contains(&SnapshotItemRevision { - item_key: item.item_key.clone(), - item_version: item.item_version, - }) - }) - { - return Err(EvaluationError::InvalidReview); - } if golden.approval.proposal_digest != classification_proposal_digest(report) { return Err(EvaluationError::SnapshotMismatch); } From fdf8b8d70c05bcb76c55cb6336c9bf31b5e42ce4 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sun, 6 Sep 2026 21:17:40 +0900 Subject: [PATCH 28/28] docs: bind scope admission contracts to verified coverage --- docs/PRD.md | 2 ++ docs/TRD.md | 4 +++- docs/UML.md | 4 ++++ docs/adr/0006-zotero-research-intake.md | 2 ++ docs/product-technical-gap-baseline.md | 2 ++ 5 files changed, 13 insertions(+), 1 deletion(-) diff --git a/docs/PRD.md b/docs/PRD.md index 498614b7..2d7d6e54 100644 --- a/docs/PRD.md +++ b/docs/PRD.md @@ -56,6 +56,8 @@ All LLM-backed induction uses `contextual-orchestrator`. Model output is untrust ### FR-9 Research evidence intake +Evaluation must reject omitted or inconsistent retained sources before requesting approval. Changing retained source metadata invalidates prior approval, even when paper predictions stay unchanged. Empty or unresolved-only inventories remain auditable evidence and must not acquire reviewed-paper counts. + Preserve every observed source, including standalone files and notes outside the bibliographic proposals. Keep unresolved source relationships visible instead of treating a completed bibliography worksheet as a completed library review. All standalone sources and records without a valid path to a bibliographic parent need explicit reconciliation; notes, files and annotations must not acquire paper labels from their titles. Retraction and correction evidence remains separate from topic classification and approval. The current producer retains this inventory; downstream reconciliation, completion admission and independent governance remain required, not implemented by inventory generation alone. Library reads must finish within a bounded observation window or fail visibly without returning a partial classification. Slowly arriving pages cannot keep a run open indefinitely, and missing time budget must not be handled by silently dropping papers. diff --git a/docs/TRD.md b/docs/TRD.md index e7f916c7..710b52f1 100644 --- a/docs/TRD.md +++ b/docs/TRD.md @@ -61,7 +61,7 @@ Evaluation must separate extraction recall, semantic correctness, structural cor `ClassificationReport.unclassified_items` retains every input record excluded from bibliographic classification, using the existing `ZoteroItem` metadata projection. Bibliographic proposals and this inventory are disjoint and together account for the observed record count on reader-admitted input. The existing child index is consumed once from bibliographic roots; records never reached remain in sorted `pending_source_item_keys`, including standalone roots, their descendants, orphan trees and cycles. The traversal is iterative, uses no new dependency and costs O(n log n) time/O(n) auxiliary space. It does not validate arbitrary offline input or preserve note bodies, attachment-specific fields and unknown provider JSON. -Consumer requirement, still pending forward integration: require both inventory fields rather than defaulting absent legacy fields to empty; validate exact snapshot identity/version/parent complement and recompute pending keys before any review, duplicate evaluation or write verifier. Keep bibliographic progress distinct from whole-library completion. An empty pending list is only ancestry accounting, never semantic approval. Full-text report-digest changes require fresh bound verification, not capture rewriting; metadata proposal-only digests do not implicitly bind the new inventory. See [source-scope evidence and integration map](doctoring/zotero_source_scope.md). +Evaluation now calls `validate_classification_report` before governance: counts, disjoint complete key/version partitions, types, direct children and recomputed pending keys must agree. Equal partition count and one successful removal of each unique snapshot coordinate prove completeness. This owner has no original parent/type snapshot coordinates, so consistency is not source authentication. Later restoration, review, duplicate and write consumers must adopt this guard and require both inventory fields without empty legacy defaults. Keep bibliographic progress distinct from whole-library completion. Full-text report-digest changes require fresh bound verification, not capture rewriting. See [source-scope evidence and integration map](doctoring/zotero_source_scope.md). The shared live reader rejects empty or whitespace-only item keys before accumulating a page or requesting another one. It preserves valid keys verbatim; this does not add a new provider key-format restriction or certify arbitrary offline classifier input. @@ -75,6 +75,8 @@ Every top-level bibliographic record receives exactly one proposed disposition. Golden-set evaluation accepts only a governance receipt verified with the complete reviewed set by a caller-owned authorization boundary. Its library version, rule revision, canonical SHA-256 content digest, and every observed parent/child item-key/item-version identity must bind the classification report. The snapshot digest covers every raw Zotero item in canonical key order. A separate required `proposal_digest` binds every field of every proposed item, including predictions, supporting evidence, and proposals outside a reviewed sample. `classification_proposal_digest` computes SHA-256 over compact JSON containing the `conceptweave-classification-proposals-v1` domain marker and proposal records sorted by item key and revision. It uses the current records, not a report's self-declared digest or a second stored source snapshot. Governance must issue and independently verify both digests together with the labels; a locally recomputed replacement digest cannot renew an old approval. Legacy approvals missing this field fail closed and require reissuance, not automatic backfill. +The proposal-only v1 format above describes the previous receipt contract. The current source-scope amendment supersedes it with `conceptweave-classification-proposals-v2`: compact JSON binds the marker, sorted proposals, key/version-sorted projected unclassified records, and sorted pending keys. Old v1 receipts fail before governance; locally rewritten digests cannot renew approval. This is not a lossless full-text digest. + Structural, source, proposal, and label checks precede the external verifier. Blank, duplicate, unknown, stale, content-mismatched, prediction-mismatched, label-mismatched, or abstention-as-truth inputs fail closed. The aggregate result retains the verified library version, rule revision, and opaque snapshot/proposal digests, but contains no item keys, reviewer identity, or bibliographic text. Production authorization remains Keyverse/governance-owned; this crate passes the complete reviewed labels and approval bindings to that boundary instead of minting authority. Provider deserialization captures each complete JSON object before projecting metadata. Snapshot hashing serializes the domain marker `conceptweave-zotero-snapshot-v2` followed by key-ordered pairs of that canonical source JSON and the actual typed classifier input. Unknown nested fields, array order, and omitted-versus-explicit default fields remain bound; changing a typed input after decoding also changes the digest. Synthetic offline typed items have no captured provider object and bind an explicit absent-source value alongside their typed input. Earlier reduced-content digests remain historical evidence and cannot establish this complete-content contract; regenerate the report and review artifacts and obtain fresh approval before any release or approved write. diff --git a/docs/UML.md b/docs/UML.md index 714c3004..0c1a99b4 100644 --- a/docs/UML.md +++ b/docs/UML.md @@ -58,5 +58,9 @@ sequenceDiagram Intake->>Report: write proposals, complete inventory and unresolved source keys Note over Report,Steward: Pending sources prevent a whole-library completion claim; inventory is not approval Report->>Steward: review dispositions and merge candidates + Steward->>Intake: reviewed labels and independently issued receipt + Intake->>Intake: validate complete partitions and recompute pending ancestry + Intake->>Intake: verify v2 proposal and retained-source binding + Note over Intake,Steward: Only locally valid reports reach independent governance verification Intake-->>Zotero: no mutation ``` diff --git a/docs/adr/0006-zotero-research-intake.md b/docs/adr/0006-zotero-research-intake.md index efcb8996..2ab03175 100644 --- a/docs/adr/0006-zotero-research-intake.md +++ b/docs/adr/0006-zotero-research-intake.md @@ -72,6 +72,8 @@ GREEN source `f6735b585022aac1c8ceff86c150d9b64fd77ec2` passed 68 tests across 1 ## Alternatives considered +The follow-up `8ccb0d5b3d7705786b6c40c3bcf5a10ff32046d9` removes duplicate evaluator identity checks subsumed by the entry validator. Equal partition lengths plus one successful removal per unique coordinate prove no leftover source; testing impossible duplicate branches would require bypassing the real entry boundary. Existing malformed-report cases remain, and `d1344c7` adds legacy-v1 rejection and empty/orphan/cycle/blank-identity regressions. The unchanged pinned coverage gate now passes all normalized owned regions and branches; raw instantiated gaps remain explicitly reported in the Gap baseline. No coverage exclusion, dependency or authority service was added. + - First-match classification was rejected because FR-9 requires ambiguous evidence to abstain rather than acquire an arbitrary priority-based disposition. - Hard-coding Zotero schema 42 was rejected because the documented compatibility contract is API v3; schema revision is instead recorded and checked for within-read drift. - Direct Zotero 9 writes were rejected because the supported Local API write capability is Zotero 10+ only. diff --git a/docs/product-technical-gap-baseline.md b/docs/product-technical-gap-baseline.md index fe5917b0..0d167f12 100644 --- a/docs/product-technical-gap-baseline.md +++ b/docs/product-technical-gap-baseline.md @@ -56,6 +56,8 @@ Protected central source is `.github/main@b5efbc2762e472e4a380b0503b1f050f76fbb0 ## September 6 source-scope admission checkpoint +Follow-up runtime `8ccb0d5b3d7705786b6c40c3bcf5a10ff32046d9` adds explicit legacy receipt/empty/orphan/cycle/blank-identity regressions and removes only checks proven redundant after shared admission. Local workspace tests: 71/14 unfiltered suites including two doctests; strict Clippy, warnings-denied rustdoc, release build, format and diff checks passed. Independent read-only review reran 15 integrity tests and found no actionable defect. Unchanged pinned coverage gate passed: functions 136/136, normalized owned regions 1,037/1,037 and branches 174/174. Raw lines 1,448/1,451, regions 2,268/2,279 and branches 169/174 remain below 100%; no raw full-coverage claim is made. Logs: `/tmp/conceptweave-admission-coverage-baseline.log`, `/tmp/conceptweave-admission-coverage-green.log`, `/tmp/conceptweave-admission-boundary-tests.log`. + PR #10 source `f6735b585022aac1c8ceff86c150d9b64fd77ec2` repairs evaluation admission and independently approved scope binding after normal integration of producer `51c7df6d03f072449422fd58ca24b2f9d6026f07`. RED `3f2cf55` failed three new integrity tests; GREEN passed 68 tests/14 unfiltered suites and strict Clippy. See the Proposed [ADR 0006 amendment](adr/0006-zotero-research-intake.md). This local result does not establish hosted GREEN, protection-compliant merge, release, downstream adoption or full coverage. Remaining work: mandatory adoption by restoration, worksheet, duplicate and write consumers without empty defaults or weakened full-text binding. Genuine reviewed decisions and approvals remain 0/3,715 bibliographic proposals, with four additional standalone sources unresolved. Native visual inspection remains unverified at this checkpoint because the Mac was locked. No UI change or new utility repository was needed.