From bacb78da8ad594ca20d55b38c99c79a756232f8a Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sat, 5 Sep 2026 02:22:36 +0900 Subject: [PATCH 01/12] test(zotero): require complete steward review coverage --- .../tests/golden_set_evaluation.rs | 30 ++++++++++++++++++- 1 file changed, 29 insertions(+), 1 deletion(-) diff --git a/crates/conceptweave-zotero/tests/golden_set_evaluation.rs b/crates/conceptweave-zotero/tests/golden_set_evaluation.rs index e416abe3..24c30e8e 100644 --- a/crates/conceptweave-zotero/tests/golden_set_evaluation.rs +++ b/crates/conceptweave-zotero/tests/golden_set_evaluation.rs @@ -1,7 +1,7 @@ use conceptweave_zotero::{ Disposition, EvaluationError, GoldenLabel, GoldenSetApproval, ItemData, ReviewedGoldenSet, SnapshotItemRevision, ZoteroItem, classification_snapshot_digest, classify_snapshot, - evaluate_reviewed_golden_set, + evaluate_complete_reviewed_classification, evaluate_reviewed_golden_set, }; fn item(key: &str, title: &str) -> ZoteroItem { @@ -20,6 +20,34 @@ fn item(key: &str, title: &str) -> ZoteroItem { } } +#[test] +fn complete_review_requires_one_steward_label_per_bibliographic_item() { + let report = report(); + assert_eq!( + evaluate_complete_reviewed_classification( + &report, + &golden(vec![ + GoldenLabel::new("A", Disposition::Generation), + GoldenLabel::new("B", Disposition::EvaluationGovernance), + ]), + verify_synthetic_approval, + ), + Err(EvaluationError::IncompleteReview) + ); + + let evaluation = evaluate_complete_reviewed_classification( + &report, + &golden(vec![ + GoldenLabel::new("A", Disposition::Generation), + GoldenLabel::new("B", Disposition::EvaluationGovernance), + GoldenLabel::new("C", Disposition::OutOfScope), + ]), + verify_synthetic_approval, + ) + .unwrap(); + assert_eq!(evaluation.reviewed_count, 3); +} + fn report() -> conceptweave_zotero::ClassificationReport { classify_snapshot( "9.0.6".into(), From 65c107d4a91203003912291a5ef0e3a4794bf1ab Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sat, 5 Sep 2026 02:23:02 +0900 Subject: [PATCH 02/12] feat(zotero): verify complete steward review coverage --- crates/conceptweave-zotero/src/lib.rs | 21 +++++++++++++++++++ .../tests/golden_set_evaluation.rs | 1 + 2 files changed, 22 insertions(+) diff --git a/crates/conceptweave-zotero/src/lib.rs b/crates/conceptweave-zotero/src/lib.rs index 66318bdf..3fbd0bfd 100644 --- a/crates/conceptweave-zotero/src/lib.rs +++ b/crates/conceptweave-zotero/src/lib.rs @@ -1136,6 +1136,8 @@ pub enum EvaluationError { UnknownItem, /// A reviewed key occurs more than once. DuplicateItem, + /// The reviewed labels do not cover every bibliographic item. + IncompleteReview, } impl fmt::Display for EvaluationError { @@ -1149,12 +1151,31 @@ impl fmt::Display for EvaluationError { } Self::UnknownItem => "golden set contains an item absent from the report", Self::DuplicateItem => "golden set contains a duplicate item", + Self::IncompleteReview => { + "complete review must label every bibliographic item exactly once" + } }) } } impl std::error::Error for EvaluationError {} +/// Evaluates a steward review only when it covers every bibliographic item. +pub fn evaluate_complete_reviewed_classification( + report: &ClassificationReport, + golden: &ReviewedGoldenSet, + verify_approval: F, +) -> Result +where + F: FnOnce(&ReviewedGoldenSet) -> bool, +{ + let evaluation = evaluate_reviewed_golden_set(report, golden, verify_approval)?; + if evaluation.reviewed_count != report.classified_items.len() { + return Err(EvaluationError::IncompleteReview); + } + Ok(evaluation) +} + /// Evaluates reviewed labels without copying item identities into the result. pub fn evaluate_reviewed_golden_set( report: &ClassificationReport, diff --git a/crates/conceptweave-zotero/tests/golden_set_evaluation.rs b/crates/conceptweave-zotero/tests/golden_set_evaluation.rs index 24c30e8e..3af9228b 100644 --- a/crates/conceptweave-zotero/tests/golden_set_evaluation.rs +++ b/crates/conceptweave-zotero/tests/golden_set_evaluation.rs @@ -262,6 +262,7 @@ fn reviewed_golden_set_rejects_stale_unknown_and_duplicate_labels() { (EvaluationError::InvalidExpectedDisposition, "abstention"), (EvaluationError::UnknownItem, "absent"), (EvaluationError::DuplicateItem, "duplicate"), + (EvaluationError::IncompleteReview, "every bibliographic"), ] { assert!(error.to_string().contains(fragment)); } From 6e8925bd13c03d94e6234048fd7b767a3e1ca234 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sat, 5 Sep 2026 02:24:12 +0900 Subject: [PATCH 03/12] docs(zotero): define full review completion KPI --- CHANGELOG.md | 1 + docs/PRD.md | 2 +- docs/TRD.md | 2 +- docs/UML.md | 2 ++ docs/adr/0006-zotero-research-intake.md | 2 +- docs/product-technical-gap-baseline.md | 2 ++ 6 files changed, 8 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ec59d9a3..c05128bd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,6 +14,7 @@ All notable changes to ConceptWeave are documented here. - Read-only delayed reconciliation receipts for indeterminate Zotero rollback operations. - Minimal, nonduplicated local abstract context for Zotero items that require steward classification. - Owner-only file permissions for sensitive local Zotero classification reports. +- A complete-review evaluator that rejects partial steward labels as full reclassification evidence. ### Security diff --git a/docs/PRD.md b/docs/PRD.md index 14d6346f..561699de 100644 --- a/docs/PRD.md +++ b/docs/PRD.md @@ -66,7 +66,7 @@ For execute-mode plans, the runtime must preflight every item before the first w The Zotero 10+ adapter can accept a caller-owned API key and server identity at runtime or consume one successful, user-approved Local API authorization. Authorization sends one bounded application name and the expected server identity to the fixed loopback endpoint; only a same-server bounded response that explicitly reports denial is classified as the user's decision. Denial and rate limiting return immediately without another prompt or automatic retry. The private 32-character key is neither serializable nor printable. Authorization, read, and write responses bind to the expected server before status classification; writes name expired authorization and matching-server stale preconditions separately. Thin public execution boundaries connect the adapter to the reviewed write and rollback cores without duplicating mutation logic. Synthetic transport evidence does not satisfy AC6's approved live Zotero 10 authorization, write, and rollback requirement. -Evaluate classifier quality only against a steward-reviewed local golden set whose governance receipt is externally verified and bound to the canonical SHA-256 digest of the complete Zotero classification report plus its item-key/item-version coordinates. Abstention is a prediction outcome, never an approved truth label. Evaluation emits the verified library revision, rule revision, opaque snapshot digest, and aggregate counts for exact matches, abstentions, and per-disposition true-positive/predicted/expected totals; it must not copy Zotero keys, reviewer identity, or bibliographic text into the result. +Evaluate classifier quality only against a steward-reviewed local golden set whose governance receipt is externally verified and bound to the canonical SHA-256 digest of the complete Zotero classification report plus its item-key/item-version coordinates. Abstention is a prediction outcome, never an approved truth label. Evaluation emits the verified library revision, rule revision, opaque snapshot digest, and aggregate counts for exact matches, abstentions, and per-disposition true-positive/predicted/expected totals; it must not copy Zotero keys, reviewer identity, or bibliographic text into the result. A full-reclassification completion result additionally requires exactly one non-abstention steward label for every top-level bibliographic item; a sampled golden set remains valid for quality measurement but cannot prove completion. Every successful classification report includes aggregate evidence for snapshot coverage, proposal coverage, provenance completeness, abstentions, duplicate candidates, disposition totals, and zero unreported failures. ## 6. First vertical slice diff --git a/docs/TRD.md b/docs/TRD.md index e9d99ea2..029ddacf 100644 --- a/docs/TRD.md +++ b/docs/TRD.md @@ -65,7 +65,7 @@ Every top-level bibliographic record receives exactly one proposed disposition. Duplicate review is independent of subject classification. A reviewed decision set must match the exact raw-snapshot digest and complete item-key/item-version coordinates, cover every duplicate candidate exactly once, select one retained key from the connected duplicate component, and pass an external governance verifier. Every operation records all component item revisions, identity mappings before and after canonicalization, and the exact rollback mapping. These mappings affect only downstream identity resolution; Zotero records are neither mutated nor deleted. -Golden-set evaluation accepts only a governance receipt verified with the complete reviewed set by a caller-owned authorization boundary. Its library version, rule revision, canonical SHA-256 content digest, and every observed parent/child item-key/item-version identity must bind the classification report. The digest covers every raw Zotero item in canonical key order. Blank, duplicate, unknown, stale, content-mismatched, label-mismatched, or abstention-as-truth inputs fail closed. The output retains the verified library version, rule revision, and opaque snapshot digest, but contains no item keys, reviewer identity, or bibliographic text. Production authorization remains Keyverse/governance-owned; this crate passes the complete reviewed labels to that boundary instead of minting authority. +Golden-set evaluation accepts only a governance receipt verified with the complete reviewed set by a caller-owned authorization boundary. Its library version, rule revision, canonical SHA-256 content digest, and every observed parent/child item-key/item-version identity must bind the classification report. The digest covers every raw Zotero item in canonical key order. Blank, duplicate, unknown, stale, content-mismatched, label-mismatched, or abstention-as-truth inputs fail closed. The full-reclassification evaluator reuses this contract and additionally requires the reviewed label count to equal the unique classified bibliographic-item count; because the base evaluator already rejects blank, duplicate, and unknown keys, equality proves complete coverage. A sampled golden set can measure quality but cannot satisfy this completion gate. The output retains the verified library version, rule revision, and opaque snapshot digest, but contains no item keys, reviewer identity, or bibliographic text. Production authorization remains Keyverse/governance-owned; this crate passes the complete reviewed labels to that boundary instead of minting authority. A successful classification report carries an `audit_summary` whose snapshot, bibliographic, proposed-disposition, provenance-complete, abstention, duplicate-candidate, failure, and per-disposition counts are derived from the same in-memory immutable snapshot. Zotero item version zero remains a valid observed coordinate for never-synced Zotero 9 records; provenance completeness rejects a missing item key rather than inventing a positive-only version invariant. Reader failures return an error instead of a partial report; therefore a returned report records `failure_count=0` rather than hiding partial failures. The local report can contain titles, tags, matched metadata, and abstention abstracts. It is sensitive steward-review material, remains outside the repository, and is not a publication artifact. diff --git a/docs/UML.md b/docs/UML.md index c055bb4b..bcdafb81 100644 --- a/docs/UML.md +++ b/docs/UML.md @@ -55,6 +55,8 @@ sequenceDiagram Intake->>Intake: classify or abstain; link children; find duplicate candidates Intake->>Report: write proposals and evidence Report->>Steward: review dispositions and merge candidates + Steward->>Intake: verified labels for every bibliographic item + Intake-->>Steward: aggregate completion evidence or incomplete-review failure Steward->>Intake: verified canonical-item decisions Intake->>Report: before/after/rollback identity manifest Report-->>Steward: reversible local mapping; source records preserved diff --git a/docs/adr/0006-zotero-research-intake.md b/docs/adr/0006-zotero-research-intake.md index 4f4dbb74..7019bde6 100644 --- a/docs/adr/0006-zotero-research-intake.md +++ b/docs/adr/0006-zotero-research-intake.md @@ -21,7 +21,7 @@ Matched metadata values are copied into the local-only evidence receipt for repl Duplicate candidates become canonical references only through externally verified steward decisions bound to the raw digest, complete item-key/item-version snapshot, and exact candidate membership. Overlapping candidates form one connected component and must select one component-level canonical item. Every resulting operation retains all component source revisions and complete before/after/rollback key mappings. It changes downstream identity resolution only; classification does not merge, delete, or mutate Zotero source records. -Classifier quality is measured only against local steward-reviewed labels whose complete reviewed set is verified outside this crate and bound to the exact library version, rule revision, canonical SHA-256 raw-snapshot digest, and every observed parent/child item-key/item-version coordinate. `NeedsStewardReview` is an abstention prediction and cannot be approved truth. Evaluation returns the verified revisions and opaque digest with aggregate integer evidence; Zotero keys, reviewer identity, and bibliographic text are omitted. Missing, stale, content- or label-mismatched, unverified, unknown, duplicate, or invalid review identities fail closed. +Classifier quality is measured only against local steward-reviewed labels whose reviewed set is verified outside this crate and bound to the exact library version, rule revision, canonical SHA-256 raw-snapshot digest, and every observed parent/child item-key/item-version coordinate. `NeedsStewardReview` is an abstention prediction and cannot be approved truth. Sampled labels may measure classifier quality, but a full-reclassification completion result requires exactly one approved label for every classified bibliographic item. Evaluation returns the verified revisions and opaque digest with aggregate integer evidence; Zotero keys, reviewer identity, and bibliographic text are omitted. Missing, incomplete, stale, content- or label-mismatched, unverified, unknown, duplicate, or invalid review identities fail closed at the applicable completion boundary. Every successful report includes an aggregate audit summary computed from the same captured snapshot. Zotero 9 item version zero is preserved as a valid never-synced source coordinate, not treated as missing provenance. Partial reads never produce a report, so successful output explicitly records zero failures alongside snapshot, proposal, provenance, abstention, duplicate, and per-disposition totals. The reader fails closed above 50,000 items or 256 MiB of cumulative response bodies, while retaining the 8 MiB per-page bound, finite request timeouts, redirect denial, total-count checks, snapshot-version checks, and duplicate-key detection. Pagination, consistency, resource-budget, and provider-contract behavior are separated from the narrow `ureq` transport so deterministic tests exercise the production reader core rather than excluding the entire reader from coverage. diff --git a/docs/product-technical-gap-baseline.md b/docs/product-technical-gap-baseline.md index e5870a37..2ffa86d2 100644 --- a/docs/product-technical-gap-baseline.md +++ b/docs/product-technical-gap-baseline.md @@ -50,6 +50,8 @@ The 3,658-item abstention queue now preserves each nonempty abstract exactly onc The golden-set evaluation contract now records aggregate precision/recall numerators and denominators, requires an externally verified governance receipt bound to the complete item-key/item-version snapshot, rejects abstention as expected truth, and retains verified revisions plus an opaque snapshot digest so detached metrics remain attributable. Item and reviewer identities stay out of its output. Successful classification reports also carry same-snapshot aggregate coverage, provenance, abstention, duplicate, disposition, and failure evidence. Connected duplicate components now produce a snapshot-bound local review manifest only after external steward verification; every operation retains all component source revisions and before/after/rollback canonical mappings while Zotero records remain unchanged. Reviewed collection/tag changes produce a default-dry-run plan bound to exact server, library, item, rule, digest, and complete metadata preconditions; externally read-only plan state prevents post-validation forgery, automatic-tag type is preserved, and Zotero 9 execute mode is rejected. The injected execution core calls nothing in dry-run mode, preflights every item before a write, stops at the first failure, reconciles a lost or invalid response with a same-boundary read, and emits rollback evidence for every item whose applied state is proven. The generic rollback executor rejects mixed-server evidence before reading, verifies all expected post-write states at one current library version before the first inverse write, follows the receipt's reverse order, advances only from verified writes, and stops with restored, failed, indeterminate, not-attempted, and remaining classifications. Unprovable state is reported as indeterminate with complete operation evidence retained separately and excluded from automatic retry until operator reconciliation. A later read-only reconciliation records the observed state, tolerates unrelated library-version advancement, and emits retry evidence only for an exact unchanged item; restoration metadata at a newer item revision proves current state but not causality. Reuse after restoration fails before writing. A fixed-loopback Zotero 10 adapter supplies stable server-pinned reads and authenticated one-item writes with atomic library/item preconditions, complete collection/tag replacement, and bounded verified responses; thin wrappers reuse both generic executors. Mock fixtures verify these contracts and secret-free failures. Korean, Japanese, Chinese, Vietnamese, Spanish, German, and French ontology-alignment metadata now have explicit fail-closed abstention coverage alongside the existing English positive case; this is safety evidence, not translated classification support. No real precision/recall, duplicate merge, write, or rollback claim exists until a steward supplies reviewed local decisions and a production authorization adapter verifies them. AC6 still requires approved live Zotero 10 write, partial-failure, and rollback evidence. Multilingual rule expansion remains a later evidence-driven change and must not reduce abstention safety. A dedicated utility repository remains unnecessary until an independently released cross-product contract exists. +The completion KPI is now executable: a sampled golden set may measure classifier quality, but it cannot prove that the library was fully reclassified. The full-review boundary returns aggregate completion evidence only when externally verified, snapshot-bound, non-abstention labels cover every unique classified bibliographic item exactly once. The current live baseline therefore remains incomplete at 0/3,715 approved labels rather than treating 57 deterministic proposals or a future sample as steward truth. + 1. **Concrete Source Observation adapter** — maintained Rust PostgreSQL driver behind `conceptweave-source-port`; adapter-local credential resolution; explicit read-only mode; statement timeout, cancellation, row/byte/concurrency budgets; complete immutable snapshot or fail closed; deterministic replay against a frozen anonymized GRC-shaped fixture. 2. **Ontology discovery** — deterministic term/concept/taxonomy/non-taxonomic-relation candidate generation with exact source receipts and abstention for unsupported semantics. 3. **Semantic-layer discovery** — dimensions, measures, grain, units, relationships and physical mappings with deterministic calculation contracts; do not infer business authority from relational structure alone. From 59d05d1ba3fe73f237ccbd1568a43a06aa40fc4c Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sat, 5 Sep 2026 02:24:53 +0900 Subject: [PATCH 04/12] test(zotero): preserve complete review validation errors --- crates/conceptweave-zotero/tests/golden_set_evaluation.rs | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/crates/conceptweave-zotero/tests/golden_set_evaluation.rs b/crates/conceptweave-zotero/tests/golden_set_evaluation.rs index 3af9228b..d118c18f 100644 --- a/crates/conceptweave-zotero/tests/golden_set_evaluation.rs +++ b/crates/conceptweave-zotero/tests/golden_set_evaluation.rs @@ -23,6 +23,14 @@ fn item(key: &str, title: &str) -> ZoteroItem { #[test] fn complete_review_requires_one_steward_label_per_bibliographic_item() { let report = report(); + assert_eq!( + evaluate_complete_reviewed_classification( + &report, + &golden(vec![]), + verify_synthetic_approval, + ), + Err(EvaluationError::InvalidReview) + ); assert_eq!( evaluate_complete_reviewed_classification( &report, From 40c1cc06342b288e6c3f230720a62cae0aaabb82 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sat, 5 Sep 2026 02:26:45 +0900 Subject: [PATCH 05/12] fix(zotero): validate reviews before approval boundary --- crates/conceptweave-zotero/src/lib.rs | 11 ++--- .../tests/golden_set_evaluation.rs | 40 +++++++++++++++---- 2 files changed, 39 insertions(+), 12 deletions(-) diff --git a/crates/conceptweave-zotero/src/lib.rs b/crates/conceptweave-zotero/src/lib.rs index 3fbd0bfd..fb4c5880 100644 --- a/crates/conceptweave-zotero/src/lib.rs +++ b/crates/conceptweave-zotero/src/lib.rs @@ -1169,10 +1169,10 @@ pub fn evaluate_complete_reviewed_classification( where F: FnOnce(&ReviewedGoldenSet) -> bool, { - let evaluation = evaluate_reviewed_golden_set(report, golden, verify_approval)?; - if evaluation.reviewed_count != report.classified_items.len() { + if golden.labels.len() != report.classified_items.len() { return Err(EvaluationError::IncompleteReview); } + let evaluation = evaluate_reviewed_golden_set(report, golden, verify_approval)?; Ok(evaluation) } @@ -1193,9 +1193,6 @@ where { return Err(EvaluationError::InvalidReview); } - if !verify_approval(golden) { - return Err(EvaluationError::UnverifiedApproval); - } let report_snapshot = report .snapshot_items .iter() @@ -1264,6 +1261,10 @@ where } } + if !verify_approval(golden) { + return Err(EvaluationError::UnverifiedApproval); + } + Ok(GoldenSetEvaluation { review_id: golden.approval.receipt_id.clone(), library_version: golden.approval.library_version, diff --git a/crates/conceptweave-zotero/tests/golden_set_evaluation.rs b/crates/conceptweave-zotero/tests/golden_set_evaluation.rs index d118c18f..246e8a1a 100644 --- a/crates/conceptweave-zotero/tests/golden_set_evaluation.rs +++ b/crates/conceptweave-zotero/tests/golden_set_evaluation.rs @@ -22,14 +22,16 @@ fn item(key: &str, title: &str) -> ZoteroItem { #[test] fn complete_review_requires_one_steward_label_per_bibliographic_item() { + use std::cell::Cell; + let report = report(); + let verifier_calls = Cell::new(0); assert_eq!( - evaluate_complete_reviewed_classification( - &report, - &golden(vec![]), - verify_synthetic_approval, - ), - Err(EvaluationError::InvalidReview) + evaluate_complete_reviewed_classification(&report, &golden(vec![]), |_| { + verifier_calls.set(verifier_calls.get() + 1); + true + },), + Err(EvaluationError::IncompleteReview) ); assert_eq!( evaluate_complete_reviewed_classification( @@ -38,10 +40,14 @@ fn complete_review_requires_one_steward_label_per_bibliographic_item() { GoldenLabel::new("A", Disposition::Generation), GoldenLabel::new("B", Disposition::EvaluationGovernance), ]), - verify_synthetic_approval, + |_| { + verifier_calls.set(verifier_calls.get() + 1); + true + }, ), Err(EvaluationError::IncompleteReview) ); + assert_eq!(verifier_calls.get(), 0); let evaluation = evaluate_complete_reviewed_classification( &report, @@ -56,6 +62,26 @@ fn complete_review_requires_one_steward_label_per_bibliographic_item() { assert_eq!(evaluation.reviewed_count, 3); } +#[test] +fn invalid_local_review_never_reaches_the_approval_verifier() { + use std::cell::Cell; + + let verifier_calls = Cell::new(0); + let result = evaluate_reviewed_golden_set( + &report(), + &golden(vec![ + GoldenLabel::new("A", Disposition::Generation), + GoldenLabel::new("A", Disposition::Generation), + ]), + |_| { + verifier_calls.set(verifier_calls.get() + 1); + true + }, + ); + assert_eq!(result, Err(EvaluationError::DuplicateItem)); + assert_eq!(verifier_calls.get(), 0); +} + fn report() -> conceptweave_zotero::ClassificationReport { classify_snapshot( "9.0.6".into(), From 64de274ed151e45f338eac1d57e3a454c72e3286 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sat, 5 Sep 2026 02:27:11 +0900 Subject: [PATCH 06/12] docs(zotero): order review validation before authority --- docs/TRD.md | 2 +- docs/adr/0006-zotero-research-intake.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/TRD.md b/docs/TRD.md index 029ddacf..7f1453f6 100644 --- a/docs/TRD.md +++ b/docs/TRD.md @@ -65,7 +65,7 @@ Every top-level bibliographic record receives exactly one proposed disposition. Duplicate review is independent of subject classification. A reviewed decision set must match the exact raw-snapshot digest and complete item-key/item-version coordinates, cover every duplicate candidate exactly once, select one retained key from the connected duplicate component, and pass an external governance verifier. Every operation records all component item revisions, identity mappings before and after canonicalization, and the exact rollback mapping. These mappings affect only downstream identity resolution; Zotero records are neither mutated nor deleted. -Golden-set evaluation accepts only a governance receipt verified with the complete reviewed set by a caller-owned authorization boundary. Its library version, rule revision, canonical SHA-256 content digest, and every observed parent/child item-key/item-version identity must bind the classification report. The digest covers every raw Zotero item in canonical key order. Blank, duplicate, unknown, stale, content-mismatched, label-mismatched, or abstention-as-truth inputs fail closed. The full-reclassification evaluator reuses this contract and additionally requires the reviewed label count to equal the unique classified bibliographic-item count; because the base evaluator already rejects blank, duplicate, and unknown keys, equality proves complete coverage. A sampled golden set can measure quality but cannot satisfy this completion gate. The output retains the verified library version, rule revision, and opaque snapshot digest, but contains no item keys, reviewer identity, or bibliographic text. Production authorization remains Keyverse/governance-owned; this crate passes the complete reviewed labels to that boundary instead of minting authority. +Golden-set evaluation accepts only a governance receipt verified with the complete reviewed set by a caller-owned authorization boundary. Its library version, rule revision, canonical SHA-256 content digest, and every observed parent/child item-key/item-version identity must bind the classification report. The digest covers every raw Zotero item in canonical key order. Blank, duplicate, unknown, stale, content-mismatched, label-mismatched, or abstention-as-truth inputs fail closed before the external approval verifier is called, so an invalid local set cannot consume approval authority. The full-reclassification evaluator checks label cardinality before that boundary and additionally requires the reviewed label count to equal the unique classified bibliographic-item count; because the base evaluator rejects blank, duplicate, and unknown keys, equality proves complete coverage. A sampled golden set can measure quality but cannot satisfy this completion gate. The output retains the verified library version, rule revision, and opaque snapshot digest, but contains no item keys, reviewer identity, or bibliographic text. Production authorization remains Keyverse/governance-owned; this crate passes the complete reviewed labels to that boundary instead of minting authority. A successful classification report carries an `audit_summary` whose snapshot, bibliographic, proposed-disposition, provenance-complete, abstention, duplicate-candidate, failure, and per-disposition counts are derived from the same in-memory immutable snapshot. Zotero item version zero remains a valid observed coordinate for never-synced Zotero 9 records; provenance completeness rejects a missing item key rather than inventing a positive-only version invariant. Reader failures return an error instead of a partial report; therefore a returned report records `failure_count=0` rather than hiding partial failures. The local report can contain titles, tags, matched metadata, and abstention abstracts. It is sensitive steward-review material, remains outside the repository, and is not a publication artifact. diff --git a/docs/adr/0006-zotero-research-intake.md b/docs/adr/0006-zotero-research-intake.md index 7019bde6..7621db9f 100644 --- a/docs/adr/0006-zotero-research-intake.md +++ b/docs/adr/0006-zotero-research-intake.md @@ -21,7 +21,7 @@ Matched metadata values are copied into the local-only evidence receipt for repl Duplicate candidates become canonical references only through externally verified steward decisions bound to the raw digest, complete item-key/item-version snapshot, and exact candidate membership. Overlapping candidates form one connected component and must select one component-level canonical item. Every resulting operation retains all component source revisions and complete before/after/rollback key mappings. It changes downstream identity resolution only; classification does not merge, delete, or mutate Zotero source records. -Classifier quality is measured only against local steward-reviewed labels whose reviewed set is verified outside this crate and bound to the exact library version, rule revision, canonical SHA-256 raw-snapshot digest, and every observed parent/child item-key/item-version coordinate. `NeedsStewardReview` is an abstention prediction and cannot be approved truth. Sampled labels may measure classifier quality, but a full-reclassification completion result requires exactly one approved label for every classified bibliographic item. Evaluation returns the verified revisions and opaque digest with aggregate integer evidence; Zotero keys, reviewer identity, and bibliographic text are omitted. Missing, incomplete, stale, content- or label-mismatched, unverified, unknown, duplicate, or invalid review identities fail closed at the applicable completion boundary. +Classifier quality is measured only against local steward-reviewed labels whose reviewed set is verified outside this crate and bound to the exact library version, rule revision, canonical SHA-256 raw-snapshot digest, and every observed parent/child item-key/item-version coordinate. `NeedsStewardReview` is an abstention prediction and cannot be approved truth. Sampled labels may measure classifier quality, but a full-reclassification completion result requires exactly one approved label for every classified bibliographic item. Cardinality, snapshot, key, disposition, and duplicate checks run before the external approval verifier so invalid local input cannot consume approval authority. Evaluation returns the verified revisions and opaque digest with aggregate integer evidence; Zotero keys, reviewer identity, and bibliographic text are omitted. Missing, incomplete, stale, content- or label-mismatched, unverified, unknown, duplicate, or invalid review identities fail closed at the applicable completion boundary. Every successful report includes an aggregate audit summary computed from the same captured snapshot. Zotero 9 item version zero is preserved as a valid never-synced source coordinate, not treated as missing provenance. Partial reads never produce a report, so successful output explicitly records zero failures alongside snapshot, proposal, provenance, abstention, duplicate, and per-disposition totals. The reader fails closed above 50,000 items or 256 MiB of cumulative response bodies, while retaining the 8 MiB per-page bound, finite request timeouts, redirect denial, total-count checks, snapshot-version checks, and duplicate-key detection. Pagination, consistency, resource-budget, and provider-contract behavior are separated from the narrow `ureq` transport so deterministic tests exercise the production reader core rather than excluding the entire reader from coverage. From 7d17861a172665e9f9fa0723baa069f65fffce17 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sat, 5 Sep 2026 02:28:07 +0900 Subject: [PATCH 07/12] test(zotero): reject invalid full review before approval --- .../tests/golden_set_evaluation.rs | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/crates/conceptweave-zotero/tests/golden_set_evaluation.rs b/crates/conceptweave-zotero/tests/golden_set_evaluation.rs index 246e8a1a..43d9f216 100644 --- a/crates/conceptweave-zotero/tests/golden_set_evaluation.rs +++ b/crates/conceptweave-zotero/tests/golden_set_evaluation.rs @@ -48,6 +48,22 @@ fn complete_review_requires_one_steward_label_per_bibliographic_item() { Err(EvaluationError::IncompleteReview) ); assert_eq!(verifier_calls.get(), 0); + assert_eq!( + evaluate_complete_reviewed_classification( + &report, + &golden(vec![ + GoldenLabel::new("A", Disposition::Generation), + GoldenLabel::new("A", Disposition::Generation), + GoldenLabel::new("B", Disposition::EvaluationGovernance), + ]), + |_| { + verifier_calls.set(verifier_calls.get() + 1); + true + }, + ), + Err(EvaluationError::DuplicateItem) + ); + assert_eq!(verifier_calls.get(), 0); let evaluation = evaluate_complete_reviewed_classification( &report, From 8c0ef441894a7db71e9b0e495504e9c3ac8c5bce Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sun, 6 Sep 2026 23:40:47 +0900 Subject: [PATCH 08/12] test(research): reject pending sources as complete review evidence --- .../tests/golden_set_evaluation.rs | 39 +++++++++++++++++++ 1 file changed, 39 insertions(+) diff --git a/crates/conceptweave-zotero/tests/golden_set_evaluation.rs b/crates/conceptweave-zotero/tests/golden_set_evaluation.rs index 58c1074a..352e7b30 100644 --- a/crates/conceptweave-zotero/tests/golden_set_evaluation.rs +++ b/crates/conceptweave-zotero/tests/golden_set_evaluation.rs @@ -80,6 +80,45 @@ fn complete_review_requires_one_steward_label_per_bibliographic_item() { assert_eq!(evaluation.reviewed_count, 3); } +#[test] +fn complete_review_rejects_pending_sources_without_blocking_sampled_evaluation() { + use std::cell::Cell; + + for parent_key in ["", "missing", "A"] { + let mut source_item = item("source", "synthetic attachment"); + source_item.data.item_type = "attachment".into(); + source_item.data.parent_item = parent_key.into(); + let report = classify_snapshot( + "9.0.6".into(), + None, + 42, + vec![item("A", "ontology learning"), source_item], + ); + let mut reviewed = golden(vec![GoldenLabel::new("A", Disposition::Generation)]); + reviewed.approval.snapshot_digest = classification_snapshot_digest(&report); + reviewed.approval.proposal_digest = classification_proposal_digest(&report); + reviewed.approval.snapshot_items = report.snapshot_items.clone(); + let issued_review = reviewed.clone(); + assert!( + evaluate_reviewed_golden_set(&report, &reviewed, |value| { value == &issued_review }) + .is_ok() + ); + + let verifier_calls = Cell::new(0); + let result = evaluate_complete_reviewed_classification(&report, &reviewed, |value| { + verifier_calls.set(verifier_calls.get() + 1); + value == &issued_review + }); + if parent_key == "A" { + assert_eq!(result.unwrap().reviewed_count, 1); + assert_eq!(verifier_calls.get(), 1); + } else { + assert_eq!(result, Err(EvaluationError::IncompleteReview)); + assert_eq!(verifier_calls.get(), 0); + } + } +} + #[test] fn invalid_local_review_never_reaches_the_approval_verifier() { use std::cell::Cell; From ac5947725be767240b8886b34986abd78cab8ca5 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sun, 6 Sep 2026 23:41:13 +0900 Subject: [PATCH 09/12] fix(research): require resolved source scope before completion --- crates/conceptweave-zotero/src/lib.rs | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/crates/conceptweave-zotero/src/lib.rs b/crates/conceptweave-zotero/src/lib.rs index eb3ac6b9..33783270 100644 --- a/crates/conceptweave-zotero/src/lib.rs +++ b/crates/conceptweave-zotero/src/lib.rs @@ -1265,7 +1265,11 @@ impl fmt::Display for EvaluationError { impl std::error::Error for EvaluationError {} -/// Evaluates a steward review only when it covers every bibliographic item. +/// Evaluates a review covering every bibliographic item with no unresolved sources. +/// +/// Standalone sources, orphan trees and disconnected cycles must be resolved +/// before completion. Sampled quality evaluation remains available separately. +/// Success proves reviewed metadata coverage, not a Zotero write or full-text approval. pub fn evaluate_complete_reviewed_classification( report: &ClassificationReport, golden: &ReviewedGoldenSet, @@ -1274,7 +1278,9 @@ pub fn evaluate_complete_reviewed_classification( where F: FnOnce(&ReviewedGoldenSet) -> bool, { - if golden.labels.len() != report.classified_items.len() { + if golden.labels.len() != report.classified_items.len() + || !report.pending_source_item_keys.is_empty() + { return Err(EvaluationError::IncompleteReview); } let evaluation = evaluate_reviewed_golden_set(report, golden, verify_approval)?; From b868f39a93fbc567610ff91d5cf96bd17a51097c Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sun, 6 Sep 2026 23:42:07 +0900 Subject: [PATCH 10/12] test(research): retain pending ancestry validation at completion boundary --- .../tests/golden_set_evaluation.rs | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/crates/conceptweave-zotero/tests/golden_set_evaluation.rs b/crates/conceptweave-zotero/tests/golden_set_evaluation.rs index 352e7b30..1d0e0a25 100644 --- a/crates/conceptweave-zotero/tests/golden_set_evaluation.rs +++ b/crates/conceptweave-zotero/tests/golden_set_evaluation.rs @@ -84,7 +84,7 @@ fn complete_review_requires_one_steward_label_per_bibliographic_item() { fn complete_review_rejects_pending_sources_without_blocking_sampled_evaluation() { use std::cell::Cell; - for parent_key in ["", "missing", "A"] { + for parent_key in ["", "missing", "source", "A"] { let mut source_item = item("source", "synthetic attachment"); source_item.data.item_type = "attachment".into(); source_item.data.parent_item = parent_key.into(); @@ -115,6 +115,17 @@ fn complete_review_rejects_pending_sources_without_blocking_sampled_evaluation() } else { assert_eq!(result, Err(EvaluationError::IncompleteReview)); assert_eq!(verifier_calls.get(), 0); + let mut forged_report = report.clone(); + forged_report.pending_source_item_keys.clear(); + reviewed.approval.proposal_digest = classification_proposal_digest(&forged_report); + assert_eq!( + evaluate_complete_reviewed_classification(&forged_report, &reviewed, |_| { + verifier_calls.set(verifier_calls.get() + 1); + true + }), + Err(EvaluationError::InvalidReview) + ); + assert_eq!(verifier_calls.get(), 0); } } } From 5b2282a053bc7a52cf70704770d502e706533974 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sun, 6 Sep 2026 23:43:31 +0900 Subject: [PATCH 11/12] fix(research): clarify complete review scope and reuse owned fixture --- crates/conceptweave-zotero/src/lib.rs | 4 ++-- crates/conceptweave-zotero/tests/golden_set_evaluation.rs | 2 +- docs/PRD.md | 2 +- docs/TRD.md | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/crates/conceptweave-zotero/src/lib.rs b/crates/conceptweave-zotero/src/lib.rs index 33783270..79c1bc10 100644 --- a/crates/conceptweave-zotero/src/lib.rs +++ b/crates/conceptweave-zotero/src/lib.rs @@ -1241,7 +1241,7 @@ pub enum EvaluationError { UnknownItem, /// A reviewed key occurs more than once. DuplicateItem, - /// The reviewed labels do not cover every bibliographic item. + /// Bibliographic labels are incomplete or source records remain unresolved. IncompleteReview, } @@ -1257,7 +1257,7 @@ impl fmt::Display for EvaluationError { Self::UnknownItem => "golden set contains an item absent from the report", Self::DuplicateItem => "golden set contains a duplicate item", Self::IncompleteReview => { - "complete review must label every bibliographic item exactly once" + "complete review must label every bibliographic item exactly once and resolve all pending sources" } }) } diff --git a/crates/conceptweave-zotero/tests/golden_set_evaluation.rs b/crates/conceptweave-zotero/tests/golden_set_evaluation.rs index 1d0e0a25..ccd6bf36 100644 --- a/crates/conceptweave-zotero/tests/golden_set_evaluation.rs +++ b/crates/conceptweave-zotero/tests/golden_set_evaluation.rs @@ -115,7 +115,7 @@ fn complete_review_rejects_pending_sources_without_blocking_sampled_evaluation() } else { assert_eq!(result, Err(EvaluationError::IncompleteReview)); assert_eq!(verifier_calls.get(), 0); - let mut forged_report = report.clone(); + let mut forged_report = report; forged_report.pending_source_item_keys.clear(); reviewed.approval.proposal_digest = classification_proposal_digest(&forged_report); assert_eq!( diff --git a/docs/PRD.md b/docs/PRD.md index 3763b872..f1212f62 100644 --- a/docs/PRD.md +++ b/docs/PRD.md @@ -88,7 +88,7 @@ The Zotero 10+ adapter can accept a caller-owned API key and server identity at Evaluate classifier quality only against a steward-reviewed local golden set whose governance receipt is externally verified and binds both the complete source/classifier-input snapshot and every current proposal field, in addition to the item-key/item-version coordinates. Same-version changes to unmodeled provider metadata, absent/default fields, classifier inputs, predictions or supporting evidence must invalidate the corresponding binding. Evaluation recomputes proposal identity before contacting governance; a locally changed digest cannot renew an approval. Legacy unbound approvals require reissuance, never automatic backfill. Abstention is a prediction outcome, never an approved truth label. Evaluation emits the verified library revision, rule revision, opaque snapshot and proposal digests, and aggregate counts for exact matches, abstentions, and per-disposition true-positive/predicted/expected totals; it must not copy Zotero keys, reviewer identity, or bibliographic text into the result. -A full-reclassification completion result additionally requires exactly one non-abstention steward label for every top-level bibliographic item; a sampled golden set remains valid for quality measurement but cannot prove completion. +A complete metadata-review result additionally requires exactly one non-abstention steward label for every top-level bibliographic item and no unresolved source records. Standalone sources, orphan trees and disconnected cycles must be resolved before completion; clearing their reported list cannot bypass inventory validation. A sampled golden set remains valid for quality measurement but cannot prove completion. Neither result proves full-text approval or an applied Zotero reclassification. Every successful classification report includes aggregate evidence for snapshot coverage, proposal coverage, provenance completeness, abstentions, duplicate candidates, disposition totals, and zero unreported failures. ## 6. First vertical slice diff --git a/docs/TRD.md b/docs/TRD.md index 68596a6a..3080f018 100644 --- a/docs/TRD.md +++ b/docs/TRD.md @@ -106,7 +106,7 @@ Structural, source, proposal, and label checks precede the external verifier. Bl Provider deserialization captures each complete JSON object before projecting metadata. Snapshot hashing serializes the domain marker `conceptweave-zotero-snapshot-v2` followed by key-ordered pairs of that canonical source JSON and the actual typed classifier input. Unknown nested fields, array order, and omitted-versus-explicit default fields remain bound; changing a typed input after decoding also changes the digest. Synthetic offline typed items have no captured provider object and bind an explicit absent-source value alongside their typed input. Earlier reduced-content digests remain historical evidence and cannot establish this complete-content contract; regenerate the report and review artifacts and obtain fresh approval before any release or approved write. -The full-reclassification evaluator checks label cardinality before that boundary and additionally requires the reviewed label count to equal the unique classified bibliographic-item count; because the base evaluator rejects blank, duplicate, and unknown keys, equality proves complete coverage. A sampled golden set can measure quality but cannot satisfy this completion gate. +The complete metadata-review evaluator rejects unequal label cardinality or nonempty `pending_source_item_keys` with `IncompleteReview` before governance. The shared evaluator then recomputes the complete inventory and pending ancestry, so clearing pending keys and rewriting the proposal digest still fails local validation. Because shared validation rejects blank, duplicate, and unknown keys, equal cardinality proves bibliographic label coverage. Sampled evaluation still supports pending sources; completion does not prove a Zotero mutation or full-text approval. A successful classification report carries an `audit_summary` whose snapshot, bibliographic, proposed-disposition, provenance-complete, abstention, duplicate-candidate, failure, and per-disposition counts are derived from the same in-memory immutable snapshot. Zotero item version zero remains a valid observed coordinate for never-synced Zotero 9 records; provenance completeness rejects a missing item key rather than inventing a positive-only version invariant. Reader failures return an error instead of a partial report; therefore a returned report records `failure_count=0` rather than hiding partial failures. The local report can contain titles, tags, matched metadata, and abstention abstracts. It is sensitive steward-review material, remains outside the repository, and is not a publication artifact. From 35c57ca4510a65cf48069285d78b95cf47db65ba Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sun, 6 Sep 2026 23:45:24 +0900 Subject: [PATCH 12/12] docs(research): record PR24 pending-source RED and verified scope --- docs/product-technical-gap-baseline.md | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/docs/product-technical-gap-baseline.md b/docs/product-technical-gap-baseline.md index 5aa39db3..10ddb3ef 100644 --- a/docs/product-technical-gap-baseline.md +++ b/docs/product-technical-gap-baseline.md @@ -6,6 +6,16 @@ This file records code-current product and technical gaps. Exact PR/check/run co ## September 6 source inventory checkpoint +### PR24 pending-source completion repair + +Final source `5b2282a` passes strict all-target Clippy, warnings-denied rustdoc, formatting, CI contract and diff checks. The unchanged pinned coverage gate passes 279/279 functions, 2,420/2,420 normalized regions and 408/408 normalized branches. Raw coverage remains 3,198/3,259 lines, 4,810/4,912 regions and 363/408 branches, not 100%. Logs: `/tmp/conceptweave-pr24-{clippy,rustdoc,coverage}-verified.log`. No predecessor or later-head coverage is attributed to this source. + +PR24 originally `1e73e1545de32ae9a349c469a7794c5c3fc2ae9b` passed 123 tests/23 suites. Normal merge `b4c16a4` retains that head and PR23 `2d32f96740c708c3f7c13b392386f8bc7a878746`; integrated tests passed 149/23. The existing complete-review evaluator accepted one reviewed bibliographic item even when a standalone attachment remained unresolved. This contradicted the source-scope requirement: a complete bibliography is insufficient when source records remain unaccounted for. + +Behavioral RED `8c0ef44` compiled and failed with an unexpected successful evaluation (`/tmp/conceptweave-pr24-pending-red.log`). Repair `ac59477` adds the pending-source condition to the existing completion boundary, without changing sampled evaluation or creating another authority mechanism. `b868f39` adds self-cycle and forged-empty-pending coverage; its initial test failed to compile because the report is not Clone. `5b2282a` moves the test-owned report instead of expanding the production API. The final workspace result is 150 tests/23 suites including three doctests (`/tmp/conceptweave-pr24-verified.log`). Standalone, orphan and cyclic sources reject completion before governance; attached sources remain eligible. Clearing pending keys and rewriting the digest still fails shared inventory validation. Independent read-only review found no additional production regression; its wording finding is reflected in PRD, TRD and the error message. + +This is complete metadata-review coverage, not full-text approval, a successful Zotero write, hosted GREEN, independent protected approval, merge or release. The root runtime has not yet adopted the repair. Actual decisions and independent approvals remain 0/3,715 plus four unresolved standalone sources; no real data, labels or authority were created for these synthetic unit tests. Visual Inspection was retried, but the Mac is locked; no new screen evidence is claimed. Preserve Proposed ADR 0006 and the existing owner stack. Next propagate into PR25 `c6b4c17e931951a2e1d4ea79ac79363f6306a5bf` and the later full-text/runtime consumers, then validate protected and live evidence separately. + The existing #9 owner now retains every nonbibliographic metadata record and derives unresolved ancestry rather than silently discarding standalone sources. [Source-scope doctoring](doctoring/zotero_source_scope.md) binds committed REDs, final source `1e95d6eb979e66ecb7dae4f81f18a6b0a91b7624`, **47 tests / 10 unfiltered suites**, strict checks and the unchanged coverage gate. The earlier inventory executable at `48c3525` genuinely reads 8,326 records into 3,715 unchanged bibliographic proposals plus 4,611 other records, with exactly the four previously audited standalone identities pending. A later shared-reader guard also rejects blank identities; no actual final-guard executable replay is implied. The earlier source findings are repaired locally, not yet propagated into root #39. Required downstream restoration/identity accounting, pending-source reconciliation, approval binding and full-library completion gates remain open; neither zero pending keys nor successful classification grants semantic or write authority. Current native Visual Inspection was attempted but the Mac is locked, so no new screenshot was verified. Historical source scope, authentic worksheet decisions/independent approvals 0/3,715, plus four unresolved sources remain distinct. This checkpoint does not refresh every historical PR coordinate below or imply protected merge/release.