diff --git a/CHANGELOG.md b/CHANGELOG.md index 836b76bb..37c41e28 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -27,6 +27,7 @@ All notable changes to ConceptWeave are documented here. - Read-only delayed reconciliation receipts for indeterminate Zotero rollback operations. - Minimal, nonduplicated local abstract context for Zotero items that require steward classification. - Owner-only file permissions for sensitive local Zotero classification reports. +- A complete-review evaluator that rejects partial steward labels as full reclassification evidence. ### Security diff --git a/crates/conceptweave-zotero/src/lib.rs b/crates/conceptweave-zotero/src/lib.rs index 9d871a0f..79c1bc10 100644 --- a/crates/conceptweave-zotero/src/lib.rs +++ b/crates/conceptweave-zotero/src/lib.rs @@ -1241,6 +1241,8 @@ pub enum EvaluationError { UnknownItem, /// A reviewed key occurs more than once. DuplicateItem, + /// Bibliographic labels are incomplete or source records remain unresolved. + IncompleteReview, } impl fmt::Display for EvaluationError { @@ -1254,12 +1256,37 @@ impl fmt::Display for EvaluationError { } Self::UnknownItem => "golden set contains an item absent from the report", Self::DuplicateItem => "golden set contains a duplicate item", + Self::IncompleteReview => { + "complete review must label every bibliographic item exactly once and resolve all pending sources" + } }) } } impl std::error::Error for EvaluationError {} +/// Evaluates a review covering every bibliographic item with no unresolved sources. +/// +/// Standalone sources, orphan trees and disconnected cycles must be resolved +/// before completion. Sampled quality evaluation remains available separately. +/// Success proves reviewed metadata coverage, not a Zotero write or full-text approval. +pub fn evaluate_complete_reviewed_classification( + report: &ClassificationReport, + golden: &ReviewedGoldenSet, + verify_approval: F, +) -> Result +where + F: FnOnce(&ReviewedGoldenSet) -> bool, +{ + if golden.labels.len() != report.classified_items.len() + || !report.pending_source_item_keys.is_empty() + { + return Err(EvaluationError::IncompleteReview); + } + let evaluation = evaluate_reviewed_golden_set(report, golden, verify_approval)?; + Ok(evaluation) +} + /// Checks that every observed item belongs to exactly one report partition. /// /// Child links and unresolved source keys are recomputed from preserved metadata. diff --git a/crates/conceptweave-zotero/tests/golden_set_evaluation.rs b/crates/conceptweave-zotero/tests/golden_set_evaluation.rs index dae82e9f..ccd6bf36 100644 --- a/crates/conceptweave-zotero/tests/golden_set_evaluation.rs +++ b/crates/conceptweave-zotero/tests/golden_set_evaluation.rs @@ -1,7 +1,8 @@ use conceptweave_zotero::{ Disposition, EvaluationError, GoldenLabel, GoldenSetApproval, ItemData, ReviewedGoldenSet, SnapshotItemRevision, ZoteroItem, classification_proposal_digest, - classification_snapshot_digest, classify_snapshot, evaluate_reviewed_golden_set, + classification_snapshot_digest, classify_snapshot, evaluate_complete_reviewed_classification, + evaluate_reviewed_golden_set, }; fn item(key: &str, title: &str) -> ZoteroItem { @@ -21,6 +22,134 @@ fn item(key: &str, title: &str) -> ZoteroItem { } } +#[test] +fn complete_review_requires_one_steward_label_per_bibliographic_item() { + use std::cell::Cell; + + let report = report(); + let verifier_calls = Cell::new(0); + assert_eq!( + evaluate_complete_reviewed_classification(&report, &golden(vec![]), |_| { + verifier_calls.set(verifier_calls.get() + 1); + true + },), + Err(EvaluationError::IncompleteReview) + ); + assert_eq!( + evaluate_complete_reviewed_classification( + &report, + &golden(vec![ + GoldenLabel::new("A", Disposition::Generation), + GoldenLabel::new("B", Disposition::EvaluationGovernance), + ]), + |_| { + verifier_calls.set(verifier_calls.get() + 1); + true + }, + ), + Err(EvaluationError::IncompleteReview) + ); + assert_eq!(verifier_calls.get(), 0); + assert_eq!( + evaluate_complete_reviewed_classification( + &report, + &golden(vec![ + GoldenLabel::new("A", Disposition::Generation), + GoldenLabel::new("A", Disposition::Generation), + GoldenLabel::new("B", Disposition::EvaluationGovernance), + ]), + |_| { + verifier_calls.set(verifier_calls.get() + 1); + true + }, + ), + Err(EvaluationError::DuplicateItem) + ); + assert_eq!(verifier_calls.get(), 0); + + let evaluation = evaluate_complete_reviewed_classification( + &report, + &golden(vec![ + GoldenLabel::new("A", Disposition::Generation), + GoldenLabel::new("B", Disposition::EvaluationGovernance), + GoldenLabel::new("C", Disposition::OutOfScope), + ]), + verify_synthetic_approval, + ) + .unwrap(); + assert_eq!(evaluation.reviewed_count, 3); +} + +#[test] +fn complete_review_rejects_pending_sources_without_blocking_sampled_evaluation() { + use std::cell::Cell; + + for parent_key in ["", "missing", "source", "A"] { + let mut source_item = item("source", "synthetic attachment"); + source_item.data.item_type = "attachment".into(); + source_item.data.parent_item = parent_key.into(); + let report = classify_snapshot( + "9.0.6".into(), + None, + 42, + vec![item("A", "ontology learning"), source_item], + ); + let mut reviewed = golden(vec![GoldenLabel::new("A", Disposition::Generation)]); + reviewed.approval.snapshot_digest = classification_snapshot_digest(&report); + reviewed.approval.proposal_digest = classification_proposal_digest(&report); + reviewed.approval.snapshot_items = report.snapshot_items.clone(); + let issued_review = reviewed.clone(); + assert!( + evaluate_reviewed_golden_set(&report, &reviewed, |value| { value == &issued_review }) + .is_ok() + ); + + let verifier_calls = Cell::new(0); + let result = evaluate_complete_reviewed_classification(&report, &reviewed, |value| { + verifier_calls.set(verifier_calls.get() + 1); + value == &issued_review + }); + if parent_key == "A" { + assert_eq!(result.unwrap().reviewed_count, 1); + assert_eq!(verifier_calls.get(), 1); + } else { + assert_eq!(result, Err(EvaluationError::IncompleteReview)); + assert_eq!(verifier_calls.get(), 0); + let mut forged_report = report; + forged_report.pending_source_item_keys.clear(); + reviewed.approval.proposal_digest = classification_proposal_digest(&forged_report); + assert_eq!( + evaluate_complete_reviewed_classification(&forged_report, &reviewed, |_| { + verifier_calls.set(verifier_calls.get() + 1); + true + }), + Err(EvaluationError::InvalidReview) + ); + assert_eq!(verifier_calls.get(), 0); + } + } +} + +#[test] +fn invalid_local_review_never_reaches_the_approval_verifier() { + use std::cell::Cell; + + let verifier_calls = Cell::new(0); + let result = evaluate_reviewed_golden_set( + &report(), + &golden(vec![ + GoldenLabel::new("A", Disposition::Generation), + GoldenLabel::new("A", Disposition::Generation), + ]), + |_| { + verifier_calls.set(verifier_calls.get() + 1); + true + }, + ); + assert_eq!(result, Err(EvaluationError::DuplicateItem)); + assert_eq!(verifier_calls.get(), 0); +} + fn report() -> conceptweave_zotero::ClassificationReport { classify_snapshot( "9.0.6".into(), @@ -236,6 +365,7 @@ fn reviewed_golden_set_rejects_stale_unknown_and_duplicate_labels() { (EvaluationError::InvalidExpectedDisposition, "abstention"), (EvaluationError::UnknownItem, "absent"), (EvaluationError::DuplicateItem, "duplicate"), + (EvaluationError::IncompleteReview, "every bibliographic"), ] { assert!(error.to_string().contains(fragment)); } diff --git a/docs/PRD.md b/docs/PRD.md index 5b094f16..f1212f62 100644 --- a/docs/PRD.md +++ b/docs/PRD.md @@ -87,6 +87,8 @@ PR #21 retains validated delayed reads without writes and complete observed meta The Zotero 10+ adapter can accept a caller-owned API key and server identity at runtime or consume one successful, user-approved Local API authorization. Authorization sends one bounded application name and the expected server identity to the fixed loopback endpoint; only a same-server bounded response that explicitly reports denial is classified as the user's decision. Denial and rate limiting return immediately without another prompt or automatic retry. The private 32-character key is neither serializable nor printable. Authorization, read, and write responses bind to the expected server before status classification; writes name expired authorization and matching-server stale preconditions separately. Thin public execution boundaries connect the adapter to the reviewed write and rollback cores without duplicating mutation logic. Synthetic transport evidence does not satisfy AC6's approved live Zotero 10 authorization, write, and rollback requirement. Evaluate classifier quality only against a steward-reviewed local golden set whose governance receipt is externally verified and binds both the complete source/classifier-input snapshot and every current proposal field, in addition to the item-key/item-version coordinates. Same-version changes to unmodeled provider metadata, absent/default fields, classifier inputs, predictions or supporting evidence must invalidate the corresponding binding. Evaluation recomputes proposal identity before contacting governance; a locally changed digest cannot renew an approval. Legacy unbound approvals require reissuance, never automatic backfill. Abstention is a prediction outcome, never an approved truth label. Evaluation emits the verified library revision, rule revision, opaque snapshot and proposal digests, and aggregate counts for exact matches, abstentions, and per-disposition true-positive/predicted/expected totals; it must not copy Zotero keys, reviewer identity, or bibliographic text into the result. + +A complete metadata-review result additionally requires exactly one non-abstention steward label for every top-level bibliographic item and no unresolved source records. Standalone sources, orphan trees and disconnected cycles must be resolved before completion; clearing their reported list cannot bypass inventory validation. A sampled golden set remains valid for quality measurement but cannot prove completion. Neither result proves full-text approval or an applied Zotero reclassification. Every successful classification report includes aggregate evidence for snapshot coverage, proposal coverage, provenance completeness, abstentions, duplicate candidates, disposition totals, and zero unreported failures. ## 6. First vertical slice diff --git a/docs/TRD.md b/docs/TRD.md index 07ffe2c9..3080f018 100644 --- a/docs/TRD.md +++ b/docs/TRD.md @@ -105,6 +105,8 @@ The proposal-only v1 format above describes the previous receipt contract. The c Structural, source, proposal, and label checks precede the external verifier. Blank, duplicate, unknown, stale, content-mismatched, prediction-mismatched, label-mismatched, or abstention-as-truth inputs fail closed. The aggregate result retains the verified library version, rule revision, and opaque snapshot/proposal digests, but contains no item keys, reviewer identity, or bibliographic text. Production authorization remains Keyverse/governance-owned; this crate passes the complete reviewed labels and approval bindings to that boundary instead of minting authority. Provider deserialization captures each complete JSON object before projecting metadata. Snapshot hashing serializes the domain marker `conceptweave-zotero-snapshot-v2` followed by key-ordered pairs of that canonical source JSON and the actual typed classifier input. Unknown nested fields, array order, and omitted-versus-explicit default fields remain bound; changing a typed input after decoding also changes the digest. Synthetic offline typed items have no captured provider object and bind an explicit absent-source value alongside their typed input. Earlier reduced-content digests remain historical evidence and cannot establish this complete-content contract; regenerate the report and review artifacts and obtain fresh approval before any release or approved write. + +The complete metadata-review evaluator rejects unequal label cardinality or nonempty `pending_source_item_keys` with `IncompleteReview` before governance. The shared evaluator then recomputes the complete inventory and pending ancestry, so clearing pending keys and rewriting the proposal digest still fails local validation. Because shared validation rejects blank, duplicate, and unknown keys, equal cardinality proves bibliographic label coverage. Sampled evaluation still supports pending sources; completion does not prove a Zotero mutation or full-text approval. A successful classification report carries an `audit_summary` whose snapshot, bibliographic, proposed-disposition, provenance-complete, abstention, duplicate-candidate, failure, and per-disposition counts are derived from the same in-memory immutable snapshot. Zotero item version zero remains a valid observed coordinate for never-synced Zotero 9 records; provenance completeness rejects a missing item key rather than inventing a positive-only version invariant. Reader failures return an error instead of a partial report; therefore a returned report records `failure_count=0` rather than hiding partial failures. The local report can contain titles, tags, matched metadata, and abstention abstracts. It is sensitive steward-review material, remains outside the repository, and is not a publication artifact. diff --git a/docs/UML.md b/docs/UML.md index 1af557e7..e20f31b1 100644 --- a/docs/UML.md +++ b/docs/UML.md @@ -58,10 +58,12 @@ sequenceDiagram Intake->>Report: write proposals, complete inventory and unresolved source keys Note over Report,Steward: Pending sources prevent a whole-library completion claim; inventory is not approval Report->>Steward: review dispositions and merge candidates + Steward->>Intake: verified labels for every bibliographic item Steward->>Intake: reviewed labels and independently issued receipt Intake->>Intake: validate complete partitions and recompute pending ancestry Intake->>Intake: verify v2 proposal and retained-source binding Note over Intake,Steward: Only locally valid reports reach independent governance verification + Intake-->>Steward: aggregate bibliographic review evidence or incomplete-review failure Steward->>Intake: verified canonical-item decisions Intake->>Report: before/after/rollback identity manifest Report-->>Steward: reversible local mapping; source records preserved diff --git a/docs/adr/0006-zotero-research-intake.md b/docs/adr/0006-zotero-research-intake.md index 3600a494..72fd8ba4 100644 --- a/docs/adr/0006-zotero-research-intake.md +++ b/docs/adr/0006-zotero-research-intake.md @@ -21,7 +21,7 @@ Matched metadata values are copied into the local-only evidence receipt for repl Duplicate candidates become canonical references only through externally verified steward decisions bound to the raw digest, complete item-key/item-version snapshot, and exact candidate membership. Overlapping candidates form one connected component and must select one component-level canonical item. Every resulting operation retains all component source revisions and complete before/after/rollback key mappings. It changes downstream identity resolution only; classification does not merge, delete, or mutate Zotero source records. -Classifier quality is measured only against local steward-reviewed labels whose complete reviewed set is verified outside this crate and bound to the exact library version, rule revision, canonical SHA-256 raw-snapshot digest, and every observed parent/child item-key/item-version coordinate. `NeedsStewardReview` is an abstention prediction and cannot be approved truth. Evaluation returns the verified revisions and opaque digest with aggregate integer evidence; Zotero keys, reviewer identity, and bibliographic text are omitted. Missing, stale, content- or label-mismatched, unverified, unknown, duplicate, or invalid review identities fail closed. +Classifier quality is measured only against local steward-reviewed labels whose reviewed set is verified outside this crate and bound to the exact library version, rule revision, canonical SHA-256 raw-snapshot digest, and every observed parent/child item-key/item-version coordinate. `NeedsStewardReview` is an abstention prediction and cannot be approved truth. Sampled labels may measure classifier quality, but a full-reclassification completion result requires exactly one approved label for every classified bibliographic item. Cardinality, snapshot, key, disposition, and duplicate checks run before the external approval verifier so invalid local input cannot consume approval authority. Evaluation returns the verified revisions and opaque digest with aggregate integer evidence; Zotero keys, reviewer identity, and bibliographic text are omitted. Missing, incomplete, stale, content- or label-mismatched, unverified, unknown, duplicate, or invalid review identities fail closed at the applicable completion boundary. Every successful report includes an aggregate audit summary computed from the same captured snapshot. Zotero 9 item version zero is preserved as a valid never-synced source coordinate, not treated as missing provenance. Partial reads never produce a report, so successful output explicitly records zero failures alongside snapshot, proposal, provenance, abstention, duplicate, and per-disposition totals. The reader fails closed above 50,000 items or 256 MiB of cumulative response bodies, while retaining the 8 MiB per-page bound, finite request timeouts, redirect denial, total-count checks, snapshot-version checks, and duplicate-key detection. Pagination, consistency, resource-budget, and provider-contract behavior are separated from the narrow `ureq` transport so deterministic tests exercise the production reader core rather than excluding the entire reader from coverage. diff --git a/docs/product-technical-gap-baseline.md b/docs/product-technical-gap-baseline.md index 5aa39db3..10ddb3ef 100644 --- a/docs/product-technical-gap-baseline.md +++ b/docs/product-technical-gap-baseline.md @@ -6,6 +6,16 @@ This file records code-current product and technical gaps. Exact PR/check/run co ## September 6 source inventory checkpoint +### PR24 pending-source completion repair + +Final source `5b2282a` passes strict all-target Clippy, warnings-denied rustdoc, formatting, CI contract and diff checks. The unchanged pinned coverage gate passes 279/279 functions, 2,420/2,420 normalized regions and 408/408 normalized branches. Raw coverage remains 3,198/3,259 lines, 4,810/4,912 regions and 363/408 branches, not 100%. Logs: `/tmp/conceptweave-pr24-{clippy,rustdoc,coverage}-verified.log`. No predecessor or later-head coverage is attributed to this source. + +PR24 originally `1e73e1545de32ae9a349c469a7794c5c3fc2ae9b` passed 123 tests/23 suites. Normal merge `b4c16a4` retains that head and PR23 `2d32f96740c708c3f7c13b392386f8bc7a878746`; integrated tests passed 149/23. The existing complete-review evaluator accepted one reviewed bibliographic item even when a standalone attachment remained unresolved. This contradicted the source-scope requirement: a complete bibliography is insufficient when source records remain unaccounted for. + +Behavioral RED `8c0ef44` compiled and failed with an unexpected successful evaluation (`/tmp/conceptweave-pr24-pending-red.log`). Repair `ac59477` adds the pending-source condition to the existing completion boundary, without changing sampled evaluation or creating another authority mechanism. `b868f39` adds self-cycle and forged-empty-pending coverage; its initial test failed to compile because the report is not Clone. `5b2282a` moves the test-owned report instead of expanding the production API. The final workspace result is 150 tests/23 suites including three doctests (`/tmp/conceptweave-pr24-verified.log`). Standalone, orphan and cyclic sources reject completion before governance; attached sources remain eligible. Clearing pending keys and rewriting the digest still fails shared inventory validation. Independent read-only review found no additional production regression; its wording finding is reflected in PRD, TRD and the error message. + +This is complete metadata-review coverage, not full-text approval, a successful Zotero write, hosted GREEN, independent protected approval, merge or release. The root runtime has not yet adopted the repair. Actual decisions and independent approvals remain 0/3,715 plus four unresolved standalone sources; no real data, labels or authority were created for these synthetic unit tests. Visual Inspection was retried, but the Mac is locked; no new screen evidence is claimed. Preserve Proposed ADR 0006 and the existing owner stack. Next propagate into PR25 `c6b4c17e931951a2e1d4ea79ac79363f6306a5bf` and the later full-text/runtime consumers, then validate protected and live evidence separately. + The existing #9 owner now retains every nonbibliographic metadata record and derives unresolved ancestry rather than silently discarding standalone sources. [Source-scope doctoring](doctoring/zotero_source_scope.md) binds committed REDs, final source `1e95d6eb979e66ecb7dae4f81f18a6b0a91b7624`, **47 tests / 10 unfiltered suites**, strict checks and the unchanged coverage gate. The earlier inventory executable at `48c3525` genuinely reads 8,326 records into 3,715 unchanged bibliographic proposals plus 4,611 other records, with exactly the four previously audited standalone identities pending. A later shared-reader guard also rejects blank identities; no actual final-guard executable replay is implied. The earlier source findings are repaired locally, not yet propagated into root #39. Required downstream restoration/identity accounting, pending-source reconciliation, approval binding and full-library completion gates remain open; neither zero pending keys nor successful classification grants semantic or write authority. Current native Visual Inspection was attempted but the Mac is locked, so no new screenshot was verified. Historical source scope, authentic worksheet decisions/independent approvals 0/3,715, plus four unresolved sources remain distinct. This checkpoint does not refresh every historical PR coordinate below or imply protected merge/release.