From f653c50a7e454a82fc00111587e8e98a652e76e9 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sat, 5 Sep 2026 06:16:01 +0900 Subject: [PATCH 01/30] test(zotero): require review batch context validation --- .../tests/steward_review_batch.rs | 43 ++++++++++++++++++- 1 file changed, 42 insertions(+), 1 deletion(-) diff --git a/crates/conceptweave-zotero/tests/steward_review_batch.rs b/crates/conceptweave-zotero/tests/steward_review_batch.rs index 18fe86a2..423cdb12 100644 --- a/crates/conceptweave-zotero/tests/steward_review_batch.rs +++ b/crates/conceptweave-zotero/tests/steward_review_batch.rs @@ -1,7 +1,7 @@ use conceptweave_zotero::{ Disposition, ItemData, StewardDecisionPatch, WorksheetError, ZoteroItem, apply_steward_decision_patch, build_steward_review_batch, build_steward_review_worksheet, - classify_snapshot, + classify_snapshot, decision_patch_from_review_batch, }; fn item(key: &str, title: &str, abstract_note: &str) -> ZoteroItem { @@ -138,3 +138,44 @@ fn review_batch_rejects_invalid_or_complete_workloads() { Err(WorksheetError::InvalidReport) ); } + +#[test] +fn completed_review_batch_must_preserve_the_context_shown_to_the_steward() { + let report = classify_snapshot( + "9.0.6".into(), + None, + 42, + vec![item("A", "unmatched", "review context")], + ); + let worksheet = build_steward_review_worksheet(&report).unwrap(); + let mut batch = build_steward_review_batch(&report, &worksheet, 1).unwrap(); + batch.decisions[0].reviewed_disposition = Some(Disposition::OutOfScope); + + let patch = decision_patch_from_review_batch(&report, &worksheet, &batch).unwrap(); + assert_eq!(patch.decisions.len(), 1); + assert_eq!(patch.decisions[0].item_key, "A"); + assert_eq!(patch.decisions[0].reviewed_disposition, Disposition::OutOfScope); + + for invalid in [ + { + let mut invalid = batch.clone(); + invalid.decisions[0].title = "different context".into(); + invalid + }, + { + let mut invalid = batch.clone(); + invalid.decisions[0].reviewed_disposition = None; + invalid + }, + { + let mut invalid = batch.clone(); + invalid.decisions[0].reviewed_disposition = Some(Disposition::NeedsStewardReview); + invalid + }, + ] { + assert_eq!( + decision_patch_from_review_batch(&report, &worksheet, &invalid), + Err(WorksheetError::InvalidReport) + ); + } +} From 247581581c7b5c59293209dcd592f8d3ee9a6395 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sat, 5 Sep 2026 06:16:30 +0900 Subject: [PATCH 02/30] fix(zotero): validate completed review batch context --- crates/conceptweave-zotero/src/lib.rs | 40 +++++++++++++++++++++++++-- 1 file changed, 38 insertions(+), 2 deletions(-) diff --git a/crates/conceptweave-zotero/src/lib.rs b/crates/conceptweave-zotero/src/lib.rs index 12e0cd03..958eb2f6 100644 --- a/crates/conceptweave-zotero/src/lib.rs +++ b/crates/conceptweave-zotero/src/lib.rs @@ -1121,7 +1121,7 @@ pub struct StewardDecisionPatch { pub const MAX_REVIEW_BATCH_ITEMS: usize = 100; /// One pending decision with the report context required for human review. -#[derive(Debug, Clone, PartialEq, Eq, Serialize)] +#[derive(Debug, Clone, PartialEq, Eq, Deserialize, Serialize)] pub struct StewardReviewBatchDecision { /// Stable Zotero item key used to apply the completed decision. pub item_key: String, @@ -1148,7 +1148,7 @@ pub struct StewardReviewBatchDecision { } /// Deterministic owner-only view of the next pending steward decisions. -#[derive(Debug, Clone, PartialEq, Eq, Serialize)] +#[derive(Debug, Clone, PartialEq, Eq, Deserialize, Serialize)] pub struct StewardReviewBatch { /// Zotero library revision shared with the report and worksheet. pub library_version: u64, @@ -1443,6 +1443,42 @@ pub fn apply_steward_decision_patch( Ok(updated) } +/// Converts a completed review batch only when its presented context is unchanged. +pub fn decision_patch_from_review_batch( + report: &ClassificationReport, + worksheet: &StewardReviewWorksheet, + batch: &StewardReviewBatch, +) -> Result { + if batch.decisions.is_empty() { + return Err(WorksheetError::InvalidReport); + } + let expected = build_steward_review_batch(report, worksheet, batch.decisions.len())?; + let mut reviewed_context = batch.clone(); + let mut decisions = Vec::with_capacity(reviewed_context.decisions.len()); + for decision in &mut reviewed_context.decisions { + let Some(reviewed_disposition) = decision.reviewed_disposition.take() else { + return Err(WorksheetError::InvalidReport); + }; + if reviewed_disposition == Disposition::NeedsStewardReview { + return Err(WorksheetError::InvalidReport); + } + decisions.push(StewardDecisionUpdate { + item_key: decision.item_key.clone(), + item_version: decision.item_version, + reviewed_disposition, + }); + } + if reviewed_context != expected { + return Err(WorksheetError::InvalidReport); + } + Ok(StewardDecisionPatch { + library_version: expected.library_version, + rule_revision: expected.rule_revision, + snapshot_digest: expected.snapshot_digest, + decisions, + }) +} + fn validate_steward_review_worksheet_against( expected: &StewardReviewWorksheet, worksheet: &StewardReviewWorksheet, From 1dadb21be3f2e6ef7fcf688c671cd33930a0819e Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sat, 5 Sep 2026 06:16:57 +0900 Subject: [PATCH 03/30] test(zotero): require validated batch apply CLI --- .../tests/steward_review_batch_cli.rs | 60 ++++++++++++++++++- 1 file changed, 59 insertions(+), 1 deletion(-) diff --git a/crates/conceptweave-zotero/tests/steward_review_batch_cli.rs b/crates/conceptweave-zotero/tests/steward_review_batch_cli.rs index f0a6f65b..74736891 100644 --- a/crates/conceptweave-zotero/tests/steward_review_batch_cli.rs +++ b/crates/conceptweave-zotero/tests/steward_review_batch_cli.rs @@ -1,7 +1,8 @@ #![cfg(unix)] use conceptweave_zotero::{ - Disposition, ItemData, ZoteroItem, build_steward_review_worksheet, classify_snapshot, + Disposition, ItemData, StewardReviewBatch, StewardReviewWorksheet, ZoteroItem, + build_steward_review_batch, build_steward_review_worksheet, classify_snapshot, }; use std::fs; use std::os::unix::fs::PermissionsExt; @@ -67,6 +68,45 @@ fn review_batch_cli_emits_nothing_for_complete_or_existing_output() { } } +#[test] +fn completed_review_batch_cli_validates_context_before_updating_worksheet() { + let report = classify_snapshot("9.0.6".into(), None, 42, vec![item("A", "unmatched")]); + let worksheet = build_steward_review_worksheet(&report).unwrap(); + let mut batch = build_steward_review_batch(&report, &worksheet, 1).unwrap(); + batch.decisions[0].reviewed_disposition = Some(Disposition::OutOfScope); + let report_path = private_input("apply-batch-report", &report); + let worksheet_path = private_input("apply-batch-worksheet", &worksheet); + let batch_path = private_input("apply-batch-input", &batch); + let output_path = temp_path("apply-batch-output"); + let _ = fs::remove_file(&output_path); + + assert!(run_apply_batch(&report_path, &worksheet_path, &batch_path, &output_path).success()); + let updated: StewardReviewWorksheet = + serde_json::from_slice(&fs::read(&output_path).unwrap()).unwrap(); + assert_eq!( + updated.decisions[0].reviewed_disposition, + Some(Disposition::OutOfScope) + ); + + let mut tampered: StewardReviewBatch = batch; + tampered.decisions[0].title = "different context".into(); + let tampered_path = private_input("apply-batch-tampered", &tampered); + let rejected_path = temp_path("apply-batch-rejected"); + let _ = fs::remove_file(&rejected_path); + assert!(!run_apply_batch(&report_path, &worksheet_path, &tampered_path, &rejected_path).success()); + assert!(!rejected_path.exists()); + + for path in [ + report_path, + worksheet_path, + batch_path, + output_path, + tampered_path, + ] { + fs::remove_file(path).unwrap(); + } +} + fn item(key: &str, title: &str) -> ZoteroItem { ZoteroItem { key: key.into(), @@ -109,6 +149,24 @@ fn run_batch( .unwrap() } +fn run_apply_batch( + report: &Path, + worksheet: &Path, + batch: &Path, + output: &Path, +) -> std::process::ExitStatus { + Command::new(env!("CARGO_BIN_EXE_conceptweave-zotero")) + .args([ + "--apply-review-batch", + report.to_str().unwrap(), + worksheet.to_str().unwrap(), + batch.to_str().unwrap(), + output.to_str().unwrap(), + ]) + .status() + .unwrap() +} + fn temp_path(name: &str) -> std::path::PathBuf { std::env::temp_dir().join(format!( "conceptweave-zotero-{}-{name}.json", From f05523d0bb1b567d9fe31d12fd1b9052bf2f3b5b Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sat, 5 Sep 2026 06:17:37 +0900 Subject: [PATCH 04/30] feat(zotero): apply validated review batches offline --- crates/conceptweave-zotero/src/main.rs | 68 ++++++++++++++++++++++++-- 1 file changed, 64 insertions(+), 4 deletions(-) diff --git a/crates/conceptweave-zotero/src/main.rs b/crates/conceptweave-zotero/src/main.rs index 2e5301d0..8fdccf9b 100644 --- a/crates/conceptweave-zotero/src/main.rs +++ b/crates/conceptweave-zotero/src/main.rs @@ -3,9 +3,9 @@ use conceptweave_zotero::{ ClassificationReport, GoldenSetApproval, MAX_REVIEW_BATCH_ITEMS, StewardDecisionPatch, - StewardReviewWorksheet, apply_steward_decision_patch, assess_steward_review_progress, - build_steward_review_batch, build_steward_review_worksheet, read_local_snapshot, - reviewed_golden_set_from_worksheet, + StewardReviewBatch, StewardReviewWorksheet, apply_steward_decision_patch, + assess_steward_review_progress, build_steward_review_batch, build_steward_review_worksheet, + decision_patch_from_review_batch, read_local_snapshot, reviewed_golden_set_from_worksheet, }; use serde::de::DeserializeOwned; use std::collections::BTreeSet; @@ -14,7 +14,7 @@ use std::fs::{self, File, OpenOptions}; use std::io::{self, BufWriter, Read, Write}; use std::path::{Path, PathBuf}; -const USAGE: &str = "usage: conceptweave-zotero /tmp/REPORT.json | --worksheet /tmp/REPORT.json /tmp/WORKSHEET.json | --review-progress /tmp/REPORT.json /tmp/WORKSHEET.json /tmp/PROGRESS.json | --review-batch /tmp/REPORT.json /tmp/CURRENT_WORKSHEET.json LIMIT /tmp/BATCH.json | --apply-decision-patch /tmp/REPORT.json /tmp/CURRENT_WORKSHEET.json /tmp/PATCH.json /tmp/UPDATED_WORKSHEET.json | --finalize /tmp/REPORT.json /tmp/WORKSHEET.json /tmp/APPROVAL.json /tmp/GOLDEN.json"; +const USAGE: &str = "usage: conceptweave-zotero /tmp/REPORT.json | --worksheet /tmp/REPORT.json /tmp/WORKSHEET.json | --review-progress /tmp/REPORT.json /tmp/WORKSHEET.json /tmp/PROGRESS.json | --review-batch /tmp/REPORT.json /tmp/CURRENT_WORKSHEET.json LIMIT /tmp/BATCH.json | --apply-review-batch /tmp/REPORT.json /tmp/CURRENT_WORKSHEET.json /tmp/COMPLETED_BATCH.json /tmp/UPDATED_WORKSHEET.json | --apply-decision-patch /tmp/REPORT.json /tmp/CURRENT_WORKSHEET.json /tmp/PATCH.json /tmp/UPDATED_WORKSHEET.json | --finalize /tmp/REPORT.json /tmp/WORKSHEET.json /tmp/APPROVAL.json /tmp/GOLDEN.json"; const MAX_ARTIFACT_BYTES: u64 = 16 * 1024 * 1024; #[derive(Debug, PartialEq, Eq)] @@ -41,6 +41,12 @@ enum OutputRequest { patch: String, output: String, }, + ApplyReviewBatch { + report: String, + worksheet: String, + batch: String, + output: String, + }, Finalize { report: String, worksheet: String, @@ -117,6 +123,36 @@ where limit, output, } + } else if first == "--apply-review-batch" { + let report = args + .next() + .ok_or("--apply-review-batch requires four artifact paths")?; + let worksheet = args + .next() + .ok_or("--apply-review-batch requires four artifact paths")?; + let batch = args + .next() + .ok_or("--apply-review-batch requires four artifact paths")?; + let output = args + .next() + .ok_or("--apply-review-batch requires four artifact paths")?; + if BTreeSet::from([ + report.as_str(), + worksheet.as_str(), + batch.as_str(), + output.as_str(), + ]) + .len() + != 4 + { + return Err("review batch application artifact paths must differ"); + } + OutputRequest::ApplyReviewBatch { + report, + worksheet, + batch, + output, + } } else if first == "--apply-decision-patch" { let report = args .next() @@ -486,6 +522,30 @@ fn main() -> Result<(), Box> { let content = serde_json::to_vec_pretty(&batch)?; write_private_output(&output, &content)?; } + OutputRequest::ApplyReviewBatch { + report, + worksheet, + batch, + output, + } => { + let output = validate_output_path(&output)?; + let (report, report_identity): (ClassificationReport, _) = + read_private_json(&report).map_err(|error| label_input("report", error))?; + let (worksheet, worksheet_identity): (StewardReviewWorksheet, _) = + read_private_json(&worksheet).map_err(|error| label_input("worksheet", error))?; + let (batch, batch_identity): (StewardReviewBatch, _) = + read_private_json(&batch).map_err(|error| label_input("review batch", error))?; + if BTreeSet::from([report_identity, worksheet_identity, batch_identity]).len() != 3 { + return Err(io::Error::new( + io::ErrorKind::InvalidInput, + "review batch application inputs must be distinct files", + ) + .into()); + } + let patch = decision_patch_from_review_batch(&report, &worksheet, &batch)?; + let updated = apply_steward_decision_patch(&report, &worksheet, &patch)?; + write_private_output(&output, &serde_json::to_vec_pretty(&updated)?)?; + } OutputRequest::ApplyDecisionPatch { report, worksheet, From 86fd7c461486f46031be49f13c70ace2b4c0a63e Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sat, 5 Sep 2026 06:17:59 +0900 Subject: [PATCH 05/30] test(zotero): reject unvalidated batch as decision patch --- .../tests/steward_review_batch.rs | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/crates/conceptweave-zotero/tests/steward_review_batch.rs b/crates/conceptweave-zotero/tests/steward_review_batch.rs index 423cdb12..d743203a 100644 --- a/crates/conceptweave-zotero/tests/steward_review_batch.rs +++ b/crates/conceptweave-zotero/tests/steward_review_batch.rs @@ -62,8 +62,13 @@ fn review_batch_is_deterministic_bounded_and_patch_compatible() { let mut completed_batch = batch.clone(); completed_batch.decisions[0].reviewed_disposition = Some(Disposition::OutOfScope); - let patch: StewardDecisionPatch = - serde_json::from_value(serde_json::to_value(&completed_batch).unwrap()).unwrap(); + assert!( + serde_json::from_value::( + serde_json::to_value(&completed_batch).unwrap() + ) + .is_err() + ); + let patch = decision_patch_from_review_batch(&report, &worksheet, &completed_batch).unwrap(); let updated = apply_steward_decision_patch(&report, &worksheet, &patch).unwrap(); assert_eq!( updated.decisions[1].reviewed_disposition, @@ -154,7 +159,10 @@ fn completed_review_batch_must_preserve_the_context_shown_to_the_steward() { let patch = decision_patch_from_review_batch(&report, &worksheet, &batch).unwrap(); assert_eq!(patch.decisions.len(), 1); assert_eq!(patch.decisions[0].item_key, "A"); - assert_eq!(patch.decisions[0].reviewed_disposition, Disposition::OutOfScope); + assert_eq!( + patch.decisions[0].reviewed_disposition, + Disposition::OutOfScope + ); for invalid in [ { From d9fb1f9e26ad64cb85bbb434930748e92db4c28a Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sat, 5 Sep 2026 06:18:22 +0900 Subject: [PATCH 06/30] fix(zotero): reject unvalidated review context --- crates/conceptweave-zotero/src/lib.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/crates/conceptweave-zotero/src/lib.rs b/crates/conceptweave-zotero/src/lib.rs index 958eb2f6..31d0f4a5 100644 --- a/crates/conceptweave-zotero/src/lib.rs +++ b/crates/conceptweave-zotero/src/lib.rs @@ -1106,6 +1106,7 @@ pub struct StewardDecisionUpdate { /// A bounded set of local steward decisions for one immutable classification snapshot. #[derive(Debug, Clone, PartialEq, Eq, Deserialize, Serialize)] +#[serde(deny_unknown_fields)] pub struct StewardDecisionPatch { /// Zotero library revision shared with the report and worksheet. pub library_version: u64, From f30179614f9254d5e82a3955d6ff7bc5c0af49d3 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sat, 5 Sep 2026 06:19:06 +0900 Subject: [PATCH 07/30] docs(zotero): bind reviewed batches to displayed evidence --- CHANGELOG.md | 1 + README.md | 6 +++++- crates/conceptweave-zotero/tests/steward_review_batch.rs | 2 +- docs/PRD.md | 1 + docs/TRD.md | 4 +++- docs/adr/0006-zotero-research-intake.md | 3 ++- docs/product-technical-gap-baseline.md | 2 +- 7 files changed, 14 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4f7f3475..f4c9bb2b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -19,6 +19,7 @@ All notable changes to ConceptWeave are documented here. - An explicit `--worksheet` CLI mode that writes the live worksheet with owner-only report protections. - Fail-closed conversion from a fully decided worksheet to the existing externally verified golden-set boundary. - Lossless owner-only classification-report deserialization for offline review finalization. +- Context-bound validation and owner-only application of completed steward review batches. ### Security diff --git a/README.md b/README.md index 02f5a394..829df78f 100644 --- a/README.md +++ b/README.md @@ -24,7 +24,11 @@ Create a small deterministic view of the next pending records for human review: cargo +1.98.0 run --bin conceptweave-zotero -- --review-batch /tmp/report.json /tmp/current-worksheet.json 25 /tmp/review-batch.json ``` -The batch repeats on unchanged input and is not a reservation or assignment. It contains sensitive bibliographic context, must remain owner-only, and becomes a decision patch only after a steward fills every `reviewed_disposition`. +The batch repeats on unchanged input and is not a reservation or assignment. It contains sensitive bibliographic context and must remain owner-only. After a steward fills every `reviewed_disposition`, validate the complete displayed context and create a new worksheet: + +```sh +cargo +1.98.0 run --bin conceptweave-zotero -- --apply-review-batch /tmp/report.json /tmp/current-worksheet.json /tmp/review-batch.json /tmp/updated-worksheet.json +``` [![Ask DeepWiki](https://deepwiki.com/badge.svg)](https://deepwiki.com/ContextualWisdomLab/ConceptWeave) diff --git a/crates/conceptweave-zotero/tests/steward_review_batch.rs b/crates/conceptweave-zotero/tests/steward_review_batch.rs index d743203a..7a8cc0e0 100644 --- a/crates/conceptweave-zotero/tests/steward_review_batch.rs +++ b/crates/conceptweave-zotero/tests/steward_review_batch.rs @@ -21,7 +21,7 @@ fn item(key: &str, title: &str, abstract_note: &str) -> ZoteroItem { } #[test] -fn review_batch_is_deterministic_bounded_and_patch_compatible() { +fn review_batch_is_deterministic_bounded_and_requires_validated_conversion() { let report = classify_snapshot( "9.0.6".into(), None, diff --git a/docs/PRD.md b/docs/PRD.md index f5b67768..70933436 100644 --- a/docs/PRD.md +++ b/docs/PRD.md @@ -75,6 +75,7 @@ During the human review campaign, operators must be able to validate a partially Operators must be able to accumulate small steward-reviewed decision sets into the canonical worksheet without hand-merging the complete JSON document. Each decision patch binds the original library version, classifier revision, snapshot digest, item key, and item revision. Empty, duplicate, unknown, stale, or abstention decisions fail atomically. Reapplying the same decision is idempotent; a different decision cannot overwrite existing review work. Applying a patch does not confer approval or record publication authority. The offline CLI must read the saved report, current worksheet, and decision patch as three distinct owner-only file identities and create a separate updated worksheet. It must never overwrite the current worksheet, reread Zotero, or emit output after invalid input. Operators must be able to extract up to 100 pending rows at a time with the exact report context needed for human review. Batch order is deterministic, decided rows are skipped, and unchanged inputs reproduce the same batch. Batch creation is neither assignment nor progress; only a completed patch accepted into the canonical worksheet increases unverified review coverage. +Applying a completed review batch must rederive the same pending view from the original report and current worksheet, compare every displayed context field, and reject blank or abstaining decisions. A review batch must not be accepted through the context-free decision-patch parser because silently ignored display-field drift would break the link between the steward decision and the evidence shown during review. Every successful classification report includes aggregate evidence for snapshot coverage, proposal coverage, provenance completeness, abstentions, duplicate candidates, disposition totals, and zero unreported failures. diff --git a/docs/TRD.md b/docs/TRD.md index bced06c9..8748c154 100644 --- a/docs/TRD.md +++ b/docs/TRD.md @@ -78,7 +78,9 @@ The owner-only report uses owned JSON values and supports lossless deserializati `conceptweave-zotero --apply-decision-patch /tmp/REPORT.json /tmp/CURRENT_WORKSHEET.json /tmp/PATCH.json /tmp/UPDATED_WORKSHEET.json` exposes that contract through the existing private artifact boundary. Four textual paths and all three opened input device/inode identities must differ. Each input is bounded, regular, single-link, exact `0600`, and opened without following the final symlink; the output is serialized before its create-new `0600` write. Invalid input leaves no output, an existing output is preserved, and identical replay may create another semantically identical worksheet. No Zotero read or authority verifier runs. -`conceptweave-zotero --review-batch /tmp/REPORT.json /tmp/CURRENT_WORKSHEET.json LIMIT /tmp/BATCH.json` validates two distinct owner-only input identities and emits the first 1–100 blank decisions in canonical item-key order. It skips decided rows and rejects invalid, abstaining, complete, or empty worksheets before output creation. Each row carries its exact key/version, item type, title, minimized review abstract, collections, typed tags, proposal, abstention reason, deterministic evidence, and a blank `reviewed_disposition`; global snapshot coordinates bind the batch. Snapshot-item arrays, child keys, model receipts, audit and duplicate aggregates, reviewer identity, and approval fields are omitted. A completed batch is wire-compatible with `StewardDecisionPatch` because its extra review fields are non-authoritative and ignored during patch deserialization; the original report and current worksheet remain authoritative. Repeated unchanged input returns the same batch and creates no lease, allocation, or exclusivity claim. +`conceptweave-zotero --review-batch /tmp/REPORT.json /tmp/CURRENT_WORKSHEET.json LIMIT /tmp/BATCH.json` validates two distinct owner-only input identities and emits the first 1–100 blank decisions in canonical item-key order. It skips decided rows and rejects invalid, abstaining, complete, or empty worksheets before output creation. Each row carries its exact key/version, item type, title, minimized review abstract, collections, typed tags, proposal, abstention reason, deterministic evidence, and a blank `reviewed_disposition`; global snapshot coordinates bind the batch. Snapshot-item arrays, child keys, model receipts, audit and duplicate aggregates, reviewer identity, and approval fields are omitted. Repeated unchanged input returns the same batch and creates no lease, allocation, or exclusivity claim. + +`decision_patch_from_review_batch` rebuilds the expected pending batch at the submitted length and compares every snapshot coordinate, row, and displayed context field after extracting complete non-abstention decisions. `StewardDecisionPatch` rejects unknown JSON fields, so a review batch cannot bypass this check through permissive deserialization. `conceptweave-zotero --apply-review-batch /tmp/REPORT.json /tmp/CURRENT_WORKSHEET.json /tmp/COMPLETED_BATCH.json /tmp/UPDATED_WORKSHEET.json` applies the validated conversion through the existing atomic patch and private artifact boundaries. It reads no Zotero state, writes no partial worksheet, and grants no approval authority. The local report can contain titles, tags, matched metadata, and abstention abstracts. It is sensitive steward-review material, remains outside the repository, and is not a publication artifact. diff --git a/docs/adr/0006-zotero-research-intake.md b/docs/adr/0006-zotero-research-intake.md index c9fb36ab..51cdc000 100644 --- a/docs/adr/0006-zotero-research-intake.md +++ b/docs/adr/0006-zotero-research-intake.md @@ -36,7 +36,8 @@ Incremental steward work is integrated by a snapshot-bound decision patch rather The offline CLI consumes the saved report, current worksheet, and patch as distinct owner-only file identities and emits only a separate create-new worksheet. Reusing the existing private artifact boundary keeps the command local, bounded, and fail-closed without introducing a second review service or repository. The CLI never rereads Zotero, overwrites the source worksheet, or verifies approval. -Human review uses deterministic batches of at most 100 pending records. A batch projects only the matching report context and blank decision slots, remains owner-only, and can become the existing patch wire shape after a steward fills each decision. We choose a repeatable view instead of reservation state because the current campaign has no independent assignment service or cross-product consumer; concurrency ownership must be added only when such a contract exists. Creating a batch does not advance review or approval KPIs. +Human review uses deterministic batches of at most 100 pending records. A batch projects only the matching report context and blank decision slots and remains owner-only. We choose a repeatable view instead of reservation state because the current campaign has no independent assignment service or cross-product consumer; concurrency ownership must be added only when such a contract exists. Creating a batch does not advance review or approval KPIs. +Completed batches cross a dedicated validation boundary before becoming decision patches. The owner rebuilds the pending view from the immutable report and current worksheet, requires every displayed context field to match, and rejects direct batch deserialization as a context-free patch. This keeps the steward's decision bound to what was shown without adding another aggregate or service. The reader fails closed above 50,000 items or 256 MiB of cumulative response bodies, while retaining the 8 MiB per-page bound, finite request timeouts, redirect denial, total-count checks, snapshot-version checks, and duplicate-key detection. Pagination, consistency, resource-budget, and provider-contract behavior are separated from the narrow `ureq` transport so deterministic tests exercise the production reader core rather than excluding the entire reader from coverage. diff --git a/docs/product-technical-gap-baseline.md b/docs/product-technical-gap-baseline.md index d390f42b..6b9f64d2 100644 --- a/docs/product-technical-gap-baseline.md +++ b/docs/product-technical-gap-baseline.md @@ -62,7 +62,7 @@ The steward campaign now has an offline progress checkpoint. It revalidates a pa The campaign integration now applies small snapshot-bound decision patches through an owner-only CLI. It revalidates the complete report and worksheet, rejects both pre-existing reviewed abstentions and abstentions in the incoming patch, applies updates atomically to a clone, permits identical replay, and rejects empty, duplicate, unknown, stale, or conflicting decisions. `--apply-decision-patch` reads three distinct bounded `0600` input identities and creates a separate `0600` worksheet without rereading Zotero or overwriting current review work. Invalid input leaves no output and an existing output remains unchanged. This removes unsafe manual whole-file merging without creating another aggregate or repository. No live steward decision has been applied, so both unverified worksheet coverage and externally approved completion remain 0/3,715; the next Gap is real steward input and measured campaign progress. -The owner CLI can now project the first 1–100 pending records into a deterministic `0600` review batch with the exact local report context needed for a human decision. Decided rows are skipped; complete, empty, tampered, aliased, or unsafe inputs produce no batch. The output omits global snapshot contents plus report audit and duplicate aggregates, repeats an abstention abstract only once, and is directly usable as a decision patch after every blank decision is filled. Unchanged inputs intentionally reproduce the same view; no assignment, lease, or concurrent ownership is claimed. This removes the remaining tooling barrier to collecting authentic labels but is not itself campaign progress: unverified worksheet coverage and externally approved completion remain 0/3,715 until a steward supplies decisions. +The owner CLI can now project the first 1–100 pending records into a deterministic `0600` review batch with the exact local report context needed for a human decision. Decided rows are skipped; complete, empty, tampered, aliased, or unsafe inputs produce no batch. The output omits global snapshot contents plus report audit and duplicate aggregates and repeats an abstention abstract only once. Unchanged inputs intentionally reproduce the same view; no assignment, lease, or concurrent ownership is claimed. A completed batch now crosses a dedicated owner-only apply command that rebuilds the expected pending view, compares every displayed context field, rejects blank or abstaining decisions, and then reuses the atomic decision-patch boundary. The patch parser rejects unknown fields, closing the former path that silently discarded edited batch context. This removes the remaining tooling barrier to collecting authentic labels but is not itself campaign progress: unverified worksheet coverage and externally approved completion remain 0/3,715 until a steward supplies decisions. On 2026-09-05, the real batch command ran against the parent-bound owner-only report/worksheet pair for library version 12341 and snapshot `sha256:c49b08066c4526e520a5f85416543ea20a620a06170e1e15f563088f6bc9e162`. It created a `0600`, 32,940-byte local batch containing 25 blank decisions from 3,715 remaining; the artifact SHA-256 was `7d1a77bd6913bd8c0c826ab60c1a4fa31afede7f7e7e694aad351c31c710b921`. Item keys and bibliographic text remain outside repository evidence. This proves executable campaign intake, not completed human review, allocation, or approval. From b94d86a9b5bbbd7208026f380dedbd524209f54f Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sat, 5 Sep 2026 06:21:14 +0900 Subject: [PATCH 08/30] test(zotero): cover review batch rejection paths --- crates/conceptweave-zotero/src/main.rs | 50 +++++++++++++++++++ .../tests/steward_review_batch.rs | 15 ++++++ 2 files changed, 65 insertions(+) diff --git a/crates/conceptweave-zotero/src/main.rs b/crates/conceptweave-zotero/src/main.rs index 8fdccf9b..e02088ff 100644 --- a/crates/conceptweave-zotero/src/main.rs +++ b/crates/conceptweave-zotero/src/main.rs @@ -759,6 +759,56 @@ mod tests { ); } + #[test] + fn apply_review_batch_mode_requires_four_distinct_artifact_paths() { + let report = "/tmp/report.json"; + let worksheet = "/tmp/worksheet.json"; + let batch = "/tmp/batch.json"; + let output = "/tmp/updated-worksheet.json"; + assert_eq!( + parse_output_request(vec![ + "--apply-review-batch", + report, + worksheet, + batch, + output, + ]), + Ok(OutputRequest::ApplyReviewBatch { + report: report.to_owned(), + worksheet: worksheet.to_owned(), + batch: batch.to_owned(), + output: output.to_owned(), + }) + ); + assert!(parse_output_request(vec!["--apply-review-batch"]).is_err()); + assert!(parse_output_request(vec!["--apply-review-batch", report]).is_err()); + assert!(parse_output_request(vec!["--apply-review-batch", report, worksheet]).is_err()); + assert!( + parse_output_request(vec!["--apply-review-batch", report, worksheet, batch]).is_err() + ); + assert!( + parse_output_request(vec![ + "--apply-review-batch", + report, + worksheet, + batch, + report, + ]) + .is_err() + ); + assert!( + parse_output_request(vec![ + "--apply-review-batch", + report, + worksheet, + batch, + output, + "extra", + ]) + .is_err() + ); + } + #[test] fn review_batch_mode_requires_distinct_paths_and_decimal_limit() { let report = "/tmp/report.json"; diff --git a/crates/conceptweave-zotero/tests/steward_review_batch.rs b/crates/conceptweave-zotero/tests/steward_review_batch.rs index 7a8cc0e0..a6cb8469 100644 --- a/crates/conceptweave-zotero/tests/steward_review_batch.rs +++ b/crates/conceptweave-zotero/tests/steward_review_batch.rs @@ -186,4 +186,19 @@ fn completed_review_batch_must_preserve_the_context_shown_to_the_steward() { Err(WorksheetError::InvalidReport) ); } + + let mut empty = batch.clone(); + empty.decisions.clear(); + assert_eq!( + decision_patch_from_review_batch(&report, &worksheet, &empty), + Err(WorksheetError::InvalidReport) + ); + let mut oversized = batch; + oversized + .decisions + .resize(101, oversized.decisions[0].clone()); + assert_eq!( + decision_patch_from_review_batch(&report, &worksheet, &oversized), + Err(WorksheetError::InvalidBatchLimit) + ); } From 38dcf085810c31d945705830952c458f0e9519ba Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sat, 5 Sep 2026 06:22:38 +0900 Subject: [PATCH 09/30] test(zotero): preserve patch identity coverage --- .../tests/finalization_artifact_identity.rs | 28 +++++++++++++++---- 1 file changed, 23 insertions(+), 5 deletions(-) diff --git a/crates/conceptweave-zotero/tests/finalization_artifact_identity.rs b/crates/conceptweave-zotero/tests/finalization_artifact_identity.rs index 773ce7c7..1162d543 100644 --- a/crates/conceptweave-zotero/tests/finalization_artifact_identity.rs +++ b/crates/conceptweave-zotero/tests/finalization_artifact_identity.rs @@ -1,8 +1,8 @@ #![cfg(unix)] use conceptweave_zotero::{ - Disposition, GoldenSetApproval, ItemData, ZoteroItem, build_steward_review_worksheet, - classify_snapshot, + Disposition, GoldenSetApproval, ItemData, StewardDecisionPatch, StewardDecisionUpdate, + ZoteroItem, build_steward_review_worksheet, classify_snapshot, }; use std::fs; use std::os::unix::fs::PermissionsExt; @@ -37,8 +37,8 @@ fn artifact_commands_reject_distinct_path_spellings_for_one_input_file() { snapshot_items: worksheet.snapshot_items.clone(), }; - // These structs intentionally accept additional owner-only fields. Without checking file - // identity, one JSON object can therefore be accepted as all three finalization inputs. + // The finalization inputs accept overlapping owner-only fields. Without checking file + // identity, one JSON object can therefore be accepted as all three inputs. let mut combined = serde_json::to_value(&report).unwrap(); let object = combined.as_object_mut().unwrap(); object.insert( @@ -64,10 +64,27 @@ fn artifact_commands_reject_distinct_path_spellings_for_one_input_file() { "conceptweave-zotero-finalize-alias-{}-output.json", std::process::id() )); + let patch_path = temp.join(format!( + "conceptweave-zotero-finalize-alias-{}-patch.json", + std::process::id() + )); let _ = fs::remove_file(&input); let _ = fs::remove_file(&output); + let _ = fs::remove_file(&patch_path); fs::write(&input, serde_json::to_vec(&combined).unwrap()).unwrap(); fs::set_permissions(&input, fs::Permissions::from_mode(0o600)).unwrap(); + let patch = StewardDecisionPatch { + library_version: worksheet.library_version, + rule_revision: worksheet.rule_revision.clone(), + snapshot_digest: worksheet.snapshot_digest.clone(), + decisions: vec![StewardDecisionUpdate { + item_key: "ITEM".into(), + item_version: 7, + reviewed_disposition: Disposition::OutOfScope, + }], + }; + fs::write(&patch_path, serde_json::to_vec(&patch).unwrap()).unwrap(); + fs::set_permissions(&patch_path, fs::Permissions::from_mode(0o600)).unwrap(); let report_path = input.to_str().unwrap().to_owned(); let worksheet_path = format!("{}/./{}", temp.display(), filename); @@ -99,7 +116,7 @@ fn artifact_commands_reject_distinct_path_spellings_for_one_input_file() { "--apply-decision-patch", &report_path, &worksheet_path, - &approval_path, + patch_path.to_str().unwrap(), output.to_str().unwrap(), ]) .status() @@ -117,6 +134,7 @@ fn artifact_commands_reject_distinct_path_spellings_for_one_input_file() { let _ = fs::remove_file(&input); let _ = fs::remove_file(&output); + let _ = fs::remove_file(&patch_path); assert!( !status.success(), "finalization must reject three path spellings that resolve to one input artifact" From 96f8a0d2b1cfeea430151ecb04935e2cb7b1228a Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sat, 5 Sep 2026 06:26:13 +0900 Subject: [PATCH 10/30] style(zotero): format batch apply assertion --- .../tests/steward_review_batch_cli.rs | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/crates/conceptweave-zotero/tests/steward_review_batch_cli.rs b/crates/conceptweave-zotero/tests/steward_review_batch_cli.rs index 74736891..73053258 100644 --- a/crates/conceptweave-zotero/tests/steward_review_batch_cli.rs +++ b/crates/conceptweave-zotero/tests/steward_review_batch_cli.rs @@ -93,7 +93,15 @@ fn completed_review_batch_cli_validates_context_before_updating_worksheet() { let tampered_path = private_input("apply-batch-tampered", &tampered); let rejected_path = temp_path("apply-batch-rejected"); let _ = fs::remove_file(&rejected_path); - assert!(!run_apply_batch(&report_path, &worksheet_path, &tampered_path, &rejected_path).success()); + assert!( + !run_apply_batch( + &report_path, + &worksheet_path, + &tampered_path, + &rejected_path + ) + .success() + ); assert!(!rejected_path.exists()); for path in [ From 0118cfebfa18919fe974c5d93068183c1781721b Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sat, 5 Sep 2026 06:29:40 +0900 Subject: [PATCH 11/30] test(zotero): reject unknown review batch context --- .../tests/steward_review_batch.rs | 38 +++++++++++++++++-- 1 file changed, 34 insertions(+), 4 deletions(-) diff --git a/crates/conceptweave-zotero/tests/steward_review_batch.rs b/crates/conceptweave-zotero/tests/steward_review_batch.rs index a6cb8469..8fcaaa87 100644 --- a/crates/conceptweave-zotero/tests/steward_review_batch.rs +++ b/crates/conceptweave-zotero/tests/steward_review_batch.rs @@ -1,7 +1,7 @@ use conceptweave_zotero::{ - Disposition, ItemData, StewardDecisionPatch, WorksheetError, ZoteroItem, - apply_steward_decision_patch, build_steward_review_batch, build_steward_review_worksheet, - classify_snapshot, decision_patch_from_review_batch, + Disposition, ItemData, ItemTag, StewardDecisionPatch, StewardReviewBatch, WorksheetError, + ZoteroItem, apply_steward_decision_patch, build_steward_review_batch, + build_steward_review_worksheet, classify_snapshot, decision_patch_from_review_batch, }; fn item(key: &str, title: &str, abstract_note: &str) -> ZoteroItem { @@ -15,7 +15,10 @@ fn item(key: &str, title: &str, abstract_note: &str) -> ZoteroItem { doi: String::new(), parent_item: String::new(), collections: vec!["COLLECTION".into()], - tags: vec![], + tags: vec![ItemTag { + tag: "review tag".into(), + tag_type: Some(1), + }], }, } } @@ -202,3 +205,30 @@ fn completed_review_batch_must_preserve_the_context_shown_to_the_steward() { Err(WorksheetError::InvalidBatchLimit) ); } + +#[test] +fn review_batch_json_rejects_unknown_context_at_every_object_boundary() { + let report = classify_snapshot( + "9.0.6".into(), + None, + 42, + vec![item("A", "unmatched", "review context")], + ); + let worksheet = build_steward_review_worksheet(&report).unwrap(); + let batch = build_steward_review_batch(&report, &worksheet, 1).unwrap(); + let original = serde_json::to_value(batch).unwrap(); + + let mut invalid = vec![ + original.clone(), + original.clone(), + original.clone(), + original, + ]; + invalid[0]["unexpected"] = serde_json::json!("root"); + invalid[1]["decisions"][0]["unexpected"] = serde_json::json!("decision"); + invalid[2]["decisions"][0]["evidence"]["unexpected"] = serde_json::json!("evidence"); + invalid[3]["decisions"][0]["tags"][0]["unexpected"] = serde_json::json!("tag"); + for invalid in invalid { + assert!(serde_json::from_value::(invalid).is_err()); + } +} From 818ad64e62a222ce28db6607e226e598cd236f9c Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sat, 5 Sep 2026 06:30:48 +0900 Subject: [PATCH 12/30] fix(zotero): parse review context fail closed --- crates/conceptweave-zotero/src/lib.rs | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/crates/conceptweave-zotero/src/lib.rs b/crates/conceptweave-zotero/src/lib.rs index 31d0f4a5..733e0c47 100644 --- a/crates/conceptweave-zotero/src/lib.rs +++ b/crates/conceptweave-zotero/src/lib.rs @@ -67,6 +67,7 @@ pub struct ItemData { /// A Zotero item tag. #[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Deserialize, Serialize)] +#[serde(deny_unknown_fields)] pub struct ItemTag { /// Tag text. pub tag: String, @@ -111,6 +112,7 @@ pub enum AbstentionReason { /// Evidence for a deterministic proposed disposition. #[derive(Debug, Clone, PartialEq, Eq, Deserialize, Serialize)] +#[serde(deny_unknown_fields)] pub struct ClassificationEvidence { /// Metadata fields whose values matched. pub fields: Vec, @@ -1123,6 +1125,7 @@ pub const MAX_REVIEW_BATCH_ITEMS: usize = 100; /// One pending decision with the report context required for human review. #[derive(Debug, Clone, PartialEq, Eq, Deserialize, Serialize)] +#[serde(deny_unknown_fields)] pub struct StewardReviewBatchDecision { /// Stable Zotero item key used to apply the completed decision. pub item_key: String, @@ -1150,6 +1153,7 @@ pub struct StewardReviewBatchDecision { /// Deterministic owner-only view of the next pending steward decisions. #[derive(Debug, Clone, PartialEq, Eq, Deserialize, Serialize)] +#[serde(deny_unknown_fields)] pub struct StewardReviewBatch { /// Zotero library revision shared with the report and worksheet. pub library_version: u64, From 9573e92715d27869c354c0ff5eec96e8c9872c53 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sat, 5 Sep 2026 06:31:27 +0900 Subject: [PATCH 13/30] test(zotero): reject unknown batch fields at CLI boundary --- .../tests/steward_review_batch_cli.rs | 37 +++++++++++++++++-- 1 file changed, 34 insertions(+), 3 deletions(-) diff --git a/crates/conceptweave-zotero/tests/steward_review_batch_cli.rs b/crates/conceptweave-zotero/tests/steward_review_batch_cli.rs index 73053258..17ebe2e3 100644 --- a/crates/conceptweave-zotero/tests/steward_review_batch_cli.rs +++ b/crates/conceptweave-zotero/tests/steward_review_batch_cli.rs @@ -1,7 +1,7 @@ #![cfg(unix)] use conceptweave_zotero::{ - Disposition, ItemData, StewardReviewBatch, StewardReviewWorksheet, ZoteroItem, + Disposition, ItemData, ItemTag, StewardReviewBatch, StewardReviewWorksheet, ZoteroItem, build_steward_review_batch, build_steward_review_worksheet, classify_snapshot, }; use std::fs; @@ -88,7 +88,7 @@ fn completed_review_batch_cli_validates_context_before_updating_worksheet() { Some(Disposition::OutOfScope) ); - let mut tampered: StewardReviewBatch = batch; + let mut tampered: StewardReviewBatch = batch.clone(); tampered.decisions[0].title = "different context".into(); let tampered_path = private_input("apply-batch-tampered", &tampered); let rejected_path = temp_path("apply-batch-rejected"); @@ -104,6 +104,34 @@ fn completed_review_batch_cli_validates_context_before_updating_worksheet() { ); assert!(!rejected_path.exists()); + let original = serde_json::to_value(batch).unwrap(); + let mut unknown = vec![ + original.clone(), + original.clone(), + original.clone(), + original, + ]; + unknown[0]["unexpected"] = serde_json::json!("root"); + unknown[1]["decisions"][0]["unexpected"] = serde_json::json!("decision"); + unknown[2]["decisions"][0]["evidence"]["unexpected"] = serde_json::json!("evidence"); + unknown[3]["decisions"][0]["tags"][0]["unexpected"] = serde_json::json!("tag"); + for (index, invalid) in unknown.into_iter().enumerate() { + let invalid_path = private_input(&format!("apply-batch-unknown-{index}"), &invalid); + let invalid_output = temp_path(&format!("apply-batch-unknown-output-{index}")); + let _ = fs::remove_file(&invalid_output); + assert!( + !run_apply_batch( + &report_path, + &worksheet_path, + &invalid_path, + &invalid_output, + ) + .success() + ); + assert!(!invalid_output.exists()); + fs::remove_file(invalid_path).unwrap(); + } + for path in [ report_path, worksheet_path, @@ -126,7 +154,10 @@ fn item(key: &str, title: &str) -> ZoteroItem { doi: String::new(), parent_item: String::new(), collections: vec![], - tags: vec![], + tags: vec![ItemTag { + tag: "review tag".into(), + tag_type: Some(1), + }], }, } } From a84e6d49aba2a4fd0b0ef303a342922c4ce909bb Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sat, 5 Sep 2026 06:32:00 +0900 Subject: [PATCH 14/30] docs(zotero): record strict review batch schema --- docs/TRD.md | 2 +- docs/adr/0006-zotero-research-intake.md | 2 +- docs/product-technical-gap-baseline.md | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/TRD.md b/docs/TRD.md index 8748c154..6a9ea53b 100644 --- a/docs/TRD.md +++ b/docs/TRD.md @@ -80,7 +80,7 @@ The owner-only report uses owned JSON values and supports lossless deserializati `conceptweave-zotero --review-batch /tmp/REPORT.json /tmp/CURRENT_WORKSHEET.json LIMIT /tmp/BATCH.json` validates two distinct owner-only input identities and emits the first 1–100 blank decisions in canonical item-key order. It skips decided rows and rejects invalid, abstaining, complete, or empty worksheets before output creation. Each row carries its exact key/version, item type, title, minimized review abstract, collections, typed tags, proposal, abstention reason, deterministic evidence, and a blank `reviewed_disposition`; global snapshot coordinates bind the batch. Snapshot-item arrays, child keys, model receipts, audit and duplicate aggregates, reviewer identity, and approval fields are omitted. Repeated unchanged input returns the same batch and creates no lease, allocation, or exclusivity claim. -`decision_patch_from_review_batch` rebuilds the expected pending batch at the submitted length and compares every snapshot coordinate, row, and displayed context field after extracting complete non-abstention decisions. `StewardDecisionPatch` rejects unknown JSON fields, so a review batch cannot bypass this check through permissive deserialization. `conceptweave-zotero --apply-review-batch /tmp/REPORT.json /tmp/CURRENT_WORKSHEET.json /tmp/COMPLETED_BATCH.json /tmp/UPDATED_WORKSHEET.json` applies the validated conversion through the existing atomic patch and private artifact boundaries. It reads no Zotero state, writes no partial worksheet, and grants no approval authority. +`decision_patch_from_review_batch` rebuilds the expected pending batch at the submitted length and compares every snapshot coordinate, row, and displayed context field after extracting complete non-abstention decisions. Batch roots, decision rows, evidence objects, and tag objects reject unknown JSON fields recursively; `StewardDecisionPatch` does the same at its root. A review batch therefore cannot bypass comparison through permissive deserialization. `conceptweave-zotero --apply-review-batch /tmp/REPORT.json /tmp/CURRENT_WORKSHEET.json /tmp/COMPLETED_BATCH.json /tmp/UPDATED_WORKSHEET.json` applies the validated conversion through the existing atomic patch and private artifact boundaries. It reads no Zotero state, writes no partial worksheet, and grants no approval authority. The local report can contain titles, tags, matched metadata, and abstention abstracts. It is sensitive steward-review material, remains outside the repository, and is not a publication artifact. diff --git a/docs/adr/0006-zotero-research-intake.md b/docs/adr/0006-zotero-research-intake.md index 51cdc000..8b38abb7 100644 --- a/docs/adr/0006-zotero-research-intake.md +++ b/docs/adr/0006-zotero-research-intake.md @@ -37,7 +37,7 @@ Incremental steward work is integrated by a snapshot-bound decision patch rather The offline CLI consumes the saved report, current worksheet, and patch as distinct owner-only file identities and emits only a separate create-new worksheet. Reusing the existing private artifact boundary keeps the command local, bounded, and fail-closed without introducing a second review service or repository. The CLI never rereads Zotero, overwrites the source worksheet, or verifies approval. Human review uses deterministic batches of at most 100 pending records. A batch projects only the matching report context and blank decision slots and remains owner-only. We choose a repeatable view instead of reservation state because the current campaign has no independent assignment service or cross-product consumer; concurrency ownership must be added only when such a contract exists. Creating a batch does not advance review or approval KPIs. -Completed batches cross a dedicated validation boundary before becoming decision patches. The owner rebuilds the pending view from the immutable report and current worksheet, requires every displayed context field to match, and rejects direct batch deserialization as a context-free patch. This keeps the steward's decision bound to what was shown without adding another aggregate or service. +Completed batches cross a dedicated validation boundary before becoming decision patches. The owner rebuilds the pending view from the immutable report and current worksheet, requires every displayed context field to match, and rejects unknown fields recursively across batch, decision, evidence, and tag objects as well as direct batch deserialization as a context-free patch. This keeps the steward's decision bound to what was shown without adding another aggregate or service. The reader fails closed above 50,000 items or 256 MiB of cumulative response bodies, while retaining the 8 MiB per-page bound, finite request timeouts, redirect denial, total-count checks, snapshot-version checks, and duplicate-key detection. Pagination, consistency, resource-budget, and provider-contract behavior are separated from the narrow `ureq` transport so deterministic tests exercise the production reader core rather than excluding the entire reader from coverage. diff --git a/docs/product-technical-gap-baseline.md b/docs/product-technical-gap-baseline.md index 6b9f64d2..82f8cf2f 100644 --- a/docs/product-technical-gap-baseline.md +++ b/docs/product-technical-gap-baseline.md @@ -62,7 +62,7 @@ The steward campaign now has an offline progress checkpoint. It revalidates a pa The campaign integration now applies small snapshot-bound decision patches through an owner-only CLI. It revalidates the complete report and worksheet, rejects both pre-existing reviewed abstentions and abstentions in the incoming patch, applies updates atomically to a clone, permits identical replay, and rejects empty, duplicate, unknown, stale, or conflicting decisions. `--apply-decision-patch` reads three distinct bounded `0600` input identities and creates a separate `0600` worksheet without rereading Zotero or overwriting current review work. Invalid input leaves no output and an existing output remains unchanged. This removes unsafe manual whole-file merging without creating another aggregate or repository. No live steward decision has been applied, so both unverified worksheet coverage and externally approved completion remain 0/3,715; the next Gap is real steward input and measured campaign progress. -The owner CLI can now project the first 1–100 pending records into a deterministic `0600` review batch with the exact local report context needed for a human decision. Decided rows are skipped; complete, empty, tampered, aliased, or unsafe inputs produce no batch. The output omits global snapshot contents plus report audit and duplicate aggregates and repeats an abstention abstract only once. Unchanged inputs intentionally reproduce the same view; no assignment, lease, or concurrent ownership is claimed. A completed batch now crosses a dedicated owner-only apply command that rebuilds the expected pending view, compares every displayed context field, rejects blank or abstaining decisions, and then reuses the atomic decision-patch boundary. The patch parser rejects unknown fields, closing the former path that silently discarded edited batch context. This removes the remaining tooling barrier to collecting authentic labels but is not itself campaign progress: unverified worksheet coverage and externally approved completion remain 0/3,715 until a steward supplies decisions. +The owner CLI can now project the first 1–100 pending records into a deterministic `0600` review batch with the exact local report context needed for a human decision. Decided rows are skipped; complete, empty, tampered, aliased, or unsafe inputs produce no batch. The output omits global snapshot contents plus report audit and duplicate aggregates and repeats an abstention abstract only once. Unchanged inputs intentionally reproduce the same view; no assignment, lease, or concurrent ownership is claimed. A completed batch now crosses a dedicated owner-only apply command that rebuilds the expected pending view, compares every displayed context field, rejects blank or abstaining decisions, and then reuses the atomic decision-patch boundary. Batch roots, rows, evidence, tags, and patch roots reject unknown JSON fields, closing the former path that silently discarded edited human-visible context. This removes the remaining tooling barrier to collecting authentic labels but is not itself campaign progress: unverified worksheet coverage and externally approved completion remain 0/3,715 until a steward supplies decisions. On 2026-09-05, the real batch command ran against the parent-bound owner-only report/worksheet pair for library version 12341 and snapshot `sha256:c49b08066c4526e520a5f85416543ea20a620a06170e1e15f563088f6bc9e162`. It created a `0600`, 32,940-byte local batch containing 25 blank decisions from 3,715 remaining; the artifact SHA-256 was `7d1a77bd6913bd8c0c826ab60c1a4fa31afede7f7e7e694aad351c31c710b921`. Item keys and bibliographic text remain outside repository evidence. This proves executable campaign intake, not completed human review, allocation, or approval. From 5325e9a9faaca5014780a7fea74b1a1efa47d833 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sat, 5 Sep 2026 07:15:44 +0900 Subject: [PATCH 15/30] test(zotero): pin canonical output parent boundary --- .../tests/output_parent_canonicalization.rs | 35 +++++++++++++++++++ 1 file changed, 35 insertions(+) create mode 100644 crates/conceptweave-zotero/tests/output_parent_canonicalization.rs diff --git a/crates/conceptweave-zotero/tests/output_parent_canonicalization.rs b/crates/conceptweave-zotero/tests/output_parent_canonicalization.rs new file mode 100644 index 00000000..51b75dcb --- /dev/null +++ b/crates/conceptweave-zotero/tests/output_parent_canonicalization.rs @@ -0,0 +1,35 @@ +#![cfg(unix)] + +#[test] +fn sensitive_output_path_is_rebuilt_from_the_validated_canonical_parent() { + let source = include_str!("../src/main.rs"); + let start = source + .find("fn validate_output_path(raw: &str) -> io::Result {") + .expect("validate_output_path must remain owned by the Zotero CLI boundary"); + let tail = &source[start..]; + let end = tail + .find("\n}\n\n/// Creates a new sensitive report file") + .expect("validate_output_path function boundary must remain inspectable"); + let function = &tail[..end]; + + assert!( + function.contains("let file_name = path.file_name()"), + "validated output must retain only the final filename after canonicalizing its parent" + ); + assert!( + function.contains("let validated_path = resolved_parent.join(file_name);"), + "output writes must use a path rebuilt from the validated canonical parent" + ); + assert!( + function.contains("fs::symlink_metadata(&validated_path)"), + "existence/symlink checks must inspect the same canonical path later opened for creation" + ); + assert!( + function.contains("Ok(validated_path)"), + "callers must receive the canonical rebuilt output path rather than the raw symlink-bearing path" + ); + assert!( + !function.contains("Ok(path)"), + "returning the raw path reintroduces an upper-parent symlink TOCTOU between validation and create_new" + ); +} From 86288cdf5959040a95221c2ca2d99e243d25dc27 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sat, 5 Sep 2026 08:15:25 +0900 Subject: [PATCH 16/30] fix(zotero): rebuild output path from canonical parent --- crates/conceptweave-zotero/src/main.rs | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/crates/conceptweave-zotero/src/main.rs b/crates/conceptweave-zotero/src/main.rs index e02088ff..603117b1 100644 --- a/crates/conceptweave-zotero/src/main.rs +++ b/crates/conceptweave-zotero/src/main.rs @@ -402,13 +402,17 @@ fn validate_output_path(raw: &str) -> io::Result { "report output must be a direct child of the system temp directory", )); } - if fs::symlink_metadata(&path).is_ok() { + let file_name = path.file_name().ok_or_else(|| { + io::Error::new(io::ErrorKind::InvalidInput, "report output has no file name") + })?; + let validated_path = resolved_parent.join(file_name); + if fs::symlink_metadata(&validated_path).is_ok() { return Err(io::Error::new( io::ErrorKind::AlreadyExists, "report output must not already exist or be a symlink", )); } - Ok(path) + Ok(validated_path) } /// Creates a new sensitive report file or fails closed on unsupported platforms. From 22030ae6c8510d9eb8f7b07d98959bb69d2bd286 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sat, 5 Sep 2026 15:22:34 +0900 Subject: [PATCH 17/30] docs(research): restore measured Zotero campaign baseline --- docs/product-technical-gap-baseline.md | 49 +++++++++++++++++++++----- 1 file changed, 40 insertions(+), 9 deletions(-) diff --git a/docs/product-technical-gap-baseline.md b/docs/product-technical-gap-baseline.md index f7ba7dea..0a577d58 100644 --- a/docs/product-technical-gap-baseline.md +++ b/docs/product-technical-gap-baseline.md @@ -2,7 +2,7 @@ **Snapshot:** 2026-09-05 -This file records code-current product and technical gaps. Exact PR/check/run coordinates are evidence snapshots, not mutable-head dependencies. Live protected-branch, PR, issue and workflow state wins whenever it advances after this snapshot. Because this documentation update creates a Foundation successor, the Foundation SHA below is the exact pre-refresh head; PR metadata must be refreshed to the resulting successor SHA. +This file records code-current product and technical gaps. Exact PR/check/run coordinates are evidence snapshots, not mutable-head dependencies. Live protected-branch, PR, issue and workflow state wins whenever it advances after this snapshot. Refresh the changed PR's metadata after each documentation commit; evidence from another PR or an earlier head does not transfer. ## Protected truth and active stack @@ -10,11 +10,11 @@ Protected/default `main` remains `f4f440dd58c77d7cd90dff8a1eb2eeb9a9940425`; onl The active roots observed immediately before this baseline refresh are: -1. Foundation PR #1 — pre-refresh exact head `5cdd319b9425989e632149b243a3308dd630c0ae`, Draft/open/mergeable. The current Foundation makes Product CI draft-aware while keeping Ready/non-Draft quality requirements intact. Product CI still cannot materialize from protected `main` because that branch does not yet contain `.github/workflows/product.yml`. -2. Product-CI bootstrap PR #35 — exact head `daa543ce2cc2b2eb6d35a7265abcf2a7466e7381`, open/non-Draft/mergeable. It adds only the pull-request form of Product CI so #1 can later be marked Ready without a no-op commit. Exact-head CodeQL PR, Security Scan and SAST Semgrep remain queued; `Security Scan / Detect changed scope` is pre-runner with no steps and no runner assignment, and no independent submitted review exists yet. -3. Client Consumption PR #5 — exact head `cbb9cda0c93d8b762195423834f1d6a27dbfa613`, Draft/open/mergeable. The current source retains language-neutral semantic-release admission, integrity, compatibility, diff/resolution and supersession validation. Previously valid review findings are source-repaired, but current protected evidence remains independently required. -4. Source Observation PR #6 — exact head `d255f5c08a621024809c7e076989eccf0662a330`, Draft/open/mergeable. PostgreSQL targeted `ON DELETE SET NULL (...)` / `SET DEFAULT (...)` column provenance and registry/ACL-resolved source identity are source-repaired. The next P0 slice is the concrete bounded read-only PostgreSQL adapter. -5. Zotero Research Classification root PR #9 — exact head `cda546672cd95b5f8bed7024f70e4e6b39a134c8`, Draft/open/mergeable. The dependent research/write-back stack remains proposal/review oriented and does not elevate local classifier output to semantic authority. +1. Foundation PR #1 — exact head `b538470c963e6524ddc0c3f652a46a4fc8265150`, Draft/open. The current Foundation makes Product CI draft-aware while keeping Ready/non-Draft quality requirements intact. Product CI still cannot materialize from protected `main` because that branch does not yet contain `.github/workflows/product.yml`. +2. Product-CI bootstrap PR #35 — exact head `a31ae0c2df920f2794f7ddb456795b04797ab472`, open/non-Draft. It adds the pull-request form of Product CI and removes no-op closed/converted-to-Draft triggers. Scope detection and review admission have executed successfully, while CodeQL, Semgrep, Noema, Strix, Trivy and Scorecard remain queued. The workflow-only diff skips Dependency Review and OSV; these skips do not prove Foundation's dependency-changing checks. No independent approval exists for this head. +3. Client Consumption PR #5 — exact head `fcf36c8a99f015b963c9f812787df127ac2e2f9e`, Draft/open. The current source retains language-neutral semantic-release admission, integrity, compatibility, diff/resolution and supersession validation. Previously valid review findings are source-repaired, but current protected evidence remains independently required. +4. Source Observation PR #6 — exact head `51a7344c6b159df8daaf2fca6540f7b712f5f8c6`, Draft/open. PostgreSQL targeted `ON DELETE SET NULL (...)` / `SET DEFAULT (...)` column provenance and registry/ACL-resolved source identity are source-repaired. The next P0 slice is the concrete bounded read-only PostgreSQL adapter. +5. Zotero Research Classification root PR #9 — exact head `256076d12dec80997960b1db89bec0809f129c90`, Draft/open. Its dependent stack ends at review-batch PR #34, measured at `062a0d9bca086d5a2aaa5d4122f58364115d4f91` before this documentation repair. The stack remains proposal/review oriented and does not elevate local classifier output to semantic authority. Predecessor reviews/checks never transfer to successor heads. No force-push, destructive rebase, self-approval, fail-open scanner substitution or routine administrator bypass is acceptance evidence. @@ -28,7 +28,8 @@ Predecessor reviews/checks never transfer to successor heads. No force-push, des | Client Consumption | ACTIVE_CHILD | Offline Published+Authoritative admission, compatibility, exact resolution/diff, canonical digest verification, detached artifact verification and explicit supersession validation exist. Current exact-head protected evidence and prerequisite integration remain outstanding. | | Quality gate | ACTIVE_PR | Rust 1.98.0, unsafe forbidden, public docs required, exact checkout, fmt, Clippy, tests, rustdoc, owned 100% coverage, Draft-2020-12 schema fixtures, lock freshness and clean-tree checks. Every head movement requires fresh exact-head evidence. | | Security / dependency review | CONSUMER_REVALIDATION_PENDING | The earlier public non-fork exact-range HTTP 403 was traced to an uninitialized repository dependency graph, not to a retryable central workflow defect. `.github#1873` was closed unmerged after enabling Dependabot vulnerability alerts initialized affected graphs and the same exact comparison returned HTTP 200. The hard gate remains fail closed; a current ConceptWeave head must still execute the pinned Dependency Review action successfully before acceptance. | -| Review / runner admission | BLOCKED_OWNER | #35's exact-head central runs are still queued before useful execution; `Detect changed scope` has no runner assignment or steps. Queueing blocks this validation lane only and is not a reason to stop Source Observation or other repository-owned work. | +| Review / runner admission | PENDING_CURRENT_CHECKS | #35's scope/admission jobs have executed successfully, while scanner and model-review jobs remain queued. The active rules require one approving review, resolved review threads and seven central workflows. Queueing is not a reason to stop repository-owned work. | +| Zotero Research Intake | CAMPAIGN_INCOMPLETE | PRD FR-9 and ADRs 0006/0007 have executable local proposal, review, dry-run and recovery contracts. Saved-snapshot proposals cover 3,715/3,715 bibliographic items; unverified steward decisions and externally approved labels both remain 0/3,715. See the campaign evidence below. | | Standards / research | REPAIRED_PENDING_CI | Doctoring remains bound to authoritative standards/primary research and exact implementation contracts; hosted exact-head evidence remains independently required after head changes. | | Release | NOT_STARTED | No immutable ConceptWeave release exists. Version/CHANGELOG/tag/package/semantic_release/SBOM/provenance/reproducibility/rollback are required on the exact protected release head. | @@ -42,11 +43,41 @@ Acceptance remains stricter than the RCA. HTTP 200 availability alone is not GRE ## Central control-plane evidence -Protected central source is `.github/main@b5efbc2762e472e4a380b0503b1f050f76fbb008` at this snapshot. This is evidence only, not a mutable ConceptWeave dependency. +Protected central source is `.github/main@6d7fbebec8aec31d88a30a36e71ca5b3925d241d` at this snapshot. This is evidence only, not a mutable ConceptWeave dependency. - The current central source includes queue/admission and changed-scope/review-runtime repairs already integrated through ordinary protected history. - `.github#1873@41935494aa234eb458f1cc08f006daaa278b9760` is closed/unmerged because repository dependency-graph initialization, not its retry/sleep source delta, was the verified root cause of the observed public-repository 403. -- #35 remains an exact consumer canary for current runner admission and Dependency Review behavior. Its central workflows are queued, so no protected recovery or dependency-review success is inferred from repository settings alone. +- #35 remains a consumer canary for runner admission and applicable workflow security checks. Its workflow-only change skips Dependency Review, so a dependency-changing Foundation run must separately prove that action's success. Already-created runs remain bound to their own central workflow revisions. + +## Zotero research campaign evidence + +The parent integration ending at `062a0d9bca086d5a2aaa5d4122f58364115d4f91` replaced the baseline with Foundation's document and removed the research section present at `a84e6d49aba2a4fd0b0ef303a342922c4ce909bb`. This section restores FR-9 traceability using the saved private artifacts and the current executable. It preserves Foundation's updated status. These records must survive later parent integrations alongside the Foundation, Client and Source Observation evidence. + +On 2026-09-05, the existing `--review-progress` command at `062a0d9...` revalidated the original report and worksheet offline. The report still binds Zotero 9.0.6, API v3/schema 42, library version 12341, rule `ontology-research-v2`, and snapshot `sha256:c49b08066c4526e520a5f85416543ea20a620a06170e1e15f563088f6bc9e162`. This replay does not claim the mutable Zotero library is still at that version. All three original artifacts retained their hashes and remained outside the repository. + +| Measure | Saved-snapshot result | Meaning / remaining work | +| --- | --- | --- | +| Observed records | 8,326 | The original read completed without reported failures; four top-level non-bibliographic items are excluded from the classification denominator. | +| Proposal and provenance coverage | 3,715/3,715 each | Every bibliographic item has a proposal and source coordinates. These counts do not establish classification correctness. | +| Proposed dispositions | 56 adjacent evidence; 1 semantic-consumption bridge; 3,658 abstentions | Deterministic evidence leaves unsupported meaning for review. | +| Duplicate candidates | 49 | Reversible candidate groups, with no record merge or deletion. | +| Unverified worksheet coverage | 0/3,715 | The replay returned `remaining_count=3715` and `complete=false`. | +| First pending batch | 0/25 decisions filled | The existing 32,940-byte batch is a repeatable review view, not an assignment or a completed review. | +| Externally approved full-review coverage | 0/3,715 | No completed full review or externally verified approval receipt is available in this campaign. A sample cannot satisfy this measure. | +| Live write / rollback evidence | Not performed | The saved report originated from Zotero 9.0.6; Zotero 10 adapter tests do not prove approved live execution. | + +Artifact SHA-256 values for reproducibility (no bibliography or item identities): + +- report: `ff13383b88f89fcef94d2f2d7284838b268fb871bed78c75ce5b53bfab2138a8`; +- worksheet: `ad32c8352cb7d84ac3bdcd3a60c975f61e2e19adc3a8294d4c680360071e752b`; +- pending batch: `7d1a77bd6913bd8c0c826ab60c1a4fa31afede7f7e7e694aad351c31c710b921`; +- replayed aggregate progress: `ac79c719037aca2d67bb4b0ea7e84babd8a701506a7d2ec274139d260328f524` (262 bytes, mode `0600`). + +The next campaign step is authentic steward input. After all 25 decisions in the existing batch are filled, `--apply-review-batch` must reconstruct the pending view from the original report/current worksheet, validate every displayed context field, reject unknown fields and blank/abstention decisions, and produce a separate owner-only worksheet. Directly reading that rich batch through `--apply-decision-patch` is rejected because it would discard the displayed context. A successful first application should produce unverified progress of 25/3,715, with 3,690 remaining; no such result is claimed here. The same process must cover all bibliographic items before external full-review approval. PRD FR-9, TRD's Research Intake contract and [ADR 0006](adr/0006-zotero-research-intake.md) define these boundaries. + +[ADR 0007](adr/0007-reviewed-zotero-write-plan.md), [the threat model](../THREAT_MODEL.md) and [PR #18's unresolved transport finding](https://github.com/ContextualWisdomLab/ConceptWeave/pull/18#discussion_r3935881086) retain the remaining write boundary: `Zotero-Server-ID` checks database continuity but does not authenticate the loopback peer or protect a key from a hostile process occupying its port. Enterprise-secure live write-back cannot be claimed without protected provider transport or explicit governance acceptance of that remaining risk, followed by approved write, partial-failure and rollback evidence. + +ConceptWeave remains the owner of research intake and semantic-model generation. `semantic-data-portal` owns catalog/consumption, `context-graph-contracts` owns versioned interop contracts, and `contextual-orchestrator` owns model calls. These are the existing Context Map boundaries, not claims of released integration. ConceptWeave has no immutable release or verified released consumer adoption yet. A separate Utility Repository has no evidenced independent consumer or deployment contract at this snapshot. ## P0 product gaps From 7af3a383c5b6a4bbf9b7d457830b5bc9e5b2aaec Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sat, 5 Sep 2026 15:38:35 +0900 Subject: [PATCH 18/30] docs(research): bind owner inventory to current Zotero evidence --- .../cwl_ontology_capability_inventory.md | 56 +++++++++++++++++++ docs/product-technical-gap-baseline.md | 14 +++++ 2 files changed, 70 insertions(+) create mode 100644 docs/doctoring/cwl_ontology_capability_inventory.md diff --git a/docs/doctoring/cwl_ontology_capability_inventory.md b/docs/doctoring/cwl_ontology_capability_inventory.md new file mode 100644 index 00000000..27ef012f --- /dev/null +++ b/docs/doctoring/cwl_ontology_capability_inventory.md @@ -0,0 +1,56 @@ +# CWL ontology capability inventory + +Evidence snapshot: 2026-09-05. Status: research inventory, not dependency-adoption approval. + +## Scope and evidence limits + +This bounded audit covers eight candidates from the existing ecosystem boundaries, not every CWL repository. Each row separates the owner's documented responsibility, its exact default-branch documentation, and published GitHub release evidence. A name, open PR or protected branch alone does not prove a usable released API. Package registries, deployed behavior, release attestations and consumer conformance were not audited here. + +GitHub repository, branch, README-at-SHA and release endpoints supplied these observations. All eight default branches reported protected. `context-graph-contracts` and `enterprise-architecture-core` default to `develop`; their unprotected `main` branches are not the adoption baseline. GitHub detected MIT for semantic-data-portal, TEPP and LineageWeave, Apache-2.0 for RankWeave, and no SPDX identifier for the other four. An undetected license is unresolved evidence, not permission to adopt. + +## Owner and maturity evidence + +| Candidate | Responsibility and boundary | Exact default-head documentation | Release evidence / next cultivation requirement | +| --- | --- | --- | --- | +| ConceptWeave | Ontology/semantic-model generation, validation, review and release; not bibliography or catalog ownership. | `main@f4f440dd58c77d7cd90dff8a1eb2eeb9a9940425`; [bootstrap README](https://github.com/ContextualWisdomLab/ConceptWeave/blob/f4f440dd58c77d7cd90dff8a1eb2eeb9a9940425/README.md). | No GitHub release returned. Repair Research Intake integrity and pass Foundation's protected gates before claiming a released generator. | +| semantic-data-portal | Catalog, graph/semantic retrieval and governed consumption. Its concept-ingestion API does not prove reviewed ConceptWeave-release consumption. | `main@e48aa13c4af7a4875d4b53e6a60b50405c265a2f`; [API documentation](https://github.com/ContextualWisdomLab/semantic-data-portal/blob/e48aa13c4af7a4875d4b53e6a60b50405c265a2f/README.md). | No GitHub release returned. Prove released ingestion with unreviewed/incompatible-input rejection; do not copy catalog persistence into ConceptWeave. | +| context-graph-contracts | Shared versioned interoperability contracts, not ownership of product meaning. | `develop@99cb5468ba3c15c5e79688f53dee74724fae2d13`; [bootstrap README](https://github.com/ContextualWisdomLab/context-graph-contracts/blob/99cb5468ba3c15c5e79688f53dee74724fae2d13/README.md). | No GitHub release returned. Complete owner contracts, licensing and conformance release before consumer import. | +| enterprise-architecture-core | Enterprise-architecture and transformation decisions; ConceptWeave does not acquire EA authority. | `develop@dd71e40a86385fb7861b0f1be19891a3f3e29ece`; [bootstrap README](https://github.com/ContextualWisdomLab/enterprise-architecture-core/blob/dd71e40a86385fb7861b0f1be19891a3f3e29ece/README.md). | No GitHub release returned. Establish a versioned decision/Context Map contract while preserving product-owned truth. | +| EmbedRelay | Embedding identity and continuity across model migrations; vector similarity remains proposal evidence. | `main@816dcacd4fc1903d91c5cae9b77e37e21811a78d`; [bootstrap README](https://github.com/ContextualWisdomLab/EmbedRelay/blob/816dcacd4fc1903d91c5cae9b77e37e21811a78d/README.md). | No GitHub release returned. Release identity/compatibility contracts before cross-model retrieval; no consumer-side substitute conversion. | +| RankWeave | Retrieval fusion/evaluation adjacent to ontology candidate retrieval; no ontology learning or approval authority. | `main@92323cb8b55baf5d840cb97fa8534a0e75ef234c`; also inspected [release-source README](https://github.com/ContextualWisdomLab/RankWeave/blob/61c49c50d3b4a24fc9bd7c6d3a7f2f4ba19d7be6/README.md). | [v0.18.0](https://github.com/ContextualWisdomLab/RankWeave/releases/tag/v0.18.0), published 2026-08-06, resolves to `61c49c50d3b4a24fc9bd7c6d3a7f2f4ba19d7be6`. This is Python 3.10+ release-source evidence, not a Rust runtime or ConceptWeave adoption proof. Production arithmetic changes belong in the owner under the Rust-first policy. | +| TEPP | Temporal/event/relation measurement supplies bounded evidence; not ontology publication ownership. | `main@a243f18da4a4ca8a8d068c39922537f1f8ed6ad0`; [workspace and limitations](https://github.com/ContextualWisdomLab/TEPP/blob/a243f18da4a4ca8a8d068c39922537f1f8ed6ad0/README.md). | No GitHub release returned. Documented Rust contracts/partial analysis are not a complete commercial estimator. Prove released wire-contract and provenance compatibility before adoption. | +| LineageWeave | Reconstructed lineage candidates supply Source Observation evidence without becoming semantic authority. | `main@83eba56149eb802cd63642c507c324c9976ec78e`; [integration boundaries](https://github.com/ContextualWisdomLab/LineageWeave/blob/83eba56149eb802cd63642c507c324c9976ec78e/README.md). | No GitHub release returned. Verify a released observation contract and real consumer conformance; documented sibling dependencies alone do not establish readiness. | + +No runtime dependency, repository, service or database was added. The [Context Map](../CONTEXT_MAP.md) and [ADR 0006](../adr/0006-zotero-research-intake.md) still place Research Intake in ConceptWeave. A separate utility owner needs an evidenced independent consumer and deployment contract. + +## Actual Zotero evidence + +The read-only executable at `22030ae6c8510d9eb8f7b07d98959bb69d2bd286` captured a distinct report/worksheet pair after detecting Zotero 10.0.1, API 3, schema 44, library version 2 and a present server identity. The earlier 9.0.6/schema-42/library-12341 pair remains historical. Zotero documents that pre-10 synced revisions could remain unchanged after local edits; version 10 uses instance-local revisions (Zotero, 2026). These version spaces must not be compared or merged. Equal record totals do not prove unchanged content. + +The new read contains 8,326 records and 3,715 bibliographic proposals: 56 adjacent-evidence proposals, one semantic-consumption bridge and 3,658 abstentions. Among abstentions, 3,505 have no deterministic rule match and 153 have unsupported rule vocabulary. An abstract exists in review context or matched evidence for 2,715/3,715 records; 1,000 lack an abstract in this report. These measure metadata availability and routing, not ontology relevance or classification accuracy. Missing abstracts and unmatched phrases cannot prove irrelevance. + +No externally approved paper-to-owner link has been demonstrated by this audit. The separate [research-to-capability register](RESEARCH_CAPABILITY_TRACEABILITY.md) supplies design hypotheses and evaluation families, not already-reviewed Zotero labels. PROV distinguishes evidence and producing activities; SKOS supplies vocabulary/label relationships; SHACL specifies graph validation. None grants business approval (Groth & Moreau, 2013; Miles & Bechhofer, 2009; Knublauch & Kontokostas, 2017). + +Two existing PR #10 findings remain source-confirmed at the capture head: [provider metadata lost before hashing](https://github.com/ContextualWisdomLab/ConceptWeave/pull/10#discussion_r3934854209) and [mutable predictions evaluated under an unchanged receipt](https://github.com/ContextualWisdomLab/ConceptWeave/pull/10#discussion_r3934854221). File SHA-256 proves saved artifact bytes, but the current source digest does not yet prove complete raw-provider-field binding. Repair and regenerate before authority promotion. No labels, approval, authorization prompt, write or rollback were performed. + +## KPI and next actions + +| Measure | Observation | Required next evidence | +| --- | --- | --- | +| Bounded owner audit | 8/8 selected candidates have exact default-head documentation and release-query evidence | Expand for actual responsibilities/consumers; this is not an organization-wide denominator. | +| GitHub release with resolved source commit | 1/8 selected candidates | Artifact/provenance and consumer conformance; a tag is insufficient adoption evidence. | +| Verified ConceptWeave adoption | 0 demonstrated in this audit | Released owner contract, exact consumer revision and passing contract/runtime evidence. | +| Unverified steward decisions | 0/3,715 on the new snapshot | Authentic snapshot-bound decisions after source-integrity repair and regeneration. | +| Externally approved full review | 0/3,715 | Complete labels and independently verified approval; no sampled denominator or generated labels. | + +First repair the PR #10 integrity findings and propagate non-force through the dependent stack. Then review the repaired current snapshot, including missing-abstract and unsupported-vocabulary records. Do not replace the full denominator with a hand-picked success sample or heuristic relevance threshold. Maturity observations select the owner for future work, not the labels to silently assign to papers. + +## References + +Groth, P., & Moreau, L. (Eds.). (2013, April 30). *PROV-overview: An overview of the PROV family of documents* (W3C Working Group Note). World Wide Web Consortium. https://www.w3.org/TR/2013/NOTE-prov-overview-20130430/ + +Knublauch, H., & Kontokostas, D. (Eds.). (2017, July 20). *Shapes constraint language (SHACL)* (W3C Recommendation). World Wide Web Consortium. https://www.w3.org/TR/2017/REC-shacl-20170720/ + +Miles, A., & Bechhofer, S. (Eds.). (2009, August 18). *SKOS simple knowledge organization system reference* (W3C Recommendation). World Wide Web Consortium. https://www.w3.org/TR/2009/REC-skos-reference-20090818/ + +Zotero. (2026, July 29). *Zotero local API*. https://www.zotero.org/support/dev/web_api/v3/local_api diff --git a/docs/product-technical-gap-baseline.md b/docs/product-technical-gap-baseline.md index 0a577d58..5453af40 100644 --- a/docs/product-technical-gap-baseline.md +++ b/docs/product-technical-gap-baseline.md @@ -51,6 +51,20 @@ Protected central source is `.github/main@6d7fbebec8aec31d88a30a36e71ca5b3925d24 ## Zotero research campaign evidence +### Current runtime transition and integrity gates + +A fresh read at `22030ae6c8510d9eb8f7b07d98959bb69d2bd286` observed Zotero 10.0.1, API 3/schema 44, library version 2 and a present server identity. It produced a distinct report/worksheet pair without overwriting the historical artifacts below. The full read still counted 8,326 records and 3,715 bibliographic proposals, with 56 adjacent-evidence proposals, one semantic-consumption bridge, 3,658 abstentions and 49 duplicate candidates. The new worksheet's aggregate checkpoint remains 0/3,715, incomplete. Equal totals do not prove unchanged content across the Zotero 9-to-10 version-space transition. + +Both new files have mode `0600`. The report is 6,890,050 bytes with file SHA-256 `d56c8ac70da7f094355748f6611ba47f9d2256bb87f0e24ab84683536e56fb9e`; the worksheet is 1,640,941 bytes with file SHA-256 `919ad3b875846c018eb92df2b2caf5d9a8ed491ede0b4718c07e56dc69bca0d9`. Their implementation-reported snapshot digest is `sha256:bcc50fdf4e16789e7d2651b431817dfc178fdcaad7e0c73360fda2d83351d7b5`, which is not yet proof of complete raw-field binding. + +PR #10's existing findings remain source-confirmed at this capture head: provider fields omitted by the typed input are lost before raw-snapshot hashing, and mutable report predictions can be evaluated under an unchanged approval receipt. These invalidate stronger integrity claims, not the observed aggregate counts. Repair the canonical owner, propagate through the stack and regenerate before approval/promotion. Zotero 10 availability alone does not resolve the loopback confidentiality finding or grant write authority. + +The [CWL ontology capability inventory](doctoring/cwl_ontology_capability_inventory.md) records eight bounded owner candidates with exact default heads. Seven returned no GitHub release; RankWeave v0.18.0 resolves to an exact commit but does not prove Rust runtime availability or ConceptWeave adoption. `context-graph-contracts` and `enterprise-architecture-core` use protected `develop`, not `main`, as their default adoption baseline. This is owner-selection evidence, not an organization-wide census or permission to bypass missing releases. + +### Historical Zotero 9 snapshot + +The following record preserves earlier measurements and execution guidance. It does not authorize applying the old worksheet or batch to the current Zotero 10 snapshot. + The parent integration ending at `062a0d9bca086d5a2aaa5d4122f58364115d4f91` replaced the baseline with Foundation's document and removed the research section present at `a84e6d49aba2a4fd0b0ef303a342922c4ce909bb`. This section restores FR-9 traceability using the saved private artifacts and the current executable. It preserves Foundation's updated status. These records must survive later parent integrations alongside the Foundation, Client and Source Observation evidence. On 2026-09-05, the existing `--review-progress` command at `062a0d9...` revalidated the original report and worksheet offline. The report still binds Zotero 9.0.6, API v3/schema 42, library version 12341, rule `ontology-research-v2`, and snapshot `sha256:c49b08066c4526e520a5f85416543ea20a620a06170e1e15f563088f6bc9e162`. This replay does not claim the mutable Zotero library is still at that version. All three original artifacts retained their hashes and remained outside the repository. From a359c5b9d1013e84f5832506f5a57aec364e6493 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sat, 5 Sep 2026 16:11:01 +0900 Subject: [PATCH 19/30] test(research): expect validated canonical output parent Signed-off-by: Seongho Bae --- crates/conceptweave-zotero/src/main.rs | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/crates/conceptweave-zotero/src/main.rs b/crates/conceptweave-zotero/src/main.rs index 603117b1..ded59b94 100644 --- a/crates/conceptweave-zotero/src/main.rs +++ b/crates/conceptweave-zotero/src/main.rs @@ -403,7 +403,10 @@ fn validate_output_path(raw: &str) -> io::Result { )); } let file_name = path.file_name().ok_or_else(|| { - io::Error::new(io::ErrorKind::InvalidInput, "report output has no file name") + io::Error::new( + io::ErrorKind::InvalidInput, + "report output has no file name", + ) })?; let validated_path = resolved_parent.join(file_name); if fs::symlink_metadata(&validated_path).is_ok() { @@ -1033,7 +1036,10 @@ mod tests { let _ = fs::remove_file(&allowed); assert_eq!( validate_output_path(allowed.to_str().unwrap()).unwrap(), - allowed + env::temp_dir() + .canonicalize() + .unwrap() + .join(allowed.file_name().unwrap()) ); assert!(validate_output_path("relative.json").is_err()); From 6f27da910abb66c4d822cfae04868e8b5c55d7cf Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sat, 5 Sep 2026 16:15:06 +0900 Subject: [PATCH 20/30] test(research): cover missing private output filename rejection --- crates/conceptweave-zotero/src/main.rs | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/crates/conceptweave-zotero/src/main.rs b/crates/conceptweave-zotero/src/main.rs index ded59b94..5912ab5c 100644 --- a/crates/conceptweave-zotero/src/main.rs +++ b/crates/conceptweave-zotero/src/main.rs @@ -1044,6 +1044,10 @@ mod tests { assert!(validate_output_path("relative.json").is_err()); assert!(validate_output_path("/").is_err()); + let missing_name = validate_output_path(env::temp_dir().join("..").to_str().unwrap()) + .expect_err("an allowed parent still requires a file name"); + assert_eq!(missing_name.kind(), io::ErrorKind::InvalidInput); + assert_eq!(missing_name.to_string(), "report output has no file name"); assert!(validate_output_path("/tmp/missing-directory/report.json").is_err()); assert!( validate_output_path( From 2a75051f0082103511222e278de24b2690fe6bfe Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sat, 5 Sep 2026 16:19:22 +0900 Subject: [PATCH 21/30] docs(research): refresh repaired Zotero campaign and owner census --- .../cwl_ontology_capability_inventory.md | 22 ++++++++----- docs/product-technical-gap-baseline.md | 33 ++++++++++++++++--- 2 files changed, 43 insertions(+), 12 deletions(-) diff --git a/docs/doctoring/cwl_ontology_capability_inventory.md b/docs/doctoring/cwl_ontology_capability_inventory.md index 27ef012f..4552a93d 100644 --- a/docs/doctoring/cwl_ontology_capability_inventory.md +++ b/docs/doctoring/cwl_ontology_capability_inventory.md @@ -4,9 +4,11 @@ Evidence snapshot: 2026-09-05. Status: research inventory, not dependency-adopti ## Scope and evidence limits -This bounded audit covers eight candidates from the existing ecosystem boundaries, not every CWL repository. Each row separates the owner's documented responsibility, its exact default-branch documentation, and published GitHub release evidence. A name, open PR or protected branch alone does not prove a usable released API. Package registries, deployed behavior, release attestations and consumer conformance were not audited here. +The paginated organization metadata census returned 76 repositories, including one archived repository and 11 forks. Name/description screening for ontology, semantic, knowledge graph, schema, RDF, SHACL, provenance and lineage was combined with the existing Context Map; metadata matches alone miss owners whose descriptions do not use those words. This expanded the original eight-candidate audit to 12 selected candidates. The other 64 repositories have not received a source-level capability audit, so this is not proof that all relevant implementations have been found. -GitHub repository, branch, README-at-SHA and release endpoints supplied these observations. All eight default branches reported protected. `context-graph-contracts` and `enterprise-architecture-core` default to `develop`; their unprotected `main` branches are not the adoption baseline. GitHub detected MIT for semantic-data-portal, TEPP and LineageWeave, Apache-2.0 for RankWeave, and no SPDX identifier for the other four. An undetected license is unresolved evidence, not permission to adopt. +Each row separates responsibility, exact default-head documentation or tree evidence, and published GitHub release evidence. A name, open PR or protected branch alone does not prove a usable released API. Package registries, deployed behavior, release attestations and consumer conformance were not audited here. CalendarWeave and four-pillars metadata also matched, but their descriptions identify domain-specific calendar/calculation consumers, not a demonstrated shared ontology library; this is a screening disposition, not an architectural exclusion. + +GitHub repository, branch, README-at-SHA, complete-tree and release endpoints supplied these observations. All 12 selected default branches reported protected. `context-graph-contracts` and `enterprise-architecture-core` default to `develop`; their unprotected `main` branches are not the adoption baseline. GitHub detected MIT for semantic-data-portal, TEPP, LineageWeave and fast-mlsirm; Apache-2.0 for RankWeave, graphify, Veilpick and mhtml-etl-gateway; and no SPDX identifier for the other four. An undetected license is unresolved evidence, not permission to adopt. DeepWiki had no indexed evidence for graphify or Veilpick, so their exact GitHub source/tree was used instead. ## Owner and maturity evidence @@ -20,30 +22,34 @@ GitHub repository, branch, README-at-SHA and release endpoints supplied these ob | RankWeave | Retrieval fusion/evaluation adjacent to ontology candidate retrieval; no ontology learning or approval authority. | `main@92323cb8b55baf5d840cb97fa8534a0e75ef234c`; also inspected [release-source README](https://github.com/ContextualWisdomLab/RankWeave/blob/61c49c50d3b4a24fc9bd7c6d3a7f2f4ba19d7be6/README.md). | [v0.18.0](https://github.com/ContextualWisdomLab/RankWeave/releases/tag/v0.18.0), published 2026-08-06, resolves to `61c49c50d3b4a24fc9bd7c6d3a7f2f4ba19d7be6`. This is Python 3.10+ release-source evidence, not a Rust runtime or ConceptWeave adoption proof. Production arithmetic changes belong in the owner under the Rust-first policy. | | TEPP | Temporal/event/relation measurement supplies bounded evidence; not ontology publication ownership. | `main@a243f18da4a4ca8a8d068c39922537f1f8ed6ad0`; [workspace and limitations](https://github.com/ContextualWisdomLab/TEPP/blob/a243f18da4a4ca8a8d068c39922537f1f8ed6ad0/README.md). | No GitHub release returned. Documented Rust contracts/partial analysis are not a complete commercial estimator. Prove released wire-contract and provenance compatibility before adoption. | | LineageWeave | Reconstructed lineage candidates supply Source Observation evidence without becoming semantic authority. | `main@83eba56149eb802cd63642c507c324c9976ec78e`; [integration boundaries](https://github.com/ContextualWisdomLab/LineageWeave/blob/83eba56149eb802cd63642c507c324c9976ec78e/README.md). | No GitHub release returned. Verify a released observation contract and real consumer conformance; documented sibling dependencies alone do not establish readiness. | +| mhtml-etl-gateway | Value-free schema proposals and source-artifact lineage; catalog/network submission remains caller-owned. | `main@e3d21b0a44ab8430009160e4005df18351bf27c9`; [capabilities and connector boundary](https://github.com/ContextualWisdomLab/mhtml-etl-gateway/blob/e3d21b0a44ab8430009160e4005df18351bf27c9/README.md). | [v0.4.0](https://github.com/ContextualWisdomLab/mhtml-etl-gateway/releases/tag/v0.4.0), published 2026-08-12, resolves to `779254927abb1e7cee80fd949907ccd03f9fc7be`; release-source README inspected. Python 3.11–3.14 source and a caller-owned handoff are not ConceptWeave runtime proof. Resolve the Rust-first runtime policy and released contract conformance at the owner before adoption. | +| fast-mlsirm | MLSIRM/MLS2PLM estimation and recovery diagnostics are candidate measurement evidence, not ontology relevance labels or semantic approval. | `main@493326f2de49ea1704da0ded19868ed05d2fe00f`; [numeric and interpretation boundaries](https://github.com/ContextualWisdomLab/fast-mlsirm/blob/493326f2de49ea1704da0ded19868ed05d2fe00f/README.md). | [v0.9.1](https://github.com/ContextualWisdomLab/fast-mlsirm/releases/tag/v0.9.1), published 2026-08-26, resolves to `09f762ded35786dd1078222a4577ff09d649816f`; release-source README inspected. It documents a Rust numeric core with Python/PyO3 API and an explicit NumPy reference path. Validate the real response design and recovery evidence before any ontology-evaluation use; do not invent relevance weights from the library name. | +| graphify | Forked code/document knowledge-graph extraction distinguishes extracted and inferred edges; that distinction alone does not implement governed semantic releases. | `v8@ac16a93bd3f31b86c82f7d90687941e6d5c9776d`; [extraction and backend documentation](https://github.com/ContextualWisdomLab/graphify/blob/ac16a93bd3f31b86c82f7d90687941e6d5c9776d/README.md). | No CWL GitHub release returned. This is a fork of Graphify-Labs/graphify; upstream package documentation is not a CWL release or adoption receipt. Evaluate a versioned evidence port without importing its graph as semantic authority or bypassing contextual-orchestrator for model work. | +| Veilpick | Repository metadata proposes ontology-guided web acquisition, outside ConceptWeave's browser-acquisition boundary. | `develop@8fd6931092ccc2076b10e9eb23ac99b404a9880e`; [complete source tree](https://github.com/ContextualWisdomLab/Veilpick/tree/8fd6931092ccc2076b10e9eb23ac99b404a9880e) contains only `LICENSE`; README endpoint returned 404. | No GitHub release returned. No implemented library, API or accepted owner contract is evidenced at this head. Keep the metadata claim separate from implementation; do not create a dependency from a description. | No runtime dependency, repository, service or database was added. The [Context Map](../CONTEXT_MAP.md) and [ADR 0006](../adr/0006-zotero-research-intake.md) still place Research Intake in ConceptWeave. A separate utility owner needs an evidenced independent consumer and deployment contract. ## Actual Zotero evidence -The read-only executable at `22030ae6c8510d9eb8f7b07d98959bb69d2bd286` captured a distinct report/worksheet pair after detecting Zotero 10.0.1, API 3, schema 44, library version 2 and a present server identity. The earlier 9.0.6/schema-42/library-12341 pair remains historical. Zotero documents that pre-10 synced revisions could remain unchanged after local edits; version 10 uses instance-local revisions (Zotero, 2026). These version spaces must not be compared or merged. Equal record totals do not prove unchanged content. +The repaired read-only executable at `a359c5b9d1013e84f5832506f5a57aec364e6493` captured a new report/worksheet pair from Zotero 10.0.1, API 3, schema 44, library version 2. It binds captured provider JSON values and the actual typed classifier inputs with the versioned snapshot digest. The earlier `22030ae6c8510d9eb8f7b07d98959bb69d2bd286` schema-44 capture is pre-repair observation; the 9.0.6/schema-42/library-12341 pair is also historical. Neither was overwritten. Zotero documents that pre-10 synced revisions could remain unchanged after local edits; version 10 uses instance-local revisions (Zotero, 2026). These version spaces must not be compared or merged. Equal record totals do not prove unchanged content. The new read contains 8,326 records and 3,715 bibliographic proposals: 56 adjacent-evidence proposals, one semantic-consumption bridge and 3,658 abstentions. Among abstentions, 3,505 have no deterministic rule match and 153 have unsupported rule vocabulary. An abstract exists in review context or matched evidence for 2,715/3,715 records; 1,000 lack an abstract in this report. These measure metadata availability and routing, not ontology relevance or classification accuracy. Missing abstracts and unmatched phrases cannot prove irrelevance. No externally approved paper-to-owner link has been demonstrated by this audit. The separate [research-to-capability register](RESEARCH_CAPABILITY_TRACEABILITY.md) supplies design hypotheses and evaluation families, not already-reviewed Zotero labels. PROV distinguishes evidence and producing activities; SKOS supplies vocabulary/label relationships; SHACL specifies graph validation. None grants business approval (Groth & Moreau, 2013; Miles & Bechhofer, 2009; Knublauch & Kontokostas, 2017). -Two existing PR #10 findings remain source-confirmed at the capture head: [provider metadata lost before hashing](https://github.com/ContextualWisdomLab/ConceptWeave/pull/10#discussion_r3934854209) and [mutable predictions evaluated under an unchanged receipt](https://github.com/ContextualWisdomLab/ConceptWeave/pull/10#discussion_r3934854221). File SHA-256 proves saved artifact bytes, but the current source digest does not yet prove complete raw-provider-field binding. Repair and regenerate before authority promotion. No labels, approval, authorization prompt, write or rollback were performed. +The PR #10 findings for [provider metadata lost before hashing](https://github.com/ContextualWisdomLab/ConceptWeave/pull/10#discussion_r3934854209) and [mutable predictions evaluated under an unchanged receipt](https://github.com/ContextualWisdomLab/ConceptWeave/pull/10#discussion_r3934854221) are source-repaired at `e7d4e59f1b55b5954c5f8436527bc96e7ef2fb13` and propagated through all 23 dependent PRs to #34. Finalization at #27 additionally recomputes the complete proposal digest; #28 proves serialized title/evidence mutation fails before the external verifier. An unchanged source digest, rebuilt worksheet or replacement locally calculated proposal digest cannot renew independent approval. Missing proposal bindings in old receipts fail closed and must not be backfilled. The [campaign baseline](../product-technical-gap-baseline.md) records the new private artifact hashes and exact local verification. No labels, approval, authorization prompt, write or rollback were performed; source repair is not a protected merge. ## KPI and next actions | Measure | Observation | Required next evidence | | --- | --- | --- | -| Bounded owner audit | 8/8 selected candidates have exact default-head documentation and release-query evidence | Expand for actual responsibilities/consumers; this is not an organization-wide denominator. | -| GitHub release with resolved source commit | 1/8 selected candidates | Artifact/provenance and consumer conformance; a tag is insufficient adoption evidence. | +| Metadata census / bounded capability audit | 76 metadata records; 12/12 selected candidates have exact default-head README or complete-tree evidence and release-query evidence | Audit actual contracts/consumers; metadata coverage is not full source-level discovery, and 64 repositories remain unaudited at that depth. | +| GitHub release with resolved source commit | 3/12 selected candidates | Artifact/provenance and consumer conformance; a tag is insufficient adoption evidence. | | Verified ConceptWeave adoption | 0 demonstrated in this audit | Released owner contract, exact consumer revision and passing contract/runtime evidence. | -| Unverified steward decisions | 0/3,715 on the new snapshot | Authentic snapshot-bound decisions after source-integrity repair and regeneration. | +| Unverified steward decisions | 0/3,715 on the repaired snapshot; first pending batch has 0/25 decisions | Authentic snapshot-bound decisions; batch generation is not review progress. | | Externally approved full review | 0/3,715 | Complete labels and independently verified approval; no sampled denominator or generated labels. | -First repair the PR #10 integrity findings and propagate non-force through the dependent stack. Then review the repaired current snapshot, including missing-abstract and unsupported-vocabulary records. Do not replace the full denominator with a hand-picked success sample or heuristic relevance threshold. Maturity observations select the owner for future work, not the labels to silently assign to papers. +Review the repaired current snapshot, including missing-abstract and unsupported-vocabulary records, while obtaining exact-head protected Foundation/review evidence. Do not replace the full denominator with a hand-picked success sample or heuristic relevance threshold. Maturity observations select the owner for future work, not the labels to silently assign to papers. ## References diff --git a/docs/product-technical-gap-baseline.md b/docs/product-technical-gap-baseline.md index 5453af40..83d89a26 100644 --- a/docs/product-technical-gap-baseline.md +++ b/docs/product-technical-gap-baseline.md @@ -13,8 +13,8 @@ The active roots observed immediately before this baseline refresh are: 1. Foundation PR #1 — exact head `b538470c963e6524ddc0c3f652a46a4fc8265150`, Draft/open. The current Foundation makes Product CI draft-aware while keeping Ready/non-Draft quality requirements intact. Product CI still cannot materialize from protected `main` because that branch does not yet contain `.github/workflows/product.yml`. 2. Product-CI bootstrap PR #35 — exact head `a31ae0c2df920f2794f7ddb456795b04797ab472`, open/non-Draft. It adds the pull-request form of Product CI and removes no-op closed/converted-to-Draft triggers. Scope detection and review admission have executed successfully, while CodeQL, Semgrep, Noema, Strix, Trivy and Scorecard remain queued. The workflow-only diff skips Dependency Review and OSV; these skips do not prove Foundation's dependency-changing checks. No independent approval exists for this head. 3. Client Consumption PR #5 — exact head `fcf36c8a99f015b963c9f812787df127ac2e2f9e`, Draft/open. The current source retains language-neutral semantic-release admission, integrity, compatibility, diff/resolution and supersession validation. Previously valid review findings are source-repaired, but current protected evidence remains independently required. -4. Source Observation PR #6 — exact head `51a7344c6b159df8daaf2fca6540f7b712f5f8c6`, Draft/open. PostgreSQL targeted `ON DELETE SET NULL (...)` / `SET DEFAULT (...)` column provenance and registry/ACL-resolved source identity are source-repaired. The next P0 slice is the concrete bounded read-only PostgreSQL adapter. -5. Zotero Research Classification root PR #9 — exact head `256076d12dec80997960b1db89bec0809f129c90`, Draft/open. Its dependent stack ends at review-batch PR #34, measured at `062a0d9bca086d5a2aaa5d4122f58364115d4f91` before this documentation repair. The stack remains proposal/review oriented and does not elevate local classifier output to semantic authority. +4. Source Observation PR #6 — exact head `8ed91afcf520efdd53c9103b332d3e277db29a03`, Draft/open. Its independent owner added explicit schema-allowlist count/UTF-8 byte admission and checked overflow rejection after `51a7344c6b159df8daaf2fca6540f7b712f5f8c6`; exact compare and current PR notes were inspected. This does not transfer Zotero tests to that branch. The concrete bounded read-only PostgreSQL adapter remains absent. +5. Zotero Research Classification root PR #9 — exact head `256076d12dec80997960b1db89bec0809f129c90`, Draft/open. Integrity root #10 is `e7d4e59f1b55b5954c5f8436527bc96e7ef2fb13`; all 23 descendants inherited it through ordinary merges ending at review-batch PR #34 `a359c5b9d1013e84f5832506f5a57aec364e6493`. The local coverage follow-up is `6f27da9` before this documentation commit. The stack remains proposal/review oriented and does not elevate local classifier output to semantic authority. Predecessor reviews/checks never transfer to successor heads. No force-push, destructive rebase, self-approval, fail-open scanner substitution or routine administrator bypass is acceptance evidence. @@ -51,7 +51,32 @@ Protected central source is `.github/main@6d7fbebec8aec31d88a30a36e71ca5b3925d24 ## Zotero research campaign evidence -### Current runtime transition and integrity gates +### Repaired current snapshot and source verification + +PR #10 `e7d4e59f1b55b5954c5f8436527bc96e7ef2fb13` binds the complete captured provider JSON value plus the typed inputs actually consumed by the classifier, including omitted/default distinctions and post-decode changes. A separately recomputed complete proposal digest binds the actual records evaluated under a governance receipt. Root tests reproduce lost provider fields and prediction replacement under an unchanged receipt before fixing them. PR #27 records finalization RED `4718073` → GREEN `61fee4c`; PR #28 `25a787a` proves that a valid saved report preserves the proposal digest while changed title/evidence is rejected before external verification. Old approval JSON without that binding fails closed; never synthesize or backfill genuine approval receipts. + +All 23 descendants (#11–#34, excluding absent #14) received the root changes by non-force merge/push, with their original deltas and Draft states preserved. Independent local testing at #34 `a359c5b9d1013e84f5832506f5a57aec364e6493` passed 143 tests across 36 suites, strict workspace Clippy, rustdoc with warnings denied, formatting and the CI contract. The first final-tip coverage run correctly failed: five owned regions in the missing-output-filename rejection were not exercised. Test-only `6f27da9` adds that boundary case without weakening file protection or excluding code; the focused test and full coverage run passed. Existing source-normalized coverage reports 3,197/3,197 regions and 596/596 branch outcomes; functions are 314/314. LLVM's raw instantiated totals remain 3,822/3,908 lines, 5,603/5,734 regions and 528/596 branch outcomes. Do not describe those raw totals as 100% or transfer local evidence to hosted checks. + +The repaired executable at `a359c5b9d1013e84f5832506f5a57aec364e6493` completed a new read-only Zotero 10.0.1/API 3/schema-44/library-2 capture. It observed 8,326 records, 3,715 bibliographic items, complete proposal/provenance counts of 3,715 each, 56 adjacent-evidence proposals, one semantic-consumption bridge, 3,658 abstentions, 49 duplicate candidates and zero reported read failures. The versioned source digest is `sha256:0666dbebfb0c5aa99deb5a6dda1fc02d84bc46d08aaaddf25f5526a18eceef6d`. A distinct first pending batch was generated at test-only follow-up `6f27da9`; generation does not supply decisions. + +All four new artifacts remain private mode `0600`, outside the repository: + +| Artifact | Bytes | File SHA-256 | +| --- | ---: | --- | +| Repaired report | 6,890,050 | `bf45248413f433a537fe8fc62c02b93eef3c7e47ff6245f31610e9ba72031d8d` | +| Repaired worksheet | 1,640,941 | `2093aeffd3907e71d310715889b87e3fbc189cfba620338bf9a81b53ced26f87` | +| Aggregate progress | 258 | `1b0a01798e03d8dff0677ef3b13605979b667cb9a74e690327d87ae7c2d0bd25` | +| First pending review batch | 32,848 | `c8c3143bb23e4ebcb15d4ca789727c96a495d5ca50739e69e9eae2d22426286b` | + +The repaired worksheet has 0/3,715 decisions and the first batch has 0/25; externally approved full-review coverage remains 0/3,715. No authorization prompt, approval, Zotero write, record merge/deletion or rollback was performed. The three historical Zotero 9 artifact hashes below were rechecked unchanged. The pre-repair schema-44 artifacts are also preserved. Stronger source binding is not classification correctness, business approval or loopback peer authentication. + +The [CWL ontology capability inventory](doctoring/cwl_ontology_capability_inventory.md) now separates a 76-repository metadata census from a 12-candidate exact-default-head capability audit. Three selected candidates have GitHub releases with resolved source commits: RankWeave, mhtml-etl-gateway and fast-mlsirm. Veilpick's protected default tree contains only a license despite its ontology-related description; graphify is an upstream fork without a returned CWL GitHub release. These are maturity observations, not adoption receipts. Source-level discovery remains incomplete for 64 repositories, and actual ConceptWeave adoption remains unproved. No additional utility owner is justified yet. + +Next: apply only authentic snapshot-bound steward decisions to the repaired batch, cover all 3,715 items and independently verify full-review approval; continue owner contract discovery and protected Foundation work while external checks/reviews are pending. The current body notes for PRs #11–#33 distinguish inherited source repair from historical head/check claims. No PR was closed, approved, merged or retargeted by this repair. + +### Historical pre-repair Zotero 10 transition + +The following schema-44 observations were captured before integrity repair and are retained for provenance, not reused as current approval inputs. A fresh read at `22030ae6c8510d9eb8f7b07d98959bb69d2bd286` observed Zotero 10.0.1, API 3/schema 44, library version 2 and a present server identity. It produced a distinct report/worksheet pair without overwriting the historical artifacts below. The full read still counted 8,326 records and 3,715 bibliographic proposals, with 56 adjacent-evidence proposals, one semantic-consumption bridge, 3,658 abstentions and 49 duplicate candidates. The new worksheet's aggregate checkpoint remains 0/3,715, incomplete. Equal totals do not prove unchanged content across the Zotero 9-to-10 version-space transition. @@ -59,7 +84,7 @@ Both new files have mode `0600`. The report is 6,890,050 bytes with file SHA-256 PR #10's existing findings remain source-confirmed at this capture head: provider fields omitted by the typed input are lost before raw-snapshot hashing, and mutable report predictions can be evaluated under an unchanged approval receipt. These invalidate stronger integrity claims, not the observed aggregate counts. Repair the canonical owner, propagate through the stack and regenerate before approval/promotion. Zotero 10 availability alone does not resolve the loopback confidentiality finding or grant write authority. -The [CWL ontology capability inventory](doctoring/cwl_ontology_capability_inventory.md) records eight bounded owner candidates with exact default heads. Seven returned no GitHub release; RankWeave v0.18.0 resolves to an exact commit but does not prove Rust runtime availability or ConceptWeave adoption. `context-graph-contracts` and `enterprise-architecture-core` use protected `develop`, not `main`, as their default adoption baseline. This is owner-selection evidence, not an organization-wide census or permission to bypass missing releases. +At that earlier checkpoint, the inventory covered eight bounded owner candidates: seven returned no GitHub release, while RankWeave v0.18.0 resolved to an exact commit. The expanded inventory above supersedes that audit denominator without rewriting its historical observation. `context-graph-contracts` and `enterprise-architecture-core` use protected `develop`, not `main`, as their default adoption baseline. Neither checkpoint permits bypassing missing owner releases. ### Historical Zotero 9 snapshot From 9ee4b99ae6764f54f09e6e8d29860d684a0da9f0 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sat, 5 Sep 2026 16:43:16 +0900 Subject: [PATCH 22/30] docs(research): bind full-text availability and provider contract findings --- CHANGELOG.md | 1 + docs/PRD.md | 2 + docs/TRD.md | 2 + docs/adr/0006-zotero-research-intake.md | 8 +- .../cwl_ontology_capability_inventory.md | 16 ++-- .../zotero_fulltext_contract_audit.md | 80 +++++++++++++++++++ .../doctoring/zotero_fulltext_read_audit.json | 61 ++++++++++++++ docs/product-technical-gap-baseline.md | 10 ++- 8 files changed, 173 insertions(+), 7 deletions(-) create mode 100644 docs/doctoring/zotero_fulltext_contract_audit.md create mode 100644 docs/doctoring/zotero_fulltext_read_audit.json diff --git a/CHANGELOG.md b/CHANGELOG.md index 2a75d813..5d0df103 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,7 @@ All notable changes to ConceptWeave are documented here. ### Added +- Full-library research-source audit separating available text, incomplete indexing and missing material from reviewed classification; no paper is excluded because its abstract or text is unavailable. - Initial ConceptWeave product, DDD, security, test, and operability baselines. - Rust 1.98.0 `conceptweave-domain` foundation with evidence-bound semantic candidate contracts. - Fail-closed Draft -> Proposed -> Validated -> Reviewed -> Published lifecycle with explicit rejection and supersession. diff --git a/docs/PRD.md b/docs/PRD.md index ae6f958b..2067000e 100644 --- a/docs/PRD.md +++ b/docs/PRD.md @@ -58,6 +58,8 @@ All LLM-backed induction uses `contextual-orchestrator`. Model output is untrust Read one immutable Zotero Local API library-version snapshot and propose exactly one research disposition for every top-level bibliographic item. Each proposal retains the item key/version, exact matched metadata values, rule revision, linked child records, and any model receipt. Weak evidence and evidence that matches multiple specific disposition families must abstain into steward review. A local abstention retains its nonempty abstract exactly once, as matched evidence when applicable or otherwise as review context; decided items omit the review-only copy. Duplicate DOI/title identities are review candidates only: intake never merges, deletes, or silently mutates Zotero records. +Full-text enrichment must distinguish listed attachments, returned nonempty text, complete or partial indexing, and reviewed meaning. Missing abstracts or unavailable text never remove papers from the campaign denominator or prove irrelevance. Newly retrieved text requires its own immutable evidence capture and renewed review of any changed proposal; it cannot silently replace evidence beneath an earlier approval. The [full-text audit](doctoring/zotero_fulltext_contract_audit.md) establishes availability only, not an implemented enrichment or completed classification. + For every connected duplicate component, accept externally verified steward decisions selecting one component-level canonical item. Produce a local-only manifest that binds the decisions to the raw snapshot, its complete item-key/item-version coordinates, and exact duplicate-candidate membership, and records every component source revision plus before, after, and rollback canonical-key mappings. Classification preserves every Zotero source record. Reviewed collection and tag changes default to a local dry-run plan. Each operation binds the authority receipt, server/library/item revisions, raw-snapshot digest, and complete before/after/rollback metadata. Execution-critical plan state is immutable outside the owner crate, so callers cannot turn a dry run into execution or alter validated operations. Zotero 9 execute requests fail closed. No plan contains credentials or permits `NeedsStewardReview`, source-record deletion, or attachment deletion. diff --git a/docs/TRD.md b/docs/TRD.md index 82825bb6..6c949e87 100644 --- a/docs/TRD.md +++ b/docs/TRD.md @@ -59,6 +59,8 @@ Evaluation must separate extraction recall, semantic correctness, structural cor ## 11. Zotero research intake +Full-text consumption is not implemented in the current metadata classifier. The [Zotero 10.0.1 audit](doctoring/zotero_fulltext_contract_audit.md) confirms a provider-contract mismatch: full-text list responses omit the documented library-version header and expose versions written by sync, local indexing and local API paths in different version spaces. Treat them as opaque observations, not metadata revisions or reliable incremental cursors. Future enrichment must enumerate from zero, validate API/schema/server and observed attachment/parent membership, bound every response and the whole capture, and bind exact content plus completeness statistics to a separate immutable receipt. A missing, empty or partially indexed response remains explicit. Stable bookend versions/digests do not prove atomicity; no full-text capture may overwrite the earlier report digest or reuse its approval. These are admission requirements, not a claim of a shipped adapter. + `conceptweave-zotero` reads only the loopback Local API with at most 100 records per page, an 8 MiB page-body limit, a 50,000-item whole-snapshot limit, a 256 MiB cumulative body limit, redirects disabled, and finite connect/response/body/global timeouts. Every request pins `Zotero-API-Version: 3`; every response must report API version 3, while its schema version is recorded and must remain stable across the snapshot. Before another request is issued, exhausted whole-snapshot budgets fail closed. Before a parsed page is accumulated, checked item-count and byte arithmetic must remain within both the advertised total and the configured whole-snapshot budgets. `Total-Results`, `Last-Modified-Version`, Zotero version, and server identity must remain identical across all pages; contract drift, malformed JSON, an empty intermediate page, duplicate keys, or an oversized response fails the run. Every top-level bibliographic record receives exactly one proposed disposition. `NeedsStewardReview` also records a deterministic abstention reason so missing classification metadata, vocabulary unsupported by the current deterministic rules, and present-but-unmatched metadata are distinguishable. Its nonempty abstract is retained exactly once in the local report: an abstract that triggered conflicting rules remains in matched evidence, while other abstention abstracts use the review-only field. Non-abstained items omit that field. DOI duplicate identity normalization treats bare DOI values, `doi:`, `doi.org`, and legacy `dx.doi.org` resolver forms as the same identity when their normalized DOI is equal. diff --git a/docs/adr/0006-zotero-research-intake.md b/docs/adr/0006-zotero-research-intake.md index 46aafa29..5cb4a8f6 100644 --- a/docs/adr/0006-zotero-research-intake.md +++ b/docs/adr/0006-zotero-research-intake.md @@ -5,7 +5,7 @@ ## Context -CWL needs a reproducible inventory of ontology research without turning keyword matches into authoritative library organization. The current desktop is Zotero 9.0.6, whose Local API supports reads but not writes. Zotero documents Local API writes only for Zotero 10+, where they require user-granted authorization and `Zotero-Server-ID`; this slice therefore has no mutation capability. The library is mutable while pagination is in progress, duplicate metadata does not prove that two records should be merged, and the local report contains titles and item keys that must not be written into the repository. +CWL needs a reproducible inventory of ontology research without turning keyword matches into authoritative library organization. At the original 2026-09-04 decision snapshot the desktop was Zotero 9.0.6, whose Local API supports reads but not writes. Zotero documents Local API writes only for Zotero 10+, where they require user-granted authorization and `Zotero-Server-ID`; the initial intake slice therefore had no mutation capability. The 2026-09-05 campaign now observes Zotero 10.0.1; write planning remains governed separately by ADR 0007 and does not follow from that upgrade. The library is mutable while pagination is in progress, duplicate metadata does not prove that two records should be merged, and the local report contains titles and item keys that must not be written into the repository. Zotero's Local API documentation states that production clients should request `Zotero-API-Version: 3`; the response exposes `Zotero-API-Version` and `Zotero-Schema-Version`. The API version is the compatibility contract. The schema version is therefore recorded and required to remain stable across the snapshot, but it is not hard-coded to the developer workstation's current schema 42 because Zotero can legitimately revise the local data schema while retaining API v3 compatibility. @@ -47,6 +47,12 @@ No dedicated utility repository or Zotero mutation path is created. A future Zot ## Consequences +### 2026-09-05 full-text evidence amendment (Proposed) + +In the context of reviewing papers with missing abstracts, facing a full-text API whose observed versions mix sync and local writes and whose list lacks the documented version header, we decided for separately captured, content-bound full-text observations and against treating attachment listings or unchanged version counters as complete snapshot evidence, to preserve review provenance and the full campaign denominator, accepting another capture/verification step and no current claim of atomic full-text enrichment. + +The [source-grounded audit](../doctoring/zotero_fulltext_contract_audit.md) retrieved all 3,473 listed entries and demonstrated nonempty text for 3,203/3,715 bibliographic items, including 800/1,000 without retained abstracts. It did not persist raw text or classify those papers. Positive consequence: available source material can guide genuine review without inventing relevance or approval. Negative consequence: content remains unbound to the metadata report until a separate immutable capture contract is implemented and reviewed. Neither a guessed incremental cursor nor adding only the provider's missing header repairs the mixed-version semantics. Direct database edits, cloud credential expansion and a new utility owner are rejected; the provider semantics require an upstream fix, while Research Intake retains the consumer admission boundary. Status remains Proposed. + ### 2026-09-05 integrity amendment (Proposed) In the context of replaying a Zotero research classification against a steward's approved labels, facing source fields lost during projection and predictions mutable after review, we decided for separate source-and-input and proposal-content digests verified with the complete reviewed set, and against typed-only source hashing or a report's self-declared cached proposal identity, to preserve the exact evidence used for evaluation, accepting a receipt-format break, report regeneration and fresh governance approval. diff --git a/docs/doctoring/cwl_ontology_capability_inventory.md b/docs/doctoring/cwl_ontology_capability_inventory.md index 4552a93d..65ee7dca 100644 --- a/docs/doctoring/cwl_ontology_capability_inventory.md +++ b/docs/doctoring/cwl_ontology_capability_inventory.md @@ -4,11 +4,11 @@ Evidence snapshot: 2026-09-05. Status: research inventory, not dependency-adopti ## Scope and evidence limits -The paginated organization metadata census returned 76 repositories, including one archived repository and 11 forks. Name/description screening for ontology, semantic, knowledge graph, schema, RDF, SHACL, provenance and lineage was combined with the existing Context Map; metadata matches alone miss owners whose descriptions do not use those words. This expanded the original eight-candidate audit to 12 selected candidates. The other 64 repositories have not received a source-level capability audit, so this is not proof that all relevant implementations have been found. +The paginated organization metadata census returned 76 repositories, including one archived repository and 11 forks. Name/description screening for ontology, semantic, knowledge graph, schema, RDF, SHACL, provenance and lineage was combined with the existing Context Map; metadata matches alone miss owners whose descriptions do not use those words. This expanded the original eight-candidate audit to 12 selected candidates. The organization product-goal directive then identified DiskSage's actual OWL use, expanding the audit to 13. The other 63 repositories have not received a source-level capability audit, so this is not proof that all relevant implementations have been found. Each row separates responsibility, exact default-head documentation or tree evidence, and published GitHub release evidence. A name, open PR or protected branch alone does not prove a usable released API. Package registries, deployed behavior, release attestations and consumer conformance were not audited here. CalendarWeave and four-pillars metadata also matched, but their descriptions identify domain-specific calendar/calculation consumers, not a demonstrated shared ontology library; this is a screening disposition, not an architectural exclusion. -GitHub repository, branch, README-at-SHA, complete-tree and release endpoints supplied these observations. All 12 selected default branches reported protected. `context-graph-contracts` and `enterprise-architecture-core` default to `develop`; their unprotected `main` branches are not the adoption baseline. GitHub detected MIT for semantic-data-portal, TEPP, LineageWeave and fast-mlsirm; Apache-2.0 for RankWeave, graphify, Veilpick and mhtml-etl-gateway; and no SPDX identifier for the other four. An undetected license is unresolved evidence, not permission to adopt. DeepWiki had no indexed evidence for graphify or Veilpick, so their exact GitHub source/tree was used instead. +GitHub repository, branch, README-at-SHA, complete-tree and release endpoints supplied these observations. All 13 selected default branches reported protected at their recorded observations. `context-graph-contracts` and `enterprise-architecture-core` default to `develop`; their unprotected `main` branches are not the adoption baseline. GitHub detected MIT for semantic-data-portal, TEPP, LineageWeave, fast-mlsirm and disksage; Apache-2.0 for RankWeave, graphify, Veilpick and mhtml-etl-gateway; and no SPDX identifier for the other four. An undetected license is unresolved evidence, not permission to adopt. DeepWiki had no indexed evidence for graphify, Veilpick or disksage, so their exact GitHub source/tree was used instead. ## Owner and maturity evidence @@ -26,15 +26,20 @@ GitHub repository, branch, README-at-SHA, complete-tree and release endpoints su | fast-mlsirm | MLSIRM/MLS2PLM estimation and recovery diagnostics are candidate measurement evidence, not ontology relevance labels or semantic approval. | `main@493326f2de49ea1704da0ded19868ed05d2fe00f`; [numeric and interpretation boundaries](https://github.com/ContextualWisdomLab/fast-mlsirm/blob/493326f2de49ea1704da0ded19868ed05d2fe00f/README.md). | [v0.9.1](https://github.com/ContextualWisdomLab/fast-mlsirm/releases/tag/v0.9.1), published 2026-08-26, resolves to `09f762ded35786dd1078222a4577ff09d649816f`; release-source README inspected. It documents a Rust numeric core with Python/PyO3 API and an explicit NumPy reference path. Validate the real response design and recovery evidence before any ontology-evaluation use; do not invent relevance weights from the library name. | | graphify | Forked code/document knowledge-graph extraction distinguishes extracted and inferred edges; that distinction alone does not implement governed semantic releases. | `v8@ac16a93bd3f31b86c82f7d90687941e6d5c9776d`; [extraction and backend documentation](https://github.com/ContextualWisdomLab/graphify/blob/ac16a93bd3f31b86c82f7d90687941e6d5c9776d/README.md). | No CWL GitHub release returned. This is a fork of Graphify-Labs/graphify; upstream package documentation is not a CWL release or adoption receipt. Evaluate a versioned evidence port without importing its graph as semantic authority or bypassing contextual-orchestrator for model work. | | Veilpick | Repository metadata proposes ontology-guided web acquisition, outside ConceptWeave's browser-acquisition boundary. | `develop@8fd6931092ccc2076b10e9eb23ac99b404a9880e`; [complete source tree](https://github.com/ContextualWisdomLab/Veilpick/tree/8fd6931092ccc2076b10e9eb23ac99b404a9880e) contains only `LICENSE`; README endpoint returned 404. | No GitHub release returned. No implemented library, API or accepted owner contract is evidenced at this head. Keep the metadata claim separate from implementation; do not create a dependency from a description. | +| DiskSage (`disksage`) | Implemented filesystem taxonomy and ontology-driven organization; candidate for bounded reuse, not ownership of general semantic-model publication. | `main@0e90f9cebadbd7f59606baaec4ca1d2f178c899a`; [bundled eight-class ontology](https://github.com/ContextualWisdomLab/disksage/blob/0e90f9cebadbd7f59606baaec4ca1d2f178c899a/src-tauri/resources/ontology/default.ttl) and [Rust named-class reasoning subset](https://github.com/ContextualWisdomLab/disksage/blob/0e90f9cebadbd7f59606baaec4ca1d2f178c899a/src-tauri/src/ontology.rs). | No GitHub release returned. Protected source implements subclass/equivalence closure and disjoint-class checks; it is not full OWL reasoning or a released library. [Accepted ADR 0010](https://github.com/ContextualWisdomLab/disksage/blob/0e90f9cebadbd7f59606baaec4ca1d2f178c899a/docs/architecture/adr/0010-rooted-organize-destinations.md) constrains organize destinations. Establish a released owner contract with provenance and locale preservation before reuse; do not copy the private module. | + +DiskSage demonstrates why metadata screening alone is insufficient. Its [desktop registration](https://github.com/ContextualWisdomLab/disksage/blob/0e90f9cebadbd7f59606baaec4ca1d2f178c899a/src-tauri/src/lib.rs#L122) connects ontology/coherence/inventory/organization commands, while its [catalog adapter](https://github.com/ContextualWisdomLab/disksage/blob/0e90f9cebadbd7f59606baaec4ca1d2f178c899a/src-tauri/src/semantic_catalog.rs) emits a bounded version-1 preview, not catalog publication. The [package](https://github.com/ContextualWisdomLab/disksage/blob/0e90f9cebadbd7f59606baaec4ca1d2f178c899a/src-tauri/Cargo.toml) is `publish=false`. Its parser retains the first label without preserving a locale map, and comments saying no reasoning/first parent only lag the implementation. These are owner cultivation findings, not authorization to extract its product truth or a claim that this audit ran its tests/runtime. Local branch-only deletion/retention additions are excluded from protected-main evidence. No runtime dependency, repository, service or database was added. The [Context Map](../CONTEXT_MAP.md) and [ADR 0006](../adr/0006-zotero-research-intake.md) still place Research Intake in ConceptWeave. A separate utility owner needs an evidenced independent consumer and deployment contract. ## Actual Zotero evidence -The repaired read-only executable at `a359c5b9d1013e84f5832506f5a57aec364e6493` captured a new report/worksheet pair from Zotero 10.0.1, API 3, schema 44, library version 2. It binds captured provider JSON values and the actual typed classifier inputs with the versioned snapshot digest. The earlier `22030ae6c8510d9eb8f7b07d98959bb69d2bd286` schema-44 capture is pre-repair observation; the 9.0.6/schema-42/library-12341 pair is also historical. Neither was overwritten. Zotero documents that pre-10 synced revisions could remain unchanged after local edits; version 10 uses instance-local revisions (Zotero, 2026). These version spaces must not be compared or merged. Equal record totals do not prove unchanged content. +The repaired read-only executable at `a359c5b9d1013e84f5832506f5a57aec364e6493` captured a new report/worksheet pair from Zotero 10.0.1, API 3, schema 44, library version 2. It binds captured provider JSON values and the actual typed classifier inputs with the versioned snapshot digest. The earlier `22030ae6c8510d9eb8f7b07d98959bb69d2bd286` schema-44 capture is pre-repair observation; the 9.0.6/schema-42/library-12341 pair is also historical. Neither was overwritten. Zotero documents that pre-10 synced revisions could remain unchanged after local edits; version 10 metadata uses instance-local revisions (Zotero, 2026). These version spaces must not be compared or merged. Equal record totals do not prove unchanged content. The separate full-text endpoints have an observed version-contract mismatch and must not inherit this metadata-version assumption. The new read contains 8,326 records and 3,715 bibliographic proposals: 56 adjacent-evidence proposals, one semantic-consumption bridge and 3,658 abstentions. Among abstentions, 3,505 have no deterministic rule match and 153 have unsupported rule vocabulary. An abstract exists in review context or matched evidence for 2,715/3,715 records; 1,000 lack an abstract in this report. These measure metadata availability and routing, not ontology relevance or classification accuracy. Missing abstracts and unmatched phrases cannot prove irrelevance. +The [subsequent full-text audit](zotero_fulltext_contract_audit.md) queried all 3,473 listed attachments: 3,432 returned HTTP 200 and 41 returned 404. Nonempty text was observed for 3,203/3,715 bibliographic parents, including 800/1,000 without retained abstracts; complete index counters accompanied nonempty text for 2,561 parents. These are separate, non-atomic availability observations, not text-bound classifier proposals. The existing report, worksheet and approvals were not changed. + No externally approved paper-to-owner link has been demonstrated by this audit. The separate [research-to-capability register](RESEARCH_CAPABILITY_TRACEABILITY.md) supplies design hypotheses and evaluation families, not already-reviewed Zotero labels. PROV distinguishes evidence and producing activities; SKOS supplies vocabulary/label relationships; SHACL specifies graph validation. None grants business approval (Groth & Moreau, 2013; Miles & Bechhofer, 2009; Knublauch & Kontokostas, 2017). The PR #10 findings for [provider metadata lost before hashing](https://github.com/ContextualWisdomLab/ConceptWeave/pull/10#discussion_r3934854209) and [mutable predictions evaluated under an unchanged receipt](https://github.com/ContextualWisdomLab/ConceptWeave/pull/10#discussion_r3934854221) are source-repaired at `e7d4e59f1b55b5954c5f8436527bc96e7ef2fb13` and propagated through all 23 dependent PRs to #34. Finalization at #27 additionally recomputes the complete proposal digest; #28 proves serialized title/evidence mutation fails before the external verifier. An unchanged source digest, rebuilt worksheet or replacement locally calculated proposal digest cannot renew independent approval. Missing proposal bindings in old receipts fail closed and must not be backfilled. The [campaign baseline](../product-technical-gap-baseline.md) records the new private artifact hashes and exact local verification. No labels, approval, authorization prompt, write or rollback were performed; source repair is not a protected merge. @@ -43,9 +48,10 @@ The PR #10 findings for [provider metadata lost before hashing](https://github.c | Measure | Observation | Required next evidence | | --- | --- | --- | -| Metadata census / bounded capability audit | 76 metadata records; 12/12 selected candidates have exact default-head README or complete-tree evidence and release-query evidence | Audit actual contracts/consumers; metadata coverage is not full source-level discovery, and 64 repositories remain unaudited at that depth. | -| GitHub release with resolved source commit | 3/12 selected candidates | Artifact/provenance and consumer conformance; a tag is insufficient adoption evidence. | +| Metadata census / bounded capability audit | 76 metadata records; 13/13 selected candidates have exact default-head documentation/source/tree and release-query evidence | Audit actual contracts/consumers; metadata coverage is not full source-level discovery, and 63 repositories remain unaudited at that depth. | +| GitHub release with resolved source commit | 3/13 selected candidates | Artifact/provenance and consumer conformance; a tag is insufficient adoption evidence. | | Verified ConceptWeave adoption | 0 demonstrated in this audit | Released owner contract, exact consumer revision and passing contract/runtime evidence. | +| Demonstrated nonempty full-text availability | 3,203/3,715 parents, including 800/1,000 without retained abstracts | Separate immutable content capture, explicit partial/unknown indexing and review; neither an atomic metadata snapshot nor classification progress. | | Unverified steward decisions | 0/3,715 on the repaired snapshot; first pending batch has 0/25 decisions | Authentic snapshot-bound decisions; batch generation is not review progress. | | Externally approved full review | 0/3,715 | Complete labels and independently verified approval; no sampled denominator or generated labels. | diff --git a/docs/doctoring/zotero_fulltext_contract_audit.md b/docs/doctoring/zotero_fulltext_contract_audit.md new file mode 100644 index 00000000..7394976e --- /dev/null +++ b/docs/doctoring/zotero_fulltext_contract_audit.md @@ -0,0 +1,80 @@ +# Zotero full-text availability and version-contract audit + +Status: observed provider-contract gap; proposed consumer requirements, not implemented full-text classification. Observation: 2026-09-05 07:36:42 UTC. Related work: [PR #34](https://github.com/ContextualWisdomLab/ConceptWeave/pull/34), [PRD FR-9](../PRD.md), [TRD Research Intake](../TRD.md), [Proposed ADR 0006](../adr/0006-zotero-research-intake.md). + +## Motivation and scope + +The repaired metadata campaign covers 3,715 bibliographic items, but 3,658 proposals abstain and 1,000 reports retain no abstract. Missing metadata is not evidence of irrelevance. Before requesting another acquisition service or model integration, this audit tests the existing Local API's full-text surface. It neither reads Zotero's database directly nor modifies source records, review decisions or earlier reports. + +The [aggregate audit record](zotero_fulltext_read_audit.json) binds this observation to ConceptWeave `2a75051f0082103511222e278de24b2690fe6bfe`, the repaired report's file/content digests, and Zotero 10.0.1/API 3/schema 44. It contains no item keys, bibliography, full text, reviewer identity, server identity or credentials. This was a bounded read-only diagnostic, not a new released adapter or a classifier experiment. + +## Observed full-denominator results + +The manifest returned 3,473 entries, all linked to the prior metadata snapshot. They identify 3,241 candidate bibliographic parents. An entry is not proof that its content exists: the subsequent sweep attempted every entry, yielding 3,432 HTTP 200 responses and 41 HTTP 404 responses, with no other status. Five returned content strings were empty after trimming. Nonempty content linked to 3,203 distinct bibliographic parents. + +| Retained metadata | Bibliographic denominator | Nonempty full text returned | Nonempty full text and complete index counters | +| --- | ---: | ---: | ---: | +| Abstract absent from report | 1,000 | 800 | 598 | +| Abstract present in report | 2,715 | 2,403 | 1,963 | +| Total | 3,715 | 3,203 | 2,561 | + +At attachment level, 2,755 responses had complete index counters, 67 partial counters, and 610 did not prove completeness under the audit predicate. That predicate requires at least one pages/characters pair; every present pair must consist of nonnegative integers, a positive total and indexed count no greater than total. Completeness additionally requires equality for every present pair. The 610 are not declared corrupt: absent, zero or otherwise unusable counters cannot establish completeness. Counter equality does not verify OCR quality, language coverage, scientific relevance or extraction accuracy. + +The remaining 512 bibliographic parents have no demonstrated nonempty full text in this sweep; 200 also lack a retained abstract. They remain in the full campaign denominator. These are follow-up research needs, not exclusion decisions. Full-text-bound proposals and independently approved labels both remain zero. + +## Reproduction and privacy boundary + +The diagnostic read the existing owner-only repaired report in memory. It requested `items?limit=1` before the sweep, `fulltext?since=0`, every listed attachment's `items/{itemKey}/fulltext`, then the manifest and one-item endpoint again. Item keys were interpolated only in process memory; no URL, content or key was printed. All requests used fixed loopback port 23119, API v3, the report's expected server identity, redirect rejection and API/schema/server continuity checks. Reads required no authorization prompt or key. + +The execution was sequential with a 20-second request timeout, an 8 MiB response bound, a 512 MiB cumulative bound and a five-minute before-request admission budget. Actual elapsed time was 10,737 ms and transferred response bodies totaled 224,842,838 bytes. These diagnostic limits are not the production metadata reader's limits or model timeouts. A deadline/budget/identity/JSON failure would make the sweep incomplete; omitted requests must remain visible in its denominator. + +Raw text was held for one response at a time and discarded. The ordered-response SHA-256 streams compact JSON arrays of attachment key, HTTP status, observed content-version header and body SHA-256, in lexicographic key order. The public record retains only the final aggregate digest, not those arrays. Body hashes are over received bytes, not canonical JSON. They identify this sweep's responses but cannot replay discarded source content or serve as an approval receipt. + +These standalone probes expose only aggregate counts and public protocol values, not response bodies or identities: + +```sh +curl --silent --show-error --fail --max-time 20 \ + -H 'Zotero-API-Version: 3' \ + 'http://127.0.0.1:23119/api/users/0/fulltext?since=0' \ + | jq '{manifest_entries: length, version_types: ([.[] | type] | unique), minimum_version: ([.[]] | min), maximum_version: ([.[]] | max)}' + +curl --silent --show-error --fail --max-time 20 --output /dev/null \ + --write-out 'HTTP %{http_code}; version=%header{last-modified-version}; API=%header{zotero-api-version}; schema=%header{zotero-schema-version}\n' \ + -H 'Zotero-API-Version: 3' \ + 'http://127.0.0.1:23119/api/users/0/fulltext?since=0' +``` + +These probes do not reproduce the parent/content sweep or establish a coherent snapshot. The complete audit additionally requires the original private report and the guarded enumeration described above. No raw content was committed or used for generated classifications. + +## Root cause: a mixed-origin cursor and a missing header + +The official documentation says Zotero 10 full-text versions are local and describes a library `Last-Modified-Version` on the list response (Zotero, 2026a, 2026b). The observed list omitted that header. The one-item endpoint reported local library version 2, while the manifest contained 41 zero versions and 3,432 versions greater than 2, with a maximum of 12,403. Each successful content response's version matched its manifest entry. + +The official `10.0.1` tag resolves to `36749bd0bd4fdac9ee46c16f7aa7bed094a0851f`. Its source explains the discrepancy: + +| Producer / endpoint | Exact behavior and evidence | +| --- | --- | +| List and individual content reads | The [endpoint source](https://github.com/zotero/zotero/blob/36749bd0bd4fdac9ee46c16f7aa7bed094a0851f/chrome/content/zotero/xpcom/server/server_localAPI.js#L1431-L1487) reads the stored full-text version directly. The list returns a raw response tuple without the library-version header. | +| Sync download and upload | The [sync engine](https://github.com/zotero/zotero/blob/36749bd0bd4fdac9ee46c16f7aa7bed094a0851f/chrome/content/zotero/xpcom/sync/syncFullTextEngine.js#L99-L168) passes remote content/server library versions into the same full-text version storage. | +| Ordinary indexing | The [index writer](https://github.com/zotero/zotero/blob/36749bd0bd4fdac9ee46c16f7aa7bed094a0851f/chrome/content/zotero/xpcom/fulltext.js#L510-L527) defaults a missing version to zero. | +| Local API writes | The [write endpoint](https://github.com/zotero/zotero/blob/36749bd0bd4fdac9ee46c16f7aa7bed094a0851f/chrome/content/zotero/xpcom/server/server_localAPI.js#L2594-L2622) supplies the incremented local library version to that same storage. No such write was performed in this audit. | +| Upgrade migration | [Migration 129](https://github.com/zotero/zotero/blob/36749bd0bd4fdac9ee46c16f7aa7bed094a0851f/chrome/content/zotero/xpcom/schema.js#L3726-L3730) adds local versions to metadata objects/libraries, not full-text records. | +| Missing-content entries | The [missing-content path](https://github.com/zotero/zotero/blob/36749bd0bd4fdac9ee46c16f7aa7bed094a0851f/chrome/content/zotero/xpcom/fulltext.js#L1450-L1459) can insert an empty zero-version row. Manifest presence is therefore weaker than retrievability. | + +This is a provider contract mismatch, not evidence of a corrupt user library. The manifest bytes and metadata library version were unchanged at the bookends, but those observations do not establish atomicity or rule out a same-version full-text edit. The mixed-origin field must not become a reliable incremental cursor, an item revision or a write precondition. Adding the missing header alone would not fix the version semantics. + +## Proposed admission and owner follow-up + +Research Intake remains in ConceptWeave. Full text needs a separate immutable capture receipt binding server/API/schema observations, attachment and bibliographic-parent identities, content and index-statistics digests, read interval, returned status and partial/unknown coverage. Reusing an old metadata digest or governance receipt for later text is forbidden. An availability sweep may guide retrieval and steward work but cannot make unsupported content authoritative or renew prior approval. + +The provider fix belongs in Zotero: cover upgrade from synced records, local indexing/reindexing, sync downloads/uploads, local API content writes, missing/pending content, and list/header consistency with regression tests. Until a released provider contract proves those semantics, a consumer must re-enumerate from zero and treat version fields as opaque observations. A complete capture can claim exactly the bytes observed, never an atomic cross-endpoint snapshot on the evidence available here. No upstream issue or provider patch was published by this audit. + +For model-assisted proposals, protected `contextual-orchestrator/main@a080297d2546bb61e89520d637cabc202db331ec` documents API use, but its queried GitHub releases/tags returned empty and the queried PyPI project endpoint returned 404. Those checks do not rule out every deployment or registry; they leave a released integration artifact unverified. Do not replace that missing evidence with provider calls, copied owner source, invented review labels or a new utility repository. + +## References + +Zotero. (2026a, July 29). *Zotero local API*. https://www.zotero.org/support/dev/web_api/v3/local_api + +Zotero. (2026b, July 29). *Zotero Web API full-text content requests*. https://www.zotero.org/support/dev/web_api/v3/fulltext_content + +Zotero. (n.d.). *Zotero* (Version 10.0.1, commit 36749bd0bd4fdac9ee46c16f7aa7bed094a0851f) [Computer software]. GitHub. https://github.com/zotero/zotero/tree/36749bd0bd4fdac9ee46c16f7aa7bed094a0851f diff --git a/docs/doctoring/zotero_fulltext_read_audit.json b/docs/doctoring/zotero_fulltext_read_audit.json new file mode 100644 index 00000000..1909136d --- /dev/null +++ b/docs/doctoring/zotero_fulltext_read_audit.json @@ -0,0 +1,61 @@ +{ + "audit_status": "completed_read_sweep_not_atomic_snapshot", + "observed_at": "2026-09-05T07:36:42.665Z", + "conceptweave_head": "2a75051f0082103511222e278de24b2690fe6bfe", + "zotero_source_commit": "36749bd0bd4fdac9ee46c16f7aa7bed094a0851f", + "zotero_version": "10.0.1", + "api_version": 3, + "schema_version": 44, + "metadata_snapshot_digest": "sha256:0666dbebfb0c5aa99deb5a6dda1fc02d84bc46d08aaaddf25f5526a18eceef6d", + "metadata_report_file_sha256": "bf45248413f433a537fe8fc62c02b93eef3c7e47ff6245f31610e9ba72031d8d", + "library_version_before": 2, + "library_version_after": 2, + "server_api_schema_continuity_checked": true, + "manifest_last_modified_version": null, + "manifest_unchanged": true, + "manifest_sha256": "3d07472323b1a2e41947068e3b0e24a57c5569e08fd2ef0d35e626f290b9b351", + "ordered_response_sha256": "43b2239528182e3d28d91ba59ea3d713a1fc9652c37bb662c54564bfa34e84c3", + "elapsed_ms": 10737, + "total_response_bytes": 224842838, + "bounds": { + "max_bytes": 536870912, + "max_response_bytes": 8388608, + "max_elapsed_ms": 300000, + "request_timeout_ms": 20000, + "concurrent_requests": 1, + "redirects_allowed": false + }, + "counts": { + "manifest_entries": 3473, + "manifest_versions_greater_than_library": 3432, + "manifest_versions_zero": 41, + "manifest_entries_in_metadata_snapshot": 3473, + "manifest_candidate_parents": 3241, + "attempted": 3473, + "http_200": 3432, + "http_404": 41, + "other_status": 0, + "empty_content": 5, + "full_index_metadata": 2755, + "partial_index_metadata": 67, + "unproven_index_completeness": 610, + "content_version_mismatch": 0, + "bibliographic_denominator": 3715, + "parents_with_nonempty_fulltext": 3203, + "parents_with_full_index_metadata": 2561, + "approved_labels": 0, + "bound_fulltext_proposals": 0 + }, + "abstract_matrix": { + "abstract_not_retained": { + "total": 1000, + "nonempty_fulltext": 800, + "nonempty_fulltext_with_full_index_metadata": 598 + }, + "abstract_present": { + "total": 2715, + "nonempty_fulltext": 2403, + "nonempty_fulltext_with_full_index_metadata": 1963 + } + } +} diff --git a/docs/product-technical-gap-baseline.md b/docs/product-technical-gap-baseline.md index 83d89a26..b2c3e11a 100644 --- a/docs/product-technical-gap-baseline.md +++ b/docs/product-technical-gap-baseline.md @@ -70,10 +70,18 @@ All four new artifacts remain private mode `0600`, outside the repository: The repaired worksheet has 0/3,715 decisions and the first batch has 0/25; externally approved full-review coverage remains 0/3,715. No authorization prompt, approval, Zotero write, record merge/deletion or rollback was performed. The three historical Zotero 9 artifact hashes below were rechecked unchanged. The pre-repair schema-44 artifacts are also preserved. Stronger source binding is not classification correctness, business approval or loopback peer authentication. -The [CWL ontology capability inventory](doctoring/cwl_ontology_capability_inventory.md) now separates a 76-repository metadata census from a 12-candidate exact-default-head capability audit. Three selected candidates have GitHub releases with resolved source commits: RankWeave, mhtml-etl-gateway and fast-mlsirm. Veilpick's protected default tree contains only a license despite its ontology-related description; graphify is an upstream fork without a returned CWL GitHub release. These are maturity observations, not adoption receipts. Source-level discovery remains incomplete for 64 repositories, and actual ConceptWeave adoption remains unproved. No additional utility owner is justified yet. +The [CWL ontology capability inventory](doctoring/cwl_ontology_capability_inventory.md) now separates a 76-repository metadata census from a 13-candidate exact-default-head capability audit. Three selected candidates have GitHub releases with resolved source commits: RankWeave, mhtml-etl-gateway and fast-mlsirm. Veilpick's protected default tree contains only a license despite its ontology-related description; graphify is an upstream fork without a returned CWL GitHub release. The organization directive exposed DiskSage's implemented Rust named-class ontology subset at protected `main@0e90f9cebadbd7f59606baaec4ca1d2f178c899a`; it owns filesystem taxonomy and organization, not general semantic publication, and has no returned GitHub release. These are maturity observations, not adoption receipts. Source-level discovery remains incomplete for 63 repositories, and actual ConceptWeave adoption remains unproved. No additional utility owner is justified yet. Next: apply only authentic snapshot-bound steward decisions to the repaired batch, cover all 3,715 items and independently verify full-review approval; continue owner contract discovery and protected Foundation work while external checks/reviews are pending. The current body notes for PRs #11–#33 distinguish inherited source repair from historical head/check claims. No PR was closed, approved, merged or retargeted by this repair. +### Full-text availability and provider-contract finding + +At `2a75051f0082103511222e278de24b2690fe6bfe`, the [read-only full-text sweep](doctoring/zotero_fulltext_contract_audit.md) attempted all 3,473 manifest entries in 10,737 ms, with 224,842,838 response bytes inside explicit diagnostic budgets. HTTP results were 3,432 successful and 41 missing, with five empty content strings. Nonempty text linked to 3,203/3,715 bibliographic parents, including 800/1,000 without retained abstracts; 2,561 parents had nonempty text with complete index counters. The [aggregate record](doctoring/zotero_fulltext_read_audit.json) preserves the full denominator, response partitions, metadata-report binding, read limits and observation digests without exposing bibliography or item identity. + +Official Zotero 10.0.1 source `36749bd0bd4fdac9ee46c16f7aa7bed094a0851f` confirms that the full-text version storage receives remote sync versions, zero-valued local indexing and local API client versions, while the list omits the documented library-version header. Unchanged bookend manifest bytes and metadata library version 2 therefore do not prove an atomic full-text snapshot or a safe incremental cursor. No source file/database repair, new text-bound proposal, steward decision, approval or Zotero write is claimed. PRD FR-9, TRD and Proposed ADR 0006 now require separately captured content evidence; lifecycle capability metric remains 25 and approved full review remains 0/3,715. + +Next: implement and verify that bounded content-capture boundary without reusing old approvals; retain partial/missing text in the denominator; track the upstream version-contract repair. A released contextual-orchestrator integration artifact remains unverified at the audited protected owner head, so source documentation alone does not authorize model-provider bypass. No new utility repository is justified by this one intake seam. + ### Historical pre-repair Zotero 10 transition The following schema-44 observations were captured before integrity repair and are retained for provenance, not reused as current approval inputs. From a2bbe0f67bf261227c1b1aa3b2a03aa2b631c732 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sat, 5 Sep 2026 16:45:39 +0900 Subject: [PATCH 23/30] docs(security): isolate Zotero diagnostic curl requests --- docs/doctoring/zotero_fulltext_contract_audit.md | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/docs/doctoring/zotero_fulltext_contract_audit.md b/docs/doctoring/zotero_fulltext_contract_audit.md index 7394976e..d9427f43 100644 --- a/docs/doctoring/zotero_fulltext_contract_audit.md +++ b/docs/doctoring/zotero_fulltext_contract_audit.md @@ -30,15 +30,17 @@ The execution was sequential with a 20-second request timeout, an 8 MiB response Raw text was held for one response at a time and discarded. The ordered-response SHA-256 streams compact JSON arrays of attachment key, HTTP status, observed content-version header and body SHA-256, in lexicographic key order. The public record retains only the final aggregate digest, not those arrays. Body hashes are over received bytes, not canonical JSON. They identify this sweep's responses but cannot replay discarded source content or serve as an approval receipt. -These standalone probes expose only aggregate counts and public protocol values, not response bodies or identities: +These standalone probes disable curl's configuration-file loading, bypass proxies, restrict the protocol to HTTP and allow no redirects. They expose only aggregate counts and public protocol values, not response bodies or identities: ```sh -curl --silent --show-error --fail --max-time 20 \ +curl --disable --noproxy '*' --proto '=http' --max-redirs 0 \ + --silent --show-error --fail --max-time 20 --max-filesize 8388608 \ -H 'Zotero-API-Version: 3' \ 'http://127.0.0.1:23119/api/users/0/fulltext?since=0' \ | jq '{manifest_entries: length, version_types: ([.[] | type] | unique), minimum_version: ([.[]] | min), maximum_version: ([.[]] | max)}' -curl --silent --show-error --fail --max-time 20 --output /dev/null \ +curl --disable --noproxy '*' --proto '=http' --max-redirs 0 \ + --silent --show-error --fail --max-time 20 --max-filesize 8388608 --output /dev/null \ --write-out 'HTTP %{http_code}; version=%header{last-modified-version}; API=%header{zotero-api-version}; schema=%header{zotero-schema-version}\n' \ -H 'Zotero-API-Version: 3' \ 'http://127.0.0.1:23119/api/users/0/fulltext?since=0' From 2e6448e896e65562ebeee2fd339dec64d9fdf6e5 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sat, 5 Sep 2026 16:46:25 +0900 Subject: [PATCH 24/30] docs(research): distinguish retrieval attempts from returned content --- docs/adr/0006-zotero-research-intake.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/adr/0006-zotero-research-intake.md b/docs/adr/0006-zotero-research-intake.md index 5cb4a8f6..b4640c37 100644 --- a/docs/adr/0006-zotero-research-intake.md +++ b/docs/adr/0006-zotero-research-intake.md @@ -7,7 +7,7 @@ CWL needs a reproducible inventory of ontology research without turning keyword matches into authoritative library organization. At the original 2026-09-04 decision snapshot the desktop was Zotero 9.0.6, whose Local API supports reads but not writes. Zotero documents Local API writes only for Zotero 10+, where they require user-granted authorization and `Zotero-Server-ID`; the initial intake slice therefore had no mutation capability. The 2026-09-05 campaign now observes Zotero 10.0.1; write planning remains governed separately by ADR 0007 and does not follow from that upgrade. The library is mutable while pagination is in progress, duplicate metadata does not prove that two records should be merged, and the local report contains titles and item keys that must not be written into the repository. -Zotero's Local API documentation states that production clients should request `Zotero-API-Version: 3`; the response exposes `Zotero-API-Version` and `Zotero-Schema-Version`. The API version is the compatibility contract. The schema version is therefore recorded and required to remain stable across the snapshot, but it is not hard-coded to the developer workstation's current schema 42 because Zotero can legitimately revise the local data schema while retaining API v3 compatibility. +Zotero's Local API documentation states that production clients should request `Zotero-API-Version: 3`; the response exposes `Zotero-API-Version` and `Zotero-Schema-Version`. The API version is the compatibility contract. The schema version is therefore recorded and required to remain stable across the snapshot, but it is not hard-coded to the originally observed schema 42 because Zotero can legitimately revise the local data schema while retaining API v3 compatibility. Primary capability references: Zotero, *Local API* (updated 2026-07-29), https://www.zotero.org/support/dev/web_api/v3/local_api; Zotero, *Basics*, https://www.zotero.org/support/dev/web_api/v3/basics. @@ -51,7 +51,7 @@ No dedicated utility repository or Zotero mutation path is created. A future Zot In the context of reviewing papers with missing abstracts, facing a full-text API whose observed versions mix sync and local writes and whose list lacks the documented version header, we decided for separately captured, content-bound full-text observations and against treating attachment listings or unchanged version counters as complete snapshot evidence, to preserve review provenance and the full campaign denominator, accepting another capture/verification step and no current claim of atomic full-text enrichment. -The [source-grounded audit](../doctoring/zotero_fulltext_contract_audit.md) retrieved all 3,473 listed entries and demonstrated nonempty text for 3,203/3,715 bibliographic items, including 800/1,000 without retained abstracts. It did not persist raw text or classify those papers. Positive consequence: available source material can guide genuine review without inventing relevance or approval. Negative consequence: content remains unbound to the metadata report until a separate immutable capture contract is implemented and reviewed. Neither a guessed incremental cursor nor adding only the provider's missing header repairs the mixed-version semantics. Direct database edits, cloud credential expansion and a new utility owner are rejected; the provider semantics require an upstream fix, while Research Intake retains the consumer admission boundary. Status remains Proposed. +The [source-grounded audit](../doctoring/zotero_fulltext_contract_audit.md) attempted all 3,473 listed entries and demonstrated nonempty text for 3,203/3,715 bibliographic items, including 800/1,000 without retained abstracts. It did not persist raw text or classify those papers. Positive consequence: available source material can guide genuine review without inventing relevance or approval. Negative consequence: content remains unbound to the metadata report until a separate immutable capture contract is implemented and reviewed. Neither a guessed incremental cursor nor adding only the provider's missing header repairs the mixed-version semantics. Direct database edits, cloud credential expansion and a new utility owner are rejected; the provider semantics require an upstream fix, while Research Intake retains the consumer admission boundary. Status remains Proposed. ### 2026-09-05 integrity amendment (Proposed) From d1e8bb78f43bb5d2886d32862bbefcf52aa89f21 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Mon, 7 Sep 2026 01:23:49 +0900 Subject: [PATCH 25/30] test: bind patch fixture without duplicating approval identity --- .../conceptweave-zotero/tests/finalization_artifact_identity.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/conceptweave-zotero/tests/finalization_artifact_identity.rs b/crates/conceptweave-zotero/tests/finalization_artifact_identity.rs index 92ee0b75..efd78da5 100644 --- a/crates/conceptweave-zotero/tests/finalization_artifact_identity.rs +++ b/crates/conceptweave-zotero/tests/finalization_artifact_identity.rs @@ -35,7 +35,6 @@ fn artifact_commands_reject_distinct_path_spellings_for_one_input_file() { library_version: worksheet.library_version, rule_revision: worksheet.rule_revision.clone(), snapshot_digest: worksheet.snapshot_digest.clone(), - proposal_digest: worksheet.proposal_digest.clone(), proposal_digest: conceptweave_zotero::classification_proposal_digest(&report), snapshot_items: worksheet.snapshot_items.clone(), }; @@ -84,6 +83,7 @@ fn artifact_commands_reject_distinct_path_spellings_for_one_input_file() { library_version: worksheet.library_version, rule_revision: worksheet.rule_revision.clone(), snapshot_digest: worksheet.snapshot_digest.clone(), + proposal_digest: worksheet.proposal_digest.clone(), decisions: vec![StewardDecisionUpdate { item_key: "ITEM".into(), item_version: 7, From e889ac8556ef3f6b43e3768f646776ee335437c1 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Mon, 7 Sep 2026 01:24:15 +0900 Subject: [PATCH 26/30] test: reject stale reviewed batch source scope before projection --- .../tests/steward_review_batch.rs | 31 +++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/crates/conceptweave-zotero/tests/steward_review_batch.rs b/crates/conceptweave-zotero/tests/steward_review_batch.rs index 2058c714..00d1c513 100644 --- a/crates/conceptweave-zotero/tests/steward_review_batch.rs +++ b/crates/conceptweave-zotero/tests/steward_review_batch.rs @@ -191,6 +191,7 @@ fn completed_review_batch_must_preserve_the_context_shown_to_the_steward() { let patch = decision_patch_from_review_batch(&report, &worksheet, &batch).unwrap(); assert_eq!(patch.decisions.len(), 1); + assert_eq!(patch.proposal_digest, batch.proposal_digest); assert_eq!(patch.decisions[0].item_key, "A"); assert_eq!( patch.decisions[0].reviewed_disposition, @@ -198,6 +199,21 @@ fn completed_review_batch_must_preserve_the_context_shown_to_the_steward() { ); for invalid in [ + { + let mut invalid = batch.clone(); + invalid.proposal_digest.clear(); + invalid + }, + { + let mut invalid = batch.clone(); + invalid.proposal_digest = "stale-binding".into(); + invalid + }, + { + let mut invalid = batch.clone(); + invalid.pending_source_count += 1; + invalid + }, { let mut invalid = batch.clone(); invalid.decisions[0].title = "different context".into(); @@ -220,6 +236,21 @@ fn completed_review_batch_must_preserve_the_context_shown_to_the_steward() { ); } + let mut changed_report = report.clone(); + changed_report.classified_items[0].review_abstract_note = Some("changed context".into()); + let fresh_worksheet = build_steward_review_worksheet(&changed_report).unwrap(); + assert_eq!(fresh_worksheet.snapshot_digest, worksheet.snapshot_digest); + assert_ne!(fresh_worksheet.proposal_digest, worksheet.proposal_digest); + assert_eq!( + decision_patch_from_review_batch(&changed_report, &fresh_worksheet, &batch), + Err(WorksheetError::InvalidReport) + ); + for required_field in ["proposal_digest", "pending_source_count"] { + let mut missing = serde_json::to_value(&batch).unwrap(); + missing.as_object_mut().unwrap().remove(required_field); + assert!(serde_json::from_value::(missing).is_err()); + } + let mut empty = batch.clone(); empty.decisions.clear(); assert_eq!( From 5a23660ce36a4f5b8ef41395c1483f69a9881e8e Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Mon, 7 Sep 2026 01:24:53 +0900 Subject: [PATCH 27/30] test: construct changed source without cloning report --- crates/conceptweave-zotero/tests/steward_review_batch.rs | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/crates/conceptweave-zotero/tests/steward_review_batch.rs b/crates/conceptweave-zotero/tests/steward_review_batch.rs index 00d1c513..4f6a3e22 100644 --- a/crates/conceptweave-zotero/tests/steward_review_batch.rs +++ b/crates/conceptweave-zotero/tests/steward_review_batch.rs @@ -236,8 +236,12 @@ fn completed_review_batch_must_preserve_the_context_shown_to_the_steward() { ); } - let mut changed_report = report.clone(); - changed_report.classified_items[0].review_abstract_note = Some("changed context".into()); + let changed_report = classify_snapshot( + "9.0.6".into(), + None, + 42, + vec![item("A", "unmatched", "changed context")], + ); let fresh_worksheet = build_steward_review_worksheet(&changed_report).unwrap(); assert_eq!(fresh_worksheet.snapshot_digest, worksheet.snapshot_digest); assert_ne!(fresh_worksheet.proposal_digest, worksheet.proposal_digest); From cb8f78c7bda9b05c001645aaf4c3cc0b42e78b46 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Mon, 7 Sep 2026 01:25:44 +0900 Subject: [PATCH 28/30] test: isolate retained context drift from raw snapshot drift --- crates/conceptweave-zotero/tests/steward_review_batch.rs | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/crates/conceptweave-zotero/tests/steward_review_batch.rs b/crates/conceptweave-zotero/tests/steward_review_batch.rs index 4f6a3e22..7462f1f6 100644 --- a/crates/conceptweave-zotero/tests/steward_review_batch.rs +++ b/crates/conceptweave-zotero/tests/steward_review_batch.rs @@ -236,12 +236,13 @@ fn completed_review_batch_must_preserve_the_context_shown_to_the_steward() { ); } - let changed_report = classify_snapshot( + let mut changed_report = classify_snapshot( "9.0.6".into(), None, 42, - vec![item("A", "unmatched", "changed context")], + vec![item("A", "unmatched", "review context")], ); + changed_report.classified_items[0].review_abstract_note = Some("changed context".into()); let fresh_worksheet = build_steward_review_worksheet(&changed_report).unwrap(); assert_eq!(fresh_worksheet.snapshot_digest, worksheet.snapshot_digest); assert_ne!(fresh_worksheet.proposal_digest, worksheet.proposal_digest); From 28a2e28a4e94b5e24d229a09309856296d5e526b Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Mon, 7 Sep 2026 01:26:44 +0900 Subject: [PATCH 29/30] docs: record reviewed view continuity and retained failure boundaries --- docs/TRD.md | 2 +- docs/adr/0006-zotero-research-intake.md | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/docs/TRD.md b/docs/TRD.md index b7896b42..60b135a3 100644 --- a/docs/TRD.md +++ b/docs/TRD.md @@ -126,7 +126,7 @@ Offline input admission pins the checked canonical parent plus file name for met `conceptweave-zotero --review-batch /tmp/REPORT.json /tmp/CURRENT_WORKSHEET.json LIMIT /tmp/BATCH.json` validates distinct private inputs and emits the first 1–100 blank bibliographic decisions in canonical order. It skips decided rows and rejects invalid worksheets, reviewed abstention or no remaining slots before output. Each sensitive row retains exact key/version, type, title, minimized abstract, collections, tags, proposal/reason/evidence and a blank decision. The batch carries snapshot coordinates plus the already validated `proposal_digest` and `pending_source_count`; remaining slots and unresolved sources are separate counts. No slots left does not prove source completion. Snapshot-item arrays, child keys, model receipts, audit/duplicate aggregates and approval identity remain omitted. Original report and current worksheet are revalidated at application. Repeated input gives an equal batch without reservation, assignment or exclusivity. -`decision_patch_from_review_batch` rebuilds the expected pending batch at the submitted length and compares every snapshot coordinate, row, and displayed context field after extracting complete non-abstention decisions. Batch roots, decision rows, evidence objects, and tag objects reject unknown JSON fields recursively; `StewardDecisionPatch` does the same at its root. A review batch therefore cannot bypass comparison through permissive deserialization. `conceptweave-zotero --apply-review-batch /tmp/REPORT.json /tmp/CURRENT_WORKSHEET.json /tmp/COMPLETED_BATCH.json /tmp/UPDATED_WORKSHEET.json` applies the validated conversion through the existing atomic patch and private artifact boundaries. It reads no Zotero state, writes no partial worksheet, and grants no approval authority. +`decision_patch_from_review_batch` rebuilds the expected pending batch at the submitted length and compares every snapshot coordinate, row, and displayed context field after extracting complete non-abstention decisions. This includes required proposal identity and pending-source count; only after equality does the returned patch receive the verified proposal identity. A fresh worksheet cannot renew an older completed view. Batch roots, decision rows, evidence objects, and tag objects reject unknown JSON fields recursively; `StewardDecisionPatch` does the same at its root. A review batch therefore cannot bypass comparison through permissive deserialization. `conceptweave-zotero --apply-review-batch /tmp/REPORT.json /tmp/CURRENT_WORKSHEET.json /tmp/COMPLETED_BATCH.json /tmp/UPDATED_WORKSHEET.json` applies the validated conversion through the existing atomic patch and private artifact boundaries. It reads no Zotero state and grants no approval authority. Input rejection creates no output; an output write failure may retain a private partial file under the inherited no-cleanup policy. The local report can contain titles, tags, matched metadata, and abstention abstracts. It is sensitive steward-review material, remains outside the repository, and is not a publication artifact. diff --git a/docs/adr/0006-zotero-research-intake.md b/docs/adr/0006-zotero-research-intake.md index de42309d..12e2dd53 100644 --- a/docs/adr/0006-zotero-research-intake.md +++ b/docs/adr/0006-zotero-research-intake.md @@ -5,6 +5,8 @@ ## Context +Proposed September 7 completed-batch amendment: a saved review view may outlive changed source context or unresolved-source counts. Preserve PR34's whole-view comparison, including the inherited proposal identity and pending-source count, then project its already verified identity into the decision patch. Do not backfill an old batch with current identity, add another hash, or deserialize a batch directly into the smaller patch: each alternative could discard what the steward actually saw. Strict object boundaries remain. Normal merge `bd8f995` retains PR33 `93faf6ab750a99469196cf71567498be83c22a6b` and the original research-source audit; `d1e8bb7` corrects the synthetic patch initializer. Tests `e889ac8` through `cb8f78c` cover missing, blank and changed identity, altered pending count, and stale context against a freshly generated worksheet. Test compilation and fixture-assumption failures are intermediate evidence, not production RED. The cost is explicit regeneration and renewed review after context drift. Neither accepted local decisions nor this conversion authenticate a reviewer, grant full-text provenance, resolve pending sources or authorize Zotero writes. Later consumers must retain this boundary; protected adoption remains outstanding. + Proposed September 7 finalization amendment: when carrying completed metadata decisions between saved artifacts, changed report evidence plus freshly supplied receipt coordinates must not refresh a stale worksheet. We choose two checks in the existing converter—nonblank worksheet proposal identity and equality with the recomputed expected worksheet—rather than a second approval mechanism or automatically rebinding old decisions. RED `f02631e` reproduces both stale-content and blank/replaced-binding admission; `d44b9fe` closes them while preserving prior error precedence. The cost is explicit worksheet regeneration/review after changed source context. Self-consistent local artifacts remain unverified: `e90a02b` demonstrates that locally rewritten digests still fail independent original-receipt verification and that pending sources prevent complete evaluation even after local conversion. This decision grants neither full-text nor Zotero write authority. Later worksheet comparators must inherit the binding check, and protected/runtime evidence remains outstanding. Proposed September 7 export-failure amendment: when exporting sensitive report/worksheet pairs, a failed write may race with pathname replacement, and destroying a buffered writer may write pending bytes after failure. We choose to preserve artifacts, disassemble the existing buffer without flushing, and propagate the original error, rejecting pathname cleanup or an automatic retry. This extends the existing private-creation policy to the common output writer and the pair's second-file failure. The cost is retained empty/partial files and operator inspection; sequential export is not a transaction or crash-durable publication. Both canonical destinations must differ before the Local API read, including aliases such as `/tmp` and `/private/tmp`. RED `68575a6` proves replacement deletion and implicit drop-flush; `ab391b2` removes both behaviors. RED `fb5b5e5` proves alias collision; `e928858` rejects it before snapshot capture. Later CLI owners must inherit this implementation instead of restoring cleanup. No Zotero mutation or approval authority follows from local output. From 9eb89b8d4e751c34e640261f4883381571c83f25 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Mon, 7 Sep 2026 01:27:52 +0900 Subject: [PATCH 30/30] docs: record PR34 exact view verification and remaining evidence gaps --- docs/product-technical-gap-baseline.md | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/docs/product-technical-gap-baseline.md b/docs/product-technical-gap-baseline.md index 98566bfa..65b43c62 100644 --- a/docs/product-technical-gap-baseline.md +++ b/docs/product-technical-gap-baseline.md @@ -6,6 +6,14 @@ This file records code-current product and technical gaps. Exact PR/check/run co ## September 6 source inventory checkpoint +### September 7 PR34 completed-view continuity verification + +Original PR34 `b9060df2cb1ea02314be429932031fc07de1de30` passed 171 tests/37 suites. Normal merge `bd8f995` preserves that delta and PR33 `93faf6ab750a99469196cf71567498be83c22a6b`: whole-view comparison, strict JSON boundaries, research-source audit, pending-source scope and inherited private-file protections all remain. The required patch identity is projected only after whole-view equality. Fixture correction `d1e8bb7` yields 221 integrated tests/37 suites. Tests `e889ac8`, `5a23660` and `cb8f78c` add tamper/missing-field/stale-view coverage; intermediate compilation and fixture-assumption failures are recorded, not claimed as production RED. The final stale-view case holds raw snapshot identity unchanged while changing retained context and regenerating the worksheet. Independent bounded review found no additional defect; it is not GitHub approval. + +Final source `cb8f78c` passes 221 tests/37 suites including three doctests, strict all-target Clippy, warnings-denied rustdoc, formatting, CI contract and diff checks. The unchanged coverage script passes 342/342 reported functions, 3,192/3,192 normalized regions and 580/580 normalized branches. Raw 4,308/4,370 lines, 6,555/6,662 regions and 535/580 branches are not 100%. Logs: `/tmp/conceptweave-pr34-{baseline,integrated,integration-fixed,verified,final,final-verified,clippy-final,rustdoc-final,coverage}.log`. `integrated` and `verified` are failed compilation runs; `final` is the failed fixture-assumption run; `final-verified` is terminal GREEN. Documentation `28a2e28` corrects the output-failure claim: admission rejection creates no output, but write failure may retain a private partial file. + +PRD/TRD/Proposed ADR0006 preserve complete-view validation without a second hash or approval mechanism. Root and later consumers have not adopted this cascade. Actual decisions and independent approvals remain 0/3,715 plus four unresolved standalone sources; synthetic fixture decisions are not paper review. Native Visual Inspection was retried, but the Mac is locked and requires manual unlock; no fresh screen evidence exists. Keep OPEN Draft. No real Zotero write, hosted GREEN, independent protected approval, protected merge or release is claimed. + ### September 7 PR33 batch scope repair Original PR33 `d82b2f2896bc4cfef5d34ac6f6f83f7cee1072f6` passed 166 tests/36 suites. Normal merge `2f3962e` retains it and PR32 `ea25437c54b4e452f008f6b1ccad92d5516dd7b9`, preserving original abstract minimization guards with the shared validator. The existing batch-to-patch test failed for missing proposal identity. Explicit RED `7373579` compiled with one passing and two failing tests for patch compatibility and pending-source projection. `ce15389` passes existing validated progress identity and pending count into the batch, without new hashing or authority. No blank slots means exhausted paper decisions, not resolved source scope. Independent bounded review found no added defect.