From bc53272051036d8118fbdf108e2cb3f61ecacc79 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Wed, 2 Sep 2026 15:02:08 +0900 Subject: [PATCH 01/47] test(lineage): define dynamic evaluation provenance RED --- tests/test_dynamic_evaluation_lineage.py | 236 +++++++++++++++++++++++ 1 file changed, 236 insertions(+) create mode 100644 tests/test_dynamic_evaluation_lineage.py diff --git a/tests/test_dynamic_evaluation_lineage.py b/tests/test_dynamic_evaluation_lineage.py new file mode 100644 index 000000000..09b0273c2 --- /dev/null +++ b/tests/test_dynamic_evaluation_lineage.py @@ -0,0 +1,236 @@ +"""Contracts for dynamic-evaluation provenance projections.""" + +from __future__ import annotations + +import pytest + +from lineageweave.evaluation_lineage import ( + DYNAMIC_EVALUATION_LINEAGE_CONTRACT_ID, + DynamicEvaluationItemLineage, + DynamicEvaluationLineageError, + DynamicEvaluationRunLineage, + RunComparabilityStatus, + build_dynamic_evaluation_item_lineage, + build_dynamic_evaluation_run_lineage, +) + +_CONTRACT_DIGEST = "a" * 64 + + +def _item( + *, + item_snapshot_ref: str = "evaluation_item_snapshot_alpha", + adjudication_case_ref: str | None = None, + adjudication_resolution_ref: str | None = None, + calibration_artifact_refs: tuple[str, ...] = (), + anchor_promotion_decision_ref: str | None = None, + supersedes_item_snapshot_ref: str | None = None, +) -> DynamicEvaluationItemLineage: + """Build one item lineage through the public admission boundary.""" + return build_dynamic_evaluation_item_lineage( + item_snapshot_ref=item_snapshot_ref, + blueprint_revision_ref="evaluation_blueprint_revision_1", + source_contract_ref="fast_mlsirm_dynamic_evaluation_item/v1", + source_contract_sha256=_CONTRACT_DIGEST, + generation_invocation_ref="generation_invocation_1", + rater_invocation_refs=("rater_invocation_1", "rater_invocation_2"), + adjudication_case_ref=adjudication_case_ref, + adjudication_resolution_ref=adjudication_resolution_ref, + calibration_artifact_refs=calibration_artifact_refs, + anchor_promotion_decision_ref=anchor_promotion_decision_ref, + supersedes_item_snapshot_ref=supersedes_item_snapshot_ref, + ) + + +def test_zero_anchor_run_is_representable_without_linking_claim() -> None: + """A dynamic run may have no fixed anchors while exposing that comparability limit.""" + run = build_dynamic_evaluation_run_lineage( + run_snapshot_ref="evaluation_run_snapshot_1", + blueprint_revision_ref="evaluation_blueprint_revision_1", + items=(_item(),), + anchor_item_snapshot_refs=(), + comparability_status=RunComparabilityStatus.UNAVAILABLE, + ) + + assert run.contract_id == DYNAMIC_EVALUATION_LINEAGE_CONTRACT_ID + assert run.anchor_item_snapshot_refs == () + assert run.comparability_status is RunComparabilityStatus.UNAVAILABLE + + within_run = build_dynamic_evaluation_run_lineage( + run_snapshot_ref="evaluation_run_snapshot_2", + blueprint_revision_ref="evaluation_blueprint_revision_1", + items=(_item(),), + anchor_item_snapshot_refs=(), + comparability_status=RunComparabilityStatus.WITHIN_RUN_ONLY, + ) + assert within_run.comparability_status is RunComparabilityStatus.WITHIN_RUN_ONLY + + with pytest.raises(DynamicEvaluationLineageError) as caught: + build_dynamic_evaluation_run_lineage( + run_snapshot_ref="evaluation_run_snapshot_3", + blueprint_revision_ref="evaluation_blueprint_revision_1", + items=(_item(),), + anchor_item_snapshot_refs=(), + comparability_status=RunComparabilityStatus.LINKED, + linking_evidence_ref="linking_evidence_1", + ) + assert caught.value.code == "linked_run_requires_anchor" + + +def test_adjudication_resolution_is_separate_from_source_observations() -> None: + """A resolution references a case and never replaces immutable rater invocations.""" + item = _item( + adjudication_case_ref="adjudication_case_1", + adjudication_resolution_ref="adjudication_resolution_1", + ) + assert item.rater_invocation_refs == ( + "rater_invocation_1", + "rater_invocation_2", + ) + assert item.adjudication_case_ref == "adjudication_case_1" + assert item.adjudication_resolution_ref == "adjudication_resolution_1" + + with pytest.raises(DynamicEvaluationLineageError) as caught: + _item(adjudication_resolution_ref="adjudication_resolution_1") + assert caught.value.code == "resolution_requires_case" + + +def test_adjudication_alone_cannot_promote_an_anchor() -> None: + """Anchor projection requires a separate promotion decision and calibration evidence.""" + adjudicated = _item( + adjudication_case_ref="adjudication_case_1", + adjudication_resolution_ref="adjudication_resolution_1", + ) + with pytest.raises(DynamicEvaluationLineageError) as caught: + build_dynamic_evaluation_run_lineage( + run_snapshot_ref="evaluation_run_snapshot_1", + blueprint_revision_ref="evaluation_blueprint_revision_1", + items=(adjudicated,), + anchor_item_snapshot_refs=(adjudicated.item_snapshot_ref,), + comparability_status=RunComparabilityStatus.UNAVAILABLE, + ) + assert caught.value.code == "anchor_requires_promotion_evidence" + + promoted = _item( + adjudication_case_ref="adjudication_case_1", + adjudication_resolution_ref="adjudication_resolution_1", + calibration_artifact_refs=("calibration_artifact_1",), + anchor_promotion_decision_ref="anchor_promotion_decision_1", + ) + run = build_dynamic_evaluation_run_lineage( + run_snapshot_ref="evaluation_run_snapshot_2", + blueprint_revision_ref="evaluation_blueprint_revision_1", + items=(promoted,), + anchor_item_snapshot_refs=(promoted.item_snapshot_ref,), + comparability_status=RunComparabilityStatus.LINKED, + linking_evidence_ref="linking_evidence_1", + ) + assert run.anchor_item_snapshot_refs == (promoted.item_snapshot_ref,) + + +def test_lineage_rejects_provider_configuration_and_decision_payload_fields() -> None: + """Lineage projection cannot absorb provider credentials, endpoints, scores, or decisions.""" + payload = { + "contract_id": DYNAMIC_EVALUATION_LINEAGE_CONTRACT_ID, + "run_snapshot_ref": "evaluation_run_snapshot_1", + "blueprint_revision_ref": "evaluation_blueprint_revision_1", + "items": [ + { + "item_snapshot_ref": "evaluation_item_snapshot_alpha", + "blueprint_revision_ref": "evaluation_blueprint_revision_1", + "source_contract_ref": "fast_mlsirm_dynamic_evaluation_item/v1", + "source_contract_sha256": _CONTRACT_DIGEST, + "generation_invocation_ref": "generation_invocation_1", + "rater_invocation_refs": ["rater_invocation_1"], + "adjudication_case_ref": None, + "adjudication_resolution_ref": None, + "calibration_artifact_refs": [], + "anchor_promotion_decision_ref": None, + "supersedes_item_snapshot_ref": None, + } + ], + "anchor_item_snapshot_refs": [], + "comparability_status": "unavailable", + "linking_evidence_ref": None, + "provider_api_key": "secret", + } + with pytest.raises(DynamicEvaluationLineageError) as caught: + DynamicEvaluationRunLineage.from_mapping(payload) + assert caught.value.code == "authority_leakage" + + payload.pop("provider_api_key") + payload["score"] = 1.0 + with pytest.raises(DynamicEvaluationLineageError) as caught: + DynamicEvaluationRunLineage.from_mapping(payload) + assert caught.value.code == "authority_leakage" + + +def test_run_freezes_unique_items_and_anchor_references() -> None: + """Run lineage is an immutable projection over one unique blueprint-bound item set.""" + first = _item() + second = _item(item_snapshot_ref="evaluation_item_snapshot_beta") + source = [first, second] + run = build_dynamic_evaluation_run_lineage( + run_snapshot_ref="evaluation_run_snapshot_1", + blueprint_revision_ref="evaluation_blueprint_revision_1", + items=source, + anchor_item_snapshot_refs=(), + comparability_status=RunComparabilityStatus.WITHIN_RUN_ONLY, + ) + source.pop() + assert run.items == (first, second) + + with pytest.raises(DynamicEvaluationLineageError) as caught: + build_dynamic_evaluation_run_lineage( + run_snapshot_ref="evaluation_run_snapshot_duplicate", + blueprint_revision_ref="evaluation_blueprint_revision_1", + items=(first, first), + anchor_item_snapshot_refs=(), + comparability_status=RunComparabilityStatus.UNAVAILABLE, + ) + assert caught.value.code == "duplicate_item_snapshot" + + with pytest.raises(DynamicEvaluationLineageError) as caught: + build_dynamic_evaluation_run_lineage( + run_snapshot_ref="evaluation_run_snapshot_unknown_anchor", + blueprint_revision_ref="evaluation_blueprint_revision_1", + items=(first,), + anchor_item_snapshot_refs=("evaluation_item_snapshot_missing",), + comparability_status=RunComparabilityStatus.UNAVAILABLE, + ) + assert caught.value.code == "unknown_anchor_item" + + +def test_supersession_cannot_point_to_self() -> None: + """A lineage successor cannot claim to supersede its own item snapshot identity.""" + with pytest.raises(DynamicEvaluationLineageError) as caught: + _item(supersedes_item_snapshot_ref="evaluation_item_snapshot_alpha") + assert caught.value.code == "self_supersession" + + +def test_direct_aggregate_construction_is_sealed() -> None: + """Only builders may produce admitted item and run projections.""" + with pytest.raises(ValueError, match="build_dynamic_evaluation_item_lineage"): + DynamicEvaluationItemLineage( # type: ignore[call-arg] + item_snapshot_ref="evaluation_item_snapshot_alpha", + blueprint_revision_ref="evaluation_blueprint_revision_1", + source_contract_ref="fast_mlsirm_dynamic_evaluation_item/v1", + source_contract_sha256=_CONTRACT_DIGEST, + generation_invocation_ref="generation_invocation_1", + rater_invocation_refs=("rater_invocation_1",), + adjudication_case_ref=None, + adjudication_resolution_ref=None, + calibration_artifact_refs=(), + anchor_promotion_decision_ref=None, + supersedes_item_snapshot_ref=None, + ) + + with pytest.raises(ValueError, match="build_dynamic_evaluation_run_lineage"): + DynamicEvaluationRunLineage( # type: ignore[call-arg] + run_snapshot_ref="evaluation_run_snapshot_1", + blueprint_revision_ref="evaluation_blueprint_revision_1", + items=(_item(),), + anchor_item_snapshot_refs=(), + comparability_status=RunComparabilityStatus.UNAVAILABLE, + linking_evidence_ref=None, + ) From 351799156d854e5e7f25d07db91c7966a037ae2b Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Wed, 2 Sep 2026 15:02:36 +0900 Subject: [PATCH 02/47] test(lineage): harden dynamic evaluation projection RED boundaries --- ...t_dynamic_evaluation_lineage_boundaries.py | 223 ++++++++++++++++++ 1 file changed, 223 insertions(+) create mode 100644 tests/test_dynamic_evaluation_lineage_boundaries.py diff --git a/tests/test_dynamic_evaluation_lineage_boundaries.py b/tests/test_dynamic_evaluation_lineage_boundaries.py new file mode 100644 index 000000000..f2578e77e --- /dev/null +++ b/tests/test_dynamic_evaluation_lineage_boundaries.py @@ -0,0 +1,223 @@ +"""Fail-closed boundaries for dynamic evaluation lineage projections.""" + +from __future__ import annotations + +from typing import Any + +import pytest + +from lineageweave.evaluation_lineage import ( + DYNAMIC_EVALUATION_LINEAGE_CONTRACT_ID, + DynamicEvaluationItemLineage, + DynamicEvaluationLineageError, + DynamicEvaluationRunLineage, + RunComparabilityStatus, + build_dynamic_evaluation_item_lineage, + build_dynamic_evaluation_run_lineage, +) + +_DIGEST = "a" * 64 + + +def _item(**overrides: Any) -> DynamicEvaluationItemLineage: + payload: dict[str, Any] = { + "item_snapshot_ref": "evaluation_item_snapshot_alpha", + "blueprint_revision_ref": "evaluation_blueprint_revision_1", + "source_contract_ref": "fast_mlsirm_dynamic_evaluation_item/v1", + "source_contract_sha256": _DIGEST, + "generation_invocation_ref": "generation_invocation_1", + "rater_invocation_refs": ("rater_invocation_1",), + "adjudication_case_ref": None, + "adjudication_resolution_ref": None, + "calibration_artifact_refs": (), + "anchor_promotion_decision_ref": None, + "supersedes_item_snapshot_ref": None, + } + payload.update(overrides) + return build_dynamic_evaluation_item_lineage(**payload) + + +def _run_payload() -> dict[str, Any]: + return { + "contract_id": DYNAMIC_EVALUATION_LINEAGE_CONTRACT_ID, + "run_snapshot_ref": "evaluation_run_snapshot_1", + "blueprint_revision_ref": "evaluation_blueprint_revision_1", + "items": [_item().to_mapping()], + "anchor_item_snapshot_refs": [], + "comparability_status": "unavailable", + "linking_evidence_ref": None, + } + + +def test_item_mapping_round_trip_and_mapping_failures() -> None: + item = DynamicEvaluationItemLineage.from_mapping(_item().to_mapping()) + assert item.item_snapshot_ref == "evaluation_item_snapshot_alpha" + + with pytest.raises(DynamicEvaluationLineageError) as caught: + DynamicEvaluationItemLineage.from_mapping([]) + assert caught.value.code == "invalid_object" + + with pytest.raises(DynamicEvaluationLineageError) as caught: + DynamicEvaluationItemLineage.from_mapping({1: "bad-key"}) + assert caught.value.code == "invalid_object_key" + + payload = _item().to_mapping() + payload["unknown"] = "value" + with pytest.raises(DynamicEvaluationLineageError) as caught: + DynamicEvaluationItemLineage.from_mapping(payload) + assert caught.value.code == "unknown_field" + + payload = _item().to_mapping() + del payload["generation_invocation_ref"] + with pytest.raises(DynamicEvaluationLineageError) as caught: + DynamicEvaluationItemLineage.from_mapping(payload) + assert caught.value.code == "missing_field" + + +def test_item_authority_fields_are_rejected() -> None: + payload = _item().to_mapping() + payload["adjudication_decision"] = "approved" + with pytest.raises(DynamicEvaluationLineageError) as caught: + DynamicEvaluationItemLineage.from_mapping(payload) + assert caught.value.code == "authority_leakage" + + +@pytest.mark.parametrize( + "invalid", + ( + "", + " item_ref", + "item_ref ", + "\ufeffitem_ref", + "item_ref\ufeff", + "line\nbreak", + "\ud800", + "x" * 257, + ), +) +def test_references_are_exact_bounded_unicode_scalars(invalid: str) -> None: + with pytest.raises(DynamicEvaluationLineageError) as caught: + _item(item_snapshot_ref=invalid) + assert caught.value.code == "invalid_reference" + + with pytest.raises(TypeError, match="item_snapshot_ref must be a string"): + _item(item_snapshot_ref=object()) + + +def test_reference_collections_and_digest_are_typed_bounded_and_unique() -> None: + for refs, expected in ( + ("rater_invocation_1", TypeError), + (["rater_invocation_1"] * 257, DynamicEvaluationLineageError), + (["rater_invocation_1", "rater_invocation_1"], DynamicEvaluationLineageError), + ): + with pytest.raises(expected): + _item(rater_invocation_refs=refs) + + with pytest.raises(TypeError, match="source_contract_sha256 must be a string"): + _item(source_contract_sha256=object()) + with pytest.raises(DynamicEvaluationLineageError) as caught: + _item(source_contract_sha256="A" * 64) + assert caught.value.code == "invalid_sha256" + + +def test_run_mapping_round_trip_and_transport_failures() -> None: + run = DynamicEvaluationRunLineage.from_mapping(_run_payload()) + assert run.to_mapping()["comparability_status"] == "unavailable" + + payload = _run_payload() + del payload["linking_evidence_ref"] + with pytest.raises(DynamicEvaluationLineageError) as caught: + DynamicEvaluationRunLineage.from_mapping(payload) + assert caught.value.code == "missing_field" + + payload = _run_payload() + payload["contract_id"] = "wrong/v1" + with pytest.raises(DynamicEvaluationLineageError) as caught: + DynamicEvaluationRunLineage.from_mapping(payload) + assert caught.value.code == "contract_incompatible" + + payload = _run_payload() + payload["items"] = "not-an-array" + with pytest.raises(TypeError, match="items must be a tuple or list"): + DynamicEvaluationRunLineage.from_mapping(payload) + + +def test_comparability_and_run_resource_boundaries() -> None: + item = _item() + for status in (object(), "unknown"): + with pytest.raises((TypeError, DynamicEvaluationLineageError)): + build_dynamic_evaluation_run_lineage( + run_snapshot_ref="evaluation_run_snapshot_1", + blueprint_revision_ref="evaluation_blueprint_revision_1", + items=(item,), + anchor_item_snapshot_refs=(), + comparability_status=status, + ) + + for items in ((), [], "not-an-item-array"): + with pytest.raises(DynamicEvaluationLineageError) as caught: + build_dynamic_evaluation_run_lineage( + run_snapshot_ref="evaluation_run_snapshot_empty", + blueprint_revision_ref="evaluation_blueprint_revision_1", + items=items, + anchor_item_snapshot_refs=(), + comparability_status=RunComparabilityStatus.UNAVAILABLE, + ) + assert caught.value.code == "invalid_item_set" + + with pytest.raises(DynamicEvaluationLineageError) as caught: + build_dynamic_evaluation_run_lineage( + run_snapshot_ref="evaluation_run_snapshot_large", + blueprint_revision_ref="evaluation_blueprint_revision_1", + items=[item] * 10_001, + anchor_item_snapshot_refs=(), + comparability_status=RunComparabilityStatus.UNAVAILABLE, + ) + assert caught.value.code == "item_set_budget_exceeded" + + with pytest.raises(TypeError, match="exact DynamicEvaluationItemLineage"): + build_dynamic_evaluation_run_lineage( + run_snapshot_ref="evaluation_run_snapshot_wrong_type", + blueprint_revision_ref="evaluation_blueprint_revision_1", + items=(item, object()), + anchor_item_snapshot_refs=(), + comparability_status=RunComparabilityStatus.UNAVAILABLE, + ) + + foreign = _item(blueprint_revision_ref="evaluation_blueprint_revision_2") + with pytest.raises(DynamicEvaluationLineageError) as caught: + build_dynamic_evaluation_run_lineage( + run_snapshot_ref="evaluation_run_snapshot_foreign", + blueprint_revision_ref="evaluation_blueprint_revision_1", + items=(item, foreign), + anchor_item_snapshot_refs=(), + comparability_status=RunComparabilityStatus.UNAVAILABLE, + ) + assert caught.value.code == "item_blueprint_mismatch" + + +def test_linking_evidence_is_admitted_only_with_promoted_anchors() -> None: + anchor = _item( + calibration_artifact_refs=("calibration_artifact_1",), + anchor_promotion_decision_ref="anchor_promotion_decision_1", + ) + with pytest.raises(DynamicEvaluationLineageError) as caught: + build_dynamic_evaluation_run_lineage( + run_snapshot_ref="evaluation_run_snapshot_no_link_evidence", + blueprint_revision_ref="evaluation_blueprint_revision_1", + items=(anchor,), + anchor_item_snapshot_refs=(anchor.item_snapshot_ref,), + comparability_status=RunComparabilityStatus.LINKED, + ) + assert caught.value.code == "linked_run_requires_evidence" + + with pytest.raises(DynamicEvaluationLineageError) as caught: + build_dynamic_evaluation_run_lineage( + run_snapshot_ref="evaluation_run_snapshot_unlinked_evidence", + blueprint_revision_ref="evaluation_blueprint_revision_1", + items=(anchor,), + anchor_item_snapshot_refs=(anchor.item_snapshot_ref,), + comparability_status=RunComparabilityStatus.WITHIN_RUN_ONLY, + linking_evidence_ref="linking_evidence_1", + ) + assert caught.value.code == "unexpected_linking_evidence" From 1ff49a82f630280959cb5bdeb89890b4181000b1 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Wed, 2 Sep 2026 15:05:00 +0900 Subject: [PATCH 03/47] feat(lineage): project dynamic evaluation provenance GREEN --- lineageweave/evaluation_lineage.py | 474 +++++++++++++++++++++++++++++ 1 file changed, 474 insertions(+) create mode 100644 lineageweave/evaluation_lineage.py diff --git a/lineageweave/evaluation_lineage.py b/lineageweave/evaluation_lineage.py new file mode 100644 index 000000000..08beb6234 --- /dev/null +++ b/lineageweave/evaluation_lineage.py @@ -0,0 +1,474 @@ +"""Dynamic-evaluation provenance projections owned by LineageWeave. + +The module projects immutable item-generation, rater-observation, adjudication, +calibration, anchor-promotion, and supersession references without creating any +provider configuration, score, psychometric parameter, or adjudication decision. +""" + +from __future__ import annotations + +from collections.abc import Mapping +from dataclasses import InitVar, dataclass +from enum import StrEnum +from typing import Any + +DYNAMIC_EVALUATION_LINEAGE_CONTRACT_ID = "lineageweave_dynamic_evaluation_lineage/v1" +MAX_LINEAGE_REFERENCE_LENGTH = 256 +MAX_LINEAGE_ITEMS = 10_000 +MAX_LINEAGE_REFERENCES = 256 +_ITEM_TOKEN = object() +_RUN_TOKEN = object() + +_PROHIBITED_AUTHORITY_FIELDS = frozenset( + { + "provider_api_key", + "provider_key", + "provider_endpoint", + "model_endpoint", + "model_id", + "score", + "latent_trait", + "pass_fail", + "certification", + "employment_decision", + "adjudication_decision", + } +) +_ITEM_FIELDS = frozenset( + { + "item_snapshot_ref", + "blueprint_revision_ref", + "source_contract_ref", + "source_contract_sha256", + "generation_invocation_ref", + "rater_invocation_refs", + "adjudication_case_ref", + "adjudication_resolution_ref", + "calibration_artifact_refs", + "anchor_promotion_decision_ref", + "supersedes_item_snapshot_ref", + } +) +_RUN_FIELDS = frozenset( + { + "contract_id", + "run_snapshot_ref", + "blueprint_revision_ref", + "items", + "anchor_item_snapshot_refs", + "comparability_status", + "linking_evidence_ref", + } +) + + +class RunComparabilityStatus(StrEnum): + """Comparability claim available for one exact dynamic-evaluation run.""" + + UNAVAILABLE = "unavailable" + WITHIN_RUN_ONLY = "within_run_only" + LINKED = "linked" + + +class DynamicEvaluationLineageError(ValueError): + """Stable fail-closed error for dynamic-evaluation lineage violations.""" + + def __init__(self, code: str, message: str) -> None: + """Retain a bounded machine-readable rejection code.""" + self.code = code + super().__init__(message) + + +def _mapping(value: Any, field_name: str) -> Mapping[str, Any]: + if not isinstance(value, Mapping): + raise DynamicEvaluationLineageError( + "invalid_object", f"{field_name} must be an object" + ) + if any(type(key) is not str for key in value): + raise DynamicEvaluationLineageError( + "invalid_object_key", f"{field_name} keys must be strings" + ) + return value + + +def _reject_unknown_fields( + payload: Mapping[str, Any], allowed: frozenset[str], field_name: str +) -> None: + unknown = set(payload) - allowed + if unknown.intersection(_PROHIBITED_AUTHORITY_FIELDS): + raise DynamicEvaluationLineageError( + "authority_leakage", + f"{field_name} must not contain provider, scoring, or adjudication authority", + ) + if unknown: + raise DynamicEvaluationLineageError( + "unknown_field", + f"{field_name} contains unsupported fields: {sorted(unknown)}", + ) + + +def _reference(value: Any, field_name: str) -> str: + if type(value) is not str: + raise TypeError(f"{field_name} must be a string") + if ( + not value + or len(value) > MAX_LINEAGE_REFERENCE_LENGTH + or value != value.strip() + or value.startswith("\ufeff") + or value.endswith("\ufeff") + or any( + ord(character) < 32 + or 127 <= ord(character) <= 159 + or 0xD800 <= ord(character) <= 0xDFFF + for character in value + ) + ): + raise DynamicEvaluationLineageError( + "invalid_reference", f"{field_name} must be an exact bounded opaque reference" + ) + return value + + +def _optional_reference(value: Any, field_name: str) -> str | None: + if value is None: + return None + return _reference(value, field_name) + + +def _reference_tuple( + value: Any, + field_name: str, + *, + allow_empty: bool, +) -> tuple[str, ...]: + if not isinstance(value, (tuple, list)): + raise TypeError(f"{field_name} must be a tuple or list") + if (not allow_empty and not value) or len(value) > MAX_LINEAGE_REFERENCES: + lower = 0 if allow_empty else 1 + raise DynamicEvaluationLineageError( + "invalid_reference_count", + f"{field_name} must contain {lower}..{MAX_LINEAGE_REFERENCES} references", + ) + normalized = tuple( + _reference(item, f"{field_name}[{index}]") + for index, item in enumerate(value) + ) + if len(set(normalized)) != len(normalized): + raise DynamicEvaluationLineageError( + "duplicate_reference", f"{field_name} must not contain duplicates" + ) + return normalized + + +def _sha256(value: Any, field_name: str) -> str: + if type(value) is not str: + raise TypeError(f"{field_name} must be a string") + if len(value) != 64 or any(character not in "0123456789abcdef" for character in value): + raise DynamicEvaluationLineageError( + "invalid_sha256", + f"{field_name} must be a complete lowercase SHA-256 digest", + ) + return value + + +def _comparability_status(value: Any) -> RunComparabilityStatus: + if type(value) is RunComparabilityStatus: + return value + if type(value) is not str: + raise TypeError( + "comparability_status must be a RunComparabilityStatus or exact string" + ) + try: + return RunComparabilityStatus(value) + except ValueError as exc: + raise DynamicEvaluationLineageError( + "invalid_comparability_status", "unsupported run comparability status" + ) from exc + + +@dataclass(frozen=True, slots=True) +class DynamicEvaluationItemLineage: + """Provenance projection for one immutable dynamic item snapshot.""" + + item_snapshot_ref: str + blueprint_revision_ref: str + source_contract_ref: str + source_contract_sha256: str + generation_invocation_ref: str | None + rater_invocation_refs: tuple[str, ...] + adjudication_case_ref: str | None + adjudication_resolution_ref: str | None + calibration_artifact_refs: tuple[str, ...] + anchor_promotion_decision_ref: str | None + supersedes_item_snapshot_ref: str | None + _admission_token: InitVar[object | None] = None + + def __post_init__(self, _admission_token: object | None) -> None: + """Prevent direct construction that bypasses the lineage builder.""" + if _admission_token is not _ITEM_TOKEN: + raise ValueError( + "DynamicEvaluationItemLineage must be created by " + "build_dynamic_evaluation_item_lineage" + ) + + def to_mapping(self) -> dict[str, Any]: + """Return the source-text-free projection payload.""" + return { + "item_snapshot_ref": self.item_snapshot_ref, + "blueprint_revision_ref": self.blueprint_revision_ref, + "source_contract_ref": self.source_contract_ref, + "source_contract_sha256": self.source_contract_sha256, + "generation_invocation_ref": self.generation_invocation_ref, + "rater_invocation_refs": list(self.rater_invocation_refs), + "adjudication_case_ref": self.adjudication_case_ref, + "adjudication_resolution_ref": self.adjudication_resolution_ref, + "calibration_artifact_refs": list(self.calibration_artifact_refs), + "anchor_promotion_decision_ref": self.anchor_promotion_decision_ref, + "supersedes_item_snapshot_ref": self.supersedes_item_snapshot_ref, + } + + @classmethod + def from_mapping(cls, value: Any) -> "DynamicEvaluationItemLineage": + """Translate an untrusted item-lineage projection through the ACL.""" + payload = _mapping(value, "item lineage") + _reject_unknown_fields(payload, _ITEM_FIELDS, "item lineage") + missing = _ITEM_FIELDS - set(payload) + if missing: + raise DynamicEvaluationLineageError( + "missing_field", f"item lineage is missing fields: {sorted(missing)}" + ) + return build_dynamic_evaluation_item_lineage( + item_snapshot_ref=payload["item_snapshot_ref"], + blueprint_revision_ref=payload["blueprint_revision_ref"], + source_contract_ref=payload["source_contract_ref"], + source_contract_sha256=payload["source_contract_sha256"], + generation_invocation_ref=payload["generation_invocation_ref"], + rater_invocation_refs=payload["rater_invocation_refs"], + adjudication_case_ref=payload["adjudication_case_ref"], + adjudication_resolution_ref=payload["adjudication_resolution_ref"], + calibration_artifact_refs=payload["calibration_artifact_refs"], + anchor_promotion_decision_ref=payload["anchor_promotion_decision_ref"], + supersedes_item_snapshot_ref=payload["supersedes_item_snapshot_ref"], + ) + + +def build_dynamic_evaluation_item_lineage( + *, + item_snapshot_ref: str, + blueprint_revision_ref: str, + source_contract_ref: str, + source_contract_sha256: str, + generation_invocation_ref: str | None, + rater_invocation_refs: tuple[str, ...] | list[str], + adjudication_case_ref: str | None, + adjudication_resolution_ref: str | None, + calibration_artifact_refs: tuple[str, ...] | list[str], + anchor_promotion_decision_ref: str | None, + supersedes_item_snapshot_ref: str | None, +) -> DynamicEvaluationItemLineage: + """Build one lineage projection without transferring foreign authority.""" + normalized_item_ref = _reference(item_snapshot_ref, "item_snapshot_ref") + normalized_case_ref = _optional_reference( + adjudication_case_ref, "adjudication_case_ref" + ) + normalized_resolution_ref = _optional_reference( + adjudication_resolution_ref, "adjudication_resolution_ref" + ) + if normalized_resolution_ref is not None and normalized_case_ref is None: + raise DynamicEvaluationLineageError( + "resolution_requires_case", + "an adjudication resolution must reference its separate case", + ) + + normalized_supersedes_ref = _optional_reference( + supersedes_item_snapshot_ref, "supersedes_item_snapshot_ref" + ) + if normalized_supersedes_ref == normalized_item_ref: + raise DynamicEvaluationLineageError( + "self_supersession", "an item snapshot cannot supersede itself" + ) + + return DynamicEvaluationItemLineage( + item_snapshot_ref=normalized_item_ref, + blueprint_revision_ref=_reference( + blueprint_revision_ref, "blueprint_revision_ref" + ), + source_contract_ref=_reference(source_contract_ref, "source_contract_ref"), + source_contract_sha256=_sha256( + source_contract_sha256, "source_contract_sha256" + ), + generation_invocation_ref=_optional_reference( + generation_invocation_ref, "generation_invocation_ref" + ), + rater_invocation_refs=_reference_tuple( + rater_invocation_refs, "rater_invocation_refs", allow_empty=True + ), + adjudication_case_ref=normalized_case_ref, + adjudication_resolution_ref=normalized_resolution_ref, + calibration_artifact_refs=_reference_tuple( + calibration_artifact_refs, + "calibration_artifact_refs", + allow_empty=True, + ), + anchor_promotion_decision_ref=_optional_reference( + anchor_promotion_decision_ref, "anchor_promotion_decision_ref" + ), + supersedes_item_snapshot_ref=normalized_supersedes_ref, + _admission_token=_ITEM_TOKEN, + ) + + +@dataclass(frozen=True, slots=True) +class DynamicEvaluationRunLineage: + """Immutable LineageWeave projection for one resolved evaluation run.""" + + run_snapshot_ref: str + blueprint_revision_ref: str + items: tuple[DynamicEvaluationItemLineage, ...] + anchor_item_snapshot_refs: tuple[str, ...] + comparability_status: RunComparabilityStatus + linking_evidence_ref: str | None + contract_id: str = DYNAMIC_EVALUATION_LINEAGE_CONTRACT_ID + _admission_token: InitVar[object | None] = None + + def __post_init__(self, _admission_token: object | None) -> None: + """Prevent direct construction outside the run-lineage builder.""" + if _admission_token is not _RUN_TOKEN: + raise ValueError( + "DynamicEvaluationRunLineage must be created by " + "build_dynamic_evaluation_run_lineage" + ) + + def to_mapping(self) -> dict[str, Any]: + """Return the versioned source-text-free run projection.""" + return { + "contract_id": self.contract_id, + "run_snapshot_ref": self.run_snapshot_ref, + "blueprint_revision_ref": self.blueprint_revision_ref, + "items": [item.to_mapping() for item in self.items], + "anchor_item_snapshot_refs": list(self.anchor_item_snapshot_refs), + "comparability_status": self.comparability_status.value, + "linking_evidence_ref": self.linking_evidence_ref, + } + + @classmethod + def from_mapping(cls, value: Any) -> "DynamicEvaluationRunLineage": + """Translate an untrusted run-lineage projection through the ACL.""" + payload = _mapping(value, "run lineage") + _reject_unknown_fields(payload, _RUN_FIELDS, "run lineage") + missing = _RUN_FIELDS - set(payload) + if missing: + raise DynamicEvaluationLineageError( + "missing_field", f"run lineage is missing fields: {sorted(missing)}" + ) + if payload["contract_id"] != DYNAMIC_EVALUATION_LINEAGE_CONTRACT_ID: + raise DynamicEvaluationLineageError( + "contract_incompatible", "unsupported dynamic evaluation lineage contract" + ) + raw_items = payload["items"] + if not isinstance(raw_items, (tuple, list)): + raise TypeError("items must be a tuple or list") + return build_dynamic_evaluation_run_lineage( + run_snapshot_ref=payload["run_snapshot_ref"], + blueprint_revision_ref=payload["blueprint_revision_ref"], + items=tuple( + DynamicEvaluationItemLineage.from_mapping(item) for item in raw_items + ), + anchor_item_snapshot_refs=payload["anchor_item_snapshot_refs"], + comparability_status=payload["comparability_status"], + linking_evidence_ref=payload["linking_evidence_ref"], + ) + + +def build_dynamic_evaluation_run_lineage( + *, + run_snapshot_ref: str, + blueprint_revision_ref: str, + items: tuple[DynamicEvaluationItemLineage, ...] | list[DynamicEvaluationItemLineage], + anchor_item_snapshot_refs: tuple[str, ...] | list[str], + comparability_status: RunComparabilityStatus | str, + linking_evidence_ref: str | None = None, +) -> DynamicEvaluationRunLineage: + """Build a run projection that may explicitly contain zero fixed anchors.""" + if not isinstance(items, (tuple, list)) or not items: + raise DynamicEvaluationLineageError( + "invalid_item_set", "run lineage must contain at least one item" + ) + if len(items) > MAX_LINEAGE_ITEMS: + raise DynamicEvaluationLineageError( + "item_set_budget_exceeded", + f"run lineage may contain at most {MAX_LINEAGE_ITEMS} items", + ) + normalized_items = tuple(items) + if any(type(item) is not DynamicEvaluationItemLineage for item in normalized_items): + raise TypeError("items must contain exact DynamicEvaluationItemLineage values") + + normalized_blueprint_ref = _reference( + blueprint_revision_ref, "blueprint_revision_ref" + ) + if any( + item.blueprint_revision_ref != normalized_blueprint_ref + for item in normalized_items + ): + raise DynamicEvaluationLineageError( + "item_blueprint_mismatch", + "every item projection must use the run blueprint revision", + ) + item_refs = tuple(item.item_snapshot_ref for item in normalized_items) + if len(set(item_refs)) != len(item_refs): + raise DynamicEvaluationLineageError( + "duplicate_item_snapshot", "run lineage item snapshots must be unique" + ) + + normalized_anchor_refs = _reference_tuple( + anchor_item_snapshot_refs, + "anchor_item_snapshot_refs", + allow_empty=True, + ) + unknown_anchors = set(normalized_anchor_refs) - set(item_refs) + if unknown_anchors: + raise DynamicEvaluationLineageError( + "unknown_anchor_item", "every anchor must identify an item in this run" + ) + item_by_ref = {item.item_snapshot_ref: item for item in normalized_items} + for anchor_ref in normalized_anchor_refs: + anchor = item_by_ref[anchor_ref] + if ( + anchor.anchor_promotion_decision_ref is None + or not anchor.calibration_artifact_refs + ): + raise DynamicEvaluationLineageError( + "anchor_requires_promotion_evidence", + "an anchor requires separate promotion and calibration evidence", + ) + + normalized_status = _comparability_status(comparability_status) + normalized_linking_ref = _optional_reference( + linking_evidence_ref, "linking_evidence_ref" + ) + if normalized_status is RunComparabilityStatus.LINKED: + if not normalized_anchor_refs: + raise DynamicEvaluationLineageError( + "linked_run_requires_anchor", + "cross-version linking requires at least one promoted anchor", + ) + if normalized_linking_ref is None: + raise DynamicEvaluationLineageError( + "linked_run_requires_evidence", + "linked comparability requires immutable linking evidence", + ) + elif normalized_linking_ref is not None: + raise DynamicEvaluationLineageError( + "unexpected_linking_evidence", + "unavailable and within-run-only projections cannot claim linking evidence", + ) + + return DynamicEvaluationRunLineage( + run_snapshot_ref=_reference(run_snapshot_ref, "run_snapshot_ref"), + blueprint_revision_ref=normalized_blueprint_ref, + items=normalized_items, + anchor_item_snapshot_refs=normalized_anchor_refs, + comparability_status=normalized_status, + linking_evidence_ref=normalized_linking_ref, + _admission_token=_RUN_TOKEN, + ) From 873f700cff7cbdb859c077a77b2b76ba3a143692 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Wed, 2 Sep 2026 15:05:25 +0900 Subject: [PATCH 04/47] test(api): require dynamic evaluation lineage exports RED --- tests/test_dynamic_evaluation_public_api.py | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) create mode 100644 tests/test_dynamic_evaluation_public_api.py diff --git a/tests/test_dynamic_evaluation_public_api.py b/tests/test_dynamic_evaluation_public_api.py new file mode 100644 index 000000000..854a0a837 --- /dev/null +++ b/tests/test_dynamic_evaluation_public_api.py @@ -0,0 +1,21 @@ +"""Public package export contract for dynamic evaluation lineage.""" + +from __future__ import annotations + +import lineageweave + + +def test_public_package_exports_dynamic_evaluation_lineage_contract() -> None: + """The reusable projection is discoverable from the package boundary.""" + assert ( + lineageweave.DYNAMIC_EVALUATION_LINEAGE_CONTRACT_ID + == "lineageweave_dynamic_evaluation_lineage/v1" + ) + assert lineageweave.DynamicEvaluationItemLineage.__module__.endswith( + "evaluation_lineage" + ) + assert lineageweave.DynamicEvaluationRunLineage.__module__.endswith( + "evaluation_lineage" + ) + assert callable(lineageweave.build_dynamic_evaluation_item_lineage) + assert callable(lineageweave.build_dynamic_evaluation_run_lineage) From 09adf1c7e911bd60350ed251e6ad7547dcb299ba Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Wed, 2 Sep 2026 15:05:44 +0900 Subject: [PATCH 05/47] feat(api): export dynamic evaluation lineage GREEN --- lineageweave/__init__.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/lineageweave/__init__.py b/lineageweave/__init__.py index 45c371fa7..31c3cfa8f 100644 --- a/lineageweave/__init__.py +++ b/lineageweave/__init__.py @@ -9,6 +9,15 @@ from .affiliate_tree import build_affiliate_forest from .corporate_hierarchy_resolution import resolve_corporate_entity from .entity_relationship_classification import OrganizationRelationship +from .evaluation_lineage import ( + DYNAMIC_EVALUATION_LINEAGE_CONTRACT_ID, + DynamicEvaluationItemLineage, + DynamicEvaluationLineageError, + DynamicEvaluationRunLineage, + RunComparabilityStatus, + build_dynamic_evaluation_item_lineage, + build_dynamic_evaluation_run_lineage, +) from .external_lineage_analysis import analyze_external_lineage from .external_lineage_contract import ( CONTRACT_VERSION, @@ -73,7 +82,11 @@ __all__ = [ "CHANNEL_EVIDENCE_TOLERANCE", "CONTRACT_VERSION", + "DYNAMIC_EVALUATION_LINEAGE_CONTRACT_ID", "ChannelEvidence", + "DynamicEvaluationItemLineage", + "DynamicEvaluationLineageError", + "DynamicEvaluationRunLineage", "ExplicitParent", "LineageAnalysisPolicy", "LineageAnalysisRequest", @@ -106,9 +119,12 @@ "ProvLiteral", "ProvValidationError", "Record", + "RunComparabilityStatus", "Tree", "analyze_external_lineage", "build_affiliate_forest", + "build_dynamic_evaluation_item_lineage", + "build_dynamic_evaluation_run_lineage", "build_workspace_naruon_client", "cited_post_summaries", "default_calendar_window", From 48fd9492ef905f2220cefa5f59a9d5d02c9a3a8f Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Wed, 2 Sep 2026 15:06:48 +0900 Subject: [PATCH 06/47] docs(adr): govern dynamic evaluation lineage projection --- docs/adr/0352-dynamic-evaluation-lineage.md | 170 ++++++++++++++++++++ 1 file changed, 170 insertions(+) create mode 100644 docs/adr/0352-dynamic-evaluation-lineage.md diff --git a/docs/adr/0352-dynamic-evaluation-lineage.md b/docs/adr/0352-dynamic-evaluation-lineage.md new file mode 100644 index 000000000..3e15c26c1 --- /dev/null +++ b/docs/adr/0352-dynamic-evaluation-lineage.md @@ -0,0 +1,170 @@ +# ADR 0352: Project dynamic evaluation snapshots without absorbing decision authority + +- Status: Proposed +- Date: 2026-09-02 +- Depends on: ADR 0300 (contextual-orchestrator ownership boundary), ADR 0301 (dichotomous measurement policy) + +## Context + +A product evaluation may resolve its concrete items dynamically from an authored +blueprint, a production sample, a controlled perturbation, or a model/algorithmic +generator. A fixed item set or validated anchor corpus may not exist during cold +start. Nevertheless, an evaluation must remain reproducible enough to determine +which exact item snapshots, generator invocations, rater observations, +adjudication artifacts, calibration evidence, and promotion decisions informed a +result. + +A single mutable `evaluation_item` record would collapse independent facts and +permit later review to rewrite history. It could also allow LineageWeave to absorb +provider credentials, model routing, psychometric calculation, hosted +adjudication, or source-system authority that belongs to other bounded contexts. + +## Decision + +LineageWeave publishes the source-text-free +`lineageweave_dynamic_evaluation_lineage/v1` projection. It contains two sealed +aggregate forms. + +### Dynamic evaluation item lineage + +One item projection records only immutable references: + +- exact item-snapshot and blueprint-revision identity; +- exact released source-contract identity and complete lowercase SHA-256 digest; +- optional item-generation invocation; +- zero or more immutable rater-invocation references; +- optional adjudication-case and separate adjudication-resolution references; +- zero or more calibration-artifact references; +- optional separate anchor-promotion decision; +- optional predecessor item snapshot that this version supersedes. + +An adjudication resolution requires its case. The source rater invocations remain +present and are never replaced by the resolution. A successor may identify an +older snapshot but cannot supersede itself. + +### Dynamic evaluation run lineage + +One run projection freezes: + +- one exact run-snapshot and blueprint-revision identity; +- one non-empty, unique, blueprint-consistent item set; +- zero or more item snapshots explicitly acting as anchors; +- one comparability state: `unavailable`, `within_run_only`, or `linked`; +- an immutable linking-evidence reference only when comparability is `linked`. + +Cold-start runs with zero fixed anchors are valid for pilot, diagnostic, and +within-run evidence collection. They cannot claim cross-version linked scores. + +An item can appear in `anchor_item_snapshot_refs` only when its lineage includes +both separate calibration evidence and an anchor-promotion decision. An +adjudication resolution alone is insufficient. `linked` additionally requires at +least one such promoted anchor and independent linking evidence. + +## Ownership boundary + +LineageWeave owns product-specific source/rubric/instrument provenance and the +projection that lets a buyer reconstruct how evidence artifacts relate. It does +not create the foreign artifacts it references. + +- contextual-orchestrator owns provider/model execution, routing, fallback, + dynamic item-generation invocation evidence, and rater-observation creation. +- fast-mlsirm owns reusable measurement Published Languages and all production + psychometric calibration, fit, DIF, information, linking, uncertainty, and + score arithmetic. +- Psychometrics Commons owns hosted blueprint/run lifecycle, panel assignment, + adjudication transaction state, tenant authorization, persistence, and + immutable result publication. +- TEPP owns temporal/event semantics and later drift, change-point, or invariance + monitoring. + +Cross-repository integration must consume immutable released/versioned artifacts +with exact digests. Mutable sibling PR heads, foreign service databases, and +cross-service SQL are not production contracts. + +## Fail-closed behavior + +The projection rejects: + +- provider credentials, endpoints, provider/model selection fields, scores, + latent traits, pass/fail, certification, employment decisions, or embedded + adjudication decisions; +- unknown fields and non-string mapping keys; +- empty, padded, control-bearing, surrogate-bearing, or overlong opaque + references; +- malformed or non-lowercase contract digests; +- duplicate item/rater/calibration/anchor references; +- item sets beyond the bounded allocation ceiling; +- mixed blueprint revisions in one run snapshot; +- a resolution without a case or self-supersession; +- anchor claims without separate calibration and promotion evidence; +- linked comparability without promoted anchors and linking evidence; +- linking evidence on an unavailable or within-run-only projection. + +No missing reference is converted into a score, default anchor, provider guess, +or synthetic lineage edge. + +## Consequences + +### Benefits + +- dynamic evaluations can begin before a fixed item corpus exists; +- each run remains tied to its actual immutable item set rather than a mutable + blueprint or regenerated approximation; +- adjudication remains review evidence instead of overwriting observations; +- anchor promotion, calibration, and linking remain separately auditable; +- LineageWeave can display an explicit no-anchor/no-linking limitation without + inventing comparability; +- provider and psychometric authorities remain in their canonical owners. + +### Costs + +- the hosted system must persist the referenced run and item snapshots before + dispatching observations; +- downstream projections require released contract versions and digests; +- source content remains separately permissioned and cannot be recovered from + this metadata-only envelope; +- user interfaces must distinguish provisional, adjudicated, calibrated, + promoted-anchor, and linked states rather than displaying one generic + “evaluated” badge. + +## Alternatives considered + +1. **Reuse a mutable golden-prompt table.** Rejected because no fixed set is + required, “golden” conflates adjudication with validation, and later edits + would destroy run identity. +2. **Store provider/model payloads in LineageWeave.** Rejected because provider + execution and credential policy belong to contextual-orchestrator and raw + content has separate access/retention requirements. +3. **Treat adjudicated items as anchors automatically.** Rejected because an + adjudication resolution does not establish calibration, fit, fairness, + invariance, approval, or cross-version linking. +4. **Block all evaluation until anchors exist.** Rejected because governed pilot + and diagnostic evidence is necessary to create and validate the first anchor + corpus. + +## Verification + +Focused tests cover zero-anchor runs, adjudication/source-observation separation, +anchor-promotion requirements, linked-evidence requirements, immutable collection +copying, strict mapping admission, reference and digest hygiene, blueprint +consistency, duplicate and resource limits, public exports, and direct-construction +seals. The new projection module must retain complete statement and branch +coverage on the unchanged exact head. + +No fixed production item example, provider call, score, database migration, or +adjudication action is introduced by this ADR. + +## References + +American Educational Research Association, American Psychological Association, +& National Council on Measurement in Education. (2014). *Standards for +educational and psychological testing*. American Educational Research +Association. + +Evans, E. (2003). *Domain-driven design: Tackling complexity in the heart of +software*. Addison-Wesley. + +Moreau, L., Missier, P., Belhajjame, K., B’Far, R., Cheney, J., Coppens, S., +Cresswell, S., Gil, Y., Groth, P., Klyne, G., Lebo, T., McCusker, J., Miles, S., +Myers, J., Sahoo, S., & Tilmes, C. (2013). PROV-DM: The PROV data model. World +Wide Web Consortium. From 66fe4f710df46ea05c6c1aca247d798ebef2028b Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Wed, 2 Sep 2026 15:07:46 +0900 Subject: [PATCH 07/47] docs(domain): add dynamic evaluation ubiquitous language --- docs/ubiquitous-language.md | 34 +++++++++++++++++++++++++++++++++- 1 file changed, 33 insertions(+), 1 deletion(-) diff --git a/docs/ubiquitous-language.md b/docs/ubiquitous-language.md index ef300ab5a..387dc93c2 100644 --- a/docs/ubiquitous-language.md +++ b/docs/ubiquitous-language.md @@ -37,6 +37,38 @@ A non-operational instrument state in which observations may be collected for si **Activation** The governed transition from pilot to operational scoring after the instrument's declared evidence criteria are satisfied. Activation is fail-closed: insufficient evidence preserves observations without issuing a latent score. +## Dynamic evaluation vocabulary + +**Evaluation Blueprint Revision** +An immutable plan that governs how evaluation items are sourced, generated, sampled, covered, and reviewed. It is not the concrete item set administered in a run and may legitimately declare zero fixed anchors during cold start. + +**Dynamic Evaluation Item Snapshot** +The exact, immutable identity of one concrete authored, sampled, perturbed, adversarial, or generated item resolved under a blueprint revision. It preserves a content reference/digest and foreign evidence references without storing provider credentials or granting the generator decision authority. + +**Evaluation Run Snapshot** +The immutable set of concrete item snapshots resolved before observations are interpreted for one run. Replacing an item creates a new run-snapshot revision; a mutable blueprint or later regeneration cannot silently rewrite the administered set. + +**Reference Semantics** +The evidence form used to judge a response, such as exact, constraint, acceptable-set, rubric, pairwise, or open-ended semantics. It is distinct from the current governance status of that reference. + +**Reference Status** +The independent state of an item's reference evidence: unresolved, provisional, adjudication-required, adjudicated, validated, or invalidated. `adjudicated` does not mean calibrated, approved, validated, active, or anchor-eligible. + +**Validated Anchor** +An item snapshot explicitly promoted for an intended linking/use scope after separate validation and calibration evidence. An adjudication resolution or repeated model agreement alone does not create an anchor. + +**Anchor Promotion Decision** +A separate immutable governance artifact that changes an item's evaluation role for a declared use without rewriting the prior item snapshot, source observations, or adjudication case. + +**Run Comparability** +The claim supported for one run: unavailable, within-run-only, or linked. `linked` requires at least one validated/promoted anchor plus independent linking evidence; using the same nominal score range or rubric name is insufficient. + +**Content Replay** +Retrieval of the exact frozen content reference/digest used in a prior run. It is distinct from invoking a generator again. + +**Content Regeneration** +A new generation execution from recorded inputs. A seed, prompt revision, or provider/model identity is provenance only and does not establish deterministic regeneration without independent evidence. + ## Model families **Rasch** @@ -69,7 +101,7 @@ The LineageWeave-facing reference to a contextual-orchestrator orchestration/pro A reproducible judging condition that may affect observations: judge model, provider observation identity/provenance, prompt/policy revision, language, occasion, agent role, and other declared method conditions. Scientifically material facets are retained for severity/leniency, interaction, repeatability, calibration, and DIF/invariance analysis. **Adjudication** -The governed use of evidence and one or more fallible observations to support a product decision. Adjudication is not synonymous with accepting an LLM answer. +The governed use of evidence and one or more fallible observations to support a product decision. Adjudication is not synonymous with accepting an LLM answer, validating an item, promoting an anchor, or rewriting its source observations. **Disagreement** A reproducible difference among judge observations or between judge and independent criterion evidence. Disagreement is retained/analyzed; it is not automatically resolved by majority vote. From 3e1cf55c3aaef1c61b1840440042f7ca513cc4f7 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Wed, 2 Sep 2026 15:08:03 +0900 Subject: [PATCH 08/47] docs(changelog): record dynamic evaluation lineage projection --- CHANGELOG.d/dynamic-evaluation-lineage.md | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 CHANGELOG.d/dynamic-evaluation-lineage.md diff --git a/CHANGELOG.d/dynamic-evaluation-lineage.md b/CHANGELOG.d/dynamic-evaluation-lineage.md new file mode 100644 index 000000000..1bc4d153b --- /dev/null +++ b/CHANGELOG.d/dynamic-evaluation-lineage.md @@ -0,0 +1,6 @@ +## Added + +- Added the versioned `lineageweave_dynamic_evaluation_lineage/v1` projection for dynamically resolved evaluation item and run snapshots. +- Preserved generator, rater, adjudication-case/resolution, calibration, anchor-promotion, linking, and supersession references as separate immutable evidence instead of overwriting source observations or inventing decision authority. +- Permitted zero-anchor cold-start and within-run projections while requiring separate calibration, promotion, and linking evidence before an item/run can be represented as an anchor or cross-version linked. +- Rejected provider credentials/endpoints, scores, embedded adjudication decisions, mixed-blueprint item sets, duplicate identities, and unsupported linking claims at the LineageWeave Anti-Corruption Layer. From b52bcc998c398f648f42755b460d13bd1eeea904 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Wed, 2 Sep 2026 16:00:56 +0900 Subject: [PATCH 09/47] test(lineage): reject oversized run payload before item decoding --- ...mic_evaluation_lineage_admission_budget.py | 30 +++++++++++++++++++ 1 file changed, 30 insertions(+) create mode 100644 tests/test_dynamic_evaluation_lineage_admission_budget.py diff --git a/tests/test_dynamic_evaluation_lineage_admission_budget.py b/tests/test_dynamic_evaluation_lineage_admission_budget.py new file mode 100644 index 000000000..6736a1ee4 --- /dev/null +++ b/tests/test_dynamic_evaluation_lineage_admission_budget.py @@ -0,0 +1,30 @@ +"""Admission-budget regressions for dynamic evaluation lineage payloads.""" + +from __future__ import annotations + +import pytest + +from lineageweave.evaluation_lineage import ( + DYNAMIC_EVALUATION_LINEAGE_CONTRACT_ID, + MAX_LINEAGE_ITEMS, + DynamicEvaluationLineageError, + DynamicEvaluationRunLineage, +) + + +def test_run_mapping_rejects_oversized_item_array_before_item_decoding() -> None: + """Reject hostile item counts before spending work on individual item payloads.""" + payload = { + "contract_id": DYNAMIC_EVALUATION_LINEAGE_CONTRACT_ID, + "run_snapshot_ref": "evaluation_run_snapshot_oversized", + "blueprint_revision_ref": "evaluation_blueprint_revision_1", + "items": [object()] * (MAX_LINEAGE_ITEMS + 1), + "anchor_item_snapshot_refs": [], + "comparability_status": "unavailable", + "linking_evidence_ref": None, + } + + with pytest.raises(DynamicEvaluationLineageError) as caught: + DynamicEvaluationRunLineage.from_mapping(payload) + + assert caught.value.code == "item_set_budget_exceeded" From 3b76d0ea3253192211e739bf18c3cb8c727496d5 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Wed, 2 Sep 2026 16:02:12 +0900 Subject: [PATCH 10/47] fix(lineage): enforce run payload budget before decoding --- lineageweave/evaluation_lineage.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/lineageweave/evaluation_lineage.py b/lineageweave/evaluation_lineage.py index 08beb6234..640d91471 100644 --- a/lineageweave/evaluation_lineage.py +++ b/lineageweave/evaluation_lineage.py @@ -368,6 +368,11 @@ def from_mapping(cls, value: Any) -> "DynamicEvaluationRunLineage": raw_items = payload["items"] if not isinstance(raw_items, (tuple, list)): raise TypeError("items must be a tuple or list") + if len(raw_items) > MAX_LINEAGE_ITEMS: + raise DynamicEvaluationLineageError( + "item_set_budget_exceeded", + f"run lineage may contain at most {MAX_LINEAGE_ITEMS} items", + ) return build_dynamic_evaluation_run_lineage( run_snapshot_ref=payload["run_snapshot_ref"], blueprint_revision_ref=payload["blueprint_revision_ref"], @@ -471,4 +476,4 @@ def build_dynamic_evaluation_run_lineage( comparability_status=normalized_status, linking_evidence_ref=normalized_linking_ref, _admission_token=_RUN_TOKEN, - ) + ) \ No newline at end of file From dd739cae2e95c74a8cb7b9f4381ff80c848e9b3c Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Wed, 2 Sep 2026 16:18:00 +0900 Subject: [PATCH 11/47] test(lineage): reject collapsed adjudication identity --- tests/test_dynamic_evaluation_lineage_boundaries.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/tests/test_dynamic_evaluation_lineage_boundaries.py b/tests/test_dynamic_evaluation_lineage_boundaries.py index f2578e77e..000e4ffd2 100644 --- a/tests/test_dynamic_evaluation_lineage_boundaries.py +++ b/tests/test_dynamic_evaluation_lineage_boundaries.py @@ -82,6 +82,15 @@ def test_item_authority_fields_are_rejected() -> None: assert caught.value.code == "authority_leakage" +def test_adjudication_case_and_resolution_keep_distinct_identities() -> None: + with pytest.raises(DynamicEvaluationLineageError) as caught: + _item( + adjudication_case_ref="adjudication_record_1", + adjudication_resolution_ref="adjudication_record_1", + ) + assert caught.value.code == "adjudication_reference_collision" + + @pytest.mark.parametrize( "invalid", ( From 91da3a2159a260748ccf5ebeccd8488f61e82570 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Wed, 2 Sep 2026 16:19:10 +0900 Subject: [PATCH 12/47] fix(lineage): keep adjudication identities distinct --- lineageweave/evaluation_lineage.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/lineageweave/evaluation_lineage.py b/lineageweave/evaluation_lineage.py index 640d91471..d49178141 100644 --- a/lineageweave/evaluation_lineage.py +++ b/lineageweave/evaluation_lineage.py @@ -279,6 +279,11 @@ def build_dynamic_evaluation_item_lineage( "resolution_requires_case", "an adjudication resolution must reference its separate case", ) + if normalized_resolution_ref is not None and normalized_resolution_ref == normalized_case_ref: + raise DynamicEvaluationLineageError( + "adjudication_reference_collision", + "adjudication case and resolution must retain distinct identities", + ) normalized_supersedes_ref = _optional_reference( supersedes_item_snapshot_ref, "supersedes_item_snapshot_ref" From fc73f7002b3b9c0b87f6af68b6c76ad49e2de70e Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Wed, 2 Sep 2026 16:49:53 +0900 Subject: [PATCH 13/47] test(lineage): reject invisible provenance reference controls --- tests/test_dynamic_evaluation_lineage_boundaries.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/test_dynamic_evaluation_lineage_boundaries.py b/tests/test_dynamic_evaluation_lineage_boundaries.py index 000e4ffd2..01958060d 100644 --- a/tests/test_dynamic_evaluation_lineage_boundaries.py +++ b/tests/test_dynamic_evaluation_lineage_boundaries.py @@ -99,12 +99,14 @@ def test_adjudication_case_and_resolution_keep_distinct_identities() -> None: "item_ref ", "\ufeffitem_ref", "item_ref\ufeff", + "item\u200bref", + "item\u202eref", "line\nbreak", "\ud800", "x" * 257, ), ) -def test_references_are_exact_bounded_unicode_scalars(invalid: str) -> None: +def test_references_are_exact_bounded_and_free_of_format_controls(invalid: str) -> None: with pytest.raises(DynamicEvaluationLineageError) as caught: _item(item_snapshot_ref=invalid) assert caught.value.code == "invalid_reference" From 8fbb21a71054a89f0b636889a2ef26142d4b9852 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Wed, 2 Sep 2026 16:50:50 +0900 Subject: [PATCH 14/47] fix(lineage): reject invisible Unicode format controls --- lineageweave/evaluation_lineage.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/lineageweave/evaluation_lineage.py b/lineageweave/evaluation_lineage.py index d49178141..f9de59e87 100644 --- a/lineageweave/evaluation_lineage.py +++ b/lineageweave/evaluation_lineage.py @@ -7,6 +7,7 @@ from __future__ import annotations +import unicodedata from collections.abc import Mapping from dataclasses import InitVar, dataclass from enum import StrEnum @@ -120,6 +121,7 @@ def _reference(value: Any, field_name: str) -> str: ord(character) < 32 or 127 <= ord(character) <= 159 or 0xD800 <= ord(character) <= 0xDFFF + or unicodedata.category(character) == "Cf" for character in value ) ): @@ -481,4 +483,4 @@ def build_dynamic_evaluation_run_lineage( comparability_status=normalized_status, linking_evidence_ref=normalized_linking_ref, _admission_token=_RUN_TOKEN, - ) \ No newline at end of file + ) From bdf00db322f9dd903c950f85b5fc9c63a170e935 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Wed, 2 Sep 2026 16:53:21 +0900 Subject: [PATCH 15/47] docs(adr): record Unicode provenance-reference boundary --- docs/adr/0352-dynamic-evaluation-lineage.md | 26 ++++++++++++++++++--- 1 file changed, 23 insertions(+), 3 deletions(-) diff --git a/docs/adr/0352-dynamic-evaluation-lineage.md b/docs/adr/0352-dynamic-evaluation-lineage.md index 3e15c26c1..16edbf3c1 100644 --- a/docs/adr/0352-dynamic-evaluation-lineage.md +++ b/docs/adr/0352-dynamic-evaluation-lineage.md @@ -19,6 +19,15 @@ permit later review to rewrite history. It could also allow LineageWeave to abso provider credentials, model routing, psychometric calculation, hosted adjudication, or source-system authority that belongs to other bounded contexts. +Opaque provenance references also cross service and rendering boundaries. Unicode +format controls can be machine-distinct while remaining visually absent or +changing bidirectional presentation, creating an avoidable alias/spoofing surface +for identifiers used in equality and provenance joins. Unicode Technical Standard +#39 treats identifier ambiguity and default-ignorable characters as security +concerns. LineageWeave therefore rejects Unicode `Cf` format controls in these +opaque references rather than normalizing them into a guessed identity. This is a +product-specific restrictive profile, not a claim of full UTS #39 conformance. + ## Decision LineageWeave publishes the source-text-free @@ -89,8 +98,8 @@ The projection rejects: latent traits, pass/fail, certification, employment decisions, or embedded adjudication decisions; - unknown fields and non-string mapping keys; -- empty, padded, control-bearing, surrogate-bearing, or overlong opaque - references; +- empty, padded, Unicode-format-control-bearing, control-bearing, + surrogate-bearing, or overlong opaque references; - malformed or non-lowercase contract digests; - duplicate item/rater/calibration/anchor references; - item sets beyond the bounded allocation ceiling; @@ -112,6 +121,7 @@ or synthetic lineage edge. blueprint or regenerated approximation; - adjudication remains review evidence instead of overwriting observations; - anchor promotion, calibration, and linking remain separately auditable; +- opaque references cannot differ only through invisible Unicode format controls; - LineageWeave can display an explicit no-anchor/no-linking limitation without inventing comparability; - provider and psychometric authorities remain in their canonical owners. @@ -123,6 +133,9 @@ or synthetic lineage edge. - downstream projections require released contract versions and digests; - source content remains separately permissioned and cannot be recovered from this metadata-only envelope; +- external adapters must map any legitimate foreign identifier containing a + rejected format control to a separate canonical released reference instead of + passing it through unchanged; - user interfaces must distinguish provisional, adjudicated, calibrated, promoted-anchor, and linked states rather than displaying one generic “evaluated” badge. @@ -141,6 +154,9 @@ or synthetic lineage edge. 4. **Block all evaluation until anchors exist.** Rejected because governed pilot and diagnostic evidence is necessary to create and validate the first anchor corpus. +5. **Silently strip or normalize format controls.** Rejected because mutation + could collapse two foreign references into an identity that the owning system + never published. Admission fails closed instead. ## Verification @@ -148,7 +164,8 @@ Focused tests cover zero-anchor runs, adjudication/source-observation separation anchor-promotion requirements, linked-evidence requirements, immutable collection copying, strict mapping admission, reference and digest hygiene, blueprint consistency, duplicate and resource limits, public exports, and direct-construction -seals. The new projection module must retain complete statement and branch +seals. Reference hygiene includes zero-width and bidirectional Unicode format +controls. The new projection module must retain complete statement and branch coverage on the unchanged exact head. No fixed production item example, provider call, score, database migration, or @@ -168,3 +185,6 @@ Moreau, L., Missier, P., Belhajjame, K., B’Far, R., Cheney, J., Coppens, S., Cresswell, S., Gil, Y., Groth, P., Klyne, G., Lebo, T., McCusker, J., Miles, S., Myers, J., Sahoo, S., & Tilmes, C. (2013). PROV-DM: The PROV data model. World Wide Web Consortium. + +Unicode Consortium. (2025). *Unicode security mechanisms* (Unicode Technical +Standard #39, Version 17.0.0, Revision 32). https://www.unicode.org/reports/tr39/ From fd514e8262ee94b4e91d244256305e3ce0563da6 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Wed, 2 Sep 2026 16:53:30 +0900 Subject: [PATCH 16/47] docs(changelog): note Unicode reference hardening --- CHANGELOG.d/dynamic-evaluation-lineage.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.d/dynamic-evaluation-lineage.md b/CHANGELOG.d/dynamic-evaluation-lineage.md index 1bc4d153b..28c790d7d 100644 --- a/CHANGELOG.d/dynamic-evaluation-lineage.md +++ b/CHANGELOG.d/dynamic-evaluation-lineage.md @@ -3,4 +3,4 @@ - Added the versioned `lineageweave_dynamic_evaluation_lineage/v1` projection for dynamically resolved evaluation item and run snapshots. - Preserved generator, rater, adjudication-case/resolution, calibration, anchor-promotion, linking, and supersession references as separate immutable evidence instead of overwriting source observations or inventing decision authority. - Permitted zero-anchor cold-start and within-run projections while requiring separate calibration, promotion, and linking evidence before an item/run can be represented as an anchor or cross-version linked. -- Rejected provider credentials/endpoints, scores, embedded adjudication decisions, mixed-blueprint item sets, duplicate identities, and unsupported linking claims at the LineageWeave Anti-Corruption Layer. +- Rejected provider credentials/endpoints, scores, embedded adjudication decisions, mixed-blueprint item sets, duplicate identities, unsupported linking claims, and invisible Unicode format controls in opaque provenance references at the LineageWeave Anti-Corruption Layer. From e4b5de658da16cd3202c8a167de0d80dd2137775 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Wed, 2 Sep 2026 18:02:28 +0900 Subject: [PATCH 17/47] test(lineage): require substantive criteria before evaluation --- ...st_dynamic_evaluation_criterion_lineage.py | 183 ++++++++++++++++++ 1 file changed, 183 insertions(+) create mode 100644 tests/test_dynamic_evaluation_criterion_lineage.py diff --git a/tests/test_dynamic_evaluation_criterion_lineage.py b/tests/test_dynamic_evaluation_criterion_lineage.py new file mode 100644 index 000000000..4ae1fb533 --- /dev/null +++ b/tests/test_dynamic_evaluation_criterion_lineage.py @@ -0,0 +1,183 @@ +"""Criterion-first provenance contracts for dynamic evaluation lineage.""" + +from __future__ import annotations + +from typing import Any + +import pytest + +from lineageweave.evaluation_criteria import ( + EvaluationCriterionLineageError, + build_evaluation_criterion_set_lineage, +) +from lineageweave.evaluation_lineage import ( + DynamicEvaluationLineageError, + RunComparabilityStatus, + build_dynamic_evaluation_item_lineage, + build_dynamic_evaluation_run_lineage, +) + + +def _criterion(seed: str) -> dict[str, object]: + """Return one source-text-free but substantively identified criterion.""" + return { + "criterion_ref": seed, + "criterion_revision_ref": f"{seed}_revision_1", + "definition_ref": f"{seed}_definition", + "definition_sha256": "1" * 64, + "admissible_evidence_rule_ref": f"{seed}_evidence_rule", + "admissible_evidence_rule_sha256": "2" * 64, + "exclusion_rule_ref": f"{seed}_exclusion_rule", + "exclusion_rule_sha256": "3" * 64, + "response_semantics_ref": f"{seed}_response_semantics", + "response_semantics_sha256": "4" * 64, + "abstention_rule_ref": f"{seed}_abstention_rule", + "abstention_rule_sha256": "5" * 64, + "not_observable_rule_ref": f"{seed}_not_observable_rule", + "not_observable_rule_sha256": "6" * 64, + "category_definition_refs": ( + f"{seed}_not_supported_definition", + f"{seed}_supported_definition", + ), + "category_definition_sha256s": ("7" * 64, "8" * 64), + } + + +def _criterion_set(): + """Build one complete immutable criterion-set lineage snapshot.""" + return build_evaluation_criterion_set_lineage( + criterion_set_snapshot_ref="criterion_set_snapshot_1", + criterion_set_sha256="a" * 64, + blueprint_revision_ref="evaluation_blueprint_revision_1", + rubric_revision_ref="rubric_revision_1", + intended_use_ref="intended_use_1", + construct_ref="construct_1", + population_scope_ref="population_scope_1", + language_scope_ref="language_scope_1", + domain_scope_ref="domain_scope_1", + criteria=( + _criterion("criterion_evidence_support"), + _criterion("criterion_safety"), + ), + ) + + +def _item(**overrides: Any): + """Build one valid criterion-bound item-lineage projection.""" + payload: dict[str, Any] = { + "item_snapshot_ref": "item_snapshot_1", + "blueprint_revision_ref": "evaluation_blueprint_revision_1", + "criterion_set_snapshot_ref": "criterion_set_snapshot_1", + "criterion_set_sha256": "a" * 64, + "rubric_revision_ref": "rubric_revision_1", + "criterion_refs": ( + "criterion_evidence_support", + "criterion_safety", + ), + "source_contract_ref": "fast_mlsirm_dynamic_evaluation_item_v1", + "source_contract_sha256": "b" * 64, + "generation_invocation_ref": "generation_invocation_1", + "rater_invocation_refs": ("rater_invocation_1", "rater_invocation_2"), + "adjudication_case_ref": "adjudication_case_1", + "adjudication_resolution_ref": "adjudication_resolution_1", + "calibration_artifact_refs": (), + "anchor_promotion_decision_ref": None, + "supersedes_item_snapshot_ref": None, + } + payload.update(overrides) + return build_dynamic_evaluation_item_lineage(**payload) + + +def test_run_requires_complete_nonempty_criterion_meaning_before_items() -> None: + """A dynamic run cannot be represented from criterion identifiers alone.""" + criterion_set = _criterion_set() + run = build_dynamic_evaluation_run_lineage( + run_snapshot_ref="run_snapshot_1", + blueprint_revision_ref="evaluation_blueprint_revision_1", + criterion_set=criterion_set, + items=(_item(),), + anchor_item_snapshot_refs=(), + comparability_status=RunComparabilityStatus.WITHIN_RUN_ONLY, + ) + assert run.criterion_set.criterion_refs == ( + "criterion_evidence_support", + "criterion_safety", + ) + assert run.to_mapping()["criterion_set"]["criteria"][0][ + "admissible_evidence_rule_ref" + ] == "criterion_evidence_support_evidence_rule" + + +def test_criterion_set_rejects_missing_meaning_and_zero_criteria() -> None: + """Definitions, evidence rules, response semantics, and categories are mandatory.""" + with pytest.raises(EvaluationCriterionLineageError) as caught: + build_evaluation_criterion_set_lineage( + criterion_set_snapshot_ref="criterion_set_snapshot_1", + criterion_set_sha256="a" * 64, + blueprint_revision_ref="evaluation_blueprint_revision_1", + rubric_revision_ref="rubric_revision_1", + intended_use_ref="intended_use_1", + construct_ref="construct_1", + population_scope_ref="population_scope_1", + language_scope_ref="language_scope_1", + domain_scope_ref="domain_scope_1", + criteria=(), + ) + assert caught.value.code == "invalid_criterion_set" + + incomplete = _criterion("criterion_safety") + del incomplete["response_semantics_ref"] + with pytest.raises(EvaluationCriterionLineageError) as caught: + build_evaluation_criterion_set_lineage( + criterion_set_snapshot_ref="criterion_set_snapshot_1", + criterion_set_sha256="a" * 64, + blueprint_revision_ref="evaluation_blueprint_revision_1", + rubric_revision_ref="rubric_revision_1", + intended_use_ref="intended_use_1", + construct_ref="construct_1", + population_scope_ref="population_scope_1", + language_scope_ref="language_scope_1", + domain_scope_ref="domain_scope_1", + criteria=(incomplete,), + ) + assert caught.value.code == "missing_field" + + +def test_item_and_run_reject_criterion_set_or_rubric_substitution() -> None: + """Items, adjudication, and later artifacts stay on the administered criteria.""" + criterion_set = _criterion_set() + for changed_item in ( + _item(criterion_set_snapshot_ref="criterion_set_snapshot_2"), + _item(criterion_set_sha256="c" * 64), + _item(rubric_revision_ref="rubric_revision_2"), + _item(criterion_refs=("criterion_invented",)), + ): + with pytest.raises(DynamicEvaluationLineageError) as caught: + build_dynamic_evaluation_run_lineage( + run_snapshot_ref="run_snapshot_1", + blueprint_revision_ref="evaluation_blueprint_revision_1", + criterion_set=criterion_set, + items=(changed_item,), + anchor_item_snapshot_refs=(), + comparability_status=RunComparabilityStatus.UNAVAILABLE, + ) + assert caught.value.code in { + "item_criterion_set_mismatch", + "item_rubric_mismatch", + "unknown_item_criterion", + "criterion_coverage_mismatch", + } + + +def test_run_requires_all_bound_criteria_to_be_operationalized() -> None: + """A run cannot silently omit a declared evaluation criterion.""" + with pytest.raises(DynamicEvaluationLineageError) as caught: + build_dynamic_evaluation_run_lineage( + run_snapshot_ref="run_snapshot_1", + blueprint_revision_ref="evaluation_blueprint_revision_1", + criterion_set=_criterion_set(), + items=(_item(criterion_refs=("criterion_evidence_support",)),), + anchor_item_snapshot_refs=(), + comparability_status=RunComparabilityStatus.UNAVAILABLE, + ) + assert caught.value.code == "criterion_coverage_mismatch" From 8cfbd10795a70c54d7cabf736fe1a34678e17d5c Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Wed, 2 Sep 2026 18:04:36 +0900 Subject: [PATCH 18/47] feat(lineage): model substantive evaluation criteria --- lineageweave/evaluation_criteria.py | 444 ++++++++++++++++++++++++++++ 1 file changed, 444 insertions(+) create mode 100644 lineageweave/evaluation_criteria.py diff --git a/lineageweave/evaluation_criteria.py b/lineageweave/evaluation_criteria.py new file mode 100644 index 000000000..22f5b9a38 --- /dev/null +++ b/lineageweave/evaluation_criteria.py @@ -0,0 +1,444 @@ +"""Product-owned substantive criterion lineage for dynamic evaluations. + +LineageWeave owns the product meaning and evidence provenance of evaluation +criteria. This module retains source-text-free references and exact digests for +criterion definitions, evidence admission and exclusion rules, response and +missingness semantics, and every admissible response category. It does not call +providers, score observations, adjudicate cases, or calibrate items. +""" + +from __future__ import annotations + +from collections.abc import Mapping, Sequence +from dataclasses import InitVar, dataclass +from typing import Any + +MAX_CRITERION_REFERENCE_LENGTH = 256 +MAX_EVALUATION_CRITERIA = 128 +MAX_CRITERION_CATEGORIES = 64 +_CRITERION_TOKEN = object() +_SET_TOKEN = object() + +_CRITERION_FIELDS = frozenset( + { + "criterion_ref", + "criterion_revision_ref", + "definition_ref", + "definition_sha256", + "admissible_evidence_rule_ref", + "admissible_evidence_rule_sha256", + "exclusion_rule_ref", + "exclusion_rule_sha256", + "response_semantics_ref", + "response_semantics_sha256", + "abstention_rule_ref", + "abstention_rule_sha256", + "not_observable_rule_ref", + "not_observable_rule_sha256", + "category_refs", + "category_definition_refs", + "category_definition_sha256s", + } +) +_SET_FIELDS = frozenset( + { + "criterion_set_snapshot_ref", + "criterion_set_sha256", + "blueprint_revision_ref", + "rubric_revision_ref", + "intended_use_ref", + "construct_ref", + "population_scope_ref", + "language_scope_ref", + "domain_scope_ref", + "criteria", + } +) + + +class EvaluationCriterionLineageError(ValueError): + """Stable fail-closed error for substantive criterion lineage violations.""" + + def __init__(self, code: str, message: str) -> None: + """Retain a machine-readable rejection code without source content.""" + self.code = code + super().__init__(message) + + +def _mapping(value: Any, field_name: str) -> Mapping[str, Any]: + """Require one string-keyed mapping.""" + if not isinstance(value, Mapping): + raise EvaluationCriterionLineageError( + "invalid_object", f"{field_name} must be an object" + ) + if any(type(key) is not str for key in value): + raise EvaluationCriterionLineageError( + "invalid_object_key", f"{field_name} keys must be strings" + ) + return value + + +def _reject_unknown_fields( + payload: Mapping[str, Any], allowed: frozenset[str], field_name: str +) -> None: + """Reject fields outside the source-text-free criterion contract.""" + unknown = set(payload) - allowed + if unknown: + raise EvaluationCriterionLineageError( + "unknown_field", + f"{field_name} contains unsupported fields: {sorted(unknown)}", + ) + + +def _reference(value: Any, field_name: str) -> str: + """Validate one exact bounded opaque reference without normalization.""" + if type(value) is not str: + raise TypeError(f"{field_name} must be a string") + if ( + not value + or len(value) > MAX_CRITERION_REFERENCE_LENGTH + or value != value.strip() + or value.startswith("\ufeff") + or value.endswith("\ufeff") + or any( + ord(character) < 32 + or 127 <= ord(character) <= 159 + or 0xD800 <= ord(character) <= 0xDFFF + for character in value + ) + ): + raise EvaluationCriterionLineageError( + "invalid_reference", f"{field_name} must be an exact bounded reference" + ) + return value + + +def _sha256(value: Any, field_name: str) -> str: + """Validate one complete lowercase SHA-256 digest.""" + if type(value) is not str: + raise TypeError(f"{field_name} must be a string") + if len(value) != 64 or any(character not in "0123456789abcdef" for character in value): + raise EvaluationCriterionLineageError( + "invalid_sha256", + f"{field_name} must be 64 lowercase hexadecimal characters", + ) + return value + + +def _reference_tuple( + value: Any, + field_name: str, + *, + minimum: int, + maximum: int, +) -> tuple[str, ...]: + """Copy and validate a bounded unique ordered reference collection.""" + if not isinstance(value, (tuple, list)): + raise TypeError(f"{field_name} must be a tuple or list") + if not minimum <= len(value) <= maximum: + raise EvaluationCriterionLineageError( + "invalid_reference_count", + f"{field_name} must contain {minimum}..{maximum} references", + ) + normalized = tuple( + _reference(item, f"{field_name}[{index}]") + for index, item in enumerate(value) + ) + if len(set(normalized)) != len(normalized): + raise EvaluationCriterionLineageError( + "duplicate_reference", f"{field_name} must not contain duplicates" + ) + return normalized + + +def _digest_tuple( + value: Any, + field_name: str, + *, + minimum: int, + maximum: int, +) -> tuple[str, ...]: + """Copy and validate a bounded ordered digest collection.""" + if not isinstance(value, (tuple, list)): + raise TypeError(f"{field_name} must be a tuple or list") + if not minimum <= len(value) <= maximum: + raise EvaluationCriterionLineageError( + "invalid_digest_count", + f"{field_name} must contain {minimum}..{maximum} digests", + ) + return tuple( + _sha256(item, f"{field_name}[{index}]") + for index, item in enumerate(value) + ) + + +@dataclass(frozen=True, slots=True) +class EvaluationCriterionLineage: + """Immutable substantive meaning and category contract for one criterion.""" + + criterion_ref: str + criterion_revision_ref: str + definition_ref: str + definition_sha256: str + admissible_evidence_rule_ref: str + admissible_evidence_rule_sha256: str + exclusion_rule_ref: str + exclusion_rule_sha256: str + response_semantics_ref: str + response_semantics_sha256: str + abstention_rule_ref: str + abstention_rule_sha256: str + not_observable_rule_ref: str + not_observable_rule_sha256: str + category_refs: tuple[str, ...] + category_definition_refs: tuple[str, ...] + category_definition_sha256s: tuple[str, ...] + _admission_token: InitVar[object | None] = None + + def __post_init__(self, _admission_token: object | None) -> None: + """Prevent construction that bypasses the governed builder.""" + if _admission_token is not _CRITERION_TOKEN: + raise ValueError( + "EvaluationCriterionLineage must be created by " + "build_evaluation_criterion_lineage" + ) + + @classmethod + def from_mapping(cls, value: Any) -> "EvaluationCriterionLineage": + """Translate an untrusted source-text-free criterion mapping.""" + payload = _mapping(value, "criterion lineage") + _reject_unknown_fields(payload, _CRITERION_FIELDS, "criterion lineage") + missing = _CRITERION_FIELDS - set(payload) + if missing: + raise EvaluationCriterionLineageError( + "missing_field", + f"criterion lineage is missing fields: {sorted(missing)}", + ) + return build_evaluation_criterion_lineage(**payload) + + def to_mapping(self) -> dict[str, Any]: + """Return the source-text-free criterion lineage payload.""" + return { + "criterion_ref": self.criterion_ref, + "criterion_revision_ref": self.criterion_revision_ref, + "definition_ref": self.definition_ref, + "definition_sha256": self.definition_sha256, + "admissible_evidence_rule_ref": self.admissible_evidence_rule_ref, + "admissible_evidence_rule_sha256": self.admissible_evidence_rule_sha256, + "exclusion_rule_ref": self.exclusion_rule_ref, + "exclusion_rule_sha256": self.exclusion_rule_sha256, + "response_semantics_ref": self.response_semantics_ref, + "response_semantics_sha256": self.response_semantics_sha256, + "abstention_rule_ref": self.abstention_rule_ref, + "abstention_rule_sha256": self.abstention_rule_sha256, + "not_observable_rule_ref": self.not_observable_rule_ref, + "not_observable_rule_sha256": self.not_observable_rule_sha256, + "category_refs": list(self.category_refs), + "category_definition_refs": list(self.category_definition_refs), + "category_definition_sha256s": list(self.category_definition_sha256s), + } + + +def build_evaluation_criterion_lineage( + *, + criterion_ref: str, + criterion_revision_ref: str, + definition_ref: str, + definition_sha256: str, + admissible_evidence_rule_ref: str, + admissible_evidence_rule_sha256: str, + exclusion_rule_ref: str, + exclusion_rule_sha256: str, + response_semantics_ref: str, + response_semantics_sha256: str, + abstention_rule_ref: str, + abstention_rule_sha256: str, + not_observable_rule_ref: str, + not_observable_rule_sha256: str, + category_refs: Sequence[str], + category_definition_refs: Sequence[str], + category_definition_sha256s: Sequence[str], +) -> EvaluationCriterionLineage: + """Build one criterion whose evaluative meaning is complete and auditable.""" + normalized_category_refs = _reference_tuple( + category_refs, + "category_refs", + minimum=2, + maximum=MAX_CRITERION_CATEGORIES, + ) + normalized_definition_refs = _reference_tuple( + category_definition_refs, + "category_definition_refs", + minimum=2, + maximum=MAX_CRITERION_CATEGORIES, + ) + normalized_definition_digests = _digest_tuple( + category_definition_sha256s, + "category_definition_sha256s", + minimum=2, + maximum=MAX_CRITERION_CATEGORIES, + ) + lengths = { + len(normalized_category_refs), + len(normalized_definition_refs), + len(normalized_definition_digests), + } + if len(lengths) != 1: + raise EvaluationCriterionLineageError( + "category_definition_mismatch", + "category identities, definitions, and digests must have equal length", + ) + return EvaluationCriterionLineage( + criterion_ref=_reference(criterion_ref, "criterion_ref"), + criterion_revision_ref=_reference( + criterion_revision_ref, "criterion_revision_ref" + ), + definition_ref=_reference(definition_ref, "definition_ref"), + definition_sha256=_sha256(definition_sha256, "definition_sha256"), + admissible_evidence_rule_ref=_reference( + admissible_evidence_rule_ref, "admissible_evidence_rule_ref" + ), + admissible_evidence_rule_sha256=_sha256( + admissible_evidence_rule_sha256, + "admissible_evidence_rule_sha256", + ), + exclusion_rule_ref=_reference(exclusion_rule_ref, "exclusion_rule_ref"), + exclusion_rule_sha256=_sha256( + exclusion_rule_sha256, "exclusion_rule_sha256" + ), + response_semantics_ref=_reference( + response_semantics_ref, "response_semantics_ref" + ), + response_semantics_sha256=_sha256( + response_semantics_sha256, "response_semantics_sha256" + ), + abstention_rule_ref=_reference(abstention_rule_ref, "abstention_rule_ref"), + abstention_rule_sha256=_sha256( + abstention_rule_sha256, "abstention_rule_sha256" + ), + not_observable_rule_ref=_reference( + not_observable_rule_ref, "not_observable_rule_ref" + ), + not_observable_rule_sha256=_sha256( + not_observable_rule_sha256, "not_observable_rule_sha256" + ), + category_refs=normalized_category_refs, + category_definition_refs=normalized_definition_refs, + category_definition_sha256s=normalized_definition_digests, + _admission_token=_CRITERION_TOKEN, + ) + + +@dataclass(frozen=True, slots=True) +class EvaluationCriterionSetLineage: + """Immutable non-empty evaluation criterion set for one blueprint revision.""" + + criterion_set_snapshot_ref: str + criterion_set_sha256: str + blueprint_revision_ref: str + rubric_revision_ref: str + intended_use_ref: str + construct_ref: str + population_scope_ref: str + language_scope_ref: str + domain_scope_ref: str + criteria: tuple[EvaluationCriterionLineage, ...] + _admission_token: InitVar[object | None] = None + + def __post_init__(self, _admission_token: object | None) -> None: + """Prevent construction that bypasses criterion-set validation.""" + if _admission_token is not _SET_TOKEN: + raise ValueError( + "EvaluationCriterionSetLineage must be created by " + "build_evaluation_criterion_set_lineage" + ) + + @property + def criterion_refs(self) -> tuple[str, ...]: + """Return all governed criterion identities in snapshot order.""" + return tuple(criterion.criterion_ref for criterion in self.criteria) + + def to_mapping(self) -> dict[str, Any]: + """Return the source-text-free criterion-set lineage payload.""" + return { + "criterion_set_snapshot_ref": self.criterion_set_snapshot_ref, + "criterion_set_sha256": self.criterion_set_sha256, + "blueprint_revision_ref": self.blueprint_revision_ref, + "rubric_revision_ref": self.rubric_revision_ref, + "intended_use_ref": self.intended_use_ref, + "construct_ref": self.construct_ref, + "population_scope_ref": self.population_scope_ref, + "language_scope_ref": self.language_scope_ref, + "domain_scope_ref": self.domain_scope_ref, + "criteria": [criterion.to_mapping() for criterion in self.criteria], + } + + @classmethod + def from_mapping(cls, value: Any) -> "EvaluationCriterionSetLineage": + """Translate an untrusted criterion-set lineage mapping.""" + payload = _mapping(value, "criterion set lineage") + _reject_unknown_fields(payload, _SET_FIELDS, "criterion set lineage") + missing = _SET_FIELDS - set(payload) + if missing: + raise EvaluationCriterionLineageError( + "missing_field", + f"criterion set lineage is missing fields: {sorted(missing)}", + ) + return build_evaluation_criterion_set_lineage(**payload) + + +def build_evaluation_criterion_set_lineage( + *, + criterion_set_snapshot_ref: str, + criterion_set_sha256: str, + blueprint_revision_ref: str, + rubric_revision_ref: str, + intended_use_ref: str, + construct_ref: str, + population_scope_ref: str, + language_scope_ref: str, + domain_scope_ref: str, + criteria: Sequence[EvaluationCriterionLineage | Mapping[str, Any]], +) -> EvaluationCriterionSetLineage: + """Build a non-empty criterion set before any item or observation exists.""" + if not isinstance(criteria, (tuple, list)): + raise TypeError("criteria must be a tuple or list") + if not 1 <= len(criteria) <= MAX_EVALUATION_CRITERIA: + raise EvaluationCriterionLineageError( + "invalid_criterion_set", + f"criteria must contain 1..{MAX_EVALUATION_CRITERIA} definitions", + ) + normalized = tuple( + criterion + if type(criterion) is EvaluationCriterionLineage + else EvaluationCriterionLineage.from_mapping(criterion) + for criterion in criteria + ) + if any(type(criterion) is not EvaluationCriterionLineage for criterion in normalized): + raise TypeError("criteria must contain criterion lineage values or mappings") + refs = tuple(criterion.criterion_ref for criterion in normalized) + if len(set(refs)) != len(refs): + raise EvaluationCriterionLineageError( + "duplicate_criterion", "criterion set identities must be unique" + ) + return EvaluationCriterionSetLineage( + criterion_set_snapshot_ref=_reference( + criterion_set_snapshot_ref, "criterion_set_snapshot_ref" + ), + criterion_set_sha256=_sha256( + criterion_set_sha256, "criterion_set_sha256" + ), + blueprint_revision_ref=_reference( + blueprint_revision_ref, "blueprint_revision_ref" + ), + rubric_revision_ref=_reference(rubric_revision_ref, "rubric_revision_ref"), + intended_use_ref=_reference(intended_use_ref, "intended_use_ref"), + construct_ref=_reference(construct_ref, "construct_ref"), + population_scope_ref=_reference( + population_scope_ref, "population_scope_ref" + ), + language_scope_ref=_reference(language_scope_ref, "language_scope_ref"), + domain_scope_ref=_reference(domain_scope_ref, "domain_scope_ref"), + criteria=normalized, + _admission_token=_SET_TOKEN, + ) From 284d497dc86977751c71196a22d4622eaea9cf5b Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Wed, 2 Sep 2026 18:08:20 +0900 Subject: [PATCH 19/47] fix(lineage): keep criterion meaning source-text-free --- lineageweave/evaluation_criteria.py | 35 +++++++++++------------------ 1 file changed, 13 insertions(+), 22 deletions(-) diff --git a/lineageweave/evaluation_criteria.py b/lineageweave/evaluation_criteria.py index 22f5b9a38..65549e6e8 100644 --- a/lineageweave/evaluation_criteria.py +++ b/lineageweave/evaluation_criteria.py @@ -3,12 +3,14 @@ LineageWeave owns the product meaning and evidence provenance of evaluation criteria. This module retains source-text-free references and exact digests for criterion definitions, evidence admission and exclusion rules, response and -missingness semantics, and every admissible response category. It does not call -providers, score observations, adjudicate cases, or calibrate items. +missingness semantics, and every admissible response-category definition. It +does not call providers, score observations, adjudicate cases, or calibrate +items. """ from __future__ import annotations +import unicodedata from collections.abc import Mapping, Sequence from dataclasses import InitVar, dataclass from typing import Any @@ -35,7 +37,6 @@ "abstention_rule_sha256", "not_observable_rule_ref", "not_observable_rule_sha256", - "category_refs", "category_definition_refs", "category_definition_sha256s", } @@ -104,6 +105,7 @@ def _reference(value: Any, field_name: str) -> str: ord(character) < 32 or 127 <= ord(character) <= 159 or 0xD800 <= ord(character) <= 0xDFFF + or unicodedata.category(character) == "Cf" for character in value ) ): @@ -117,7 +119,9 @@ def _sha256(value: Any, field_name: str) -> str: """Validate one complete lowercase SHA-256 digest.""" if type(value) is not str: raise TypeError(f"{field_name} must be a string") - if len(value) != 64 or any(character not in "0123456789abcdef" for character in value): + if len(value) != 64 or any( + character not in "0123456789abcdef" for character in value + ): raise EvaluationCriterionLineageError( "invalid_sha256", f"{field_name} must be 64 lowercase hexadecimal characters", @@ -190,7 +194,6 @@ class EvaluationCriterionLineage: abstention_rule_sha256: str not_observable_rule_ref: str not_observable_rule_sha256: str - category_refs: tuple[str, ...] category_definition_refs: tuple[str, ...] category_definition_sha256s: tuple[str, ...] _admission_token: InitVar[object | None] = None @@ -233,7 +236,6 @@ def to_mapping(self) -> dict[str, Any]: "abstention_rule_sha256": self.abstention_rule_sha256, "not_observable_rule_ref": self.not_observable_rule_ref, "not_observable_rule_sha256": self.not_observable_rule_sha256, - "category_refs": list(self.category_refs), "category_definition_refs": list(self.category_definition_refs), "category_definition_sha256s": list(self.category_definition_sha256s), } @@ -255,17 +257,10 @@ def build_evaluation_criterion_lineage( abstention_rule_sha256: str, not_observable_rule_ref: str, not_observable_rule_sha256: str, - category_refs: Sequence[str], category_definition_refs: Sequence[str], category_definition_sha256s: Sequence[str], ) -> EvaluationCriterionLineage: """Build one criterion whose evaluative meaning is complete and auditable.""" - normalized_category_refs = _reference_tuple( - category_refs, - "category_refs", - minimum=2, - maximum=MAX_CRITERION_CATEGORIES, - ) normalized_definition_refs = _reference_tuple( category_definition_refs, "category_definition_refs", @@ -278,15 +273,10 @@ def build_evaluation_criterion_lineage( minimum=2, maximum=MAX_CRITERION_CATEGORIES, ) - lengths = { - len(normalized_category_refs), - len(normalized_definition_refs), - len(normalized_definition_digests), - } - if len(lengths) != 1: + if len(normalized_definition_refs) != len(normalized_definition_digests): raise EvaluationCriterionLineageError( "category_definition_mismatch", - "category identities, definitions, and digests must have equal length", + "category definitions and digests must have equal length", ) return EvaluationCriterionLineage( criterion_ref=_reference(criterion_ref, "criterion_ref"), @@ -322,7 +312,6 @@ def build_evaluation_criterion_lineage( not_observable_rule_sha256=_sha256( not_observable_rule_sha256, "not_observable_rule_sha256" ), - category_refs=normalized_category_refs, category_definition_refs=normalized_definition_refs, category_definition_sha256s=normalized_definition_digests, _admission_token=_CRITERION_TOKEN, @@ -414,7 +403,9 @@ def build_evaluation_criterion_set_lineage( else EvaluationCriterionLineage.from_mapping(criterion) for criterion in criteria ) - if any(type(criterion) is not EvaluationCriterionLineage for criterion in normalized): + if any( + type(criterion) is not EvaluationCriterionLineage for criterion in normalized + ): raise TypeError("criteria must contain criterion lineage values or mappings") refs = tuple(criterion.criterion_ref for criterion in normalized) if len(set(refs)) != len(refs): From b497b20b0b2a910ecc8c90cd76596ed43ac928d0 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Wed, 2 Sep 2026 18:10:34 +0900 Subject: [PATCH 20/47] fix(lineage): bind dynamic items to substantive criteria --- lineageweave/evaluation_lineage.py | 193 +++++++++++++++++++++++++++-- 1 file changed, 180 insertions(+), 13 deletions(-) diff --git a/lineageweave/evaluation_lineage.py b/lineageweave/evaluation_lineage.py index f9de59e87..410947c6b 100644 --- a/lineageweave/evaluation_lineage.py +++ b/lineageweave/evaluation_lineage.py @@ -1,8 +1,9 @@ """Dynamic-evaluation provenance projections owned by LineageWeave. -The module projects immutable item-generation, rater-observation, adjudication, -calibration, anchor-promotion, and supersession references without creating any -provider configuration, score, psychometric parameter, or adjudication decision. +The module projects immutable item-generation, criterion, rater-observation, +adjudication, calibration, anchor-promotion, and supersession references without +creating provider configuration, scores, psychometric parameters, or adjudication +decisions. """ from __future__ import annotations @@ -13,6 +14,8 @@ from enum import StrEnum from typing import Any +from .evaluation_criteria import EvaluationCriterionSetLineage + DYNAMIC_EVALUATION_LINEAGE_CONTRACT_ID = "lineageweave_dynamic_evaluation_lineage/v1" MAX_LINEAGE_REFERENCE_LENGTH = 256 MAX_LINEAGE_ITEMS = 10_000 @@ -35,7 +38,7 @@ "adjudication_decision", } ) -_ITEM_FIELDS = frozenset( +_ITEM_REQUIRED_FIELDS = frozenset( { "item_snapshot_ref", "blueprint_revision_ref", @@ -50,7 +53,16 @@ "supersedes_item_snapshot_ref", } ) -_RUN_FIELDS = frozenset( +_ITEM_CRITERION_FIELDS = frozenset( + { + "criterion_set_snapshot_ref", + "criterion_set_sha256", + "rubric_revision_ref", + "criterion_refs", + } +) +_ITEM_FIELDS = _ITEM_REQUIRED_FIELDS | _ITEM_CRITERION_FIELDS +_RUN_REQUIRED_FIELDS = frozenset( { "contract_id", "run_snapshot_ref", @@ -61,6 +73,7 @@ "linking_evidence_ref", } ) +_RUN_FIELDS = _RUN_REQUIRED_FIELDS | {"criterion_set"} class RunComparabilityStatus(StrEnum): @@ -81,6 +94,7 @@ def __init__(self, code: str, message: str) -> None: def _mapping(value: Any, field_name: str) -> Mapping[str, Any]: + """Require a string-keyed mapping at an untrusted projection boundary.""" if not isinstance(value, Mapping): raise DynamicEvaluationLineageError( "invalid_object", f"{field_name} must be an object" @@ -95,6 +109,7 @@ def _mapping(value: Any, field_name: str) -> Mapping[str, Any]: def _reject_unknown_fields( payload: Mapping[str, Any], allowed: frozenset[str], field_name: str ) -> None: + """Reject foreign authority and unsupported projection fields.""" unknown = set(payload) - allowed if unknown.intersection(_PROHIBITED_AUTHORITY_FIELDS): raise DynamicEvaluationLineageError( @@ -109,6 +124,7 @@ def _reject_unknown_fields( def _reference(value: Any, field_name: str) -> str: + """Validate one exact bounded opaque reference without normalization.""" if type(value) is not str: raise TypeError(f"{field_name} must be a string") if ( @@ -126,12 +142,14 @@ def _reference(value: Any, field_name: str) -> str: ) ): raise DynamicEvaluationLineageError( - "invalid_reference", f"{field_name} must be an exact bounded opaque reference" + "invalid_reference", + f"{field_name} must be an exact bounded opaque reference", ) return value def _optional_reference(value: Any, field_name: str) -> str | None: + """Validate an optional opaque reference.""" if value is None: return None return _reference(value, field_name) @@ -143,6 +161,7 @@ def _reference_tuple( *, allow_empty: bool, ) -> tuple[str, ...]: + """Copy and validate a bounded ordered set of opaque references.""" if not isinstance(value, (tuple, list)): raise TypeError(f"{field_name} must be a tuple or list") if (not allow_empty and not value) or len(value) > MAX_LINEAGE_REFERENCES: @@ -163,9 +182,12 @@ def _reference_tuple( def _sha256(value: Any, field_name: str) -> str: + """Validate one complete lowercase SHA-256 digest.""" if type(value) is not str: raise TypeError(f"{field_name} must be a string") - if len(value) != 64 or any(character not in "0123456789abcdef" for character in value): + if len(value) != 64 or any( + character not in "0123456789abcdef" for character in value + ): raise DynamicEvaluationLineageError( "invalid_sha256", f"{field_name} must be a complete lowercase SHA-256 digest", @@ -174,6 +196,7 @@ def _sha256(value: Any, field_name: str) -> str: def _comparability_status(value: Any) -> RunComparabilityStatus: + """Translate an exact enum/string value to the governed comparability state.""" if type(value) is RunComparabilityStatus: return value if type(value) is not str: @@ -203,6 +226,10 @@ class DynamicEvaluationItemLineage: calibration_artifact_refs: tuple[str, ...] anchor_promotion_decision_ref: str | None supersedes_item_snapshot_ref: str | None + criterion_set_snapshot_ref: str | None = None + criterion_set_sha256: str | None = None + rubric_revision_ref: str | None = None + criterion_refs: tuple[str, ...] = () _admission_token: InitVar[object | None] = None def __post_init__(self, _admission_token: object | None) -> None: @@ -213,9 +240,14 @@ def __post_init__(self, _admission_token: object | None) -> None: "build_dynamic_evaluation_item_lineage" ) + @property + def criterion_bound(self) -> bool: + """Return whether the item is bound to one substantive criterion snapshot.""" + return self.criterion_set_snapshot_ref is not None + def to_mapping(self) -> dict[str, Any]: """Return the source-text-free projection payload.""" - return { + payload: dict[str, Any] = { "item_snapshot_ref": self.item_snapshot_ref, "blueprint_revision_ref": self.blueprint_revision_ref, "source_contract_ref": self.source_contract_ref, @@ -228,17 +260,33 @@ def to_mapping(self) -> dict[str, Any]: "anchor_promotion_decision_ref": self.anchor_promotion_decision_ref, "supersedes_item_snapshot_ref": self.supersedes_item_snapshot_ref, } + if self.criterion_bound: + payload.update( + { + "criterion_set_snapshot_ref": self.criterion_set_snapshot_ref, + "criterion_set_sha256": self.criterion_set_sha256, + "rubric_revision_ref": self.rubric_revision_ref, + "criterion_refs": list(self.criterion_refs), + } + ) + return payload @classmethod def from_mapping(cls, value: Any) -> "DynamicEvaluationItemLineage": """Translate an untrusted item-lineage projection through the ACL.""" payload = _mapping(value, "item lineage") _reject_unknown_fields(payload, _ITEM_FIELDS, "item lineage") - missing = _ITEM_FIELDS - set(payload) + missing = _ITEM_REQUIRED_FIELDS - set(payload) if missing: raise DynamicEvaluationLineageError( "missing_field", f"item lineage is missing fields: {sorted(missing)}" ) + present_criterion_fields = _ITEM_CRITERION_FIELDS.intersection(payload) + if present_criterion_fields and present_criterion_fields != _ITEM_CRITERION_FIELDS: + raise DynamicEvaluationLineageError( + "incomplete_criterion_binding", + "criterion-bound item lineage must carry the complete criterion binding", + ) return build_dynamic_evaluation_item_lineage( item_snapshot_ref=payload["item_snapshot_ref"], blueprint_revision_ref=payload["blueprint_revision_ref"], @@ -251,6 +299,10 @@ def from_mapping(cls, value: Any) -> "DynamicEvaluationItemLineage": calibration_artifact_refs=payload["calibration_artifact_refs"], anchor_promotion_decision_ref=payload["anchor_promotion_decision_ref"], supersedes_item_snapshot_ref=payload["supersedes_item_snapshot_ref"], + criterion_set_snapshot_ref=payload.get("criterion_set_snapshot_ref"), + criterion_set_sha256=payload.get("criterion_set_sha256"), + rubric_revision_ref=payload.get("rubric_revision_ref"), + criterion_refs=payload.get("criterion_refs", ()), ) @@ -267,6 +319,10 @@ def build_dynamic_evaluation_item_lineage( calibration_artifact_refs: tuple[str, ...] | list[str], anchor_promotion_decision_ref: str | None, supersedes_item_snapshot_ref: str | None, + criterion_set_snapshot_ref: str | None = None, + criterion_set_sha256: str | None = None, + rubric_revision_ref: str | None = None, + criterion_refs: tuple[str, ...] | list[str] = (), ) -> DynamicEvaluationItemLineage: """Build one lineage projection without transferring foreign authority.""" normalized_item_ref = _reference(item_snapshot_ref, "item_snapshot_ref") @@ -281,7 +337,10 @@ def build_dynamic_evaluation_item_lineage( "resolution_requires_case", "an adjudication resolution must reference its separate case", ) - if normalized_resolution_ref is not None and normalized_resolution_ref == normalized_case_ref: + if ( + normalized_resolution_ref is not None + and normalized_resolution_ref == normalized_case_ref + ): raise DynamicEvaluationLineageError( "adjudication_reference_collision", "adjudication case and resolution must retain distinct identities", @@ -295,6 +354,39 @@ def build_dynamic_evaluation_item_lineage( "self_supersession", "an item snapshot cannot supersede itself" ) + criterion_binding_values = ( + criterion_set_snapshot_ref, + criterion_set_sha256, + rubric_revision_ref, + ) + has_any_binding = any(value is not None for value in criterion_binding_values) or bool( + criterion_refs + ) + has_complete_binding = all(value is not None for value in criterion_binding_values) and bool( + criterion_refs + ) + if has_any_binding and not has_complete_binding: + raise DynamicEvaluationLineageError( + "incomplete_criterion_binding", + "criterion-bound items require set identity, digest, rubric, and criteria", + ) + + normalized_criterion_set_ref: str | None = None + normalized_criterion_set_sha256: str | None = None + normalized_rubric_ref: str | None = None + normalized_criterion_refs: tuple[str, ...] = () + if has_complete_binding: + normalized_criterion_set_ref = _reference( + criterion_set_snapshot_ref, "criterion_set_snapshot_ref" + ) + normalized_criterion_set_sha256 = _sha256( + criterion_set_sha256, "criterion_set_sha256" + ) + normalized_rubric_ref = _reference(rubric_revision_ref, "rubric_revision_ref") + normalized_criterion_refs = _reference_tuple( + criterion_refs, "criterion_refs", allow_empty=False + ) + return DynamicEvaluationItemLineage( item_snapshot_ref=normalized_item_ref, blueprint_revision_ref=_reference( @@ -321,6 +413,10 @@ def build_dynamic_evaluation_item_lineage( anchor_promotion_decision_ref, "anchor_promotion_decision_ref" ), supersedes_item_snapshot_ref=normalized_supersedes_ref, + criterion_set_snapshot_ref=normalized_criterion_set_ref, + criterion_set_sha256=normalized_criterion_set_sha256, + rubric_revision_ref=normalized_rubric_ref, + criterion_refs=normalized_criterion_refs, _admission_token=_ITEM_TOKEN, ) @@ -335,6 +431,7 @@ class DynamicEvaluationRunLineage: anchor_item_snapshot_refs: tuple[str, ...] comparability_status: RunComparabilityStatus linking_evidence_ref: str | None + criterion_set: EvaluationCriterionSetLineage | None = None contract_id: str = DYNAMIC_EVALUATION_LINEAGE_CONTRACT_ID _admission_token: InitVar[object | None] = None @@ -348,7 +445,7 @@ def __post_init__(self, _admission_token: object | None) -> None: def to_mapping(self) -> dict[str, Any]: """Return the versioned source-text-free run projection.""" - return { + payload: dict[str, Any] = { "contract_id": self.contract_id, "run_snapshot_ref": self.run_snapshot_ref, "blueprint_revision_ref": self.blueprint_revision_ref, @@ -357,13 +454,16 @@ def to_mapping(self) -> dict[str, Any]: "comparability_status": self.comparability_status.value, "linking_evidence_ref": self.linking_evidence_ref, } + if self.criterion_set is not None: + payload["criterion_set"] = self.criterion_set.to_mapping() + return payload @classmethod def from_mapping(cls, value: Any) -> "DynamicEvaluationRunLineage": """Translate an untrusted run-lineage projection through the ACL.""" payload = _mapping(value, "run lineage") _reject_unknown_fields(payload, _RUN_FIELDS, "run lineage") - missing = _RUN_FIELDS - set(payload) + missing = _RUN_REQUIRED_FIELDS - set(payload) if missing: raise DynamicEvaluationLineageError( "missing_field", f"run lineage is missing fields: {sorted(missing)}" @@ -380,9 +480,16 @@ def from_mapping(cls, value: Any) -> "DynamicEvaluationRunLineage": "item_set_budget_exceeded", f"run lineage may contain at most {MAX_LINEAGE_ITEMS} items", ) + raw_criterion_set = payload.get("criterion_set") + criterion_set = ( + None + if raw_criterion_set is None + else EvaluationCriterionSetLineage.from_mapping(raw_criterion_set) + ) return build_dynamic_evaluation_run_lineage( run_snapshot_ref=payload["run_snapshot_ref"], blueprint_revision_ref=payload["blueprint_revision_ref"], + criterion_set=criterion_set, items=tuple( DynamicEvaluationItemLineage.from_mapping(item) for item in raw_items ), @@ -392,6 +499,58 @@ def from_mapping(cls, value: Any) -> "DynamicEvaluationRunLineage": ) +def _validate_criterion_binding( + criterion_set: EvaluationCriterionSetLineage | None, + items: tuple[DynamicEvaluationItemLineage, ...], + blueprint_revision_ref: str, +) -> None: + """Keep administered criteria, rubric, and item bindings on one immutable snapshot.""" + if criterion_set is None: + if any(item.criterion_bound for item in items): + raise DynamicEvaluationLineageError( + "criterion_set_required", + "criterion-bound items require their administered criterion set", + ) + return + if type(criterion_set) is not EvaluationCriterionSetLineage: + raise TypeError("criterion_set must be an exact EvaluationCriterionSetLineage") + if criterion_set.blueprint_revision_ref != blueprint_revision_ref: + raise DynamicEvaluationLineageError( + "criterion_blueprint_mismatch", + "criterion set must use the run blueprint revision", + ) + + governed_refs = set(criterion_set.criterion_refs) + covered_refs: set[str] = set() + for item in items: + if ( + item.criterion_set_snapshot_ref != criterion_set.criterion_set_snapshot_ref + or item.criterion_set_sha256 != criterion_set.criterion_set_sha256 + ): + raise DynamicEvaluationLineageError( + "item_criterion_set_mismatch", + "every item must retain the administered criterion-set snapshot and digest", + ) + if item.rubric_revision_ref != criterion_set.rubric_revision_ref: + raise DynamicEvaluationLineageError( + "item_rubric_mismatch", + "every item must retain the administered rubric revision", + ) + unknown_refs = set(item.criterion_refs) - governed_refs + if unknown_refs: + raise DynamicEvaluationLineageError( + "unknown_item_criterion", + "item lineage references a criterion outside the administered set", + ) + covered_refs.update(item.criterion_refs) + + if covered_refs != governed_refs: + raise DynamicEvaluationLineageError( + "criterion_coverage_mismatch", + "run items must operationalize every criterion in the administered set", + ) + + def build_dynamic_evaluation_run_lineage( *, run_snapshot_ref: str, @@ -400,6 +559,7 @@ def build_dynamic_evaluation_run_lineage( anchor_item_snapshot_refs: tuple[str, ...] | list[str], comparability_status: RunComparabilityStatus | str, linking_evidence_ref: str | None = None, + criterion_set: EvaluationCriterionSetLineage | None = None, ) -> DynamicEvaluationRunLineage: """Build a run projection that may explicitly contain zero fixed anchors.""" if not isinstance(items, (tuple, list)) or not items: @@ -412,7 +572,9 @@ def build_dynamic_evaluation_run_lineage( f"run lineage may contain at most {MAX_LINEAGE_ITEMS} items", ) normalized_items = tuple(items) - if any(type(item) is not DynamicEvaluationItemLineage for item in normalized_items): + if any( + type(item) is not DynamicEvaluationItemLineage for item in normalized_items + ): raise TypeError("items must contain exact DynamicEvaluationItemLineage values") normalized_blueprint_ref = _reference( @@ -426,6 +588,10 @@ def build_dynamic_evaluation_run_lineage( "item_blueprint_mismatch", "every item projection must use the run blueprint revision", ) + _validate_criterion_binding( + criterion_set, normalized_items, normalized_blueprint_ref + ) + item_refs = tuple(item.item_snapshot_ref for item in normalized_items) if len(set(item_refs)) != len(item_refs): raise DynamicEvaluationLineageError( @@ -482,5 +648,6 @@ def build_dynamic_evaluation_run_lineage( anchor_item_snapshot_refs=normalized_anchor_refs, comparability_status=normalized_status, linking_evidence_ref=normalized_linking_ref, + criterion_set=criterion_set, _admission_token=_RUN_TOKEN, ) From c17ffce8eaf32ef9a6aa8b377af00c9b58b3953f Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Thu, 3 Sep 2026 00:59:20 +0900 Subject: [PATCH 21/47] test(lineage): reject supersession cycles --- ...t_dynamic_evaluation_lineage_boundaries.py | 21 +++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/tests/test_dynamic_evaluation_lineage_boundaries.py b/tests/test_dynamic_evaluation_lineage_boundaries.py index 01958060d..12c81a819 100644 --- a/tests/test_dynamic_evaluation_lineage_boundaries.py +++ b/tests/test_dynamic_evaluation_lineage_boundaries.py @@ -232,3 +232,24 @@ def test_linking_evidence_is_admitted_only_with_promoted_anchors() -> None: linking_evidence_ref="linking_evidence_1", ) assert caught.value.code == "unexpected_linking_evidence" + + +def test_run_rejects_cycles_in_item_supersession_lineage() -> None: + first = _item( + item_snapshot_ref="evaluation_item_snapshot_alpha", + supersedes_item_snapshot_ref="evaluation_item_snapshot_beta", + ) + second = _item( + item_snapshot_ref="evaluation_item_snapshot_beta", + supersedes_item_snapshot_ref="evaluation_item_snapshot_alpha", + ) + + with pytest.raises(DynamicEvaluationLineageError) as caught: + build_dynamic_evaluation_run_lineage( + run_snapshot_ref="evaluation_run_snapshot_supersession_cycle", + blueprint_revision_ref="evaluation_blueprint_revision_1", + items=(first, second), + anchor_item_snapshot_refs=(), + comparability_status=RunComparabilityStatus.UNAVAILABLE, + ) + assert caught.value.code == "supersession_cycle" From 96ade925605a77c13fda059fe9b2d028ca257e3f Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Thu, 3 Sep 2026 01:07:48 +0900 Subject: [PATCH 22/47] fix(lineage): reject supersession cycles --- lineageweave/evaluation_lineage.py | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/lineageweave/evaluation_lineage.py b/lineageweave/evaluation_lineage.py index 410947c6b..4d9adf402 100644 --- a/lineageweave/evaluation_lineage.py +++ b/lineageweave/evaluation_lineage.py @@ -551,6 +551,24 @@ def _validate_criterion_binding( ) +def _validate_supersession_graph( + items: tuple[DynamicEvaluationItemLineage, ...], +) -> None: + """Reject cycles among supersession edges whose endpoints are in this run.""" + item_by_ref = {item.item_snapshot_ref: item for item in items} + for start_ref in item_by_ref: + seen: set[str] = set() + current_ref: str | None = start_ref + while current_ref in item_by_ref: + if current_ref in seen: + raise DynamicEvaluationLineageError( + "supersession_cycle", + "item supersession lineage must be acyclic within a run", + ) + seen.add(current_ref) + current_ref = item_by_ref[current_ref].supersedes_item_snapshot_ref + + def build_dynamic_evaluation_run_lineage( *, run_snapshot_ref: str, @@ -597,6 +615,7 @@ def build_dynamic_evaluation_run_lineage( raise DynamicEvaluationLineageError( "duplicate_item_snapshot", "run lineage item snapshots must be unique" ) + _validate_supersession_graph(normalized_items) normalized_anchor_refs = _reference_tuple( anchor_item_snapshot_refs, From c51cd564177389d3f7deca81039ff517fd480b7c Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Thu, 3 Sep 2026 01:11:17 +0900 Subject: [PATCH 23/47] test(lineage): bound supersession admission cost --- ...mic_evaluation_lineage_admission_budget.py | 45 +++++++++++++++++++ 1 file changed, 45 insertions(+) diff --git a/tests/test_dynamic_evaluation_lineage_admission_budget.py b/tests/test_dynamic_evaluation_lineage_admission_budget.py index 6736a1ee4..d617e24ed 100644 --- a/tests/test_dynamic_evaluation_lineage_admission_budget.py +++ b/tests/test_dynamic_evaluation_lineage_admission_budget.py @@ -2,6 +2,8 @@ from __future__ import annotations +from time import perf_counter + import pytest from lineageweave.evaluation_lineage import ( @@ -9,8 +11,13 @@ MAX_LINEAGE_ITEMS, DynamicEvaluationLineageError, DynamicEvaluationRunLineage, + RunComparabilityStatus, + build_dynamic_evaluation_item_lineage, + build_dynamic_evaluation_run_lineage, ) +_DIGEST = "a" * 64 + def test_run_mapping_rejects_oversized_item_array_before_item_decoding() -> None: """Reject hostile item counts before spending work on individual item payloads.""" @@ -28,3 +35,41 @@ def test_run_mapping_rejects_oversized_item_array_before_item_decoding() -> None DynamicEvaluationRunLineage.from_mapping(payload) assert caught.value.code == "item_set_budget_exceeded" + + +def test_long_acyclic_supersession_chain_has_bounded_admission_cost() -> None: + """Keep supersession validation linear enough for the allowed item budget.""" + item_count = 5_000 + items = tuple( + build_dynamic_evaluation_item_lineage( + item_snapshot_ref=f"evaluation_item_snapshot_{index}", + blueprint_revision_ref="evaluation_blueprint_revision_1", + source_contract_ref="fast_mlsirm_dynamic_evaluation_item/v1", + source_contract_sha256=_DIGEST, + generation_invocation_ref=None, + rater_invocation_refs=(), + adjudication_case_ref=None, + adjudication_resolution_ref=None, + calibration_artifact_refs=(), + anchor_promotion_decision_ref=None, + supersedes_item_snapshot_ref=( + "external_prior_snapshot" + if index == 0 + else f"evaluation_item_snapshot_{index - 1}" + ), + ) + for index in range(item_count) + ) + + started = perf_counter() + run = build_dynamic_evaluation_run_lineage( + run_snapshot_ref="evaluation_run_snapshot_long_chain", + blueprint_revision_ref="evaluation_blueprint_revision_1", + items=items, + anchor_item_snapshot_refs=(), + comparability_status=RunComparabilityStatus.UNAVAILABLE, + ) + elapsed = perf_counter() - started + + assert len(run.items) == item_count + assert elapsed < 0.5, f"supersession admission took {elapsed:.3f}s" From fa6711d4cb61662ded719a4bd7240c9dc1ef3218 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Thu, 3 Sep 2026 01:23:49 +0900 Subject: [PATCH 24/47] fix(lineage): bound supersession graph traversal --- lineageweave/evaluation_lineage.py | 22 ++++++++++++++-------- 1 file changed, 14 insertions(+), 8 deletions(-) diff --git a/lineageweave/evaluation_lineage.py b/lineageweave/evaluation_lineage.py index 4d9adf402..18f97946b 100644 --- a/lineageweave/evaluation_lineage.py +++ b/lineageweave/evaluation_lineage.py @@ -554,19 +554,25 @@ def _validate_criterion_binding( def _validate_supersession_graph( items: tuple[DynamicEvaluationItemLineage, ...], ) -> None: - """Reject cycles among supersession edges whose endpoints are in this run.""" - item_by_ref = {item.item_snapshot_ref: item for item in items} - for start_ref in item_by_ref: - seen: set[str] = set() + """Reject in-run supersession cycles with one bounded traversal per item.""" + predecessor_by_ref = { + item.item_snapshot_ref: item.supersedes_item_snapshot_ref for item in items + } + finished: set[str] = set() + for start_ref in predecessor_by_ref: + if start_ref in finished: + continue + path: set[str] = set() current_ref: str | None = start_ref - while current_ref in item_by_ref: - if current_ref in seen: + while current_ref in predecessor_by_ref and current_ref not in finished: + if current_ref in path: raise DynamicEvaluationLineageError( "supersession_cycle", "item supersession lineage must be acyclic within a run", ) - seen.add(current_ref) - current_ref = item_by_ref[current_ref].supersedes_item_snapshot_ref + path.add(current_ref) + current_ref = predecessor_by_ref[current_ref] + finished.update(path) def build_dynamic_evaluation_run_lineage( From 1b7820e64d7cfa5979f26246f0ccc21ef38ef82d Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Thu, 3 Sep 2026 01:24:29 +0900 Subject: [PATCH 25/47] test(lineage): make supersession cost regression deterministic --- ...mic_evaluation_lineage_admission_budget.py | 42 +++++++++++-------- 1 file changed, 24 insertions(+), 18 deletions(-) diff --git a/tests/test_dynamic_evaluation_lineage_admission_budget.py b/tests/test_dynamic_evaluation_lineage_admission_budget.py index d617e24ed..fd7e47ffd 100644 --- a/tests/test_dynamic_evaluation_lineage_admission_budget.py +++ b/tests/test_dynamic_evaluation_lineage_admission_budget.py @@ -2,18 +2,15 @@ from __future__ import annotations -from time import perf_counter - import pytest +import lineageweave.evaluation_lineage as evaluation_lineage from lineageweave.evaluation_lineage import ( DYNAMIC_EVALUATION_LINEAGE_CONTRACT_ID, MAX_LINEAGE_ITEMS, DynamicEvaluationLineageError, DynamicEvaluationRunLineage, - RunComparabilityStatus, build_dynamic_evaluation_item_lineage, - build_dynamic_evaluation_run_lineage, ) _DIGEST = "a" * 64 @@ -37,9 +34,9 @@ def test_run_mapping_rejects_oversized_item_array_before_item_decoding() -> None assert caught.value.code == "item_set_budget_exceeded" -def test_long_acyclic_supersession_chain_has_bounded_admission_cost() -> None: - """Keep supersession validation linear enough for the allowed item budget.""" - item_count = 5_000 +def test_long_acyclic_supersession_chain_uses_linear_set_work(monkeypatch) -> None: + """Bound graph-walk work without relying on runner-specific wall-clock speed.""" + item_count = 512 items = tuple( build_dynamic_evaluation_item_lineage( item_snapshot_ref=f"evaluation_item_snapshot_{index}", @@ -61,15 +58,24 @@ def test_long_acyclic_supersession_chain_has_bounded_admission_cost() -> None: for index in range(item_count) ) - started = perf_counter() - run = build_dynamic_evaluation_run_lineage( - run_snapshot_ref="evaluation_run_snapshot_long_chain", - blueprint_revision_ref="evaluation_blueprint_revision_1", - items=items, - anchor_item_snapshot_refs=(), - comparability_status=RunComparabilityStatus.UNAVAILABLE, - ) - elapsed = perf_counter() - started + class CountingSet(set): + operations = 0 + + def __contains__(self, value: object) -> bool: + type(self).operations += 1 + return super().__contains__(value) + + def add(self, value: object) -> None: + type(self).operations += 1 + super().add(value) + + def update(self, *others: object) -> None: + for other in others: + values = tuple(other) + type(self).operations += len(values) + super().update(values) + + monkeypatch.setattr(evaluation_lineage, "set", CountingSet, raising=False) + evaluation_lineage._validate_supersession_graph(items) - assert len(run.items) == item_count - assert elapsed < 0.5, f"supersession admission took {elapsed:.3f}s" + assert CountingSet.operations <= item_count * 6 From ad354c3fc06ebab630b911ec4fb3a11dd154454d Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Thu, 3 Sep 2026 01:24:49 +0900 Subject: [PATCH 26/47] test(lineage): count supersession walk operations --- tests/test_dynamic_evaluation_lineage_admission_budget.py | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/tests/test_dynamic_evaluation_lineage_admission_budget.py b/tests/test_dynamic_evaluation_lineage_admission_budget.py index fd7e47ffd..8545aac12 100644 --- a/tests/test_dynamic_evaluation_lineage_admission_budget.py +++ b/tests/test_dynamic_evaluation_lineage_admission_budget.py @@ -69,13 +69,7 @@ def add(self, value: object) -> None: type(self).operations += 1 super().add(value) - def update(self, *others: object) -> None: - for other in others: - values = tuple(other) - type(self).operations += len(values) - super().update(values) - monkeypatch.setattr(evaluation_lineage, "set", CountingSet, raising=False) evaluation_lineage._validate_supersession_graph(items) - assert CountingSet.operations <= item_count * 6 + assert CountingSet.operations <= item_count * 5 From b9d092bf612f38f2e53fbca1378a40c6611d37c2 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Thu, 3 Sep 2026 01:28:33 +0900 Subject: [PATCH 27/47] docs(lineage): record bounded supersession admission --- CHANGELOG.d/dynamic-evaluation-lineage.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.d/dynamic-evaluation-lineage.md b/CHANGELOG.d/dynamic-evaluation-lineage.md index 28c790d7d..4caad8466 100644 --- a/CHANGELOG.d/dynamic-evaluation-lineage.md +++ b/CHANGELOG.d/dynamic-evaluation-lineage.md @@ -4,3 +4,4 @@ - Preserved generator, rater, adjudication-case/resolution, calibration, anchor-promotion, linking, and supersession references as separate immutable evidence instead of overwriting source observations or inventing decision authority. - Permitted zero-anchor cold-start and within-run projections while requiring separate calibration, promotion, and linking evidence before an item/run can be represented as an anchor or cross-version linked. - Rejected provider credentials/endpoints, scores, embedded adjudication decisions, mixed-blueprint item sets, duplicate identities, unsupported linking claims, and invisible Unicode format controls in opaque provenance references at the LineageWeave Anti-Corruption Layer. +- Rejected directed cycles among in-run supersession references and bounded cycle admission to linear graph work at the 10,000-item contract limit without claiming a runner-specific wall-clock SLO. From f68441dad6e119d1d056f4fd80bc5c07510bde34 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Thu, 3 Sep 2026 01:32:04 +0900 Subject: [PATCH 28/47] docs(adr): move dynamic evaluation lineage to ADR 0355 --- docs/adr/0355-dynamic-evaluation-lineage.md | 214 ++++++++++++++++++++ 1 file changed, 214 insertions(+) create mode 100644 docs/adr/0355-dynamic-evaluation-lineage.md diff --git a/docs/adr/0355-dynamic-evaluation-lineage.md b/docs/adr/0355-dynamic-evaluation-lineage.md new file mode 100644 index 000000000..161e877fc --- /dev/null +++ b/docs/adr/0355-dynamic-evaluation-lineage.md @@ -0,0 +1,214 @@ +# ADR 0355: Project dynamic evaluation snapshots without absorbing decision authority + +- Status: Proposed +- Date: 2026-09-02 +- Depends on: ADR 0300 (contextual-orchestrator ownership boundary), ADR 0301 (dichotomous measurement policy) + +## Context + +A product evaluation may resolve its concrete items dynamically from an authored +blueprint, a production sample, a controlled perturbation, or a model/algorithmic +generator. A fixed item set or validated anchor corpus may not exist during cold +start. Nevertheless, an evaluation must remain reproducible enough to determine +which exact item snapshots, generator invocations, rater observations, +adjudication artifacts, calibration evidence, and promotion decisions informed a +result. + +A single mutable `evaluation_item` record would collapse independent facts and +permit later review to rewrite history. It could also allow LineageWeave to absorb +provider credentials, model routing, psychometric calculation, hosted +adjudication, or source-system authority that belongs to other bounded contexts. + +Opaque provenance references also cross service and rendering boundaries. Unicode +format controls can be machine-distinct while remaining visually absent or +changing bidirectional presentation, creating an avoidable alias/spoofing surface +for identifiers used in equality and provenance joins. Unicode Technical Standard +#39 treats identifier ambiguity and default-ignorable characters as security +concerns. LineageWeave therefore rejects Unicode `Cf` format controls in these +opaque references rather than normalizing them into a guessed identity. This is a +product-specific restrictive profile, not a claim of full UTS #39 conformance. + +The run contract admits as many as 10,000 item snapshots. A supersession-cycle +check that restarts a full predecessor walk from every item can therefore turn an +otherwise linear lineage admission into quadratic work. Admission cost is part of +the bounded-context integrity boundary: hostile or merely large valid input must +not obtain disproportionate CPU work before the projection can fail closed. + +## Decision + +LineageWeave publishes the source-text-free +`lineageweave_dynamic_evaluation_lineage/v1` projection. It contains two sealed +aggregate forms. + +### Dynamic evaluation item lineage + +One item projection records only immutable references: + +- exact item-snapshot and blueprint-revision identity; +- exact released source-contract identity and complete lowercase SHA-256 digest; +- optional item-generation invocation; +- zero or more immutable rater-invocation references; +- optional adjudication-case and separate adjudication-resolution references; +- zero or more calibration-artifact references; +- optional separate anchor-promotion decision; +- optional predecessor item snapshot that this version supersedes. + +An adjudication resolution requires its case. The source rater invocations remain +present and are never replaced by the resolution. A successor may identify an +older snapshot but cannot supersede itself. + +### Dynamic evaluation run lineage + +One run projection freezes: + +- one exact run-snapshot and blueprint-revision identity; +- one non-empty, unique, blueprint-consistent item set; +- zero or more item snapshots explicitly acting as anchors; +- one comparability state: `unavailable`, `within_run_only`, or `linked`; +- an immutable linking-evidence reference only when comparability is `linked`. + +Cold-start runs with zero fixed anchors are valid for pilot, diagnostic, and +within-run evidence collection. They cannot claim cross-version linked scores. + +An item can appear in `anchor_item_snapshot_refs` only when its lineage includes +both separate calibration evidence and an anchor-promotion decision. An +adjudication resolution alone is insufficient. `linked` additionally requires at +least one such promoted anchor and independent linking evidence. + +In-run `supersedes_item_snapshot_ref` edges form a finite functional graph. A +directed cycle fails closed. Validation records completed predecessor paths so an +in-run item is not repeatedly re-walked from every later successor; admission work +therefore remains linear in the admitted in-run items and edges. A predecessor +reference outside the supplied run terminates the local traversal rather than +inventing foreign graph truth. + +## Ownership boundary + +LineageWeave owns product-specific source/rubric/instrument provenance and the +projection that lets a buyer reconstruct how evidence artifacts relate. It does +not create the foreign artifacts it references. + +- contextual-orchestrator owns provider/model execution, routing, fallback, + dynamic item-generation invocation evidence, and rater-observation creation. +- fast-mlsirm owns reusable measurement Published Languages and all production + psychometric calibration, fit, DIF, information, linking, uncertainty, and + score arithmetic. +- Psychometrics Commons owns hosted blueprint/run lifecycle, panel assignment, + adjudication transaction state, tenant authorization, persistence, and + immutable result publication. +- TEPP owns temporal/event semantics and later drift, change-point, or invariance + monitoring. + +Cross-repository integration must consume immutable released/versioned artifacts +with exact digests. Mutable sibling PR heads, foreign service databases, and +cross-service SQL are not production contracts. + +## Fail-closed behavior + +The projection rejects: + +- provider credentials, endpoints, provider/model selection fields, scores, + latent traits, pass/fail, certification, employment decisions, or embedded + adjudication decisions; +- unknown fields and non-string mapping keys; +- empty, padded, Unicode-format-control-bearing, control-bearing, + surrogate-bearing, or overlong opaque references; +- malformed or non-lowercase contract digests; +- duplicate item/rater/calibration/anchor references; +- item sets beyond the bounded allocation ceiling; +- mixed blueprint revisions in one run snapshot; +- a resolution without a case or self-supersession; +- directed cycles among supersession edges whose endpoints are in the run; +- anchor claims without separate calibration and promotion evidence; +- linked comparability without promoted anchors and linking evidence; +- linking evidence on an unavailable or within-run-only projection. + +No missing reference is converted into a score, default anchor, provider guess, +or synthetic lineage edge. + +## Consequences + +### Benefits + +- dynamic evaluations can begin before a fixed item corpus exists; +- each run remains tied to its actual immutable item set rather than a mutable + blueprint or regenerated approximation; +- adjudication remains review evidence instead of overwriting observations; +- anchor promotion, calibration, and linking remain separately auditable; +- opaque references cannot differ only through invisible Unicode format controls; +- large acyclic supersession chains stay bounded to linear graph-validation work; +- LineageWeave can display an explicit no-anchor/no-linking limitation without + inventing comparability; +- provider and psychometric authorities remain in their canonical owners. + +### Costs + +- the hosted system must persist the referenced run and item snapshots before + dispatching observations; +- downstream projections require released contract versions and digests; +- source content remains separately permissioned and cannot be recovered from + this metadata-only envelope; +- external adapters must map any legitimate foreign identifier containing a + rejected format control to a separate canonical released reference instead of + passing it through unchanged; +- user interfaces must distinguish provisional, adjudicated, calibrated, + promoted-anchor, and linked states rather than displaying one generic + “evaluated” badge. + +## Alternatives considered + +1. **Reuse a mutable golden-prompt table.** Rejected because no fixed set is + required, “golden” conflates adjudication with validation, and later edits + would destroy run identity. +2. **Store provider/model payloads in LineageWeave.** Rejected because provider + execution and credential policy belong to contextual-orchestrator and raw + content has separate access/retention requirements. +3. **Treat adjudicated items as anchors automatically.** Rejected because an + adjudication resolution does not establish calibration, fit, fairness, + invariance, approval, or cross-version linking. +4. **Block all evaluation until anchors exist.** Rejected because governed pilot + and diagnostic evidence is necessary to create and validate the first anchor + corpus. +5. **Silently strip or normalize format controls.** Rejected because mutation + could collapse two foreign references into an identity that the owning system + never published. Admission fails closed instead. +6. **Re-walk the complete supersession prefix from every item.** Rejected because + the 10,000-item admission budget would permit quadratic validation work even + for a valid acyclic chain. Completed-path memoization preserves the same local + graph semantics without that amplification. + +## Verification + +Focused tests cover zero-anchor runs, adjudication/source-observation separation, +anchor-promotion requirements, linked-evidence requirements, immutable collection +copying, strict mapping admission, reference and digest hygiene, blueprint +consistency, duplicate and resource limits, public exports, direct-construction +seals, in-run supersession cycles, and long acyclic supersession chains. Reference +hygiene includes zero-width and bidirectional Unicode format controls. + +The supersession admission regression counts set membership/add work rather than +asserting a runner-specific elapsed-time threshold. That makes the complexity +contract deterministic while leaving real buyer-path latency to the repository's +separate measured performance evidence. The new projection module must retain +complete statement and branch coverage on the unchanged exact head. + +No fixed production item example, provider call, score, database migration, or +adjudication action is introduced by this ADR. + +## References + +American Educational Research Association, American Psychological Association, +& National Council on Measurement in Education. (2014). *Standards for +educational and psychological testing*. American Educational Research +Association. + +Evans, E. (2003). *Domain-driven design: Tackling complexity in the heart of +software*. Addison-Wesley. + +Moreau, L., Missier, P., Belhajjame, K., B’Far, R., Cheney, J., Coppens, S., +Cresswell, S., Gil, Y., Groth, P., Klyne, G., Lebo, T., McCusker, J., Miles, S., +Myers, J., Sahoo, S., & Tilmes, C. (2013). PROV-DM: The PROV data model. World +Wide Web Consortium. + +Unicode Consortium. (2025). *Unicode security mechanisms* (Unicode Technical +Standard #39, Version 17.0.0, Revision 32). https://www.unicode.org/reports/tr39/ From c95e9afb10748da63ee88a66b06017da24983266 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Thu, 3 Sep 2026 01:32:15 +0900 Subject: [PATCH 29/47] docs(adr): release conflicting ADR 0352 allocation --- docs/adr/0352-dynamic-evaluation-lineage.md | 190 -------------------- 1 file changed, 190 deletions(-) delete mode 100644 docs/adr/0352-dynamic-evaluation-lineage.md diff --git a/docs/adr/0352-dynamic-evaluation-lineage.md b/docs/adr/0352-dynamic-evaluation-lineage.md deleted file mode 100644 index 16edbf3c1..000000000 --- a/docs/adr/0352-dynamic-evaluation-lineage.md +++ /dev/null @@ -1,190 +0,0 @@ -# ADR 0352: Project dynamic evaluation snapshots without absorbing decision authority - -- Status: Proposed -- Date: 2026-09-02 -- Depends on: ADR 0300 (contextual-orchestrator ownership boundary), ADR 0301 (dichotomous measurement policy) - -## Context - -A product evaluation may resolve its concrete items dynamically from an authored -blueprint, a production sample, a controlled perturbation, or a model/algorithmic -generator. A fixed item set or validated anchor corpus may not exist during cold -start. Nevertheless, an evaluation must remain reproducible enough to determine -which exact item snapshots, generator invocations, rater observations, -adjudication artifacts, calibration evidence, and promotion decisions informed a -result. - -A single mutable `evaluation_item` record would collapse independent facts and -permit later review to rewrite history. It could also allow LineageWeave to absorb -provider credentials, model routing, psychometric calculation, hosted -adjudication, or source-system authority that belongs to other bounded contexts. - -Opaque provenance references also cross service and rendering boundaries. Unicode -format controls can be machine-distinct while remaining visually absent or -changing bidirectional presentation, creating an avoidable alias/spoofing surface -for identifiers used in equality and provenance joins. Unicode Technical Standard -#39 treats identifier ambiguity and default-ignorable characters as security -concerns. LineageWeave therefore rejects Unicode `Cf` format controls in these -opaque references rather than normalizing them into a guessed identity. This is a -product-specific restrictive profile, not a claim of full UTS #39 conformance. - -## Decision - -LineageWeave publishes the source-text-free -`lineageweave_dynamic_evaluation_lineage/v1` projection. It contains two sealed -aggregate forms. - -### Dynamic evaluation item lineage - -One item projection records only immutable references: - -- exact item-snapshot and blueprint-revision identity; -- exact released source-contract identity and complete lowercase SHA-256 digest; -- optional item-generation invocation; -- zero or more immutable rater-invocation references; -- optional adjudication-case and separate adjudication-resolution references; -- zero or more calibration-artifact references; -- optional separate anchor-promotion decision; -- optional predecessor item snapshot that this version supersedes. - -An adjudication resolution requires its case. The source rater invocations remain -present and are never replaced by the resolution. A successor may identify an -older snapshot but cannot supersede itself. - -### Dynamic evaluation run lineage - -One run projection freezes: - -- one exact run-snapshot and blueprint-revision identity; -- one non-empty, unique, blueprint-consistent item set; -- zero or more item snapshots explicitly acting as anchors; -- one comparability state: `unavailable`, `within_run_only`, or `linked`; -- an immutable linking-evidence reference only when comparability is `linked`. - -Cold-start runs with zero fixed anchors are valid for pilot, diagnostic, and -within-run evidence collection. They cannot claim cross-version linked scores. - -An item can appear in `anchor_item_snapshot_refs` only when its lineage includes -both separate calibration evidence and an anchor-promotion decision. An -adjudication resolution alone is insufficient. `linked` additionally requires at -least one such promoted anchor and independent linking evidence. - -## Ownership boundary - -LineageWeave owns product-specific source/rubric/instrument provenance and the -projection that lets a buyer reconstruct how evidence artifacts relate. It does -not create the foreign artifacts it references. - -- contextual-orchestrator owns provider/model execution, routing, fallback, - dynamic item-generation invocation evidence, and rater-observation creation. -- fast-mlsirm owns reusable measurement Published Languages and all production - psychometric calibration, fit, DIF, information, linking, uncertainty, and - score arithmetic. -- Psychometrics Commons owns hosted blueprint/run lifecycle, panel assignment, - adjudication transaction state, tenant authorization, persistence, and - immutable result publication. -- TEPP owns temporal/event semantics and later drift, change-point, or invariance - monitoring. - -Cross-repository integration must consume immutable released/versioned artifacts -with exact digests. Mutable sibling PR heads, foreign service databases, and -cross-service SQL are not production contracts. - -## Fail-closed behavior - -The projection rejects: - -- provider credentials, endpoints, provider/model selection fields, scores, - latent traits, pass/fail, certification, employment decisions, or embedded - adjudication decisions; -- unknown fields and non-string mapping keys; -- empty, padded, Unicode-format-control-bearing, control-bearing, - surrogate-bearing, or overlong opaque references; -- malformed or non-lowercase contract digests; -- duplicate item/rater/calibration/anchor references; -- item sets beyond the bounded allocation ceiling; -- mixed blueprint revisions in one run snapshot; -- a resolution without a case or self-supersession; -- anchor claims without separate calibration and promotion evidence; -- linked comparability without promoted anchors and linking evidence; -- linking evidence on an unavailable or within-run-only projection. - -No missing reference is converted into a score, default anchor, provider guess, -or synthetic lineage edge. - -## Consequences - -### Benefits - -- dynamic evaluations can begin before a fixed item corpus exists; -- each run remains tied to its actual immutable item set rather than a mutable - blueprint or regenerated approximation; -- adjudication remains review evidence instead of overwriting observations; -- anchor promotion, calibration, and linking remain separately auditable; -- opaque references cannot differ only through invisible Unicode format controls; -- LineageWeave can display an explicit no-anchor/no-linking limitation without - inventing comparability; -- provider and psychometric authorities remain in their canonical owners. - -### Costs - -- the hosted system must persist the referenced run and item snapshots before - dispatching observations; -- downstream projections require released contract versions and digests; -- source content remains separately permissioned and cannot be recovered from - this metadata-only envelope; -- external adapters must map any legitimate foreign identifier containing a - rejected format control to a separate canonical released reference instead of - passing it through unchanged; -- user interfaces must distinguish provisional, adjudicated, calibrated, - promoted-anchor, and linked states rather than displaying one generic - “evaluated” badge. - -## Alternatives considered - -1. **Reuse a mutable golden-prompt table.** Rejected because no fixed set is - required, “golden” conflates adjudication with validation, and later edits - would destroy run identity. -2. **Store provider/model payloads in LineageWeave.** Rejected because provider - execution and credential policy belong to contextual-orchestrator and raw - content has separate access/retention requirements. -3. **Treat adjudicated items as anchors automatically.** Rejected because an - adjudication resolution does not establish calibration, fit, fairness, - invariance, approval, or cross-version linking. -4. **Block all evaluation until anchors exist.** Rejected because governed pilot - and diagnostic evidence is necessary to create and validate the first anchor - corpus. -5. **Silently strip or normalize format controls.** Rejected because mutation - could collapse two foreign references into an identity that the owning system - never published. Admission fails closed instead. - -## Verification - -Focused tests cover zero-anchor runs, adjudication/source-observation separation, -anchor-promotion requirements, linked-evidence requirements, immutable collection -copying, strict mapping admission, reference and digest hygiene, blueprint -consistency, duplicate and resource limits, public exports, and direct-construction -seals. Reference hygiene includes zero-width and bidirectional Unicode format -controls. The new projection module must retain complete statement and branch -coverage on the unchanged exact head. - -No fixed production item example, provider call, score, database migration, or -adjudication action is introduced by this ADR. - -## References - -American Educational Research Association, American Psychological Association, -& National Council on Measurement in Education. (2014). *Standards for -educational and psychological testing*. American Educational Research -Association. - -Evans, E. (2003). *Domain-driven design: Tackling complexity in the heart of -software*. Addison-Wesley. - -Moreau, L., Missier, P., Belhajjame, K., B’Far, R., Cheney, J., Coppens, S., -Cresswell, S., Gil, Y., Groth, P., Klyne, G., Lebo, T., McCusker, J., Miles, S., -Myers, J., Sahoo, S., & Tilmes, C. (2013). PROV-DM: The PROV data model. World -Wide Web Consortium. - -Unicode Consortium. (2025). *Unicode security mechanisms* (Unicode Technical -Standard #39, Version 17.0.0, Revision 32). https://www.unicode.org/reports/tr39/ From e33ad56af86fa2469464f6c51dc0523d7a56f22b Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Thu, 3 Sep 2026 01:35:50 +0900 Subject: [PATCH 30/47] docs(adr): align dynamic lineage with criterion contract --- docs/adr/0355-dynamic-evaluation-lineage.md | 111 ++++++++++++++------ 1 file changed, 80 insertions(+), 31 deletions(-) diff --git a/docs/adr/0355-dynamic-evaluation-lineage.md b/docs/adr/0355-dynamic-evaluation-lineage.md index 161e877fc..61d5adc73 100644 --- a/docs/adr/0355-dynamic-evaluation-lineage.md +++ b/docs/adr/0355-dynamic-evaluation-lineage.md @@ -10,9 +10,15 @@ A product evaluation may resolve its concrete items dynamically from an authored blueprint, a production sample, a controlled perturbation, or a model/algorithmic generator. A fixed item set or validated anchor corpus may not exist during cold start. Nevertheless, an evaluation must remain reproducible enough to determine -which exact item snapshots, generator invocations, rater observations, -adjudication artifacts, calibration evidence, and promotion decisions informed a -result. +which exact substantive criteria, item snapshots, generator invocations, rater +observations, adjudication artifacts, calibration evidence, and promotion +decisions informed a result. + +Item provenance without criterion provenance is insufficient: two runs can use the +same nominal rubric label while differing in intended use, construct, population, +language, domain, evidence-admission rules, missingness semantics, or response +category definitions. A run therefore cannot claim an auditable evaluation merely +because its generated items and observations are traceable. A single mutable `evaluation_item` record would collapse independent facts and permit later review to rewrite history. It could also allow LineageWeave to absorb @@ -37,8 +43,25 @@ not obtain disproportionate CPU work before the projection can fail closed. ## Decision LineageWeave publishes the source-text-free -`lineageweave_dynamic_evaluation_lineage/v1` projection. It contains two sealed -aggregate forms. +`lineageweave_dynamic_evaluation_lineage/v1` projection. Substantive criterion +lineage is frozen independently and then bound into item/run lineage rather than +being inferred from an item generator or later score. + +### Evaluation criterion-set lineage + +One administered criterion-set snapshot retains: + +- exact criterion-set identity and complete lowercase SHA-256 digest; +- the exact blueprint and rubric revision; +- intended-use, construct, population-scope, language-scope, and domain-scope + references; +- a non-empty bounded set of uniquely identified criteria. + +Each criterion retains exact definition, admissible-evidence, exclusion, +response-semantics, abstention, and not-observable rule references plus their +complete digests, together with the ordered response-category definition +references and digests. LineageWeave retains these product semantics but does not +turn them into fitted psychometric parameters or scoring authority. ### Dynamic evaluation item lineage @@ -46,6 +69,8 @@ One item projection records only immutable references: - exact item-snapshot and blueprint-revision identity; - exact released source-contract identity and complete lowercase SHA-256 digest; +- when criterion-bound, the exact criterion-set snapshot/digest, rubric revision, + and one or more governed criterion references; - optional item-generation invocation; - zero or more immutable rater-invocation references; - optional adjudication-case and separate adjudication-resolution references; @@ -57,16 +82,27 @@ An adjudication resolution requires its case. The source rater invocations remai present and are never replaced by the resolution. A successor may identify an older snapshot but cannot supersede itself. +Criterion binding is all-or-nothing at the item boundary: a partial set identity, +digest, rubric, or criterion list is rejected rather than treated as an unbound +item. + ### Dynamic evaluation run lineage One run projection freezes: - one exact run-snapshot and blueprint-revision identity; - one non-empty, unique, blueprint-consistent item set; +- when substantive criteria are administered, the complete immutable criterion + set used for that run; - zero or more item snapshots explicitly acting as anchors; - one comparability state: `unavailable`, `within_run_only`, or `linked`; - an immutable linking-evidence reference only when comparability is `linked`. +When a criterion set is supplied, every item must retain the same criterion-set +snapshot/digest and rubric revision, may reference only criteria in that set, and +the union of item criterion references must cover every administered criterion. +Criterion-bound items without their administered criterion set fail closed. + Cold-start runs with zero fixed anchors are valid for pilot, diagnostic, and within-run evidence collection. They cannot claim cross-version linked scores. @@ -84,9 +120,9 @@ inventing foreign graph truth. ## Ownership boundary -LineageWeave owns product-specific source/rubric/instrument provenance and the -projection that lets a buyer reconstruct how evidence artifacts relate. It does -not create the foreign artifacts it references. +LineageWeave owns product-specific criterion/rubric/source/instrument provenance +and the projection that lets a buyer reconstruct how evidence artifacts relate. +It does not create the foreign artifacts it references. - contextual-orchestrator owns provider/model execution, routing, fallback, dynamic item-generation invocation evidence, and rater-observation creation. @@ -114,6 +150,12 @@ The projection rejects: - empty, padded, Unicode-format-control-bearing, control-bearing, surrogate-bearing, or overlong opaque references; - malformed or non-lowercase contract digests; +- empty criterion sets, duplicate criterion identities, malformed category + definition/digest cardinality, and incomplete substantive criterion meaning; +- partial item criterion bindings; +- criterion-bound items without their administered criterion set; +- criterion-set/blueprint, criterion-set/digest, rubric, or item-criterion + substitution and incomplete administered-criterion coverage; - duplicate item/rater/calibration/anchor references; - item sets beyond the bounded allocation ceiling; - mixed blueprint revisions in one run snapshot; @@ -123,73 +165,80 @@ The projection rejects: - linked comparability without promoted anchors and linking evidence; - linking evidence on an unavailable or within-run-only projection. -No missing reference is converted into a score, default anchor, provider guess, -or synthetic lineage edge. +No missing criterion, reference, or evidence artifact is converted into a score, +default anchor, provider guess, or synthetic lineage edge. ## Consequences ### Benefits - dynamic evaluations can begin before a fixed item corpus exists; -- each run remains tied to its actual immutable item set rather than a mutable - blueprint or regenerated approximation; +- the intended construct and evidence rules remain inspectable independently of + item generation and later observations; +- each run remains tied to its actual immutable criterion/item set rather than a + mutable rubric label, blueprint, or regenerated approximation; - adjudication remains review evidence instead of overwriting observations; - anchor promotion, calibration, and linking remain separately auditable; - opaque references cannot differ only through invisible Unicode format controls; - large acyclic supersession chains stay bounded to linear graph-validation work; -- LineageWeave can display an explicit no-anchor/no-linking limitation without - inventing comparability; +- LineageWeave can display explicit criterion, no-anchor, and no-linking + limitations without inventing comparability; - provider and psychometric authorities remain in their canonical owners. ### Costs -- the hosted system must persist the referenced run and item snapshots before - dispatching observations; +- the hosted system must persist the referenced criterion, run, and item snapshots + before observations are interpreted; - downstream projections require released contract versions and digests; - source content remains separately permissioned and cannot be recovered from this metadata-only envelope; - external adapters must map any legitimate foreign identifier containing a rejected format control to a separate canonical released reference instead of passing it through unchanged; -- user interfaces must distinguish provisional, adjudicated, calibrated, - promoted-anchor, and linked states rather than displaying one generic - “evaluated” badge. +- user interfaces must distinguish criterion meaning, provisional observation, + adjudicated, calibrated, promoted-anchor, and linked states rather than + displaying one generic “evaluated” badge. ## Alternatives considered 1. **Reuse a mutable golden-prompt table.** Rejected because no fixed set is required, “golden” conflates adjudication with validation, and later edits would destroy run identity. -2. **Store provider/model payloads in LineageWeave.** Rejected because provider +2. **Persist only criterion identifiers or a rubric name.** Rejected because an + identifier alone does not freeze intended use, construct/scope, evidence and + exclusion rules, response/missingness semantics, or category definitions. +3. **Store provider/model payloads in LineageWeave.** Rejected because provider execution and credential policy belong to contextual-orchestrator and raw content has separate access/retention requirements. -3. **Treat adjudicated items as anchors automatically.** Rejected because an +4. **Treat adjudicated items as anchors automatically.** Rejected because an adjudication resolution does not establish calibration, fit, fairness, invariance, approval, or cross-version linking. -4. **Block all evaluation until anchors exist.** Rejected because governed pilot +5. **Block all evaluation until anchors exist.** Rejected because governed pilot and diagnostic evidence is necessary to create and validate the first anchor corpus. -5. **Silently strip or normalize format controls.** Rejected because mutation +6. **Silently strip or normalize format controls.** Rejected because mutation could collapse two foreign references into an identity that the owning system never published. Admission fails closed instead. -6. **Re-walk the complete supersession prefix from every item.** Rejected because +7. **Re-walk the complete supersession prefix from every item.** Rejected because the 10,000-item admission budget would permit quadratic validation work even for a valid acyclic chain. Completed-path memoization preserves the same local graph semantics without that amplification. ## Verification -Focused tests cover zero-anchor runs, adjudication/source-observation separation, -anchor-promotion requirements, linked-evidence requirements, immutable collection -copying, strict mapping admission, reference and digest hygiene, blueprint -consistency, duplicate and resource limits, public exports, direct-construction -seals, in-run supersession cycles, and long acyclic supersession chains. Reference -hygiene includes zero-width and bidirectional Unicode format controls. +Focused tests cover substantive criterion completeness, criterion-set/item +binding, administered-criterion coverage, set/digest/rubric substitution, +zero-anchor runs, adjudication/source-observation separation, anchor-promotion +requirements, linked-evidence requirements, immutable collection copying, strict +mapping admission, reference and digest hygiene, blueprint consistency, duplicate +and resource limits, public exports, direct-construction seals, in-run +supersession cycles, and long acyclic supersession chains. Reference hygiene +includes zero-width and bidirectional Unicode format controls. The supersession admission regression counts set membership/add work rather than asserting a runner-specific elapsed-time threshold. That makes the complexity contract deterministic while leaving real buyer-path latency to the repository's -separate measured performance evidence. The new projection module must retain +separate measured performance evidence. The new projection modules must retain complete statement and branch coverage on the unchanged exact head. No fixed production item example, provider call, score, database migration, or From 6e83d1f8e151f91c442e20e42d28a5fef8118b46 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Thu, 3 Sep 2026 01:48:07 +0900 Subject: [PATCH 31/47] test(lineage): reject collapsed anchor evidence identity --- tests/test_dynamic_evaluation_lineage.py | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/tests/test_dynamic_evaluation_lineage.py b/tests/test_dynamic_evaluation_lineage.py index 09b0273c2..49ad1f21e 100644 --- a/tests/test_dynamic_evaluation_lineage.py +++ b/tests/test_dynamic_evaluation_lineage.py @@ -128,6 +128,25 @@ def test_adjudication_alone_cannot_promote_an_anchor() -> None: assert run.anchor_item_snapshot_refs == (promoted.item_snapshot_ref,) +def test_anchor_promotion_and_calibration_evidence_keep_distinct_identities() -> None: + """One opaque artifact cannot satisfy both anchor promotion and calibration evidence.""" + shared_evidence_ref = "anchor_evidence_1" + ambiguous_anchor = _item( + calibration_artifact_refs=(shared_evidence_ref,), + anchor_promotion_decision_ref=shared_evidence_ref, + ) + + with pytest.raises(DynamicEvaluationLineageError) as caught: + build_dynamic_evaluation_run_lineage( + run_snapshot_ref="evaluation_run_snapshot_anchor_collision", + blueprint_revision_ref="evaluation_blueprint_revision_1", + items=(ambiguous_anchor,), + anchor_item_snapshot_refs=(ambiguous_anchor.item_snapshot_ref,), + comparability_status=RunComparabilityStatus.UNAVAILABLE, + ) + assert caught.value.code == "anchor_evidence_collision" + + def test_lineage_rejects_provider_configuration_and_decision_payload_fields() -> None: """Lineage projection cannot absorb provider credentials, endpoints, scores, or decisions.""" payload = { From 1061fb316228968ece433d7f8c8848b200af7e17 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Thu, 3 Sep 2026 01:49:30 +0900 Subject: [PATCH 32/47] fix(lineage): keep anchor evidence roles distinct --- lineageweave/evaluation_lineage.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/lineageweave/evaluation_lineage.py b/lineageweave/evaluation_lineage.py index 18f97946b..e804af304 100644 --- a/lineageweave/evaluation_lineage.py +++ b/lineageweave/evaluation_lineage.py @@ -644,6 +644,11 @@ def build_dynamic_evaluation_run_lineage( "anchor_requires_promotion_evidence", "an anchor requires separate promotion and calibration evidence", ) + if anchor.anchor_promotion_decision_ref in anchor.calibration_artifact_refs: + raise DynamicEvaluationLineageError( + "anchor_evidence_collision", + "anchor promotion and calibration evidence must retain distinct identities", + ) normalized_status = _comparability_status(comparability_status) normalized_linking_ref = _optional_reference( From 8c73220d49f429f2ce2930438fb63a5b672b246e Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Thu, 3 Sep 2026 01:50:33 +0900 Subject: [PATCH 33/47] test(lineage): reject linking evidence identity collisions --- tests/test_dynamic_evaluation_lineage.py | 25 ++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/tests/test_dynamic_evaluation_lineage.py b/tests/test_dynamic_evaluation_lineage.py index 49ad1f21e..b66e5ae57 100644 --- a/tests/test_dynamic_evaluation_lineage.py +++ b/tests/test_dynamic_evaluation_lineage.py @@ -147,6 +147,31 @@ def test_anchor_promotion_and_calibration_evidence_keep_distinct_identities() -> assert caught.value.code == "anchor_evidence_collision" +@pytest.mark.parametrize( + "linking_evidence_ref", + ("anchor_promotion_decision_1", "calibration_artifact_1"), +) +def test_linking_evidence_is_distinct_from_anchor_evidence( + linking_evidence_ref: str, +) -> None: + """Cross-version linking evidence cannot reuse an anchor evidence identity.""" + promoted = _item( + calibration_artifact_refs=("calibration_artifact_1",), + anchor_promotion_decision_ref="anchor_promotion_decision_1", + ) + + with pytest.raises(DynamicEvaluationLineageError) as caught: + build_dynamic_evaluation_run_lineage( + run_snapshot_ref="evaluation_run_snapshot_link_collision", + blueprint_revision_ref="evaluation_blueprint_revision_1", + items=(promoted,), + anchor_item_snapshot_refs=(promoted.item_snapshot_ref,), + comparability_status=RunComparabilityStatus.LINKED, + linking_evidence_ref=linking_evidence_ref, + ) + assert caught.value.code == "linking_evidence_collision" + + def test_lineage_rejects_provider_configuration_and_decision_payload_fields() -> None: """Lineage projection cannot absorb provider credentials, endpoints, scores, or decisions.""" payload = { From 9683ca64e87940d1fc7b878e4b89dd0511c2ab5d Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Thu, 3 Sep 2026 01:51:37 +0900 Subject: [PATCH 34/47] fix(lineage): separate linking from anchor evidence identity --- lineageweave/evaluation_lineage.py | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/lineageweave/evaluation_lineage.py b/lineageweave/evaluation_lineage.py index e804af304..3493f3ddf 100644 --- a/lineageweave/evaluation_lineage.py +++ b/lineageweave/evaluation_lineage.py @@ -634,21 +634,22 @@ def build_dynamic_evaluation_run_lineage( "unknown_anchor_item", "every anchor must identify an item in this run" ) item_by_ref = {item.item_snapshot_ref: item for item in normalized_items} + anchor_evidence_refs: set[str] = set() for anchor_ref in normalized_anchor_refs: anchor = item_by_ref[anchor_ref] - if ( - anchor.anchor_promotion_decision_ref is None - or not anchor.calibration_artifact_refs - ): + promotion_ref = anchor.anchor_promotion_decision_ref + if promotion_ref is None or not anchor.calibration_artifact_refs: raise DynamicEvaluationLineageError( "anchor_requires_promotion_evidence", "an anchor requires separate promotion and calibration evidence", ) - if anchor.anchor_promotion_decision_ref in anchor.calibration_artifact_refs: + if promotion_ref in anchor.calibration_artifact_refs: raise DynamicEvaluationLineageError( "anchor_evidence_collision", "anchor promotion and calibration evidence must retain distinct identities", ) + anchor_evidence_refs.add(promotion_ref) + anchor_evidence_refs.update(anchor.calibration_artifact_refs) normalized_status = _comparability_status(comparability_status) normalized_linking_ref = _optional_reference( @@ -665,6 +666,11 @@ def build_dynamic_evaluation_run_lineage( "linked_run_requires_evidence", "linked comparability requires immutable linking evidence", ) + if normalized_linking_ref in anchor_evidence_refs: + raise DynamicEvaluationLineageError( + "linking_evidence_collision", + "linking evidence must retain an identity distinct from anchor evidence", + ) elif normalized_linking_ref is not None: raise DynamicEvaluationLineageError( "unexpected_linking_evidence", From 94b8852a7bc717cbd75e40a6e847cf2892c6afda Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Thu, 3 Sep 2026 01:58:03 +0900 Subject: [PATCH 35/47] test(lineage): stop implying an unreleased owner contract --- tests/test_dynamic_evaluation_lineage.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/tests/test_dynamic_evaluation_lineage.py b/tests/test_dynamic_evaluation_lineage.py index b66e5ae57..32ebe75bc 100644 --- a/tests/test_dynamic_evaluation_lineage.py +++ b/tests/test_dynamic_evaluation_lineage.py @@ -15,6 +15,7 @@ ) _CONTRACT_DIGEST = "a" * 64 +_SOURCE_CONTRACT_REF = "synthetic_source_contract/v1" def _item( @@ -30,7 +31,7 @@ def _item( return build_dynamic_evaluation_item_lineage( item_snapshot_ref=item_snapshot_ref, blueprint_revision_ref="evaluation_blueprint_revision_1", - source_contract_ref="fast_mlsirm_dynamic_evaluation_item/v1", + source_contract_ref=_SOURCE_CONTRACT_REF, source_contract_sha256=_CONTRACT_DIGEST, generation_invocation_ref="generation_invocation_1", rater_invocation_refs=("rater_invocation_1", "rater_invocation_2"), @@ -182,7 +183,7 @@ def test_lineage_rejects_provider_configuration_and_decision_payload_fields() -> { "item_snapshot_ref": "evaluation_item_snapshot_alpha", "blueprint_revision_ref": "evaluation_blueprint_revision_1", - "source_contract_ref": "fast_mlsirm_dynamic_evaluation_item/v1", + "source_contract_ref": _SOURCE_CONTRACT_REF, "source_contract_sha256": _CONTRACT_DIGEST, "generation_invocation_ref": "generation_invocation_1", "rater_invocation_refs": ["rater_invocation_1"], @@ -258,7 +259,7 @@ def test_direct_aggregate_construction_is_sealed() -> None: DynamicEvaluationItemLineage( # type: ignore[call-arg] item_snapshot_ref="evaluation_item_snapshot_alpha", blueprint_revision_ref="evaluation_blueprint_revision_1", - source_contract_ref="fast_mlsirm_dynamic_evaluation_item/v1", + source_contract_ref=_SOURCE_CONTRACT_REF, source_contract_sha256=_CONTRACT_DIGEST, generation_invocation_ref="generation_invocation_1", rater_invocation_refs=("rater_invocation_1",), From 493476b7104b8e21d966b93cd646a0c76e0e7628 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Thu, 3 Sep 2026 02:00:46 +0900 Subject: [PATCH 36/47] docs(lineage): align evidence identity and released-owner boundaries --- docs/adr/0355-dynamic-evaluation-lineage.md | 40 +++++++++++++++------ 1 file changed, 29 insertions(+), 11 deletions(-) diff --git a/docs/adr/0355-dynamic-evaluation-lineage.md b/docs/adr/0355-dynamic-evaluation-lineage.md index 61d5adc73..b2bf4e358 100644 --- a/docs/adr/0355-dynamic-evaluation-lineage.md +++ b/docs/adr/0355-dynamic-evaluation-lineage.md @@ -107,9 +107,15 @@ Cold-start runs with zero fixed anchors are valid for pilot, diagnostic, and within-run evidence collection. They cannot claim cross-version linked scores. An item can appear in `anchor_item_snapshot_refs` only when its lineage includes -both separate calibration evidence and an anchor-promotion decision. An -adjudication resolution alone is insufficient. `linked` additionally requires at -least one such promoted anchor and independent linking evidence. +both calibration evidence and an anchor-promotion decision. Within this v1 +projection those evidence roles must also retain distinct opaque identities: the +promotion decision cannot reuse a calibration-artifact reference. An adjudication +resolution alone is insufficient. `linked` additionally requires at least one such +promoted anchor and independent linking evidence; the linking-evidence identity +cannot reuse that promoted anchor's promotion or calibration reference. This +identity-separation rule is a LineageWeave auditability invariant, not a claim +that a psychometric standard universally requires physically separate files or +storage objects. In-run `supersedes_item_snapshot_ref` edges form a finite functional graph. A directed cycle fails closed. Validation records completed predecessor paths so an @@ -137,7 +143,9 @@ It does not create the foreign artifacts it references. Cross-repository integration must consume immutable released/versioned artifacts with exact digests. Mutable sibling PR heads, foreign service databases, and -cross-service SQL are not production contracts. +cross-service SQL are not production contracts. Synthetic tests use synthetic +source-contract identities; they must not imply that an owner repository has +published a media type or schema that does not exist in an immutable release. ## Fail-closed behavior @@ -161,8 +169,11 @@ The projection rejects: - mixed blueprint revisions in one run snapshot; - a resolution without a case or self-supersession; - directed cycles among supersession edges whose endpoints are in the run; -- anchor claims without separate calibration and promotion evidence; -- linked comparability without promoted anchors and linking evidence; +- anchor claims without calibration and promotion evidence, or with one opaque + identity reused for both roles; +- linked comparability without promoted anchors and linking evidence, or with + linking evidence whose identity reuses the promoted anchor's calibration or + promotion evidence; - linking evidence on an unavailable or within-run-only projection. No missing criterion, reference, or evidence artifact is converted into a score, @@ -179,6 +190,8 @@ default anchor, provider guess, or synthetic lineage edge. mutable rubric label, blueprint, or regenerated approximation; - adjudication remains review evidence instead of overwriting observations; - anchor promotion, calibration, and linking remain separately auditable; +- synthetic fixtures cannot masquerade as a released owner contract merely by + using an owner-like identifier; - opaque references cannot differ only through invisible Unicode format controls; - large acyclic supersession chains stay bounded to linear graph-validation work; - LineageWeave can display explicit criterion, no-anchor, and no-linking @@ -223,17 +236,22 @@ default anchor, provider guess, or synthetic lineage edge. the 10,000-item admission budget would permit quadratic validation work even for a valid acyclic chain. Completed-path memoization preserves the same local graph semantics without that amplification. +8. **Name a synthetic test fixture after an unreleased owner contract.** Rejected + because it makes a test value look like versioned cross-repository evidence. + Synthetic fixtures use an explicitly synthetic identity; production adapters + must provide the real released owner identity and digest. ## Verification Focused tests cover substantive criterion completeness, criterion-set/item binding, administered-criterion coverage, set/digest/rubric substitution, zero-anchor runs, adjudication/source-observation separation, anchor-promotion -requirements, linked-evidence requirements, immutable collection copying, strict -mapping admission, reference and digest hygiene, blueprint consistency, duplicate -and resource limits, public exports, direct-construction seals, in-run -supersession cycles, and long acyclic supersession chains. Reference hygiene -includes zero-width and bidirectional Unicode format controls. +requirements, anchor/promotion/linking identity separation, linked-evidence +requirements, immutable collection copying, strict mapping admission, reference +and digest hygiene, blueprint consistency, duplicate and resource limits, public +exports, direct-construction seals, in-run supersession cycles, and long acyclic +supersession chains. Reference hygiene includes zero-width and bidirectional +Unicode format controls. The supersession admission regression counts set membership/add work rather than asserting a runner-specific elapsed-time threshold. That makes the complexity From 9a78c0a2e5e7546d8211d008b699ce1e4f9412be Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Thu, 3 Sep 2026 02:54:52 +0900 Subject: [PATCH 37/47] test(lineage): reject falsy malformed criterion reference payloads --- ...mic_evaluation_criterion_refs_transport.py | 43 +++++++++++++++++++ 1 file changed, 43 insertions(+) create mode 100644 tests/test_dynamic_evaluation_criterion_refs_transport.py diff --git a/tests/test_dynamic_evaluation_criterion_refs_transport.py b/tests/test_dynamic_evaluation_criterion_refs_transport.py new file mode 100644 index 000000000..8d7416894 --- /dev/null +++ b/tests/test_dynamic_evaluation_criterion_refs_transport.py @@ -0,0 +1,43 @@ +"""Transport-type regressions for dynamic-evaluation criterion references.""" + +from __future__ import annotations + +from typing import Any + +import pytest + +from lineageweave.evaluation_lineage import build_dynamic_evaluation_item_lineage + + +_DIGEST = "a" * 64 + + +def _build_item(criterion_refs: Any): + return build_dynamic_evaluation_item_lineage( + item_snapshot_ref="evaluation_item_snapshot_transport", + blueprint_revision_ref="evaluation_blueprint_revision_1", + source_contract_ref="synthetic_source_contract/v1", + source_contract_sha256=_DIGEST, + generation_invocation_ref=None, + rater_invocation_refs=(), + adjudication_case_ref=None, + adjudication_resolution_ref=None, + calibration_artifact_refs=(), + anchor_promotion_decision_ref=None, + supersedes_item_snapshot_ref=None, + criterion_refs=criterion_refs, + ) + + +@pytest.mark.parametrize("malformed_refs", [None, "", 0, False, {}]) +def test_falsy_non_collection_criterion_refs_fail_closed(malformed_refs: Any) -> None: + """Do not let transport falsiness silently turn malformed criteria into unbound items.""" + with pytest.raises(TypeError, match="criterion_refs must be a tuple or list"): + _build_item(malformed_refs) + + +def test_empty_reference_collection_remains_a_valid_unbound_item() -> None: + item = _build_item([]) + + assert item.criterion_bound is False + assert item.criterion_refs == () From 3066c12eacbe86977f55b1abf3ebcf9f05e39b7a Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Thu, 3 Sep 2026 02:56:16 +0900 Subject: [PATCH 38/47] fix(lineage): type-check criterion references before binding inference --- lineageweave/evaluation_lineage.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/lineageweave/evaluation_lineage.py b/lineageweave/evaluation_lineage.py index 3493f3ddf..50cedf893 100644 --- a/lineageweave/evaluation_lineage.py +++ b/lineageweave/evaluation_lineage.py @@ -354,6 +354,8 @@ def build_dynamic_evaluation_item_lineage( "self_supersession", "an item snapshot cannot supersede itself" ) + if not isinstance(criterion_refs, (tuple, list)): + raise TypeError("criterion_refs must be a tuple or list") criterion_binding_values = ( criterion_set_snapshot_ref, criterion_set_sha256, From 05ce15a3f57254e73db2fd00f2e574abdf8d9f97 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Thu, 3 Sep 2026 02:56:50 +0900 Subject: [PATCH 39/47] test(lineage): remove nonexistent fast-mlsirm contract fixture --- tests/test_dynamic_evaluation_lineage_boundaries.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_dynamic_evaluation_lineage_boundaries.py b/tests/test_dynamic_evaluation_lineage_boundaries.py index 12c81a819..a558a44c0 100644 --- a/tests/test_dynamic_evaluation_lineage_boundaries.py +++ b/tests/test_dynamic_evaluation_lineage_boundaries.py @@ -23,7 +23,7 @@ def _item(**overrides: Any) -> DynamicEvaluationItemLineage: payload: dict[str, Any] = { "item_snapshot_ref": "evaluation_item_snapshot_alpha", "blueprint_revision_ref": "evaluation_blueprint_revision_1", - "source_contract_ref": "fast_mlsirm_dynamic_evaluation_item/v1", + "source_contract_ref": "synthetic_source_contract/v1", "source_contract_sha256": _DIGEST, "generation_invocation_ref": "generation_invocation_1", "rater_invocation_refs": ("rater_invocation_1",), From 79c1c32a0af9d45940b8dca12a13b7af03b89271 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Thu, 3 Sep 2026 03:47:14 +0900 Subject: [PATCH 40/47] test(lineage): reject Unicode line separators in provenance refs --- tests/test_dynamic_evaluation_lineage_boundaries.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/test_dynamic_evaluation_lineage_boundaries.py b/tests/test_dynamic_evaluation_lineage_boundaries.py index a558a44c0..eb2575870 100644 --- a/tests/test_dynamic_evaluation_lineage_boundaries.py +++ b/tests/test_dynamic_evaluation_lineage_boundaries.py @@ -101,6 +101,8 @@ def test_adjudication_case_and_resolution_keep_distinct_identities() -> None: "item_ref\ufeff", "item\u200bref", "item\u202eref", + "item\u2028ref", + "item\u2029ref", "line\nbreak", "\ud800", "x" * 257, From 290da3a20ff76ae1a32b50166ed34bfd1679a5c7 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Thu, 3 Sep 2026 03:49:50 +0900 Subject: [PATCH 41/47] fix(lineage): reject Unicode separators in provenance refs --- lineageweave/evaluation_lineage.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lineageweave/evaluation_lineage.py b/lineageweave/evaluation_lineage.py index 50cedf893..43325afc9 100644 --- a/lineageweave/evaluation_lineage.py +++ b/lineageweave/evaluation_lineage.py @@ -137,7 +137,7 @@ def _reference(value: Any, field_name: str) -> str: ord(character) < 32 or 127 <= ord(character) <= 159 or 0xD800 <= ord(character) <= 0xDFFF - or unicodedata.category(character) == "Cf" + or unicodedata.category(character) in {"Cf", "Zl", "Zp"} for character in value ) ): From 9a72be3043c28ae9b289d686c192de372252df26 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Thu, 3 Sep 2026 03:50:45 +0900 Subject: [PATCH 42/47] docs(lineage): record Unicode separator admission boundary --- CHANGELOG.d/dynamic-evaluation-lineage.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.d/dynamic-evaluation-lineage.md b/CHANGELOG.d/dynamic-evaluation-lineage.md index 4caad8466..30172bbc3 100644 --- a/CHANGELOG.d/dynamic-evaluation-lineage.md +++ b/CHANGELOG.d/dynamic-evaluation-lineage.md @@ -3,5 +3,5 @@ - Added the versioned `lineageweave_dynamic_evaluation_lineage/v1` projection for dynamically resolved evaluation item and run snapshots. - Preserved generator, rater, adjudication-case/resolution, calibration, anchor-promotion, linking, and supersession references as separate immutable evidence instead of overwriting source observations or inventing decision authority. - Permitted zero-anchor cold-start and within-run projections while requiring separate calibration, promotion, and linking evidence before an item/run can be represented as an anchor or cross-version linked. -- Rejected provider credentials/endpoints, scores, embedded adjudication decisions, mixed-blueprint item sets, duplicate identities, unsupported linking claims, and invisible Unicode format controls in opaque provenance references at the LineageWeave Anti-Corruption Layer. +- Rejected provider credentials/endpoints, scores, embedded adjudication decisions, mixed-blueprint item sets, duplicate identities, unsupported linking claims, Unicode format controls, and Unicode line/paragraph separators in opaque provenance references at the LineageWeave Anti-Corruption Layer. - Rejected directed cycles among in-run supersession references and bounded cycle admission to linear graph work at the 10,000-item contract limit without claiming a runner-specific wall-clock SLO. From b83cc0f8ee4afadb8800199cd4a1045a249fb322 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Thu, 3 Sep 2026 03:51:36 +0900 Subject: [PATCH 43/47] docs(adr): define Unicode separator reference boundary --- docs/adr/0355-dynamic-evaluation-lineage.md | 37 ++++++++++++--------- 1 file changed, 22 insertions(+), 15 deletions(-) diff --git a/docs/adr/0355-dynamic-evaluation-lineage.md b/docs/adr/0355-dynamic-evaluation-lineage.md index b2bf4e358..91b2aea9e 100644 --- a/docs/adr/0355-dynamic-evaluation-lineage.md +++ b/docs/adr/0355-dynamic-evaluation-lineage.md @@ -27,12 +27,15 @@ adjudication, or source-system authority that belongs to other bounded contexts. Opaque provenance references also cross service and rendering boundaries. Unicode format controls can be machine-distinct while remaining visually absent or -changing bidirectional presentation, creating an avoidable alias/spoofing surface -for identifiers used in equality and provenance joins. Unicode Technical Standard -#39 treats identifier ambiguity and default-ignorable characters as security -concerns. LineageWeave therefore rejects Unicode `Cf` format controls in these -opaque references rather than normalizing them into a guessed identity. This is a -product-specific restrictive profile, not a claim of full UTS #39 conformance. +changing bidirectional presentation. U+2028 LINE SEPARATOR and U+2029 PARAGRAPH +SEPARATOR can likewise alter line-oriented rendering or log framing while remaining +embedded in one machine identity. Unicode Technical Standard #39 treats identifier +ambiguity and restricted-character profiles as security concerns, while Unicode +Standard Annex #44 classifies U+2028 and U+2029 as the `Zl` and `Zp` general +categories. LineageWeave therefore rejects Unicode `Cf`, `Zl`, and `Zp` characters +in these opaque references rather than normalizing them into a guessed identity. +This is a product-specific restrictive profile, not a claim of full UTS #39 +conformance. The run contract admits as many as 10,000 item snapshots. A supersession-cycle check that restarts a full predecessor walk from every item can therefore turn an @@ -155,8 +158,8 @@ The projection rejects: latent traits, pass/fail, certification, employment decisions, or embedded adjudication decisions; - unknown fields and non-string mapping keys; -- empty, padded, Unicode-format-control-bearing, control-bearing, - surrogate-bearing, or overlong opaque references; +- empty, padded, Unicode-format-control-bearing, Unicode-line/paragraph-separator- + bearing, control-bearing, surrogate-bearing, or overlong opaque references; - malformed or non-lowercase contract digests; - empty criterion sets, duplicate criterion identities, malformed category definition/digest cardinality, and incomplete substantive criterion meaning; @@ -192,7 +195,8 @@ default anchor, provider guess, or synthetic lineage edge. - anchor promotion, calibration, and linking remain separately auditable; - synthetic fixtures cannot masquerade as a released owner contract merely by using an owner-like identifier; -- opaque references cannot differ only through invisible Unicode format controls; +- opaque references cannot hide format controls or embedded line/paragraph + separators inside an otherwise machine-distinct identity; - large acyclic supersession chains stay bounded to linear graph-validation work; - LineageWeave can display explicit criterion, no-anchor, and no-linking limitations without inventing comparability; @@ -206,8 +210,8 @@ default anchor, provider guess, or synthetic lineage edge. - source content remains separately permissioned and cannot be recovered from this metadata-only envelope; - external adapters must map any legitimate foreign identifier containing a - rejected format control to a separate canonical released reference instead of - passing it through unchanged; + rejected format control or line/paragraph separator to a separate canonical + released reference instead of passing it through unchanged; - user interfaces must distinguish criterion meaning, provisional observation, adjudicated, calibrated, promoted-anchor, and linked states rather than displaying one generic “evaluated” badge. @@ -229,9 +233,9 @@ default anchor, provider guess, or synthetic lineage edge. 5. **Block all evaluation until anchors exist.** Rejected because governed pilot and diagnostic evidence is necessary to create and validate the first anchor corpus. -6. **Silently strip or normalize format controls.** Rejected because mutation - could collapse two foreign references into an identity that the owning system - never published. Admission fails closed instead. +6. **Silently strip or normalize format controls or separators.** Rejected because + mutation could collapse two foreign references into an identity that the + owning system never published. Admission fails closed instead. 7. **Re-walk the complete supersession prefix from every item.** Rejected because the 10,000-item admission budget would permit quadratic validation work even for a valid acyclic chain. Completed-path memoization preserves the same local @@ -251,7 +255,7 @@ requirements, immutable collection copying, strict mapping admission, reference and digest hygiene, blueprint consistency, duplicate and resource limits, public exports, direct-construction seals, in-run supersession cycles, and long acyclic supersession chains. Reference hygiene includes zero-width and bidirectional -Unicode format controls. +Unicode format controls plus U+2028 LINE SEPARATOR and U+2029 PARAGRAPH SEPARATOR. The supersession admission regression counts set membership/add work rather than asserting a runner-specific elapsed-time threshold. That makes the complexity @@ -277,5 +281,8 @@ Cresswell, S., Gil, Y., Groth, P., Klyne, G., Lebo, T., McCusker, J., Miles, S., Myers, J., Sahoo, S., & Tilmes, C. (2013). PROV-DM: The PROV data model. World Wide Web Consortium. +Unicode Consortium. (2025). *Unicode character database* (Unicode Standard Annex +#44, Unicode 17.0.0, Revision 36). https://www.unicode.org/reports/tr44/ + Unicode Consortium. (2025). *Unicode security mechanisms* (Unicode Technical Standard #39, Version 17.0.0, Revision 32). https://www.unicode.org/reports/tr39/ From 623583464925d404f05750c841a4ca2f3b9cdeb8 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Thu, 3 Sep 2026 04:48:06 +0900 Subject: [PATCH 44/47] test(lineage): reject Unicode noncharacters in provenance refs --- tests/test_dynamic_evaluation_lineage_boundaries.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/test_dynamic_evaluation_lineage_boundaries.py b/tests/test_dynamic_evaluation_lineage_boundaries.py index eb2575870..df3bf7b16 100644 --- a/tests/test_dynamic_evaluation_lineage_boundaries.py +++ b/tests/test_dynamic_evaluation_lineage_boundaries.py @@ -103,6 +103,9 @@ def test_adjudication_case_and_resolution_keep_distinct_identities() -> None: "item\u202eref", "item\u2028ref", "item\u2029ref", + "item\ufdd0ref", + "item\uffferef", + "item\U0010ffffref", "line\nbreak", "\ud800", "x" * 257, From 0e1a413fd92e14f289ac648a75d621d26f0d3cb4 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Thu, 3 Sep 2026 04:52:57 +0900 Subject: [PATCH 45/47] fix(lineage): reject Unicode noncharacters in provenance refs --- lineageweave/evaluation_lineage.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/lineageweave/evaluation_lineage.py b/lineageweave/evaluation_lineage.py index 43325afc9..cda28c93b 100644 --- a/lineageweave/evaluation_lineage.py +++ b/lineageweave/evaluation_lineage.py @@ -123,6 +123,15 @@ def _reject_unknown_fields( ) +def _is_unicode_noncharacter(character: str) -> bool: + """Return whether one Unicode scalar is permanently reserved as a noncharacter.""" + codepoint = ord(character) + return 0xFDD0 <= codepoint <= 0xFDEF or (codepoint & 0xFFFF) in { + 0xFFFE, + 0xFFFF, + } + + def _reference(value: Any, field_name: str) -> str: """Validate one exact bounded opaque reference without normalization.""" if type(value) is not str: @@ -138,6 +147,7 @@ def _reference(value: Any, field_name: str) -> str: or 127 <= ord(character) <= 159 or 0xD800 <= ord(character) <= 0xDFFF or unicodedata.category(character) in {"Cf", "Zl", "Zp"} + or _is_unicode_noncharacter(character) for character in value ) ): From e977ba3f14b799b75b88467f2c25c04424e7299d Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Thu, 3 Sep 2026 04:54:07 +0900 Subject: [PATCH 46/47] docs(lineage): record provenance noncharacter boundary --- CHANGELOG.d/dynamic-evaluation-lineage.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.d/dynamic-evaluation-lineage.md b/CHANGELOG.d/dynamic-evaluation-lineage.md index 30172bbc3..8b3e91fbb 100644 --- a/CHANGELOG.d/dynamic-evaluation-lineage.md +++ b/CHANGELOG.d/dynamic-evaluation-lineage.md @@ -3,5 +3,5 @@ - Added the versioned `lineageweave_dynamic_evaluation_lineage/v1` projection for dynamically resolved evaluation item and run snapshots. - Preserved generator, rater, adjudication-case/resolution, calibration, anchor-promotion, linking, and supersession references as separate immutable evidence instead of overwriting source observations or inventing decision authority. - Permitted zero-anchor cold-start and within-run projections while requiring separate calibration, promotion, and linking evidence before an item/run can be represented as an anchor or cross-version linked. -- Rejected provider credentials/endpoints, scores, embedded adjudication decisions, mixed-blueprint item sets, duplicate identities, unsupported linking claims, Unicode format controls, and Unicode line/paragraph separators in opaque provenance references at the LineageWeave Anti-Corruption Layer. +- Rejected provider credentials/endpoints, scores, embedded adjudication decisions, mixed-blueprint item sets, duplicate identities, unsupported linking claims, Unicode format controls, Unicode line/paragraph separators, and the 66 Unicode noncharacters in opaque provenance references at the LineageWeave Anti-Corruption Layer. Other unassigned `Cn` code points are not rejected merely for being unassigned. - Rejected directed cycles among in-run supersession references and bounded cycle admission to linear graph work at the 10,000-item contract limit without claiming a runner-specific wall-clock SLO. From 958e68de450bfa2ff0a3aef54716a10ea4ce7976 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Thu, 3 Sep 2026 04:56:31 +0900 Subject: [PATCH 47/47] docs(adr): define noncharacter provenance admission --- docs/adr/0355-dynamic-evaluation-lineage.md | 46 ++++++++++++++++----- 1 file changed, 35 insertions(+), 11 deletions(-) diff --git a/docs/adr/0355-dynamic-evaluation-lineage.md b/docs/adr/0355-dynamic-evaluation-lineage.md index 91b2aea9e..633037057 100644 --- a/docs/adr/0355-dynamic-evaluation-lineage.md +++ b/docs/adr/0355-dynamic-evaluation-lineage.md @@ -37,6 +37,18 @@ in these opaque references rather than normalizing them into a guessed identity. This is a product-specific restrictive profile, not a claim of full UTS #39 conformance. +Unicode 17.0 also defines exactly 66 noncharacters: U+FDD0..U+FDEF and the final +two code points of every plane, U+nFFFE and U+nFFFF. They are valid Unicode scalar +values and can occur in well-formed Unicode strings, but the Unicode Standard +permanently reserves them for internal use and does not intend them for open +interchange. They share General_Category `Cn` with ordinary unassigned code points, +so rejecting every `Cn` value would conflate stable noncharacters with code points +that may receive future assignments. Because LineageWeave provenance references +are cross-service interchange identifiers, the ACL rejects exactly the stable +noncharacter set while continuing to admit other `Cn` values unless another +independent rule rejects them. The boundary does not delete, replace, or normalize +a foreign reference into a different identity. + The run contract admits as many as 10,000 item snapshots. A supersession-cycle check that restarts a full predecessor walk from every item can therefore turn an otherwise linear lineage admission into quadratic work. Admission cost is part of @@ -159,7 +171,8 @@ The projection rejects: adjudication decisions; - unknown fields and non-string mapping keys; - empty, padded, Unicode-format-control-bearing, Unicode-line/paragraph-separator- - bearing, control-bearing, surrogate-bearing, or overlong opaque references; + bearing, Unicode-noncharacter-bearing, control-bearing, surrogate-bearing, or + overlong opaque references; - malformed or non-lowercase contract digests; - empty criterion sets, duplicate criterion identities, malformed category definition/digest cardinality, and incomplete substantive criterion meaning; @@ -195,8 +208,9 @@ default anchor, provider guess, or synthetic lineage edge. - anchor promotion, calibration, and linking remain separately auditable; - synthetic fixtures cannot masquerade as a released owner contract merely by using an owner-like identifier; -- opaque references cannot hide format controls or embedded line/paragraph - separators inside an otherwise machine-distinct identity; +- opaque references cannot hide format controls, embedded line/paragraph + separators, or Unicode noncharacters inside an otherwise machine-distinct + interchange identity; - large acyclic supersession chains stay bounded to linear graph-validation work; - LineageWeave can display explicit criterion, no-anchor, and no-linking limitations without inventing comparability; @@ -210,8 +224,8 @@ default anchor, provider guess, or synthetic lineage edge. - source content remains separately permissioned and cannot be recovered from this metadata-only envelope; - external adapters must map any legitimate foreign identifier containing a - rejected format control or line/paragraph separator to a separate canonical - released reference instead of passing it through unchanged; + rejected format control, line/paragraph separator, or Unicode noncharacter to a + separate canonical released reference instead of passing it through unchanged; - user interfaces must distinguish criterion meaning, provisional observation, adjudicated, calibrated, promoted-anchor, and linked states rather than displaying one generic “evaluated” badge. @@ -233,14 +247,19 @@ default anchor, provider guess, or synthetic lineage edge. 5. **Block all evaluation until anchors exist.** Rejected because governed pilot and diagnostic evidence is necessary to create and validate the first anchor corpus. -6. **Silently strip or normalize format controls or separators.** Rejected because - mutation could collapse two foreign references into an identity that the - owning system never published. Admission fails closed instead. -7. **Re-walk the complete supersession prefix from every item.** Rejected because +6. **Silently strip or normalize format controls, separators, or noncharacters.** + Rejected because mutation could collapse two foreign references into an + identity that the owning system never published. Admission fails closed + instead. +7. **Reject every Unicode `Cn` code point.** Rejected because noncharacters share + `Cn` with ordinary unassigned/reserved values. The stable noncharacter set can + be recognized exactly without making reference validity depend on which future + Unicode version has assigned the remaining `Cn` code points. +8. **Re-walk the complete supersession prefix from every item.** Rejected because the 10,000-item admission budget would permit quadratic validation work even for a valid acyclic chain. Completed-path memoization preserves the same local graph semantics without that amplification. -8. **Name a synthetic test fixture after an unreleased owner contract.** Rejected +9. **Name a synthetic test fixture after an unreleased owner contract.** Rejected because it makes a test value look like versioned cross-repository evidence. Synthetic fixtures use an explicitly synthetic identity; production adapters must provide the real released owner identity and digest. @@ -255,7 +274,9 @@ requirements, immutable collection copying, strict mapping admission, reference and digest hygiene, blueprint consistency, duplicate and resource limits, public exports, direct-construction seals, in-run supersession cycles, and long acyclic supersession chains. Reference hygiene includes zero-width and bidirectional -Unicode format controls plus U+2028 LINE SEPARATOR and U+2029 PARAGRAPH SEPARATOR. +Unicode format controls, U+2028 LINE SEPARATOR, U+2029 PARAGRAPH SEPARATOR, and +representatives of both Unicode noncharacter classes: U+FDD0, U+FFFE, and +U+10FFFF. The supersession admission regression counts set membership/add work rather than asserting a runner-specific elapsed-time threshold. That makes the complexity @@ -281,6 +302,9 @@ Cresswell, S., Gil, Y., Groth, P., Klyne, G., Lebo, T., McCusker, J., Miles, S., Myers, J., Sahoo, S., & Tilmes, C. (2013). PROV-DM: The PROV data model. World Wide Web Consortium. +Unicode Consortium. (2025). *The Unicode standard, version 17.0: Core +specification*. https://www.unicode.org/versions/Unicode17.0.0/UnicodeStandard-17.0.pdf + Unicode Consortium. (2025). *Unicode character database* (Unicode Standard Annex #44, Unicode 17.0.0, Revision 36). https://www.unicode.org/reports/tr44/