From 24188863d9fa8c6f5de7acc1536af23c794caf3f Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Wed, 12 Aug 2026 06:44:20 +0900 Subject: [PATCH 1/5] test(rubric): require explicit DIF applicability evidence --- tests/test_item_bank_dif_applicability.py | 10 ++++++++++ 1 file changed, 10 insertions(+) create mode 100644 tests/test_item_bank_dif_applicability.py diff --git a/tests/test_item_bank_dif_applicability.py b/tests/test_item_bank_dif_applicability.py new file mode 100644 index 000000000..220c2a8f8 --- /dev/null +++ b/tests/test_item_bank_dif_applicability.py @@ -0,0 +1,10 @@ +"""Fail-first contract for scientifically inapplicable DIF evidence.""" + +from fast_mlsirm.rubric.item_bank import ItemBankEvidenceKind + + +def test_item_bank_evidence_domain_can_represent_dif_not_applicable() -> None: + """Calibration must not require fabricated DIF when no comparison design exists.""" + evidence_kinds = {kind.value for kind in ItemBankEvidenceKind} + + assert "dif_not_applicable" in evidence_kinds From 989fb53040a47bb126bde866a8ab0f16efea2db9 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Wed, 12 Aug 2026 06:47:45 +0900 Subject: [PATCH 2/5] fix(rubric): distinguish inapplicable DIF evidence --- python/fast_mlsirm/rubric/item_bank.py | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/python/fast_mlsirm/rubric/item_bank.py b/python/fast_mlsirm/rubric/item_bank.py index f1077a122..a92bd87c7 100644 --- a/python/fast_mlsirm/rubric/item_bank.py +++ b/python/fast_mlsirm/rubric/item_bank.py @@ -50,6 +50,7 @@ class ItemBankEvidenceKind(str, Enum): CALIBRATION = "calibration" ITEM_FIT = "item_fit" DIF = "dif" + DIF_NOT_APPLICABLE = "dif_not_applicable" ITEM_INFORMATION = "item_information" LINKING = "linking" EXPOSURE = "exposure" @@ -504,9 +505,14 @@ def _missing_required_kinds( required = { ItemBankEvidenceKind.CALIBRATION, ItemBankEvidenceKind.ITEM_FIT, - ItemBankEvidenceKind.DIF, ItemBankEvidenceKind.ITEM_INFORMATION, } + missing = [kind.value for kind in required - supplied_kinds] + if not supplied_kinds.intersection( + {ItemBankEvidenceKind.DIF, ItemBankEvidenceKind.DIF_NOT_APPLICABLE} + ): + missing.append("dif_or_dif_not_applicable") + return tuple(sorted(missing)) elif target_state is ItemBankLifecycleState.APPROVED: required = {ItemBankEvidenceKind.APPROVAL} elif ( @@ -561,6 +567,16 @@ def transition_item_bank_record( error_type=ItemBankLifecycleError, ) supplied_kinds = {reference.evidence_kind for reference in additions} + if ( + target is ItemBankLifecycleState.CALIBRATED + and ItemBankEvidenceKind.DIF in supplied_kinds + and ItemBankEvidenceKind.DIF_NOT_APPLICABLE in supplied_kinds + ): + raise ItemBankLifecycleError( + "conflicting_dif_applicability", + "$.evidence_references", + "calibration requires exactly one DIF applicability evidence class", + ) missing = _missing_required_kinds( current.lifecycle_state, target, From cb060b7496a6a8f890d9ed0127c6ace974b42a65 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Wed, 12 Aug 2026 06:48:22 +0900 Subject: [PATCH 3/5] test(rubric): cover DIF applicability transition semantics --- tests/test_item_bank_dif_applicability.py | 130 +++++++++++++++++++++- 1 file changed, 128 insertions(+), 2 deletions(-) diff --git a/tests/test_item_bank_dif_applicability.py b/tests/test_item_bank_dif_applicability.py index 220c2a8f8..48248857a 100644 --- a/tests/test_item_bank_dif_applicability.py +++ b/tests/test_item_bank_dif_applicability.py @@ -1,6 +1,52 @@ -"""Fail-first contract for scientifically inapplicable DIF evidence.""" +"""Contracts for scientifically explicit DIF applicability evidence.""" -from fast_mlsirm.rubric.item_bank import ItemBankEvidenceKind +from __future__ import annotations + +from pathlib import Path +import runpy + +import pytest + +from fast_mlsirm.rubric.item_bank import ( + ItemBankEvidenceKind, + ItemBankEvidenceReference, + ItemBankLifecycleError, + ItemBankLifecycleState, + build_item_bank_pilot_record, + transition_item_bank_record, +) + +_LIFECYCLE_FIXTURES = runpy.run_path( + str(Path(__file__).with_name("test_rubric_item_bank_lifecycle.py")) +) + + +def _evidence( + kind: ItemBankEvidenceKind, + suffix: str, + fingerprint_character: str, +) -> ItemBankEvidenceReference: + """Return one bounded source-text-free lifecycle evidence identity.""" + return ItemBankEvidenceReference( + evidence_kind=kind, + evidence_id=f"{suffix}_evidence", + evidence_fingerprint=fingerprint_character * 64, + ) + + +def _pilot_record(): + """Return one verified pilot lifecycle record from canonical test fixtures.""" + pilot = _LIFECYCLE_FIXTURES["_pilot_record"]() + return build_item_bank_pilot_record(pilot, item_version="1.0.0") + + +def _base_calibration_evidence() -> tuple[ItemBankEvidenceReference, ...]: + """Return calibration evidence common to both DIF applicability branches.""" + return ( + _evidence(ItemBankEvidenceKind.CALIBRATION, "calibration", "a"), + _evidence(ItemBankEvidenceKind.ITEM_FIT, "item_fit", "b"), + _evidence(ItemBankEvidenceKind.ITEM_INFORMATION, "item_information", "c"), + ) def test_item_bank_evidence_domain_can_represent_dif_not_applicable() -> None: @@ -8,3 +54,83 @@ def test_item_bank_evidence_domain_can_represent_dif_not_applicable() -> None: evidence_kinds = {kind.value for kind in ItemBankEvidenceKind} assert "dif_not_applicable" in evidence_kinds + + +def test_calibration_accepts_governed_dif_not_applicable_evidence() -> None: + """A governed N/A determination can satisfy only the DIF applicability gate.""" + calibrated = transition_item_bank_record( + _pilot_record(), + ItemBankLifecycleState.CALIBRATED, + evidence_references=( + *_base_calibration_evidence(), + _evidence( + ItemBankEvidenceKind.DIF_NOT_APPLICABLE, + "dif_not_applicable", + "d", + ), + ), + transition_reason_id="calibration_completed", + ) + + assert calibrated.lifecycle_state is ItemBankLifecycleState.CALIBRATED + assert ItemBankEvidenceKind.DIF_NOT_APPLICABLE in { + reference.evidence_kind for reference in calibrated.evidence_references + } + assert ItemBankEvidenceKind.DIF not in { + reference.evidence_kind for reference in calibrated.evidence_references + } + + +def test_calibration_still_requires_an_explicit_dif_applicability_decision() -> None: + """Omitting both measured DIF and governed N/A evidence fails closed.""" + with pytest.raises(ItemBankLifecycleError) as caught: + transition_item_bank_record( + _pilot_record(), + ItemBankLifecycleState.CALIBRATED, + evidence_references=_base_calibration_evidence(), + transition_reason_id="calibration_completed", + ) + + assert caught.value.code == "missing_transition_evidence" + assert caught.value.path == "$.evidence_references" + assert "dif_or_dif_not_applicable" in caught.value.message + + +def test_calibration_rejects_conflicting_dif_applicability_evidence() -> None: + """A calibration cannot claim both measured DIF and DIF-not-applicable.""" + with pytest.raises(ItemBankLifecycleError) as caught: + transition_item_bank_record( + _pilot_record(), + ItemBankLifecycleState.CALIBRATED, + evidence_references=( + *_base_calibration_evidence(), + _evidence(ItemBankEvidenceKind.DIF, "dif", "d"), + _evidence( + ItemBankEvidenceKind.DIF_NOT_APPLICABLE, + "dif_not_applicable", + "e", + ), + ), + transition_reason_id="calibration_completed", + ) + + assert caught.value.code == "conflicting_dif_applicability" + assert caught.value.path == "$.evidence_references" + + +def test_existing_measured_dif_calibration_path_is_unchanged() -> None: + """A comparison design with real DIF evidence retains the prior contract.""" + calibrated = transition_item_bank_record( + _pilot_record(), + ItemBankLifecycleState.CALIBRATED, + evidence_references=( + *_base_calibration_evidence(), + _evidence(ItemBankEvidenceKind.DIF, "dif", "d"), + ), + transition_reason_id="calibration_completed", + ) + + assert calibrated.lifecycle_state is ItemBankLifecycleState.CALIBRATED + assert ItemBankEvidenceKind.DIF in { + reference.evidence_kind for reference in calibrated.evidence_references + } From 46c9a9298684e202dbc86f21f3f3130a60141afd Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Wed, 12 Aug 2026 06:48:55 +0900 Subject: [PATCH 4/5] docs(rubric): clarify DIF applicability gate --- docs/doctoring/governed_item_bank_lifecycle.md | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/docs/doctoring/governed_item_bank_lifecycle.md b/docs/doctoring/governed_item_bank_lifecycle.md index ccbe94f43..ca7853447 100644 --- a/docs/doctoring/governed_item_bank_lifecycle.md +++ b/docs/doctoring/governed_item_bank_lifecycle.md @@ -34,7 +34,14 @@ A lifecycle evidence reference stores only: The referenced artifact may contain Rust-backed calibration, fit, DIF, information, linking, exposure, drift, approval, suspension, or retirement evidence. The lifecycle layer does not reproduce or reinterpret those calculations. -The initial calibration gate requires references for calibration, item fit, DIF, and item information. This requirement proves only that the governed evidence classes are present; later slices must validate their schemas, estimator identity, uncertainty, population/design scope, parameter recovery, and decision thresholds. +The initial calibration gate always requires calibration, item-fit, and item-information evidence plus **exactly one DIF-applicability branch**: + +- `dif` when the governed calibration design includes a comparison/grouping structure for which DIF evidence is applicable; or +- `dif_not_applicable` when a separate governed evidence artifact establishes that the calibration design has no applicable DIF comparison. + +Supplying neither branch fails closed; supplying both fails closed as conflicting applicability evidence. `dif_not_applicable` is not a fairness, invariance, validity, or cross-population equivalence finding. It only prevents callers from fabricating a DIF result for a design in which DIF is scientifically inapplicable. Any approved use, population, language, domain, release, or score-comparability claim that requires DIF/invariance evidence must still provide the applicable evidence before that claim is made. + +These requirements prove only that the governed evidence classes are present; later slices must validate their schemas, estimator identity, uncertainty, population/design scope, parameter recovery, and decision thresholds. Approval is use-specific. An item cannot enter `approved` without at least one descriptive approved-use identifier, and an `active` record cannot erase that scope. @@ -50,7 +57,7 @@ A low-information safety-critical criterion may therefore remain operationally r ## Suspension and retirement -Suspension is reversible but requires both a governance suspension record and evidence of a measured concern such as drift or DIF. Reactivation requires new approval plus new drift evidence. Retirement is terminal for new operational use, while historical evidence remains content-addressed for audit reconstruction. +Suspension is reversible but requires both a governance suspension record and evidence of a measured concern such as drift or DIF. A `dif_not_applicable` calibration determination is not evidence of a later operational concern and therefore does not satisfy the suspension gate. Reactivation requires new approval plus new drift evidence. Retirement is terminal for new operational use, while historical evidence remains content-addressed for audit reconstruction. The contract does not authorize physical deletion, retention exceptions, or erasure behavior. Those controls belong to the downstream persistence and governance system. @@ -70,6 +77,7 @@ The implementation requires deterministic tests for: - direct-construction refusal; - transition graph enforcement; - required evidence classes; +- mutually exclusive measured-DIF versus governed-DIF-not-applicable calibration evidence; - use-specific approval; - cumulative evidence and previous-record linkage; - evidence-order invariance; From 5641d547e7fca90da0d85b22beef5216eecfd181 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Wed, 12 Aug 2026 06:49:08 +0900 Subject: [PATCH 5/5] docs(changelog): record DIF applicability evidence --- docs/changelog.d/609-governed-item-bank-lifecycle.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/changelog.d/609-governed-item-bank-lifecycle.md b/docs/changelog.d/609-governed-item-bank-lifecycle.md index dc741f44a..6528feee0 100644 --- a/docs/changelog.d/609-governed-item-bank-lifecycle.md +++ b/docs/changelog.d/609-governed-item-bank-lifecycle.md @@ -2,6 +2,6 @@ ## Added -- Add a factory-sealed, content-addressed post-pilot item-bank lifecycle that requires exact calibration, item-fit, DIF, information, approval, drift, suspension, and retirement evidence before an item can advance through `piloting`, `calibrated`, `approved`, `active`, `suspended`, reactivated, or terminal `retired` states. +- Add a factory-sealed, content-addressed post-pilot item-bank lifecycle that requires exact calibration, item-fit, item-information, explicit DIF-applicability, approval, drift, suspension, and retirement evidence before an item can advance through `piloting`, `calibrated`, `approved`, `active`, `suspended`, reactivated, or terminal `retired` states. Calibration accepts either measured `dif` evidence or a separately governed `dif_not_applicable` determination, never both and never neither. - Preserve policy criticality independently of psychometric discrimination, require use-specific approval, link every successor to the exact previous record fingerprint, and retain only source-text-free evidence identities while leaving numerical calibration and item-bank arithmetic Rust-owned. - Keep tenancy, authorization, identity mapping, persistence, encryption, retention, deletion, human governance, provider SDKs, new estimators, version bumps, and releases outside this reusable-core slice.