Execute the real TinyDev E02 gate

.github/workflows/tiny-dev-detector.yml

@@ -17,11 +17,15 @@ on:
     paths:
       - ".github/workflows/tiny-dev-detector.yml"
       - "fuckmark/experiments/tiny_dev_detector_evidence.py"
+      - "fuckmark/experiments/tiny_dev_e02_evidence.py"
+      - "fuckmark/experiments/development_calibration.py"
+      - "fuckmark/experiments/e02_pristine.py"
       - "fuckmark/tiny_dev_detector_hf.py"
       - "fuckmark/corpus/tiny_dev_io.py"
       - "fuckmark/corpus/tiny_dev_generation.py"
       - "fuckmark/tiny_dev_corpus_hf.py"
       - "tests/test_tiny_dev_detector_evidence.py"
+      - "tests/test_tiny_dev_e02_evidence.py"
       - "requirements-smoke.txt"
 
 permissions:
@@ -55,12 +59,13 @@ jobs:
           --top-k 50
           --top-p 0.95
           --json artifacts/tiny-dev-corpus.json
-      - name: Calibrate and evaluate Mean and Weighted Mean
+      - name: Calibrate detectors and execute E02
         run: >-
           python -m fuckmark.tiny_dev_detector_hf
           --corpus-json artifacts/tiny-dev-corpus.json
           --device cpu
           --json artifacts/tiny-dev-detector-evidence.json
+          --e02-json artifacts/tiny-dev-e02-evidence.json
       - uses: actions/upload-artifact@v4
         if: always()
         with:

evidence/tiny_dev/real-hf-gpt2-e02-2026-08-18.json

@@ -0,0 +1,96 @@
+{
+  "algorithm_version": "real-tiny-dev-e02-evidence-record-v1",
+  "evidence_kind": "development-pristine-detectability",
+  "scientific_scope": "DEVELOPMENT_ONLY; Mean and Weighted Mean on the 64-token Hugging Face GPT-2 TinyDev corpus; not M3 complete because Bayesian evidence is not included; not M6 readiness or a confirmatory result",
+  "source": {
+    "repository": "byte271/FuckMark",
+    "head_commit": "4f4189ab9f09f1533713b2df7f67b35a5605cdfb",
+    "workflow_run_id": 32088935838,
+    "workflow_run_number": 4,
+    "workflow_name": "Real TinyDev Detector Evidence",
+    "workflow_conclusion": "success",
+    "artifact_id": 9307861756,
+    "artifact_name": "real-tiny-dev-detector-evidence",
+    "artifact_zip_sha256": "cd2f9b647de2f1cfea754b2ef2e4baae79abf394b7c3dae6d17beb439b288b87",
+    "corpus_json_sha256": "ee09527d06abc58e612f7157b69908d5abe06101865714ce13a1287d1f0d7664",
+    "detector_json_sha256": "086770a9d8e9e7d639bad6c04365f1a239042ae2799b729831199d704188ed99",
+    "e02_json_sha256": "93ec9830d5300cb3482264bbcc239a82c275af5c741643e18856e74af7e35d7b"
+  },
+  "bindings": {
+    "tiny_dev_artifact_hash": "42240f72e17df1a048f44197f216a0b5f30128c26e2da93afbe92820946e4561",
+    "corpus_manifest_hash": "693788e76bceea43e72425b98f326bc09e4840d069eb47dacb18784ec7c2e388",
+    "detector_artifact_hash": "d676033f294a60a6a63fd354b1604838ff6abae2c6f8f93f847156e7ca67c219",
+    "e02_artifact_hash": "11bcbc69b953cc703983bfbb2a38933f63c4334263a30c457de479ce3c967ef9"
+  },
+  "evaluation": {
+    "positive_count": 4,
+    "negative_count": 4,
+    "interpretability_floor": 0.8,
+    "families": [
+      {
+        "detector_family": "mean",
+        "calibration_binding_hash": "e8ff7240a8b0fa4ea3b555eb755804f50edca4d8f53a00636ff286d775368b86",
+        "result_hash": "71c6903f29a30dc13f48597a8e826355bdc76e4c22d798f7b859ef0e1a5213b9",
+        "status": "PASS",
+        "auc": 1.0,
+        "operating_points": [
+          {
+            "target_fpr": 0.05,
+            "threshold_value": 0.538888888888889,
+            "positive_detected_count": 4,
+            "negative_detected_count": 0,
+            "tpr": 1.0,
+            "evaluation_fpr": 0.0,
+            "tpr_ci95": [0.3976353643835254, 1.0],
+            "evaluation_fpr_ci95": [0.0, 0.6023646356164745]
+          },
+          {
+            "target_fpr": 0.01,
+            "threshold_value": 0.5555555555555555,
+            "positive_detected_count": 4,
+            "negative_detected_count": 0,
+            "tpr": 1.0,
+            "evaluation_fpr": 0.0,
+            "tpr_ci95": [0.3976353643835254, 1.0],
+            "evaluation_fpr_ci95": [0.0, 0.6023646356164745]
+          }
+        ]
+      },
+      {
+        "detector_family": "weighted_mean",
+        "calibration_binding_hash": "b8e364bf4f15575959124bfdd9d03dcd6ce9e69cf94f6ae70d3642c4929724f9",
+        "result_hash": "150b4a2bf541bc076e86739e3f2f5824673dc8aab3a161bfaf2c084f558ab6b4",
+        "status": "PASS",
+        "auc": 1.0,
+        "operating_points": [
+          {
+            "target_fpr": 0.05,
+            "threshold_value": 0.5455401472350625,
+            "positive_detected_count": 4,
+            "negative_detected_count": 0,
+            "tpr": 1.0,
+            "evaluation_fpr": 0.0,
+            "tpr_ci95": [0.3976353643835254, 1.0],
+            "evaluation_fpr_ci95": [0.0, 0.6023646356164745]
+          },
+          {
+            "target_fpr": 0.01,
+            "threshold_value": 0.5616883116883117,
+            "positive_detected_count": 4,
+            "negative_detected_count": 0,
+            "tpr": 1.0,
+            "evaluation_fpr": 0.0,
+            "tpr_ci95": [0.3976353643835254, 1.0],
+            "evaluation_fpr_ci95": [0.0, 0.6023646356164745]
+          }
+        ]
+      }
+    ]
+  },
+  "limitations": [
+    "The attack-development evaluation contains only four watermarked and four unwatermarked texts, so exact confidence intervals are wide.",
+    "The artifact covers Mean and Weighted Mean only; a source-compatible trained Bayesian detector remains required for M3.",
+    "The corpus is the 64-token Hugging Face GPT-2 TinyDev development corpus and does not establish model, tokenizer, length, key, or adapter generalization."
+  ],
+  "evidence_hash": "212f196ff8dfdf33aee93bbcc5bc76a3ddcd717914fbdd0f0063e4deda7ec741"
+}

fuckmark/experiments/tiny_dev_e02_evidence.py

@@ -0,0 +1,160 @@
+from __future__ import annotations
+
+from dataclasses import dataclass
+
+from .._validation import require_clean_string, require_sha256
+from ..corpus import TinyDevCorpusArtifact
+from ..detectors import DetectorFamily
+from ..hashing import sha256_json
+from .development_calibration import DevelopmentCalibrationBinding, calibrate_tiny_dev_detector
+from .e02_pristine import E02PristineDetectabilityResult, run_e02_pristine_detectability
+from .tiny_dev_detector_evidence import TinyDevDetectorEvidenceArtifact, TinyDevDetectorFamilyEvidence
+
+
+TINY_DEV_E02_FAMILY_ALGORITHM_VERSION = "tiny-dev-e02-family-evidence-v1"
+TINY_DEV_E02_EVIDENCE_ALGORITHM_VERSION = "tiny-dev-e02-evidence-v1"
+TINY_DEV_E02_SCIENTIFIC_STATUS = "DEVELOPMENT_ONLY"
+
+
+class TinyDevE02EvidenceError(ValueError):
+    pass
+
+
+@dataclass(frozen=True, slots=True)
+class TinyDevE02FamilyEvidence:
+    algorithm_version: str
+    detector_family: DetectorFamily
+    detector_source_family_hash: str
+    calibration_binding: DevelopmentCalibrationBinding
+    result: E02PristineDetectabilityResult
+    family_hash: str
+
+    def __post_init__(self) -> None:
+        require_clean_string("algorithm_version", self.algorithm_version)
+        if self.algorithm_version != TINY_DEV_E02_FAMILY_ALGORITHM_VERSION:
+            raise ValueError("unsupported TinyDev E02 family evidence version")
+        if self.detector_family not in (DetectorFamily.MEAN, DetectorFamily.WEIGHTED_MEAN):
+            raise ValueError("TinyDev E02 evidence currently supports Mean and Weighted Mean only")
+        require_sha256("detector_source_family_hash", self.detector_source_family_hash)
+        if not isinstance(self.calibration_binding, DevelopmentCalibrationBinding):
+            raise TypeError("calibration_binding must be a DevelopmentCalibrationBinding")
+        if not isinstance(self.result, E02PristineDetectabilityResult):
+            raise TypeError("result must be an E02PristineDetectabilityResult")
+        identity = self.calibration_binding.calibration_bundle.detector_identity
+        if identity.detector_family is not self.detector_family:
+            raise ValueError("E02 calibration detector family does not match family evidence")
+        if self.result.calibration_binding_hash != self.calibration_binding.binding_hash:
+            raise ValueError("E02 result does not bind the supplied development calibration")
+        if self.result.detector_identity_hash != identity.identity_hash:
+            raise ValueError("E02 result detector identity does not match development calibration")
+        require_sha256("family_hash", self.family_hash)
+        if self.family_hash != sha256_json(self._payload()):
+            raise ValueError("family_hash does not match TinyDev E02 family evidence")
+
+    def _payload(self) -> dict[str, object]:
+        return {
+            "algorithm_version": self.algorithm_version,
+            "detector_family": self.detector_family.value,
+            "detector_source_family_hash": self.detector_source_family_hash,
+            "calibration_binding": self.calibration_binding,
+            "result": self.result,
+        }
+
+
+@dataclass(frozen=True, slots=True)
+class TinyDevE02EvidenceArtifact:
+    algorithm_version: str
+    tiny_dev_artifact_hash: str
+    corpus_manifest_hash: str
+    detector_artifact_hash: str
+    families: tuple[TinyDevE02FamilyEvidence, ...]
+    scientific_status: str
+    artifact_hash: str
+
+    def __post_init__(self) -> None:
+        require_clean_string("algorithm_version", self.algorithm_version)
+        if self.algorithm_version != TINY_DEV_E02_EVIDENCE_ALGORITHM_VERSION:
+            raise ValueError("unsupported TinyDev E02 evidence version")
+        for name, value in (
+            ("tiny_dev_artifact_hash", self.tiny_dev_artifact_hash),
+            ("corpus_manifest_hash", self.corpus_manifest_hash),
+            ("detector_artifact_hash", self.detector_artifact_hash),
+            ("artifact_hash", self.artifact_hash),
+        ):
+            require_sha256(name, value)
+        if not isinstance(self.families, tuple) or any(
+            not isinstance(value, TinyDevE02FamilyEvidence) for value in self.families
+        ):
+            raise TypeError("families must contain TinyDevE02FamilyEvidence values")
+        if tuple(value.detector_family for value in self.families) != (
+            DetectorFamily.MEAN,
+            DetectorFamily.WEIGHTED_MEAN,
+        ):
+            raise ValueError("TinyDev E02 evidence must contain Mean then Weighted Mean")
+        if any(value.result.tiny_dev_artifact_hash != self.tiny_dev_artifact_hash for value in self.families):
+            raise ValueError("E02 family result does not match top-level TinyDev artifact")
+        require_clean_string("scientific_status", self.scientific_status)
+        if self.scientific_status != TINY_DEV_E02_SCIENTIFIC_STATUS:
+            raise ValueError("TinyDev E02 evidence must remain development-only")
+        if self.artifact_hash != sha256_json(self._payload()):
+            raise ValueError("artifact_hash does not match TinyDev E02 evidence")
+
+    def _payload(self) -> dict[str, object]:
+        return {
+            "algorithm_version": self.algorithm_version,
+            "tiny_dev_artifact_hash": self.tiny_dev_artifact_hash,
+            "corpus_manifest_hash": self.corpus_manifest_hash,
+            "detector_artifact_hash": self.detector_artifact_hash,
+            "families": self.families,
+            "scientific_status": self.scientific_status,
+        }
+
+
+def _build_family(
+    corpus: TinyDevCorpusArtifact,
+    source: TinyDevDetectorFamilyEvidence,
+) -> TinyDevE02FamilyEvidence:
+    binding = calibrate_tiny_dev_detector(corpus, source.calibration_evidence)
+    result = run_e02_pristine_detectability(corpus, binding, source.attack_evidence)
+    payload = {
+        "algorithm_version": TINY_DEV_E02_FAMILY_ALGORITHM_VERSION,
+        "detector_family": source.detector_family.value,
+        "detector_source_family_hash": source.family_hash,
+        "calibration_binding": binding,
+        "result": result,
+    }
+    return TinyDevE02FamilyEvidence(
+        algorithm_version=TINY_DEV_E02_FAMILY_ALGORITHM_VERSION,
+        detector_family=source.detector_family,
+        detector_source_family_hash=source.family_hash,
+        calibration_binding=binding,
+        result=result,
+        family_hash=sha256_json(payload),
+    )
+
+
+def build_tiny_dev_e02_evidence(
+    corpus: TinyDevCorpusArtifact,
+    detector_evidence: TinyDevDetectorEvidenceArtifact,
+) -> TinyDevE02EvidenceArtifact:
+    if not isinstance(corpus, TinyDevCorpusArtifact):
+        raise TypeError("corpus must be a TinyDevCorpusArtifact")
+    if not isinstance(detector_evidence, TinyDevDetectorEvidenceArtifact):
+        raise TypeError("detector_evidence must be a TinyDevDetectorEvidenceArtifact")
+    if detector_evidence.tiny_dev_artifact_hash != corpus.artifact_hash:
+        raise TinyDevE02EvidenceError("detector evidence does not belong to the supplied TinyDev corpus")
+    if detector_evidence.corpus_manifest_hash != corpus.manifest.manifest_hash:
+        raise TinyDevE02EvidenceError("detector evidence corpus manifest does not match the supplied TinyDev corpus")
+    families = tuple(_build_family(corpus, value) for value in detector_evidence.family_evidence)
+    payload = {
+        "algorithm_version": TINY_DEV_E02_EVIDENCE_ALGORITHM_VERSION,
+        "tiny_dev_artifact_hash": corpus.artifact_hash,
+        "corpus_manifest_hash": corpus.manifest.manifest_hash,
+        "detector_artifact_hash": detector_evidence.artifact_hash,
+        "families": families,
+        "scientific_status": TINY_DEV_E02_SCIENTIFIC_STATUS,
+    }
+    return TinyDevE02EvidenceArtifact(
+        **payload,
+        artifact_hash=sha256_json(payload),
+    )

fuckmark/tiny_dev_detector_hf.py

@@ -11,6 +11,7 @@
     TINY_DEV_PRIMARY_FPR,
     build_tiny_dev_detector_evidence,
 )
+from .experiments.tiny_dev_e02_evidence import build_tiny_dev_e02_evidence
 from .hashing import sha256_json
 from .tiny_dev_corpus_hf import DEFAULT_KEYS, DEFAULT_NGRAM_LEN
 
@@ -46,6 +47,11 @@ def _parser() -> argparse.ArgumentParser:
         type=Path,
         default=Path("artifacts/tiny-dev-detector-evidence.json"),
     )
+    parser.add_argument(
+        "--e02-json",
+        type=Path,
+        default=Path("artifacts/tiny-dev-e02-evidence.json"),
+    )
     parser.add_argument("--device", choices=("cpu", "cuda"), default="cpu")
     return parser
 
@@ -71,10 +77,14 @@ def main(argv: list[str] | None = None) -> int:
         adapter,
         expected_watermark_config_hash=default_watermark_config_hash(),
     )
+    e02 = build_tiny_dev_e02_evidence(corpus, evidence)
     args.json.parent.mkdir(parents=True, exist_ok=True)
+    args.e02_json.parent.mkdir(parents=True, exist_ok=True)
     args.json.write_text(canonical_json_text(evidence) + "\n", encoding="utf-8")
+    args.e02_json.write_text(canonical_json_text(e02) + "\n", encoding="utf-8")
 
     sys.stdout.write(f"artifact_hash={evidence.artifact_hash}\n")
+    sys.stdout.write(f"e02_artifact_hash={e02.artifact_hash}\n")
     sys.stdout.write(f"corpus_artifact_hash={evidence.tiny_dev_artifact_hash}\n")
     sys.stdout.write(f"adapter_config_hash={evidence.adapter_config_hash}\n")
     sys.stdout.write(f"sampling_table_hash={adapter.sampling_table_hash}\n")
@@ -102,7 +112,16 @@ def main(argv: list[str] | None = None) -> int:
             f"{primary.pristine_baseline.sample_count} "
             f"status={primary.pristine_baseline.status.value}\n"
         )
+    for family in e02.families:
+        primary = next(value for value in family.result.operating_points if value.target_fpr == 0.01)
+        sys.stdout.write(
+            f"e02 detector={family.detector_family.value} "
+            f"auc={family.result.auc:.17g} "
+            f"primary_tpr={primary.tpr:.17g} "
+            f"status={family.result.status.value}\n"
+        )
     sys.stdout.write(f"json={args.json.as_posix()}\n")
+    sys.stdout.write(f"e02_json={args.e02_json.as_posix()}\n")
     return 0