diff --git a/.github/workflows/tiny-dev-detector.yml b/.github/workflows/tiny-dev-detector.yml index 62b53e72..ebd08cdb 100644 --- a/.github/workflows/tiny-dev-detector.yml +++ b/.github/workflows/tiny-dev-detector.yml @@ -17,11 +17,15 @@ on: paths: - ".github/workflows/tiny-dev-detector.yml" - "fuckmark/experiments/tiny_dev_detector_evidence.py" + - "fuckmark/experiments/tiny_dev_e02_evidence.py" + - "fuckmark/experiments/development_calibration.py" + - "fuckmark/experiments/e02_pristine.py" - "fuckmark/tiny_dev_detector_hf.py" - "fuckmark/corpus/tiny_dev_io.py" - "fuckmark/corpus/tiny_dev_generation.py" - "fuckmark/tiny_dev_corpus_hf.py" - "tests/test_tiny_dev_detector_evidence.py" + - "tests/test_tiny_dev_e02_evidence.py" - "requirements-smoke.txt" permissions: @@ -55,12 +59,13 @@ jobs: --top-k 50 --top-p 0.95 --json artifacts/tiny-dev-corpus.json - - name: Calibrate and evaluate Mean and Weighted Mean + - name: Calibrate detectors and execute E02 run: >- python -m fuckmark.tiny_dev_detector_hf --corpus-json artifacts/tiny-dev-corpus.json --device cpu --json artifacts/tiny-dev-detector-evidence.json + --e02-json artifacts/tiny-dev-e02-evidence.json - uses: actions/upload-artifact@v4 if: always() with: diff --git a/evidence/tiny_dev/real-hf-gpt2-e02-2026-08-18.json b/evidence/tiny_dev/real-hf-gpt2-e02-2026-08-18.json new file mode 100644 index 00000000..37e17bd4 --- /dev/null +++ b/evidence/tiny_dev/real-hf-gpt2-e02-2026-08-18.json @@ -0,0 +1,96 @@ +{ + "algorithm_version": "real-tiny-dev-e02-evidence-record-v1", + "evidence_kind": "development-pristine-detectability", + "scientific_scope": "DEVELOPMENT_ONLY; Mean and Weighted Mean on the 64-token Hugging Face GPT-2 TinyDev corpus; not M3 complete because Bayesian evidence is not included; not M6 readiness or a confirmatory result", + "source": { + "repository": "byte271/FuckMark", + "head_commit": "4f4189ab9f09f1533713b2df7f67b35a5605cdfb", + "workflow_run_id": 32088935838, + "workflow_run_number": 4, + "workflow_name": "Real TinyDev Detector Evidence", + "workflow_conclusion": "success", + "artifact_id": 9307861756, + "artifact_name": "real-tiny-dev-detector-evidence", + "artifact_zip_sha256": "cd2f9b647de2f1cfea754b2ef2e4baae79abf394b7c3dae6d17beb439b288b87", + "corpus_json_sha256": "ee09527d06abc58e612f7157b69908d5abe06101865714ce13a1287d1f0d7664", + "detector_json_sha256": "086770a9d8e9e7d639bad6c04365f1a239042ae2799b729831199d704188ed99", + "e02_json_sha256": "93ec9830d5300cb3482264bbcc239a82c275af5c741643e18856e74af7e35d7b" + }, + "bindings": { + "tiny_dev_artifact_hash": "42240f72e17df1a048f44197f216a0b5f30128c26e2da93afbe92820946e4561", + "corpus_manifest_hash": "693788e76bceea43e72425b98f326bc09e4840d069eb47dacb18784ec7c2e388", + "detector_artifact_hash": "d676033f294a60a6a63fd354b1604838ff6abae2c6f8f93f847156e7ca67c219", + "e02_artifact_hash": "11bcbc69b953cc703983bfbb2a38933f63c4334263a30c457de479ce3c967ef9" + }, + "evaluation": { + "positive_count": 4, + "negative_count": 4, + "interpretability_floor": 0.8, + "families": [ + { + "detector_family": "mean", + "calibration_binding_hash": "e8ff7240a8b0fa4ea3b555eb755804f50edca4d8f53a00636ff286d775368b86", + "result_hash": "71c6903f29a30dc13f48597a8e826355bdc76e4c22d798f7b859ef0e1a5213b9", + "status": "PASS", + "auc": 1.0, + "operating_points": [ + { + "target_fpr": 0.05, + "threshold_value": 0.538888888888889, + "positive_detected_count": 4, + "negative_detected_count": 0, + "tpr": 1.0, + "evaluation_fpr": 0.0, + "tpr_ci95": [0.3976353643835254, 1.0], + "evaluation_fpr_ci95": [0.0, 0.6023646356164745] + }, + { + "target_fpr": 0.01, + "threshold_value": 0.5555555555555555, + "positive_detected_count": 4, + "negative_detected_count": 0, + "tpr": 1.0, + "evaluation_fpr": 0.0, + "tpr_ci95": [0.3976353643835254, 1.0], + "evaluation_fpr_ci95": [0.0, 0.6023646356164745] + } + ] + }, + { + "detector_family": "weighted_mean", + "calibration_binding_hash": "b8e364bf4f15575959124bfdd9d03dcd6ce9e69cf94f6ae70d3642c4929724f9", + "result_hash": "150b4a2bf541bc076e86739e3f2f5824673dc8aab3a161bfaf2c084f558ab6b4", + "status": "PASS", + "auc": 1.0, + "operating_points": [ + { + "target_fpr": 0.05, + "threshold_value": 0.5455401472350625, + "positive_detected_count": 4, + "negative_detected_count": 0, + "tpr": 1.0, + "evaluation_fpr": 0.0, + "tpr_ci95": [0.3976353643835254, 1.0], + "evaluation_fpr_ci95": [0.0, 0.6023646356164745] + }, + { + "target_fpr": 0.01, + "threshold_value": 0.5616883116883117, + "positive_detected_count": 4, + "negative_detected_count": 0, + "tpr": 1.0, + "evaluation_fpr": 0.0, + "tpr_ci95": [0.3976353643835254, 1.0], + "evaluation_fpr_ci95": [0.0, 0.6023646356164745] + } + ] + } + ] + }, + "limitations": [ + "The attack-development evaluation contains only four watermarked and four unwatermarked texts, so exact confidence intervals are wide.", + "The artifact covers Mean and Weighted Mean only; a source-compatible trained Bayesian detector remains required for M3.", + "The corpus is the 64-token Hugging Face GPT-2 TinyDev development corpus and does not establish model, tokenizer, length, key, or adapter generalization." + ], + "evidence_hash": "212f196ff8dfdf33aee93bbcc5bc76a3ddcd717914fbdd0f0063e4deda7ec741" +} diff --git a/fuckmark/experiments/tiny_dev_e02_evidence.py b/fuckmark/experiments/tiny_dev_e02_evidence.py new file mode 100644 index 00000000..517ae773 --- /dev/null +++ b/fuckmark/experiments/tiny_dev_e02_evidence.py @@ -0,0 +1,160 @@ +from __future__ import annotations + +from dataclasses import dataclass + +from .._validation import require_clean_string, require_sha256 +from ..corpus import TinyDevCorpusArtifact +from ..detectors import DetectorFamily +from ..hashing import sha256_json +from .development_calibration import DevelopmentCalibrationBinding, calibrate_tiny_dev_detector +from .e02_pristine import E02PristineDetectabilityResult, run_e02_pristine_detectability +from .tiny_dev_detector_evidence import TinyDevDetectorEvidenceArtifact, TinyDevDetectorFamilyEvidence + + +TINY_DEV_E02_FAMILY_ALGORITHM_VERSION = "tiny-dev-e02-family-evidence-v1" +TINY_DEV_E02_EVIDENCE_ALGORITHM_VERSION = "tiny-dev-e02-evidence-v1" +TINY_DEV_E02_SCIENTIFIC_STATUS = "DEVELOPMENT_ONLY" + + +class TinyDevE02EvidenceError(ValueError): + pass + + +@dataclass(frozen=True, slots=True) +class TinyDevE02FamilyEvidence: + algorithm_version: str + detector_family: DetectorFamily + detector_source_family_hash: str + calibration_binding: DevelopmentCalibrationBinding + result: E02PristineDetectabilityResult + family_hash: str + + def __post_init__(self) -> None: + require_clean_string("algorithm_version", self.algorithm_version) + if self.algorithm_version != TINY_DEV_E02_FAMILY_ALGORITHM_VERSION: + raise ValueError("unsupported TinyDev E02 family evidence version") + if self.detector_family not in (DetectorFamily.MEAN, DetectorFamily.WEIGHTED_MEAN): + raise ValueError("TinyDev E02 evidence currently supports Mean and Weighted Mean only") + require_sha256("detector_source_family_hash", self.detector_source_family_hash) + if not isinstance(self.calibration_binding, DevelopmentCalibrationBinding): + raise TypeError("calibration_binding must be a DevelopmentCalibrationBinding") + if not isinstance(self.result, E02PristineDetectabilityResult): + raise TypeError("result must be an E02PristineDetectabilityResult") + identity = self.calibration_binding.calibration_bundle.detector_identity + if identity.detector_family is not self.detector_family: + raise ValueError("E02 calibration detector family does not match family evidence") + if self.result.calibration_binding_hash != self.calibration_binding.binding_hash: + raise ValueError("E02 result does not bind the supplied development calibration") + if self.result.detector_identity_hash != identity.identity_hash: + raise ValueError("E02 result detector identity does not match development calibration") + require_sha256("family_hash", self.family_hash) + if self.family_hash != sha256_json(self._payload()): + raise ValueError("family_hash does not match TinyDev E02 family evidence") + + def _payload(self) -> dict[str, object]: + return { + "algorithm_version": self.algorithm_version, + "detector_family": self.detector_family.value, + "detector_source_family_hash": self.detector_source_family_hash, + "calibration_binding": self.calibration_binding, + "result": self.result, + } + + +@dataclass(frozen=True, slots=True) +class TinyDevE02EvidenceArtifact: + algorithm_version: str + tiny_dev_artifact_hash: str + corpus_manifest_hash: str + detector_artifact_hash: str + families: tuple[TinyDevE02FamilyEvidence, ...] + scientific_status: str + artifact_hash: str + + def __post_init__(self) -> None: + require_clean_string("algorithm_version", self.algorithm_version) + if self.algorithm_version != TINY_DEV_E02_EVIDENCE_ALGORITHM_VERSION: + raise ValueError("unsupported TinyDev E02 evidence version") + for name, value in ( + ("tiny_dev_artifact_hash", self.tiny_dev_artifact_hash), + ("corpus_manifest_hash", self.corpus_manifest_hash), + ("detector_artifact_hash", self.detector_artifact_hash), + ("artifact_hash", self.artifact_hash), + ): + require_sha256(name, value) + if not isinstance(self.families, tuple) or any( + not isinstance(value, TinyDevE02FamilyEvidence) for value in self.families + ): + raise TypeError("families must contain TinyDevE02FamilyEvidence values") + if tuple(value.detector_family for value in self.families) != ( + DetectorFamily.MEAN, + DetectorFamily.WEIGHTED_MEAN, + ): + raise ValueError("TinyDev E02 evidence must contain Mean then Weighted Mean") + if any(value.result.tiny_dev_artifact_hash != self.tiny_dev_artifact_hash for value in self.families): + raise ValueError("E02 family result does not match top-level TinyDev artifact") + require_clean_string("scientific_status", self.scientific_status) + if self.scientific_status != TINY_DEV_E02_SCIENTIFIC_STATUS: + raise ValueError("TinyDev E02 evidence must remain development-only") + if self.artifact_hash != sha256_json(self._payload()): + raise ValueError("artifact_hash does not match TinyDev E02 evidence") + + def _payload(self) -> dict[str, object]: + return { + "algorithm_version": self.algorithm_version, + "tiny_dev_artifact_hash": self.tiny_dev_artifact_hash, + "corpus_manifest_hash": self.corpus_manifest_hash, + "detector_artifact_hash": self.detector_artifact_hash, + "families": self.families, + "scientific_status": self.scientific_status, + } + + +def _build_family( + corpus: TinyDevCorpusArtifact, + source: TinyDevDetectorFamilyEvidence, +) -> TinyDevE02FamilyEvidence: + binding = calibrate_tiny_dev_detector(corpus, source.calibration_evidence) + result = run_e02_pristine_detectability(corpus, binding, source.attack_evidence) + payload = { + "algorithm_version": TINY_DEV_E02_FAMILY_ALGORITHM_VERSION, + "detector_family": source.detector_family.value, + "detector_source_family_hash": source.family_hash, + "calibration_binding": binding, + "result": result, + } + return TinyDevE02FamilyEvidence( + algorithm_version=TINY_DEV_E02_FAMILY_ALGORITHM_VERSION, + detector_family=source.detector_family, + detector_source_family_hash=source.family_hash, + calibration_binding=binding, + result=result, + family_hash=sha256_json(payload), + ) + + +def build_tiny_dev_e02_evidence( + corpus: TinyDevCorpusArtifact, + detector_evidence: TinyDevDetectorEvidenceArtifact, +) -> TinyDevE02EvidenceArtifact: + if not isinstance(corpus, TinyDevCorpusArtifact): + raise TypeError("corpus must be a TinyDevCorpusArtifact") + if not isinstance(detector_evidence, TinyDevDetectorEvidenceArtifact): + raise TypeError("detector_evidence must be a TinyDevDetectorEvidenceArtifact") + if detector_evidence.tiny_dev_artifact_hash != corpus.artifact_hash: + raise TinyDevE02EvidenceError("detector evidence does not belong to the supplied TinyDev corpus") + if detector_evidence.corpus_manifest_hash != corpus.manifest.manifest_hash: + raise TinyDevE02EvidenceError("detector evidence corpus manifest does not match the supplied TinyDev corpus") + families = tuple(_build_family(corpus, value) for value in detector_evidence.family_evidence) + payload = { + "algorithm_version": TINY_DEV_E02_EVIDENCE_ALGORITHM_VERSION, + "tiny_dev_artifact_hash": corpus.artifact_hash, + "corpus_manifest_hash": corpus.manifest.manifest_hash, + "detector_artifact_hash": detector_evidence.artifact_hash, + "families": families, + "scientific_status": TINY_DEV_E02_SCIENTIFIC_STATUS, + } + return TinyDevE02EvidenceArtifact( + **payload, + artifact_hash=sha256_json(payload), + ) diff --git a/fuckmark/tiny_dev_detector_hf.py b/fuckmark/tiny_dev_detector_hf.py index f75b9f96..8558d353 100644 --- a/fuckmark/tiny_dev_detector_hf.py +++ b/fuckmark/tiny_dev_detector_hf.py @@ -11,6 +11,7 @@ TINY_DEV_PRIMARY_FPR, build_tiny_dev_detector_evidence, ) +from .experiments.tiny_dev_e02_evidence import build_tiny_dev_e02_evidence from .hashing import sha256_json from .tiny_dev_corpus_hf import DEFAULT_KEYS, DEFAULT_NGRAM_LEN @@ -46,6 +47,11 @@ def _parser() -> argparse.ArgumentParser: type=Path, default=Path("artifacts/tiny-dev-detector-evidence.json"), ) + parser.add_argument( + "--e02-json", + type=Path, + default=Path("artifacts/tiny-dev-e02-evidence.json"), + ) parser.add_argument("--device", choices=("cpu", "cuda"), default="cpu") return parser @@ -71,10 +77,14 @@ def main(argv: list[str] | None = None) -> int: adapter, expected_watermark_config_hash=default_watermark_config_hash(), ) + e02 = build_tiny_dev_e02_evidence(corpus, evidence) args.json.parent.mkdir(parents=True, exist_ok=True) + args.e02_json.parent.mkdir(parents=True, exist_ok=True) args.json.write_text(canonical_json_text(evidence) + "\n", encoding="utf-8") + args.e02_json.write_text(canonical_json_text(e02) + "\n", encoding="utf-8") sys.stdout.write(f"artifact_hash={evidence.artifact_hash}\n") + sys.stdout.write(f"e02_artifact_hash={e02.artifact_hash}\n") sys.stdout.write(f"corpus_artifact_hash={evidence.tiny_dev_artifact_hash}\n") sys.stdout.write(f"adapter_config_hash={evidence.adapter_config_hash}\n") sys.stdout.write(f"sampling_table_hash={adapter.sampling_table_hash}\n") @@ -102,7 +112,16 @@ def main(argv: list[str] | None = None) -> int: f"{primary.pristine_baseline.sample_count} " f"status={primary.pristine_baseline.status.value}\n" ) + for family in e02.families: + primary = next(value for value in family.result.operating_points if value.target_fpr == 0.01) + sys.stdout.write( + f"e02 detector={family.detector_family.value} " + f"auc={family.result.auc:.17g} " + f"primary_tpr={primary.tpr:.17g} " + f"status={family.result.status.value}\n" + ) sys.stdout.write(f"json={args.json.as_posix()}\n") + sys.stdout.write(f"e02_json={args.e02_json.as_posix()}\n") return 0 diff --git a/tests/test_real_tiny_dev_e02_manifest.py b/tests/test_real_tiny_dev_e02_manifest.py new file mode 100644 index 00000000..f561e917 --- /dev/null +++ b/tests/test_real_tiny_dev_e02_manifest.py @@ -0,0 +1,72 @@ +import json +from pathlib import Path + +from fuckmark.hashing import sha256_json + + +_EVIDENCE_PATH = Path("evidence/tiny_dev/real-hf-gpt2-e02-2026-08-18.json") + + +def _evidence() -> dict[str, object]: + return json.loads(_EVIDENCE_PATH.read_text(encoding="utf-8")) + + +def test_real_tiny_dev_e02_manifest_is_self_hashed_and_development_only() -> None: + evidence = _evidence() + evidence_hash = evidence.pop("evidence_hash") + assert evidence_hash == sha256_json(evidence) + assert evidence["algorithm_version"] == "real-tiny-dev-e02-evidence-record-v1" + assert evidence["evidence_kind"] == "development-pristine-detectability" + assert evidence["scientific_scope"].startswith("DEVELOPMENT_ONLY;") + assert "not M3 complete" in evidence["scientific_scope"] + assert "not M6 readiness" in evidence["scientific_scope"] + + +def test_real_tiny_dev_e02_manifest_binds_successful_workflow_artifact() -> None: + evidence = _evidence() + source = evidence["source"] + bindings = evidence["bindings"] + assert source["head_commit"] == "4f4189ab9f09f1533713b2df7f67b35a5605cdfb" + assert source["workflow_run_id"] == 32088935838 + assert source["workflow_conclusion"] == "success" + assert source["artifact_id"] == 9307861756 + assert source["artifact_zip_sha256"] == ( + "cd2f9b647de2f1cfea754b2ef2e4baae79abf394b7c3dae6d17beb439b288b87" + ) + assert source["corpus_json_sha256"] == ( + "ee09527d06abc58e612f7157b69908d5abe06101865714ce13a1287d1f0d7664" + ) + assert source["detector_json_sha256"] == ( + "086770a9d8e9e7d639bad6c04365f1a239042ae2799b729831199d704188ed99" + ) + assert source["e02_json_sha256"] == ( + "93ec9830d5300cb3482264bbcc239a82c275af5c741643e18856e74af7e35d7b" + ) + assert bindings["e02_artifact_hash"] == ( + "11bcbc69b953cc703983bfbb2a38933f63c4334263a30c457de479ce3c967ef9" + ) + + +def test_real_tiny_dev_e02_manifest_records_both_mean_families_and_wide_uncertainty() -> None: + evaluation = _evidence()["evaluation"] + assert evaluation["positive_count"] == 4 + assert evaluation["negative_count"] == 4 + families = evaluation["families"] + assert [value["detector_family"] for value in families] == ["mean", "weighted_mean"] + for family in families: + assert family["status"] == "PASS" + assert family["auc"] == 1.0 + assert [value["target_fpr"] for value in family["operating_points"]] == [0.05, 0.01] + for point in family["operating_points"]: + assert point["positive_detected_count"] == 4 + assert point["negative_detected_count"] == 0 + assert point["tpr"] == 1.0 + assert point["evaluation_fpr"] == 0.0 + assert point["tpr_ci95"][0] < 0.4 + assert point["evaluation_fpr_ci95"][1] > 0.6 + + +def test_real_tiny_dev_e02_manifest_does_not_claim_bayesian_or_generalization() -> None: + limitations = _evidence()["limitations"] + assert any("Bayesian detector remains required for M3" in value for value in limitations) + assert any("does not establish model, tokenizer, length, key, or adapter generalization" in value for value in limitations) diff --git a/tests/test_tiny_dev_e02_evidence.py b/tests/test_tiny_dev_e02_evidence.py new file mode 100644 index 00000000..8cec0968 --- /dev/null +++ b/tests/test_tiny_dev_e02_evidence.py @@ -0,0 +1,64 @@ +from dataclasses import replace + +import pytest + +from fuckmark.detectors import DetectorFamily +from fuckmark.experiments.e02_pristine import E02Status +from fuckmark.experiments.tiny_dev_detector_evidence import build_tiny_dev_detector_evidence +from fuckmark.experiments.tiny_dev_e02_evidence import ( + TINY_DEV_E02_SCIENTIFIC_STATUS, + build_tiny_dev_e02_evidence, +) + +from test_tiny_dev_detector_evidence import _WATERMARK_HASH, _adapter, _artifact + + +def _e02(): + corpus = _artifact() + detector = build_tiny_dev_detector_evidence( + corpus, + _adapter(), + expected_watermark_config_hash=_WATERMARK_HASH, + ) + return corpus, detector, build_tiny_dev_e02_evidence(corpus, detector) + + +def test_tiny_dev_e02_replays_detector_rows_through_canonical_e02_gate() -> None: + corpus, detector, evidence = _e02() + assert evidence.tiny_dev_artifact_hash == corpus.artifact_hash + assert evidence.corpus_manifest_hash == corpus.manifest.manifest_hash + assert evidence.detector_artifact_hash == detector.artifact_hash + assert evidence.scientific_status == TINY_DEV_E02_SCIENTIFIC_STATUS + assert tuple(value.detector_family for value in evidence.families) == ( + DetectorFamily.MEAN, + DetectorFamily.WEIGHTED_MEAN, + ) + for source, family in zip(detector.family_evidence, evidence.families): + assert family.detector_source_family_hash == source.family_hash + assert family.calibration_binding.tiny_dev_artifact_hash == corpus.artifact_hash + assert family.calibration_binding.calibration_bundle.negative_count == 100 + assert family.calibration_binding.calibration_bundle.scope.token_track == "generation" + assert family.calibration_binding.calibration_bundle.scope.prompt_boundary_mode == "continuation_only" + assert family.result.calibration_binding_hash == family.calibration_binding.binding_hash + assert family.result.status in (E02Status.PASS, E02Status.UNDERPOWERED) + assert tuple(value.target_fpr for value in family.result.operating_points) == (0.05, 0.01) + assert family.result.result_hash + + +def test_tiny_dev_e02_is_tamper_evident() -> None: + _, _, evidence = _e02() + with pytest.raises(ValueError, match="artifact_hash"): + replace(evidence, artifact_hash="0" * 64) + with pytest.raises(ValueError, match="family_hash"): + replace(evidence.families[0], family_hash="0" * 64) + + +def test_tiny_dev_e02_keeps_source_detector_artifact_immutable() -> None: + _, detector, evidence = _e02() + for source, family in zip(detector.family_evidence, evidence.families): + assert family.detector_source_family_hash == source.family_hash + assert family.calibration_binding.calibration_bundle.bundle_hash != source.calibration_bundle.bundle_hash + assert tuple(value.value for value in (source.detector_family, family.detector_family)) == ( + source.detector_family.value, + source.detector_family.value, + )