From 4b14f65a6d6672cd6ddcd96b55c2842fd6549d2e Mon Sep 17 00:00:00 2001 From: Byte271 Date: Mon, 17 Aug 2026 21:08:52 -0400 Subject: [PATCH 1/7] Add real TinyDev detector evidence builder --- .../experiments/tiny_dev_detector_evidence.py | 519 ++++++++++++++++++ 1 file changed, 519 insertions(+) create mode 100644 fuckmark/experiments/tiny_dev_detector_evidence.py diff --git a/fuckmark/experiments/tiny_dev_detector_evidence.py b/fuckmark/experiments/tiny_dev_detector_evidence.py new file mode 100644 index 00000000..75c78bd2 --- /dev/null +++ b/fuckmark/experiments/tiny_dev_detector_evidence.py @@ -0,0 +1,519 @@ +from __future__ import annotations + +import math +from dataclasses import dataclass + +from .._validation import require_clean_string, require_int, require_sha256 +from ..adapters.base import WatermarkAdapter +from ..corpus import CorpusSplit, TinyDevCorpusArtifact, WatermarkLabel +from ..detectors import ( + BaselineStatus, + CalibratedDetectorResult, + CalibrationBundle, + CalibrationScope, + DetectorFamily, + ExactBinomialInterval, + PristineBaselineSummary, + UncalibratedDetectorEvidence, + apply_calibration, + calibrate_detector, + evaluate_pristine_baseline, + exact_binomial_interval, + mean_evidence, + weighted_mean_evidence, +) +from ..detectors.calibration_identity import DetectorCalibrationIdentity +from ..hashing import sha256_json +from ..native_observations import build_native_observations + + +TINY_DEV_DETECTOR_EVIDENCE_ALGORITHM_VERSION = "tiny-dev-detector-evidence-v1" +TINY_DEV_THRESHOLD_EVALUATION_ALGORITHM_VERSION = "tiny-dev-threshold-evaluation-v1" +TINY_DEV_DETECTOR_FAMILY_ALGORITHM_VERSION = "tiny-dev-detector-family-evidence-v1" +TINY_DEV_HEADLINE_FPRS = (0.05, 0.01) +TINY_DEV_PRIMARY_FPR = 0.01 +TINY_DEV_BASELINE_INTERPRETABILITY_FLOOR = 0.80 +TINY_DEV_TOKEN_TRACK = "original-generation-token-ids" +TINY_DEV_PROMPT_BOUNDARY_MODE = "continuation-only" +TINY_DEV_DETECTOR_STATUS = "DEVELOPMENT_ONLY" + + +class TinyDevDetectorEvidenceError(ValueError): + pass + + +def _require_probability(name: str, value: float) -> float: + if isinstance(value, bool) or not isinstance(value, (int, float)): + raise TypeError(f"{name} must be a real number") + number = float(value) + if not math.isfinite(number) or number < 0.0 or number > 1.0: + raise ValueError(f"{name} must be finite and in [0, 1]") + return number + + +@dataclass(frozen=True, slots=True) +class TinyDevThresholdEvaluation: + algorithm_version: str + target_fpr: float + threshold_hash: str + threshold_value: float + achieved_calibration_fpr: float + calibration_fpr_interval: ExactBinomialInterval + positive_results: tuple[CalibratedDetectorResult, ...] + negative_results: tuple[CalibratedDetectorResult, ...] + pristine_baseline: PristineBaselineSummary + attack_negative_detected_count: int + attack_negative_fpr: float + attack_negative_fpr_interval: ExactBinomialInterval + evaluation_hash: str + + def __post_init__(self) -> None: + require_clean_string("algorithm_version", self.algorithm_version) + if self.algorithm_version != TINY_DEV_THRESHOLD_EVALUATION_ALGORITHM_VERSION: + raise ValueError("unsupported TinyDev threshold evaluation version") + target = _require_probability("target_fpr", self.target_fpr) + if target not in TINY_DEV_HEADLINE_FPRS: + raise ValueError("target_fpr must be a frozen TinyDev headline FPR") + object.__setattr__(self, "target_fpr", target) + require_sha256("threshold_hash", self.threshold_hash) + threshold_value = _require_probability("threshold_value", self.threshold_value) + object.__setattr__(self, "threshold_value", threshold_value) + achieved = _require_probability("achieved_calibration_fpr", self.achieved_calibration_fpr) + object.__setattr__(self, "achieved_calibration_fpr", achieved) + if not isinstance(self.calibration_fpr_interval, ExactBinomialInterval): + raise TypeError("calibration_fpr_interval must be an ExactBinomialInterval") + if not isinstance(self.positive_results, tuple) or not isinstance(self.negative_results, tuple): + raise TypeError("threshold results must be tuples") + if len(self.positive_results) != 4 or len(self.negative_results) != 4: + raise ValueError("TinyDev threshold evaluation requires four positive and four negative attack rows") + if any(not isinstance(value, CalibratedDetectorResult) for value in (*self.positive_results, *self.negative_results)): + raise TypeError("threshold result tuples must contain CalibratedDetectorResult values") + all_results = (*self.positive_results, *self.negative_results) + if len({value.sample_id for value in all_results}) != 8: + raise ValueError("TinyDev attack threshold results must use eight unique samples") + for value in all_results: + if value.target_fpr != target: + raise ValueError("calibrated attack result target FPR does not match threshold evaluation") + if value.threshold_hash != self.threshold_hash: + raise ValueError("calibrated attack result threshold hash does not match threshold evaluation") + if value.threshold_value != threshold_value: + raise ValueError("calibrated attack result threshold value does not match threshold evaluation") + if value.achieved_calibration_fpr != achieved: + raise ValueError("calibrated attack result achieved FPR does not match threshold evaluation") + if value.calibration_fpr_interval != self.calibration_fpr_interval: + raise ValueError("calibrated attack result calibration interval does not match threshold evaluation") + if tuple(sorted(self.positive_results, key=lambda value: value.sample_id)) != self.positive_results: + raise ValueError("positive_results must use canonical sample ordering") + if tuple(sorted(self.negative_results, key=lambda value: value.sample_id)) != self.negative_results: + raise ValueError("negative_results must use canonical sample ordering") + if not isinstance(self.pristine_baseline, PristineBaselineSummary): + raise TypeError("pristine_baseline must be a PristineBaselineSummary") + if self.pristine_baseline.threshold_hash != self.threshold_hash: + raise ValueError("pristine baseline threshold does not match threshold evaluation") + expected_positive_evidence = tuple(sorted(value.evidence_hash for value in self.positive_results)) + if self.pristine_baseline.evidence_manifest_hash != sha256_json(expected_positive_evidence): + raise ValueError("pristine baseline evidence does not match positive attack results") + require_int("attack_negative_detected_count", self.attack_negative_detected_count) + detected = sum(value.decision for value in self.negative_results) + if self.attack_negative_detected_count != detected: + raise ValueError("attack_negative_detected_count does not match negative decisions") + expected_fpr = detected / len(self.negative_results) + attack_fpr = _require_probability("attack_negative_fpr", self.attack_negative_fpr) + if attack_fpr != expected_fpr: + raise ValueError("attack_negative_fpr does not match negative decisions") + object.__setattr__(self, "attack_negative_fpr", attack_fpr) + if not isinstance(self.attack_negative_fpr_interval, ExactBinomialInterval): + raise TypeError("attack_negative_fpr_interval must be an ExactBinomialInterval") + if self.attack_negative_fpr_interval != exact_binomial_interval(detected, len(self.negative_results)): + raise ValueError("attack negative FPR interval does not match decisions") + require_sha256("evaluation_hash", self.evaluation_hash) + if self.evaluation_hash != sha256_json(self._payload()): + raise ValueError("evaluation_hash does not match TinyDev threshold evaluation") + + def _payload(self) -> dict[str, object]: + return { + "algorithm_version": self.algorithm_version, + "target_fpr": self.target_fpr, + "threshold_hash": self.threshold_hash, + "threshold_value": self.threshold_value, + "achieved_calibration_fpr": self.achieved_calibration_fpr, + "calibration_fpr_interval": self.calibration_fpr_interval, + "positive_results": self.positive_results, + "negative_results": self.negative_results, + "pristine_baseline": self.pristine_baseline, + "attack_negative_detected_count": self.attack_negative_detected_count, + "attack_negative_fpr": self.attack_negative_fpr, + "attack_negative_fpr_interval": self.attack_negative_fpr_interval, + } + + +@dataclass(frozen=True, slots=True) +class TinyDevDetectorFamilyEvidence: + algorithm_version: str + detector_family: DetectorFamily + calibration_evidence: tuple[UncalibratedDetectorEvidence, ...] + attack_evidence: tuple[UncalibratedDetectorEvidence, ...] + calibration_bundle: CalibrationBundle + threshold_evaluations: tuple[TinyDevThresholdEvaluation, ...] + family_hash: str + + def __post_init__(self) -> None: + require_clean_string("algorithm_version", self.algorithm_version) + if self.algorithm_version != TINY_DEV_DETECTOR_FAMILY_ALGORITHM_VERSION: + raise ValueError("unsupported TinyDev detector-family evidence version") + if self.detector_family not in (DetectorFamily.MEAN, DetectorFamily.WEIGHTED_MEAN): + raise ValueError("TinyDev v1 real detector evidence supports Mean and Weighted Mean only") + if not isinstance(self.calibration_evidence, tuple) or not isinstance(self.attack_evidence, tuple): + raise TypeError("detector evidence rows must be tuples") + if len(self.calibration_evidence) != 100: + raise ValueError("TinyDev detector calibration requires exactly 100 negative rows") + if len(self.attack_evidence) != 8: + raise ValueError("TinyDev detector attack evaluation requires exactly eight rows") + for values in (self.calibration_evidence, self.attack_evidence): + if any(not isinstance(value, UncalibratedDetectorEvidence) for value in values): + raise TypeError("detector evidence tuples must contain UncalibratedDetectorEvidence values") + if tuple(sorted(values, key=lambda value: value.sample_id)) != values: + raise ValueError("detector evidence rows must use canonical sample ordering") + if len({value.sample_id for value in values}) != len(values): + raise ValueError("detector evidence sample IDs must be unique") + if any(value.detector_family is not self.detector_family for value in values): + raise ValueError("detector evidence family does not match family container") + if set(value.sample_id for value in self.calibration_evidence) & set(value.sample_id for value in self.attack_evidence): + raise ValueError("calibration and attack evidence populations must be disjoint") + if not isinstance(self.calibration_bundle, CalibrationBundle): + raise TypeError("calibration_bundle must be a CalibrationBundle") + if self.calibration_bundle.negative_count != 100: + raise ValueError("calibration bundle must bind 100 TinyDev negatives") + identity = DetectorCalibrationIdentity.from_evidence(self.calibration_evidence[0]) + if self.calibration_bundle.detector_identity != identity: + raise ValueError("calibration bundle detector identity does not match calibration evidence") + if any(DetectorCalibrationIdentity.from_evidence(value) != identity for value in (*self.calibration_evidence, *self.attack_evidence)): + raise ValueError("TinyDev family evidence mixes detector identities") + if tuple(value.target_fpr for value in self.calibration_bundle.thresholds) != TINY_DEV_HEADLINE_FPRS: + raise ValueError("calibration bundle must contain the frozen 5% and 1% headline FPRs") + if not isinstance(self.threshold_evaluations, tuple): + raise TypeError("threshold_evaluations must be a tuple") + if tuple(value.target_fpr for value in self.threshold_evaluations) != TINY_DEV_HEADLINE_FPRS: + raise ValueError("threshold evaluations must use the frozen 5% and 1% FPR order") + if any(not isinstance(value, TinyDevThresholdEvaluation) for value in self.threshold_evaluations): + raise TypeError("threshold_evaluations must contain TinyDevThresholdEvaluation values") + for evaluation, threshold in zip(self.threshold_evaluations, self.calibration_bundle.thresholds): + if evaluation.threshold_hash != threshold.threshold_hash: + raise ValueError("threshold evaluation does not match calibration bundle") + require_sha256("family_hash", self.family_hash) + if self.family_hash != sha256_json(self._payload()): + raise ValueError("family_hash does not match TinyDev detector evidence") + + def _payload(self) -> dict[str, object]: + return { + "algorithm_version": self.algorithm_version, + "detector_family": self.detector_family.value, + "calibration_evidence": self.calibration_evidence, + "attack_evidence": self.attack_evidence, + "calibration_bundle": self.calibration_bundle, + "threshold_evaluations": self.threshold_evaluations, + } + + +@dataclass(frozen=True, slots=True) +class TinyDevDetectorEvidenceArtifact: + algorithm_version: str + tiny_dev_artifact_hash: str + corpus_manifest_hash: str + watermark_config_hash: str + adapter_id: str + adapter_algorithm_version: str + adapter_config_hash: str + adapter_source_id: str + adapter_source_commit: str + token_track: str + prompt_boundary_mode: str + headline_fprs: tuple[float, ...] + primary_fpr: float + baseline_interpretability_floor: float + family_evidence: tuple[TinyDevDetectorFamilyEvidence, ...] + scientific_status: str + artifact_hash: str + + def __post_init__(self) -> None: + require_clean_string("algorithm_version", self.algorithm_version) + if self.algorithm_version != TINY_DEV_DETECTOR_EVIDENCE_ALGORITHM_VERSION: + raise ValueError("unsupported TinyDev detector evidence artifact version") + for name, value in ( + ("tiny_dev_artifact_hash", self.tiny_dev_artifact_hash), + ("corpus_manifest_hash", self.corpus_manifest_hash), + ("watermark_config_hash", self.watermark_config_hash), + ("adapter_config_hash", self.adapter_config_hash), + ("artifact_hash", self.artifact_hash), + ): + require_sha256(name, value) + for name, value in ( + ("adapter_id", self.adapter_id), + ("adapter_algorithm_version", self.adapter_algorithm_version), + ("adapter_source_id", self.adapter_source_id), + ("adapter_source_commit", self.adapter_source_commit), + ("token_track", self.token_track), + ("prompt_boundary_mode", self.prompt_boundary_mode), + ("scientific_status", self.scientific_status), + ): + require_clean_string(name, value) + if self.token_track != TINY_DEV_TOKEN_TRACK: + raise ValueError("TinyDev detector evidence must use original generation token IDs") + if self.prompt_boundary_mode != TINY_DEV_PROMPT_BOUNDARY_MODE: + raise ValueError("TinyDev detector evidence must use continuation-only prompt boundaries") + if self.headline_fprs != TINY_DEV_HEADLINE_FPRS: + raise ValueError("headline_fprs do not match the frozen 5% and 1% profile") + if self.primary_fpr != TINY_DEV_PRIMARY_FPR: + raise ValueError("primary_fpr must be 1%") + if self.baseline_interpretability_floor != TINY_DEV_BASELINE_INTERPRETABILITY_FLOOR: + raise ValueError("baseline interpretability floor must be 80%") + if self.scientific_status != TINY_DEV_DETECTOR_STATUS: + raise ValueError("TinyDev detector evidence must remain development-only") + if not isinstance(self.family_evidence, tuple): + raise TypeError("family_evidence must be a tuple") + if tuple(value.detector_family for value in self.family_evidence) != ( + DetectorFamily.MEAN, + DetectorFamily.WEIGHTED_MEAN, + ): + raise ValueError("TinyDev detector artifact must contain Mean then Weighted Mean evidence") + for family in self.family_evidence: + if family.calibration_evidence[0].adapter_id != self.adapter_id: + raise ValueError("detector family adapter identity does not match artifact") + if family.calibration_evidence[0].adapter_algorithm_version != self.adapter_algorithm_version: + raise ValueError("detector family adapter version does not match artifact") + if family.calibration_evidence[0].adapter_config_hash != self.adapter_config_hash: + raise ValueError("detector family adapter configuration does not match artifact") + if family.calibration_evidence[0].source_id != self.adapter_source_id: + raise ValueError("detector family adapter source does not match artifact") + if family.calibration_evidence[0].source_commit != self.adapter_source_commit: + raise ValueError("detector family adapter source commit does not match artifact") + if self.artifact_hash != sha256_json(self._payload()): + raise ValueError("artifact_hash does not match TinyDev detector evidence artifact") + + def _payload(self) -> dict[str, object]: + return { + "algorithm_version": self.algorithm_version, + "tiny_dev_artifact_hash": self.tiny_dev_artifact_hash, + "corpus_manifest_hash": self.corpus_manifest_hash, + "watermark_config_hash": self.watermark_config_hash, + "adapter_id": self.adapter_id, + "adapter_algorithm_version": self.adapter_algorithm_version, + "adapter_config_hash": self.adapter_config_hash, + "adapter_source_id": self.adapter_source_id, + "adapter_source_commit": self.adapter_source_commit, + "token_track": self.token_track, + "prompt_boundary_mode": self.prompt_boundary_mode, + "headline_fprs": self.headline_fprs, + "primary_fpr": self.primary_fpr, + "baseline_interpretability_floor": self.baseline_interpretability_floor, + "family_evidence": self.family_evidence, + "scientific_status": self.scientific_status, + } + + +def _sample_populations(artifact: TinyDevCorpusArtifact): + calibration = tuple( + sample + for sample in artifact.manifest.samples + if sample.split is CorpusSplit.THRESHOLD_CALIBRATION + and sample.label is WatermarkLabel.UNWATERMARKED + ) + attack = tuple( + sample + for sample in artifact.manifest.samples + if sample.split is CorpusSplit.ATTACK_DEVELOPMENT + ) + if len(calibration) != 100 or len(attack) != 8: + raise TinyDevDetectorEvidenceError("TinyDev corpus does not match the frozen detector-evidence populations") + return calibration, attack + + +def _score_samples(samples, adapter: WatermarkAdapter, evidence_builder): + rows: list[UncalibratedDetectorEvidence] = [] + for sample in samples: + eos_token_id = sample.model.eos_token_id + if eos_token_id is None: + raise TinyDevDetectorEvidenceError("TinyDev model identity must define eos_token_id") + batch = build_native_observations( + sample.sample_id, + sample.generation_tokens.continuation_token_ids, + eos_token_id, + adapter, + ) + rows.append(evidence_builder(batch)) + return tuple(sorted(rows, key=lambda value: value.sample_id)) + + +def _threshold_evaluation( + target_fpr: float, + bundle: CalibrationBundle, + attack_evidence: tuple[UncalibratedDetectorEvidence, ...], + attack_labels: dict[str, WatermarkLabel], +) -> TinyDevThresholdEvaluation: + calibrated = tuple( + sorted( + (apply_calibration(value, bundle, target_fpr) for value in attack_evidence), + key=lambda value: value.sample_id, + ) + ) + positives = tuple(value for value in calibrated if attack_labels[value.sample_id] is WatermarkLabel.WATERMARKED) + negatives = tuple(value for value in calibrated if attack_labels[value.sample_id] is WatermarkLabel.UNWATERMARKED) + baseline = evaluate_pristine_baseline( + positives, + interpretability_floor=TINY_DEV_BASELINE_INTERPRETABILITY_FLOOR, + ) + threshold = next(value for value in bundle.thresholds if value.target_fpr == target_fpr) + negative_detected = sum(value.decision for value in negatives) + interval = exact_binomial_interval(negative_detected, len(negatives)) + payload = { + "algorithm_version": TINY_DEV_THRESHOLD_EVALUATION_ALGORITHM_VERSION, + "target_fpr": target_fpr, + "threshold_hash": threshold.threshold_hash, + "threshold_value": threshold.value, + "achieved_calibration_fpr": threshold.achieved_fpr, + "calibration_fpr_interval": threshold.fpr_interval, + "positive_results": positives, + "negative_results": negatives, + "pristine_baseline": baseline, + "attack_negative_detected_count": negative_detected, + "attack_negative_fpr": negative_detected / len(negatives), + "attack_negative_fpr_interval": interval, + } + return TinyDevThresholdEvaluation( + algorithm_version=TINY_DEV_THRESHOLD_EVALUATION_ALGORITHM_VERSION, + target_fpr=target_fpr, + threshold_hash=threshold.threshold_hash, + threshold_value=threshold.value, + achieved_calibration_fpr=threshold.achieved_fpr, + calibration_fpr_interval=threshold.fpr_interval, + positive_results=positives, + negative_results=negatives, + pristine_baseline=baseline, + attack_negative_detected_count=negative_detected, + attack_negative_fpr=negative_detected / len(negatives), + attack_negative_fpr_interval=interval, + evaluation_hash=sha256_json(payload), + ) + + +def _family_evidence( + family: DetectorFamily, + calibration_samples, + attack_samples, + adapter: WatermarkAdapter, +) -> TinyDevDetectorFamilyEvidence: + builder = mean_evidence if family is DetectorFamily.MEAN else weighted_mean_evidence + calibration_rows = _score_samples(calibration_samples, adapter, builder) + attack_rows = _score_samples(attack_samples, adapter, builder) + scope = CalibrationScope.create( + corpus_id="tiny-dev-real", + population_id="threshold-calibration-unwatermarked", + length_policy_id="generated-64", + token_track=TINY_DEV_TOKEN_TRACK, + prompt_boundary_mode=TINY_DEV_PROMPT_BOUNDARY_MODE, + ) + bundle = calibrate_detector( + calibration_rows, + scope, + target_fprs=TINY_DEV_HEADLINE_FPRS, + ) + attack_labels = {sample.sample_id: sample.label for sample in attack_samples} + evaluations = tuple( + _threshold_evaluation(target, bundle, attack_rows, attack_labels) + for target in TINY_DEV_HEADLINE_FPRS + ) + payload = { + "algorithm_version": TINY_DEV_DETECTOR_FAMILY_ALGORITHM_VERSION, + "detector_family": family.value, + "calibration_evidence": calibration_rows, + "attack_evidence": attack_rows, + "calibration_bundle": bundle, + "threshold_evaluations": evaluations, + } + return TinyDevDetectorFamilyEvidence( + algorithm_version=TINY_DEV_DETECTOR_FAMILY_ALGORITHM_VERSION, + detector_family=family, + calibration_evidence=calibration_rows, + attack_evidence=attack_rows, + calibration_bundle=bundle, + threshold_evaluations=evaluations, + family_hash=sha256_json(payload), + ) + + +def build_tiny_dev_detector_evidence( + artifact: TinyDevCorpusArtifact, + adapter: WatermarkAdapter, + *, + expected_watermark_config_hash: str, +) -> TinyDevDetectorEvidenceArtifact: + if not isinstance(artifact, TinyDevCorpusArtifact): + raise TypeError("artifact must be a TinyDevCorpusArtifact") + if not isinstance(adapter, WatermarkAdapter): + raise TypeError("adapter must satisfy the WatermarkAdapter protocol") + require_sha256("expected_watermark_config_hash", expected_watermark_config_hash) + if artifact.watermark_condition_hash != artifact.manifest.samples[0].watermark.condition_hash: + raise TinyDevDetectorEvidenceError("TinyDev artifact watermark condition does not match corpus samples") + watermark_hashes = {sample.watermark.watermark_config_hash for sample in artifact.manifest.samples} + if watermark_hashes != {expected_watermark_config_hash}: + raise TinyDevDetectorEvidenceError("TinyDev corpus watermark configuration does not match the scoring adapter contract") + if any(sample.generation_realized_length != 64 for sample in artifact.manifest.samples): + raise TinyDevDetectorEvidenceError("TinyDev detector evidence requires exact 64-token generation tracks") + calibration_samples, attack_samples = _sample_populations(artifact) + families = tuple( + _family_evidence(family, calibration_samples, attack_samples, adapter) + for family in (DetectorFamily.MEAN, DetectorFamily.WEIGHTED_MEAN) + ) + source_pin = adapter.source_pin + payload = { + "algorithm_version": TINY_DEV_DETECTOR_EVIDENCE_ALGORITHM_VERSION, + "tiny_dev_artifact_hash": artifact.artifact_hash, + "corpus_manifest_hash": artifact.manifest.manifest_hash, + "watermark_config_hash": expected_watermark_config_hash, + "adapter_id": adapter.adapter_id, + "adapter_algorithm_version": adapter.algorithm_version, + "adapter_config_hash": adapter.configuration_fingerprint(), + "adapter_source_id": source_pin.source_id, + "adapter_source_commit": source_pin.commit, + "token_track": TINY_DEV_TOKEN_TRACK, + "prompt_boundary_mode": TINY_DEV_PROMPT_BOUNDARY_MODE, + "headline_fprs": TINY_DEV_HEADLINE_FPRS, + "primary_fpr": TINY_DEV_PRIMARY_FPR, + "baseline_interpretability_floor": TINY_DEV_BASELINE_INTERPRETABILITY_FLOOR, + "family_evidence": families, + "scientific_status": TINY_DEV_DETECTOR_STATUS, + } + return TinyDevDetectorEvidenceArtifact( + algorithm_version=TINY_DEV_DETECTOR_EVIDENCE_ALGORITHM_VERSION, + tiny_dev_artifact_hash=artifact.artifact_hash, + corpus_manifest_hash=artifact.manifest.manifest_hash, + watermark_config_hash=expected_watermark_config_hash, + adapter_id=adapter.adapter_id, + adapter_algorithm_version=adapter.algorithm_version, + adapter_config_hash=adapter.configuration_fingerprint(), + adapter_source_id=source_pin.source_id, + adapter_source_commit=source_pin.commit, + token_track=TINY_DEV_TOKEN_TRACK, + prompt_boundary_mode=TINY_DEV_PROMPT_BOUNDARY_MODE, + headline_fprs=TINY_DEV_HEADLINE_FPRS, + primary_fpr=TINY_DEV_PRIMARY_FPR, + baseline_interpretability_floor=TINY_DEV_BASELINE_INTERPRETABILITY_FLOOR, + family_evidence=families, + scientific_status=TINY_DEV_DETECTOR_STATUS, + artifact_hash=sha256_json(payload), + ) + + +def primary_baseline_statuses( + artifact: TinyDevDetectorEvidenceArtifact, +) -> tuple[tuple[DetectorFamily, BaselineStatus], ...]: + if not isinstance(artifact, TinyDevDetectorEvidenceArtifact): + raise TypeError("artifact must be a TinyDevDetectorEvidenceArtifact") + output = [] + for family in artifact.family_evidence: + evaluation = next( + value for value in family.threshold_evaluations + if value.target_fpr == TINY_DEV_PRIMARY_FPR + ) + output.append((family.detector_family, evaluation.pristine_baseline.status)) + return tuple(output) From 108e344515e32d0ff8fccfef328dbed7da44b3ef Mon Sep 17 00:00:00 2001 From: Byte271 Date: Mon, 17 Aug 2026 21:11:21 -0400 Subject: [PATCH 2/7] Harden TinyDev detector evidence contracts --- .../experiments/tiny_dev_detector_evidence.py | 275 ++++++++---------- 1 file changed, 118 insertions(+), 157 deletions(-) diff --git a/fuckmark/experiments/tiny_dev_detector_evidence.py b/fuckmark/experiments/tiny_dev_detector_evidence.py index 75c78bd2..f6707309 100644 --- a/fuckmark/experiments/tiny_dev_detector_evidence.py +++ b/fuckmark/experiments/tiny_dev_detector_evidence.py @@ -1,11 +1,12 @@ from __future__ import annotations import math +from collections.abc import Callable, Sequence from dataclasses import dataclass from .._validation import require_clean_string, require_int, require_sha256 from ..adapters.base import WatermarkAdapter -from ..corpus import CorpusSplit, TinyDevCorpusArtifact, WatermarkLabel +from ..corpus import CorpusSample, CorpusSplit, TinyDevCorpusArtifact, WatermarkLabel from ..detectors import ( BaselineStatus, CalibratedDetectorResult, @@ -24,7 +25,7 @@ ) from ..detectors.calibration_identity import DetectorCalibrationIdentity from ..hashing import sha256_json -from ..native_observations import build_native_observations +from ..native_observations import NativeObservationBatch, build_native_observations TINY_DEV_DETECTOR_EVIDENCE_ALGORITHM_VERSION = "tiny-dev-detector-evidence-v1" @@ -42,7 +43,7 @@ class TinyDevDetectorEvidenceError(ValueError): pass -def _require_probability(name: str, value: float) -> float: +def _probability(name: str, value: float) -> float: if isinstance(value, bool) or not isinstance(value, (int, float)): raise TypeError(f"{name} must be a real number") number = float(value) @@ -71,60 +72,55 @@ def __post_init__(self) -> None: require_clean_string("algorithm_version", self.algorithm_version) if self.algorithm_version != TINY_DEV_THRESHOLD_EVALUATION_ALGORITHM_VERSION: raise ValueError("unsupported TinyDev threshold evaluation version") - target = _require_probability("target_fpr", self.target_fpr) + target = _probability("target_fpr", self.target_fpr) if target not in TINY_DEV_HEADLINE_FPRS: raise ValueError("target_fpr must be a frozen TinyDev headline FPR") object.__setattr__(self, "target_fpr", target) require_sha256("threshold_hash", self.threshold_hash) - threshold_value = _require_probability("threshold_value", self.threshold_value) + threshold_value = _probability("threshold_value", self.threshold_value) object.__setattr__(self, "threshold_value", threshold_value) - achieved = _require_probability("achieved_calibration_fpr", self.achieved_calibration_fpr) + achieved = _probability("achieved_calibration_fpr", self.achieved_calibration_fpr) object.__setattr__(self, "achieved_calibration_fpr", achieved) if not isinstance(self.calibration_fpr_interval, ExactBinomialInterval): raise TypeError("calibration_fpr_interval must be an ExactBinomialInterval") - if not isinstance(self.positive_results, tuple) or not isinstance(self.negative_results, tuple): - raise TypeError("threshold results must be tuples") - if len(self.positive_results) != 4 or len(self.negative_results) != 4: - raise ValueError("TinyDev threshold evaluation requires four positive and four negative attack rows") - if any(not isinstance(value, CalibratedDetectorResult) for value in (*self.positive_results, *self.negative_results)): - raise TypeError("threshold result tuples must contain CalibratedDetectorResult values") + for name, values in (("positive_results", self.positive_results), ("negative_results", self.negative_results)): + if not isinstance(values, tuple): + raise TypeError(f"{name} must be a tuple") + if len(values) != 4: + raise ValueError(f"{name} must contain four TinyDev attack rows") + if any(not isinstance(value, CalibratedDetectorResult) for value in values): + raise TypeError(f"{name} must contain CalibratedDetectorResult values") + if tuple(sorted(values, key=lambda value: value.sample_id)) != values: + raise ValueError(f"{name} must use canonical sample ordering") all_results = (*self.positive_results, *self.negative_results) if len({value.sample_id for value in all_results}) != 8: - raise ValueError("TinyDev attack threshold results must use eight unique samples") + raise ValueError("threshold evaluation must contain eight unique attack samples") for value in all_results: - if value.target_fpr != target: - raise ValueError("calibrated attack result target FPR does not match threshold evaluation") - if value.threshold_hash != self.threshold_hash: - raise ValueError("calibrated attack result threshold hash does not match threshold evaluation") - if value.threshold_value != threshold_value: - raise ValueError("calibrated attack result threshold value does not match threshold evaluation") - if value.achieved_calibration_fpr != achieved: - raise ValueError("calibrated attack result achieved FPR does not match threshold evaluation") - if value.calibration_fpr_interval != self.calibration_fpr_interval: - raise ValueError("calibrated attack result calibration interval does not match threshold evaluation") - if tuple(sorted(self.positive_results, key=lambda value: value.sample_id)) != self.positive_results: - raise ValueError("positive_results must use canonical sample ordering") - if tuple(sorted(self.negative_results, key=lambda value: value.sample_id)) != self.negative_results: - raise ValueError("negative_results must use canonical sample ordering") + if ( + value.target_fpr != target + or value.threshold_hash != self.threshold_hash + or value.threshold_value != threshold_value + or value.achieved_calibration_fpr != achieved + or value.calibration_fpr_interval != self.calibration_fpr_interval + ): + raise ValueError("calibrated attack result does not match threshold evaluation") if not isinstance(self.pristine_baseline, PristineBaselineSummary): raise TypeError("pristine_baseline must be a PristineBaselineSummary") if self.pristine_baseline.threshold_hash != self.threshold_hash: raise ValueError("pristine baseline threshold does not match threshold evaluation") - expected_positive_evidence = tuple(sorted(value.evidence_hash for value in self.positive_results)) - if self.pristine_baseline.evidence_manifest_hash != sha256_json(expected_positive_evidence): + positive_manifest = sha256_json(tuple(value.evidence_hash for value in self.positive_results)) + if self.pristine_baseline.evidence_manifest_hash != positive_manifest: raise ValueError("pristine baseline evidence does not match positive attack results") require_int("attack_negative_detected_count", self.attack_negative_detected_count) detected = sum(value.decision for value in self.negative_results) if self.attack_negative_detected_count != detected: raise ValueError("attack_negative_detected_count does not match negative decisions") - expected_fpr = detected / len(self.negative_results) - attack_fpr = _require_probability("attack_negative_fpr", self.attack_negative_fpr) - if attack_fpr != expected_fpr: + attack_fpr = _probability("attack_negative_fpr", self.attack_negative_fpr) + if attack_fpr != detected / 4: raise ValueError("attack_negative_fpr does not match negative decisions") object.__setattr__(self, "attack_negative_fpr", attack_fpr) - if not isinstance(self.attack_negative_fpr_interval, ExactBinomialInterval): - raise TypeError("attack_negative_fpr_interval must be an ExactBinomialInterval") - if self.attack_negative_fpr_interval != exact_binomial_interval(detected, len(self.negative_results)): + expected_interval = exact_binomial_interval(detected, 4) + if self.attack_negative_fpr_interval != expected_interval: raise ValueError("attack negative FPR interval does not match decisions") require_sha256("evaluation_hash", self.evaluation_hash) if self.evaluation_hash != sha256_json(self._payload()): @@ -162,47 +158,49 @@ def __post_init__(self) -> None: if self.algorithm_version != TINY_DEV_DETECTOR_FAMILY_ALGORITHM_VERSION: raise ValueError("unsupported TinyDev detector-family evidence version") if self.detector_family not in (DetectorFamily.MEAN, DetectorFamily.WEIGHTED_MEAN): - raise ValueError("TinyDev v1 real detector evidence supports Mean and Weighted Mean only") - if not isinstance(self.calibration_evidence, tuple) or not isinstance(self.attack_evidence, tuple): - raise TypeError("detector evidence rows must be tuples") - if len(self.calibration_evidence) != 100: - raise ValueError("TinyDev detector calibration requires exactly 100 negative rows") - if len(self.attack_evidence) != 8: - raise ValueError("TinyDev detector attack evaluation requires exactly eight rows") - for values in (self.calibration_evidence, self.attack_evidence): + raise ValueError("TinyDev v1 detector evidence supports Mean and Weighted Mean only") + for name, values, expected in ( + ("calibration_evidence", self.calibration_evidence, 100), + ("attack_evidence", self.attack_evidence, 8), + ): + if not isinstance(values, tuple): + raise TypeError(f"{name} must be a tuple") + if len(values) != expected: + raise ValueError(f"{name} must contain exactly {expected} rows") if any(not isinstance(value, UncalibratedDetectorEvidence) for value in values): - raise TypeError("detector evidence tuples must contain UncalibratedDetectorEvidence values") + raise TypeError(f"{name} must contain UncalibratedDetectorEvidence values") if tuple(sorted(values, key=lambda value: value.sample_id)) != values: - raise ValueError("detector evidence rows must use canonical sample ordering") + raise ValueError(f"{name} must use canonical sample ordering") if len({value.sample_id for value in values}) != len(values): - raise ValueError("detector evidence sample IDs must be unique") + raise ValueError(f"{name} sample IDs must be unique") if any(value.detector_family is not self.detector_family for value in values): - raise ValueError("detector evidence family does not match family container") + raise ValueError(f"{name} detector family does not match container") if set(value.sample_id for value in self.calibration_evidence) & set(value.sample_id for value in self.attack_evidence): - raise ValueError("calibration and attack evidence populations must be disjoint") + raise ValueError("calibration and attack populations must be disjoint") if not isinstance(self.calibration_bundle, CalibrationBundle): raise TypeError("calibration_bundle must be a CalibrationBundle") if self.calibration_bundle.negative_count != 100: - raise ValueError("calibration bundle must bind 100 TinyDev negatives") + raise ValueError("calibration bundle must bind 100 negative rows") identity = DetectorCalibrationIdentity.from_evidence(self.calibration_evidence[0]) + all_evidence = (*self.calibration_evidence, *self.attack_evidence) if self.calibration_bundle.detector_identity != identity: - raise ValueError("calibration bundle detector identity does not match calibration evidence") - if any(DetectorCalibrationIdentity.from_evidence(value) != identity for value in (*self.calibration_evidence, *self.attack_evidence)): + raise ValueError("calibration bundle detector identity does not match evidence") + if any(DetectorCalibrationIdentity.from_evidence(value) != identity for value in all_evidence): raise ValueError("TinyDev family evidence mixes detector identities") if tuple(value.target_fpr for value in self.calibration_bundle.thresholds) != TINY_DEV_HEADLINE_FPRS: - raise ValueError("calibration bundle must contain the frozen 5% and 1% headline FPRs") + raise ValueError("calibration bundle must contain frozen 5% and 1% thresholds") if not isinstance(self.threshold_evaluations, tuple): raise TypeError("threshold_evaluations must be a tuple") - if tuple(value.target_fpr for value in self.threshold_evaluations) != TINY_DEV_HEADLINE_FPRS: - raise ValueError("threshold evaluations must use the frozen 5% and 1% FPR order") if any(not isinstance(value, TinyDevThresholdEvaluation) for value in self.threshold_evaluations): raise TypeError("threshold_evaluations must contain TinyDevThresholdEvaluation values") + if tuple(value.target_fpr for value in self.threshold_evaluations) != TINY_DEV_HEADLINE_FPRS: + raise ValueError("threshold evaluations must use frozen 5% and 1% order") for evaluation, threshold in zip(self.threshold_evaluations, self.calibration_bundle.thresholds): if evaluation.threshold_hash != threshold.threshold_hash: raise ValueError("threshold evaluation does not match calibration bundle") require_sha256("family_hash", self.family_hash) if self.family_hash != sha256_json(self._payload()): - raise ValueError("family_hash does not match TinyDev detector evidence") + raise ValueError("family_hash does not match TinyDev detector family evidence") def _payload(self) -> dict[str, object]: return { @@ -257,36 +255,33 @@ def __post_init__(self) -> None: ("scientific_status", self.scientific_status), ): require_clean_string(name, value) - if self.token_track != TINY_DEV_TOKEN_TRACK: - raise ValueError("TinyDev detector evidence must use original generation token IDs") - if self.prompt_boundary_mode != TINY_DEV_PROMPT_BOUNDARY_MODE: - raise ValueError("TinyDev detector evidence must use continuation-only prompt boundaries") - if self.headline_fprs != TINY_DEV_HEADLINE_FPRS: - raise ValueError("headline_fprs do not match the frozen 5% and 1% profile") - if self.primary_fpr != TINY_DEV_PRIMARY_FPR: - raise ValueError("primary_fpr must be 1%") + if self.token_track != TINY_DEV_TOKEN_TRACK or self.prompt_boundary_mode != TINY_DEV_PROMPT_BOUNDARY_MODE: + raise ValueError("TinyDev detector evidence must use generation tokens and continuation-only boundaries") + if self.headline_fprs != TINY_DEV_HEADLINE_FPRS or self.primary_fpr != TINY_DEV_PRIMARY_FPR: + raise ValueError("TinyDev detector FPR profile does not match the frozen 5%/1% design") if self.baseline_interpretability_floor != TINY_DEV_BASELINE_INTERPRETABILITY_FLOOR: raise ValueError("baseline interpretability floor must be 80%") if self.scientific_status != TINY_DEV_DETECTOR_STATUS: raise ValueError("TinyDev detector evidence must remain development-only") - if not isinstance(self.family_evidence, tuple): - raise TypeError("family_evidence must be a tuple") + if not isinstance(self.family_evidence, tuple) or any( + not isinstance(value, TinyDevDetectorFamilyEvidence) for value in self.family_evidence + ): + raise TypeError("family_evidence must contain TinyDevDetectorFamilyEvidence values") if tuple(value.detector_family for value in self.family_evidence) != ( DetectorFamily.MEAN, DetectorFamily.WEIGHTED_MEAN, ): raise ValueError("TinyDev detector artifact must contain Mean then Weighted Mean evidence") for family in self.family_evidence: - if family.calibration_evidence[0].adapter_id != self.adapter_id: - raise ValueError("detector family adapter identity does not match artifact") - if family.calibration_evidence[0].adapter_algorithm_version != self.adapter_algorithm_version: - raise ValueError("detector family adapter version does not match artifact") - if family.calibration_evidence[0].adapter_config_hash != self.adapter_config_hash: - raise ValueError("detector family adapter configuration does not match artifact") - if family.calibration_evidence[0].source_id != self.adapter_source_id: - raise ValueError("detector family adapter source does not match artifact") - if family.calibration_evidence[0].source_commit != self.adapter_source_commit: - raise ValueError("detector family adapter source commit does not match artifact") + first = family.calibration_evidence[0] + if ( + first.adapter_id != self.adapter_id + or first.adapter_algorithm_version != self.adapter_algorithm_version + or first.adapter_config_hash != self.adapter_config_hash + or first.source_id != self.adapter_source_id + or first.source_commit != self.adapter_source_commit + ): + raise ValueError("detector family adapter identity does not match top-level artifact") if self.artifact_hash != sha256_json(self._payload()): raise ValueError("artifact_hash does not match TinyDev detector evidence artifact") @@ -311,24 +306,25 @@ def _payload(self) -> dict[str, object]: } -def _sample_populations(artifact: TinyDevCorpusArtifact): +def _populations(artifact: TinyDevCorpusArtifact) -> tuple[tuple[CorpusSample, ...], tuple[CorpusSample, ...]]: calibration = tuple( - sample - for sample in artifact.manifest.samples - if sample.split is CorpusSplit.THRESHOLD_CALIBRATION - and sample.label is WatermarkLabel.UNWATERMARKED + sample for sample in artifact.manifest.samples + if sample.split is CorpusSplit.THRESHOLD_CALIBRATION and sample.label is WatermarkLabel.UNWATERMARKED ) attack = tuple( - sample - for sample in artifact.manifest.samples + sample for sample in artifact.manifest.samples if sample.split is CorpusSplit.ATTACK_DEVELOPMENT ) if len(calibration) != 100 or len(attack) != 8: - raise TinyDevDetectorEvidenceError("TinyDev corpus does not match the frozen detector-evidence populations") + raise TinyDevDetectorEvidenceError("TinyDev corpus does not match the frozen detector populations") return calibration, attack -def _score_samples(samples, adapter: WatermarkAdapter, evidence_builder): +def _score_samples( + samples: Sequence[CorpusSample], + adapter: WatermarkAdapter, + builder: Callable[[NativeObservationBatch], UncalibratedDetectorEvidence], +) -> tuple[UncalibratedDetectorEvidence, ...]: rows: list[UncalibratedDetectorEvidence] = [] for sample in samples: eos_token_id = sample.model.eos_token_id @@ -340,31 +336,29 @@ def _score_samples(samples, adapter: WatermarkAdapter, evidence_builder): eos_token_id, adapter, ) - rows.append(evidence_builder(batch)) + rows.append(builder(batch)) return tuple(sorted(rows, key=lambda value: value.sample_id)) -def _threshold_evaluation( +def _evaluate_threshold( target_fpr: float, bundle: CalibrationBundle, attack_evidence: tuple[UncalibratedDetectorEvidence, ...], - attack_labels: dict[str, WatermarkLabel], + labels: dict[str, WatermarkLabel], ) -> TinyDevThresholdEvaluation: - calibrated = tuple( - sorted( - (apply_calibration(value, bundle, target_fpr) for value in attack_evidence), - key=lambda value: value.sample_id, - ) - ) - positives = tuple(value for value in calibrated if attack_labels[value.sample_id] is WatermarkLabel.WATERMARKED) - negatives = tuple(value for value in calibrated if attack_labels[value.sample_id] is WatermarkLabel.UNWATERMARKED) + calibrated = tuple(sorted( + (apply_calibration(value, bundle, target_fpr) for value in attack_evidence), + key=lambda value: value.sample_id, + )) + positives = tuple(value for value in calibrated if labels[value.sample_id] is WatermarkLabel.WATERMARKED) + negatives = tuple(value for value in calibrated if labels[value.sample_id] is WatermarkLabel.UNWATERMARKED) baseline = evaluate_pristine_baseline( positives, interpretability_floor=TINY_DEV_BASELINE_INTERPRETABILITY_FLOOR, ) threshold = next(value for value in bundle.thresholds if value.target_fpr == target_fpr) negative_detected = sum(value.decision for value in negatives) - interval = exact_binomial_interval(negative_detected, len(negatives)) + negative_interval = exact_binomial_interval(negative_detected, 4) payload = { "algorithm_version": TINY_DEV_THRESHOLD_EVALUATION_ALGORITHM_VERSION, "target_fpr": target_fpr, @@ -376,50 +370,33 @@ def _threshold_evaluation( "negative_results": negatives, "pristine_baseline": baseline, "attack_negative_detected_count": negative_detected, - "attack_negative_fpr": negative_detected / len(negatives), - "attack_negative_fpr_interval": interval, + "attack_negative_fpr": negative_detected / 4, + "attack_negative_fpr_interval": negative_interval, } - return TinyDevThresholdEvaluation( - algorithm_version=TINY_DEV_THRESHOLD_EVALUATION_ALGORITHM_VERSION, - target_fpr=target_fpr, - threshold_hash=threshold.threshold_hash, - threshold_value=threshold.value, - achieved_calibration_fpr=threshold.achieved_fpr, - calibration_fpr_interval=threshold.fpr_interval, - positive_results=positives, - negative_results=negatives, - pristine_baseline=baseline, - attack_negative_detected_count=negative_detected, - attack_negative_fpr=negative_detected / len(negatives), - attack_negative_fpr_interval=interval, - evaluation_hash=sha256_json(payload), - ) + return TinyDevThresholdEvaluation(**payload, evaluation_hash=sha256_json(payload)) -def _family_evidence( +def _build_family( family: DetectorFamily, - calibration_samples, - attack_samples, + corpus_id: str, + calibration_samples: tuple[CorpusSample, ...], + attack_samples: tuple[CorpusSample, ...], adapter: WatermarkAdapter, ) -> TinyDevDetectorFamilyEvidence: builder = mean_evidence if family is DetectorFamily.MEAN else weighted_mean_evidence calibration_rows = _score_samples(calibration_samples, adapter, builder) attack_rows = _score_samples(attack_samples, adapter, builder) scope = CalibrationScope.create( - corpus_id="tiny-dev-real", + corpus_id=corpus_id, population_id="threshold-calibration-unwatermarked", length_policy_id="generated-64", token_track=TINY_DEV_TOKEN_TRACK, prompt_boundary_mode=TINY_DEV_PROMPT_BOUNDARY_MODE, ) - bundle = calibrate_detector( - calibration_rows, - scope, - target_fprs=TINY_DEV_HEADLINE_FPRS, - ) - attack_labels = {sample.sample_id: sample.label for sample in attack_samples} + bundle = calibrate_detector(calibration_rows, scope, target_fprs=TINY_DEV_HEADLINE_FPRS) + labels = {sample.sample_id: sample.label for sample in attack_samples} evaluations = tuple( - _threshold_evaluation(target, bundle, attack_rows, attack_labels) + _evaluate_threshold(target, bundle, attack_rows, labels) for target in TINY_DEV_HEADLINE_FPRS ) payload = { @@ -452,16 +429,15 @@ def build_tiny_dev_detector_evidence( if not isinstance(adapter, WatermarkAdapter): raise TypeError("adapter must satisfy the WatermarkAdapter protocol") require_sha256("expected_watermark_config_hash", expected_watermark_config_hash) - if artifact.watermark_condition_hash != artifact.manifest.samples[0].watermark.condition_hash: - raise TinyDevDetectorEvidenceError("TinyDev artifact watermark condition does not match corpus samples") - watermark_hashes = {sample.watermark.watermark_config_hash for sample in artifact.manifest.samples} - if watermark_hashes != {expected_watermark_config_hash}: - raise TinyDevDetectorEvidenceError("TinyDev corpus watermark configuration does not match the scoring adapter contract") + if {sample.watermark.watermark_config_hash for sample in artifact.manifest.samples} != { + expected_watermark_config_hash + }: + raise TinyDevDetectorEvidenceError("TinyDev watermark configuration does not match scoring contract") if any(sample.generation_realized_length != 64 for sample in artifact.manifest.samples): raise TinyDevDetectorEvidenceError("TinyDev detector evidence requires exact 64-token generation tracks") - calibration_samples, attack_samples = _sample_populations(artifact) + calibration_samples, attack_samples = _populations(artifact) families = tuple( - _family_evidence(family, calibration_samples, attack_samples, adapter) + _build_family(family, artifact.manifest.corpus_id, calibration_samples, attack_samples, adapter) for family in (DetectorFamily.MEAN, DetectorFamily.WEIGHTED_MEAN) ) source_pin = adapter.source_pin @@ -483,25 +459,7 @@ def build_tiny_dev_detector_evidence( "family_evidence": families, "scientific_status": TINY_DEV_DETECTOR_STATUS, } - return TinyDevDetectorEvidenceArtifact( - algorithm_version=TINY_DEV_DETECTOR_EVIDENCE_ALGORITHM_VERSION, - tiny_dev_artifact_hash=artifact.artifact_hash, - corpus_manifest_hash=artifact.manifest.manifest_hash, - watermark_config_hash=expected_watermark_config_hash, - adapter_id=adapter.adapter_id, - adapter_algorithm_version=adapter.algorithm_version, - adapter_config_hash=adapter.configuration_fingerprint(), - adapter_source_id=source_pin.source_id, - adapter_source_commit=source_pin.commit, - token_track=TINY_DEV_TOKEN_TRACK, - prompt_boundary_mode=TINY_DEV_PROMPT_BOUNDARY_MODE, - headline_fprs=TINY_DEV_HEADLINE_FPRS, - primary_fpr=TINY_DEV_PRIMARY_FPR, - baseline_interpretability_floor=TINY_DEV_BASELINE_INTERPRETABILITY_FLOOR, - family_evidence=families, - scientific_status=TINY_DEV_DETECTOR_STATUS, - artifact_hash=sha256_json(payload), - ) + return TinyDevDetectorEvidenceArtifact(**payload, artifact_hash=sha256_json(payload)) def primary_baseline_statuses( @@ -509,11 +467,14 @@ def primary_baseline_statuses( ) -> tuple[tuple[DetectorFamily, BaselineStatus], ...]: if not isinstance(artifact, TinyDevDetectorEvidenceArtifact): raise TypeError("artifact must be a TinyDevDetectorEvidenceArtifact") - output = [] - for family in artifact.family_evidence: - evaluation = next( - value for value in family.threshold_evaluations - if value.target_fpr == TINY_DEV_PRIMARY_FPR + return tuple( + ( + family.detector_family, + next( + value.pristine_baseline.status + for value in family.threshold_evaluations + if value.target_fpr == TINY_DEV_PRIMARY_FPR + ), ) - output.append((family.detector_family, evaluation.pristine_baseline.status)) - return tuple(output) + for family in artifact.family_evidence + ) From 7c3694a90ba1a36f642a602b4d6dcd0e85b0b410 Mon Sep 17 00:00:00 2001 From: Byte271 Date: Mon, 17 Aug 2026 21:11:53 -0400 Subject: [PATCH 3/7] Test TinyDev fixed-FPR detector evidence --- tests/test_tiny_dev_detector_evidence.py | 186 +++++++++++++++++++++++ 1 file changed, 186 insertions(+) create mode 100644 tests/test_tiny_dev_detector_evidence.py diff --git a/tests/test_tiny_dev_detector_evidence.py b/tests/test_tiny_dev_detector_evidence.py new file mode 100644 index 00000000..63cb5409 --- /dev/null +++ b/tests/test_tiny_dev_detector_evidence.py @@ -0,0 +1,186 @@ +from __future__ import annotations + +from dataclasses import replace + +import pytest + +from fuckmark.adapters import DeepMindReferenceAdapter, DeepMindReferenceConfig +from fuckmark.corpus import ( + CorpusSample, + GenerationParameters, + GenerationTokenRecord, + KeySplit, + ModelTokenizerIdentity, + PaddingSide, + WatermarkCondition, + WatermarkLabel, + build_tiny_dev_corpus, +) +from fuckmark.corpus.tiny_dev_generation import build_tiny_dev_prompt_records +from fuckmark.detectors import DetectorFamily +from fuckmark.experiments.tiny_dev_detector_evidence import ( + TINY_DEV_DETECTOR_STATUS, + TINY_DEV_HEADLINE_FPRS, + TINY_DEV_PRIMARY_FPR, + TinyDevDetectorEvidenceError, + build_tiny_dev_detector_evidence, + primary_baseline_statuses, +) +from fuckmark.hashing import sha256_json, sha256_text + + +_WATERMARK_HASH = sha256_json({"fixture": "tiny-dev-detector"}) + + +def _model() -> ModelTokenizerIdentity: + return ModelTokenizerIdentity.create( + model_id="fixture/model", + model_revision="a" * 40, + tokenizer_id="fixture/tokenizer", + tokenizer_revision="b" * 40, + chat_template_present=False, + chat_template_hash=sha256_text(""), + special_token_map_hash=sha256_json({"eos_token": ""}), + padding_side=PaddingSide.LEFT, + bos_token_id=None, + eos_token_id=2, + pad_token_id=None, + add_bos_token=False, + add_eos_token=False, + ) + + +def _generation(seed: int) -> GenerationParameters: + return GenerationParameters.create( + seed=seed, + seed_policy_id="fixture-paired-seed-v1", + temperature=0.8, + top_k=50, + top_p=0.95, + max_new_tokens=64, + do_sample=True, + dtype="float32", + device="cpu", + backend_id="fixture", + backend_version="1", + ) + + +def _tokens(identity: ModelTokenizerIdentity, pair_index: int, label_offset: int) -> GenerationTokenRecord: + continuation = tuple( + 1000 + ((pair_index * 104729 + label_offset * 100003 + index * index * 7919 + index * 997) % 1_000_000) + for index in range(64) + ) + input_ids = (101, 102, 103) + return GenerationTokenRecord.create( + input_token_ids=input_ids, + attention_mask=(1, 1, 1), + generated_sequence_ids=input_ids + continuation, + continuation_start_index=len(input_ids), + continuation_token_ids=continuation, + prompt_length_after_templating=3, + model_tokenizer_identity_hash=identity.identity_hash, + ) + + +def _artifact(): + identity = _model() + watermark = WatermarkCondition.create(_WATERMARK_HASH, KeySplit.DEV, "fixture-dev-key") + prompts = build_tiny_dev_prompt_records() + samples = [] + for pair_index, prompt in enumerate(prompts): + generation = _generation(50_000 + pair_index) + for label, offset in ( + (WatermarkLabel.UNWATERMARKED, 0), + (WatermarkLabel.WATERMARKED, 1), + ): + tokens = _tokens(identity, pair_index, offset) + samples.append( + CorpusSample.create( + sample_id=f"{prompt.prompt_id}-{label.value}", + match_id=f"match-{prompt.prompt_id}", + prompt_id=prompt.prompt_id, + prompt_family_id=prompt.prompt_family_id, + domain=prompt.domain, + split=prompt.split, + label=label, + text=f"fixture {label.value} output {pair_index}", + model=identity, + generation=generation, + watermark=watermark, + target_length=64, + generation_tokens=tokens, + ) + ) + return build_tiny_dev_corpus("fixture-real-tiny-dev", prompts, samples) + + +def _adapter() -> DeepMindReferenceAdapter: + return DeepMindReferenceAdapter( + DeepMindReferenceConfig( + ngram_len=5, + keys=(11, 23, 37, 41, 53, 67, 79, 83, 97), + context_history_size=1024, + ) + ) + + +def test_tiny_dev_detector_evidence_uses_real_generation_tracks_and_fixed_fprs() -> None: + source = _artifact() + result = build_tiny_dev_detector_evidence( + source, + _adapter(), + expected_watermark_config_hash=_WATERMARK_HASH, + ) + assert result.tiny_dev_artifact_hash == source.artifact_hash + assert result.corpus_manifest_hash == source.manifest.manifest_hash + assert result.headline_fprs == TINY_DEV_HEADLINE_FPRS + assert result.primary_fpr == TINY_DEV_PRIMARY_FPR + assert result.scientific_status == TINY_DEV_DETECTOR_STATUS + assert tuple(value.detector_family for value in result.family_evidence) == ( + DetectorFamily.MEAN, + DetectorFamily.WEIGHTED_MEAN, + ) + for family in result.family_evidence: + assert len(family.calibration_evidence) == 100 + assert len(family.attack_evidence) == 8 + assert family.calibration_bundle.scope.corpus_id == "fixture-real-tiny-dev" + assert family.calibration_bundle.scope.token_track == "original-generation-token-ids" + assert tuple(value.target_fpr for value in family.calibration_bundle.thresholds) == (0.05, 0.01) + assert tuple(value.target_fpr for value in family.threshold_evaluations) == (0.05, 0.01) + for evaluation in family.threshold_evaluations: + assert len(evaluation.positive_results) == 4 + assert len(evaluation.negative_results) == 4 + assert evaluation.pristine_baseline.sample_count == 4 + + +def test_tiny_dev_detector_evidence_fails_closed_on_wrong_watermark_contract() -> None: + with pytest.raises(TinyDevDetectorEvidenceError, match="watermark configuration"): + build_tiny_dev_detector_evidence( + _artifact(), + _adapter(), + expected_watermark_config_hash="0" * 64, + ) + + +def test_tiny_dev_detector_evidence_hash_is_tamper_evident() -> None: + result = build_tiny_dev_detector_evidence( + _artifact(), + _adapter(), + expected_watermark_config_hash=_WATERMARK_HASH, + ) + with pytest.raises(ValueError, match="artifact_hash"): + replace(result, artifact_hash="0" * 64) + + +def test_primary_baseline_statuses_cover_both_detector_families() -> None: + result = build_tiny_dev_detector_evidence( + _artifact(), + _adapter(), + expected_watermark_config_hash=_WATERMARK_HASH, + ) + statuses = primary_baseline_statuses(result) + assert tuple(value[0] for value in statuses) == ( + DetectorFamily.MEAN, + DetectorFamily.WEIGHTED_MEAN, + ) From 31a7e0c5ca48e5dddd926247b72292bd32f5eb22 Mon Sep 17 00:00:00 2001 From: Byte271 Date: Mon, 17 Aug 2026 21:12:22 -0400 Subject: [PATCH 4/7] Add pinned TinyDev detector evidence runner --- fuckmark/tiny_dev_detector_hf.py | 110 +++++++++++++++++++++++++++++++ 1 file changed, 110 insertions(+) create mode 100644 fuckmark/tiny_dev_detector_hf.py diff --git a/fuckmark/tiny_dev_detector_hf.py b/fuckmark/tiny_dev_detector_hf.py new file mode 100644 index 00000000..f75b9f96 --- /dev/null +++ b/fuckmark/tiny_dev_detector_hf.py @@ -0,0 +1,110 @@ +from __future__ import annotations + +import argparse +import sys +from pathlib import Path + +from .adapters import HuggingFaceSynthIDAdapter, HuggingFaceSynthIDConfig +from .config import canonical_json_text +from .corpus import load_tiny_dev_corpus_json +from .experiments.tiny_dev_detector_evidence import ( + TINY_DEV_PRIMARY_FPR, + build_tiny_dev_detector_evidence, +) +from .hashing import sha256_json +from .tiny_dev_corpus_hf import DEFAULT_KEYS, DEFAULT_NGRAM_LEN + + +DEFAULT_CONTEXT_HISTORY_SIZE = 1024 +DEFAULT_SAMPLING_TABLE_SEED = 0 +DEFAULT_SAMPLING_TABLE_SIZE = 2**16 +DEFAULT_SKIP_FIRST_NGRAM_CALLS = False +DEFAULT_DEBUG_MODE = False + + +def default_watermark_payload() -> dict[str, object]: + return { + "ngram_len": DEFAULT_NGRAM_LEN, + "keys": DEFAULT_KEYS, + "context_history_size": DEFAULT_CONTEXT_HISTORY_SIZE, + "sampling_table_seed": DEFAULT_SAMPLING_TABLE_SEED, + "sampling_table_size": DEFAULT_SAMPLING_TABLE_SIZE, + "skip_first_ngram_calls": DEFAULT_SKIP_FIRST_NGRAM_CALLS, + "debug_mode": DEFAULT_DEBUG_MODE, + } + + +def default_watermark_config_hash() -> str: + return sha256_json(default_watermark_payload()) + + +def _parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser(prog="fuckmark-tiny-dev-detector-hf") + parser.add_argument("--corpus-json", type=Path, required=True) + parser.add_argument( + "--json", + type=Path, + default=Path("artifacts/tiny-dev-detector-evidence.json"), + ) + parser.add_argument("--device", choices=("cpu", "cuda"), default="cpu") + return parser + + +def main(argv: list[str] | None = None) -> int: + args = _parser().parse_args(argv) + corpus = load_tiny_dev_corpus_json(args.corpus_json) + payload = default_watermark_payload() + adapter = HuggingFaceSynthIDAdapter.from_torch( + HuggingFaceSynthIDConfig( + ngram_len=payload["ngram_len"], + keys=payload["keys"], + context_history_size=payload["context_history_size"], + sampling_table_seed=payload["sampling_table_seed"], + sampling_table_size=payload["sampling_table_size"], + skip_first_ngram_calls=payload["skip_first_ngram_calls"], + debug_mode=payload["debug_mode"], + ), + device=args.device, + ) + evidence = build_tiny_dev_detector_evidence( + corpus, + adapter, + expected_watermark_config_hash=default_watermark_config_hash(), + ) + args.json.parent.mkdir(parents=True, exist_ok=True) + args.json.write_text(canonical_json_text(evidence) + "\n", encoding="utf-8") + + sys.stdout.write(f"artifact_hash={evidence.artifact_hash}\n") + sys.stdout.write(f"corpus_artifact_hash={evidence.tiny_dev_artifact_hash}\n") + sys.stdout.write(f"adapter_config_hash={evidence.adapter_config_hash}\n") + sys.stdout.write(f"sampling_table_hash={adapter.sampling_table_hash}\n") + sys.stdout.write(f"sampling_table_provenance={adapter.sampling_table_provenance}\n") + for family in evidence.family_evidence: + sys.stdout.write(f"detector={family.detector_family.value}\n") + for evaluation in family.threshold_evaluations: + sys.stdout.write( + "threshold " + f"target_fpr={evaluation.target_fpr:g} " + f"value={evaluation.threshold_value:.17g} " + f"achieved_calibration_fpr={evaluation.achieved_calibration_fpr:.17g} " + f"pristine_tpr={evaluation.pristine_baseline.tpr:.17g} " + f"baseline_status={evaluation.pristine_baseline.status.value} " + f"attack_negative_fpr={evaluation.attack_negative_fpr:.17g}\n" + ) + primary = next( + value for value in family.threshold_evaluations + if value.target_fpr == TINY_DEV_PRIMARY_FPR + ) + sys.stdout.write( + f"primary detector={family.detector_family.value} " + f"target_fpr={TINY_DEV_PRIMARY_FPR:g} " + f"pristine_detected={primary.pristine_baseline.detected_count}/" + f"{primary.pristine_baseline.sample_count} " + f"status={primary.pristine_baseline.status.value}\n" + ) + sys.stdout.write(f"json={args.json.as_posix()}\n") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) From 4e69de04f1121c7cffcca2d9b032209006f8cfca Mon Sep 17 00:00:00 2001 From: Byte271 Date: Mon, 17 Aug 2026 21:12:38 -0400 Subject: [PATCH 5/7] Run real TinyDev fixed-FPR detector evidence --- .github/workflows/tiny-dev-detector.yml | 69 +++++++++++++++++++++++++ 1 file changed, 69 insertions(+) create mode 100644 .github/workflows/tiny-dev-detector.yml diff --git a/.github/workflows/tiny-dev-detector.yml b/.github/workflows/tiny-dev-detector.yml new file mode 100644 index 00000000..62b53e72 --- /dev/null +++ b/.github/workflows/tiny-dev-detector.yml @@ -0,0 +1,69 @@ +name: Real TinyDev Detector Evidence + +on: + workflow_dispatch: + inputs: + seed_base: + description: First deterministic paired generation seed + required: true + default: "401000" + type: string + max_attempts: + description: Deterministic retry budget for exact 64-token unique pairs + required: true + default: "16" + type: string + pull_request: + paths: + - ".github/workflows/tiny-dev-detector.yml" + - "fuckmark/experiments/tiny_dev_detector_evidence.py" + - "fuckmark/tiny_dev_detector_hf.py" + - "fuckmark/corpus/tiny_dev_io.py" + - "fuckmark/corpus/tiny_dev_generation.py" + - "fuckmark/tiny_dev_corpus_hf.py" + - "tests/test_tiny_dev_detector_evidence.py" + - "requirements-smoke.txt" + +permissions: + contents: read + +jobs: + real-detector-evidence: + runs-on: ubuntu-24.04 + timeout-minutes: 180 + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: "3.12" + cache: pip + - name: Install FuckMark + run: python -m pip install -e . + - name: Install exact CPU Torch runtime + run: python -m pip install "torch==2.13.0" --index-url https://download.pytorch.org/whl/cpu + - name: Install maintained Transformers SynthID dependencies + run: python -m pip install -r requirements-smoke.txt + - name: Reproduce real TinyDev corpus + run: >- + python -m fuckmark.tiny_dev_corpus_hf + --model openai-community/gpt2 + --model-revision 607a30d783dfa663caf39e06633721c8d4cfcd7e + --device cpu + --seed-base "${{ inputs.seed_base || '401000' }}" + --max-attempts "${{ inputs.max_attempts || '16' }}" + --temperature 0.8 + --top-k 50 + --top-p 0.95 + --json artifacts/tiny-dev-corpus.json + - name: Calibrate and evaluate Mean and Weighted Mean + run: >- + python -m fuckmark.tiny_dev_detector_hf + --corpus-json artifacts/tiny-dev-corpus.json + --device cpu + --json artifacts/tiny-dev-detector-evidence.json + - uses: actions/upload-artifact@v4 + if: always() + with: + name: real-tiny-dev-detector-evidence + path: artifacts/ + if-no-files-found: error From 6efcd2d932ed8576fda14674065ab1e0a05d8e75 Mon Sep 17 00:00:00 2001 From: Byte271 Date: Mon, 17 Aug 2026 21:24:49 -0400 Subject: [PATCH 6/7] Record verified real TinyDev detector evidence --- .../real-hf-gpt2-detectors-2026-08-18.json | 203 ++++++++++++++++++ 1 file changed, 203 insertions(+) create mode 100644 evidence/tiny_dev/real-hf-gpt2-detectors-2026-08-18.json diff --git a/evidence/tiny_dev/real-hf-gpt2-detectors-2026-08-18.json b/evidence/tiny_dev/real-hf-gpt2-detectors-2026-08-18.json new file mode 100644 index 00000000..e3f1dff2 --- /dev/null +++ b/evidence/tiny_dev/real-hf-gpt2-detectors-2026-08-18.json @@ -0,0 +1,203 @@ +{ + "algorithm_version": "real-tiny-dev-detector-evidence-v1", + "corpus": { + "artifact_hash": "42240f72e17df1a048f44197f216a0b5f30128c26e2da93afbe92820946e4561", + "attack_negative_count": 4, + "attack_positive_count": 4, + "calibration_negative_count": 100, + "manifest_hash": "693788e76bceea43e72425b98f326bc09e4840d069eb47dacb18784ec7c2e388", + "prompt_boundary_mode": "continuation-only", + "prompt_count": 104, + "sample_count": 208, + "target_length": 64, + "token_track": "original-generation-token-ids", + "watermark_config_hash": "d5918a5b92bd723b78f393d986b812f786b2855c767eb7ab60192e63d2a34086" + }, + "detector_artifact": { + "adapter_algorithm_version": "huggingface-synthid-observation-v1", + "adapter_config_hash": "0d9b6e5681240db5b28123042cd2ea23ad9b5886f0c3cb80d6efbe5089cb651f", + "adapter_id": "huggingface-transformers-synthid", + "artifact_hash": "d676033f294a60a6a63fd354b1604838ff6abae2c6f8f93f847156e7ca67c219", + "baseline_interpretability_floor": 0.8, + "headline_fprs": [ + 0.05, + 0.01 + ], + "primary_fpr": 0.01, + "scientific_status": "DEVELOPMENT_ONLY" + }, + "evidence_hash": "e83cef475c5d60be02e5694f7835ac024a1e62aa9b96cdc975e51f10b105b662", + "evidence_kind": "development-detector-calibration-and-pristine-baseline", + "families": { + "mean": { + "calibration_bundle_hash": "8951996af029f636597e991a0c701a1e16f6b3b191da831ae912dcddc0b24a78", + "family_hash": "627484ed3b211487a9fd16ccf71848de090342f1e3d94fd8819f2afbe779aeb7", + "thresholds": [ + { + "achieved_calibration_fpr": 0.05, + "attack_negative_detected_count": 0, + "attack_negative_fpr": 0.0, + "attack_negative_fpr_interval": { + "confidence_level": 0.95, + "lower": 0.0, + "method": "clopper-pearson-equal-tailed", + "upper": 0.6023646356164745 + }, + "baseline_status": "PASS", + "calibration_fpr_interval": { + "confidence_level": 0.95, + "lower": 0.01643187918205198, + "method": "clopper-pearson-equal-tailed", + "upper": 0.11283491110546334 + }, + "evaluation_hash": "c03cab481edf692bb553304500cfbe164a615432c5befdd713e71cf951da879b", + "pristine_detected_count": 4, + "pristine_sample_count": 4, + "pristine_tpr": 1.0, + "pristine_tpr_interval": { + "confidence_level": 0.95, + "lower": 0.3976353643835254, + "method": "clopper-pearson-equal-tailed", + "upper": 1.0 + }, + "target_fpr": 0.05, + "threshold_hash": "b7f9955d22432de1ddba6d753a7e26ed292f0c3b7679fa73556240ea12a979c8", + "threshold_value": 0.538888888888889 + }, + { + "achieved_calibration_fpr": 0.01, + "attack_negative_detected_count": 0, + "attack_negative_fpr": 0.0, + "attack_negative_fpr_interval": { + "confidence_level": 0.95, + "lower": 0.0, + "method": "clopper-pearson-equal-tailed", + "upper": 0.6023646356164745 + }, + "baseline_status": "PASS", + "calibration_fpr_interval": { + "confidence_level": 0.95, + "lower": 0.0002531460329774256, + "method": "clopper-pearson-equal-tailed", + "upper": 0.05445938539208044 + }, + "evaluation_hash": "2abf61653c1719a5605fb96a19487c2a96c0695635f2df1c514bd7a5af3c8acb", + "pristine_detected_count": 4, + "pristine_sample_count": 4, + "pristine_tpr": 1.0, + "pristine_tpr_interval": { + "confidence_level": 0.95, + "lower": 0.3976353643835254, + "method": "clopper-pearson-equal-tailed", + "upper": 1.0 + }, + "target_fpr": 0.01, + "threshold_hash": "ee158a3a242773fa5a7093cb422dd476a431278619930888e20c3b6d59fa74b3", + "threshold_value": 0.5555555555555555 + } + ] + }, + "weighted_mean": { + "calibration_bundle_hash": "452a2ecd65958f2bfb968730c85c96e480af84f69e40da6a27ff020b7e39c30f", + "family_hash": "c8a134afacec52258ba31ec1e7f6fa945ac2d5c4ad6592f9a8bff0a5747d3dfc", + "thresholds": [ + { + "achieved_calibration_fpr": 0.05, + "attack_negative_detected_count": 0, + "attack_negative_fpr": 0.0, + "attack_negative_fpr_interval": { + "confidence_level": 0.95, + "lower": 0.0, + "method": "clopper-pearson-equal-tailed", + "upper": 0.6023646356164745 + }, + "baseline_status": "PASS", + "calibration_fpr_interval": { + "confidence_level": 0.95, + "lower": 0.01643187918205198, + "method": "clopper-pearson-equal-tailed", + "upper": 0.11283491110546334 + }, + "evaluation_hash": "0730644cf85866d1dc0132da9cde8eac67cf6eeaf1dc82d2a1578a93991b1b3d", + "pristine_detected_count": 4, + "pristine_sample_count": 4, + "pristine_tpr": 1.0, + "pristine_tpr_interval": { + "confidence_level": 0.95, + "lower": 0.3976353643835254, + "method": "clopper-pearson-equal-tailed", + "upper": 1.0 + }, + "target_fpr": 0.05, + "threshold_hash": "1b20028af2ec9ffd30438597b750884ebddd23fb88cfb8107f14630444ca8d7e", + "threshold_value": 0.5455401472350625 + }, + { + "achieved_calibration_fpr": 0.01, + "attack_negative_detected_count": 0, + "attack_negative_fpr": 0.0, + "attack_negative_fpr_interval": { + "confidence_level": 0.95, + "lower": 0.0, + "method": "clopper-pearson-equal-tailed", + "upper": 0.6023646356164745 + }, + "baseline_status": "PASS", + "calibration_fpr_interval": { + "confidence_level": 0.95, + "lower": 0.0002531460329774256, + "method": "clopper-pearson-equal-tailed", + "upper": 0.05445938539208044 + }, + "evaluation_hash": "c69a407c4c2b5eac94fcad34da05e9aeb1ca897492ae8badf7f98213b9b3bec4", + "pristine_detected_count": 4, + "pristine_sample_count": 4, + "pristine_tpr": 1.0, + "pristine_tpr_interval": { + "confidence_level": 0.95, + "lower": 0.3976353643835254, + "method": "clopper-pearson-equal-tailed", + "upper": 1.0 + }, + "target_fpr": 0.01, + "threshold_hash": "fb518c977ed45daa128e9530e5d55989c1b6a2072f976a9ce3ebf9953311045a", + "threshold_value": 0.5616883116883117 + } + ] + } + }, + "runtime": { + "device": "cpu", + "python_version": "3.12.13", + "runner_image": "ubuntu-24.04", + "sampling_table_hash": "a3e9e18ea34546849c5136cf9b06c7bfd03e50e601da14af47cd926f9e623a85", + "sampling_table_provenance": "torch=2.13.0+cpu;device=cpu", + "torch_version": "2.13.0+cpu", + "transformers_source_commit": "a61d5f9e4fc184cff66938ff6c521cc358b5e024", + "transformers_version": "5.16.0.dev0" + }, + "scientific_scope": "DEV_KEYS-only TinyDev Mean/Weighted Mean fixed-FPR evidence; not M6 readiness and not confirmatory evidence", + "source": { + "artifact_id": 9307352458, + "artifact_name": "real-tiny-dev-detector-evidence", + "artifact_zip_sha256": "d1174d706aa4a0b5e5124a8e8c457423cea32bb2e4d3b3161cf546c503e282d9", + "corpus_json_sha256": "ee09527d06abc58e612f7157b69908d5abe06101865714ce13a1287d1f0d7664", + "detector_json_sha256": "086770a9d8e9e7d639bad6c04365f1a239042ae2799b729831199d704188ed99", + "repository": "byte271/FuckMark", + "source_head_commit": "4e69de04f1121c7cffcca2d9b032209006f8cfca", + "workflow_conclusion": "success", + "workflow_name": "Real TinyDev Detector Evidence", + "workflow_run_id": 32087405132, + "workflow_run_number": 1 + }, + "verification": { + "corpus_reproduced_original_sha256": true, + "detector_json_canonical": true, + "detector_top_level_hash_recomputed": true, + "family_hashes_recomputed": true, + "mean_primary_baseline_passed": true, + "source_workflow_succeeded": true, + "threshold_evaluation_hashes_recomputed": true, + "weighted_mean_primary_baseline_passed": true + } +} \ No newline at end of file From 8510a0d2cb3ea9c84aeed0efd56990f95c59e5a9 Mon Sep 17 00:00:00 2001 From: Byte271 Date: Mon, 17 Aug 2026 21:25:09 -0400 Subject: [PATCH 7/7] Test committed real TinyDev detector evidence --- tests/test_real_tiny_dev_detector_manifest.py | 92 +++++++++++++++++++ 1 file changed, 92 insertions(+) create mode 100644 tests/test_real_tiny_dev_detector_manifest.py diff --git a/tests/test_real_tiny_dev_detector_manifest.py b/tests/test_real_tiny_dev_detector_manifest.py new file mode 100644 index 00000000..73719183 --- /dev/null +++ b/tests/test_real_tiny_dev_detector_manifest.py @@ -0,0 +1,92 @@ +import json +from pathlib import Path + +from fuckmark.hashing import sha256_json + + +_EVIDENCE_PATH = Path("evidence/tiny_dev/real-hf-gpt2-detectors-2026-08-18.json") + + +def _evidence() -> dict[str, object]: + return json.loads(_EVIDENCE_PATH.read_text(encoding="utf-8")) + + +def test_real_tiny_dev_detector_manifest_is_self_hashed_and_development_only() -> None: + evidence = _evidence() + evidence_hash = evidence.pop("evidence_hash") + assert evidence_hash == sha256_json(evidence) + assert evidence["algorithm_version"] == "real-tiny-dev-detector-evidence-v1" + assert evidence["scientific_scope"] == ( + "DEV_KEYS-only TinyDev Mean/Weighted Mean fixed-FPR evidence; " + "not M6 readiness and not confirmatory evidence" + ) + assert evidence["detector_artifact"]["scientific_status"] == "DEVELOPMENT_ONLY" + + +def test_real_tiny_dev_detector_manifest_binds_source_run_and_reproduced_corpus() -> None: + evidence = _evidence() + source = evidence["source"] + corpus = evidence["corpus"] + assert source["source_head_commit"] == "4e69de04f1121c7cffcca2d9b032209006f8cfca" + assert source["workflow_run_id"] == 32087405132 + assert source["workflow_conclusion"] == "success" + assert source["artifact_id"] == 9307352458 + assert source["artifact_zip_sha256"] == ( + "d1174d706aa4a0b5e5124a8e8c457423cea32bb2e4d3b3161cf546c503e282d9" + ) + assert source["corpus_json_sha256"] == ( + "ee09527d06abc58e612f7157b69908d5abe06101865714ce13a1287d1f0d7664" + ) + assert source["detector_json_sha256"] == ( + "086770a9d8e9e7d639bad6c04365f1a239042ae2799b729831199d704188ed99" + ) + assert corpus["artifact_hash"] == ( + "42240f72e17df1a048f44197f216a0b5f30128c26e2da93afbe92820946e4561" + ) + assert corpus["calibration_negative_count"] == 100 + assert corpus["attack_positive_count"] == 4 + assert corpus["attack_negative_count"] == 4 + assert corpus["target_length"] == 64 + + +def test_real_tiny_dev_detector_manifest_binds_exact_runtime_and_adapter() -> None: + evidence = _evidence() + runtime = evidence["runtime"] + detector = evidence["detector_artifact"] + assert runtime["python_version"] == "3.12.13" + assert runtime["torch_version"] == "2.13.0+cpu" + assert runtime["transformers_version"] == "5.16.0.dev0" + assert runtime["transformers_source_commit"] == "a61d5f9e4fc184cff66938ff6c521cc358b5e024" + assert runtime["sampling_table_hash"] == ( + "a3e9e18ea34546849c5136cf9b06c7bfd03e50e601da14af47cd926f9e623a85" + ) + assert detector["adapter_id"] == "huggingface-transformers-synthid" + assert detector["adapter_config_hash"] == ( + "0d9b6e5681240db5b28123042cd2ea23ad9b5886f0c3cb80d6efbe5089cb651f" + ) + assert detector["headline_fprs"] == [0.05, 0.01] + assert detector["primary_fpr"] == 0.01 + + +def test_real_tiny_dev_primary_baselines_pass_but_keep_small_n_uncertainty() -> None: + families = _evidence()["families"] + expected = { + "mean": (0.5555555555555555, "ee158a3a242773fa5a7093cb422dd476a431278619930888e20c3b6d59fa74b3"), + "weighted_mean": (0.5616883116883117, "fb518c977ed45daa128e9530e5d55989c1b6a2072f976a9ce3ebf9953311045a"), + } + for family_name, (threshold, threshold_hash) in expected.items(): + primary = next( + row for row in families[family_name]["thresholds"] + if row["target_fpr"] == 0.01 + ) + assert primary["threshold_value"] == threshold + assert primary["threshold_hash"] == threshold_hash + assert primary["achieved_calibration_fpr"] == 0.01 + assert primary["pristine_detected_count"] == 4 + assert primary["pristine_sample_count"] == 4 + assert primary["pristine_tpr"] == 1.0 + assert primary["baseline_status"] == "PASS" + assert primary["pristine_tpr_interval"]["lower"] == 0.3976353643835254 + assert primary["attack_negative_detected_count"] == 0 + assert primary["attack_negative_fpr"] == 0.0 + assert primary["attack_negative_fpr_interval"]["upper"] == 0.6023646356164745