From 29d7a401b1e5a2b9e954ea75b2a525bbd28420a3 Mon Sep 17 00:00:00 2001 From: Cyinh Date: Sat, 22 Aug 2026 20:05:39 -0400 Subject: [PATCH 01/30] Freeze budget-scaled coverage effectiveness protocol v1 --- ...ed-coverage-effectiveness-v1.contract.json | 113 +++++++++++++++ ...aled-coverage-effectiveness-v1.freeze.json | 10 ++ ...aled-coverage-effectiveness-v1.protocol.md | 134 ++++++++++++++++++ 3 files changed, 257 insertions(+) create mode 100644 specs/fuckmark-budget-scaled-coverage-effectiveness-v1.contract.json create mode 100644 specs/fuckmark-budget-scaled-coverage-effectiveness-v1.freeze.json create mode 100644 specs/fuckmark-budget-scaled-coverage-effectiveness-v1.protocol.md diff --git a/specs/fuckmark-budget-scaled-coverage-effectiveness-v1.contract.json b/specs/fuckmark-budget-scaled-coverage-effectiveness-v1.contract.json new file mode 100644 index 00000000..8ab47c0a --- /dev/null +++ b/specs/fuckmark-budget-scaled-coverage-effectiveness-v1.contract.json @@ -0,0 +1,113 @@ +{ + "protocol_id": "budget-scaled-coverage-effectiveness-v1", + "algorithm_version": "budget-scaled-coverage-effectiveness-contract-v1", + "recorded_date": "2026-08-22", + "base_main_commit": "16e153fdfa3739a88bca9b1559543a10ded2d859", + "claim_boundary": { + "status": "EXPLORATORY_FIXED_OPEN_DETECTOR_ONLY", + "watermark_removal_claim": false, + "undetectability_claim": false, + "unknown_key_claim": false, + "proprietary_detector_claim": false, + "cleaning_durability_claim": false, + "release_authorized": false, + "level_b_product_admissible": false, + "level_b_failure_reason": "candidate intentionally edits visible text (spacing and contractions)" + }, + "candidate_profile": { + "profile_id": "key-blind-full-pool-coverage-v1", + "ruleset": "development forward contractions + surface + lexical + syntax (B16 line ruleset)", + "schedule_policy": "COVERAGE_GREEDY_KEY_BLIND", + "geometry_mode": "TOKENIZER_AWARE_PUBLIC", + "ngram_len": 5, + "replicate_count": 1, + "budget_selection": "frozen rule in protocol section 7", + "development_sweep_budgets": [16, 24, 32, 48], + "selection_access": { + "detector_scores": false, + "detector_decisions": false, + "watermark_secret_keys": false, + "confirmation_results": false, + "public_tokenizer_geometry": true, + "public_detector_ngram_len": true + } + }, + "corpora": { + "model_id": "openai-community/gpt2", + "model_revision": "607a30d783dfa663caf39e06633721c8d4cfcd7e", + "generation": { + "device": "cpu", + "temperature": 0.8, + "top_k": 50, + "top_p": 0.95, + "target_length": 64, + "max_attempts": 16 + }, + "watermark_keys_public_dev": [654, 400, 836, 123, 340, 443, 597, 160, 57], + "development": [ + {"seed_base": 401000, "profile": "tiny-dev-v2-canonical", "attack_pairs_per_domain": 1}, + {"seed_base": 402000, "profile": "tiny-dev-v3-parameterized", "attack_pairs_per_domain": 8} + ], + "confirmation": { + "seed_base": 403000, + "profile": "tiny-dev-v3-parameterized", + "attack_pairs_per_domain": 16, + "watermarked_attack_sources": 64, + "unwatermarked_attack_controls": 64, + "calibration_pairs_per_domain": 25 + }, + "disjointness": "pairwise-distinct text sha256 across corpora; confirmation disjoint from all development texts; asserted at scoring time" + }, + "detector": { + "family": "Hugging Face SynthID Weighted Mean", + "ngram_len": 5, + "input_path": "uncleaned text-only re-tokenized track", + "calibration": "100 unwatermarked calibration negatives of the same corpus", + "target_fpr": 0.01, + "comparison_operator": ">=", + "threshold_scope": "recomputed per corpus from its own calibration split" + }, + "fidelity_gates": { + "hard_invariants": "PASS required for every row", + "protected_span_violations": 0, + "word_edit_rate_max": 0.15, + "ascii_gate": true, + "forbidden_codepoint_categories": ["Cc", "Cf", "Mn", "Mc", "Me", "Zl", "Zp"], + "allowed_space": "U+0020 only", + "normalization_noop": ["NFC", "NFD", "NFKC", "NFKD"], + "cf_stripping_noop": true + }, + "controls": [ + "pristine_watermarked", + "transformed_watermarked", + "pristine_unwatermarked", + "transformed_unwatermarked" + ], + "budget_selection_rule": { + "eligibility": "development transformed-watermarked detected count == 0 AND all fidelity gates pass on all development rows", + "selection": "smallest eligible budget from [16, 24, 32, 48]", + "fallback": "smallest development transformed-watermarked detected count, ties toward smaller budget", + "freeze": "selected budget recorded in freeze-record appendix before any confirmation scoring" + }, + "primary_endpoint": { + "target": "confirmation transformed watermarked detected count == 0 out of 64", + "interpretation_if_met": "narrow exploratory detector-specific target met for this detector, keys, and input path only" + }, + "secondary_endpoints": { + "pristine_watermarked_detected_min": 60, + "transformed_unwatermarked_detected_max": 0, + "gate_pass_fraction_required": 1.0 + }, + "artifacts": [ + "specs/fuckmark-budget-scaled-coverage-effectiveness-v1.freeze.json", + "artifacts/budget-scaled/dev-corpus-402000.json", + "artifacts/budget-scaled/dev-plan-{budget}.json", + "artifacts/budget-scaled/dev-evidence-{budget}.json", + "artifacts/budget-scaled/confirmation-corpus-403000.json", + "artifacts/budget-scaled/confirmation-plan.json", + "artifacts/budget-scaled/confirmation-evidence.json", + "artifacts/budget-scaled/robustness-report.json", + "artifacts/budget-scaled/rendering-manifest.json", + "artifacts/budget-scaled/final-report.md" + ] +} diff --git a/specs/fuckmark-budget-scaled-coverage-effectiveness-v1.freeze.json b/specs/fuckmark-budget-scaled-coverage-effectiveness-v1.freeze.json new file mode 100644 index 00000000..2918d93b --- /dev/null +++ b/specs/fuckmark-budget-scaled-coverage-effectiveness-v1.freeze.json @@ -0,0 +1,10 @@ +{ + "appendix_confirmation_budget": null, + "base_main_commit": "16e153fdfa3739a88bca9b1559543a10ded2d859", + "contract_sha256": "e9cec62e6f208db7ed8f13e47b0e7657fc9808d74ce0979115a8ab30f4f8043e", + "freeze_record_version": "budget-scaled-coverage-freeze-v1", + "frozen_at_utc": "2026-08-23T00:05:32.588787+00:00", + "pre_score_amendments": [], + "protocol_id": "budget-scaled-coverage-effectiveness-v1", + "protocol_sha256": "ad657b3ff9a719a5e3c06f268b1d8281dcf84208e5735ec0a173cc8e886fd688" +} diff --git a/specs/fuckmark-budget-scaled-coverage-effectiveness-v1.protocol.md b/specs/fuckmark-budget-scaled-coverage-effectiveness-v1.protocol.md new file mode 100644 index 00000000..709e3fb8 --- /dev/null +++ b/specs/fuckmark-budget-scaled-coverage-effectiveness-v1.protocol.md @@ -0,0 +1,134 @@ +# Frozen Research Protocol: Budget-Scaled Key-Blind Coverage Effectiveness (v1) + +Protocol ID: `budget-scaled-coverage-effectiveness-v1` +Frozen before any detector scoring for this cycle. See the machine-readable contract in +`specs/fuckmark-budget-scaled-coverage-effectiveness-v1.contract.json` and the hash-binding +freeze record in `specs/fuckmark-budget-scaled-coverage-effectiveness-v1.freeze.json`. + +## 1. Objective and claim boundary + +Level A (this protocol): for the fixed, documented open detector and the existing uncleaned +text-only input path, measure whether a budget-scaled, detector-blind, key-blind transform +profile lowers watermarked held-out detection on a new source-disjoint confirmation corpus while +preserving all pre-registered fidelity gates. The stretch exploratory target is 0 detected +watermarked sources out of at least 64. A 0/64 result is a narrow detector-specific measurement +under the documented fixed input path; it is not watermark removal, universal undetectability, +immunity to input cleaning, or safety against unknown keys or proprietary detectors. + +Level B (product/release admissibility): this candidate family intentionally edits visible text +(spacing and contractions), so it is NOT a product candidate this cycle regardless of Level A +outcome. No candidate output will be added to default, development-release, or CLI registries. + +## 2. Candidate mechanism + +Profile `key-blind-full-pool-coverage-v1`: the frozen development ruleset already used by the +B16 line (ruleset hash bound in artifacts; forward contractions, surface spacing, lexical, and +syntax rules), the COVERAGE_GREEDY_KEY_BLIND scheduler over public-tokenizer n-gram disruption +geometry (ngram_len 5), one replicate, and schedule seeds derived from the frozen seed base plus +sorted source index. The only new degree of freedom is the requested budget, swept on +development data and then frozen by the pre-registered rule in section 7. Selection reads the +public tokenizer and public detector n-gram length only; it never reads detector scores, +decisions, keys beyond the public development keys, or confirmation results. + +## 3. Data splits and source disjointness + +- Development: the canonical pinned TinyDev corpus (seed base 401000, corpus artifact hash + recorded at generation time) plus one fresh development corpus at seed base 402000 with 8 + attack pairs per domain (32 watermarked attack sources). Pooled development watermarked + attack sources: 36. +- Confirmation: one fresh corpus at seed base 403000 with 16 attack pairs per domain + (64 watermarked attack sources plus 64 unwatermarked attack controls), generated only after + the protocol, contract, and freeze record exist, and scored exactly once. +- Every corpus keeps the frozen 25 threshold-calibration pairs per domain (100 negatives). +- Disjointness gates: no text SHA-256 may appear in more than one corpus used by this protocol; + confirmation text hashes must be disjoint from all development text hashes; recorded + automatically at scoring time. Neither corpus reuses any example from the archived B10 or + B16 holdout corpora; disjointness from those corpora is asserted by artifact-hash inequality + and text-hash disjointness against any locally available recorded holdout text hashes. + +## 4. Generation and detector configuration + +- Model: `openai-community/gpt2` at revision `607a30d783dfa663caf39e06633721c8d4cfcd7e`, + CPU, temperature 0.8, top-k 50, top-p 0.95, 64 target continuation tokens, max 16 attempts + per pair, seeds derived from the corpus seed base by the existing frozen seed policy. +- Watermarking: Hugging Face SynthID text integration with the public development keys + (654, 400, 836, 123, 340, 443, 597, 160, 57), ngram_len 5, context history 1024, sampling + table seed 0, size 65536, debug off. +- Detector: the Hugging Face SynthID weighted-mean open detector over the text-only + re-tokenized track, ngram_len 5, public development keys (the detector is the open research + harness detector; the keys are public development keys, not provider secrets). +- Threshold: text-only calibration on the 100 unwatermarked calibration negatives of the SAME + corpus being scored, target FPR 0.01, decision rule score >= threshold. Threshold value and + identity hashes are bound inside every scoring artifact. The confirmation threshold is + computed only from the confirmation corpus calibration split. + +## 5. Controls + +All four paired controls are scored for every source: pristine watermarked, transformed +watermarked, pristine unwatermarked, transformed unwatermarked. Summaries report raw counts, +rates, mean scores, mean per-source score change, and false positives, overall and per domain. + +## 6. Fidelity, representation, and robustness gates + +Gates evaluated on every plan row, development and confirmation: + +1. Hard invariants: status PASS (enforced at plan build; any failure aborts). +2. Protected spans: zero violations under the symmetric protected-span counter. +3. Word edit rate: word-level edit distance divided by source word count must be <= 0.15. +4. ASCII gate: transformed text `isascii()` must be True (rules are ASCII-only by design). +5. Codepoint audit: transformed text must contain no Unicode Cf, Cc, Mn, Mc, Me, Zl, Zp, or Zs + characters other than U+0020, and no character outside the source alphabet union rule + alphabet. +6. Normalization no-op: NFC, NFD, NFKC, and NFKD of every transformed text must equal the + transformed text itself; Unicode Cf stripping must be the identity function on every + transformed text. Any non-identity result is an adverse finding and a Level B failure. +7. Copy/DOM: transformed text must contain no control or format characters, so DOM + textContent equals the transformed string; verified by the codepoint audit and archived + fixed-browser renders. +8. Fixed-browser rendering: every confirmation watermarked source/transform pair is rendered + side by side in a fixed browser and screenshotted; the rendered DOM text must equal the + plan transformed text. Screenshots are archived; the deterministic gates above carry the + decision weight. + +## 7. Pre-registered budget selection rule (frozen) + +Development sweep budgets: 16, 24, 32, 48. For each budget, compute on the pooled 36 +development watermarked attack sources: transformed detected count at the development +thresholds, and gate status for every development row. A budget is eligible when its +development transformed-watermarked detected count is 0 and all gates pass on all development +rows at that budget. The confirmation budget is the smallest eligible budget. If no budget is +eligible, the confirmation budget is the budget with the smallest development +transformed-watermarked detected count (ties broken toward the smaller budget); in that case +the primary target is expected to fail and the confirmation run proceeds and reports honestly. +The rule is fully determined before any confirmation scoring; no post-hoc budget change is +permitted. No confirmation score may be computed before the confirmation budget is recorded in +the freeze record appendix. + +## 8. Success and falsification criteria + +Primary (Level A stretch target): transformed watermarked detected count == 0 out of 64 +confirmation watermarked sources at the frozen confirmation threshold. + +Mandatory secondary gates on the confirmation corpus: pristine watermarked detected >= 60/64; +transformed unwatermarked detected == 0/64; every gate in section 6 passes on 100% of +confirmation rows; all four controls' artifacts replay and hash-bind. + +Adverse results that must be reported, never hidden: any residual transformed-watermarked +detection; any gate failure; any normalization or Cf-stripping non-identity; any pristine +watermarked source below threshold (weak watermark); any false positive; per-domain or +per-length heterogeneity. + +Permitted conclusion language is exactly: detector-specific detection reduction under the +documented fixed open detector, public development keys, and uncleaned text-only input path. +Prohibited: watermark removal, undetectability, untraceability, robustness against cleaning, +unknown-key or proprietary-detector claims, release readiness. + +## 9. Reproducibility and artifacts + +Canonical JSON artifacts with SHA-256 hashes: corpora (+ provenance), plans (+ provenance), +scoring evidence (+ provenance), robustness report, rendering archive manifest, final report, +and the freeze record appendix recording the rule-selected confirmation budget before +confirmation scoring. All artifacts bind the source code commit, corpus artifact hash, +plan hash, profile hash, ruleset hash, detector identity hash, calibration bundle hash, and +threshold hash. Full test suite, `git diff --check`, and `git status --short --branch` run at +the end. No pushes, PRs, merges, or releases in this cycle. From 19a38015a4b1549b5bb1a44ad2efbab5e2fde167 Mon Sep 17 00:00:00 2001 From: Cyinh Date: Sat, 22 Aug 2026 20:14:34 -0400 Subject: [PATCH 02/30] Add corpus profile v3, full-pool coverage profile, parameterized runners, and frozen protocol amendments --- fuckmark/corpus/__init__.py | 24 ++ fuckmark/corpus/tiny_dev_io_v3.py | 132 +++++++++ fuckmark/corpus/tiny_dev_v3.py | 265 ++++++++++++++++++ fuckmark/corpus/tiny_dev_v3_generation.py | 211 ++++++++++++++ fuckmark/tiny_dev_corpus_hf.py | 45 ++- fuckmark/tiny_dev_effectiveness_plan_hf.py | 43 ++- fuckmark/tiny_dev_effectiveness_score_hf.py | 56 +++- fuckmark/transforms/__init__.py | 8 + fuckmark/transforms/effectiveness_profile.py | 40 +++ ...ed-coverage-effectiveness-v1.contract.json | 8 +- ...aled-coverage-effectiveness-v1.freeze.json | 36 ++- ...aled-coverage-effectiveness-v1.protocol.md | 4 +- tests/test_effectiveness_full_pool_profile.py | 152 ++++++++++ tests/test_tiny_dev_corpus_v3.py | 179 ++++++++++++ 14 files changed, 1171 insertions(+), 32 deletions(-) create mode 100644 fuckmark/corpus/tiny_dev_io_v3.py create mode 100644 fuckmark/corpus/tiny_dev_v3.py create mode 100644 fuckmark/corpus/tiny_dev_v3_generation.py create mode 100644 tests/test_effectiveness_full_pool_profile.py create mode 100644 tests/test_tiny_dev_corpus_v3.py diff --git a/fuckmark/corpus/__init__.py b/fuckmark/corpus/__init__.py index 1166bd81..73ebc2b3 100644 --- a/fuckmark/corpus/__init__.py +++ b/fuckmark/corpus/__init__.py @@ -38,6 +38,21 @@ load_tiny_dev_corpus_json, parse_tiny_dev_corpus_json, ) +from .tiny_dev_v3 import ( + TINY_DEV_V3_CORPUS_ALGORITHM_VERSION, + TINY_DEV_V3_MAX_ATTACK_PAIRS_PER_DOMAIN, + TinyDevV3CorpusArtifact, + build_tiny_dev_v3_corpus, +) +from .tiny_dev_v3_generation import ( + build_real_tiny_dev_v3_corpus, + build_tiny_dev_v3_prompt_records, +) +from .tiny_dev_io_v3 import ( + load_tiny_dev_corpus_by_version_json, + load_tiny_dev_v3_corpus_json, + parse_tiny_dev_v3_corpus_json, +) from .tokenization import GenerationTokenRecord, TextOnlyTokenRecord, TokenTrack from .mid_dev_calibration_independence_v3 import ( MID_DEV_CALIBRATION_INDEPENDENCE_V3_SELECTION_RULE, @@ -78,6 +93,8 @@ "TINY_DEV_JSON_MAX_BYTES", "TINY_DEV_SPLITS", "TINY_DEV_TARGET_LENGTH", + "TINY_DEV_V3_CORPUS_ALGORITHM_VERSION", + "TINY_DEV_V3_MAX_ATTACK_PAIRS_PER_DOMAIN", "CorpusDomain", "CorpusIntegrityError", "CorpusLeakageError", @@ -107,14 +124,20 @@ "TinyDevCorpusCell", "TinyDevCorpusError", "TinyDevCorpusJsonError", + "TinyDevV3CorpusArtifact", "TokenTrack", "WatermarkCondition", "WatermarkLabel", "build_corpus_manifest", "build_calibration_independence_v3", "build_calibration_independence_v3_from_samples", + "build_real_tiny_dev_v3_corpus", "build_tiny_dev_corpus", + "build_tiny_dev_v3_corpus", + "build_tiny_dev_v3_prompt_records", "load_tiny_dev_corpus_json", + "load_tiny_dev_corpus_by_version_json", + "load_tiny_dev_v3_corpus_json", "load_mid_dev_calibration_independence_v3_artifact_json", "load_mid_dev_calibration_independence_v3_exclusion_json", "load_mid_dev_calibration_independence_v3_manifest_json", @@ -122,4 +145,5 @@ "parse_mid_dev_calibration_independence_v3_exclusion_json", "parse_mid_dev_calibration_independence_v3_manifest_json", "parse_tiny_dev_corpus_json", + "parse_tiny_dev_v3_corpus_json", ] diff --git a/fuckmark/corpus/tiny_dev_io_v3.py b/fuckmark/corpus/tiny_dev_io_v3.py new file mode 100644 index 00000000..59c0306c --- /dev/null +++ b/fuckmark/corpus/tiny_dev_io_v3.py @@ -0,0 +1,132 @@ +from __future__ import annotations + +import json +from pathlib import Path + +from ..config import canonical_json_text +from .schema import CorpusDomain, CorpusSplit +from .tiny_dev_io import ( + TINY_DEV_JSON_MAX_BYTES, + TinyDevCorpusJsonError, + _array, + _manifest, + _mapping, + _reject_constant, + _unique_object, +) +from .tiny_dev import TinyDevCorpusError +from .tiny_dev_v3 import ( + TINY_DEV_V3_CORPUS_ALGORITHM_VERSION, + TINY_DEV_V3_MAX_ATTACK_PAIRS_PER_DOMAIN, + TinyDevV3CorpusArtifact, + TinyDevV3CorpusCell, +) + + +def _cell(value: object) -> TinyDevV3CorpusCell: + data = _mapping("cell", value, ("split", "domain", "pair_count")) + return TinyDevV3CorpusCell( + split=CorpusSplit(data["split"]), + domain=CorpusDomain(data["domain"]), + pair_count=data["pair_count"], + ) + + +def _artifact_v3(value: object) -> TinyDevV3CorpusArtifact: + data = _mapping( + "artifact", + value, + ( + "algorithm_version", + "manifest", + "target_length", + "calibration_pairs_per_domain", + "attack_pairs_per_domain", + "required_splits", + "required_domains", + "model_identity_hash", + "generation_matching_signature_hash", + "watermark_condition_hash", + "cells", + "artifact_hash", + ), + ) + return TinyDevV3CorpusArtifact( + algorithm_version=data["algorithm_version"], + manifest=_manifest(data["manifest"]), + target_length=data["target_length"], + calibration_pairs_per_domain=data["calibration_pairs_per_domain"], + attack_pairs_per_domain=data["attack_pairs_per_domain"], + required_splits=tuple( + CorpusSplit(item) for item in _array("artifact.required_splits", data["required_splits"]) + ), + required_domains=tuple( + CorpusDomain(item) for item in _array("artifact.required_domains", data["required_domains"]) + ), + model_identity_hash=data["model_identity_hash"], + generation_matching_signature_hash=data["generation_matching_signature_hash"], + watermark_condition_hash=data["watermark_condition_hash"], + cells=tuple(_cell(item) for item in _array("artifact.cells", data["cells"])), + artifact_hash=data["artifact_hash"], + ) + + +def parse_tiny_dev_v3_corpus_json(text: str, *, require_canonical: bool = True) -> TinyDevV3CorpusArtifact: + if not isinstance(text, str): + raise TypeError("text must be a string") + if len(text.encode("utf-8")) > TINY_DEV_JSON_MAX_BYTES: + raise TinyDevCorpusJsonError("tiny-dev v3 JSON exceeds the size limit") + try: + decoded = json.loads( + text, + object_pairs_hook=_unique_object, + parse_constant=_reject_constant, + ) + except TinyDevCorpusJsonError: + raise + except (json.JSONDecodeError, UnicodeError) as error: + raise TinyDevCorpusJsonError("tiny-dev v3 JSON is not valid JSON") from error + try: + artifact = _artifact_v3(decoded) + except (KeyError, TypeError, ValueError) as error: + if isinstance(error, TinyDevCorpusJsonError): + raise + raise TinyDevCorpusJsonError("tiny-dev v3 JSON failed artifact validation") from error + if require_canonical: + canonical = canonical_json_text(artifact) + if text not in (canonical, canonical + "\n"): + raise TinyDevCorpusJsonError("tiny-dev v3 JSON is not in canonical serialized form") + return artifact + + +def load_tiny_dev_v3_corpus_json(path: str | Path, *, require_canonical: bool = True) -> TinyDevV3CorpusArtifact: + if not isinstance(path, (str, Path)): + raise TypeError("path must be a string or Path") + file_path = Path(path) + size = file_path.stat().st_size + if size > TINY_DEV_JSON_MAX_BYTES: + raise TinyDevCorpusJsonError("tiny-dev v3 JSON exceeds the size limit") + try: + text = file_path.read_text(encoding="utf-8") + except UnicodeDecodeError as error: + raise TinyDevCorpusJsonError("tiny-dev v3 JSON must be UTF-8") from error + return parse_tiny_dev_v3_corpus_json(text, require_canonical=require_canonical) + + +def load_tiny_dev_corpus_by_version_json(path: str | Path): + if not isinstance(path, (str, Path)): + raise TypeError("path must be a string or Path") + file_path = Path(path) + text = file_path.read_text(encoding="utf-8") + try: + decoded = json.loads(text) + except json.JSONDecodeError as error: + raise TinyDevCorpusJsonError("tiny-dev corpus JSON is not valid JSON") from error + if not isinstance(decoded, dict): + raise TinyDevCorpusJsonError("tiny-dev corpus JSON must be an object") + version = decoded.get("algorithm_version") + if version == TINY_DEV_V3_CORPUS_ALGORITHM_VERSION: + return parse_tiny_dev_v3_corpus_json(text) + from .tiny_dev_io import parse_tiny_dev_corpus_json + + return parse_tiny_dev_corpus_json(text) diff --git a/fuckmark/corpus/tiny_dev_v3.py b/fuckmark/corpus/tiny_dev_v3.py new file mode 100644 index 00000000..dca17b38 --- /dev/null +++ b/fuckmark/corpus/tiny_dev_v3.py @@ -0,0 +1,265 @@ +from __future__ import annotations + +from collections import Counter +from collections.abc import Sequence +from dataclasses import dataclass + +from .._validation import require_clean_string, require_int, require_sha256 +from ..hashing import sha256_json +from .manifest import CorpusManifest, build_corpus_manifest +from .prompt import PromptRecord +from .sample import CorpusSample +from .schema import CorpusDomain, CorpusSplit, KeySplit, TARGET_LENGTHS +from .tiny_dev import ( + TINY_DEV_CALIBRATION_PAIRS_PER_DOMAIN, + TINY_DEV_SPLITS, + TINY_DEV_TARGET_LENGTH, + TINY_DEV_DOMAINS, + TinyDevCorpusError, +) + + +TINY_DEV_V3_CORPUS_ALGORITHM_VERSION = "tiny-dev-corpus-v3" +TINY_DEV_V3_MAX_ATTACK_PAIRS_PER_DOMAIN = 16 + + +def _validate_attack_pairs_per_domain(attack_pairs_per_domain: int) -> None: + require_int("attack_pairs_per_domain", attack_pairs_per_domain) + if attack_pairs_per_domain < 1 or attack_pairs_per_domain > TINY_DEV_V3_MAX_ATTACK_PAIRS_PER_DOMAIN: + raise TinyDevCorpusError( + "v3 attack pairs per domain must lie between 1 and " + f"{TINY_DEV_V3_MAX_ATTACK_PAIRS_PER_DOMAIN}" + ) + + +@dataclass(frozen=True, slots=True) +class TinyDevV3CorpusCell: + split: CorpusSplit + domain: CorpusDomain + pair_count: int + + def __post_init__(self) -> None: + if not isinstance(self.split, CorpusSplit): + raise TypeError("split must be a CorpusSplit") + if not isinstance(self.domain, CorpusDomain): + raise TypeError("domain must be a CorpusDomain") + require_int("pair_count", self.pair_count) + if self.pair_count < 1: + raise ValueError("tiny development v3 corpus cell pair count must be positive") + + +@dataclass(frozen=True, slots=True) +class TinyDevV3CorpusArtifact: + algorithm_version: str + manifest: CorpusManifest + target_length: int + calibration_pairs_per_domain: int + attack_pairs_per_domain: int + required_splits: tuple[CorpusSplit, ...] + required_domains: tuple[CorpusDomain, ...] + model_identity_hash: str + generation_matching_signature_hash: str + watermark_condition_hash: str + cells: tuple[TinyDevV3CorpusCell, ...] + artifact_hash: str + + def __post_init__(self) -> None: + require_clean_string("algorithm_version", self.algorithm_version) + if self.algorithm_version != TINY_DEV_V3_CORPUS_ALGORITHM_VERSION: + raise ValueError("unsupported tiny development v3 corpus algorithm version") + if not isinstance(self.manifest, CorpusManifest): + raise TypeError("manifest must be a CorpusManifest") + require_int("target_length", self.target_length) + if self.target_length != TINY_DEV_TARGET_LENGTH or self.target_length not in TARGET_LENGTHS: + raise ValueError("tiny development v3 corpus target length must be 64") + require_int("calibration_pairs_per_domain", self.calibration_pairs_per_domain) + if self.calibration_pairs_per_domain != TINY_DEV_CALIBRATION_PAIRS_PER_DOMAIN: + raise ValueError("v3 calibration pair count must match the frozen calibration profile") + _validate_attack_pairs_per_domain(self.attack_pairs_per_domain) + if self.required_splits != TINY_DEV_SPLITS: + raise ValueError("tiny development v3 corpus split profile does not match the frozen profile") + if self.required_domains != TINY_DEV_DOMAINS: + raise ValueError("tiny development v3 corpus domain profile does not match the frozen profile") + require_sha256("model_identity_hash", self.model_identity_hash) + require_sha256("generation_matching_signature_hash", self.generation_matching_signature_hash) + require_sha256("watermark_condition_hash", self.watermark_condition_hash) + if not isinstance(self.cells, tuple): + raise TypeError("cells must be a tuple") + if any(not isinstance(value, TinyDevV3CorpusCell) for value in self.cells): + raise TypeError("cells must contain TinyDevV3CorpusCell values") + expected_cells = _expected_v3_cells(self.attack_pairs_per_domain) + if self.cells != expected_cells: + raise ValueError("tiny development v3 corpus cells do not match the parameterized matrix") + _validate_v3_manifest( + self.manifest, + self.model_identity_hash, + self.generation_matching_signature_hash, + self.watermark_condition_hash, + self.attack_pairs_per_domain, + ) + require_sha256("artifact_hash", self.artifact_hash) + if self.artifact_hash != sha256_json(self._payload()): + raise ValueError("artifact_hash does not match tiny development v3 corpus artifact") + + def _payload(self) -> dict[str, object]: + return { + "algorithm_version": self.algorithm_version, + "manifest_hash": self.manifest.manifest_hash, + "target_length": self.target_length, + "calibration_pairs_per_domain": self.calibration_pairs_per_domain, + "attack_pairs_per_domain": self.attack_pairs_per_domain, + "required_splits": tuple(value.value for value in self.required_splits), + "required_domains": tuple(value.value for value in self.required_domains), + "model_identity_hash": self.model_identity_hash, + "generation_matching_signature_hash": self.generation_matching_signature_hash, + "watermark_condition_hash": self.watermark_condition_hash, + "cells": self.cells, + } + + +def _expected_v3_cells(attack_pairs_per_domain: int) -> tuple[TinyDevV3CorpusCell, ...]: + return tuple( + TinyDevV3CorpusCell(split, domain, _v3_pairs_for_split(split, attack_pairs_per_domain)) + for split in TINY_DEV_SPLITS + for domain in TINY_DEV_DOMAINS + ) + + +def _v3_pairs_for_split(split: CorpusSplit, attack_pairs_per_domain: int) -> int: + if split is CorpusSplit.THRESHOLD_CALIBRATION: + return TINY_DEV_CALIBRATION_PAIRS_PER_DOMAIN + if split is CorpusSplit.ATTACK_DEVELOPMENT: + return attack_pairs_per_domain + raise TinyDevCorpusError("split is outside the frozen tiny development profile") + + +def _validate_v3_manifest( + manifest: CorpusManifest, + model_identity_hash: str, + generation_matching_signature_hash: str, + watermark_condition_hash: str, + attack_pairs_per_domain: int, +) -> None: + prompts = manifest.prompts + samples = manifest.samples + expected_pair_count = sum( + _v3_pairs_for_split(split, attack_pairs_per_domain) * len(TINY_DEV_DOMAINS) + for split in TINY_DEV_SPLITS + ) + expected_sample_count = expected_pair_count * 2 + if len(samples) != expected_sample_count: + raise TinyDevCorpusError( + f"tiny development v3 corpus must contain exactly {expected_sample_count} samples" + ) + if len({sample.match_id for sample in samples}) != expected_pair_count: + raise TinyDevCorpusError( + f"tiny development v3 corpus must contain exactly {expected_pair_count} matched pairs" + ) + if len(prompts) != expected_pair_count: + raise TinyDevCorpusError("tiny development v3 corpus must contain exactly one prompt per matched pair") + allowed_splits = set(TINY_DEV_SPLITS) + allowed_domains = set(TINY_DEV_DOMAINS) + if any(prompt.split not in allowed_splits for prompt in prompts): + raise TinyDevCorpusError("tiny development v3 prompts may use only calibration or attack-development splits") + if any(prompt.domain not in allowed_domains for prompt in prompts): + raise TinyDevCorpusError("tiny development v3 prompts must use the frozen four-domain profile") + if any(sample.split not in allowed_splits for sample in samples): + raise TinyDevCorpusError("tiny development v3 samples may use only calibration or attack-development splits") + if any(sample.domain not in allowed_domains for sample in samples): + raise TinyDevCorpusError("tiny development v3 samples must use the frozen four-domain profile") + if any(sample.target_length != TINY_DEV_TARGET_LENGTH for sample in samples): + raise TinyDevCorpusError("tiny development v3 samples must target 64 continuation tokens") + if any(sample.watermark.key_split is not KeySplit.DEV for sample in samples): + raise TinyDevCorpusError("tiny development v3 samples must use DEV_KEYS only") + if {sample.model.identity_hash for sample in samples} != {model_identity_hash}: + raise TinyDevCorpusError("tiny development v3 corpus must use one frozen model/tokenizer identity") + if {sample.generation.matching_signature_hash for sample in samples} != { + generation_matching_signature_hash + }: + raise TinyDevCorpusError("tiny development v3 corpus must use one matched generation-parameter signature") + if {sample.watermark.condition_hash for sample in samples} != {watermark_condition_hash}: + raise TinyDevCorpusError("tiny development v3 corpus must use one frozen DEV watermark condition") + prompt_counts = Counter(sample.prompt_id for sample in samples) + if any(prompt_counts[prompt.prompt_id] != 2 for prompt in prompts): + raise TinyDevCorpusError("each tiny development v3 prompt must bind exactly one matched on/off pair") + cell_pairs: Counter[tuple[CorpusSplit, CorpusDomain]] = Counter() + seen_match_ids: set[str] = set() + for sample in samples: + if sample.match_id in seen_match_ids: + continue + seen_match_ids.add(sample.match_id) + cell_pairs[(sample.split, sample.domain)] += 1 + expected_cells = { + (split, domain): _v3_pairs_for_split(split, attack_pairs_per_domain) + for split in TINY_DEV_SPLITS + for domain in TINY_DEV_DOMAINS + } + if dict(cell_pairs) != expected_cells: + raise TinyDevCorpusError("tiny development v3 corpus split/domain pair counts do not match the profile") + calibration_negatives = sum( + 1 + for sample in samples + if sample.split is CorpusSplit.THRESHOLD_CALIBRATION + and sample.label.value == "unwatermarked" + ) + if calibration_negatives < 100: + raise TinyDevCorpusError("tiny development v3 corpus must contain at least 100 calibration negatives") + + +def build_tiny_dev_v3_corpus( + corpus_id: str, + prompts: Sequence[PromptRecord], + samples: Sequence[CorpusSample], + *, + attack_pairs_per_domain: int, +) -> TinyDevV3CorpusArtifact: + require_clean_string("corpus_id", corpus_id) + _validate_attack_pairs_per_domain(attack_pairs_per_domain) + manifest = build_corpus_manifest(corpus_id, prompts, samples) + model_hashes = {sample.model.identity_hash for sample in manifest.samples} + generation_hashes = {sample.generation.matching_signature_hash for sample in manifest.samples} + watermark_hashes = {sample.watermark.condition_hash for sample in manifest.samples} + if len(model_hashes) != 1: + raise TinyDevCorpusError("tiny development v3 corpus must use exactly one model/tokenizer identity") + if len(generation_hashes) != 1: + raise TinyDevCorpusError("tiny development v3 corpus must use exactly one generation matching signature") + if len(watermark_hashes) != 1: + raise TinyDevCorpusError("tiny development v3 corpus must use exactly one watermark condition") + model_identity_hash = next(iter(model_hashes)) + generation_matching_signature_hash = next(iter(generation_hashes)) + watermark_condition_hash = next(iter(watermark_hashes)) + _validate_v3_manifest( + manifest, + model_identity_hash, + generation_matching_signature_hash, + watermark_condition_hash, + attack_pairs_per_domain, + ) + cells = _expected_v3_cells(attack_pairs_per_domain) + payload = { + "algorithm_version": TINY_DEV_V3_CORPUS_ALGORITHM_VERSION, + "manifest_hash": manifest.manifest_hash, + "target_length": TINY_DEV_TARGET_LENGTH, + "calibration_pairs_per_domain": TINY_DEV_CALIBRATION_PAIRS_PER_DOMAIN, + "attack_pairs_per_domain": attack_pairs_per_domain, + "required_splits": tuple(value.value for value in TINY_DEV_SPLITS), + "required_domains": tuple(value.value for value in TINY_DEV_DOMAINS), + "model_identity_hash": model_identity_hash, + "generation_matching_signature_hash": generation_matching_signature_hash, + "watermark_condition_hash": watermark_condition_hash, + "cells": cells, + } + return TinyDevV3CorpusArtifact( + TINY_DEV_V3_CORPUS_ALGORITHM_VERSION, + manifest, + TINY_DEV_TARGET_LENGTH, + TINY_DEV_CALIBRATION_PAIRS_PER_DOMAIN, + attack_pairs_per_domain, + TINY_DEV_SPLITS, + TINY_DEV_DOMAINS, + model_identity_hash, + generation_matching_signature_hash, + watermark_condition_hash, + cells, + sha256_json(payload), + ) diff --git a/fuckmark/corpus/tiny_dev_v3_generation.py b/fuckmark/corpus/tiny_dev_v3_generation.py new file mode 100644 index 00000000..5db28ed2 --- /dev/null +++ b/fuckmark/corpus/tiny_dev_v3_generation.py @@ -0,0 +1,211 @@ +from __future__ import annotations + +from ..hashing import sha256_json, sha256_text +from .prompt import PromptRecord +from .schema import CorpusDomain, CorpusSplit, KeySplit, MAX_GENERATION_SEED, WatermarkLabel +from .tiny_dev import ( + TINY_DEV_CALIBRATION_PAIRS_PER_DOMAIN, + TINY_DEV_SPLITS, + TINY_DEV_TARGET_LENGTH, + TinyDevCorpusError, +) +from .tiny_dev_generation import ( + TINY_DEV_DEFAULT_MAX_ATTEMPTS, + TINY_DEV_PAIR_SEED_STRIDE, + TINY_DEV_PROMPT_LICENSE_ID, + TINY_DEV_PROMPT_PROVENANCE, + TINY_DEV_PROMPT_SOURCE_ID, + TINY_DEV_SEED_POLICY_ID, + TinyDevGenerationBackend, + TinyDevGenerationError, + _TINY_DEV_TEMPLATES, + _TINY_DEV_TOPICS, + _build_sample, +) +from .tiny_dev_v3 import ( + TINY_DEV_V3_MAX_ATTACK_PAIRS_PER_DOMAIN, + TinyDevV3CorpusArtifact, + build_tiny_dev_v3_corpus, +) + + +TINY_DEV_V3_GENERATION_ALGORITHM_VERSION = "tiny-dev-v3-generation-v1" +TINY_DEV_V3_PROMPT_SOURCE_ID = "fuckmark-tiny-dev-v3-prompts-v1" +TINY_DEV_V3_DOMAINS = tuple(CorpusDomain) + +_TINY_DEV_V3_ATTACK_TOPICS = ( + "research claims", + "effect size honesty", + "preregistration discipline", + "randomization integrity", + "baseline drift", + "reporting bias", + "measurement error", + "replication crisis", + "statistical power", + "selection effects", + "confound control", + "outcome switching", + "sample size planning", + "analysis robustness", + "evidence strength", + "uncertainty reporting", +) + + +def _v3_topics() -> tuple[str, ...]: + if _TINY_DEV_TOPICS[25] != _TINY_DEV_V3_ATTACK_TOPICS[0]: + raise TinyDevCorpusError("v3 attack topic pool must extend the v2 attack topic") + return (*_TINY_DEV_TOPICS[:25], *_TINY_DEV_V3_ATTACK_TOPICS) + + +def _v3_prompt_source_hash(attack_pairs_per_domain: int) -> str: + return sha256_json( + { + "algorithm_version": TINY_DEV_V3_GENERATION_ALGORITHM_VERSION, + "v2_source_id": TINY_DEV_PROMPT_SOURCE_ID, + "source_id": TINY_DEV_V3_PROMPT_SOURCE_ID, + "license_id": TINY_DEV_PROMPT_LICENSE_ID, + "provenance": TINY_DEV_PROMPT_PROVENANCE, + "attack_pairs_per_domain": attack_pairs_per_domain, + "calibration_topics": _TINY_DEV_TOPICS[:25], + "attack_topics": _TINY_DEV_V3_ATTACK_TOPICS, + "templates": tuple( + (domain.value, _TINY_DEV_TEMPLATES[domain]) + for domain in TINY_DEV_V3_DOMAINS + ), + } + ) + + +def build_tiny_dev_v3_prompt_records(attack_pairs_per_domain: int) -> tuple[PromptRecord, ...]: + if type(attack_pairs_per_domain) is not int or attack_pairs_per_domain < 1: + raise ValueError("attack_pairs_per_domain must be a positive integer") + if attack_pairs_per_domain > TINY_DEV_V3_MAX_ATTACK_PAIRS_PER_DOMAIN: + raise TinyDevCorpusError( + "v3 attack pairs per domain exceeds the frozen maximum of " + f"{TINY_DEV_V3_MAX_ATTACK_PAIRS_PER_DOMAIN}" + ) + topics = _v3_topics() + source_hash = _v3_prompt_source_hash(attack_pairs_per_domain) + records: list[PromptRecord] = [] + for split in TINY_DEV_SPLITS: + pair_count = ( + TINY_DEV_CALIBRATION_PAIRS_PER_DOMAIN + if split is CorpusSplit.THRESHOLD_CALIBRATION + else attack_pairs_per_domain + ) + topic_offset = ( + 0 + if split is CorpusSplit.THRESHOLD_CALIBRATION + else TINY_DEV_CALIBRATION_PAIRS_PER_DOMAIN + ) + for domain in TINY_DEV_V3_DOMAINS: + template = _TINY_DEV_TEMPLATES[domain] + for local_index in range(pair_count): + topic_index = topic_offset + local_index + topic = topics[topic_index] + prompt_id = f"tiny-dev-{split.value}-{domain.value}-{topic_index:02d}" + records.append( + PromptRecord.create( + prompt_id=prompt_id, + prompt_family_id=f"tiny-dev-family-{domain.value}-{topic_index:02d}", + domain=domain, + split=split, + source_id=TINY_DEV_V3_PROMPT_SOURCE_ID, + source_hash=source_hash, + license_id=TINY_DEV_PROMPT_LICENSE_ID, + provenance=TINY_DEV_PROMPT_PROVENANCE, + text=template.format(topic=topic), + ) + ) + return tuple(sorted(records, key=lambda value: value.prompt_id)) + + +def build_real_tiny_dev_v3_corpus( + backend: TinyDevGenerationBackend, + *, + corpus_id: str = "fuckmark-tiny-dev-real-v3", + seed_base: int = 402000, + attack_pairs_per_domain: int = 1, + max_attempts: int = TINY_DEV_DEFAULT_MAX_ATTEMPTS, +) -> TinyDevV3CorpusArtifact: + if type(seed_base) is not int or seed_base < 0: + raise ValueError("seed_base must be a non-negative integer") + if type(max_attempts) is not int or max_attempts <= 0: + raise ValueError("max_attempts must be a positive integer") + if type(attack_pairs_per_domain) is not int or attack_pairs_per_domain < 1: + raise ValueError("attack_pairs_per_domain must be a positive integer") + if attack_pairs_per_domain > TINY_DEV_V3_MAX_ATTACK_PAIRS_PER_DOMAIN: + raise ValueError("attack_pairs_per_domain exceeds the frozen v3 maximum") + prompts = build_tiny_dev_v3_prompt_records(attack_pairs_per_domain) + if seed_base + len(prompts) * TINY_DEV_PAIR_SEED_STRIDE + max_attempts > MAX_GENERATION_SEED: + raise ValueError("seed schedule exceeds the 64-bit generation-seed range") + + samples: list[object] = [] + seen_text_hashes: set[str] = set() + seen_token_hashes: set[str] = set() + + for pair_index, prompt in enumerate(prompts): + pair_seed_base = seed_base + pair_index * TINY_DEV_PAIR_SEED_STRIDE + accepted = False + for attempt in range(max_attempts): + seed = pair_seed_base + attempt + control = backend.generate(prompt.text, seed, watermarked=False) + watermarked = backend.generate(prompt.text, seed, watermarked=True) + candidates = ( + (WatermarkLabel.UNWATERMARKED, control), + (WatermarkLabel.WATERMARKED, watermarked), + ) + if any( + len(value.continuation_token_ids) != TINY_DEV_TARGET_LENGTH + for _, value in candidates + ): + continue + text_hashes = tuple(sha256_text(value.text) for _, value in candidates) + token_hashes = tuple( + sha256_json(value.continuation_token_ids) + for _, value in candidates + ) + if len(set(text_hashes)) != len(text_hashes) or len(set(token_hashes)) != len(token_hashes): + continue + if any(value in seen_text_hashes for value in text_hashes): + continue + if any(value in seen_token_hashes for value in token_hashes): + continue + pair_samples = tuple( + _build_sample( + prompt=prompt, + label=label, + generated=value, + backend=backend, + seed=seed, + ) + for label, value in candidates + ) + samples.extend(pair_samples) + seen_text_hashes.update(text_hashes) + seen_token_hashes.update(token_hashes) + accepted = True + break + if not accepted: + raise TinyDevGenerationError( + f"could not produce an exact, unique 64-token pair for {prompt.prompt_id} " + f"within {max_attempts} deterministic attempts" + ) + + artifact = build_tiny_dev_v3_corpus( + corpus_id, + prompts, + samples, + attack_pairs_per_domain=attack_pairs_per_domain, + ) + if any(sample.watermark.key_split is not KeySplit.DEV for sample in artifact.manifest.samples): + raise TinyDevGenerationError("real tiny development v3 generation must use DEV_KEYS") + for match_id in sorted({sample.match_id for sample in artifact.manifest.samples}): + pair = tuple(sample for sample in artifact.manifest.samples if sample.match_id == match_id) + if len({sample.generation.seed for sample in pair}) != 1: + raise TinyDevGenerationError("real tiny development v3 runner requires one shared seed per on/off pair") + if any(sample.generation.seed_policy_id != TINY_DEV_SEED_POLICY_ID for sample in pair): + raise TinyDevGenerationError("real tiny development v3 pair seed policy drifted") + return artifact diff --git a/fuckmark/tiny_dev_corpus_hf.py b/fuckmark/tiny_dev_corpus_hf.py index f37b5f82..8a1c001e 100644 --- a/fuckmark/tiny_dev_corpus_hf.py +++ b/fuckmark/tiny_dev_corpus_hf.py @@ -11,6 +11,9 @@ ModelTokenizerIdentity, PaddingSide, WatermarkCondition, + build_real_tiny_dev_v3_corpus, + load_tiny_dev_corpus_json, + load_tiny_dev_v3_corpus_json, ) from .corpus.tiny_dev import TINY_DEV_TARGET_LENGTH from .corpus.tiny_dev_generation import ( @@ -215,12 +218,26 @@ def _parser() -> argparse.ArgumentParser: parser.add_argument("--temperature", type=float, default=0.8) parser.add_argument("--top-k", type=int, default=50) parser.add_argument("--top-p", type=float, default=0.95) + parser.add_argument( + "--profile", + choices=("v2", "v3"), + default="v2", + ) + parser.add_argument( + "--attack-pairs-per-domain", + type=int, + default=1, + ) parser.add_argument("--json", type=Path, default=Path("artifacts/tiny-dev-corpus.json")) return parser def main(argv: list[str] | None = None) -> int: args = _parser().parse_args(argv) + if args.profile == "v3" and args.attack_pairs_per_domain < 1: + raise ValueError("the v3 profile requires at least one attack pair per domain") + if args.profile == "v2" and args.attack_pairs_per_domain != 1: + raise ValueError("the v2 profile is frozen at one attack pair per domain") backend = HuggingFaceTinyDevBackend( args.model, args.model_revision, @@ -229,13 +246,28 @@ def main(argv: list[str] | None = None) -> int: top_k=args.top_k, top_p=args.top_p, ) - artifact = build_real_tiny_dev_corpus( - backend, - seed_base=args.seed_base, - max_attempts=args.max_attempts, - ) + if args.profile == "v3": + artifact = build_real_tiny_dev_v3_corpus( + backend, + seed_base=args.seed_base, + attack_pairs_per_domain=args.attack_pairs_per_domain, + max_attempts=args.max_attempts, + ) + else: + artifact = build_real_tiny_dev_corpus( + backend, + seed_base=args.seed_base, + max_attempts=args.max_attempts, + ) args.json.parent.mkdir(parents=True, exist_ok=True) args.json.write_text(canonical_json_text(artifact) + "\n", encoding="utf-8") + loaded = ( + load_tiny_dev_v3_corpus_json(args.json) + if args.profile == "v3" + else load_tiny_dev_corpus_json(args.json) + ) + if loaded.artifact_hash != artifact.artifact_hash: + raise RuntimeError("canonical corpus JSON did not replay the generated artifact hash") sys.stdout.write(f"artifact_hash={artifact.artifact_hash}\n") sys.stdout.write(f"manifest_hash={artifact.manifest.manifest_hash}\n") sys.stdout.write(f"prompt_count={len(artifact.manifest.prompts)}\n") @@ -245,6 +277,9 @@ def main(argv: list[str] | None = None) -> int: f"generation_signature_hash={artifact.generation_matching_signature_hash}\n" ) sys.stdout.write(f"watermark_condition_hash={artifact.watermark_condition_hash}\n") + sys.stdout.write(f"profile={args.profile}\n") + if args.profile == "v3": + sys.stdout.write(f"attack_pairs_per_domain={args.attack_pairs_per_domain}\n") sys.stdout.write(f"json={args.json.as_posix()}\n") return 0 diff --git a/fuckmark/tiny_dev_effectiveness_plan_hf.py b/fuckmark/tiny_dev_effectiveness_plan_hf.py index ca3cbd56..9a4a3187 100644 --- a/fuckmark/tiny_dev_effectiveness_plan_hf.py +++ b/fuckmark/tiny_dev_effectiveness_plan_hf.py @@ -6,7 +6,7 @@ from datetime import datetime, timezone from pathlib import Path -from .corpus import load_tiny_dev_corpus_json +from .corpus import load_tiny_dev_corpus_by_version_json from .durable_io import write_canonical_json_fsynced from .experiments.effectiveness_plan import ( build_key_blind_high_coverage_plan, @@ -17,7 +17,11 @@ DEFAULT_MODEL_REVISION, runtime_tokenizer_identity_public, ) -from .transforms import KEY_BLIND_HIGH_COVERAGE_PROFILE +from .transforms import ( + KEY_BLIND_HIGH_COVERAGE_PROFILE_ID, + KEY_BLIND_FULL_POOL_COVERAGE_PROFILE_ID, + resolve_effectiveness_profile, +) EFFECTIVENESS_PLAN_PROVENANCE_VERSION = "effectiveness-plan-provenance-v1" @@ -53,12 +57,36 @@ def _provenance( return {**payload, "provenance_hash": sha256_json(payload)} +def _parse_budgets(raw: str) -> tuple[int, ...]: + if not raw: + return () + values: list[int] = [] + for chunk in raw.split(","): + stripped = chunk.strip() + if not stripped: + raise ValueError("budget list contains an empty entry") + value = int(stripped) + if value <= 0: + raise ValueError("budgets must be positive integers") + values.append(value) + budgets = tuple(sorted(set(values))) + if budgets != tuple(values): + raise ValueError("budgets must be provided in ascending order without duplicates") + return budgets + + def _parser() -> argparse.ArgumentParser: parser = argparse.ArgumentParser(prog="fuckmark-tiny-dev-effectiveness-plan-hf") parser.add_argument("--corpus-json", type=Path, required=True) parser.add_argument("--model", default=DEFAULT_MODEL_ID) parser.add_argument("--model-revision", default=DEFAULT_MODEL_REVISION) parser.add_argument("--source-code-commit", required=True) + parser.add_argument( + "--profile-id", + default=KEY_BLIND_HIGH_COVERAGE_PROFILE_ID, + choices=(KEY_BLIND_HIGH_COVERAGE_PROFILE_ID, KEY_BLIND_FULL_POOL_COVERAGE_PROFILE_ID), + ) + parser.add_argument("--budgets", default="") parser.add_argument( "--plan-json", type=Path, @@ -79,7 +107,8 @@ def main(argv: list[str] | None = None) -> int: except ImportError as error: raise RuntimeError("Install the pinned TinyDev Transformers dependencies first") from error - corpus = load_tiny_dev_corpus_json(args.corpus_json) + profile = resolve_effectiveness_profile(args.profile_id, _parse_budgets(args.budgets)) + corpus = load_tiny_dev_corpus_by_version_json(args.corpus_json) tokenizer = AutoTokenizer.from_pretrained( args.model, revision=args.model_revision, @@ -96,7 +125,7 @@ def main(argv: list[str] | None = None) -> int: plan = build_key_blind_high_coverage_plan( corpus, tokenizer, - profile=KEY_BLIND_HIGH_COVERAGE_PROFILE, + profile=profile, source_code_commit=args.source_code_commit, ) write_canonical_json_fsynced(args.plan_json, plan) @@ -104,14 +133,14 @@ def main(argv: list[str] | None = None) -> int: provenance = _provenance( source_code_commit=args.source_code_commit, plan_hash=plan["plan_hash"], - profile_hash=KEY_BLIND_HIGH_COVERAGE_PROFILE.profile_hash, + profile_hash=profile.profile_hash, plan_started_at=started_at, plan_fsynced_at=fsynced_at, ) write_canonical_json_fsynced(args.provenance_json, provenance) - sys.stdout.write(f"profile_id={KEY_BLIND_HIGH_COVERAGE_PROFILE.profile_id}\n") - sys.stdout.write(f"profile_hash={KEY_BLIND_HIGH_COVERAGE_PROFILE.profile_hash}\n") + sys.stdout.write(f"profile_id={profile.profile_id}\n") + sys.stdout.write(f"profile_hash={profile.profile_hash}\n") sys.stdout.write(f"plan_hash={plan['plan_hash']}\n") sys.stdout.write(f"variant_count={len(plan['variants'])}\n") sys.stdout.write(f"plan_json={args.plan_json.as_posix()}\n") diff --git a/fuckmark/tiny_dev_effectiveness_score_hf.py b/fuckmark/tiny_dev_effectiveness_score_hf.py index 1ac2f5b6..3df8c1e6 100644 --- a/fuckmark/tiny_dev_effectiveness_score_hf.py +++ b/fuckmark/tiny_dev_effectiveness_score_hf.py @@ -10,7 +10,7 @@ from typing import Any from .adapters import HuggingFaceSynthIDAdapter, HuggingFaceSynthIDConfig -from .corpus import load_tiny_dev_corpus_json +from .corpus import load_tiny_dev_corpus_by_version_json from .detectors import weighted_mean_evidence from .durable_io import write_canonical_json_fsynced from .experiments.effectiveness_plan import validate_key_blind_high_coverage_plan @@ -31,7 +31,13 @@ _threshold, _word_edit_distance, ) -from .transforms import KEY_BLIND_HIGH_COVERAGE_PROFILE +from .transforms import ( + KEY_BLIND_HIGH_COVERAGE_PROFILE, + KEY_BLIND_HIGH_COVERAGE_PROFILE_ID, + KEY_BLIND_FULL_POOL_COVERAGE_PROFILE_ID, + EffectivenessTransformProfile, + resolve_effectiveness_profile, +) KEY_BLIND_HIGH_COVERAGE_EVIDENCE_VERSION = "key-blind-high-coverage-evidence-v1" @@ -87,9 +93,12 @@ def score_key_blind_high_coverage_plan( tokenizer: Any, plan: Mapping[str, object], adapter: HuggingFaceSynthIDAdapter, + *, + profile: EffectivenessTransformProfile | None = None, ) -> dict[str, object]: - validate_key_blind_high_coverage_plan(plan, corpus, KEY_BLIND_HIGH_COVERAGE_PROFILE) - if adapter.ngram_len != KEY_BLIND_HIGH_COVERAGE_PROFILE.ngram_len: + active_profile = profile if profile is not None else KEY_BLIND_HIGH_COVERAGE_PROFILE + validate_key_blind_high_coverage_plan(plan, corpus, active_profile) + if adapter.ngram_len != active_profile.ngram_len: raise ValueError("detector ngram length does not match the frozen effectiveness profile") calibration = _text_only_calibration(corpus, adapter) threshold = _threshold(calibration, PRIMARY_TARGET_FPR) @@ -146,9 +155,9 @@ def score_key_blind_high_coverage_plan( "tiny_dev_artifact_hash": corpus.artifact_hash, "corpus_manifest_hash": corpus.manifest.manifest_hash, "plan_hash": plan["plan_hash"], - "profile_id": KEY_BLIND_HIGH_COVERAGE_PROFILE.profile_id, - "profile_hash": KEY_BLIND_HIGH_COVERAGE_PROFILE.profile_hash, - "ruleset_hash": KEY_BLIND_HIGH_COVERAGE_PROFILE.ruleset_hash, + "profile_id": active_profile.profile_id, + "profile_hash": active_profile.profile_hash, + "ruleset_hash": active_profile.ruleset_hash, "detector_identity_hash": calibration.detector_identity.identity_hash, "calibration_bundle_hash": calibration.bundle_hash, "threshold_hash": threshold.threshold_hash, @@ -188,6 +197,24 @@ def _provenance( return {**payload, "provenance_hash": sha256_json(payload)} +def _parse_budgets(raw: str) -> tuple[int, ...]: + if not raw: + return () + values: list[int] = [] + for chunk in raw.split(","): + stripped = chunk.strip() + if not stripped: + raise ValueError("budget list contains an empty entry") + value = int(stripped) + if value <= 0: + raise ValueError("budgets must be positive integers") + values.append(value) + budgets = tuple(sorted(set(values))) + if budgets != tuple(values): + raise ValueError("budgets must be provided in ascending order without duplicates") + return budgets + + def _parser() -> argparse.ArgumentParser: parser = argparse.ArgumentParser(prog="fuckmark-tiny-dev-effectiveness-score-hf") parser.add_argument("--corpus-json", type=Path, required=True) @@ -196,6 +223,12 @@ def _parser() -> argparse.ArgumentParser: parser.add_argument("--model-revision", default=DEFAULT_MODEL_REVISION) parser.add_argument("--device", choices=("cpu", "cuda"), default="cpu") parser.add_argument("--source-code-commit", required=True) + parser.add_argument( + "--profile-id", + default=KEY_BLIND_HIGH_COVERAGE_PROFILE_ID, + choices=(KEY_BLIND_HIGH_COVERAGE_PROFILE_ID, KEY_BLIND_FULL_POOL_COVERAGE_PROFILE_ID), + ) + parser.add_argument("--budgets", default="") parser.add_argument( "--json", type=Path, @@ -216,9 +249,10 @@ def main(argv: list[str] | None = None) -> int: except ImportError as error: raise RuntimeError("Install the pinned TinyDev Transformers dependencies first") from error - corpus = load_tiny_dev_corpus_json(args.corpus_json) + profile = resolve_effectiveness_profile(args.profile_id, _parse_budgets(args.budgets)) + corpus = load_tiny_dev_corpus_by_version_json(args.corpus_json) plan = _load_json(args.plan_json) - validate_key_blind_high_coverage_plan(plan, corpus, KEY_BLIND_HIGH_COVERAGE_PROFILE) + validate_key_blind_high_coverage_plan(plan, corpus, profile) if plan["source_code_commit"] != args.source_code_commit: raise ValueError("score invocation commit does not match the frozen plan") tokenizer = AutoTokenizer.from_pretrained( @@ -247,7 +281,7 @@ def main(argv: list[str] | None = None) -> int: device=args.device, ) started_at = _now() - evidence = score_key_blind_high_coverage_plan(corpus, tokenizer, plan, adapter) + evidence = score_key_blind_high_coverage_plan(corpus, tokenizer, plan, adapter, profile=profile) write_canonical_json_fsynced(args.json, evidence) fsynced_at = _now() provenance = _provenance( @@ -259,7 +293,7 @@ def main(argv: list[str] | None = None) -> int: ) write_canonical_json_fsynced(args.provenance_json, provenance) - sys.stdout.write(f"profile_id={KEY_BLIND_HIGH_COVERAGE_PROFILE.profile_id}\n") + sys.stdout.write(f"profile_id={profile.profile_id}\n") sys.stdout.write(f"plan_hash={plan['plan_hash']}\n") sys.stdout.write(f"artifact_hash={evidence['artifact_hash']}\n") for summary in evidence["summaries"]: diff --git a/fuckmark/transforms/__init__.py b/fuckmark/transforms/__init__.py index cca9ca93..e91ef4f6 100644 --- a/fuckmark/transforms/__init__.py +++ b/fuckmark/transforms/__init__.py @@ -1,12 +1,16 @@ from .candidate_artifacts import CandidateConflict, CandidateEnumeration, CandidateRejection, TransformCandidate from .effectiveness_profile import ( EFFECTIVENESS_PROFILE_ALGORITHM_VERSION, + KEY_BLIND_FULL_POOL_COVERAGE_PROFILE_ID, + KEY_BLIND_FULL_POOL_COVERAGE_SEED_BASE, KEY_BLIND_HIGH_COVERAGE_BUDGETS, KEY_BLIND_HIGH_COVERAGE_PROFILE, KEY_BLIND_HIGH_COVERAGE_PROFILE_ID, KEY_BLIND_HIGH_COVERAGE_SEED_BASE, EffectivenessTransformProfile, + key_blind_full_pool_coverage_profile, key_blind_high_coverage_transform_registry, + resolve_effectiveness_profile, validate_effectiveness_profile_registry, ) from .fidelity_readiness import ( @@ -196,6 +200,8 @@ "HardInvariantSignature", "hard_invariant_signature", "key_blind_high_coverage_transform_registry", + "key_blind_full_pool_coverage_profile", + "resolve_effectiveness_profile", "validate_hard_invariants", "InvariantDifference", "InvariantStatus", @@ -204,6 +210,8 @@ "KEY_BLIND_HIGH_COVERAGE_PROFILE", "KEY_BLIND_HIGH_COVERAGE_PROFILE_ID", "KEY_BLIND_HIGH_COVERAGE_SEED_BASE", + "KEY_BLIND_FULL_POOL_COVERAGE_PROFILE_ID", + "KEY_BLIND_FULL_POOL_COVERAGE_SEED_BASE", "LEXICAL_RETOKENIZATION_FIXTURE_ALGORITHM_VERSION", "LEXICAL_RULE_AUDIT_ALGORITHM_VERSION", "LEXICAL_TEMPLATE_RULE_ALGORITHM_VERSION", diff --git a/fuckmark/transforms/effectiveness_profile.py b/fuckmark/transforms/effectiveness_profile.py index 06c724e7..224a2c26 100644 --- a/fuckmark/transforms/effectiveness_profile.py +++ b/fuckmark/transforms/effectiveness_profile.py @@ -23,6 +23,8 @@ KEY_BLIND_HIGH_COVERAGE_PROFILE_ID = "key-blind-high-coverage-v1" KEY_BLIND_HIGH_COVERAGE_BUDGETS = (16,) KEY_BLIND_HIGH_COVERAGE_SEED_BASE = 1_120_000 +KEY_BLIND_FULL_POOL_COVERAGE_PROFILE_ID = "key-blind-full-pool-coverage-v1" +KEY_BLIND_FULL_POOL_COVERAGE_SEED_BASE = 1_130_000 @dataclass(frozen=True, slots=True) @@ -167,6 +169,44 @@ def key_blind_high_coverage_transform_registry( ) +def key_blind_full_pool_coverage_profile( + budgets: tuple[int, ...], +) -> EffectivenessTransformProfile: + if not isinstance(budgets, tuple): + raise TypeError("budgets must be a tuple") + return EffectivenessTransformProfile.create( + profile_id=KEY_BLIND_FULL_POOL_COVERAGE_PROFILE_ID, + budgets=budgets, + budget_unit="operation", + schedule_policy_id="COVERAGE_GREEDY_KEY_BLIND", + schedule_seed_base=KEY_BLIND_FULL_POOL_COVERAGE_SEED_BASE, + replicate_count=1, + ngram_len=5, + ruleset_hash=key_blind_high_coverage_transform_registry().ruleset_hash, + scientific_scope=( + "Frozen detector-blind and key-blind budget-scaled coverage profile over the " + "development ruleset with one fixed source-index seed per sorted source; " + "exploratory effectiveness evidence only and not release authorization" + ), + ) + + +def resolve_effectiveness_profile( + profile_id: str, + budgets: tuple[int, ...] = (), +) -> EffectivenessTransformProfile: + require_clean_string("profile_id", profile_id) + if profile_id == KEY_BLIND_HIGH_COVERAGE_PROFILE_ID: + if budgets: + raise ValueError("the frozen B16 profile does not accept custom budgets") + return KEY_BLIND_HIGH_COVERAGE_PROFILE + if profile_id == KEY_BLIND_FULL_POOL_COVERAGE_PROFILE_ID: + if not budgets: + raise ValueError("the full-pool coverage profile requires explicit budgets") + return key_blind_full_pool_coverage_profile(budgets) + raise ValueError("unknown effectiveness profile id") + + def validate_effectiveness_profile_registry( profile: EffectivenessTransformProfile, registry: TransformRegistry, diff --git a/specs/fuckmark-budget-scaled-coverage-effectiveness-v1.contract.json b/specs/fuckmark-budget-scaled-coverage-effectiveness-v1.contract.json index 8ab47c0a..be05434a 100644 --- a/specs/fuckmark-budget-scaled-coverage-effectiveness-v1.contract.json +++ b/specs/fuckmark-budget-scaled-coverage-effectiveness-v1.contract.json @@ -46,10 +46,10 @@ "watermark_keys_public_dev": [654, 400, 836, 123, 340, 443, 597, 160, 57], "development": [ {"seed_base": 401000, "profile": "tiny-dev-v2-canonical", "attack_pairs_per_domain": 1}, - {"seed_base": 402000, "profile": "tiny-dev-v3-parameterized", "attack_pairs_per_domain": 8} + {"seed_base": 410000, "profile": "tiny-dev-v3-parameterized", "attack_pairs_per_domain": 8} ], "confirmation": { - "seed_base": 403000, + "seed_base": 420000, "profile": "tiny-dev-v3-parameterized", "attack_pairs_per_domain": 16, "watermarked_attack_sources": 64, @@ -100,10 +100,10 @@ }, "artifacts": [ "specs/fuckmark-budget-scaled-coverage-effectiveness-v1.freeze.json", - "artifacts/budget-scaled/dev-corpus-402000.json", + "artifacts/budget-scaled/dev-corpus-410000.json", "artifacts/budget-scaled/dev-plan-{budget}.json", "artifacts/budget-scaled/dev-evidence-{budget}.json", - "artifacts/budget-scaled/confirmation-corpus-403000.json", + "artifacts/budget-scaled/confirmation-corpus-420000.json", "artifacts/budget-scaled/confirmation-plan.json", "artifacts/budget-scaled/confirmation-evidence.json", "artifacts/budget-scaled/robustness-report.json", diff --git a/specs/fuckmark-budget-scaled-coverage-effectiveness-v1.freeze.json b/specs/fuckmark-budget-scaled-coverage-effectiveness-v1.freeze.json index 2918d93b..fd437ff8 100644 --- a/specs/fuckmark-budget-scaled-coverage-effectiveness-v1.freeze.json +++ b/specs/fuckmark-budget-scaled-coverage-effectiveness-v1.freeze.json @@ -1,10 +1,40 @@ { "appendix_confirmation_budget": null, "base_main_commit": "16e153fdfa3739a88bca9b1559543a10ded2d859", - "contract_sha256": "e9cec62e6f208db7ed8f13e47b0e7657fc9808d74ce0979115a8ab30f4f8043e", + "contract_sha256": "d7b96e2b11b5a54e0518e9dcb74d6809269f6a3b9bc7aaedb6fabb89fde91753", "freeze_record_version": "budget-scaled-coverage-freeze-v1", "frozen_at_utc": "2026-08-23T00:05:32.588787+00:00", - "pre_score_amendments": [], + "pre_score_amendments": [ + { + "amendment_id": "pre-score-amendment-1", + "change": { + "confirmation_corpus_seed_base": { + "from": 403000, + "to": 420000 + }, + "development_second_corpus_seed_base": { + "from": 402000, + "to": 410000 + } + }, + "justification": "pair seed stride is 32; corpora at 402000/403000 would have overlapping seed ranges (dev spans ~4200 seeds, confirmation ~5250), and shared calibration prompt texts at equal seeds could reproduce identical texts across corpora; bases 410000 and 420000 are disjoint from each other and from the canonical 401000 corpus (which ends near 404300)", + "reason": "seed-range overlap arithmetic hazard found during implementation, before any scoring", + "recorded_at_utc": "2026-08-23T00:12:24.926060+00:00" + }, + { + "amendment_id": "pre-score-amendment-2", + "change": { + "development_canonical_corpus": { + "from": "CI-pinned ee0952 artifact", + "to": "locally generated seed-401000 corpus artifact 42240f72 as development data only" + } + }, + "finding": "generating the canonical seed-401000 corpus with the pinned model revision, pinned torch 2.13.0+cpu, and the pinned transformers git commit produced artifact_hash 42240f72e17df1a048f44197f216a0b5f30128c26e2da93afbe92820946e4561 and file sha256 57b580c8839ea85a650993e302443fec4e16f76ea4d1ff7156fcb5468017f7ce instead of the CI pin ee09527d06abc58e612f7157b69908d5abe06101865714ce13a1287d1f0d7664", + "interpretation": "generation is platform-sensitive (CPU float nondeterminism across operating systems); detector scoring arithmetic over fixed token sequences is platform-stable; thresholds are computed per corpus from that same corpus calibration split, so all within-cycle comparisons remain internally consistent", + "reason": "canonical corpus pin is not byte-reproducible on the local Windows CPU runtime", + "recorded_at_utc": "2026-08-23T00:14:34.573438+00:00" + } + ], "protocol_id": "budget-scaled-coverage-effectiveness-v1", - "protocol_sha256": "ad657b3ff9a719a5e3c06f268b1d8281dcf84208e5735ec0a173cc8e886fd688" + "protocol_sha256": "e242a93b4d81614ead88946dcd4ccc95462c4590ab3a83de09978a755518b63e" } diff --git a/specs/fuckmark-budget-scaled-coverage-effectiveness-v1.protocol.md b/specs/fuckmark-budget-scaled-coverage-effectiveness-v1.protocol.md index 709e3fb8..a9e806dd 100644 --- a/specs/fuckmark-budget-scaled-coverage-effectiveness-v1.protocol.md +++ b/specs/fuckmark-budget-scaled-coverage-effectiveness-v1.protocol.md @@ -33,10 +33,10 @@ decisions, keys beyond the public development keys, or confirmation results. ## 3. Data splits and source disjointness - Development: the canonical pinned TinyDev corpus (seed base 401000, corpus artifact hash - recorded at generation time) plus one fresh development corpus at seed base 402000 with 8 + recorded at generation time) plus one fresh development corpus at seed base 410000 with 8 attack pairs per domain (32 watermarked attack sources). Pooled development watermarked attack sources: 36. -- Confirmation: one fresh corpus at seed base 403000 with 16 attack pairs per domain +- Confirmation: one fresh corpus at seed base 420000 with 16 attack pairs per domain (64 watermarked attack sources plus 64 unwatermarked attack controls), generated only after the protocol, contract, and freeze record exist, and scored exactly once. - Every corpus keeps the frozen 25 threshold-calibration pairs per domain (100 negatives). diff --git a/tests/test_effectiveness_full_pool_profile.py b/tests/test_effectiveness_full_pool_profile.py new file mode 100644 index 00000000..a3fa96c9 --- /dev/null +++ b/tests/test_effectiveness_full_pool_profile.py @@ -0,0 +1,152 @@ +from __future__ import annotations + +from types import SimpleNamespace + +import pytest + +from fuckmark.corpus import CorpusDomain, CorpusSplit, WatermarkLabel +from fuckmark.experiments.effectiveness_plan import ( + build_key_blind_high_coverage_plan, + validate_key_blind_high_coverage_plan, +) +from fuckmark.hashing import sha256_text +from fuckmark.transforms import ( + KEY_BLIND_FULL_POOL_COVERAGE_PROFILE_ID, + KEY_BLIND_FULL_POOL_COVERAGE_SEED_BASE, + KEY_BLIND_HIGH_COVERAGE_PROFILE_ID, + key_blind_full_pool_coverage_profile, + resolve_effectiveness_profile, +) +from fuckmark.tiny_dev_effectiveness_plan_hf import _parse_budgets +from fuckmark.tiny_dev_effectiveness_score_hf import _parse_budgets as _parse_budgets_score + + +class FakeTokenizer: + def __call__( + self, + text: str, + *, + add_special_tokens: bool, + return_offsets_mapping: bool = False, + ) -> dict[str, object]: + assert add_special_tokens is False + ids = [index + 1 for index in range(len(text))] + value: dict[str, object] = {"input_ids": ids} + if return_offsets_mapping: + value["offset_mapping"] = [(index, index + 1) for index in range(len(text))] + return value + + +def _sample(sample_id: str, label: WatermarkLabel, text: str, identity_hash: str): + token_ids = tuple(index + 1 for index in range(len(text))) + return SimpleNamespace( + sample_id=sample_id, + split=CorpusSplit.ATTACK_DEVELOPMENT, + label=label, + prompt_family_id=f"prompt-{sample_id}", + domain=CorpusDomain.GENERAL_EXPLANATORY, + text=text, + text_sha256=sha256_text(text), + text_only_tokens=SimpleNamespace(token_ids=token_ids), + model=SimpleNamespace(identity_hash=identity_hash, eos_token_id=50256), + ) + + +def _corpus(): + identity_hash = sha256_text("fake-tokenizer") + text = "You are not ready, and we do not stop when the system is in use." + samples = ( + _sample("negative-1", WatermarkLabel.UNWATERMARKED, text, identity_hash), + _sample("positive-1", WatermarkLabel.WATERMARKED, text, identity_hash), + ) + return SimpleNamespace( + artifact_hash=sha256_text("fake-corpus"), + model_identity_hash=identity_hash, + manifest=SimpleNamespace(samples=samples, manifest_hash=sha256_text("fake-manifest")), + ) + + +def test_full_pool_profile_is_deterministic_and_binds_budgets() -> None: + first = key_blind_full_pool_coverage_profile((16, 24, 32, 48)) + second = key_blind_full_pool_coverage_profile((16, 24, 32, 48)) + assert first == second + assert first.profile_id == KEY_BLIND_FULL_POOL_COVERAGE_PROFILE_ID + assert first.budgets == (16, 24, 32, 48) + assert first.schedule_seed_base == KEY_BLIND_FULL_POOL_COVERAGE_SEED_BASE + assert first.ruleset_hash == resolve_effectiveness_profile( + KEY_BLIND_HIGH_COVERAGE_PROFILE_ID + ).ruleset_hash + other = key_blind_full_pool_coverage_profile((32,)) + assert other.profile_hash != first.profile_hash + + +def test_resolve_effectiveness_profile_rejects_invalid_requests() -> None: + with pytest.raises(ValueError): + resolve_effectiveness_profile("unknown-profile") + with pytest.raises(ValueError): + resolve_effectiveness_profile(KEY_BLIND_HIGH_COVERAGE_PROFILE_ID, (16,)) + with pytest.raises(ValueError): + resolve_effectiveness_profile(KEY_BLIND_FULL_POOL_COVERAGE_PROFILE_ID) + with pytest.raises(ValueError): + key_blind_full_pool_coverage_profile((48, 16)) + + +def test_full_pool_plan_covers_every_budget_with_derived_seeds() -> None: + corpus = _corpus() + profile = key_blind_full_pool_coverage_profile((24, 32)) + plan = build_key_blind_high_coverage_plan( + corpus, + FakeTokenizer(), + profile=profile, + source_code_commit="a" * 40, + ) + validate_key_blind_high_coverage_plan(plan, corpus, profile) + assert plan["budgets"] == (24, 32) + assert len(plan["variants"]) == 4 + assert {row["requested_budget"] for row in plan["variants"]} == {24, 32} + assert [row["schedule_seed"] for row in plan["variants"]] == [ + KEY_BLIND_FULL_POOL_COVERAGE_SEED_BASE, + KEY_BLIND_FULL_POOL_COVERAGE_SEED_BASE, + KEY_BLIND_FULL_POOL_COVERAGE_SEED_BASE + 1, + KEY_BLIND_FULL_POOL_COVERAGE_SEED_BASE + 1, + ] + assert all(row["detector_access_observed"] is False for row in plan["variants"]) + assert all(row["secret_access_observed"] is False for row in plan["variants"]) + + +def test_full_pool_plan_rejects_mismatched_profile_on_validation() -> None: + corpus = _corpus() + profile = key_blind_full_pool_coverage_profile((24,)) + plan = build_key_blind_high_coverage_plan( + corpus, + FakeTokenizer(), + profile=profile, + source_code_commit="a" * 40, + ) + with pytest.raises(ValueError): + validate_key_blind_high_coverage_plan( + plan, + corpus, + key_blind_full_pool_coverage_profile((32,)), + ) + + +@pytest.mark.parametrize( + ("raw", "expected"), + ( + ("", ()), + ("16", (16,)), + ("16,24,32,48", (16, 24, 32, 48)), + ), +) +def test_budget_parsing_accepts_valid_lists(raw: str, expected: tuple[int, ...]) -> None: + assert _parse_budgets(raw) == expected + assert _parse_budgets_score(raw) == expected + + +@pytest.mark.parametrize("raw", ("16,,24", "0", "-4", "24,16", "16,16")) +def test_budget_parsing_rejects_invalid_lists(raw: str) -> None: + with pytest.raises(ValueError): + _parse_budgets(raw) + with pytest.raises(ValueError): + _parse_budgets_score(raw) diff --git a/tests/test_tiny_dev_corpus_v3.py b/tests/test_tiny_dev_corpus_v3.py new file mode 100644 index 00000000..343f57be --- /dev/null +++ b/tests/test_tiny_dev_corpus_v3.py @@ -0,0 +1,179 @@ +from __future__ import annotations + +from collections import defaultdict + +import pytest + +from fuckmark.config import canonical_json_text +from fuckmark.corpus import ( + CorpusDomain, + CorpusSplit, + GenerationParameters, + KeySplit, + ModelTokenizerIdentity, + PaddingSide, + WatermarkCondition, + WatermarkLabel, + TINY_DEV_V3_CORPUS_ALGORITHM_VERSION, + TINY_DEV_V3_MAX_ATTACK_PAIRS_PER_DOMAIN, + build_tiny_dev_v3_prompt_records, + build_real_tiny_dev_v3_corpus, + parse_tiny_dev_v3_corpus_json, +) +from fuckmark.corpus.tiny_dev_generation import ( + TINY_DEV_SEED_POLICY_ID, + TinyDevGeneratedContinuation, + build_tiny_dev_prompt_records, +) +from fuckmark.corpus.tiny_dev_io_v3 import load_tiny_dev_corpus_by_version_json +from fuckmark.hashing import sha256_json, sha256_text + + +class FakeTinyDevV3Backend: + def __init__(self) -> None: + self._model = ModelTokenizerIdentity.create( + model_id="example/tiny-dev-model", + model_revision="a" * 40, + tokenizer_id="example/tiny-dev-tokenizer", + tokenizer_revision="b" * 40, + chat_template_present=False, + chat_template_hash=sha256_text(""), + special_token_map_hash=sha256_json({"eos_token": "", "pad_token": ""}), + padding_side=PaddingSide.LEFT, + bos_token_id=None, + eos_token_id=2, + pad_token_id=0, + add_bos_token=False, + add_eos_token=False, + ) + self._watermark = WatermarkCondition.create( + sha256_json({"ngram_len": 5, "keys": (1, 2, 3)}), + KeySplit.DEV, + "dev-test-key", + ) + + @property + def model_identity(self) -> ModelTokenizerIdentity: + return self._model + + @property + def watermark_condition(self) -> WatermarkCondition: + return self._watermark + + def generation_parameters(self, seed: int) -> GenerationParameters: + return GenerationParameters.create( + seed=seed, + seed_policy_id=TINY_DEV_SEED_POLICY_ID, + temperature=0.8, + top_k=50, + top_p=0.95, + max_new_tokens=64, + do_sample=True, + dtype="float32", + device="cpu", + backend_id="fake-tiny-dev", + backend_version="1", + ) + + def generate(self, prompt: str, seed: int, *, watermarked: bool) -> TinyDevGeneratedContinuation: + prompt_token = 10 + (sum(prompt.encode("utf-8")) % 1000) + label_offset = 100_000 if watermarked else 200_000 + continuation = tuple( + label_offset + ((seed * 131 + index) % 90_000) + for index in range(64) + ) + text = ( + f"{'watermarked' if watermarked else 'control'} output for " + f"{sha256_text(prompt)[:12]} seed {seed}" + ) + text_only = tuple(300_000 + ((seed * 17 + index) % 80_000) for index in range(16)) + return TinyDevGeneratedContinuation( + text=text, + input_token_ids=(prompt_token, prompt_token + 1), + attention_mask=(1, 1), + continuation_token_ids=continuation, + text_only_token_ids=text_only, + ) + + +def test_v3_prompt_records_preserve_v2_calibration_and_extend_attack() -> None: + v2 = build_tiny_dev_prompt_records() + v3 = build_tiny_dev_v3_prompt_records(16) + v2_calibration = {prompt.text for prompt in v2 if prompt.split is CorpusSplit.THRESHOLD_CALIBRATION} + v3_calibration = {prompt.text for prompt in v3 if prompt.split is CorpusSplit.THRESHOLD_CALIBRATION} + assert v2_calibration == v3_calibration + v2_attack = {prompt.text for prompt in v2 if prompt.split is CorpusSplit.ATTACK_DEVELOPMENT} + v3_one = build_tiny_dev_v3_prompt_records(1) + v3_one_attack = {prompt.text for prompt in v3_one if prompt.split is CorpusSplit.ATTACK_DEVELOPMENT} + assert v2_attack == v3_one_attack + counts: defaultdict[tuple[CorpusSplit, CorpusDomain], int] = defaultdict(int) + for prompt in v3: + counts[(prompt.split, prompt.domain)] += 1 + for domain in CorpusDomain: + assert counts[(CorpusSplit.THRESHOLD_CALIBRATION, domain)] == 25 + assert counts[(CorpusSplit.ATTACK_DEVELOPMENT, domain)] == 16 + assert len({prompt.prompt_id for prompt in v3}) == 164 + assert len({prompt.prompt_family_id for prompt in v3}) == 164 + + +@pytest.mark.parametrize("count", (0, -1, TINY_DEV_V3_MAX_ATTACK_PAIRS_PER_DOMAIN + 1)) +def test_v3_prompt_records_reject_out_of_range_attack_pairs(count: int) -> None: + with pytest.raises((ValueError, Exception)): + build_tiny_dev_v3_prompt_records(count) + + +def test_real_v3_runner_builds_parameterized_paired_corpus() -> None: + backend = FakeTinyDevV3Backend() + artifact = build_real_tiny_dev_v3_corpus( + backend, + seed_base=5000, + attack_pairs_per_domain=8, + ) + assert artifact.algorithm_version == TINY_DEV_V3_CORPUS_ALGORITHM_VERSION + assert artifact.attack_pairs_per_domain == 8 + assert artifact.calibration_pairs_per_domain == 25 + assert len(artifact.manifest.prompts) == 132 + assert len(artifact.manifest.samples) == 264 + attack_watermarked = [ + sample + for sample in artifact.manifest.samples + if sample.split is CorpusSplit.ATTACK_DEVELOPMENT + and sample.label is WatermarkLabel.WATERMARKED + ] + assert len(attack_watermarked) == 32 + assert all(sample.watermark.key_split is KeySplit.DEV for sample in artifact.manifest.samples) + assert len({sample.text_sha256 for sample in artifact.manifest.samples}) == 264 + + +def test_v3_corpus_json_replays_canonically_and_version_dispatch_loads() -> None: + backend = FakeTinyDevV3Backend() + artifact = build_real_tiny_dev_v3_corpus( + backend, + seed_base=9000, + attack_pairs_per_domain=2, + ) + text = canonical_json_text(artifact) + "\n" + replay = parse_tiny_dev_v3_corpus_json(text) + assert replay == artifact + assert replay.artifact_hash == artifact.artifact_hash + + +def test_v3_builder_rejects_mismatched_prompt_profile() -> None: + from fuckmark.corpus import build_tiny_dev_v3_corpus + + backend = FakeTinyDevV3Backend() + prompts = build_tiny_dev_v3_prompt_records(4) + with pytest.raises(Exception): + build_tiny_dev_v3_corpus( + "mismatch-corpus", + prompts, + (), + attack_pairs_per_domain=8, + ) + + +def test_version_dispatch_loader_rejects_unknown_version(tmp_path) -> None: + bad = tmp_path / "corpus.json" + bad.write_text('{"algorithm_version": "tiny-dev-corpus-v9"}', encoding="utf-8") + with pytest.raises(Exception): + load_tiny_dev_corpus_by_version_json(bad) From a5efc7780637842d0dad93c75996bf635b09f261 Mon Sep 17 00:00:00 2001 From: Cyinh Date: Sat, 22 Aug 2026 20:21:42 -0400 Subject: [PATCH 03/30] Add budget-scaled robustness audit module with tests and gate amendment --- fuckmark/tiny_dev_budget_scaled_robustness.py | 161 ++++++++++++++++++ ...ed-coverage-effectiveness-v1.contract.json | 2 +- ...aled-coverage-effectiveness-v1.freeze.json | 16 +- ...aled-coverage-effectiveness-v1.protocol.md | 2 +- tests/test_budget_scaled_robustness.py | 106 ++++++++++++ 5 files changed, 283 insertions(+), 4 deletions(-) create mode 100644 fuckmark/tiny_dev_budget_scaled_robustness.py create mode 100644 tests/test_budget_scaled_robustness.py diff --git a/fuckmark/tiny_dev_budget_scaled_robustness.py b/fuckmark/tiny_dev_budget_scaled_robustness.py new file mode 100644 index 00000000..fa8b404a --- /dev/null +++ b/fuckmark/tiny_dev_budget_scaled_robustness.py @@ -0,0 +1,161 @@ +from __future__ import annotations + +import argparse +import json +import sys +import unicodedata +from pathlib import Path + +from .corpus import load_tiny_dev_corpus_by_version_json +from .experiments.mid_dev_quality import protected_span_violation_count, word_edit_rate +from .hashing import sha256_json, sha256_text +from .transforms import validate_hard_invariants, InvariantStatus + + +BUDGET_SCALED_ROBUSTNESS_VERSION = "budget-scaled-robustness-v1" +FORBIDDEN_CATEGORIES = ("Cc", "Cf", "Mn", "Mc", "Me", "Zl", "Zp") +WORD_EDIT_RATE_MAX = 0.30 + + +def _forbidden_codepoints(text: str) -> tuple[str, ...]: + return tuple( + f"U+{ord(character):04X}({unicodedata.category(character)})" + for character in text + if unicodedata.category(character) in FORBIDDEN_CATEGORIES + or (unicodedata.category(character) == "Zs" and character != " ") + ) + + +def _cf_stripped(text: str) -> str: + return "".join( + character + for character in text + if unicodedata.category(character) != "Cf" + ) + + +def _robustness_row(source_text: str, transformed_text: str) -> dict[str, object]: + normalization_noops = { + "nfc": unicodedata.normalize("NFC", transformed_text) == transformed_text, + "nfd": unicodedata.normalize("NFD", transformed_text) == transformed_text, + "nfkc": unicodedata.normalize("NFKC", transformed_text) == transformed_text, + "nfkd": unicodedata.normalize("NFKD", transformed_text) == transformed_text, + } + invariant_status = validate_hard_invariants(source_text, transformed_text).status + row = { + "source_text_hash": sha256_text(source_text), + "transformed_text_hash": sha256_text(transformed_text), + "ascii_gate": transformed_text.isascii(), + "forbidden_codepoints": _forbidden_codepoints(transformed_text), + "normalization_noops": normalization_noops, + "cf_strip_noop": _cf_stripped(transformed_text) == transformed_text, + "word_edit_rate": word_edit_rate(source_text, transformed_text), + "protected_span_violation_count": protected_span_violation_count(source_text, transformed_text), + "hard_invariant_status": invariant_status.value, + } + gates_pass = ( + row["ascii_gate"] + and not row["forbidden_codepoints"] + and all(normalization_noops.values()) + and row["cf_strip_noop"] + and row["word_edit_rate"] <= WORD_EDIT_RATE_MAX + and row["protected_span_violation_count"] == 0 + and invariant_status is InvariantStatus.PASS + ) + return {**row, "gates_pass": gates_pass} + + +def build_budget_scaled_robustness_report( + corpora: tuple[object, ...], + plans: tuple[dict[str, object], ...], +) -> dict[str, object]: + sources: dict[str, object] = {} + for corpus in corpora: + for sample in corpus.manifest.samples: + sources[sample.sample_id] = sample + corpus_text_hashes: list[set[str]] = [ + {sample.text_sha256 for sample in corpus.manifest.samples} for corpus in corpora + ] + pairwise_overlaps = {} + for left in range(len(corpora)): + for right in range(left + 1, len(corpora)): + pairwise_overlaps[f"{left}-{right}"] = len( + corpus_text_hashes[left] & corpus_text_hashes[right] + ) + rows: list[dict[str, object]] = [] + for plan in plans: + for variant in plan["variants"]: + sample = sources[variant["source_sample_id"]] + rows.append( + { + "plan_hash": plan["plan_hash"], + "source_sample_id": variant["source_sample_id"], + "requested_budget": variant["requested_budget"], + **_robustness_row(sample.text, variant["transformed_text"]), + } + ) + row_tuple = tuple(rows) + summary = { + "row_count": len(row_tuple), + "gate_pass_fraction": sum(1 for row in row_tuple if row["gates_pass"]) / len(row_tuple) if row_tuple else 1.0, + "all_ascii": all(row["ascii_gate"] for row in row_tuple), + "all_forbidden_codepoints_empty": all(not row["forbidden_codepoints"] for row in row_tuple), + "all_normalization_noops": all(all(row["normalization_noops"].values()) for row in row_tuple), + "all_cf_strip_noop": all(row["cf_strip_noop"] for row in row_tuple), + "max_word_edit_rate": max((row["word_edit_rate"] for row in row_tuple), default=0.0), + "total_protected_span_violations": sum( + row["protected_span_violation_count"] for row in row_tuple + ), + "all_hard_invariants_pass": all( + row["hard_invariant_status"] == InvariantStatus.PASS.value for row in row_tuple + ), + "corpus_pairwise_text_hash_overlaps": pairwise_overlaps, + } + payload = { + "algorithm_version": BUDGET_SCALED_ROBUSTNESS_VERSION, + "word_edit_rate_max": WORD_EDIT_RATE_MAX, + "forbidden_categories": FORBIDDEN_CATEGORIES, + "corpus_artifact_hashes": tuple(corpus.artifact_hash for corpus in corpora), + "plan_hashes": tuple(plan["plan_hash"] for plan in plans), + "rows": row_tuple, + "summary": summary, + } + return {**payload, "artifact_hash": sha256_json(payload)} + + +def _load_plan(path: Path) -> dict[str, object]: + value = json.loads(path.read_text(encoding="utf-8")) + if not isinstance(value, dict): + raise ValueError("expected a JSON object") + return value + + +def _parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser(prog="fuckmark-tiny-dev-budget-scaled-robustness") + parser.add_argument("--corpus-json", type=Path, action="append", required=True) + parser.add_argument("--plan-json", type=Path, action="append", required=True) + parser.add_argument("--json", type=Path, required=True) + return parser + + +def main(argv: list[str] | None = None) -> int: + args = _parser().parse_args(argv) + corpora = tuple( + load_tiny_dev_corpus_by_version_json(path) for path in args.corpus_json + ) + plans = tuple(_load_plan(path) for path in args.plan_json) + report = build_budget_scaled_robustness_report(corpora, plans) + args.json.parent.mkdir(parents=True, exist_ok=True) + args.json.write_text( + json.dumps(report, indent=2, sort_keys=True, ensure_ascii=False) + "\n", + encoding="utf-8", + ) + sys.stdout.write(f"artifact_hash={report['artifact_hash']}\n") + sys.stdout.write(f"row_count={report['summary']['row_count']}\n") + sys.stdout.write(f"gate_pass_fraction={report['summary']['gate_pass_fraction']}\n") + sys.stdout.write(f"json={args.json.as_posix()}\n") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/specs/fuckmark-budget-scaled-coverage-effectiveness-v1.contract.json b/specs/fuckmark-budget-scaled-coverage-effectiveness-v1.contract.json index be05434a..536f5f14 100644 --- a/specs/fuckmark-budget-scaled-coverage-effectiveness-v1.contract.json +++ b/specs/fuckmark-budget-scaled-coverage-effectiveness-v1.contract.json @@ -70,7 +70,7 @@ "fidelity_gates": { "hard_invariants": "PASS required for every row", "protected_span_violations": 0, - "word_edit_rate_max": 0.15, + "word_edit_rate_max": 0.3, "ascii_gate": true, "forbidden_codepoint_categories": ["Cc", "Cf", "Mn", "Mc", "Me", "Zl", "Zp"], "allowed_space": "U+0020 only", diff --git a/specs/fuckmark-budget-scaled-coverage-effectiveness-v1.freeze.json b/specs/fuckmark-budget-scaled-coverage-effectiveness-v1.freeze.json index fd437ff8..cd5c9335 100644 --- a/specs/fuckmark-budget-scaled-coverage-effectiveness-v1.freeze.json +++ b/specs/fuckmark-budget-scaled-coverage-effectiveness-v1.freeze.json @@ -1,7 +1,7 @@ { "appendix_confirmation_budget": null, "base_main_commit": "16e153fdfa3739a88bca9b1559543a10ded2d859", - "contract_sha256": "d7b96e2b11b5a54e0518e9dcb74d6809269f6a3b9bc7aaedb6fabb89fde91753", + "contract_sha256": "47873d48f3a0c3a3eb1a3872c65426a5a440a65e33d3c05e7b801e9bfe8ca1f8", "freeze_record_version": "budget-scaled-coverage-freeze-v1", "frozen_at_utc": "2026-08-23T00:05:32.588787+00:00", "pre_score_amendments": [ @@ -33,8 +33,20 @@ "interpretation": "generation is platform-sensitive (CPU float nondeterminism across operating systems); detector scoring arithmetic over fixed token sequences is platform-stable; thresholds are computed per corpus from that same corpus calibration split, so all within-cycle comparisons remain internally consistent", "reason": "canonical corpus pin is not byte-reproducible on the local Windows CPU runtime", "recorded_at_utc": "2026-08-23T00:14:34.573438+00:00" + }, + { + "amendment_id": "pre-score-amendment-3", + "change": { + "word_edit_rate_max": { + "from": 0.15, + "to": 0.3 + } + }, + "finding": "a contraction rewrite such as \"are not\" to \"aren't\" costs two word-level edit operations (substitute plus delete), so even a semantically faithful, rules-only transformation of a short source reaches a word edit rate near 0.27; the original 0.15 gate would fail rows for reasons unrelated to the research question", + "reason": "word edit rate gate was set below the mechanical floor of the ruleset before any scoring", + "recorded_at_utc": "2026-08-23T00:16:07.299567+00:00" } ], "protocol_id": "budget-scaled-coverage-effectiveness-v1", - "protocol_sha256": "e242a93b4d81614ead88946dcd4ccc95462c4590ab3a83de09978a755518b63e" + "protocol_sha256": "8b2e5aac454705e3a346a737e3f7854aa5688c6c3ebd3c27cc55984a0e9a87e7" } diff --git a/specs/fuckmark-budget-scaled-coverage-effectiveness-v1.protocol.md b/specs/fuckmark-budget-scaled-coverage-effectiveness-v1.protocol.md index a9e806dd..0450557b 100644 --- a/specs/fuckmark-budget-scaled-coverage-effectiveness-v1.protocol.md +++ b/specs/fuckmark-budget-scaled-coverage-effectiveness-v1.protocol.md @@ -74,7 +74,7 @@ Gates evaluated on every plan row, development and confirmation: 1. Hard invariants: status PASS (enforced at plan build; any failure aborts). 2. Protected spans: zero violations under the symmetric protected-span counter. -3. Word edit rate: word-level edit distance divided by source word count must be <= 0.15. +3. Word edit rate: word-level edit distance divided by source word count must be <= 0.30. 4. ASCII gate: transformed text `isascii()` must be True (rules are ASCII-only by design). 5. Codepoint audit: transformed text must contain no Unicode Cf, Cc, Mn, Mc, Me, Zl, Zp, or Zs characters other than U+0020, and no character outside the source alphabet union rule diff --git a/tests/test_budget_scaled_robustness.py b/tests/test_budget_scaled_robustness.py new file mode 100644 index 00000000..aeec6b42 --- /dev/null +++ b/tests/test_budget_scaled_robustness.py @@ -0,0 +1,106 @@ +from __future__ import annotations + +from types import SimpleNamespace + +from fuckmark.corpus import CorpusDomain, CorpusSplit, WatermarkLabel +from fuckmark.experiments.effectiveness_plan import build_key_blind_high_coverage_plan +from fuckmark.hashing import sha256_text +from fuckmark.tiny_dev_budget_scaled_robustness import ( + _cf_stripped, + _forbidden_codepoints, + build_budget_scaled_robustness_report, +) +from fuckmark.transforms import key_blind_full_pool_coverage_profile + + +class FakeTokenizer: + def __call__( + self, + text: str, + *, + add_special_tokens: bool, + return_offsets_mapping: bool = False, + ) -> dict[str, object]: + assert add_special_tokens is False + ids = [index + 1 for index in range(len(text))] + value: dict[str, object] = {"input_ids": ids} + if return_offsets_mapping: + value["offset_mapping"] = [(index, index + 1) for index in range(len(text))] + return value + + +def _sample(sample_id: str, label: WatermarkLabel, text: str, identity_hash: str): + token_ids = tuple(index + 1 for index in range(len(text))) + return SimpleNamespace( + sample_id=sample_id, + split=CorpusSplit.ATTACK_DEVELOPMENT, + label=label, + prompt_family_id=f"prompt-{sample_id}", + domain=CorpusDomain.GENERAL_EXPLANATORY, + text=text, + text_sha256=sha256_text(text), + text_only_tokens=SimpleNamespace(token_ids=token_ids), + model=SimpleNamespace(identity_hash=identity_hash, eos_token_id=50256), + ) + + +def _corpus(text: str, salt: str): + identity_hash = sha256_text("fake-tokenizer") + samples = ( + _sample(f"negative-{salt}", WatermarkLabel.UNWATERMARKED, text, identity_hash), + _sample(f"positive-{salt}", WatermarkLabel.WATERMARKED, text, identity_hash), + ) + return SimpleNamespace( + artifact_hash=sha256_text(f"fake-corpus-{salt}"), + model_identity_hash=identity_hash, + manifest=SimpleNamespace(samples=samples, manifest_hash=sha256_text("fake-manifest")), + ) + + +def test_forbidden_codepoint_audit_detects_hidden_representation() -> None: + clean = "plain ascii text with spaces only" + assert _forbidden_codepoints(clean) == () + zero_width = "hidden" + chr(0x200C) + "join" + combining = "combining" + chr(0x0301) + nbsp = "nbsp" + chr(0x00A0) + soft_hyphen = "soft" + chr(0x00AD) + assert _forbidden_codepoints(zero_width) == ("U+200C(Cf)",) + assert _forbidden_codepoints(combining) == ("U+0301(Mn)",) + assert _forbidden_codepoints(nbsp) == ("U+00A0(Zs)",) + assert _forbidden_codepoints(soft_hyphen) == ("U+00AD(Cf)",) + assert _cf_stripped(zero_width) == "hiddenjoin" + assert _cf_stripped(clean) == clean + + +def test_robustness_report_gates_pass_on_rule_transforms() -> None: + text = "You are not ready, and we do not stop when the system is in use." + corpus = _corpus(text, "1") + plan = build_key_blind_high_coverage_plan( + corpus, + FakeTokenizer(), + profile=key_blind_full_pool_coverage_profile((32,)), + source_code_commit="a" * 40, + ) + report = build_budget_scaled_robustness_report((corpus,), (plan,)) + assert report["summary"]["row_count"] == 2 + assert report["summary"]["gate_pass_fraction"] == 1.0 + assert report["summary"]["all_normalization_noops"] is True + assert report["summary"]["all_cf_strip_noop"] is True + assert report["summary"]["all_ascii"] is True + assert report["summary"]["corpus_pairwise_text_hash_overlaps"] == {} + assert all(row["gates_pass"] for row in report["rows"]) + assert report["artifact_hash"] + + +def test_robustness_report_detects_cross_corpus_text_overlap() -> None: + text = "Shared generation text that appears in two corpora at once." + first = _corpus(text, "1") + second = _corpus(text, "2") + plan = build_key_blind_high_coverage_plan( + first, + FakeTokenizer(), + profile=key_blind_full_pool_coverage_profile((16,)), + source_code_commit="a" * 40, + ) + report = build_budget_scaled_robustness_report((first, second), (plan,)) + assert report["summary"]["corpus_pairwise_text_hash_overlaps"] == {"0-1": 1} From 453560f3949bc54ab3ec2fd537049a5ec452e55a Mon Sep 17 00:00:00 2001 From: Cyinh Date: Sat, 22 Aug 2026 20:24:45 -0400 Subject: [PATCH 04/30] Gate candidate-introduced codepoints instead of absolute text content --- fuckmark/tiny_dev_budget_scaled_robustness.py | 37 +++++++++++++++---- ...ed-coverage-effectiveness-v1.contract.json | 7 ++-- ...aled-coverage-effectiveness-v1.freeze.json | 11 +++++- ...aled-coverage-effectiveness-v1.protocol.md | 10 +++-- tests/test_budget_scaled_robustness.py | 18 ++++++++- 5 files changed, 66 insertions(+), 17 deletions(-) diff --git a/fuckmark/tiny_dev_budget_scaled_robustness.py b/fuckmark/tiny_dev_budget_scaled_robustness.py index fa8b404a..2e5ac131 100644 --- a/fuckmark/tiny_dev_budget_scaled_robustness.py +++ b/fuckmark/tiny_dev_budget_scaled_robustness.py @@ -20,12 +20,31 @@ def _forbidden_codepoints(text: str) -> tuple[str, ...]: return tuple( f"U+{ord(character):04X}({unicodedata.category(character)})" - for character in text + for character in sorted(set(text)) if unicodedata.category(character) in FORBIDDEN_CATEGORIES or (unicodedata.category(character) == "Zs" and character != " ") ) +def _introduced_forbidden_codepoints(source_text: str, transformed_text: str) -> tuple[str, ...]: + source_alphabet = set(source_text) + return tuple( + f"U+{ord(character):04X}({unicodedata.category(character)})" + for character in sorted(set(transformed_text) - source_alphabet) + if unicodedata.category(character) in FORBIDDEN_CATEGORIES + or (unicodedata.category(character) == "Zs" and character != " ") + ) + + +def _introduced_non_ascii(source_text: str, transformed_text: str) -> tuple[str, ...]: + source_alphabet = set(source_text) + return tuple( + f"U+{ord(character):04X}" + for character in sorted(set(transformed_text) - source_alphabet) + if not character.isascii() + ) + + def _cf_stripped(text: str) -> str: return "".join( character @@ -45,8 +64,10 @@ def _robustness_row(source_text: str, transformed_text: str) -> dict[str, object row = { "source_text_hash": sha256_text(source_text), "transformed_text_hash": sha256_text(transformed_text), - "ascii_gate": transformed_text.isascii(), - "forbidden_codepoints": _forbidden_codepoints(transformed_text), + "transformed_is_ascii": transformed_text.isascii(), + "preexisting_forbidden_codepoints": _forbidden_codepoints(transformed_text), + "introduced_forbidden_codepoints": _introduced_forbidden_codepoints(source_text, transformed_text), + "introduced_non_ascii_codepoints": _introduced_non_ascii(source_text, transformed_text), "normalization_noops": normalization_noops, "cf_strip_noop": _cf_stripped(transformed_text) == transformed_text, "word_edit_rate": word_edit_rate(source_text, transformed_text), @@ -54,8 +75,8 @@ def _robustness_row(source_text: str, transformed_text: str) -> dict[str, object "hard_invariant_status": invariant_status.value, } gates_pass = ( - row["ascii_gate"] - and not row["forbidden_codepoints"] + not row["introduced_forbidden_codepoints"] + and not row["introduced_non_ascii_codepoints"] and all(normalization_noops.values()) and row["cf_strip_noop"] and row["word_edit_rate"] <= WORD_EDIT_RATE_MAX @@ -98,8 +119,10 @@ def build_budget_scaled_robustness_report( summary = { "row_count": len(row_tuple), "gate_pass_fraction": sum(1 for row in row_tuple if row["gates_pass"]) / len(row_tuple) if row_tuple else 1.0, - "all_ascii": all(row["ascii_gate"] for row in row_tuple), - "all_forbidden_codepoints_empty": all(not row["forbidden_codepoints"] for row in row_tuple), + "all_introduced_codepoints_clean": all( + not row["introduced_forbidden_codepoints"] and not row["introduced_non_ascii_codepoints"] + for row in row_tuple + ), "all_normalization_noops": all(all(row["normalization_noops"].values()) for row in row_tuple), "all_cf_strip_noop": all(row["cf_strip_noop"] for row in row_tuple), "max_word_edit_rate": max((row["word_edit_rate"] for row in row_tuple), default=0.0), diff --git a/specs/fuckmark-budget-scaled-coverage-effectiveness-v1.contract.json b/specs/fuckmark-budget-scaled-coverage-effectiveness-v1.contract.json index 536f5f14..c4922288 100644 --- a/specs/fuckmark-budget-scaled-coverage-effectiveness-v1.contract.json +++ b/specs/fuckmark-budget-scaled-coverage-effectiveness-v1.contract.json @@ -71,9 +71,10 @@ "hard_invariants": "PASS required for every row", "protected_span_violations": 0, "word_edit_rate_max": 0.3, - "ascii_gate": true, - "forbidden_codepoint_categories": ["Cc", "Cf", "Mn", "Mc", "Me", "Zl", "Zp"], - "allowed_space": "U+0020 only", + "introduced_codepoints_must_be_ascii": true, + "introduced_forbidden_codepoint_categories": ["Cc", "Cf", "Mn", "Mc", "Me", "Zl", "Zp"], + "introduced_allowed_space": "U+0020 only", + "preexisting_source_codepoints": "reported, not gated", "normalization_noop": ["NFC", "NFD", "NFKC", "NFKD"], "cf_stripping_noop": true }, diff --git a/specs/fuckmark-budget-scaled-coverage-effectiveness-v1.freeze.json b/specs/fuckmark-budget-scaled-coverage-effectiveness-v1.freeze.json index cd5c9335..62c924ea 100644 --- a/specs/fuckmark-budget-scaled-coverage-effectiveness-v1.freeze.json +++ b/specs/fuckmark-budget-scaled-coverage-effectiveness-v1.freeze.json @@ -1,7 +1,7 @@ { "appendix_confirmation_budget": null, "base_main_commit": "16e153fdfa3739a88bca9b1559543a10ded2d859", - "contract_sha256": "47873d48f3a0c3a3eb1a3872c65426a5a440a65e33d3c05e7b801e9bfe8ca1f8", + "contract_sha256": "c878ff9fdbf4c412361f6058a72af1fa8c925e78997c89825d255c8d437f598f", "freeze_record_version": "budget-scaled-coverage-freeze-v1", "frozen_at_utc": "2026-08-23T00:05:32.588787+00:00", "pre_score_amendments": [ @@ -45,8 +45,15 @@ "finding": "a contraction rewrite such as \"are not\" to \"aren't\" costs two word-level edit operations (substitute plus delete), so even a semantically faithful, rules-only transformation of a short source reaches a word edit rate near 0.27; the original 0.15 gate would fail rows for reasons unrelated to the research question", "reason": "word edit rate gate was set below the mechanical floor of the ruleset before any scoring", "recorded_at_utc": "2026-08-23T00:16:07.299567+00:00" + }, + { + "amendment_id": "pre-score-amendment-4", + "change": "gates now bind to characters INTRODUCED by the transformation (present in transformed, absent from source alphabet): introduced characters must be ASCII and must include no Cc, Cf, Mn, Mc, Me, Zl, Zp, or non-space Zs codepoint; pre-existing source codepoints are reported without gating", + "finding": "the audit gated on absolute presence of category Cc characters in transformed text, which flags ordinary newlines (U+000A) that the GPT-2 source generation already contains; 28 of 32 development rows failed a gate that measures the corpus, not the candidate", + "reason": "codepoint audit scope error found on development robustness evidence before confirmation scoring", + "recorded_at_utc": "2026-08-23T00:24:31.440654+00:00" } ], "protocol_id": "budget-scaled-coverage-effectiveness-v1", - "protocol_sha256": "8b2e5aac454705e3a346a737e3f7854aa5688c6c3ebd3c27cc55984a0e9a87e7" + "protocol_sha256": "e9ed870944a32ad7bcfe38993d1fdd6287618a04d638192734b82bcfaf9de833" } diff --git a/specs/fuckmark-budget-scaled-coverage-effectiveness-v1.protocol.md b/specs/fuckmark-budget-scaled-coverage-effectiveness-v1.protocol.md index 0450557b..3df810bf 100644 --- a/specs/fuckmark-budget-scaled-coverage-effectiveness-v1.protocol.md +++ b/specs/fuckmark-budget-scaled-coverage-effectiveness-v1.protocol.md @@ -75,10 +75,12 @@ Gates evaluated on every plan row, development and confirmation: 1. Hard invariants: status PASS (enforced at plan build; any failure aborts). 2. Protected spans: zero violations under the symmetric protected-span counter. 3. Word edit rate: word-level edit distance divided by source word count must be <= 0.30. -4. ASCII gate: transformed text `isascii()` must be True (rules are ASCII-only by design). -5. Codepoint audit: transformed text must contain no Unicode Cf, Cc, Mn, Mc, Me, Zl, Zp, or Zs - characters other than U+0020, and no character outside the source alphabet union rule - alphabet. +4. Introduced-codepoint gate: every character the transformation introduces (present in the + transformed text but absent from the source alphabet) must be ASCII; pre-existing source + characters such as newlines are corpus properties and are reported, not gated. +5. Codepoint audit: the transformation must introduce no Unicode Cf, Cc, Mn, Mc, Me, Zl, Zp, + or Zs characters other than U+0020. Pre-existing forbidden-category characters in the + source are reported separately and do not gate the candidate. 6. Normalization no-op: NFC, NFD, NFKC, and NFKD of every transformed text must equal the transformed text itself; Unicode Cf stripping must be the identity function on every transformed text. Any non-identity result is an adverse finding and a Level B failure. diff --git a/tests/test_budget_scaled_robustness.py b/tests/test_budget_scaled_robustness.py index aeec6b42..3677bfdb 100644 --- a/tests/test_budget_scaled_robustness.py +++ b/tests/test_budget_scaled_robustness.py @@ -72,6 +72,22 @@ def test_forbidden_codepoint_audit_detects_hidden_representation() -> None: assert _cf_stripped(clean) == clean +def test_introduced_codepoint_gate_ignores_preexisting_source_characters() -> None: + from fuckmark.tiny_dev_budget_scaled_robustness import ( + _introduced_forbidden_codepoints, + _introduced_non_ascii, + ) + + source = "source with\nnewline and " + chr(0x2019) + "curly quote" + transformed_same = source + " and one more plain ascii word" + assert _introduced_forbidden_codepoints(source, transformed_same) == () + assert _introduced_non_ascii(source, transformed_same) == () + transformed_hidden = transformed_same + chr(0x200B) + assert _introduced_forbidden_codepoints(source, transformed_hidden) == ("U+200B(Cf)",) + transformed_visible_unicode = transformed_same + chr(0x00E9) + assert _introduced_non_ascii(source, transformed_visible_unicode) == ("U+00E9",) + + def test_robustness_report_gates_pass_on_rule_transforms() -> None: text = "You are not ready, and we do not stop when the system is in use." corpus = _corpus(text, "1") @@ -86,7 +102,7 @@ def test_robustness_report_gates_pass_on_rule_transforms() -> None: assert report["summary"]["gate_pass_fraction"] == 1.0 assert report["summary"]["all_normalization_noops"] is True assert report["summary"]["all_cf_strip_noop"] is True - assert report["summary"]["all_ascii"] is True + assert report["summary"]["all_introduced_codepoints_clean"] is True assert report["summary"]["corpus_pairwise_text_hash_overlaps"] == {} assert all(row["gates_pass"] for row in report["rows"]) assert report["artifact_hash"] From 87941e2856eba6206fe32423f1a2f0fb1c95d76a Mon Sep 17 00:00:00 2001 From: Cyinh Date: Sat, 22 Aug 2026 20:28:03 -0400 Subject: [PATCH 05/30] Add render page generator for fixed-browser pair checks --- tools/budget_scaled_render_page.py | 88 ++++++++++++++++++++++++++++++ 1 file changed, 88 insertions(+) create mode 100644 tools/budget_scaled_render_page.py diff --git a/tools/budget_scaled_render_page.py b/tools/budget_scaled_render_page.py new file mode 100644 index 00000000..9e0fbd7b --- /dev/null +++ b/tools/budget_scaled_render_page.py @@ -0,0 +1,88 @@ +from __future__ import annotations + +import argparse +import html +import json +from pathlib import Path + + +def _load(path: Path) -> dict: + value = json.loads(path.read_text(encoding="utf-8")) + if not isinstance(value, dict): + raise ValueError("expected a JSON object") + return value + + +def _load_corpus_texts(corpus_path: Path) -> dict[str, str]: + corpus = _load(corpus_path) + return {sample["sample_id"]: sample["text"] for sample in corpus["manifest"]["samples"]} + + +def build_render_page(plan: dict, corpus_texts: dict[str, str]) -> tuple[str, list[dict]]: + blocks: list[str] = [] + manifest: list[dict] = [] + for index, variant in enumerate(plan["variants"]): + source_id = variant["source_sample_id"] + source_text = corpus_texts[source_id] + transformed_text = variant["transformed_text"] + blocks.append( + "
" + "

Pair {index}: {source_id} (budget {budget})

" + "
{source}
" + "
{transformed}
" + "
".format( + index=index, + source_id=html.escape(source_id), + budget=variant["requested_budget"], + source=html.escape(source_text), + transformed=html.escape(transformed_text), + ) + ) + manifest.append( + { + "pair_index": index, + "source_sample_id": source_id, + "requested_budget": variant["requested_budget"], + "source_text_hash": variant["source_text_hash"], + "transformed_text_hash": variant["transformed_text_hash"], + "variant_hash": variant["variant_hash"], + } + ) + page = ( + "" + "FuckMark budget-scaled render check" + "" + + "".join(blocks) + + "" + ) + return page, manifest + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(prog="budget-scaled-render-page") + parser.add_argument("--corpus-json", type=Path, required=True) + parser.add_argument("--plan-json", type=Path, required=True) + parser.add_argument("--html", type=Path, required=True) + parser.add_argument("--manifest-json", type=Path, required=True) + args = parser.parse_args(argv) + plan = _load(args.plan_json) + corpus_texts = _load_corpus_texts(args.corpus_json) + page, manifest = build_render_page(plan, corpus_texts) + args.html.parent.mkdir(parents=True, exist_ok=True) + args.html.write_text(page, encoding="utf-8") + args.manifest_json.write_text( + json.dumps({"pair_count": len(manifest), "plan_hash": plan["plan_hash"], "pairs": manifest}, indent=2, sort_keys=True) + "\n", + encoding="utf-8", + ) + print(f"pair_count={len(manifest)}") + print(f"html={args.html.as_posix()}") + print(f"manifest_json={args.manifest_json.as_posix()}") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) From 87101b0cd41bb78106b8445415a935051d4580ac Mon Sep 17 00:00:00 2001 From: Cyinh Date: Sat, 22 Aug 2026 20:30:07 -0400 Subject: [PATCH 06/30] Bind robustness plan joins to corpus artifact hashes --- fuckmark/tiny_dev_budget_scaled_robustness.py | 10 +++++--- tests/test_budget_scaled_robustness.py | 23 +++++++++++++++++++ 2 files changed, 30 insertions(+), 3 deletions(-) diff --git a/fuckmark/tiny_dev_budget_scaled_robustness.py b/fuckmark/tiny_dev_budget_scaled_robustness.py index 2e5ac131..7730e142 100644 --- a/fuckmark/tiny_dev_budget_scaled_robustness.py +++ b/fuckmark/tiny_dev_budget_scaled_robustness.py @@ -90,10 +90,9 @@ def build_budget_scaled_robustness_report( corpora: tuple[object, ...], plans: tuple[dict[str, object], ...], ) -> dict[str, object]: - sources: dict[str, object] = {} + corpus_by_artifact: dict[str, object] = {} for corpus in corpora: - for sample in corpus.manifest.samples: - sources[sample.sample_id] = sample + corpus_by_artifact[corpus.artifact_hash] = corpus corpus_text_hashes: list[set[str]] = [ {sample.text_sha256 for sample in corpus.manifest.samples} for corpus in corpora ] @@ -105,11 +104,16 @@ def build_budget_scaled_robustness_report( ) rows: list[dict[str, object]] = [] for plan in plans: + corpus = corpus_by_artifact.get(plan["tiny_dev_artifact_hash"]) + if corpus is None: + raise ValueError("robustness report received a plan whose corpus was not supplied") + sources = {sample.sample_id: sample for sample in corpus.manifest.samples} for variant in plan["variants"]: sample = sources[variant["source_sample_id"]] rows.append( { "plan_hash": plan["plan_hash"], + "corpus_artifact_hash": corpus.artifact_hash, "source_sample_id": variant["source_sample_id"], "requested_budget": variant["requested_budget"], **_robustness_row(sample.text, variant["transformed_text"]), diff --git a/tests/test_budget_scaled_robustness.py b/tests/test_budget_scaled_robustness.py index 3677bfdb..dfe8aed7 100644 --- a/tests/test_budget_scaled_robustness.py +++ b/tests/test_budget_scaled_robustness.py @@ -108,6 +108,29 @@ def test_robustness_report_gates_pass_on_rule_transforms() -> None: assert report["artifact_hash"] +def test_robustness_report_joins_plans_to_their_own_corpus_by_artifact_hash() -> None: + identity_hash = sha256_text("fake-tokenizer") + first_text = "You are not ready, and we do not stop when the system is in use." + second_text = "A second corpus shares sample IDs but holds different source texts here." + first = _corpus(first_text, "1") + second = _corpus(second_text, "1") + first.artifact_hash = sha256_text("corpus-a") + second.artifact_hash = sha256_text("corpus-b") + for corpus, text in ((first, first_text), (second, second_text)): + for sample in corpus.manifest.samples: + sample.text = text + sample.text_sha256 = sha256_text(text) + plan = build_key_blind_high_coverage_plan( + first, + FakeTokenizer(), + profile=key_blind_full_pool_coverage_profile((16,)), + source_code_commit="a" * 40, + ) + report = build_budget_scaled_robustness_report((first, second), (plan,)) + assert all(row["corpus_artifact_hash"] == sha256_text("corpus-a") for row in report["rows"]) + assert report["summary"]["gate_pass_fraction"] == 1.0 + + def test_robustness_report_detects_cross_corpus_text_overlap() -> None: text = "Shared generation text that appears in two corpora at once." first = _corpus(text, "1") From 46c220f1c0dfe4c2b401cc32ecf4aaf70ac5ed7b Mon Sep 17 00:00:00 2001 From: Cyinh Date: Sat, 22 Aug 2026 20:30:07 -0400 Subject: [PATCH 07/30] Record frozen-rule budget selection in protocol appendix --- ...aled-coverage-effectiveness-v1.freeze.json | 35 ++++++++++++++++++- 1 file changed, 34 insertions(+), 1 deletion(-) diff --git a/specs/fuckmark-budget-scaled-coverage-effectiveness-v1.freeze.json b/specs/fuckmark-budget-scaled-coverage-effectiveness-v1.freeze.json index 62c924ea..a4f6af56 100644 --- a/specs/fuckmark-budget-scaled-coverage-effectiveness-v1.freeze.json +++ b/specs/fuckmark-budget-scaled-coverage-effectiveness-v1.freeze.json @@ -1,5 +1,38 @@ { - "appendix_confirmation_budget": null, + "appendix_confirmation_budget": { + "development_gate_pass_fraction": 1.0, + "evidence_hashes": { + "dev_evidence_401000": "3ab52d1adf9ffaa934ad088dd5b425042e3b5ad440b24eacf16f60574adf59c2", + "dev_evidence_410000": "2d9776f05ccea7e113449dde08394a498b630e61e14f354487b979b5395e8511", + "dev_robustness_pooled": "eacd39a761997d4f66eb214dac3a86a17df9776a189854714dfcb41d9083005a" + }, + "pooled_development_watermarked_sources": 36, + "recorded_at_utc": "2026-08-23T00:30:07.496640+00:00", + "rule_input": { + "16": { + "detected_401000": 0, + "detected_410000": 1, + "pooled": 1 + }, + "24": { + "detected_401000": 0, + "detected_410000": 1, + "pooled": 1 + }, + "32": { + "detected_401000": 0, + "detected_410000": 1, + "pooled": 1 + }, + "48": { + "detected_401000": 0, + "detected_410000": 1, + "pooled": 1 + } + }, + "rule_path": "fallback rule (no eligible budget)", + "selected_confirmation_budget": 16 + }, "base_main_commit": "16e153fdfa3739a88bca9b1559543a10ded2d859", "contract_sha256": "c878ff9fdbf4c412361f6058a72af1fa8c925e78997c89825d255c8d437f598f", "freeze_record_version": "budget-scaled-coverage-freeze-v1", From bc826021e464273dc10a2f88c3467ebb01663840 Mon Sep 17 00:00:00 2001 From: Cyinh Date: Sat, 22 Aug 2026 20:53:35 -0400 Subject: [PATCH 08/30] Record budget-scaled coverage research cycle evidence and final report --- .../artifacts-SHA256SUMS.txt | 22 + .../binding-cross-check.json | 20 + .../final-report.md | 162 +++ .../rendering-manifest.json | 1030 +++++++++++++++++ ...aled-coverage-effectiveness-v1.freeze.json | 9 +- 5 files changed, 1242 insertions(+), 1 deletion(-) create mode 100644 evidence/budget-scaled-coverage-2026-08-22/artifacts-SHA256SUMS.txt create mode 100644 evidence/budget-scaled-coverage-2026-08-22/binding-cross-check.json create mode 100644 evidence/budget-scaled-coverage-2026-08-22/final-report.md create mode 100644 evidence/budget-scaled-coverage-2026-08-22/rendering-manifest.json diff --git a/evidence/budget-scaled-coverage-2026-08-22/artifacts-SHA256SUMS.txt b/evidence/budget-scaled-coverage-2026-08-22/artifacts-SHA256SUMS.txt new file mode 100644 index 00000000..1711638f --- /dev/null +++ b/evidence/budget-scaled-coverage-2026-08-22/artifacts-SHA256SUMS.txt @@ -0,0 +1,22 @@ +4a6744ae64b783f770dd426012cd4439c1e2d772416655107682ed38143b2067 *binding-cross-check.json +49b105cecfeec14880d55ff59e578855f4ce9588574f1acd167799e0f9937038 *confirmation-corpus-420000.json +5d08679efcb3bfbe418beb07ec94480f6eefab017708a8ba9ed2afcb472dc07d *confirmation-evidence-420000-provenance.json +7fb0ce6c21e9f7c8a599889591e1ce24ff8cec7c8263da4a87444ffd4949ff5d *confirmation-evidence-420000.json +06cd3d9c496e65a0c7987e5e71d9ea57d25e704b59f14cae4f0d28bf634e1386 *confirmation-plan-420000-provenance.json +3079622ccdb4d86f48b1a77a64a00d6cff41b1918655d5526ae783025185181c *confirmation-plan-420000.json +c9d2686b16e7c0b213bff14475e845f06c53ac64601b53ccdbed1fea2a1287df *confirmation-robustness-420000.json +57b580c8839ea85a650993e302443fec4e16f76ea4d1ff7156fcb5468017f7ce *dev-corpus-401000-local.json +956e9a5c2994dc5a511ea535968cd4bbed68ba1a3ce50a1ca410c56df03ab62f *dev-corpus-410000.json +d0bec9524b623fb582b6f8699bbe4ef412ccd42b2a50662675480133e0f793e8 *dev-evidence-401000-provenance.json +d352c3e4393f088dc47fe40af3c256c71472853dd96432df48350de0673f9ec6 *dev-evidence-401000.json +12a63869c5f68845adfa88967faaae5669845f7e18fca882e98e50300ea3528a *dev-evidence-410000-provenance.json +2222d4293183071ecb479633bd279ddf858655c5539fd6f291030fd1faf4e88a *dev-evidence-410000.json +642cee2ba8b9ce853b020517e85b35ea34867e77b736d53ed95915e291301439 *dev-plan-401000-provenance.json +0fffc9284e502387e9743a281197e7bf989abd3b06dc2b0e72074c98ee941d40 *dev-plan-401000.json +e866f83819de7e4e89d7bad95123c4f8e41d499bfc876e1d056685ada61853dc *dev-plan-410000-provenance.json +18e05ee9d8be733af401e96e4f07dcab9f22048cb2b7b8c6dd860b97a3428a5a *dev-plan-410000.json +cbf6135f43fa6b6718d95320e2c6d5174e916353560efab1c2232f3af2183d1c *dev-robustness-401000.json +3cf84fedf6fc2818cf494873e98ca63c98c179e433d269102fc6f82a7750e99a *dev-robustness-pooled.json +6a8d031b883c1cc7eddfbb46f40d10de9be7233d9a93ca5cedb1b8dc4cf8c013 *rendering-manifest.json +59345fdece25c4ebe5c1123fabd525c957b665f6fc39db8bd069122eda484281 *confirmation-render-pairs.html +b0c740c804e2555f2b4a5616590cd342a2c60275686a5e7f8a44f29b92eb608c *confirmation-render-screenshot.png diff --git a/evidence/budget-scaled-coverage-2026-08-22/binding-cross-check.json b/evidence/budget-scaled-coverage-2026-08-22/binding-cross-check.json new file mode 100644 index 00000000..e02b619e --- /dev/null +++ b/evidence/budget-scaled-coverage-2026-08-22/binding-cross-check.json @@ -0,0 +1,20 @@ +{ + "corpus_artifact_hash_replay": "85cdda3bef549c961caac07767126ec2924f360f299b5d5a6b43f4272aa7e2ba", + "evidence_binds_corpus": true, + "evidence_binds_plan": true, + "evidence_hash_replay": true, + "evidence_profile": "key-blind-full-pool-coverage-v1", + "evidence_row_hash_replays": "128/128", + "evidence_row_texts_match_plan": true, + "evidence_threshold": 0.5584848484848485, + "evidence_threshold_hash_present": true, + "plan_binds_commit": true, + "plan_binds_corpus": true, + "plan_binds_manifest": true, + "plan_detector_access_false": true, + "plan_hash_replay": true, + "render_manifest_binds_plan": true, + "render_manifest_pair_count": 128, + "robustness_binds_plan": true, + "robustness_hash_replay": true +} diff --git a/evidence/budget-scaled-coverage-2026-08-22/final-report.md b/evidence/budget-scaled-coverage-2026-08-22/final-report.md new file mode 100644 index 00000000..de3a8dd7 --- /dev/null +++ b/evidence/budget-scaled-coverage-2026-08-22/final-report.md @@ -0,0 +1,162 @@ +# Budget-Scaled Key-Blind Coverage Effectiveness - Research Cycle Report + +Cycle date: 2026-08-22/23. Research branch: `research/ngram-window-coverage-v1` (local only; +nothing pushed). Base commit: `16e153fdfa3739a88bca9b1559543a10ded2d859` (`origin/main`, which +already contains merged PR #80). + +## Executive summary + +The pre-registered stretch target was NOT met. On a new, source-disjoint confirmation corpus of +64 watermarked attack sources, the frozen detector-blind key-blind budget-16 coverage profile +reduced fixed-open-detector detection from 63/64 (pristine) to 11/64 (transformed) - an 82.5% +relative reduction in the detected count under the documented condition - but did not reach 0/64. +All fidelity and representation gates passed on every confirmation row. The evidence supports +only: detector-specific detection reduction under the fixed open detector, public development +keys, and uncleaned text-only input path. It does not support watermark removal, +undetectability, unknown-key, proprietary-detector, cleaning-durability, or release claims. +Level B (product/release admissibility) is explicitly NOT met and was not pursued: the +candidate intentionally edits visible text. + +## Protocol provenance + +- Protocol, contract, and hash-binding freeze record committed before any scoring + (commit `29d7a40`). +- Five timestamped pre-score amendments, all recorded before confirmation scoring: + 1. Seed bases moved to 410000 (dev) / 420000 (confirmation) - pair-stride overlap arithmetic. + 2. CI canonical corpus pin (`ee09527d...`) is not byte-reproducible on the local Windows CPU + runtime (locally regenerated 401000 corpus `42240f72...` / file `57b580c8...` used as + development data only; adverse reproducibility finding, reported not hidden). + 3. Word-edit-rate gate raised from 0.15 to 0.30 (contractions cost two word edits each; the + original gate was below the mechanical floor of the ruleset). + 4. Codepoint audit re-scoped to candidate-INTRODUCED characters (absolute ban flagged + pre-existing source newlines, category Cc). + 5. Normalization gate re-scoped to introduced instability (precomposed accented source + characters are NFD-unstable; the candidate introduces ASCII only). +- Final protocol SHA-256 `4fa68c7519a2064c2bf131393598c5b181b92f2ae5fd62f9fd1959b7ebea0367`, + contract SHA-256 `c878ff9fdbf4c412361f6058a72af1fa8c925e78997c89825d255c8d437f598f`. +- Budget selection rule applied exactly as frozen: no budget reached 0/36 pooled dev + transformed-watermarked detections (all budgets tied at 1/36 because n-gram-window coverage + saturates at 9-13 edits, below the smallest swept budget), so the fallback selected + the smallest budget, 16. Recorded in the freeze-record appendix before confirmation scoring. + +## Development results (iteration data; separate from confirmation) + +| Corpus (seed base) | Watermarked sources | Transformed detected | False positives (unwm) | Mean score drop | +| --- | --- | --- | --- | --- | +| 401000 local | 4 | 0/4 | 0/4 | 0.0980 | +| 410000 (v3, 8 attack pairs/domain) | 32 | 1/32 | 0/32 | 0.1052 | +| Pooled | 36 | 1/36 | 0/36 | - | + +Budget sweep {16, 24, 32, 48} produced identical detections at every budget on every dev source +(coverage saturation; realized edit costs 9-13). Dev robustness gates: 288/288 rows pass. + +## Confirmation results (scored once, at commit `46c220f`, profile `key-blind-full-pool-coverage-v1`, budget 16) + +Detector: HF SynthID weighted mean, ngram_len 5, public development keys, uncleaned text-only +re-tokenized path. Threshold 0.558485 calibrated at target FPR 0.01 on the confirmation corpus's +own 100 unwatermarked calibration negatives (>= comparison). + +| Condition | Detected | Rate | Mean score | +| --- | --- | --- | --- | +| Pristine watermarked | 63/64 | 98.4% | 0.6333 | +| Transformed watermarked | 11/64 | 17.2% | 0.5315 | +| Pristine unwatermarked | 1/64 | 1.6% | - | +| Transformed unwatermarked | 0/64 | 0.0% | 0.5029 | + +Mean per-source watermarked score drop: 0.1018. Residual detection by domain: +technical_explanation 5/16, conversational_prose 3/16, structured_instructional 2/16, +general_explanatory 1/16. Eight of eleven residuals sit within +0.024 of the threshold; four +non-detected sources sit within -0.012 - the residual is margin-driven. Realized edit costs: +mean 11.89 of a mean candidate pool of 18.6; only 2/64 sources exhausted the budget-16 cap; +one source enumerated zero candidates and passed through unchanged (still detected). + +Pre-registered secondary gates: pristine watermarked >= 60/64 PASSED (63; one weak-watermark +source below threshold at 0.5569 vs 0.5585, reported as an adverse observation). +Transformed-unwatermarked == 0/64 PASSED. Section-6 fidelity gates on 128/128 confirmation +rows PASSED (hard invariants, protected spans zero, word edit rate max 0.1333 <= 0.30, no +introduced non-ASCII or hidden-category codepoints, no introduced normalization instability, +Cf stripping is the identity on every introduced character by construction and per row). + +Primary endpoint (0/64): FAILED. + +## Representation, normalization, and rendering evidence + +- Codepoint audit: no introduced Cc/Cf/Mn/Mc/Me/Zl/Zp or non-space Zs characters on any of the + 128 rows; no introduced non-ASCII characters at all. Pre-existing source newlines and two + precomposed accented characters (NFD-unstable sources) are corpus properties, reported. +- Normalization: NFC/NFD/NFKC/NFKD stability preserved on all rows (sources stable under a form + imply transformed stable under it). The two NFD-unstable sources were unstable before + transformation. +- Fixed-browser rendering: all 128 source/transform pairs rendered in a fixed browser page; + DOM textContent equals the plan texts byte-for-byte on 128/128 pairs (0 source mismatches, + 0 transformed mismatches); screenshot archived + (`confirmation-render-screenshot.png`, SHA-256 `b0c740c8...`). +- Copy/DOM divergence: none possible by construction and none observed (no control or format + characters introduced). + +## Independent hash cross-check + +Reloaded artifacts from disk and recomputed every binding: corpus artifact hash replays; +plan hash replays and binds corpus artifact hash, manifest hash, code commit +`46c220f1c0dfe4c2b401cc32ecf4aaf70ac5ed7b`, detector_access_observed=False, +secret_access_observed=False; evidence artifact hash replays, binds the plan hash, corpus hash, +profile id `key-blind-full-pool-coverage-v1`, detector identity, calibration bundle, and +threshold; 128/128 row hashes replay; every scored transformed text hash matches the plan +variant's transformed text; robustness artifact hash replays and binds the plan; rendering +manifest binds the plan with 128 pairs. See `artifacts/budget-scaled/binding-cross-check.json` +(copied into this evidence directory). + +## Adverse and deviation findings (all reported, none suppressed) + +1. Canonical CI corpus pin is not byte-reproducible on this Windows CPU runtime despite pinned + versions; corpus generation is platform-sensitive. All within-cycle comparisons use one + runtime and per-corpus calibration, so the science is internally consistent, but + cross-platform regeneration of the same corpus artifact is not guaranteed. +2. Disjointness deviation: one confirmation attack text is byte-identical to one development + corpus (401000) watermarked calibration sample - a degenerate GPT-2 repetition under + different prompts and seeds. Calibration thresholds use only unwatermarked negatives, and + no development tuning ever read this text's detector score, so no optimization leak is + possible; the text-hash disjointness gate (all pairwise corpus overlaps zero) was violated + once (1 shared hash, dev-corpus-401000-local to confirmation corpus) and is reported as a + protocol deviation. +3. Development underpredicted confirmation residual detection (1/36 dev vs 11/64 + confirmation). The budget sweep showed saturation, so no frozen-rule budget choice could + have changed the outcome. +4. One pristine unwatermarked source scored above threshold (1.6% observed vs 1% nominal FPR; + consistent with binomial noise at n=64, P >= 1 at 1% is about 47%). +5. One pristine watermarked source scored below threshold (weak watermark signal). +6. One confirmation source enumerated zero transform candidates (no rule matched) and was + passed through unchanged. + +## Classification + +- Product-safe code delivered: corpus profile v3 (parameterized attack pairs, frozen + calibration), full-pool coverage profile factory with resolver, parameterized plan/score + runners (defaults preserve the frozen B16 CI behavior), robustness audit module with + introduced-codepoint and normalization-stability semantics, render-page tool, and 24 new + tests. +- Quarantined experiment status: unchanged; the U+200C visible-projection experiment was not + touched and remains excluded from default, development-release, and CLI paths. +- Candidate outcome: exploratory partial effectiveness under one fixed condition; primary + target failed; NOT product/release admissible (visible-text edits); no overclaim. +- Deviations: enumerated above; the corpus-generation platform sensitivity is the most + significant reproducibility limitation. + +## Reproducibility + +Full suite at final state: `python -m pytest -o addopts=` -> 1415 passed, 8 skipped, 1 failed +(`test_frozen_revision_2_spec_is_exact`, the pre-existing Windows CRLF-only `spec.md` freeze +assertion documented in PR #80; `git show HEAD:spec.md` hashes exactly to the pinned value). +`git diff --check` clean. Artifact SHA-256 manifest: `artifacts-SHA256SUMS.txt` in this +directory. Environment: Python 3.11.15, torch 2.13.0+cpu, transformers 5.16.0.dev0 at the +pinned git commit, Windows-10-10.0.26220. + +## Permitted conclusion, precisely + +Under the fixed Hugging Face SynthID weighted-mean open detector with public development keys +and the uncleaned text-only input path, at the pre-registered frozen threshold, the budget-16 +key-blind coverage profile reduced watermarked detection from 63/64 to 11/64 on a new, +source-disjoint 64-source confirmation corpus while passing all pre-registered fidelity, +representation, normalization, and rendering gates. It did not reach zero detection. Nothing in +this cycle demonstrates watermark removal, undetectability, robustness against input cleaning, +unknown keys, proprietary detectors, or release readiness. diff --git a/evidence/budget-scaled-coverage-2026-08-22/rendering-manifest.json b/evidence/budget-scaled-coverage-2026-08-22/rendering-manifest.json new file mode 100644 index 00000000..cf53a892 --- /dev/null +++ b/evidence/budget-scaled-coverage-2026-08-22/rendering-manifest.json @@ -0,0 +1,1030 @@ +{ + "pair_count": 128, + "pairs": [ + { + "pair_index": 0, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-conversational_prose-25-unwatermarked", + "source_text_hash": "d6708285d65976877cd638c856e189cc2c8731a817539480434f2c4f46d2165f", + "transformed_text_hash": "2dedf2b5211b13505c17ed937b95fb6aeb7119db8833a52175c9dd74a03a26ee", + "variant_hash": "b0453702a822d2590421804f9af59b7caebb6f23b402f05353bb935014963df9" + }, + { + "pair_index": 1, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-conversational_prose-25-watermarked", + "source_text_hash": "9a286feb31c48e9ed0c9999789711b2579248fb083202063af1c669f37fecce4", + "transformed_text_hash": "ccb2c55f42aedd03f029ac83debc2edc3b873ff2b45dcfcc9b7c1c76250edc06", + "variant_hash": "9c08407ebff44ab776b64b61359cf13f44b956f72e9c7b9f44f10aec346af42b" + }, + { + "pair_index": 2, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-conversational_prose-26-unwatermarked", + "source_text_hash": "28a63d542a728713d0004d489be3895ec891f42df3ade3474381d3ab0e5f5ef0", + "transformed_text_hash": "a2b44c5407188a9969010a6e789828b06dc79ba16fd0080943fd385a76f91aba", + "variant_hash": "79be387834ba3bc142b78c77c9518559ab02fc500ee9e761f3883545492b5196" + }, + { + "pair_index": 3, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-conversational_prose-26-watermarked", + "source_text_hash": "1d936ad5aedeb613dc0b674eeae26f5cda6b24002664c061138008810fdf4de9", + "transformed_text_hash": "01a1e1aa606b9c9b6c584451302cce0d453c9ac1deeb543fc52ec56ef37d5ec7", + "variant_hash": "60197714bdbf8d5108b4b6663c138ebd8ae1406e39fe2c92278956b7946f6b3b" + }, + { + "pair_index": 4, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-conversational_prose-27-unwatermarked", + "source_text_hash": "e093007ba4fd439eecb20a9aff6088aa297f19447b26043c1bf8f753e2c829c4", + "transformed_text_hash": "07664b12c1af72e2f7be4829f11c97c008005de86faf15610cb9519abd303e07", + "variant_hash": "8403a8a97f960c31d0110a83b3aab6e262b6b464088e67ba2426f45965ac953e" + }, + { + "pair_index": 5, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-conversational_prose-27-watermarked", + "source_text_hash": "1997a2737eaa043176c546ab32aa604dcb6f2201bbc1454ab7965dcc3de57f7e", + "transformed_text_hash": "2e167e73821cb00702a32b3d3b620681fbb4c9bde0450c583e58dc372cbd8c4b", + "variant_hash": "6703efa875bbbabca3f65fe7c5fcb74d1eb06ce2b38a7bffe7c7d6dcd92871cf" + }, + { + "pair_index": 6, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-conversational_prose-28-unwatermarked", + "source_text_hash": "a671440b0822fcf6fdfd38074e7646e659fffe950db6c360b79eb35877b5240e", + "transformed_text_hash": "0959df19e6e37e313a8ed9112ac26880c7367f043fa8dec7d1f8015ec502be53", + "variant_hash": "f63a8709b060103b6f7bb5c629ac06a5ed4a023a3a1a44161e6d31daf742559a" + }, + { + "pair_index": 7, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-conversational_prose-28-watermarked", + "source_text_hash": "6a61fb0e5c1bd11ca922cf7549c3c08a05fc767c038fb3ffa5d90d0f43123f65", + "transformed_text_hash": "e424251a96f0c0e161ce276bd2163cad2ba2e80e01555ec368bdcacf0fd88ab6", + "variant_hash": "4a25eaca09e5299552edcb911d7408fc0699bae0ea24c32ea232721566ad4d1c" + }, + { + "pair_index": 8, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-conversational_prose-29-unwatermarked", + "source_text_hash": "62076d10fb04dc787d008b7cb4a77e9b5caf195029ecb5bed15a22519eaee486", + "transformed_text_hash": "658772dbb731fed08c020d90bac7dcf226cec94b2aef742a3f320bc992b5bfcc", + "variant_hash": "2745bb294cb9fe86777f26629332f204d718bfca8baf4fc8887f1c039cf204c9" + }, + { + "pair_index": 9, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-conversational_prose-29-watermarked", + "source_text_hash": "efe80422df6408b2e344bf490cc52fb67affc50f8626368df5d75d49ece9f003", + "transformed_text_hash": "a9ded56ce0f189d27f942931df5be047bca68d66574a14d75ab07ce4efde56cf", + "variant_hash": "b53d7dc17e715c5c465651235845acbf0ca9d3ec4d287a3fcb1ff13fe062f75b" + }, + { + "pair_index": 10, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-conversational_prose-30-unwatermarked", + "source_text_hash": "46110e06178e781b93329a52260164d0da26a9e55ed57a19900b913874b6901e", + "transformed_text_hash": "911b00f3456af46f3c0431dd35b1b45513c7144005bb83ad32be55dea7c8e131", + "variant_hash": "5acb8cabb0653736272cd6394170e436e58478cf2aec2172f4f3f6b6e0f11856" + }, + { + "pair_index": 11, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-conversational_prose-30-watermarked", + "source_text_hash": "f32ccdb871662f4113ff2adbb9eb6677ac0d8f6cc0d66afcc768093e68b1741c", + "transformed_text_hash": "810a16a234f5850f56dbcc92c3a0716a32646ce10d1283a8bbf4ee2693a2bd12", + "variant_hash": "3fb6693f7f40c4d2f6b141dcb0ba933e3ebc6b2eab3808c10f976af1df959dcf" + }, + { + "pair_index": 12, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-conversational_prose-31-unwatermarked", + "source_text_hash": "081245380d9b579084680a58e4e00faa4138a9e6593251d946505fdbae1b867c", + "transformed_text_hash": "0c07b8bf910d57a0ac858c83a23a57e192bead38a65bd3715e598d94e7caa468", + "variant_hash": "32dd000602ad7a62290b0537bc5deb6ae0da8dfa0ceee2837c81dc63e26ecb09" + }, + { + "pair_index": 13, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-conversational_prose-31-watermarked", + "source_text_hash": "84f33bd813a752f02a66f3d54d3a6795ee50f49ca0b17437187f62788ee52d10", + "transformed_text_hash": "67e7c3192f3a9c08529225b77eb59a7f3e9edd413c9b4b7ce3132c6984021d94", + "variant_hash": "a52bd68ac8affa345789476c33c6035ab3dac9f128cea6b19789909935243b49" + }, + { + "pair_index": 14, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-conversational_prose-32-unwatermarked", + "source_text_hash": "3258bc7eaee6dc1ec7cef4af96fc255c14b5b9ad018303678aa4d4c7fb66943b", + "transformed_text_hash": "c41ecdea48958aa641dcf27b4ab317913447f5ca8ed5303e6551d9b8ef7e3bd2", + "variant_hash": "8f89499f17f534c52164fe6700544838186284b806ac1034a75c6d9fb3593da2" + }, + { + "pair_index": 15, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-conversational_prose-32-watermarked", + "source_text_hash": "86366f2768d30b0be0407e38d772381c497dea2737135a99accb152c37a9c303", + "transformed_text_hash": "2903add455715008ede4664065862a1b8b968880ee7d962f449d5c56e3608aa8", + "variant_hash": "c0929a16ea739268db7732a513ceabc0b08b854e27214467ec079298522c581a" + }, + { + "pair_index": 16, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-conversational_prose-33-unwatermarked", + "source_text_hash": "4b0e4f32c2b572aa6fa7f42aef596bd0a468cd10f849562e1ba491e0354ab6bb", + "transformed_text_hash": "78416f6bf9af092afbdea548a9c2fe7f8c0f54028c2a511dc2e4f2bd75de3885", + "variant_hash": "33b98dceb17f76910e618d8d98f6ec004e3cf6f020addf25b8549d764dd04e1b" + }, + { + "pair_index": 17, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-conversational_prose-33-watermarked", + "source_text_hash": "d3b2df8701d5bfd247b4a1fe53e961d65c64048122c8ae2d4a9e6ba4e9538dc3", + "transformed_text_hash": "391d4624d6b6291d6f4a4ba6a5c1db90352979e1cb90975069298cce8007cec3", + "variant_hash": "b821af27ee2e7d84fb0af0d74a007c7461d80048cf06abc631c9b855fbf1ba1f" + }, + { + "pair_index": 18, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-conversational_prose-34-unwatermarked", + "source_text_hash": "7409ca9b2af0fa76a7c8d1641b78cfb24eca034777ae5a0b5779e5327186cd0b", + "transformed_text_hash": "09af6858fabeaafa6604fe1d831177382d904048cd8bb55a34e23c96a3fee93e", + "variant_hash": "ad8b3042f88c3a00e2b969d38f830da91eb8334f1a64098537bc1d37fd7a30fd" + }, + { + "pair_index": 19, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-conversational_prose-34-watermarked", + "source_text_hash": "2c951d293a0ef7aaa3cc150a5b57fd316b292701b344606aba0ebeb1bf39e2bd", + "transformed_text_hash": "2de55ce96a831afd1021efdff4b07909ad85e623c223c8166bfc61e41f6ab97d", + "variant_hash": "26e613e25b69c6940a2214265fefb7248b0dc6f085c5dbd488e7758dcedd660b" + }, + { + "pair_index": 20, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-conversational_prose-35-unwatermarked", + "source_text_hash": "b4b4eadc592ece2aa02c5cad703b16e32084f3ed6b8e377e60c7b3abe29d9b43", + "transformed_text_hash": "fa91884b4482f3494279dcbdc2713cec9e865d515978d6ed3a5781ef15fc79c1", + "variant_hash": "8adbf2c7fbd8bd01733424c40a5961978b333c02d8ab2cdc53679af619ad713c" + }, + { + "pair_index": 21, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-conversational_prose-35-watermarked", + "source_text_hash": "39f706f158f225d80afb4f92fe3f9c96edb1ec68a2d0046d3dc2453c1cebbe87", + "transformed_text_hash": "cbb85345f10b963dc32d4face2e70175949ead599b9f8f916ccda4cf4067812a", + "variant_hash": "621a9f3627902c5fb4e7f90bd8e38f15bc6c95c3c71a41ec180509bdfb78172e" + }, + { + "pair_index": 22, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-conversational_prose-36-unwatermarked", + "source_text_hash": "e8c5dc04c7d846973f1fb1eb16a61a630e3c54cd151af0fd3d241907a7018bd0", + "transformed_text_hash": "2bfa9f53db8927820e985fbbc0fb0554f1d56065ae1ac116154863a334f1d6ba", + "variant_hash": "551f08c82df5839e9ffa154f6bcd9f43ce7b5c3a571ed379ffd6f6cc4f525c7b" + }, + { + "pair_index": 23, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-conversational_prose-36-watermarked", + "source_text_hash": "71d5675b314dbc6683c6755efe4e4930a96816e85cd09d1d365b08443f2d8f6a", + "transformed_text_hash": "6597a4e811a00a590ed4f6ac07cf9c7c4326025f29c41e9aabd90944dc9f321d", + "variant_hash": "5e0a898cd8d29645733d5716fada331ee2f45bed3cf4570e4615e3ca69cd55c7" + }, + { + "pair_index": 24, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-conversational_prose-37-unwatermarked", + "source_text_hash": "36ba5db85f67575b54624c78f9b566063b0f58bfe410ae8e8f36517ac4811551", + "transformed_text_hash": "2cd123475ec3cd7c75d9696f382044f6f1f43eed09cc6842a8438c67fa166cf1", + "variant_hash": "61cfde568b39167933043f28e0bb87681de933cc540c5c6eca2dd848151166a3" + }, + { + "pair_index": 25, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-conversational_prose-37-watermarked", + "source_text_hash": "e2f26924c9d3b4f2bc024ba7e73e30f923485177423c59a518e3e40a3d8ed6ba", + "transformed_text_hash": "edeab024e8acb049c29d09330bfd53468223e454bdbbbd02a4a0a33a236605e6", + "variant_hash": "432c7f27ac9d2fbb00f938328b6de4d7370fb4c1197b2d1a0e275a555433f44b" + }, + { + "pair_index": 26, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-conversational_prose-38-unwatermarked", + "source_text_hash": "3bcca26dc68d190d03e98d96124babd18dac89a144583a0d1566c2ff588de1d3", + "transformed_text_hash": "8dce6a2ff68f1d07fc60482c67a9f7f5f891c4e1c0a3a3a0d4ba1ee9aee6d938", + "variant_hash": "aba89a6172474674623e5d783290c558916b5f0db55011c60fe774b8d5ea6482" + }, + { + "pair_index": 27, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-conversational_prose-38-watermarked", + "source_text_hash": "f31c7adbbca6d62a5726eeed8fca7d4aabfb9f9b4c8a36671d976c8789c0c5b4", + "transformed_text_hash": "07233fd02d3266a1cd910142e0c1aaf7b1f7ac579578c8b79a5eae364e733728", + "variant_hash": "7855fdd337c1865973232afda20fc01af6c76e6d1fa9c79db9b0524142fd3ad3" + }, + { + "pair_index": 28, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-conversational_prose-39-unwatermarked", + "source_text_hash": "59fbd805973ae2c003e7ca3ac05503290701665bf809a644a16dadff4be52291", + "transformed_text_hash": "bae50882f9377a2050be86a8808dea5839dd68ec9d1c114f9f5eec2abf02034e", + "variant_hash": "82d6672286411d0eaa7318b220e0095a4e7fe87dc3fc11b80fc45a7195c45b76" + }, + { + "pair_index": 29, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-conversational_prose-39-watermarked", + "source_text_hash": "df5150a41694687e719a836d2fa02c0e7200514fd8839a03a965584874d7bc77", + "transformed_text_hash": "b6381f2bff3b7499590bc2545ddbc599869a3dd3672239ecb8ae74ef7f4d1162", + "variant_hash": "20d19dfaee48c5a90ba8381249e014d917827e9cd6dd685dca27124c28de9f4e" + }, + { + "pair_index": 30, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-conversational_prose-40-unwatermarked", + "source_text_hash": "8c37969af9ad8ba0100a37ca6034167f671a2f3fab2662a96160c253846d3a03", + "transformed_text_hash": "cdf14cc71f5fe95995c1f94b0e35e4320c2731fdfbde63d0d3d95aaf8219d308", + "variant_hash": "4b99a27f0383ec93748124a8fed5a76dcfee3d55b12feb8eabe2570e6b34a758" + }, + { + "pair_index": 31, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-conversational_prose-40-watermarked", + "source_text_hash": "8d7c6ee86d885114b21f4d1d0701aa2b1172e3a72a7ffc55089f17d03e79a632", + "transformed_text_hash": "fb498ef4516bd6c69a1ac585c90f1ddade1e5efca68e1597a2741040c200d652", + "variant_hash": "87b82a74867f3da51f3bc11d469d012d204e7c952556e971c7a8f36e4c29a970" + }, + { + "pair_index": 32, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-general_explanatory-25-unwatermarked", + "source_text_hash": "fbb24992cd2c56529e2e67830c2f1b7e085cfbe8b9fa3165cdb155a8b0322efe", + "transformed_text_hash": "91d3c1a54ff9c2298f70b576662c9d4d03284edb78c7254482b29057c793fcf9", + "variant_hash": "40ee0209412c287928861d56be4cc1a9a678f69510b2b2b601cf00eec6a3547f" + }, + { + "pair_index": 33, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-general_explanatory-25-watermarked", + "source_text_hash": "92dd95e4efb92c47acf64fa85d32484d0cdf2abd6dc1f6134f25f523fd618d31", + "transformed_text_hash": "dead23c8aa8a8a183a1f6f481483b1cf49254dc35cad96c769db012dc7a06c36", + "variant_hash": "39cb74587a9fb1eabfb55325985e71b2eb92d40c57191409da123dee61fa0b47" + }, + { + "pair_index": 34, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-general_explanatory-26-unwatermarked", + "source_text_hash": "7ee55c3c1859567a1e62f0d2a5a5a907da34ca9cb506a733fd364d65a2e1a6a9", + "transformed_text_hash": "9558bb0e746d4039ede2704c9e7e51455dd8862fb420858d4cdfbfe5b07ed15f", + "variant_hash": "d2348c9ba55f78942612e8406bf510e42cd0785b4c127939a10962f6270faba4" + }, + { + "pair_index": 35, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-general_explanatory-26-watermarked", + "source_text_hash": "c6c4688cd77773ef7e81e66dd1e120657b79a936b3bc15316b7cc9f962b4c36a", + "transformed_text_hash": "18169915d727a1d811b6dce37ff12de3cdf1f7f435f5c8c21c599834818048f7", + "variant_hash": "e183c9a315734f620f3f0d16f7ffd10a4b535a12acecaa59b563a9b5c69727ed" + }, + { + "pair_index": 36, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-general_explanatory-27-unwatermarked", + "source_text_hash": "e22bec68fa599895b14f7a67e9fb25b8eb8ed8e2f19418a9cfc8dc166abcd5fd", + "transformed_text_hash": "c10300f3915f526defc1ad88c83246e0ea78a78216accd24b643148e93e0fed7", + "variant_hash": "a30c8504f1d215aad8fa100ab6d303aa9510d5b9c4d50ab445786c2de7aa7936" + }, + { + "pair_index": 37, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-general_explanatory-27-watermarked", + "source_text_hash": "3725849e088ec7704fcadd7de1e7b958a262e2161ca7438a14a8b8dedb492622", + "transformed_text_hash": "88089edf8f3be3d6c6a00c38af378ddc1158a74986abefaec2d3fa2ee4f5d59f", + "variant_hash": "bdef41ca87da0a48341cdbdf8ec7d16c66381739a73ea9dc2e08a0ec95de77c5" + }, + { + "pair_index": 38, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-general_explanatory-28-unwatermarked", + "source_text_hash": "76a08256c0f54cfd3f73dc2c6960b216cd95a02e729f998b33cb68af99879904", + "transformed_text_hash": "d60fffb793330229d29bfdeb198d308faa9837061585b28f04866e28a2fb72d7", + "variant_hash": "3de4e1b3628b84b6841c7e0e4faae6dade4d4dcb30f6b1b5bb55a18e5d9f6e3b" + }, + { + "pair_index": 39, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-general_explanatory-28-watermarked", + "source_text_hash": "afecfe7217a45d67e7dddf27f3d943670d1ebc4aff3f53dadc23f47441ecb97e", + "transformed_text_hash": "1bd61092e33d01d0aad2804b4186fdeed09b5c646159413f6a238f54f4c3a849", + "variant_hash": "447e8486345d0aa13749c38272125177a4577c26bd6a726e16539ccf23b72021" + }, + { + "pair_index": 40, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-general_explanatory-29-unwatermarked", + "source_text_hash": "a18560c164f77e6e4cdfbe07c5a0630476f9eb043e20fb97f4945c7523eb7fd3", + "transformed_text_hash": "60b2e43535824fd120c5c09e2f70ba7bfb18006973ab56c323fe6f0a87b8b5e0", + "variant_hash": "136eec6e4cb72623faba16c2206612609801a047155408294914a4a9d07037f5" + }, + { + "pair_index": 41, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-general_explanatory-29-watermarked", + "source_text_hash": "d34ffbeb248bb79fbe4461bfa05dcced87a608c01d2b72bf137285300ad022a7", + "transformed_text_hash": "e484ec1c9861ca847a4369c7a7e080fa325d7751cfde3a866af54b1da7cb54bf", + "variant_hash": "6d63eacc156c16b72e6bbe610dd8a9fa1f7feb1a34b1e5637a575b001604fd01" + }, + { + "pair_index": 42, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-general_explanatory-30-unwatermarked", + "source_text_hash": "63b695b0b3095c5999332e768f70374cf781414cf40d3c4dd645045538912fc4", + "transformed_text_hash": "5507061be4038001805837243516fe91d6d4f0505341dfa94ecf2f603ad9c756", + "variant_hash": "fabba121784618552cbbd7bb6a4e83eeb5354fa04eccaa225888731c5a1a4aa8" + }, + { + "pair_index": 43, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-general_explanatory-30-watermarked", + "source_text_hash": "ff476d79c406a3a7201e13f1efe45d15e8a82febbca45e28549c736329fd320c", + "transformed_text_hash": "68569314a56901a307636571791947b6686e1b7e9707d33db9f5631316187853", + "variant_hash": "e01da9ca178e8833154f80e3c1b0a5484b71e847eec5d0bc074ceca19d8d0387" + }, + { + "pair_index": 44, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-general_explanatory-31-unwatermarked", + "source_text_hash": "e233350d3c2830701e988fe9eb89e7b7c1fe7772e3e98a46e8a5c564ce5fba68", + "transformed_text_hash": "199b6bd6bf6ef33eee0e22c00f3f2935f15869bde16501a1508a526e2db938b6", + "variant_hash": "1fb60691ebed77a878169b6b84a744afaf03a2a66b6bb366486ec242bb8bca53" + }, + { + "pair_index": 45, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-general_explanatory-31-watermarked", + "source_text_hash": "1cd4dcdaad6b8682be53683493a326f80da5878f112437958c7de96aae23c588", + "transformed_text_hash": "8ddc87de99b05722531dc03d91c8c4abc0cb4e63f71ae5f93f5c482070bad468", + "variant_hash": "3853c2fcd01c45b2bee2d6ba6db7fb6c763dd09e643132ba4425bec5a0b57fad" + }, + { + "pair_index": 46, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-general_explanatory-32-unwatermarked", + "source_text_hash": "4f7e460f3f9baab5213a2b877339f83ed726b54973c9217c00e4f27d2987f409", + "transformed_text_hash": "a59558cc39d9dd8b6cd60ff5c2aa54938216926a3b543d6601a82da68f5b18d0", + "variant_hash": "02e4bc12dde4ac14e557088baf8eafa66474522c3bb01dbe81855988b503d0fb" + }, + { + "pair_index": 47, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-general_explanatory-32-watermarked", + "source_text_hash": "c8a32fcfcde6639a3d5f66aaafeeb9ba1883ebf19fe6aea7aad67d2806f5feb7", + "transformed_text_hash": "9fa24d9906ffb87d65b00c2905dc9a1f28defcd5c2d4f5ecb4353a8b041520e8", + "variant_hash": "ce32cc63d6bddcce7e015fcd1f7a9032e7f15e77459b8163353a61bb266cc9c9" + }, + { + "pair_index": 48, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-general_explanatory-33-unwatermarked", + "source_text_hash": "6d78c10093957ecff0e6ec0384c48a1460a2708d1790a450aaf7248e439f9943", + "transformed_text_hash": "6d78c10093957ecff0e6ec0384c48a1460a2708d1790a450aaf7248e439f9943", + "variant_hash": "b56435fe9583008fa031aed539a01b0121d857c0f976ddd51bd4749e93f9a67e" + }, + { + "pair_index": 49, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-general_explanatory-33-watermarked", + "source_text_hash": "dd09e6a66240dd1aa8859c9be952b0788f07bdbee4f4d486034e281e3e0b1a82", + "transformed_text_hash": "e247fb3ac95c5b6f1ac1b8ded1f13fd6e3af0ad39316b6c62e9b7d0aa1ac5025", + "variant_hash": "b262bafc81817c01401fdce749b5c1d89ca10f04df5ec180e3c4aee816b80279" + }, + { + "pair_index": 50, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-general_explanatory-34-unwatermarked", + "source_text_hash": "5b721d62dfa8ca30b711dc008912332182c4b43d0c9180ca295fc0a87a45755f", + "transformed_text_hash": "1f42c4509fc90f69d5007443dd56da99541c4af1bf5001259ead8552f032623f", + "variant_hash": "e63e0fee07a46a8e0cb5fabb7ed30ff43b4bf34b712b94515ba230ee420fd4bb" + }, + { + "pair_index": 51, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-general_explanatory-34-watermarked", + "source_text_hash": "193af4f5f4fcc333a30857b687a3aa199488bc118058dc0d403b11263f004a0c", + "transformed_text_hash": "2c9b77a7735ba1ba594ddb0944a81b0683e7033489d0db11a1a1f99e94f77efd", + "variant_hash": "f50321257fdbe3f15ba1b9745888d256306d63a6e92735c41194506325466c97" + }, + { + "pair_index": 52, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-general_explanatory-35-unwatermarked", + "source_text_hash": "9dc34c5dd7adf1e6bfccc838b803df7edc0b94bd0c1676600685f4a294ff0685", + "transformed_text_hash": "7d35791a32750bd62bf272e5ff1e6d3567edf0479c2167e3cdc42739f7a868ac", + "variant_hash": "29a49b12e03c2aa1b3d0f298c440e9c263da5f7b1da7f78a474fb4754daf4ca5" + }, + { + "pair_index": 53, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-general_explanatory-35-watermarked", + "source_text_hash": "e4b4d7897a0afee76dc29a7b91f09df662698f27433a2ec6c09128b6f389b1f5", + "transformed_text_hash": "2778e3f89efc5e0447bed25b6782fa02b5afcf5e5c60eea3b3ae3f849a02729d", + "variant_hash": "cdd6b8abe247ba6c236eea2022f0673665b7e8e56355251c27d183d44741a336" + }, + { + "pair_index": 54, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-general_explanatory-36-unwatermarked", + "source_text_hash": "190d2bbff1e708f7d11f156db266447b976baad7f5c825f0ae8c21e20bf9b4ec", + "transformed_text_hash": "a349dbbfb630fce308d29e69d1ebfeb688de91955bb37eab60cc96e5f4161017", + "variant_hash": "8890380b8d884bcff83115e8a63076398cdb4b47b112f76eb1c2237fee7b32e8" + }, + { + "pair_index": 55, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-general_explanatory-36-watermarked", + "source_text_hash": "2137ae71b3644ec82423ca4e75b545d8a3c5eb7a788dcb0182752a82305ee887", + "transformed_text_hash": "e5d80cd3eb50a7db3eae3cd2d0704a6a6ee3abc8085db5934049f5316636c7f4", + "variant_hash": "fc38182d2c0a63328b08db052af33bf4c841e15fd704349d138d25b9804d7a75" + }, + { + "pair_index": 56, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-general_explanatory-37-unwatermarked", + "source_text_hash": "d6813d5634c7657976a49e54e1ee3fb0c262743ba41d03186f3f47ae76f0950f", + "transformed_text_hash": "6c5922fb7facab8b2336feb3144d085a8d23d6b7121ff67e5fe678fe657802ce", + "variant_hash": "49a9836ab8ff2e6294f9d0b241f5b0eccefbce0cb4ca9450e05b4d5cf7305d8c" + }, + { + "pair_index": 57, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-general_explanatory-37-watermarked", + "source_text_hash": "c026a3b7a76822e5d5fb419554eb33859cb34f470b3f3fd022081958eb095df0", + "transformed_text_hash": "5470b665d53db233e8ed64e494338b60ee86e405f77a76d9cc96c6ba1dc9933f", + "variant_hash": "3274da063da5db8affb703e59b06f4bb6a8e5b0f4e6258037f9ff38be0ba34aa" + }, + { + "pair_index": 58, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-general_explanatory-38-unwatermarked", + "source_text_hash": "d945e74d3d6302e33d785c0f6aa31218e28de62d562d44b3b20d8139f297309b", + "transformed_text_hash": "631dd78c4f8e0f88af9e417255b8d56f11f935111176bc021ff6c68d4170e6b0", + "variant_hash": "52ffdb16e60b85637203ac06f95a0886302dd23ade638d5557ee9d34205b504d" + }, + { + "pair_index": 59, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-general_explanatory-38-watermarked", + "source_text_hash": "4863ba551a30470274c01823d63cf6b8da30fb125157a69bb2a313a2c67192ba", + "transformed_text_hash": "444297244587797a065cbce91ada0063fb5266e274c06ccba27738dbb29b623f", + "variant_hash": "ffae6e5c940f21c65d04d579e0cb844459bc72ccba3f6f11290dc7f173a56ee8" + }, + { + "pair_index": 60, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-general_explanatory-39-unwatermarked", + "source_text_hash": "f3cee919e65c87e7a778a3615d48e5f83461f050cefc4d1ef1220dc3a9347b0b", + "transformed_text_hash": "b9c520ff45f3729cb83ac8829562e2e182482d6a5da0a541dda14822b79ed91f", + "variant_hash": "665904e5852e2b1a03701312f2252e78af96483e732a1c17a943f11d97e02121" + }, + { + "pair_index": 61, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-general_explanatory-39-watermarked", + "source_text_hash": "58a323a4e75b4ff9a9f3ce1e154280754e04c6834903e20d5e62c2db8fcbef16", + "transformed_text_hash": "e9ce6f20bc711e3de9d05f6b851678847068c3bffb9fc3b738cb11e1becadd4f", + "variant_hash": "add68877a3c99b5f90334f467c2a72cbf7b3b6ff3b68c83879d63c8a4b2e95f6" + }, + { + "pair_index": 62, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-general_explanatory-40-unwatermarked", + "source_text_hash": "6bcfd4018abf123952fc2f97dc7c72cd3559e0c3cf6f12896093d85662acc9d8", + "transformed_text_hash": "1d50801f2efa974968106f09dc90944c5f2d77511ee705cdd038cf6f0753280d", + "variant_hash": "1605095faa8955ad813e822cf7dcd0c2e9da8b7df1afeaee5e3fd5ac572a1ca8" + }, + { + "pair_index": 63, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-general_explanatory-40-watermarked", + "source_text_hash": "4d92361e194b5afbbb52973066877852f23b8d8f884aed764ac1d26b78d3b718", + "transformed_text_hash": "186ecbea9be417d64340d31ffe8517dd37cdbe0ff5808353a452a3547a8cedbc", + "variant_hash": "0f32e7df37649cb4df387bf78e8a37dd059c41815cdc22eb0ae23b194c92c177" + }, + { + "pair_index": 64, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-structured_instructional-25-unwatermarked", + "source_text_hash": "025cb365b51494a2bdb43d1a2716adb08bb4f381510b14326ba914c03c553220", + "transformed_text_hash": "2b81386abec26cfca7f2416a2e625f1d9afb48a07d80870d3aec81bf0a739e04", + "variant_hash": "ae4be47fecf6dedc26a38fdeea3bb4feb0ecec819d06ce2a9d4807a0465b216a" + }, + { + "pair_index": 65, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-structured_instructional-25-watermarked", + "source_text_hash": "2b62c7cc0ade63a8eee3e94785107948f74d8876b02966bc4f1c29f5bf4ec80c", + "transformed_text_hash": "7beff0b797a693836a34ba440003fe5455a88d40456b3ee91363c9a9f7252483", + "variant_hash": "7d112173b225d8bd408f4e91ffaa265699139a550ce3190ada5ed5479e64eee7" + }, + { + "pair_index": 66, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-structured_instructional-26-unwatermarked", + "source_text_hash": "4efb4170d38a7c8d0b0aba17f018f1ec6deb003b21e4267d02b0d380c0d83395", + "transformed_text_hash": "2b195d06ba3cb0430145970d0fe2df00d26b636ab170cc063ab9098db6a32e44", + "variant_hash": "b9518e71c504d3445f1461deaca2ed121ecf8e0c2f991ce7d0f7ae2417f9028d" + }, + { + "pair_index": 67, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-structured_instructional-26-watermarked", + "source_text_hash": "28bda76ec6e3b385896853cf26d0bd038f9b4fd1448990da55dcb3c795b10ddb", + "transformed_text_hash": "f34c47a3a8f747ab6d5b8b0c8df6b6edbe043f14802ce1c546790f666d18efee", + "variant_hash": "74bc2667e7f1e6f12af4a6c93df28dcfe7abf2b9b45b5734d1f26f871a325a17" + }, + { + "pair_index": 68, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-structured_instructional-27-unwatermarked", + "source_text_hash": "97ab427fd1c2a9c4a7b0b4f578528e95b26f914838ace8fa1ca397f2e2ce0011", + "transformed_text_hash": "5397a002107d780a168dedbde9eb4f25dc5f348f78510c1cb45cec8c61393f8f", + "variant_hash": "c574dae9cad08d58df7bf0447b28dcf187d15ec7e507b189d716ffa8193b10ff" + }, + { + "pair_index": 69, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-structured_instructional-27-watermarked", + "source_text_hash": "1be7efc24e0caf1b1fb4d64b3426ec98b5a29ced2570568d60c78f22357eb9b1", + "transformed_text_hash": "b8297d63941cde67ffa3c72363f1d900a6ec3e2ee6736b0878b334bf38b434d3", + "variant_hash": "4d4c1b67698fa63cbd4815eba8bff9e6e74f683f48a43132c8f465928eb61d4a" + }, + { + "pair_index": 70, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-structured_instructional-28-unwatermarked", + "source_text_hash": "ec7e8b2891ea5a91b2d8b93208f815ea35ac1e134b7f14e66b81857f897133cc", + "transformed_text_hash": "2f8ad2074f959c76f29d46e006fd65bdc9b98b56805c80ae27b76ce4d04e3904", + "variant_hash": "a337da48e9edebd0e704cdaa4937742b1bf559268de0014da46efb94b1d10b6e" + }, + { + "pair_index": 71, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-structured_instructional-28-watermarked", + "source_text_hash": "f8ab3216d6a5fc3973aa44cd1f7dbd1496453e12cc44ab9a711d037dfcdda2f0", + "transformed_text_hash": "cc9fd1b35102fe884339bcb8bd962b56aa5066138c8499e5f45d16f8b236a8b6", + "variant_hash": "c2024adad04cf780ca99b1d82133e5daf30e280cfc41b97891d32b2a47d52956" + }, + { + "pair_index": 72, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-structured_instructional-29-unwatermarked", + "source_text_hash": "b57b98a14c7fbf36bef83c9df141069355982d7bb94502fcefec7702c6c61538", + "transformed_text_hash": "8ee7eeaa3e68582bff11d7d76edd7b6354d3dff7fdfc5ea74f7464ad560edf0a", + "variant_hash": "289e59beafb5ae374633a5bb185890f9cefd9d3b64c7d7c891f333e2fe855bf3" + }, + { + "pair_index": 73, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-structured_instructional-29-watermarked", + "source_text_hash": "12c51d46529466ebb0a2420afb0aa277eddab8f323e4420066d0f30ca4616185", + "transformed_text_hash": "e6e22770d7d62c9fce1afdd40936fb07dd457dcd0c89ce01395e4dbb9f7cebd3", + "variant_hash": "fb9a511fe33d9049b92cbffd4b85ef7135069f1f7bcd8cbd3fe98e07d7a6e5e3" + }, + { + "pair_index": 74, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-structured_instructional-30-unwatermarked", + "source_text_hash": "b1eca7f8c9b995751ccb280ab6ea6365f6d6d80bfd21684a511fc7759d42e4b2", + "transformed_text_hash": "88252014a95fa840d3d2dc85eefa6ebb93c5dc8637c07316b84fe7e2829cf869", + "variant_hash": "7d52382dc5621551b86cd4d76f3d955fa384a72832e152d5fab9a36f7a5348a0" + }, + { + "pair_index": 75, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-structured_instructional-30-watermarked", + "source_text_hash": "7943ff3211ce0255b57fcada8a7d80e3954a891b42998ca163a07a086e676eba", + "transformed_text_hash": "1cd6a59b4b6dca583da2397e7ea0322ef961058d7800e66637466f7d503c5ae9", + "variant_hash": "c293cb3d9243249f8374a6842cbb3bfc6efe996bf476008f5c832ebe8089c2c8" + }, + { + "pair_index": 76, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-structured_instructional-31-unwatermarked", + "source_text_hash": "b13fef25d155ae159a1957e0877d3de50129a7d2c3699d2ddd1a7b84cac67c7c", + "transformed_text_hash": "4c94e399423997263863e5f9b935ae5a02627dc7bbcf55aca4ac0b841cd982d7", + "variant_hash": "831b48dd7ba0c4e9f4c1058bf2da69b09b21ef8311e81f2ea721430e89e46cf5" + }, + { + "pair_index": 77, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-structured_instructional-31-watermarked", + "source_text_hash": "599477f09b19c1c0c01d8022013f5a0bb9bdf0ea4095a78d3de0be476c1eb591", + "transformed_text_hash": "8a0dcac7ba78eca5333732e2d09f03b3d85099be3d239335d763d383c6636a9f", + "variant_hash": "b648543b2aa780ffe2da3ea0f9a4126cf168c3e85b817af4a76a174b35014d17" + }, + { + "pair_index": 78, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-structured_instructional-32-unwatermarked", + "source_text_hash": "a216b9ae97e02f6dc4c52b1cbb4afc7ca6f445c38d8bc0c49f72af7351ff50e8", + "transformed_text_hash": "d7b23bf24f464fbcc5121758712f47149651688969c5128f9f638f7090f5a1e0", + "variant_hash": "609331274f1dc1d9abf8e5bfce75587531cf223f9c131e1e44c9989dc06b8551" + }, + { + "pair_index": 79, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-structured_instructional-32-watermarked", + "source_text_hash": "535192d753ecb53f4aa4ce28ff9589ccb0ee271f1ffef76c24d6e6c7d4635f29", + "transformed_text_hash": "b071dff4be47bcad25ebdfb8b5bab2f40ff4f1a85e1ae37ae6ed0414d575deeb", + "variant_hash": "a05a0819cc0e2fe2d4e4ac29db2b0dfc433cf92ab693aa7a52eac96b60019dbb" + }, + { + "pair_index": 80, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-structured_instructional-33-unwatermarked", + "source_text_hash": "41bac0e0e10b3b70ac7044cd2bd06b3ec638cb9469f1ecd7f9c985ff9060d748", + "transformed_text_hash": "c4342e1245d0c9f85fdc505b6fb29f360c7c52718cf5ed01ad46f46a4b56cf1a", + "variant_hash": "6d8dd46c2c7b8c4bed606142e3cc48dc75ab75e9038c08a105c17725cef616ea" + }, + { + "pair_index": 81, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-structured_instructional-33-watermarked", + "source_text_hash": "8f4d92c4f2d69e444775fc921d1088fe519fff017ee6bc55b39a0fa8e4b36ef3", + "transformed_text_hash": "59dae99693e5310c7e885a08576daa2683f5f96d9bac50c3016cdb32fea10d3a", + "variant_hash": "b1a1787c6f93f55b5dd167a86920f660f2142c28248af13401dc369810a750b2" + }, + { + "pair_index": 82, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-structured_instructional-34-unwatermarked", + "source_text_hash": "d73c7437a3670cb5a2a5027681cbec9f105af66233f025ab8d757b3c13b6ec4e", + "transformed_text_hash": "c8f1d0e6b46a8f9d0a8eb46cc0ac35abf6d7657043e92596bffe437a317ab30a", + "variant_hash": "585a4ec39c03123d065ea6662c37e93ebab3877807e4c0b440da8cd21765e210" + }, + { + "pair_index": 83, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-structured_instructional-34-watermarked", + "source_text_hash": "f260b4814fb7944dc7d68aa88290ce8a0f5669af4b07adf85ee0e0c27a2957e7", + "transformed_text_hash": "2d980c23ae252bc2b7003e70e4bc8d2542bce20adaa385bac509fd8ff914617a", + "variant_hash": "9cd1180c34ede01a4800993231ae7574369870fdf82c594e6b6c80c1ad61a73b" + }, + { + "pair_index": 84, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-structured_instructional-35-unwatermarked", + "source_text_hash": "66fc607dde40bc3279c1188a5f918afb7805d47ceff11ad49f02d4480c887da2", + "transformed_text_hash": "aa5f467276273194989745b298d195ab45ed4bea26b681dc1e96cea489182a45", + "variant_hash": "fb5f7b4cb5ca88b0800883856100957ff374bac34db0db257e58c0564df6db13" + }, + { + "pair_index": 85, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-structured_instructional-35-watermarked", + "source_text_hash": "6ce7960a22fffef0b10c622c7df3c4733d005e81047a4ca592adbd8d44dd79bc", + "transformed_text_hash": "8d3e8b257a1252c69a7470b49f1030004361445a3e4443e00945b03e1e248251", + "variant_hash": "6f608f138b59772821880aafaeb09036491f17e8892e62c1437171b08bbaddff" + }, + { + "pair_index": 86, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-structured_instructional-36-unwatermarked", + "source_text_hash": "b9fae73db16fdb07bd7d09c8cd7d1453fe6718a375749b7d6e13f9320511e8ac", + "transformed_text_hash": "88dda196e468dab97fecae1d7850418d9f764a41e5d4c0ad7f1a91ee600ac3da", + "variant_hash": "be549314a745443ea23a13a23cf7a02f5d17e7780d5015c24ac000744d377636" + }, + { + "pair_index": 87, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-structured_instructional-36-watermarked", + "source_text_hash": "6937c1fc3e5eabaf18ceceb43b016e09d2d25120448a96c4db6ccbd47b411ba5", + "transformed_text_hash": "6937c1fc3e5eabaf18ceceb43b016e09d2d25120448a96c4db6ccbd47b411ba5", + "variant_hash": "e9c4c6e5cee1eaa1ae1c345a701030e56c2e611ed5676713d0085990886e0b08" + }, + { + "pair_index": 88, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-structured_instructional-37-unwatermarked", + "source_text_hash": "d971912c4e81ad68c7660fc6a33227b9ebf2e04ccb877df22ca2470edac8c5c9", + "transformed_text_hash": "e7e1a9ba890702e3f482543ccd51244d2401618eca3ebc4c800c6e757ce69a29", + "variant_hash": "041b1e76b08a22151466f16457b0524c9574c251f4143b7cc763a0370e2d2218" + }, + { + "pair_index": 89, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-structured_instructional-37-watermarked", + "source_text_hash": "81e85ca4adb64ff68d692f9617155415217c1c14b87e0918b579d741ff16e691", + "transformed_text_hash": "80ad71d5e0b77b765e012cbc5df0ce91c8367a1b7bbc23000cf652c3786b16a4", + "variant_hash": "2853fc7a000ce089795b26d5953a9b041492154760b650b269cf13b2e4531efb" + }, + { + "pair_index": 90, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-structured_instructional-38-unwatermarked", + "source_text_hash": "af61bfaa7335bde2b4eb156421d8fa7263ba9b042f91f35b0d64d64084ec89fe", + "transformed_text_hash": "ddccf6482ec4a8d48b1870130a0429ccf2e3f7e51b729bc4cdabb2756e0fbc56", + "variant_hash": "a64666bb99a293af8649d8d2ea1e3eaec82951576156683e4eee13b49779552c" + }, + { + "pair_index": 91, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-structured_instructional-38-watermarked", + "source_text_hash": "d1cb44f88b79e088131054890b7b05cb53bbd7c92d4df6efea28f205ad03abc1", + "transformed_text_hash": "6bca44635dc944364c27708ad36be643c68407c8c0cabf787f85de74749d7614", + "variant_hash": "2d1519a50547889603a0552179be0ecfad676964c41b0ab1806451f53282132d" + }, + { + "pair_index": 92, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-structured_instructional-39-unwatermarked", + "source_text_hash": "34eef3b23ba205782c5828edc4c645d0276cde52e40a175894edd0f50223791e", + "transformed_text_hash": "b18ff457a5ef82be912e1d96fbb7d2393e44ef7d5b3a005195a6fef357462a9f", + "variant_hash": "258f0b091e09980b23ab864d80a6e2c801d02a7daff74c833443f835801bf133" + }, + { + "pair_index": 93, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-structured_instructional-39-watermarked", + "source_text_hash": "92be15a5321024b93ad2fb0cef2f5bbf29f9b485a6382e0341459bed04b40664", + "transformed_text_hash": "8a9c0b478abebd27403dd97a36b0bc54d9709b2cdac4f1a546e948258d61ac30", + "variant_hash": "d9c84e6c4a64cbdf44c4ebd826bf983937d10b246accf23f1ef025b531f36300" + }, + { + "pair_index": 94, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-structured_instructional-40-unwatermarked", + "source_text_hash": "7d769536562ef15be541999941cb806911f884f36da1b7b759eafceb82723b2d", + "transformed_text_hash": "513f65a1969b891e0cba62ddbaf3079332a2fc27653ce8955886da94ab594fe9", + "variant_hash": "cdfc4fa923862c2cb448a70bec44eb3fab2897983fba3c02f2dcfe1312a2feb3" + }, + { + "pair_index": 95, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-structured_instructional-40-watermarked", + "source_text_hash": "44ae6a7ec136ae7e66e55bd2ba0721862ce70ad1862dcd773875ad601fbe9ffc", + "transformed_text_hash": "a86f84dec6c0149e7384aff7569e2eeefdbc40bff8c99fd8db4d3374189e20d7", + "variant_hash": "cf723d83e28180d60454c0e1f44b89405e19f06a6a95e126fb5312e3f3274d77" + }, + { + "pair_index": 96, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-technical_explanation-25-unwatermarked", + "source_text_hash": "f464fe5dd77e49d97663110b42028017254ddff2bf703aa830c70a491d1afe1d", + "transformed_text_hash": "c834109fabed0d78ee188336a2274eb9fb3f10fd3cde794468fbfacb390d7f5f", + "variant_hash": "7079195b32f233204c6c0cf0718bc8f64810b75e772fc3e34496dbba6c4991f3" + }, + { + "pair_index": 97, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-technical_explanation-25-watermarked", + "source_text_hash": "b92ab33eda4c23ad09d46982047a047ac7519db101054bccdd8692a66f873a80", + "transformed_text_hash": "ceab1fefb5abb2e8f13ff08a0182b172a4b0900fd304a3919aa6f14c8a6f2a27", + "variant_hash": "b21bf72e5e534a9e0a45483a717a3d6ce36b767d6f2403e5e7b24fc7858ef395" + }, + { + "pair_index": 98, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-technical_explanation-26-unwatermarked", + "source_text_hash": "638b518cb906138ae63dbb6f054db29361d4b453f09f3ed0cb94f54eaea2ea7b", + "transformed_text_hash": "936c59e657595a50caef360777a611f0a156eb60d39f9e83f8ae847f1fc92317", + "variant_hash": "9485089156ff14456c831902be3cabb24edafa8c69a173392fd5b7dea1423019" + }, + { + "pair_index": 99, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-technical_explanation-26-watermarked", + "source_text_hash": "bdc4461bce35c07a1927dbf598b5809e2d50071bde32b9ac8ae5c18e6b32519b", + "transformed_text_hash": "f6329d37338f0c3e49e54a834ec6411b69ea6069d588e51eefd6801ceceb924a", + "variant_hash": "71d51c7fa965207398e419319835981fe19b62ded1b9b2ae746c141fedc9cd20" + }, + { + "pair_index": 100, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-technical_explanation-27-unwatermarked", + "source_text_hash": "ef6b15b678fc3448315915f6296fd4e0d0e58d2413bf637015a11e23c2282354", + "transformed_text_hash": "bad14c2f83880468f4326f972cfc26818fed495a85c4cf591bdf890496c43704", + "variant_hash": "203e1e06eb9fc71351229bc5ad25948906704cdcff9acb405ff05ce082bce296" + }, + { + "pair_index": 101, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-technical_explanation-27-watermarked", + "source_text_hash": "a6ba58ebe6cd82e56ec0a2b20ef0cff8204f8401ef697b989b08132bf34314ce", + "transformed_text_hash": "3b899135d9c844e876de686a3acab79c10ce2be9e851219affd491a1e4e1ce16", + "variant_hash": "78e2fb373786cf778ed94664926b9e147273440ec31426551cf3602831975cd0" + }, + { + "pair_index": 102, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-technical_explanation-28-unwatermarked", + "source_text_hash": "c9e67b9c40119311fbb5b562ad9afb0bd43252edd286705b9241fafefff2389d", + "transformed_text_hash": "445f9ae528fca119e4e36944631999110fe1944f4ac92367d65b6f54937f33ec", + "variant_hash": "8ede6ccaa5eebe866e1add38abd37984406457a0f4cb28bb1b5e6645143d9178" + }, + { + "pair_index": 103, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-technical_explanation-28-watermarked", + "source_text_hash": "4001ba60d8426872b3e703ab1c6128133564b26f60de106f5680a0f90ca6059f", + "transformed_text_hash": "a394992f13eaf7b42e956e093df019fe448d244a7cc7e5a3aa55aec8af86428f", + "variant_hash": "101edf0542389a3632279541dd410e5ff11d87bcb4d1f40c00af9a72856499b0" + }, + { + "pair_index": 104, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-technical_explanation-29-unwatermarked", + "source_text_hash": "af70907cafd4785f441751a7bf4a7f28d120225e13183426715167a8124aacf1", + "transformed_text_hash": "2b658e6a925ae3d136eaf7548a1bcc85b4f1a0d5ca577480800d5afd64c301ad", + "variant_hash": "5aa0130cbc61d32d3705d675f8ad3a9d69c492bd69f3d692c567e9bb54ca5bf8" + }, + { + "pair_index": 105, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-technical_explanation-29-watermarked", + "source_text_hash": "c7f6fbd4a3a0ff9b172c2457c58d03ed10e7685cf05dbe2bdf9e875313b64fa9", + "transformed_text_hash": "60d3de4ab47e84a6da8f5facc55089dfd8c9fe52c6dca7c7e55195547e98a260", + "variant_hash": "76d9d13a1d571e24dac6e90b1afde35bac45c34424c41d742b67f26dedd13511" + }, + { + "pair_index": 106, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-technical_explanation-30-unwatermarked", + "source_text_hash": "094c4c617e17470fcaf797ccee0bcb48948d0d2ba0e76d7cfec918dc9faf3509", + "transformed_text_hash": "15ad814dca73485a43954fb79516fb79afc6619bae65f2551f7838a3ee1d3d94", + "variant_hash": "fc2369ac1c4a922d17c0ee1f85d57eee7644c182ceeefeb1630762f33422eb4a" + }, + { + "pair_index": 107, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-technical_explanation-30-watermarked", + "source_text_hash": "d65c37c8c61d01f8cf570d56c0caaae2edae2e88c3462332d6462b49e8003f9a", + "transformed_text_hash": "bed2060dba326f55e33077cb220fb16db0e29d6d3378fa232d42e9638dd1c117", + "variant_hash": "81e080adfcfb45dd69cfc9f32ca291ac72cdd2bfc26ef2a2fd686a6354b78427" + }, + { + "pair_index": 108, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-technical_explanation-31-unwatermarked", + "source_text_hash": "f87342601a1f3b7d8883725c6d07dcd9a482b46eda7839e464fb7842561bddf6", + "transformed_text_hash": "625296697efb81d18cf4f09117f19616e3287c94e0609e0cf2fb8d00f1230af4", + "variant_hash": "4728cf5cf89c08015dc022ff4d112f1831dc7887bba208780ae10bbe7d9209eb" + }, + { + "pair_index": 109, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-technical_explanation-31-watermarked", + "source_text_hash": "005d6e251ea9cebf868c324dfe2a2c2cf7f11825579aaab0451f8e7b98bbc98b", + "transformed_text_hash": "005d6e251ea9cebf868c324dfe2a2c2cf7f11825579aaab0451f8e7b98bbc98b", + "variant_hash": "7fc1e764877aedcd81a4fc2d8f9dd188447246baf7c269f05ad3776673f68fc5" + }, + { + "pair_index": 110, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-technical_explanation-32-unwatermarked", + "source_text_hash": "00c2abda02876c7d1571e18044b3a5a43221be9a127655b7741ed82911011585", + "transformed_text_hash": "da725dbfa5490d0f037b5f2e92e61d5e407c6951ebb00b99c766e8632cbe9fa0", + "variant_hash": "0e205bc0450566c7499f2ae50f6e0a62f7a57f9b1594f3c372527e02d9f0a248" + }, + { + "pair_index": 111, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-technical_explanation-32-watermarked", + "source_text_hash": "e53e90ee0a99b0064a3e47f2e08ba23cc187d92467309b8cd05df6f8965606bc", + "transformed_text_hash": "eb04e7f9f46744eeb14402f7c5533befdb02bc2c15bb09e5ec6351561c2c10f6", + "variant_hash": "5b6242917e8d9db7e14e095124cc02417edc633c1b437b9e68bf7476670021b3" + }, + { + "pair_index": 112, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-technical_explanation-33-unwatermarked", + "source_text_hash": "0e18166a22e16d906c5da852a06f3b5e3908fa7d5fc95701b854ff7502fe4b8e", + "transformed_text_hash": "ad0add174f7f566c0b5eeb7f661a1f0e771c620aa52a115c8c64babd0a396f8c", + "variant_hash": "1b94a7a18f204935787f784f4f318f9551c1c1fbecf709756c06a3c64b8d1773" + }, + { + "pair_index": 113, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-technical_explanation-33-watermarked", + "source_text_hash": "25a761f2aea706cfd8e5381b8786f42c422724304cf893209cb44ad6a35ffd8d", + "transformed_text_hash": "3d2d5457644383a9c6e990ed5791c2ffae915cbd2c827e66be08e3b043f36745", + "variant_hash": "08ab5edf49599d169e6f70a82bb3f2296560458c36442b642f115711bb4c1eb5" + }, + { + "pair_index": 114, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-technical_explanation-34-unwatermarked", + "source_text_hash": "d5da2b31912adb1247d8d5b71530808a672a2714234d3f17f876ca40353353d9", + "transformed_text_hash": "fb86dee23e7c44227d89c0bf112abc5f2b7aff2120a5b0a14878f890eef7fc7d", + "variant_hash": "07893994f7ce917655496e315eacb7c4cc48033dad62ef0683cba01d1eea3dbe" + }, + { + "pair_index": 115, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-technical_explanation-34-watermarked", + "source_text_hash": "1db050472cb804d5bcb93a9c2cba244ffbcc1e7a3ab61e189c063ccf44e897de", + "transformed_text_hash": "cac31cf2a605e3efe5bc38c0df6ef9f34aec4651d74827839d85f4b3da14a84b", + "variant_hash": "137e708f493af406fa334ff5451c3dd4a95b29a54ae55a33eb30354cca0d8e23" + }, + { + "pair_index": 116, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-technical_explanation-35-unwatermarked", + "source_text_hash": "ea88cf9308284c3682c879505e1bc1fbbae7f53e9f6c1ccfb3302bfc971f76e0", + "transformed_text_hash": "358ca37fd9c40ce020701ed2eedbb85c980abc9c0ea21308f61e3fea5484535a", + "variant_hash": "e937a3829e7ad5d84c2453629c271a0eec617eaf50758ac7be5e3d0cd224360e" + }, + { + "pair_index": 117, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-technical_explanation-35-watermarked", + "source_text_hash": "4ca94603c26cae02a9e23dace0a4daa5879337937f9724765f8d0585a7b83835", + "transformed_text_hash": "9dd21d3b346d4e5e3755d4c700df1b56acc1ffbd816cabb47a60050681c8d036", + "variant_hash": "ac0959e59b01f8c47d6cea45020f23c0e20c6a1bc0948a2a79ada8a9691eaa4c" + }, + { + "pair_index": 118, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-technical_explanation-36-unwatermarked", + "source_text_hash": "110953fbe47df723a63c6ffd3ef36e58932d09fb78dcdd731e5d3b65fffada11", + "transformed_text_hash": "e935bfdda819854aec69abc28d5481e45e5c9a83f042e487153b36a6fb13e918", + "variant_hash": "9d2215739c2050d7ac5475d3b656b5e24cedd1a8edf79ab763caedd639cc864c" + }, + { + "pair_index": 119, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-technical_explanation-36-watermarked", + "source_text_hash": "90c59b226b947891c69066d1f1fdbc69e33fc4d3fcd43592f6293d2cdbaf6339", + "transformed_text_hash": "383d545b2547abf1ffa6f77f8bb68a711a094b6b833e564f942f0dcb96cec01b", + "variant_hash": "b45660a5e42d53c994e726847f919e5782329d74b59ea1be137a2c96d90154bc" + }, + { + "pair_index": 120, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-technical_explanation-37-unwatermarked", + "source_text_hash": "36bc0fcae3d2b596a144dbc7fca15fb6ec75a4c8e48fa83c4aaa95b90969894b", + "transformed_text_hash": "5d24b1e22028ec3057edd9383a6e251b8b28eb8a7e119d5bc2e87062ea6d3524", + "variant_hash": "c618d24cd56aa86c227fc09fa7eeca12a6728f24f854628efe2ba619386d9595" + }, + { + "pair_index": 121, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-technical_explanation-37-watermarked", + "source_text_hash": "a52f86968e94b15c8bbce9cd1b7a4cc5b8f1c6035b8a6e69fbb73c6dec2928cc", + "transformed_text_hash": "e81fbf6b76b77c30d5bcbd05f44f23318653ae876229815bd4441fe6f2356ee6", + "variant_hash": "6e40c9325747fab7f7e22a6e83331e33d7c11cda1805b37501dca63aae89ae3d" + }, + { + "pair_index": 122, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-technical_explanation-38-unwatermarked", + "source_text_hash": "7619dacb1f52101c1572eed45418ef28b31d5ec9f69335429e0d76424a7940cb", + "transformed_text_hash": "57701bb6849780319f4cd928ba4be15e06351c23c6e00c147e51a27bc34053ad", + "variant_hash": "0d5e7b500afc636ef61a6a123fc40ee38231190f5d71767af5e2c41974968005" + }, + { + "pair_index": 123, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-technical_explanation-38-watermarked", + "source_text_hash": "f0c3446335e6e0a8a18b61dcc1e5f23536f67bffb8366188babe08b6570f004a", + "transformed_text_hash": "a939179ae219bb3ecb63658742509b035001eb609f9b479a9db0e33c146e0c72", + "variant_hash": "7265f38fc927641972f017d475e41a2e05d85d627a447913cf037a4d69fbae4d" + }, + { + "pair_index": 124, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-technical_explanation-39-unwatermarked", + "source_text_hash": "0cef400b372e42544f6460d2f9c654d60d2fc21787032c936c1b63e514871817", + "transformed_text_hash": "cedd91591392eded86d4899846a6fb0f119504c2c3cecc47dc3a9bff19f24f84", + "variant_hash": "b391c6b95a6104f1b7db930e8215dbb62b6cc3cfc9cbe045d4ac5ef8e11c9329" + }, + { + "pair_index": 125, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-technical_explanation-39-watermarked", + "source_text_hash": "2ea43120ab9a29e13e6a28b2d4d425a4b4004487c8736d14e612f9844d8d30b4", + "transformed_text_hash": "d1e744f85b98b70c2b61b7b1b877f05926989fc06c083d6907093fc006917b71", + "variant_hash": "84d3b8bfc6abb22b000c4d0aea31a5c300662c5913103b8d4e6cee2024856831" + }, + { + "pair_index": 126, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-technical_explanation-40-unwatermarked", + "source_text_hash": "8bf20401901d858b677f85fe7cda63ae176833821aaf2293f7af849019104058", + "transformed_text_hash": "d3e26a7c91922b0cc0f78e88551d2bf65026e39d72bbd4bf1527f9f61343f5ba", + "variant_hash": "ef0a18812a4c9e7c8df488057ca29dc175282acffe75c66c8e88d6feb902f91c" + }, + { + "pair_index": 127, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-technical_explanation-40-watermarked", + "source_text_hash": "b3d351fbb6710b347298cfe6848d1aa16861865e4577a6f05de46e92c825515d", + "transformed_text_hash": "1bf4b9649fc65c6b4a558d408b02486ef7fd2a7a70428321028cb22f538ad4ff", + "variant_hash": "671bf1aab76e4d27b4069f2d19c9dc425c13b022eb311d84398ac2bea0181da7" + } + ], + "plan_hash": "0cce41214c3e55ee88cb1ca665d0ebf6c6376bea0d0113a3f4ea481049a5199d" +} diff --git a/specs/fuckmark-budget-scaled-coverage-effectiveness-v1.freeze.json b/specs/fuckmark-budget-scaled-coverage-effectiveness-v1.freeze.json index a4f6af56..f512a898 100644 --- a/specs/fuckmark-budget-scaled-coverage-effectiveness-v1.freeze.json +++ b/specs/fuckmark-budget-scaled-coverage-effectiveness-v1.freeze.json @@ -85,8 +85,15 @@ "finding": "the audit gated on absolute presence of category Cc characters in transformed text, which flags ordinary newlines (U+000A) that the GPT-2 source generation already contains; 28 of 32 development rows failed a gate that measures the corpus, not the candidate", "reason": "codepoint audit scope error found on development robustness evidence before confirmation scoring", "recorded_at_utc": "2026-08-23T00:24:31.440654+00:00" + }, + { + "amendment_id": "pre-score-amendment-5", + "change": "normalization gates now bind to introduced instability only: for each normalization form, source stability implies transformed stability; pre-existing source instability is reported without gating", + "finding": "2 of 128 confirmation robustness rows failed NFD/NFKD no-op checks because the GPT-2 source text itself contains precomposed accented characters that decompose under NFD; the candidate introduced only ASCII characters and cannot have caused the instability", + "reason": "normalization gate inherited the same absolute-text scope error corrected for codepoints in amendment 4", + "recorded_at_utc": "2026-08-23T00:49:12.978531+00:00" } ], "protocol_id": "budget-scaled-coverage-effectiveness-v1", - "protocol_sha256": "e9ed870944a32ad7bcfe38993d1fdd6287618a04d638192734b82bcfaf9de833" + "protocol_sha256": "4fa68c7519a2064c2bf131393598c5b181b92f2ae5fd62f9fd1959b7ebea0367" } From aefcbee0cd43cef415d9caf279f03074c86aee39 Mon Sep 17 00:00:00 2001 From: Cyinh Date: Sat, 22 Aug 2026 20:58:03 -0400 Subject: [PATCH 09/30] Gate normalization stability on candidate-introduced instability only --- fuckmark/tiny_dev_budget_scaled_robustness.py | 24 ++++++++++++++++-- ...aled-coverage-effectiveness-v1.protocol.md | 10 +++++--- tests/test_budget_scaled_robustness.py | 25 ++++++++++++++++++- 3 files changed, 53 insertions(+), 6 deletions(-) diff --git a/fuckmark/tiny_dev_budget_scaled_robustness.py b/fuckmark/tiny_dev_budget_scaled_robustness.py index 7730e142..1e8f19cc 100644 --- a/fuckmark/tiny_dev_budget_scaled_robustness.py +++ b/fuckmark/tiny_dev_budget_scaled_robustness.py @@ -60,6 +60,17 @@ def _robustness_row(source_text: str, transformed_text: str) -> dict[str, object "nfkc": unicodedata.normalize("NFKC", transformed_text) == transformed_text, "nfkd": unicodedata.normalize("NFKD", transformed_text) == transformed_text, } + source_normalization_noops = { + "nfc": unicodedata.normalize("NFC", source_text) == source_text, + "nfd": unicodedata.normalize("NFD", source_text) == source_text, + "nfkc": unicodedata.normalize("NFKC", source_text) == source_text, + "nfkd": unicodedata.normalize("NFKD", source_text) == source_text, + } + introduced_instability = tuple( + form + for form in normalization_noops + if source_normalization_noops[form] and not normalization_noops[form] + ) invariant_status = validate_hard_invariants(source_text, transformed_text).status row = { "source_text_hash": sha256_text(source_text), @@ -69,6 +80,8 @@ def _robustness_row(source_text: str, transformed_text: str) -> dict[str, object "introduced_forbidden_codepoints": _introduced_forbidden_codepoints(source_text, transformed_text), "introduced_non_ascii_codepoints": _introduced_non_ascii(source_text, transformed_text), "normalization_noops": normalization_noops, + "source_normalization_noops": source_normalization_noops, + "introduced_normalization_instability": introduced_instability, "cf_strip_noop": _cf_stripped(transformed_text) == transformed_text, "word_edit_rate": word_edit_rate(source_text, transformed_text), "protected_span_violation_count": protected_span_violation_count(source_text, transformed_text), @@ -77,7 +90,7 @@ def _robustness_row(source_text: str, transformed_text: str) -> dict[str, object gates_pass = ( not row["introduced_forbidden_codepoints"] and not row["introduced_non_ascii_codepoints"] - and all(normalization_noops.values()) + and not row["introduced_normalization_instability"] and row["cf_strip_noop"] and row["word_edit_rate"] <= WORD_EDIT_RATE_MAX and row["protected_span_violation_count"] == 0 @@ -127,7 +140,14 @@ def build_budget_scaled_robustness_report( not row["introduced_forbidden_codepoints"] and not row["introduced_non_ascii_codepoints"] for row in row_tuple ), - "all_normalization_noops": all(all(row["normalization_noops"].values()) for row in row_tuple), + "all_normalization_stability_preserved": all( + not row["introduced_normalization_instability"] for row in row_tuple + ), + "rows_with_preexisting_normalization_instability": sum( + 1 + for row in row_tuple + if not all(row["normalization_noops"].values()) + ), "all_cf_strip_noop": all(row["cf_strip_noop"] for row in row_tuple), "max_word_edit_rate": max((row["word_edit_rate"] for row in row_tuple), default=0.0), "total_protected_span_violations": sum( diff --git a/specs/fuckmark-budget-scaled-coverage-effectiveness-v1.protocol.md b/specs/fuckmark-budget-scaled-coverage-effectiveness-v1.protocol.md index 3df810bf..59ee92fd 100644 --- a/specs/fuckmark-budget-scaled-coverage-effectiveness-v1.protocol.md +++ b/specs/fuckmark-budget-scaled-coverage-effectiveness-v1.protocol.md @@ -81,9 +81,13 @@ Gates evaluated on every plan row, development and confirmation: 5. Codepoint audit: the transformation must introduce no Unicode Cf, Cc, Mn, Mc, Me, Zl, Zp, or Zs characters other than U+0020. Pre-existing forbidden-category characters in the source are reported separately and do not gate the candidate. -6. Normalization no-op: NFC, NFD, NFKC, and NFKD of every transformed text must equal the - transformed text itself; Unicode Cf stripping must be the identity function on every - transformed text. Any non-identity result is an adverse finding and a Level B failure. +6. Normalization stability: the transformation must not introduce normalization instability; + for each of NFC, NFD, NFKC, NFKD, if the source text is stable under the form then the + transformed text must be stable under it too. Sources that are themselves normalization + unstable (for example precomposed accented characters under NFD) are corpus properties, + reported without gating the candidate. Unicode Cf stripping must not alter any + candidate-introduced character; candidates introduce ASCII only, so this holds by + construction and is verified per row. 7. Copy/DOM: transformed text must contain no control or format characters, so DOM textContent equals the transformed string; verified by the codepoint audit and archived fixed-browser renders. diff --git a/tests/test_budget_scaled_robustness.py b/tests/test_budget_scaled_robustness.py index dfe8aed7..8d6b3675 100644 --- a/tests/test_budget_scaled_robustness.py +++ b/tests/test_budget_scaled_robustness.py @@ -76,6 +76,7 @@ def test_introduced_codepoint_gate_ignores_preexisting_source_characters() -> No from fuckmark.tiny_dev_budget_scaled_robustness import ( _introduced_forbidden_codepoints, _introduced_non_ascii, + _robustness_row, ) source = "source with\nnewline and " + chr(0x2019) + "curly quote" @@ -88,6 +89,28 @@ def test_introduced_codepoint_gate_ignores_preexisting_source_characters() -> No assert _introduced_non_ascii(source, transformed_visible_unicode) == ("U+00E9",) +def test_normalization_gate_ignores_source_instability_but_flags_introduced_instability() -> None: + from fuckmark.tiny_dev_budget_scaled_robustness import _robustness_row + + stable_source = "plain stable ascii source text" + stable_row = _robustness_row(stable_source, stable_source + " edited") + assert stable_row["gates_pass"] is True + assert stable_row["introduced_normalization_instability"] == () + decomposed_source = "caf" + chr(0x0065) + chr(0x0301) + " source text with several more words here" + inherited_decomposed = _robustness_row(decomposed_source, decomposed_source + " edited") + assert inherited_decomposed["normalization_noops"]["nfc"] is False + assert inherited_decomposed["introduced_normalization_instability"] == () + assert inherited_decomposed["gates_pass"] is True + introduced_unstable = _robustness_row(stable_source, stable_source + " caf" + chr(0x0065) + chr(0x0301)) + assert introduced_unstable["introduced_normalization_instability"] == ("nfc", "nfkc") + assert introduced_unstable["gates_pass"] is False + unstable_source = "caf" + chr(0x00E9) + " source text with several more words here" + inherited_row = _robustness_row(unstable_source, unstable_source + " edited") + assert inherited_row["normalization_noops"]["nfd"] is False + assert inherited_row["introduced_normalization_instability"] == () + assert inherited_row["gates_pass"] is True + + def test_robustness_report_gates_pass_on_rule_transforms() -> None: text = "You are not ready, and we do not stop when the system is in use." corpus = _corpus(text, "1") @@ -100,7 +123,7 @@ def test_robustness_report_gates_pass_on_rule_transforms() -> None: report = build_budget_scaled_robustness_report((corpus,), (plan,)) assert report["summary"]["row_count"] == 2 assert report["summary"]["gate_pass_fraction"] == 1.0 - assert report["summary"]["all_normalization_noops"] is True + assert report["summary"]["all_normalization_stability_preserved"] is True assert report["summary"]["all_cf_strip_noop"] is True assert report["summary"]["all_introduced_codepoints_clean"] is True assert report["summary"]["corpus_pairwise_text_hash_overlaps"] == {} From b83418022940efd423ad41160daebfe825c888f4 Mon Sep 17 00:00:00 2001 From: Cyinh Date: Sat, 22 Aug 2026 21:24:33 -0400 Subject: [PATCH 10/30] Freeze coverage-completion v2 protocol and add coverage diagnosis tool --- ...uckmark-coverage-completion-v2.protocol.md | 85 ++++++++++ tools/budget_scaled_coverage_diagnosis.py | 157 ++++++++++++++++++ 2 files changed, 242 insertions(+) create mode 100644 specs/fuckmark-coverage-completion-v2.protocol.md create mode 100644 tools/budget_scaled_coverage_diagnosis.py diff --git a/specs/fuckmark-coverage-completion-v2.protocol.md b/specs/fuckmark-coverage-completion-v2.protocol.md new file mode 100644 index 00000000..142c1cd0 --- /dev/null +++ b/specs/fuckmark-coverage-completion-v2.protocol.md @@ -0,0 +1,85 @@ +# Frozen Research Protocol: Key-Blind Coverage Completion (v2) + +Protocol ID: `coverage-completion-v2` +Cycle 2 of the budget-scaled effectiveness line. Frozen before any cycle-2 scoring. Hash-binding +freeze record: `specs/fuckmark-coverage-completion-v2.freeze.json`. + +## 1. Objective and claim boundary + +Level A: for the same fixed open detector and uncleaned text-only input path as cycle 1, test +whether a coverage-completion ruleset - the cycle-1 development ruleset plus an extended +surface spacing word list chosen from standard general-English function words - lowers +watermarked held-out detection on a new source-disjoint 64-source confirmation corpus relative +to cycle 1's 11/64 residual, with stretch target 0/64. Mechanistic endpoint: mean achievable +detector-n-gram coverage of the confirmation watermarked sources must be at least 0.80 (cycle 1 +measured 0.819 overall with 0.682 among residual sources). The claim boundary is identical to +cycle 1: detector-specific detection reduction under the documented fixed condition only; no +removal, undetectability, unknown-key, proprietary-detector, cleaning-durability, or release +claims. Level B is not pursued; the candidate edits visible text. + +## 2. Candidate mechanism + +Profile `key-blind-coverage-completion-v2`: the cycle-1 registry (forward contractions, surface +spacing, lexical, syntax) EXTENDED by 65 additional surface spacing words, frozen below; +COVERAGE_GREEDY_KEY_BLIND scheduling over public-tokenizer n-gram disruption geometry +(ngram_len 5); replicate count 1; schedule seed base 1_140_000 plus sorted source index; +budget fixed at 16 operations (cycle 1 demonstrated coverage saturation below 16 on every +source, so a budget sweep carries no information; this is pre-registered, not tuned). +Selection accesses only public tokenizer geometry and the public detector n-gram length. + +Frozen extension words (65, lowercase alphabetic, length >= 2, none already in the cycle-1 +surface list, chosen from standard general-English function-word rankings, not mined from any +project corpus): + +an, he, she, his, her, its, our, their, them, him, us, me, my, your, if, when, where, which, +who, what, how, why, because, so, than, then, there, here, all, any, each, more, most, some, +such, only, very, just, also, into, about, over, under, after, before, between, during, +through, without, while, do, does, been, being, would, could, should, may, might, must, were, +one, even, still, often + +## 3. Data splits and source disjointness + +- Development: fresh corpus at seed base 430000 (v3, 8 attack pairs per domain, 32 watermarked + attack sources) plus the cycle-1 development corpus at seed base 410000 (repeat use, + labeled; overfitting caveat noted) and the locally regenerated 401000 corpus. No cycle-2 + scoring may read the cycle-1 confirmation corpus at 420000. +- Confirmation: fresh corpus at seed base 440000 (v3, 16 attack pairs per domain, 64 + watermarked attack sources plus 64 unwatermarked controls), generated after this protocol is + frozen and scored exactly once. +- Disjointness: pairwise text-hash overlap zero across 410000, 430000, and 440000 corpora; + degenerate-generation collisions (cycle 1 recorded one against 401000) are checked and + reported; any collision is a reportable deviation. + +## 4. Generation and detector configuration + +Identical to cycle 1 (pinned GPT-2 revision, CPU, temperature 0.8, top-k 50, top-p 0.95, +64 tokens, 16 attempts; HF SynthID weighted mean, ngram_len 5, public development keys, +per-corpus text-only calibration at target FPR 0.01, decision score >= threshold). + +## 5. Controls, gates, robustness + +All four paired controls scored. All cycle-1 section-6 gates apply unchanged (hard invariants, +protected spans zero, word edit rate <= 0.30, no introduced non-ASCII or hidden-category +codepoints, no introduced normalization instability, Cf-strip identity on introduced +characters, fixed-browser DOM text equality with archived screenshot). + +## 6. Pre-registered endpoints and success criteria + +Primary: confirmation transformed-watermarked detected count, target 0/64. +Improvement criterion (secondary, must hold for the cycle to count as progress): detected +count strictly below 11 (cycle 1 residual on an equally constructed corpus). +Mechanistic criterion: mean achievable coverage >= 0.80 on confirmation watermarked sources. +Control criteria: pristine watermarked >= 60/64 detected; transformed unwatermarked == 0/64; +all fidelity gates pass on 100% of rows. +Adverse results (residual detection, any gate failure, any cross-corpus text collision, +pristine-unwatermarked exceedances, zero-candidate sources) are reportable, never hidden. +The frozen budget (16) is recorded in the freeze-record appendix before confirmation scoring; +no post-hoc change. + +## 7. Reproducibility + +Canonical JSON artifacts with SHA-256 hashes; plan binds corpus, manifest, code commit, and +explicit detector_access_observed=False / secret_access_observed=False; evidence binds plan, +profile, ruleset, detector identity, calibration bundle, threshold; robustness report binds +plans and corpora; rendering manifest binds the plan; independent hash cross-check at the end; +full test suite, `git diff --check`, and clean status; local commits only, no pushes. diff --git a/tools/budget_scaled_coverage_diagnosis.py b/tools/budget_scaled_coverage_diagnosis.py new file mode 100644 index 00000000..e73dd5f2 --- /dev/null +++ b/tools/budget_scaled_coverage_diagnosis.py @@ -0,0 +1,157 @@ +from __future__ import annotations + +import argparse +import json +from pathlib import Path + +from fuckmark.corpus import load_tiny_dev_corpus_by_version_json +from fuckmark.experiments.effectiveness_plan import _attack_samples, _encode_with_offsets +from fuckmark.hashing import sha256_json +from fuckmark.transforms import ( + SchedulePolicy, + CandidateScheduler, + KeyBlindScheduleInput, + ScheduleGeometryMode, + build_candidate_tokenizer_geometry, + key_blind_high_coverage_transform_registry, +) +from fuckmark.coverage import union_size, merge_intervals + + +DIAGNOSIS_VERSION = "budget-scaled-coverage-diagnosis-v1" + + +def diagnose_sources(corpus, tokenizer, plan, evidence) -> dict[str, object]: + registry = key_blind_high_coverage_transform_registry() + scheduler = CandidateScheduler() + evidence_rows = { + row["variant_hash"]: row for row in evidence["rows"] + } + rows = [] + for variant in plan["variants"]: + if variant["requested_budget"] != plan["budgets"][0]: + continue + row_evidence = evidence_rows[variant["variant_hash"]] + source = next( + sample + for sample in corpus.manifest.samples + if sample.sample_id == variant["source_sample_id"] + ) + token_ids, offsets = _encode_with_offsets(tokenizer, source) + enumeration = registry.enumerate(source.text) + geometry = build_candidate_tokenizer_geometry( + source.text, + enumeration, + token_ids, + offsets, + tokenizer_identity_hash=source.model.identity_hash, + ngram_len=5, + ) + coverage = geometry.coverage_mapping() + observation_count = max(0, len(token_ids) - 5 + 1) + achievable = union_size( + interval + for intervals in coverage.values() + for interval in intervals + ) if coverage else 0 + selected_ids = set(variant["selected_candidate_ids"]) + realized = union_size( + interval + for candidate_id, intervals in coverage.items() + if candidate_id in selected_ids + for interval in intervals + ) if coverage else 0 + hole_intervals = [] + covered = merge_intervals( + interval + for intervals in coverage.values() + for interval in intervals + ) if coverage else () + cursor = 0 + for interval in covered: + start, end = interval.start, interval.end_exclusive + if start > cursor: + hole_intervals.append((cursor, start)) + cursor = max(cursor, end) + if cursor < observation_count: + hole_intervals.append((cursor, observation_count)) + rows.append( + { + "source_sample_id": variant["source_sample_id"], + "source_label": variant["source_label"], + "domain": variant["domain"], + "observation_count": observation_count, + "candidate_count": variant["candidate_count"], + "realized_edit_cost": variant["realized_edit_cost"], + "achievable_coverage_fraction": achievable / observation_count if observation_count else 0.0, + "realized_coverage_fraction": realized / observation_count if observation_count else 0.0, + "hole_count": len(hole_intervals), + "hole_token_total": sum(end - start for start, end in hole_intervals), + "pristine_score": row_evidence["pristine_score"], + "transformed_score": row_evidence["transformed_score"], + "transformed_detected": row_evidence["transformed_detected"], + "score_drop": row_evidence["score_drop"], + } + ) + watermarked = [row for row in rows if row["source_label"] == "watermarked"] + detected = [row for row in watermarked if row["transformed_detected"]] + escaped = [row for row in watermarked if not row["transformed_detected"]] + + def _mean(values): + return sum(values) / len(values) if values else None + + payload = { + "algorithm_version": DIAGNOSIS_VERSION, + "analysis_scope": "post-hoc explanatory analysis of the completed budget-scaled confirmation cycle; not used to tune any candidate against this corpus", + "plan_hash": plan["plan_hash"], + "evidence_hash": evidence["artifact_hash"], + "corpus_hash": corpus.artifact_hash, + "rows": tuple(rows), + "watermarked_summary": { + "total": len(watermarked), + "detected": len(detected), + "mean_achievable_coverage_detected": _mean([r["achievable_coverage_fraction"] for r in detected]), + "mean_achievable_coverage_escaped": _mean([r["achievable_coverage_fraction"] for r in escaped]), + "mean_realized_coverage_detected": _mean([r["realized_coverage_fraction"] for r in detected]), + "mean_realized_coverage_escaped": _mean([r["realized_coverage_fraction"] for r in escaped]), + "mean_hole_token_total_detected": _mean([r["hole_token_total"] for r in detected]), + "mean_hole_token_total_escaped": _mean([r["hole_token_total"] for r in escaped]), + "achievable_below_70pct_detected": sum(1 for r in detected if r["achievable_coverage_fraction"] < 0.7), + "achievable_below_70pct_escaped": sum(1 for r in escaped if r["achievable_coverage_fraction"] < 0.7), + }, + } + return {**payload, "artifact_hash": sha256_json(payload)} + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(prog="budget-scaled-coverage-diagnosis") + parser.add_argument("--corpus-json", type=Path, required=True) + parser.add_argument("--plan-json", type=Path, required=True) + parser.add_argument("--evidence-json", type=Path, required=True) + parser.add_argument("--json", type=Path, required=True) + args = parser.parse_args(argv) + from transformers import AutoTokenizer + + corpus = load_tiny_dev_corpus_by_version_json(args.corpus_json) + plan = json.loads(args.plan_json.read_text(encoding="utf-8")) + evidence = json.loads(args.evidence_json.read_text(encoding="utf-8")) + tokenizer = AutoTokenizer.from_pretrained( + "openai-community/gpt2", + revision="607a30d783dfa663caf39e06633721c8d4cfcd7e", + use_fast=True, + padding_side="left", + ) + report = diagnose_sources(corpus, tokenizer, plan, evidence) + args.json.parent.mkdir(parents=True, exist_ok=True) + args.json.write_text( + json.dumps(report, indent=2, sort_keys=True) + "\n", encoding="utf-8" + ) + summary = report["watermarked_summary"] + print(f"artifact_hash={report['artifact_hash']}") + for key, value in summary.items(): + print(f"{key}={value}") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) From 202c840baa8be5d32e052dca05be5489f8a97d9f Mon Sep 17 00:00:00 2001 From: Cyinh Date: Sat, 22 Aug 2026 21:24:49 -0400 Subject: [PATCH 11/30] Add coverage-completion v2 contract and freeze record --- ...kmark-coverage-completion-v2.contract.json | 136 ++++++++++++++++++ ...uckmark-coverage-completion-v2.freeze.json | 10 ++ 2 files changed, 146 insertions(+) create mode 100644 specs/fuckmark-coverage-completion-v2.contract.json create mode 100644 specs/fuckmark-coverage-completion-v2.freeze.json diff --git a/specs/fuckmark-coverage-completion-v2.contract.json b/specs/fuckmark-coverage-completion-v2.contract.json new file mode 100644 index 00000000..500689f4 --- /dev/null +++ b/specs/fuckmark-coverage-completion-v2.contract.json @@ -0,0 +1,136 @@ +{ + "algorithm_version": "coverage-completion-contract-v2", + "base_main_commit": "16e153fdfa3739a88bca9b1559543a10ded2d859", + "candidate_profile": { + "budget": 16, + "budget_justification": "cycle-1 coverage saturation below 16 on every source; no sweep needed", + "geometry_mode": "TOKENIZER_AWARE_PUBLIC", + "ngram_len": 5, + "profile_id": "key-blind-coverage-completion-v2", + "replicate_count": 1, + "ruleset": "cycle-1 development ruleset plus 65 frozen extension surface words", + "schedule_policy": "COVERAGE_GREEDY_KEY_BLIND", + "schedule_seed_base": 1140000 + }, + "claim_boundary": { + "level_b_product_admissible": false, + "release_authorized": false, + "status": "EXPLORATORY_FIXED_OPEN_DETECTOR_ONLY", + "undetectability_claim": false, + "watermark_removal_claim": false + }, + "corpora": { + "confirmation": { + "attack_pairs_per_domain": 16, + "profile": "tiny-dev-v3", + "seed_base": 440000 + }, + "development": [ + { + "attack_pairs_per_domain": 8, + "profile": "tiny-dev-v3", + "seed_base": 430000 + }, + { + "attack_pairs_per_domain": 8, + "note": "repeat use from cycle 1, labeled", + "profile": "tiny-dev-v3", + "seed_base": 410000 + } + ], + "forbidden": "cycle-1 confirmation corpus at seed base 420000 may not be scored in cycle 2" + }, + "cycle1_reference": { + "confirmation_residual_detected": 11, + "confirmation_watermarked_sources": 64, + "mean_achievable_coverage_all": 0.8187, + "mean_achievable_coverage_detected": 0.6818, + "mean_achievable_coverage_escaped": 0.8469 + }, + "detector": { + "comparison_operator": ">=", + "family": "Hugging Face SynthID Weighted Mean", + "keys": "public development keys", + "ngram_len": 5, + "target_fpr": 0.01 + }, + "endpoints": { + "controls": { + "gate_pass_fraction_required": 1.0, + "pristine_watermarked_detected_min": 60, + "transformed_unwatermarked_detected_max": 0 + }, + "improvement": "detected count < 11", + "mechanistic": "mean achievable coverage >= 0.80 on confirmation watermarked sources", + "primary": "confirmation transformed watermarked detected == 0/64" + }, + "extension_words": [ + "an", + "he", + "she", + "his", + "her", + "its", + "our", + "their", + "them", + "him", + "us", + "me", + "my", + "your", + "if", + "when", + "where", + "which", + "who", + "what", + "how", + "why", + "because", + "so", + "than", + "then", + "there", + "here", + "all", + "any", + "each", + "more", + "most", + "some", + "such", + "only", + "very", + "just", + "also", + "into", + "about", + "over", + "under", + "after", + "before", + "between", + "during", + "through", + "without", + "while", + "do", + "does", + "been", + "being", + "would", + "could", + "should", + "may", + "might", + "must", + "were", + "one", + "even", + "still", + "often" + ], + "protocol_id": "coverage-completion-v2", + "recorded_date": "2026-08-23" +} diff --git a/specs/fuckmark-coverage-completion-v2.freeze.json b/specs/fuckmark-coverage-completion-v2.freeze.json new file mode 100644 index 00000000..a5e1fb00 --- /dev/null +++ b/specs/fuckmark-coverage-completion-v2.freeze.json @@ -0,0 +1,10 @@ +{ + "appendix_confirmation_budget": 16, + "base_main_commit": "16e153fdfa3739a88bca9b1559543a10ded2d859", + "contract_sha256": "20ba6145793a89f0aa5cbc530305d9345731a60d60ad906199b323c74942d825", + "freeze_record_version": "coverage-completion-v2-freeze-v1", + "frozen_at_utc": "2026-08-23T01:24:49.558549+00:00", + "pre_score_amendments": [], + "protocol_id": "coverage-completion-v2", + "protocol_sha256": "65856295a595acaa404392239ccd9db9fa43d3f45a86ae61ee0d88b0d1e0851a" +} From fee05d11c403eebf5322ae52a0401ae846abb2b1 Mon Sep 17 00:00:00 2001 From: Cyinh Date: Sat, 22 Aug 2026 21:27:01 -0400 Subject: [PATCH 12/30] Add coverage-completion ruleset, profile, and registry selection --- fuckmark/experiments/effectiveness_plan.py | 10 +- fuckmark/tiny_dev_effectiveness_plan_hf.py | 7 +- fuckmark/tiny_dev_effectiveness_score_hf.py | 7 +- fuckmark/transforms/__init__.py | 8 ++ fuckmark/transforms/effectiveness_profile.py | 44 +++++- fuckmark/transforms/surface_rules.py | 98 +++++++++++++ tests/test_coverage_completion_profile.py | 139 +++++++++++++++++++ 7 files changed, 309 insertions(+), 4 deletions(-) create mode 100644 tests/test_coverage_completion_profile.py diff --git a/fuckmark/experiments/effectiveness_plan.py b/fuckmark/experiments/effectiveness_plan.py index 7fc10183..f1e7d81f 100644 --- a/fuckmark/experiments/effectiveness_plan.py +++ b/fuckmark/experiments/effectiveness_plan.py @@ -14,9 +14,11 @@ ScheduleGeometryMode, SchedulePolicy, build_candidate_tokenizer_geometry, + key_blind_coverage_completion_transform_registry, key_blind_high_coverage_transform_registry, validate_effectiveness_profile_registry, validate_hard_invariants, + KEY_BLIND_COVERAGE_COMPLETION_PROFILE_ID, ) @@ -151,6 +153,12 @@ def _schedule_seed( return seed +def _registry_for_profile(profile: EffectivenessTransformProfile): + if profile.profile_id == KEY_BLIND_COVERAGE_COMPLETION_PROFILE_ID: + return key_blind_coverage_completion_transform_registry() + return key_blind_high_coverage_transform_registry() + + def build_key_blind_high_coverage_plan( corpus: Any, tokenizer: Any, @@ -159,7 +167,7 @@ def build_key_blind_high_coverage_plan( source_code_commit: str, ) -> dict[str, object]: _require_git_object_id("source_code_commit", source_code_commit) - registry = key_blind_high_coverage_transform_registry() + registry = _registry_for_profile(profile) validate_effectiveness_profile_registry(profile, registry) policy = SchedulePolicy(profile.schedule_policy_id) if policy is not SchedulePolicy.COVERAGE_GREEDY_KEY_BLIND: diff --git a/fuckmark/tiny_dev_effectiveness_plan_hf.py b/fuckmark/tiny_dev_effectiveness_plan_hf.py index 9a4a3187..d638310d 100644 --- a/fuckmark/tiny_dev_effectiveness_plan_hf.py +++ b/fuckmark/tiny_dev_effectiveness_plan_hf.py @@ -20,6 +20,7 @@ from .transforms import ( KEY_BLIND_HIGH_COVERAGE_PROFILE_ID, KEY_BLIND_FULL_POOL_COVERAGE_PROFILE_ID, + KEY_BLIND_COVERAGE_COMPLETION_PROFILE_ID, resolve_effectiveness_profile, ) @@ -84,7 +85,11 @@ def _parser() -> argparse.ArgumentParser: parser.add_argument( "--profile-id", default=KEY_BLIND_HIGH_COVERAGE_PROFILE_ID, - choices=(KEY_BLIND_HIGH_COVERAGE_PROFILE_ID, KEY_BLIND_FULL_POOL_COVERAGE_PROFILE_ID), + choices=( + KEY_BLIND_HIGH_COVERAGE_PROFILE_ID, + KEY_BLIND_FULL_POOL_COVERAGE_PROFILE_ID, + KEY_BLIND_COVERAGE_COMPLETION_PROFILE_ID, + ), ) parser.add_argument("--budgets", default="") parser.add_argument( diff --git a/fuckmark/tiny_dev_effectiveness_score_hf.py b/fuckmark/tiny_dev_effectiveness_score_hf.py index 3df8c1e6..3e18a0fa 100644 --- a/fuckmark/tiny_dev_effectiveness_score_hf.py +++ b/fuckmark/tiny_dev_effectiveness_score_hf.py @@ -35,6 +35,7 @@ KEY_BLIND_HIGH_COVERAGE_PROFILE, KEY_BLIND_HIGH_COVERAGE_PROFILE_ID, KEY_BLIND_FULL_POOL_COVERAGE_PROFILE_ID, + KEY_BLIND_COVERAGE_COMPLETION_PROFILE_ID, EffectivenessTransformProfile, resolve_effectiveness_profile, ) @@ -226,7 +227,11 @@ def _parser() -> argparse.ArgumentParser: parser.add_argument( "--profile-id", default=KEY_BLIND_HIGH_COVERAGE_PROFILE_ID, - choices=(KEY_BLIND_HIGH_COVERAGE_PROFILE_ID, KEY_BLIND_FULL_POOL_COVERAGE_PROFILE_ID), + choices=( + KEY_BLIND_HIGH_COVERAGE_PROFILE_ID, + KEY_BLIND_FULL_POOL_COVERAGE_PROFILE_ID, + KEY_BLIND_COVERAGE_COMPLETION_PROFILE_ID, + ), ) parser.add_argument("--budgets", default="") parser.add_argument( diff --git a/fuckmark/transforms/__init__.py b/fuckmark/transforms/__init__.py index e91ef4f6..5906818b 100644 --- a/fuckmark/transforms/__init__.py +++ b/fuckmark/transforms/__init__.py @@ -1,6 +1,8 @@ from .candidate_artifacts import CandidateConflict, CandidateEnumeration, CandidateRejection, TransformCandidate from .effectiveness_profile import ( EFFECTIVENESS_PROFILE_ALGORITHM_VERSION, + KEY_BLIND_COVERAGE_COMPLETION_PROFILE_ID, + KEY_BLIND_COVERAGE_COMPLETION_SEED_BASE, KEY_BLIND_FULL_POOL_COVERAGE_PROFILE_ID, KEY_BLIND_FULL_POOL_COVERAGE_SEED_BASE, KEY_BLIND_HIGH_COVERAGE_BUDGETS, @@ -8,6 +10,8 @@ KEY_BLIND_HIGH_COVERAGE_PROFILE_ID, KEY_BLIND_HIGH_COVERAGE_SEED_BASE, EffectivenessTransformProfile, + key_blind_coverage_completion_profile, + key_blind_coverage_completion_transform_registry, key_blind_full_pool_coverage_profile, key_blind_high_coverage_transform_registry, resolve_effectiveness_profile, @@ -212,6 +216,10 @@ "KEY_BLIND_HIGH_COVERAGE_SEED_BASE", "KEY_BLIND_FULL_POOL_COVERAGE_PROFILE_ID", "KEY_BLIND_FULL_POOL_COVERAGE_SEED_BASE", + "KEY_BLIND_COVERAGE_COMPLETION_PROFILE_ID", + "KEY_BLIND_COVERAGE_COMPLETION_SEED_BASE", + "key_blind_coverage_completion_profile", + "key_blind_coverage_completion_transform_registry", "LEXICAL_RETOKENIZATION_FIXTURE_ALGORITHM_VERSION", "LEXICAL_RULE_AUDIT_ALGORITHM_VERSION", "LEXICAL_TEMPLATE_RULE_ALGORITHM_VERSION", diff --git a/fuckmark/transforms/effectiveness_profile.py b/fuckmark/transforms/effectiveness_profile.py index 224a2c26..c1fe54c4 100644 --- a/fuckmark/transforms/effectiveness_profile.py +++ b/fuckmark/transforms/effectiveness_profile.py @@ -14,7 +14,7 @@ TransformRegistry, ) from .scheduler import CANDIDATE_SCHEDULER_ALGORITHM_VERSION -from .surface_rules import development_surface_rules +from .surface_rules import coverage_completion_surface_rules, development_surface_rules from .syntax_rules import development_syntax_rules from .tokenizer_geometry import TOKENIZER_GEOMETRY_ALGORITHM_VERSION @@ -25,6 +25,8 @@ KEY_BLIND_HIGH_COVERAGE_SEED_BASE = 1_120_000 KEY_BLIND_FULL_POOL_COVERAGE_PROFILE_ID = "key-blind-full-pool-coverage-v1" KEY_BLIND_FULL_POOL_COVERAGE_SEED_BASE = 1_130_000 +KEY_BLIND_COVERAGE_COMPLETION_PROFILE_ID = "key-blind-coverage-completion-v2" +KEY_BLIND_COVERAGE_COMPLETION_SEED_BASE = 1_140_000 @dataclass(frozen=True, slots=True) @@ -204,9 +206,49 @@ def resolve_effectiveness_profile( if not budgets: raise ValueError("the full-pool coverage profile requires explicit budgets") return key_blind_full_pool_coverage_profile(budgets) + if profile_id == KEY_BLIND_COVERAGE_COMPLETION_PROFILE_ID: + if not budgets: + raise ValueError("the coverage completion profile requires explicit budgets") + return key_blind_coverage_completion_profile(budgets) raise ValueError("unknown effectiveness profile id") +def key_blind_coverage_completion_transform_registry( + identifiers: Sequence[str] = (), +) -> TransformRegistry: + return TransformRegistry( + ( + *development_forward_contraction_rules(), + *coverage_completion_surface_rules(), + *development_lexical_rules(), + *development_syntax_rules(), + ), + identifiers, + ) + + +def key_blind_coverage_completion_profile( + budgets: tuple[int, ...], +) -> EffectivenessTransformProfile: + if not isinstance(budgets, tuple): + raise TypeError("budgets must be a tuple") + return EffectivenessTransformProfile.create( + profile_id=KEY_BLIND_COVERAGE_COMPLETION_PROFILE_ID, + budgets=budgets, + budget_unit="operation", + schedule_policy_id="COVERAGE_GREEDY_KEY_BLIND", + schedule_seed_base=KEY_BLIND_COVERAGE_COMPLETION_SEED_BASE, + replicate_count=1, + ngram_len=5, + ruleset_hash=key_blind_coverage_completion_transform_registry().ruleset_hash, + scientific_scope=( + "Frozen detector-blind and key-blind coverage-completion profile extending the " + "development ruleset surface list; exploratory effectiveness evidence only and " + "not release authorization" + ), + ) + + def validate_effectiveness_profile_registry( profile: EffectivenessTransformProfile, registry: TransformRegistry, diff --git a/fuckmark/transforms/surface_rules.py b/fuckmark/transforms/surface_rules.py index 8e822752..45d61f90 100644 --- a/fuckmark/transforms/surface_rules.py +++ b/fuckmark/transforms/surface_rules.py @@ -65,3 +65,101 @@ def development_surface_rules() -> tuple[SurfaceSpacingRule, ...]: for name, source in punctuation ) return (*word_rules, *punctuation_rules) + + +COVERAGE_COMPLETION_EXTENSION_ALGORITHM_VERSION = "coverage-completion-extension-words-v1" + + +def coverage_completion_extension_words() -> tuple[str, ...]: + return ( + "an", + "he", + "she", + "his", + "her", + "its", + "our", + "their", + "them", + "him", + "us", + "me", + "my", + "your", + "if", + "when", + "where", + "which", + "who", + "what", + "how", + "why", + "because", + "so", + "than", + "then", + "there", + "here", + "all", + "any", + "each", + "more", + "most", + "some", + "such", + "only", + "very", + "just", + "also", + "into", + "about", + "over", + "under", + "after", + "before", + "between", + "during", + "through", + "without", + "while", + "do", + "does", + "been", + "being", + "would", + "could", + "should", + "may", + "might", + "must", + "were", + "one", + "even", + "still", + "often", + ) + + +def coverage_completion_surface_rules() -> tuple[SurfaceSpacingRule, ...]: + base = development_surface_rules() + base_ids = {rule.rule_id for rule in base} + base_words = {rule.source for rule in base if rule.source.isalpha()} + extension_words = coverage_completion_extension_words() + if len(set(extension_words)) != len(extension_words): + raise ValueError("coverage completion extension words must be unique") + if any(word in base_words for word in extension_words): + raise ValueError("coverage completion extension words must not duplicate the base surface list") + if any(not word.isalpha() or word != word.lower() or len(word) < 2 for word in extension_words): + raise ValueError("coverage completion extension words must be lowercase alphabetic words of length >= 2") + extension_rules = tuple( + SurfaceSpacingRule.create( + rule_id=f"surface-space-after-{word}", + version="v1", + source=word, + replacement=word + " ", + ) + for word in extension_words + ) + if any(rule.rule_id in base_ids for rule in extension_rules): + raise ValueError("coverage completion extension rule IDs must not collide with base rules") + return (*base, *extension_rules) diff --git a/tests/test_coverage_completion_profile.py b/tests/test_coverage_completion_profile.py new file mode 100644 index 00000000..fc55a725 --- /dev/null +++ b/tests/test_coverage_completion_profile.py @@ -0,0 +1,139 @@ +from __future__ import annotations + +from types import SimpleNamespace + +import pytest + +from fuckmark.corpus import CorpusDomain, CorpusSplit, WatermarkLabel +from fuckmark.experiments.effectiveness_plan import build_key_blind_high_coverage_plan +from fuckmark.hashing import sha256_text +from fuckmark.transforms import ( + KEY_BLIND_COVERAGE_COMPLETION_PROFILE_ID, + KEY_BLIND_COVERAGE_COMPLETION_SEED_BASE, + KEY_BLIND_HIGH_COVERAGE_PROFILE, + key_blind_coverage_completion_profile, + key_blind_coverage_completion_transform_registry, + key_blind_high_coverage_transform_registry, + resolve_effectiveness_profile, +) +from fuckmark.transforms.surface_rules import ( + coverage_completion_extension_words, + coverage_completion_surface_rules, + development_surface_rules, +) + + +class FakeTokenizer: + def __call__( + self, + text: str, + *, + add_special_tokens: bool, + return_offsets_mapping: bool = False, + ) -> dict[str, object]: + assert add_special_tokens is False + ids = [index + 1 for index in range(len(text))] + value: dict[str, object] = {"input_ids": ids} + if return_offsets_mapping: + value["offset_mapping"] = [(index, index + 1) for index in range(len(text))] + return value + + +def _sample(sample_id: str, label: WatermarkLabel, text: str, identity_hash: str): + token_ids = tuple(index + 1 for index in range(len(text))) + return SimpleNamespace( + sample_id=sample_id, + split=CorpusSplit.ATTACK_DEVELOPMENT, + label=label, + prompt_family_id=f"prompt-{sample_id}", + domain=CorpusDomain.GENERAL_EXPLANATORY, + text=text, + text_sha256=sha256_text(text), + text_only_tokens=SimpleNamespace(token_ids=token_ids), + model=SimpleNamespace(identity_hash=identity_hash, eos_token_id=50256), + ) + + +def _corpus(): + identity_hash = sha256_text("fake-tokenizer") + text = "He said they were not sure, because more work was needed before one can decide." + samples = ( + _sample("negative-1", WatermarkLabel.UNWATERMARKED, text, identity_hash), + _sample("positive-1", WatermarkLabel.WATERMARKED, text, identity_hash), + ) + return SimpleNamespace( + artifact_hash=sha256_text("fake-corpus"), + model_identity_hash=identity_hash, + manifest=SimpleNamespace(samples=samples, manifest_hash=sha256_text("fake-manifest")), + ) + + +def test_extension_words_obey_the_frozen_surface_contract() -> None: + words = coverage_completion_extension_words() + assert len(words) == 65 + assert len(set(words)) == len(words) + base_words = {rule.source for rule in development_surface_rules() if rule.source.isalpha()} + assert not (set(words) & base_words) + assert all(word.isalpha() and word == word.lower() and len(word) >= 2 for word in words) + rules = coverage_completion_surface_rules() + base = development_surface_rules() + assert len(rules) == len(base) + 65 + assert set(rule.rule_id for rule in base) < set(rule.rule_id for rule in rules) + + +def test_extension_rules_do_not_split_words_mid_token() -> None: + registry = key_blind_coverage_completion_transform_registry() + text = "The weather, whether he measured then, was often more random than before." + enumeration = registry.enumerate(text) + replaced_words = set() + for candidate in enumeration.candidates: + replaced_words.add(text[candidate.start : candidate.end].strip()) + assert "he" in replaced_words or "then" in replaced_words + joined = "".join( + candidate.replacement_text + for candidate in enumeration.candidates[:1] + ) + assert "random" not in joined or joined == "random" + transformed_spans = [text[c.start : c.end] for c in enumeration.candidates] + assert all(span.strip().isalpha() or span in (". ", ", ", "; ", ": ", "? ", "! ") for span in transformed_spans) + + +def test_coverage_completion_registry_extends_and_rebinds_ruleset() -> None: + base_registry = key_blind_high_coverage_transform_registry() + registry = key_blind_coverage_completion_transform_registry() + assert registry.ruleset_hash != base_registry.ruleset_hash + base_ids = {rule.rule_id for rule in base_registry.rules} + extended_ids = {rule.rule_id for rule in registry.rules} + assert base_ids < extended_ids + + +def test_frozen_b16_profile_hash_is_unchanged() -> None: + assert ( + KEY_BLIND_HIGH_COVERAGE_PROFILE.profile_hash + == "6ad142262bfb11a714565d7bd43daa859657fe817bdafa9c8dcf0f4884c07512" + ) + + +def test_coverage_completion_profile_resolves_and_builds_plans() -> None: + profile = resolve_effectiveness_profile(KEY_BLIND_COVERAGE_COMPLETION_PROFILE_ID, (16,)) + assert profile.profile_id == KEY_BLIND_COVERAGE_COMPLETION_PROFILE_ID + assert profile.schedule_seed_base == KEY_BLIND_COVERAGE_COMPLETION_SEED_BASE + assert key_blind_coverage_completion_profile((16,)) == profile + corpus = _corpus() + plan = build_key_blind_high_coverage_plan( + corpus, + FakeTokenizer(), + profile=profile, + source_code_commit="a" * 40, + ) + assert plan["ruleset_hash"] == profile.ruleset_hash + assert len(plan["variants"]) == 2 + assert all(row["hard_invariant_status"] == "pass" for row in plan["variants"]) + assert all(row["detector_access_observed"] is False for row in plan["variants"]) + + +def test_resolve_rejects_invalid_coverage_completion_requests() -> None: + with pytest.raises(ValueError): + resolve_effectiveness_profile(KEY_BLIND_COVERAGE_COMPLETION_PROFILE_ID) + with pytest.raises(ValueError): + key_blind_coverage_completion_profile((48, 16)) From 6d0ad2dc40fc7335266138d263a5370f64f6e432 Mon Sep 17 00:00:00 2001 From: Cyinh Date: Sat, 22 Aug 2026 21:39:54 -0400 Subject: [PATCH 13/30] Record cycle-2 development outcome before confirmation scoring --- .../fuckmark-coverage-completion-v2.freeze.json | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/specs/fuckmark-coverage-completion-v2.freeze.json b/specs/fuckmark-coverage-completion-v2.freeze.json index a5e1fb00..e06ddbb6 100644 --- a/specs/fuckmark-coverage-completion-v2.freeze.json +++ b/specs/fuckmark-coverage-completion-v2.freeze.json @@ -1,5 +1,21 @@ { "appendix_confirmation_budget": 16, + "appendix_development_outcome": { + "cycle1_ruleset_paired_baseline_on_430000": 4, + "cycle2_dev_detected": { + "410000": 0, + "430000": 5, + "pooled_of_64": 5 + }, + "evidence_hashes": { + "dev1_paired_baseline_430000": "7c7f9624e2a95dfb16fac5b17dac7778bbe7d87be6af988c2412f09aadb26e9e", + "dev2_evidence_410000": "2eda8a0d1c3599e35df38d6779f0ed395e56a13fc958c31f7cf3d689a6b736db", + "dev2_evidence_430000": "cf46b933ce3b7ae17a893432d4230db9b7b4f04bae8905844e30f74ad2f9a49a" + }, + "interpretation": "pooled dev detection identical between rulesets (5/64 each); the extension grew candidate pools but residuals concentrate in sources with high pristine scores and content-word-dense low-candidate regions; confirmation proceeds as pre-registered and may return a neutral result", + "phase": "development outcome recorded before confirmation scoring", + "recorded_at_utc": "2026-08-23T01:39:54.423362+00:00" + }, "base_main_commit": "16e153fdfa3739a88bca9b1559543a10ded2d859", "contract_sha256": "20ba6145793a89f0aa5cbc530305d9345731a60d60ad906199b323c74942d825", "freeze_record_version": "coverage-completion-v2-freeze-v1", From 4ddd84fa9bc184b1159946c56f8a370111d20f22 Mon Sep 17 00:00:00 2001 From: Cyinh Date: Sat, 22 Aug 2026 21:58:59 -0400 Subject: [PATCH 14/30] Record cycle-2 coverage-completion confirmation evidence and report --- .../binding-cross-check-2.json | 14 + .../cycle-2-report.md | 100 ++ .../cycle2-artifacts-SHA256SUMS.txt | 24 + .../rendering-manifest-2.json | 1030 +++++++++++++++++ 4 files changed, 1168 insertions(+) create mode 100644 evidence/budget-scaled-coverage-2026-08-22/binding-cross-check-2.json create mode 100644 evidence/budget-scaled-coverage-2026-08-22/cycle-2-report.md create mode 100644 evidence/budget-scaled-coverage-2026-08-22/cycle2-artifacts-SHA256SUMS.txt create mode 100644 evidence/budget-scaled-coverage-2026-08-22/rendering-manifest-2.json diff --git a/evidence/budget-scaled-coverage-2026-08-22/binding-cross-check-2.json b/evidence/budget-scaled-coverage-2026-08-22/binding-cross-check-2.json new file mode 100644 index 00000000..3cef3ef7 --- /dev/null +++ b/evidence/budget-scaled-coverage-2026-08-22/binding-cross-check-2.json @@ -0,0 +1,14 @@ +{ + "corpus_artifact_hash_replay": "ccc9d5d6ffe5168f8059f7b2ab5f418050ed11b487e34e68ebf1634284a96760", + "evidence_binds_plan": true, + "evidence_hash_replay": true, + "evidence_profile": "key-blind-coverage-completion-v2", + "evidence_row_hash_replays": "128/128", + "plan_binds_corpus": true, + "plan_detector_access_false": true, + "plan_hash_replay": true, + "render_manifest_binds_plan": true, + "render_manifest_pair_count": 128, + "robustness_binds_plan": true, + "robustness_hash_replay": true +} diff --git a/evidence/budget-scaled-coverage-2026-08-22/cycle-2-report.md b/evidence/budget-scaled-coverage-2026-08-22/cycle-2-report.md new file mode 100644 index 00000000..9c91d322 --- /dev/null +++ b/evidence/budget-scaled-coverage-2026-08-22/cycle-2-report.md @@ -0,0 +1,100 @@ +# Coverage-Completion Cycle 2 Report - 2026-08-23 + +Branch `research/ngram-window-coverage-v1` (local only, nothing pushed). Confirmation scored +once at commit `6d0ad2d2...` on corpus `ccc9d5d6...`. Protocol SHA-256 +`65856295a595acaa404392239ccd9db9fa43d3f45a86ae61ee0d88b0d1e0851a`, contract SHA-256 +`20ba6145793a89f0aa5cbc530305d9345731a60d60ad906199b323c74942d825`. + +## Executive summary + +On a new, source-disjoint 64-watermarked-source confirmation corpus, the frozen +detector-blind key-blind coverage-completion profile achieved **0/64 transformed watermarked +detections** (primary stretch target met for this condition) with **0/64 transformed +unwatermarked false positives** and all 128 fidelity/representation gates passing. One +pre-registered secondary gate FAILED and is reported as such: pristine watermarked detection +was 58/64, below the required 60, because this corpus's per-corpus calibration threshold rose +to 0.580261 (cycle 1: 0.558485), leaving six watermarked sources below threshold before any +transformation. Conditional on initial detectability, 0 of the 58 pristine-detected sources +remained detected after transformation. The permitted conclusion is unchanged in kind: +detector-specific detection reduction under the fixed open detector, public development keys, +and uncleaned text-only input path - here reaching the pre-registered narrow 0/64 target on +this corpus. It is not universal undetectability, watermark removal, or any unknown-key, +proprietary-detector, or cleaning-durability claim, and Level B (product admissibility) +remains unmet. + +## Mechanism and diagnosis chain + +Cycle 1 ended at 11/64 residual detection. The post-hoc coverage diagnosis (tool: +`tools/budget_scaled_coverage_diagnosis.py`) showed residual sources had mean achievable +n-gram coverage 0.682 vs 0.847 for escaped sources, with realized coverage equal to achievable +everywhere - the candidate pool, not the scheduler, was the bottleneck. Cycle 2 extended the +surface spacing word list by 65 frozen general-English function words (protocol section 2), +raising mean achievable coverage on confirmation watermarked sources to 0.827 (mechanistic +endpoint >= 0.80 met) and mean candidate pools to 21.9 per source. + +## Development phase (recorded before confirmation scoring; freeze appendix) + +| Corpus | Cycle-2 ruleset | Cycle-1 ruleset (paired) | +| --- | --- | --- | +| 410000 (repeat use, labeled) | 0/32 detected | 1/32 (cycle 1) | +| 430000 (fresh) | 5/32 detected | 4/32 (paired baseline) | +| Pooled dev | 5/64 | 5/64 | + +Dev showed NO pooled improvement; the extension grew pools where function words exist but +residual sources are content-word-dense with low candidate density (one had 5 candidates +total). Residual sets overlapped strongly across rulesets, tied to high pristine scores plus +sparse candidate regions. The confirmation proceeded as pre-registered and the outcome +exceeded the dev signal; corpus-to-corpus variance is high in both directions (cycle 1: +1/36 dev to 11/64 confirmation; cycle 2: 5/64 dev to 0/64 confirmation). This variance is +itself a finding: single-corpus dev counts are weak predictors. + +## Confirmation results (64 wm + 64 unwm, budget 16, threshold 0.580261 at 1% target FPR) + +| Condition | Detected | Rate | Mean score | +| --- | --- | --- | --- | +| Pristine watermarked | 58/64 | 90.6% | 0.6348 | +| Transformed watermarked | 0/64 | 0.0% | 0.5283 | +| Pristine unwatermarked | 0/64 | 0.0% | - | +| Transformed unwatermarked | 0/64 | 0.0% | - | + +Mean watermarked score drop 0.1064. Closest transformed margin to threshold: -0.0020 (one +source), then -0.0115. Realized edit cost mean 12.72 of mean pool 21.9; zero-candidate +sources: 0. Six pristine-watermarked sources below threshold (0.5590-0.5775) are listed in +the evidence and constitute the failed >= 60/64 control gate. + +## Gates, robustness, rendering, bindings + +- Section-6 gates: 128/128 confirmation rows pass (hard invariants, protected spans zero, + max word edit rate 0.1333 <= 0.30, no introduced non-ASCII or hidden-category codepoints, + no introduced normalization instability, Cf-strip identity on introduced characters). +- Cross-corpus text-hash disjointness: all pairwise overlaps zero among 410000, 430000, + 440000; also zero against cycle-1 corpora 401000 and 420000 (no repeat of the cycle-1 + degenerate collision). +- Fixed-browser rendering: DOM textContent equals plan texts on 128/128 pairs; screenshot + archived (`confirmation2-render-screenshot.png`). +- Independent cross-check: corpus/plan/evidence/robustness/render-manifest hashes all replay + and mutually bind; 128/128 evidence row hashes replay (`binding-cross-check-2.json`). + +## Adverse findings and caveats + +1. The >= 60/64 pristine-watermarked control gate failed at 58/64 (threshold sensitivity of + per-corpus calibration); the 0/64 primary result must be read together with this. +2. Dev predicted no improvement (5/64 both rulesets); the confirmation result therefore + carries corpus-sampling uncertainty. A single 0/64 does not establish a stable zero rate. +3. One transformed source sits 0.0020 below threshold - margin-level sensitivity persists. +4. Cycle-2 word extension did not reduce dev-phase residuals; residuals concentrate in + content-word-dense, low-candidate-density sources that word-list extension cannot reach. +5. Canonical-corpus cross-platform reproducibility limitation from cycle 1 still applies. + +## Classification + +- Product-safe code: extended surface ruleset (superset, base ruleset and frozen B16 profile + hash `6ad14226...` unchanged and test-pinned), coverage-completion profile + resolver + + registry selection, diagnosis tool, 8 new tests (total suite 1422 passing plus the known + Windows-only `spec.md` environmental failure). +- Candidate outcome: narrow pre-registered 0/64 target MET on this corpus and condition, + with the control-gate failure reported; exploratory only; NOT product/release admissible + (visible-text edits); no overclaim. +- Next research lever identified by diagnosis: candidate generation for content-word-dense + regions (rules that can place a visible-text-safe edit anywhere), which word-list + extension cannot solve. diff --git a/evidence/budget-scaled-coverage-2026-08-22/cycle2-artifacts-SHA256SUMS.txt b/evidence/budget-scaled-coverage-2026-08-22/cycle2-artifacts-SHA256SUMS.txt new file mode 100644 index 00000000..24e56987 --- /dev/null +++ b/evidence/budget-scaled-coverage-2026-08-22/cycle2-artifacts-SHA256SUMS.txt @@ -0,0 +1,24 @@ +59ae8aa407b4f9b93b93c8f50b968cfd154d84d0795b38360a9be9e54fa981cd *dev2-evidence-410000-provenance.json +e377fc52d0bfd766ec5824f2658d9897466c9d975bcd658623c2e579ff84a95a *dev2-evidence-410000.json +ea1b3b9dff40f955d8761ec7683725c5f9319f3ac37fdf30c58cab8bfa5d050e *dev2-evidence-430000-provenance.json +7d9fbe1229a511b16a1460954d947f8fa6cad58afd8262f2122393bbb0f36c5d *dev2-evidence-430000.json +3dd70d23a852c220d91d719c8e78cdbc1894e1bdc596ec0c0a69bbb9844339f3 *dev2-plan-410000-provenance.json +0573f47ff38e71e59e282482f796c0d963ddcee31b476d3a7805e82dc8af13a9 *dev2-plan-410000.json +bdbb1c2148acb1fa4e23e6ca3ff3d4185384e5eff6081d9b0a5b1a5d735a09e4 *dev2-plan-430000-provenance.json +3863cc67c67e357d5c67266f42d84dd7ddb3ad4869847f387e84ae0fd8f5a334 *dev2-plan-430000.json +0bc77aea8e70b3f8a97b67e1824440e7959004a9fdf785c2eb514ccab53ba1a3 *dev2-robustness-pooled.json +2129f7dc5c5b16d7fece545c4cf1ae156ccd6b16caa9dc27af1500e1887de56e *dev1-plan-430000.json +97fd81a8346c2c51acc3299e364f37a9c83a7ae9b07b74cacb57cc0476194a48 *dev1-evidence-430000.json +fb066647ad15c86e5c09ba5b1753ddbcb53e490d1a1a97e32e36a26016e93036 *confirmation2-corpus-440000.json +13cdd4aa2ddbb359096aa3fa1e95e3c32a7d8b576696a6455280caee2bc2bbe9 *confirmation2-evidence-440000-provenance.json +bde997a0103c60b7604815a074200543cd157fd3af28f59575eb9acef24cc568 *confirmation2-evidence-440000.json +1b8608f4a23ad21e1090582787b645c8e48b38d6a20cfc287abdcbaa65f888cf *confirmation2-plan-440000-provenance.json +a801ad020a339ba802467b0b0e48f501f95a9fc4b177ac8ea68b4153095c38e8 *confirmation2-plan-440000.json +48ad930bf3ef424595ec77910c48fa15dc89e1e63ca1724a988556a9a30d23b0 *confirmation2-robustness-440000.json +acb612a2294e5a2b269ee27169195afcf54309d5eb73d5ff6aaf3da840fef835 *coverage-diagnosis-420000.json +b8636ec8334c8b5b24fbacd6cd0731a79f4ddd05426a37bdce4946de5ae45aee *coverage-diagnosis-confirmation2-440000.json +2e3e904c035fcd169b6b4a8cc49b95338028fdbd4871c900f39a772517aa7cb1 *coverage-diagnosis-dev2-410000.json +36508360e70baf33b3eb8cb316a12a3aae3fbbbc87bacb9b0addc5db60d8df2e *coverage-diagnosis-dev2-430000.json +5d04b758e9d6964cf9501aa0416a455bd00b985c0fdb03dacbfbdb51fef5ce58 *binding-cross-check-2.json +132ec9541ca134c0f5f2f15932da5e3775653c34314ffef8241ef8d5ac3bebf5 *rendering-manifest-2.json +3e6de60f70a82e6b6fabc20ef010cec7051295a9199d8ee159e40bbeb1e8d7fb *dev-corpus-430000.json diff --git a/evidence/budget-scaled-coverage-2026-08-22/rendering-manifest-2.json b/evidence/budget-scaled-coverage-2026-08-22/rendering-manifest-2.json new file mode 100644 index 00000000..4821bb29 --- /dev/null +++ b/evidence/budget-scaled-coverage-2026-08-22/rendering-manifest-2.json @@ -0,0 +1,1030 @@ +{ + "pair_count": 128, + "pairs": [ + { + "pair_index": 0, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-conversational_prose-25-unwatermarked", + "source_text_hash": "27f8ba0dbcce150c40fe33527c222a7defee9c6b505c282a353cdb3f6e1b98ea", + "transformed_text_hash": "e1a0ad09e1835bdd68e4d4e8c1afb198938292cf924df5bc94ec448917eadc00", + "variant_hash": "f30f275162a5f0d65fcac8f6de949eef3ab98f412a5cb03c55a62d57088e8126" + }, + { + "pair_index": 1, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-conversational_prose-25-watermarked", + "source_text_hash": "2d0b08c069db8ffefad2a49d48ae1ce4fb2b7984ac535dfc639fabc00cda87f7", + "transformed_text_hash": "c4ca400a079c7da663718bcc076546b0835292c4a38adaf5a8df9342bf0edeb3", + "variant_hash": "a8b8194e53923943225868e4e16bcd2ff340be63c3afa65f2890b9f917415068" + }, + { + "pair_index": 2, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-conversational_prose-26-unwatermarked", + "source_text_hash": "fd861020eaacb6c8b20a6614c6f89f571b884aa4daf851c5be9ef21f38de39ae", + "transformed_text_hash": "a50078d6a071e29e39b66acf7d5fd76306194891c5817e6d88022bee0c1a1c4b", + "variant_hash": "ae1433cf011acc6039ce11ffb53f54e2996ad819783333ec270c0e6144e2e201" + }, + { + "pair_index": 3, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-conversational_prose-26-watermarked", + "source_text_hash": "e11ff30e17146ba71ab13b87113c1e5e02c06258001f91ef745f2ce82cea2632", + "transformed_text_hash": "25024f4f126fff75a102704dab378d875debe1d677281f5a67b63445d56347f7", + "variant_hash": "9bce3d49124167aac1e5ed18bcb963bb6ff43dece4432b214f5186844662589b" + }, + { + "pair_index": 4, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-conversational_prose-27-unwatermarked", + "source_text_hash": "b966102c29e33ea669c11a19c1cb936e0b9240de92ab4d927eb3a1fe01733b24", + "transformed_text_hash": "23bc52b91ec0e30f622b356954b814b4b699d32799a82ed8c3a6ed02df4ec3ca", + "variant_hash": "0e85e03929638a31f062dfdb7d4fee9bd0bc6c4cd965c1563a0aefb09711e14a" + }, + { + "pair_index": 5, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-conversational_prose-27-watermarked", + "source_text_hash": "c87df08809f5344784f7ee04b81c1ba6394f6a2bad6f779a26f9cb1e19bf969a", + "transformed_text_hash": "bf52c8dc94478da05ab037fa177005640071419d68ad6018a52e29422264681a", + "variant_hash": "6752a3a3f3379d7c92fe07c004df1c483b8ef9f757918060dd73e1cf7b544d69" + }, + { + "pair_index": 6, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-conversational_prose-28-unwatermarked", + "source_text_hash": "72ac69c200379ad6025858414a2e84bbf45ab91cb70135a4487beee1b5f86884", + "transformed_text_hash": "dc0ad761e1bf2e26aa018bf76812e728cdad166f5c6f387b9e31f899d9069d91", + "variant_hash": "b6375d95dc90731b45718a1c16a09a3f5f70ccc1442e31610e20bb82d05cad88" + }, + { + "pair_index": 7, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-conversational_prose-28-watermarked", + "source_text_hash": "1faea3a583e510b5d458a9044035fd2d10a34669acc5a63fa8c721cfd6d6c639", + "transformed_text_hash": "7049a4318d7e8c5995e6cf436c025d3ecc40bfa0c7d698d0da6c9a37915dcd60", + "variant_hash": "4311d8a7d3f822a4775fc3aef230132cbf59d193632fc4b9435883e21ab6c452" + }, + { + "pair_index": 8, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-conversational_prose-29-unwatermarked", + "source_text_hash": "8fb32ae0f622e56fd8670d15ef6ae922525f178800d9fd7fb3b12b7d420e9a78", + "transformed_text_hash": "007870db7404383d6a4c8234208ab4e652179c5b6db9cb460059a016d72656a1", + "variant_hash": "a9e778e9b903909c5305f2b93876661e469df5c5e3db8c60043b94571403daac" + }, + { + "pair_index": 9, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-conversational_prose-29-watermarked", + "source_text_hash": "ceb303e7722bbf5e4215cb309dd99cc67b8652155486e66432fa83023b68032e", + "transformed_text_hash": "3dab095166fcb0f2befd8b0299e52c8b7911e2935b467becb7ede3cb1918de56", + "variant_hash": "9cc8e6b68236c7923c3c6ce589cf4ea6e2a0c61d8c303164b29311806d7ff891" + }, + { + "pair_index": 10, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-conversational_prose-30-unwatermarked", + "source_text_hash": "f7efa61bcc45f3aaee42d618be9add3622fd5e71187060cb49b28f68c2ae74e5", + "transformed_text_hash": "808be4a549e1d7c02bc3a3032c3a50e9468d8a0c90a3eae46684dc7473c10391", + "variant_hash": "4a46572049851c82efebe06b90ba3f9eb9f79e5e48188b41503661ab65d0448b" + }, + { + "pair_index": 11, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-conversational_prose-30-watermarked", + "source_text_hash": "ca283ea301fb9c7536a9b489e69003074fb19cc3a21147e959078115e63e2fec", + "transformed_text_hash": "7c2de3cf637e56e94ace3bbb211b6d13e25d2d5fb946be538fc6c5f7af000d60", + "variant_hash": "1aa86d41fce41101293a50ef0b15ae4a50a01ef3a8b301fc539e5850cf282972" + }, + { + "pair_index": 12, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-conversational_prose-31-unwatermarked", + "source_text_hash": "1d322d153d43fe1943cbe21da3039e7e51ab738279fd87a9e630d112ccc86d5a", + "transformed_text_hash": "f846bbdf1efcd91b5865074187e3912900ec9541d8107e8bc13c74ec9a783dd8", + "variant_hash": "71dcf1f90268fa5ffb46f44a615fa6918d1b5b94e7a8c7ad8746556e294a4d5d" + }, + { + "pair_index": 13, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-conversational_prose-31-watermarked", + "source_text_hash": "57caa5c5f83673b2d39621b009482f075bde0c8b850aee23964829b9f2a61106", + "transformed_text_hash": "08bbb8b63974ad82d9e1060edd1047a36c7df550be4d7fbd30143972e5281233", + "variant_hash": "87ac1a170a2468c0df00ad05829764b7860ac238ebfc84a120d65941ef788d5a" + }, + { + "pair_index": 14, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-conversational_prose-32-unwatermarked", + "source_text_hash": "eb2187803c4ab790422393b22f4f3f0ac2847e7b40c7402363d22ceceb55e550", + "transformed_text_hash": "ef07cb61b9338e7d5c139c766a754b8c3ba3f37880b7b14505e82c62898e4b6e", + "variant_hash": "1334d911e33e7b016c211ca494a3a90032dbc541612ee4d519f78ff14a703a3e" + }, + { + "pair_index": 15, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-conversational_prose-32-watermarked", + "source_text_hash": "3673bb2d00c0c6fb331c3730ce9d4f53f0edcdc771f1e5e163f4c4a7d4cfd77a", + "transformed_text_hash": "7809e783d2a9ce6fefa846615b3950cc342a0d2504a9af2b02b7d0798999905b", + "variant_hash": "938c430fb5d4a21eb46ad0c03d5b263a33f52ec379f31879b4a390b8c4927df2" + }, + { + "pair_index": 16, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-conversational_prose-33-unwatermarked", + "source_text_hash": "507507a54c85af546ab81435c144622b9f8ad3f9161db63a1da535ba8bd6d108", + "transformed_text_hash": "4e6e50470de1454483d901a2c83de19719590371501e801c4d5962bad80773a0", + "variant_hash": "0cb9e6bdb3d9d1e729ff11e66256995a54ad1c9c5cc55720f9094c22e0d48a38" + }, + { + "pair_index": 17, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-conversational_prose-33-watermarked", + "source_text_hash": "3f0db812a783757f321c6f20893b4aed0ca926aa1948a625f3dab16d5d09dee4", + "transformed_text_hash": "31ea2f30cc4b26caca6a27fdabfe0348b6bb0966cdfdb78a81d98214608ca2aa", + "variant_hash": "1d3dbd96c449db4a5962f9f3ffca7b18327bbb0cc3b3e43a47907f8c7241e00d" + }, + { + "pair_index": 18, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-conversational_prose-34-unwatermarked", + "source_text_hash": "279966844e8bfc586f968bb0109e2a8520f7daf194755df67b827595cb5c174a", + "transformed_text_hash": "e5bbc32c5fb1438986f5cf3706bb16aeaaa4d6fabeccff833817d0865fe8bc9e", + "variant_hash": "87b740fc9990b5d2862dff25c617d760e9dbf15c1c64cde4fd1bf01ed5a451db" + }, + { + "pair_index": 19, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-conversational_prose-34-watermarked", + "source_text_hash": "fbc5e96d6d7ffa5ad1386319618620275700ff7a466c1a6e6cf2ed038a48c604", + "transformed_text_hash": "a303df732ebf31d0696a2c1e79d1f220e15298decd23de24a86c48ca1bb8918f", + "variant_hash": "cc55e0ec03951e1994d8e8f5d8a9d76cb696940a02ac39be1a0999690135e28a" + }, + { + "pair_index": 20, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-conversational_prose-35-unwatermarked", + "source_text_hash": "c15304545085decb50a67ef45be068b39c6136c031e0d616260d0dc8bbfa4647", + "transformed_text_hash": "7716ba3651596f791d0e8161cb9221767363b86967b2c84a102fd16534af27c0", + "variant_hash": "f578e1b260ce3ed1aafeeb7cf52dfd8eb1e076dc9ab1c3139b02b053ba17b3a0" + }, + { + "pair_index": 21, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-conversational_prose-35-watermarked", + "source_text_hash": "b4d4e1253dc6206e88cd8534f581b59d2255dda1b94c2f534cdf9b986d09f112", + "transformed_text_hash": "b2bd220c36ac86f65ba7355b5fe721ca888822d07ed144a3b939896452a2f1cd", + "variant_hash": "9b14753fac3e3d019023c2de8505e318147df6566da70ec10bff91e09f7046f3" + }, + { + "pair_index": 22, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-conversational_prose-36-unwatermarked", + "source_text_hash": "fdac0cd2fd8ebc7f4d2c4f21aace43d3ad6d9bf11d837e95d3fb6346f04561c2", + "transformed_text_hash": "09230c53357f5975bcb9ef3b84feffbb0fca8371c47af816e94b00b567762e4c", + "variant_hash": "a7d60b9025239cc51098967b545822bc02b4fc16ca01285ab24e59b3f70058d9" + }, + { + "pair_index": 23, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-conversational_prose-36-watermarked", + "source_text_hash": "1c711925c07461a390f3f6848bbed27848555f12092a17544ec97064953e3408", + "transformed_text_hash": "5bb3c2157098fea095ccf98011a54cb4908923174c83fe124627add3c8222d0d", + "variant_hash": "487367a761d46a89e04e1c77fa2c23cc6ef21d6f580dc0809bcc1ca4de8be5bc" + }, + { + "pair_index": 24, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-conversational_prose-37-unwatermarked", + "source_text_hash": "54f4dc986c86fdd126b8a4026997166be45dd5f67f31ed1b40c59704c9e04c45", + "transformed_text_hash": "914d0f213f0e2c5050ba1722a2bffb3dd9422f06b3a7746145c9a7b12421214e", + "variant_hash": "bc0710ab56e7bd7feaef35fb434e4cb93db947cf138832ba4114b8536dc6c416" + }, + { + "pair_index": 25, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-conversational_prose-37-watermarked", + "source_text_hash": "a4b79edc43e35ff68ea5b98bf0f2aa8cfe0c46b9e11abb4b7989c1c11db32601", + "transformed_text_hash": "bd2c89353617a5dbc9a26b931c4d9a5d3d7b9581443dade7a10e13635be4886f", + "variant_hash": "e20b9cd446c4f4c834c414d5124dd67e10a72f0dd73877250bbf5968de8bee24" + }, + { + "pair_index": 26, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-conversational_prose-38-unwatermarked", + "source_text_hash": "c5da8c3adf4cf67e5e65a8ac73c86b2ee5ad14fea3985dbea64a90cbd1716906", + "transformed_text_hash": "9f2b459c9cef0b32584008b66e4ec0d1ffa067ad52fa1210d2bfd7895dad6210", + "variant_hash": "d5edc2cdc64459f0316e4f2d01a861aea87d8f14e28a6a86c202c5bdf59676e8" + }, + { + "pair_index": 27, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-conversational_prose-38-watermarked", + "source_text_hash": "01cc3eb0a3f4234007ec2961ae4391d01bd7a0a322ba53427ea0a87aecc60c66", + "transformed_text_hash": "024173e995b2e51ca9311054fed1610c5787e1085b8ed87649e5e2ba8f5e61ff", + "variant_hash": "ff0dbbb5aa15ad315c679dc04b8f57c94128df3eddfe120bb0894e373c0fcd89" + }, + { + "pair_index": 28, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-conversational_prose-39-unwatermarked", + "source_text_hash": "3296acb17fac31c6486a6cbbe779cea9e321a5fdd21d93b83bfc93be2aecca83", + "transformed_text_hash": "ffb0a356f35679273bf9542da37a2def6a6c2dd13cc5f9eda5378cb92f971a63", + "variant_hash": "a5e0e4c1527eb9a6b09779c3491ec33ebe3be5a227f90c339846f5c8e499a391" + }, + { + "pair_index": 29, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-conversational_prose-39-watermarked", + "source_text_hash": "dfcd1303b5acb0410b50a005a5a43b0370f61e060715e71720bc3d26bf1d6ac4", + "transformed_text_hash": "b8b575549e963128f5ea4042d91a2bdd55a7d336cb279033e0decf5c164c42b3", + "variant_hash": "4c2cc73761a40c4a0f77b606109e8dc69ff61ea7127a50e6bc9d622489dc4201" + }, + { + "pair_index": 30, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-conversational_prose-40-unwatermarked", + "source_text_hash": "651810b18e48404c3e16d5c783dea4763647ab398dc1f9371a116eff9ccbdc2a", + "transformed_text_hash": "ba7c90cd2bf9a6dd16eac63ee8d7b3112e3d27276f0d08c7b3951c0c7405efbc", + "variant_hash": "765c053e85ae6e01b95f52f6a1db41f5ce2adc444605aaf29b738753c04e022c" + }, + { + "pair_index": 31, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-conversational_prose-40-watermarked", + "source_text_hash": "bca8b1bc83ddac15d0d4510292e10c20868ffeb9ae757c72b13b23a07d8c0902", + "transformed_text_hash": "400ed37ca6088e8a0bc3b792036d8918daa8e075298331fdf35dcfd5fa6a7ed4", + "variant_hash": "6bd293b9baa2fcfb1973458cf08d16fcdbe97c62e8dadb34aea41d9572811117" + }, + { + "pair_index": 32, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-general_explanatory-25-unwatermarked", + "source_text_hash": "64d4c84f0594b3f769e3608dd3592b62dd624eff3f8f6ba4fc044a67bc00ffb2", + "transformed_text_hash": "50207fd93e040bb5217affd6fe3472156cd7dc8001ae78c5a6f09a97b2fa0a45", + "variant_hash": "f16d1145f5f2202e26be6c253f19f51eb4a14cc3aaa30c12fd33605354990568" + }, + { + "pair_index": 33, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-general_explanatory-25-watermarked", + "source_text_hash": "4e6f13e90961b4684822da2837b680520f1e6f29d6e9dc37f5c35c8343e3b533", + "transformed_text_hash": "bc655b9fb2a19d8ecae5a8747d0015967f2cae6ecb00d78580bfc2d4c6e1a04d", + "variant_hash": "b306bef89eda2c3fb5b57151d74c8a48acd97edc9c6b9c2838b62055b081b4da" + }, + { + "pair_index": 34, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-general_explanatory-26-unwatermarked", + "source_text_hash": "1b59f0e235ed66350dd80f401d9668effe33a8ad1ca99fcf0de41c5773d9b283", + "transformed_text_hash": "4a27147fd5820482fea92862efec34a21f7da3f1ab9d88dcdcc014558780e553", + "variant_hash": "bf891ebf6b08dead01f337fc86790a5817334e73c7cc9755b8681f7104b87127" + }, + { + "pair_index": 35, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-general_explanatory-26-watermarked", + "source_text_hash": "1ec2baf65366438dab096c8e76ba0f44d6a06246ed38628d783520982cea25e2", + "transformed_text_hash": "ad12189310f4e819d5a629ab43692ea49ab960364f7706bca1574481979f7b88", + "variant_hash": "d4f9f7b586cfdb65028792462b0344a770423c84fe060348d0aec416ce438a7b" + }, + { + "pair_index": 36, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-general_explanatory-27-unwatermarked", + "source_text_hash": "0cdb401a36f5ba3d546c6ea80120e82d109ee2f9ffbe275c6c06b82c27cbca0a", + "transformed_text_hash": "2608d812ab29c7a6de5b5310d0d39983723efb0b3b4dfe82259c4df58aaa11f4", + "variant_hash": "d3940cc0eb45b2e770720715965db3d08e7e3209db11d382f83bbfba73356de7" + }, + { + "pair_index": 37, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-general_explanatory-27-watermarked", + "source_text_hash": "5ac906a0035a7581c7e599203408e23eeb789e96ed328e185a8547fc279cecaf", + "transformed_text_hash": "3c17de9099ffbe2b88f4550ba40d1c97b25a5a4a2c09346cd5ea95294d5157cf", + "variant_hash": "186d2d0c6e441ca27d18fa72ebc94ac5445f82c9459ef8b56c16741df2e67f86" + }, + { + "pair_index": 38, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-general_explanatory-28-unwatermarked", + "source_text_hash": "5f5c1958fc8fd7da24eec5d964bd3971ea5a227c76984829cd834842cd89792f", + "transformed_text_hash": "6af1734b5a7844286d1618eec9e3456f702d82e88b58a0d1563601ee5400a1b2", + "variant_hash": "296cd8250bb884f090deb3d003e116c495f906ac36ee7b58df29d0bc72a357bb" + }, + { + "pair_index": 39, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-general_explanatory-28-watermarked", + "source_text_hash": "ae531606f8dc91563b1eb72b11285b63b946c77580bbcfec51ef43bf527b218e", + "transformed_text_hash": "def9ced085beee4418db8d106577083e4ed5f04227312ba5586dad80d024cb47", + "variant_hash": "b575df1102e750272cc0dc60d518d04bc29a8836adac60d82967fc653ff6fabf" + }, + { + "pair_index": 40, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-general_explanatory-29-unwatermarked", + "source_text_hash": "f30a40a314e972eea4a8e3c3598ec5ed36f8d054d902d71a542e1b71d157e1f8", + "transformed_text_hash": "89052f64fea16f320ea1a0a47eebadf9c884f73351a6e9ef74ec5dc3a4646de0", + "variant_hash": "bd2eca7387e0733dc06c8acf7c345b632a3de34d6191d1badda9108537fff126" + }, + { + "pair_index": 41, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-general_explanatory-29-watermarked", + "source_text_hash": "6ae0c002c314555f07c733930aa59fded696c8a8a5a52734ebcdb04a8b9029f5", + "transformed_text_hash": "a2f92fbd0c7fa7dacb1b6df61fbbc2436bbb6ea1a8b64dc65b56c08059151c46", + "variant_hash": "ad995ed9c7a2081d7ad25818854acf23000b33a591b83e24d616dedfa655e4f1" + }, + { + "pair_index": 42, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-general_explanatory-30-unwatermarked", + "source_text_hash": "8a6ad16474327d7fcf33c22adb7b25854e17d196e37a59b120ffdbb95c72aded", + "transformed_text_hash": "8a6ad16474327d7fcf33c22adb7b25854e17d196e37a59b120ffdbb95c72aded", + "variant_hash": "c231c6e155815752f5c2939d27f3f9eb027a62e53399ac79c1a4cb03ae8b9f7b" + }, + { + "pair_index": 43, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-general_explanatory-30-watermarked", + "source_text_hash": "400b0eed4dffb52b8ca99d304a4032b83f6a3e5be8d42f299fde80dfe930e976", + "transformed_text_hash": "f8e4d1a4929fabd3197dac513bda669e15ea8b1feae8acb58fb194d62dca73e0", + "variant_hash": "7baf4df9ec6faa148b8fc63615e2c6e9335f78e51114aad1434a0835baef6980" + }, + { + "pair_index": 44, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-general_explanatory-31-unwatermarked", + "source_text_hash": "54496cefaa027f97af0732455f7fb4c962f719c2b89c4c8c1bd5a5841c102efc", + "transformed_text_hash": "0e950f725b8fe728236bad7d6c8466f2d1284ee7423995ffb5155950e0adbcda", + "variant_hash": "016cd11dbb5267b6c25204e7b0bf4e84da562d12907983bfc25e6a8979eb0e58" + }, + { + "pair_index": 45, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-general_explanatory-31-watermarked", + "source_text_hash": "c3b77d72c89575514a45f691a3374ddb07a8972a9c4fa07f011a7768ed8ccdf3", + "transformed_text_hash": "29340c2a00db97545bad8d833ec545e087189eaa036ebb6a92b4e4b245c5f5d0", + "variant_hash": "ee53839c17d26e69476b76d6aec1e49e0c55c2800b37f52e6c374dbe23c8ceab" + }, + { + "pair_index": 46, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-general_explanatory-32-unwatermarked", + "source_text_hash": "1b53b9dc18ed374a92992304a3a61740fe8d91b62a64f65695445e354070d768", + "transformed_text_hash": "5d3fdfaeac059449d67939126b7be653a5e23eb0494009f68538c71747afb54f", + "variant_hash": "07f7c6e82b1ed69b96fb4685a99ded89c96e143e85e921a58385fdc0277a2990" + }, + { + "pair_index": 47, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-general_explanatory-32-watermarked", + "source_text_hash": "79e949d224cce0d6b94e7f03595ca7a3e6a3baf9d5e7c43fbd249c42a30c911d", + "transformed_text_hash": "a0c18d5c512e5dad95f4efadd57b33ac4e320d428cc014e7b92015dd5d2fe2b5", + "variant_hash": "18efa2d538f9ec99e022c15adceec04ee589cf2f444b6d6de9cf53f3400556d4" + }, + { + "pair_index": 48, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-general_explanatory-33-unwatermarked", + "source_text_hash": "cc95dca9ee838fc79fae57e7d77212150ed21e8f287cee06a3d338ae51db6ecb", + "transformed_text_hash": "60116f36ee74e829ecea5d2f0c936fdf36cb9984d9365d9e189713d536a29777", + "variant_hash": "cbca89241095bf0b24528161a6e153bca3801b80a6006dc0b00ed0fc9f1287c0" + }, + { + "pair_index": 49, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-general_explanatory-33-watermarked", + "source_text_hash": "cafa503fbe6d77226ca63debacc1c0649b1b59a4d9f0425fd98e0ed91600305c", + "transformed_text_hash": "3863b7c5596772d81451bf745024305aa7eccc221caeb6275d43b0304e44353e", + "variant_hash": "e6fa4b204deae24f70b449098003ecd63928f40c69509ad09574a49056e228ce" + }, + { + "pair_index": 50, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-general_explanatory-34-unwatermarked", + "source_text_hash": "5cab5bf4fb05aac31ad15865fb99fc35c57992693896c8ed0e492ed64bcaeed5", + "transformed_text_hash": "2543bbff9f611b9a8303639399ab2305091a819f0fb6b5519586c494363ea8f6", + "variant_hash": "fc9e0d1025953737b93c52f2d7f6c4bab1b7d185d1747c910e94ddc23837092e" + }, + { + "pair_index": 51, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-general_explanatory-34-watermarked", + "source_text_hash": "928bc10e2d6c5a41d1f561bbf64d4ba765c024ddcf2dd21696743eb7c6c19c2e", + "transformed_text_hash": "e70be8ed37005556570c3b3fbcf50d532c0c47bea4a266018b95dd4674d80bfe", + "variant_hash": "ca29157c5d745375943f4203e451e1edfd012e39cb97ca06bbcf8d1e4dcff5bc" + }, + { + "pair_index": 52, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-general_explanatory-35-unwatermarked", + "source_text_hash": "9140098da50315235a8d1ec094885b4c51c27ae4f56ef062267dd4075028b335", + "transformed_text_hash": "c44119142a78e6330c3f4abfe0d05fb6b8c79eb50b976c8c413711c2639f8f23", + "variant_hash": "725ce569010e753bf70a77e1a8e59e8b55b58a76ac8891b68e7c006e83bc3e9d" + }, + { + "pair_index": 53, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-general_explanatory-35-watermarked", + "source_text_hash": "9c9df6d690837da7ff7903d2f6016e3adfb48b644cc6ed6fc99f28a8d413ddff", + "transformed_text_hash": "280248d6e85d80254d0f8084b66a0f6f58d58886b9ab63826ea82a58c9c38694", + "variant_hash": "28b91e57e3271b4729ce1d1cc2b48842eb7fb09a55043e0e13e9ccf4069b1e72" + }, + { + "pair_index": 54, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-general_explanatory-36-unwatermarked", + "source_text_hash": "3cfe4787b7773091d142f30ea2661345c71d82f8256ceddf758e4eed344da680", + "transformed_text_hash": "5620e50a96ee709cf2dd6eb4de33f386f39cea121179b79a0dcbd3ca0fe22b0b", + "variant_hash": "ed793f0d57e558535d9b8796d3921da0631c6e30cfd06a83bb9ab7f11c7d8bf7" + }, + { + "pair_index": 55, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-general_explanatory-36-watermarked", + "source_text_hash": "c9200bf5a1a801bd60934e04ce7f99dc6a5a1e6948fa27ad26cbe6f9099f548a", + "transformed_text_hash": "0290cd74b23b28cedd2abd043970d6024723436cfa31e3451fd9867c42dd3dc0", + "variant_hash": "2ce80a7da23e10b35499f1abdadc0c2ced2730832e7cc08c12f2912299212a35" + }, + { + "pair_index": 56, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-general_explanatory-37-unwatermarked", + "source_text_hash": "9cf92f6b6852ef462b22958a885a292a42546f59948878147c83cf84bcd3ea05", + "transformed_text_hash": "2d39dddf69d787ce225420434182e3917251f8f37e2ee331ee4f0c83d97f3138", + "variant_hash": "ff5b754012337b21320b6beb2a47e1785bcca543019ff354f3beed334a6959ed" + }, + { + "pair_index": 57, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-general_explanatory-37-watermarked", + "source_text_hash": "407d05e67bada2a54e52fe3eea2b78cdf5ef6f173b21246c02304c2e537c08e1", + "transformed_text_hash": "83770906e4d47e4da910572a1b2e52594ce22bc2f1f887fd46ea9fd47c138434", + "variant_hash": "faa90c8ca5ca2151d84768b27759df313df46ce4d65a906d9876356d4c5e617d" + }, + { + "pair_index": 58, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-general_explanatory-38-unwatermarked", + "source_text_hash": "144daf8835cedbab9c52ec0d8a8f45623194d30365751389ef367d702785e375", + "transformed_text_hash": "bbc42635a077fbcf69f52cdc0490d691eee0b98be5449f82f9d4879598d696e3", + "variant_hash": "853b93c14f568a02959d113b720f07764bd8a81f24dac725f3af3f79c9b8cd1f" + }, + { + "pair_index": 59, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-general_explanatory-38-watermarked", + "source_text_hash": "0f402676d7a30dab8744ccee6aa84766020eaff1aea4f8ea23a8f05a0e6da17e", + "transformed_text_hash": "fffd924a463e71566be8b31ab852030e21d01da0538e55310111b6d888c803cd", + "variant_hash": "cb645278241df8a09d89787106274562611d9182eeaa068a8525a06bc89c92f9" + }, + { + "pair_index": 60, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-general_explanatory-39-unwatermarked", + "source_text_hash": "3073de9328a5c9fd78ee05c6dc76b9bfde508190c95aeaecd8d157a7fd2534e5", + "transformed_text_hash": "6764b5f671a274196a9b8bb5254275b987a17d35650d57f0178a371c461e4482", + "variant_hash": "7dc11514fc89fdf1f5f01beb7981d6eff89fbc0e5ab5e01323e9dbcae86a04a5" + }, + { + "pair_index": 61, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-general_explanatory-39-watermarked", + "source_text_hash": "9da014e543010c6fa0ee2ab4198d5122949f0e801eaf0c96716846a5deb580ee", + "transformed_text_hash": "9c7d9afaac0149a70b5e3887300f366058e3f0b68b672df18bfd6e0b004dd588", + "variant_hash": "b2d9ea80061143310277f8298a5b6caccf77bd61a3224eb4e34ebbe765fb34b2" + }, + { + "pair_index": 62, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-general_explanatory-40-unwatermarked", + "source_text_hash": "31ea5a7c73690ffe0f2f589311ecdeb5cbecac1654f4a16c68a6c3c688b629bf", + "transformed_text_hash": "a5b36ae4e1eac579ad5e6b99b390e2a9fd71852f35b81dcd453646f284ceecd1", + "variant_hash": "411b8d1331e205d6d24d4aff75b85eb3c23440922130f85fd8fd5dd70e3f1b3a" + }, + { + "pair_index": 63, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-general_explanatory-40-watermarked", + "source_text_hash": "edc97fd1bb32d49384a48c7ebdff0ffbe97479a7308b5d2ab6c0389de3e28f05", + "transformed_text_hash": "1d771aca57968fb446b586455b538a574261f7b3ea4b9ea53fb3ab6a0ca9dbf8", + "variant_hash": "d2bdc1037cbab19bb9f67a2fc5af3539bc96e812505437a20da2c64b69b88206" + }, + { + "pair_index": 64, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-structured_instructional-25-unwatermarked", + "source_text_hash": "5bb4745b09af9bf63338822a5689b1ae8fcc27e1721f1d4d37530544c8dc4fef", + "transformed_text_hash": "6c0f671e1100abec01b4b39d6177b9a058e08ae79457c78479e3a49321e4395b", + "variant_hash": "6872300b27365a75ed11f29c0bf39f660382511e7132ed40f75ea0ae12536943" + }, + { + "pair_index": 65, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-structured_instructional-25-watermarked", + "source_text_hash": "aed5ad7a77dca6acd4d6378a28344e57ae06b0e6a6419bf8e6a2435740a01d1a", + "transformed_text_hash": "2b668f53abc034c37713ef25d666f497b5348c92b73e7ede462aff7719817232", + "variant_hash": "011999063cb51475ab8520b3bc90be6afff8f5314419770523a58b8f1a6b0834" + }, + { + "pair_index": 66, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-structured_instructional-26-unwatermarked", + "source_text_hash": "d1116a86430404f4bc1c3b596e7f51cd190073b68bd9e23b6fa4f6e6db64d59a", + "transformed_text_hash": "bb59d191e08819be8abd424eb247d49868328cc9b64891a533e59004ed4ff70d", + "variant_hash": "81dac4291c33931df4cdef2811c9154951b30153c80201de2a04994b9f8c0313" + }, + { + "pair_index": 67, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-structured_instructional-26-watermarked", + "source_text_hash": "384904d576e4a71c0c5c59684872d84d39bb804cc73e5a74b197556f197db9c3", + "transformed_text_hash": "175de2f48ea0f1c3ed75c40bfb118f6199760b005001b2842fd06ada18c84a3d", + "variant_hash": "bdd9c8967d2b1b810dd6e8d088cf0985832483ca390ceb11ff7edc885f17de16" + }, + { + "pair_index": 68, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-structured_instructional-27-unwatermarked", + "source_text_hash": "780d4ec4b4fee5cb5225d151b98cc16ab1d8a2341c4b750e8e59df3489268b34", + "transformed_text_hash": "6bbf930cfa73c477fa2b1a519961dbcffe8632599bdc8fcc8ac69b7e958d39d4", + "variant_hash": "3845ac3b2cd3c9445f29602c2bd618648ebca883601ee0875f3653d15a5680b4" + }, + { + "pair_index": 69, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-structured_instructional-27-watermarked", + "source_text_hash": "22c734e69ad9807003e7c22e7b26bcf829bc4e153f43952059db41509ac4015b", + "transformed_text_hash": "b9fd26e80a1381363b6ee752503f2d11ecacde7d910ebf27ce6e61431d16d1d9", + "variant_hash": "c3a4eb62f960b08005784001d5b1d6f3fcc3bc33abb1089249e2ed20657bb6ab" + }, + { + "pair_index": 70, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-structured_instructional-28-unwatermarked", + "source_text_hash": "bb1c9bc74a791f29cf45dd4662d763e4f53b1cf9fd4c63254c6e950e4ba83340", + "transformed_text_hash": "0520a858ee4db5292bded1b87b0073e3dd1fa90c9f25f337fe22555e63117bbf", + "variant_hash": "0240ff026ab0a011afd060c7697d4cef89e288b91e9186a41e45a002ecec9f0d" + }, + { + "pair_index": 71, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-structured_instructional-28-watermarked", + "source_text_hash": "0bc2d9d51e8647d381f6477dec495bba3f49dd9fa8f43c61a836d8f2381d31ba", + "transformed_text_hash": "752e910b14ad9fe2286ae0cd97408f9b5a8157c21362a8f27edd266f3abfd67a", + "variant_hash": "bb45a4504988755b7daabed9be62e3e6e30ea53c8aa521d5770ca45b19a81aa5" + }, + { + "pair_index": 72, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-structured_instructional-29-unwatermarked", + "source_text_hash": "4c1786c1847c73a9a031c3db02e3b48e3e4ef1c8b3c1e851d202e4e62379e526", + "transformed_text_hash": "4c1786c1847c73a9a031c3db02e3b48e3e4ef1c8b3c1e851d202e4e62379e526", + "variant_hash": "07eb67bd92cb0d915f43eaac7a0344804f294c3fb6efa04e81c6ff3fecc0194f" + }, + { + "pair_index": 73, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-structured_instructional-29-watermarked", + "source_text_hash": "fb5f9a9a4c6563df1cb26fa9f4eef2ab0f08ec10d25d8f54b4c17d2c53a3edf6", + "transformed_text_hash": "6270c695aff64675c80ee23e1de2c71c335350a5faf9858d544e37d27ff2bdfe", + "variant_hash": "08b4566319ec0ca23838e199cc88ff3ab7fe27f61fc671a25905be7f5e569132" + }, + { + "pair_index": 74, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-structured_instructional-30-unwatermarked", + "source_text_hash": "f9ab7744eab4c9d6f5eb9089ec9d8de486cea286d4206f40f9eeff7279554d2d", + "transformed_text_hash": "e803f90413aa18f49ab16d5f245dc5f17a790c923dca3bdc46e131815acaeaf4", + "variant_hash": "c8334aa4000a348586d69516b12cf65ae1ddae09e91ba32e85c86a6171c054dd" + }, + { + "pair_index": 75, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-structured_instructional-30-watermarked", + "source_text_hash": "6cded08a5270a56bd2255abfabba5c6cef36a2c18863e19048aacde2a6f6447f", + "transformed_text_hash": "28d9d88742e63ca9e4b5168b345987caac2a06cb9738b0313fe93c9104afcc40", + "variant_hash": "995c3323605dc3cd74a194b9ca0646f399e049e99a601d35ed4a88ba4fd7bb87" + }, + { + "pair_index": 76, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-structured_instructional-31-unwatermarked", + "source_text_hash": "c568eafe1b4e46d6b752120089b183a3578eb4784a7181eccd13dd545aa3b886", + "transformed_text_hash": "0c7123afee521c072eb5dc1b5ddb0c9559c13bd5cf8daf6a350bddeb783da5bf", + "variant_hash": "16d72fda07b382b545896de3a66e012bd7c55a82d80328210aabcf03790f156b" + }, + { + "pair_index": 77, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-structured_instructional-31-watermarked", + "source_text_hash": "c0c6611b034adf895d19a3959be3c8c89a56eec7e6c8691adf23e93064ecedd2", + "transformed_text_hash": "1163d4708154dd79b62925dea84251163734c5137749bb44e9de63de6d720709", + "variant_hash": "aee20f5d364a9430037662f91f70bc7d9a6347d82e8048868d1724ca0d7728cb" + }, + { + "pair_index": 78, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-structured_instructional-32-unwatermarked", + "source_text_hash": "98599963e22a674e4fce06f2ffb96e3a8a91986fbd1aea33265785cbb52efd53", + "transformed_text_hash": "a7b6d0643cd3a482d2106e214ed98fd7ef7f8198116c9e252ef6355ee2a06ad6", + "variant_hash": "3ebdc3bafd7795087ff032b901c78a50d01c5bd97dd348df28497d7ac7553f36" + }, + { + "pair_index": 79, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-structured_instructional-32-watermarked", + "source_text_hash": "90220632596b59b79311a5e66b8a12ce09833b1e004e09dd1cc363c15ce643a1", + "transformed_text_hash": "78d177606b19de94e532d35f2f1ce17ba9e2253b16370fc1288c1d3b88ed3c13", + "variant_hash": "52e615e5166840fab16b000a751a819abef26b5f9ced7a448714a1ddb134a15d" + }, + { + "pair_index": 80, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-structured_instructional-33-unwatermarked", + "source_text_hash": "7aebca0a135c5b4730034979c958f5427af2b513a6ad6ea68869abc01db28a48", + "transformed_text_hash": "9064e063cdd3784408b3dc30827e700a7ed28fa2bb707ff82712ead7fe7268b5", + "variant_hash": "57b26b52b80b797dfdd4bc88c2f150efc8d405f3de3ca9cf2bc1e1473c372f82" + }, + { + "pair_index": 81, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-structured_instructional-33-watermarked", + "source_text_hash": "b82736478591c828f943e9044326debc97e267f0850cc1dcf35e1fd23ff14233", + "transformed_text_hash": "51f1216a22fad29a682f2b65a494984ce2575621f4028c6b9e0563cad6735c1d", + "variant_hash": "fc233b169a152932847ff73937cf21dd56667b7044f99228f16640713e7637d1" + }, + { + "pair_index": 82, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-structured_instructional-34-unwatermarked", + "source_text_hash": "1cd7294e7a2f1575dc79db384a3eda1630cde1a9d8e963a8d6a8b5e4bdaedbb8", + "transformed_text_hash": "fa0a47954fc656fb4010bc1f705e76b85226044df49fd11aea288ed5a6fb0d91", + "variant_hash": "368a70dfdb12463c12b5c4e12027e08d4948bad0d622fd39e6b19e368d1ac69c" + }, + { + "pair_index": 83, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-structured_instructional-34-watermarked", + "source_text_hash": "346f243cf31760acf46d33c3e1d6499f08c8453b7b266baf04d44843962dee3e", + "transformed_text_hash": "a4a17dc944276aa5fecb97aab34cc42d6938ba307c32dfbafe7e6b218a13e894", + "variant_hash": "1544b0089f2ab1458e1f884654f315807186dfa1ddba252132331043e0a3b923" + }, + { + "pair_index": 84, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-structured_instructional-35-unwatermarked", + "source_text_hash": "616ca6891205a12645aee1377e57d4761e68a4085ecb5d79ce6e51e231c14820", + "transformed_text_hash": "35ce805479e7f4e71b38bfee4e5c7e18ec7747cc6a75b24543a8a103da7eb924", + "variant_hash": "0a91ba5dd48b375186e3f2f2349011a335a56e0a8316daac8defe9e787b63554" + }, + { + "pair_index": 85, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-structured_instructional-35-watermarked", + "source_text_hash": "39c01b143d77b384217262f7ef268d0af99128747b0968d2ec59bff6602451ac", + "transformed_text_hash": "4938376f8a2b3945d8c52401111552f0293e38b181d825dc72c4e4d8876aebb9", + "variant_hash": "4388dd0e9b61b397d8206b4ebdd414b142ca5cc058d87808a559f03f2b63e717" + }, + { + "pair_index": 86, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-structured_instructional-36-unwatermarked", + "source_text_hash": "87b32c5e6eeee104ae7615b0e394d8b59739642b33e91b9f9dead2f449d166e8", + "transformed_text_hash": "8d82a1faede58e5e7b4f7c4a278544601486b6f29dffc6f5092bc41c81193426", + "variant_hash": "4d20990cf8650d96ca87f6be6ad6353bc0d87d2a3f56db937ac1abf6c91433e3" + }, + { + "pair_index": 87, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-structured_instructional-36-watermarked", + "source_text_hash": "7055cc1d5e8e7f573aa2fb39a85727cc7c230165e7424652cd5c9b51b479004d", + "transformed_text_hash": "896de0c356b34c4d82444d64d80ddd3e9b9794736dda3ec8401cd4a8b4d85504", + "variant_hash": "04464e9c45bbae2d80976008f540029edf6ac532a0fb591c8fbe174e0c2cabbe" + }, + { + "pair_index": 88, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-structured_instructional-37-unwatermarked", + "source_text_hash": "492708f849be84ec13f86e072d85367f6a6bbb23b748d80f4336a9d0bd8e4dd8", + "transformed_text_hash": "4a094455390426aa52d63c6cd568f0cefeca9a5322b355bc8ed7869df06f6894", + "variant_hash": "8ff83965ddb1f349e0185118911f32b1adbddcb2ff0f39b8b9de9a7cf3f4c462" + }, + { + "pair_index": 89, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-structured_instructional-37-watermarked", + "source_text_hash": "7f97f4cb67b3e6be44ca6ab92093fc28c439e5f86a740c000a827800dea45848", + "transformed_text_hash": "2fc8cb1143d7cf522906b476da5d88fd1df2271ec2aa847a5e9d3ee089f4aed3", + "variant_hash": "81b89bd46c67a6fc68a87aa81a3b925cb37e17821651d3f2066a9fa6c2fbf792" + }, + { + "pair_index": 90, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-structured_instructional-38-unwatermarked", + "source_text_hash": "ec82880014f1772169c66e1710b5cbe8d8489ce826647d5b42533595e1624791", + "transformed_text_hash": "5880212659c686e2b1fc63f583e08fee2ec6e03065ee7819a236b31f5985a9cd", + "variant_hash": "5946863740842e31c7c7e48dc3a8e304c4472821c19889484f88ddb826d94fdd" + }, + { + "pair_index": 91, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-structured_instructional-38-watermarked", + "source_text_hash": "53546512fd56196dd6adb439ca7ae2c19d201444166b0d1ac1aa1aec03fedc71", + "transformed_text_hash": "08fa50bc0d0c977297209e4a7bd14b9dfab1b0a1cf098dff8316468a0beff1d9", + "variant_hash": "3408b7498c46a65d2aaa774967decb7bfe7d5e9591b6508acc150f24486937bb" + }, + { + "pair_index": 92, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-structured_instructional-39-unwatermarked", + "source_text_hash": "50831f201237267205f03267665fea3c53045d886482b688e08fbe044c1bda37", + "transformed_text_hash": "4c43c0a53a2b8d523c4acceaca6a40a63345702d8c34e81aa7b558aa5fbd4a64", + "variant_hash": "0432057ddccdad4f2063d8088a657255d8d7618753eef6a93fbcac1da3ab91dd" + }, + { + "pair_index": 93, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-structured_instructional-39-watermarked", + "source_text_hash": "3b75893d1bba9bfdc57c785305fa9e9f31603241adb81833c9f42a3d10adb47e", + "transformed_text_hash": "492b78bf6b28addccc579b2dced8589fac096f29333d9259568455cb013653ef", + "variant_hash": "c8d630e311a88692f2378de488605a9477d6d61e287f821fc8328b14c3965662" + }, + { + "pair_index": 94, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-structured_instructional-40-unwatermarked", + "source_text_hash": "916a39bcd1fc1ae4c7ef8db9c939c70dc94af86dc79777903c622575f6911c1d", + "transformed_text_hash": "55c3aa3b77dfcac5275f7d3cea3af491408fc177d86d16e54c5579ed568f0182", + "variant_hash": "1b32bb54fc095e90e5521cc3aa5034dd59f8ed444950471ec1fbd8b7d86b7c2a" + }, + { + "pair_index": 95, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-structured_instructional-40-watermarked", + "source_text_hash": "9e099e477443eccbf265b856c6c752b317710877277fa283b1107970bd005578", + "transformed_text_hash": "54cf379b5bdb32a201fa20480082265c43d4b68de9ffaf9cac9a53e82858500e", + "variant_hash": "c26ec27920c51f83e3f159b1e7b81acade353f37b0df59aab6374682317d5372" + }, + { + "pair_index": 96, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-technical_explanation-25-unwatermarked", + "source_text_hash": "693f010802429c5a391e636b64674554c9439333c59177c344cdfe34349c0f74", + "transformed_text_hash": "aaef6670e09bc69f64d62348a393df75288d4d47b59d2d0ed98e7c1b4edc9c5b", + "variant_hash": "fe1898db88ed2120a9a9ff1b50608554666a4289254bfa2797df67ec5225d593" + }, + { + "pair_index": 97, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-technical_explanation-25-watermarked", + "source_text_hash": "f87c451c1e5bb1e5fd0bbea9846e2aa5cb299ae1c05f281326e83ce01c036ff4", + "transformed_text_hash": "959f557639d722f4b525d3b58608df5c57ac42d1773e98a63fc183182bbf169a", + "variant_hash": "888d03f1cc5ecabae9a85c0a3f00ac15493ad0c94056546aae590990df3c6f41" + }, + { + "pair_index": 98, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-technical_explanation-26-unwatermarked", + "source_text_hash": "ae70a810b354f0bde1420f8001fb557ca029481730dfbafde19f53a5c35d99fc", + "transformed_text_hash": "4fa2f1a68d491ea4fcc043c5a031491c30cfaaa47282aaade7ee6c30dccc763b", + "variant_hash": "0edd197e9c67057ac7c53ec19af1bfad245d4ab421d82beb53301b60dcdd4f93" + }, + { + "pair_index": 99, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-technical_explanation-26-watermarked", + "source_text_hash": "d14b412f0fea10ce41459e8de3f86cc56409a655f1f1097e56de20b33ee3896c", + "transformed_text_hash": "397641946a36a3eccd89c7f14e4f24f8b1e8813732e54dc3479139d55abc19f2", + "variant_hash": "b4cd8a2c3a127484c12a531673bc540de7b809253257050eacfeee72fb8197ea" + }, + { + "pair_index": 100, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-technical_explanation-27-unwatermarked", + "source_text_hash": "02baf2d5bdff1c291b82ebf73778f8f8250fc03e194351e8974acc55c11b2cb6", + "transformed_text_hash": "9444826c237aa03c696343310f6865695f0a12d25a0fa0c65cd511f93f70ddd9", + "variant_hash": "73d51ee08b4ac5aa3bbdfa9d64c17af048eb362201655c3ad433d7dd44fc2140" + }, + { + "pair_index": 101, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-technical_explanation-27-watermarked", + "source_text_hash": "e58f4536dfdda95658b18e89c5af52c445bdf69eb6a3c8f8e85f4eb2491f38a6", + "transformed_text_hash": "1a3c5a2ec9fbd523501a9e6f23263935f3bdf177d98072750b7b69e34335e675", + "variant_hash": "0b9e979219188525a5a954ffb48dc8c8201e8de898049a8060282cd17b2b7a23" + }, + { + "pair_index": 102, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-technical_explanation-28-unwatermarked", + "source_text_hash": "188ced327bc1ad5840c2f6762b19421d68c21c0bc4d7e3aa39689b399883ab5f", + "transformed_text_hash": "499244c6351794071327e75d41bd90c3c21b1ad273a795aebf986464ec58624f", + "variant_hash": "e6db1e6902c15a806edd0fffb6c79c5febbf421508b43b28d9d3839a126c4809" + }, + { + "pair_index": 103, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-technical_explanation-28-watermarked", + "source_text_hash": "4be2a84a7c3887a5521578da9fd62129604e7cf720aa75032e139b3816303a5c", + "transformed_text_hash": "f6ead7823af5701780f87c5807cc46c4873a35ea537764c245af7ffb3d9da5ec", + "variant_hash": "20f750854949c29e1ef317c2a429cdfd0e260a21ba440a5a16e69c7b1c3dd1b8" + }, + { + "pair_index": 104, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-technical_explanation-29-unwatermarked", + "source_text_hash": "ea662d006f84898fdc34bfd3823f4df393af497201cbc450f2deb65992d4eea4", + "transformed_text_hash": "f61c4d1ec5cdb0a3ef2a7c37eae6ccc9e9f28738be64265845c1674f6baf899c", + "variant_hash": "a52e337b58a1f6d04f34cab6218172ac090e24907157dd6fea1485b6d4c352dd" + }, + { + "pair_index": 105, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-technical_explanation-29-watermarked", + "source_text_hash": "228c69e523de8eccd980812adabd554167f987b6abb214e6b40ed91d551adec3", + "transformed_text_hash": "081636737aec7087d647837bc64e87fb256833fc6d41253a7c3133889c04f148", + "variant_hash": "456ccca47a4229fee4496f5823df5496d909d842e2fdc039a761864b462b55e0" + }, + { + "pair_index": 106, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-technical_explanation-30-unwatermarked", + "source_text_hash": "3286e0c2bcd84d8134e7c5273c93095269f89094f363dceeb6d82ff42d1f7e1c", + "transformed_text_hash": "47b46fe53690c8adf7f1dc1f4d6cd7f5b261bd42b9c3405b1c24e7efce0368dd", + "variant_hash": "15f76efa2ad2cdee4275c352f2e7c0a34567cfd9963c287c5b5bd403d34784ab" + }, + { + "pair_index": 107, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-technical_explanation-30-watermarked", + "source_text_hash": "9cda66531dc58a651ecb8e78c96c153837a1de28d1ac542a59d5fe3fd61cecbd", + "transformed_text_hash": "e7d8d96faab7d8e25dde4188a3cbf5dd79272e61336c3c108c8cb8fb144c9a9b", + "variant_hash": "3276c61811d776993f6173e6d2eb473f25b01fefdebcf950916f9ec5cbd4d190" + }, + { + "pair_index": 108, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-technical_explanation-31-unwatermarked", + "source_text_hash": "c41537c178441c2538edbd08822e29876e913ade4859c390e75af53a55beef9f", + "transformed_text_hash": "2481d75afd75258382bfdaec27b3b5e66cb17bfe4be118e3c3f800709aea0947", + "variant_hash": "a0bdec7f6180d962aa779e5ae8636d8636632e92b5175a621942201416788a6b" + }, + { + "pair_index": 109, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-technical_explanation-31-watermarked", + "source_text_hash": "b31e060a90b80a18adaa3f0501c40d721d799fdf76904a4c1fc17cab665f881b", + "transformed_text_hash": "eb2ada38ba2c9aec3fa19b004e4219406569bff1acb07a082a744badb4de78b4", + "variant_hash": "397c70485121d4aeabf1474f17c319823c36d61bfa8ecccadce70d4281a2851b" + }, + { + "pair_index": 110, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-technical_explanation-32-unwatermarked", + "source_text_hash": "f833cdf060358c0f09302a8083369c8c3d4eb04260a126b4ebfb506566821b0a", + "transformed_text_hash": "6f23b7c3bbdf0b073fb3f4e6ca31e432d587391197ae7f966c2b524bacb64e6b", + "variant_hash": "9f0a1b945220a6c42ef8ccaa2a8ba89db8af51ea91d17511d325db3469759829" + }, + { + "pair_index": 111, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-technical_explanation-32-watermarked", + "source_text_hash": "1faa86cfebefea78b01ceaa69190983e135356e749a973197ea6375564712604", + "transformed_text_hash": "4152d424481435a8cd1a45e4a4d2dc73799c479afd40978d4a27f1fee5fec7ec", + "variant_hash": "bf4814068f60afee4e6b8c05a2899c0f6128d858f75c57bc37c64e8423a90537" + }, + { + "pair_index": 112, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-technical_explanation-33-unwatermarked", + "source_text_hash": "49ae9db1725a03387dcd66d63730725a14caf52e51755c5570905060b6a99261", + "transformed_text_hash": "34e0d47189cc27644a63ba44900000f7a485dcdfb60e26664549bf87e923a6d0", + "variant_hash": "09e5fbcf3847dbd1b9cd9c14571e77d990ab67772cd4d8e94d1be40f572f356d" + }, + { + "pair_index": 113, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-technical_explanation-33-watermarked", + "source_text_hash": "fa6f418c07b10618c0a06a0d2de7e42545cb8abcd25a4d03306e477bc04d5ac3", + "transformed_text_hash": "ff38758a226b18a02f88a7449b1ef32007c9ea0c0fc3ec8b0ae22bcfd5e43b6f", + "variant_hash": "54799da0fd837ba1d1be42c9c99966cc4b1b13d41c0f3e9ecc8df47cc80277a0" + }, + { + "pair_index": 114, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-technical_explanation-34-unwatermarked", + "source_text_hash": "d3e5788af180dd752d441df7622cf67a7eece306754395dd96a43ac281b8401c", + "transformed_text_hash": "a509ad063162e9d6ee88005d4445705407bfc4ed0a450c176821cc07c9ff451f", + "variant_hash": "8f7ade50cc99a1ab82e3b7abdae6d2feff2d3ae9827566cb3eb75b1934037d27" + }, + { + "pair_index": 115, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-technical_explanation-34-watermarked", + "source_text_hash": "c51081f9dc2f63d01d2175098348e6c15bad2df9fd72bc3e6a3d6d3583ca2c91", + "transformed_text_hash": "367dbefe2eee0e4829d055a74a4eb3be1888eed0e932443f15812f6e062848b7", + "variant_hash": "4002557d39567b84f48881c30574b7dec5031e98c9351ccf7bece266e4a6fbf1" + }, + { + "pair_index": 116, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-technical_explanation-35-unwatermarked", + "source_text_hash": "9f2d689d9bbe2b31eabe058ec8553869bd2fde6a7de2f542dc6e1b0f78abc11a", + "transformed_text_hash": "82fda0f4ef61b8de23de8fcebe8f9afc55ed3efa08c550a0646fd1da73deee62", + "variant_hash": "2443798d2c43a1683c2ede10c46dbf8fdf8de4589502ed09d177a975bf6ad8db" + }, + { + "pair_index": 117, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-technical_explanation-35-watermarked", + "source_text_hash": "abd419da92c1b2126d45ab71b4899a043d82ac97830adb1c3fa0f977ea21ea2e", + "transformed_text_hash": "58cfadd5fdfef177f70f71e9cec1dcfdbd45cb0c49db8a3f1bd0a1b32bb50822", + "variant_hash": "4f3f28fc7d5db623a1e97a39024d1d2c98ddd6965a54b9bee6d22197d636be49" + }, + { + "pair_index": 118, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-technical_explanation-36-unwatermarked", + "source_text_hash": "ca2aa16a76f084cd634af09eb3e24ea460c99ab0049074ef46076a318338002e", + "transformed_text_hash": "0c05ca66a89b7655b97bfac2a8faacf5ea3e0b22ca8f9c984080ca22191e6fca", + "variant_hash": "a9bf4e570588b7f5b50089180359a0e095da6271f2391c574cbf7d3a960d5bea" + }, + { + "pair_index": 119, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-technical_explanation-36-watermarked", + "source_text_hash": "6992b279f0cbf3f65a3e1901b62d1fa169aeefaa081727cd9d269fff3021703f", + "transformed_text_hash": "cf51a4e460de1e6026223d11dcc825fc2f8d9c1cba5206439ba333926414d0bb", + "variant_hash": "cc7b8f41c91c6a013cce81f4ffa3ad0a0ac7a8df9ea0f7e762d1298fe6f2ef4d" + }, + { + "pair_index": 120, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-technical_explanation-37-unwatermarked", + "source_text_hash": "d36bb6caba912e611280f86c8cae820e68abb5a60d3a5d56bebf9fe26cec8c8f", + "transformed_text_hash": "ad7500c4a6bb08e574cf4874c5dcd4113b5d84bb9cc9e30be5b8c156eb5babf5", + "variant_hash": "2641388b1372d7c8a6ce195f2810aee6be47c7f23f007edeb2008f141acd0b59" + }, + { + "pair_index": 121, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-technical_explanation-37-watermarked", + "source_text_hash": "39587cdb2aa7a627a7e225f21d489114690935132face2b29c8e9b3404726994", + "transformed_text_hash": "01f67e763dea7200fc8a1e028940bf96c278e95f838a0139bd4c5c24a6531d98", + "variant_hash": "fb7457d59c433b1c2cbf75e20a50c67de2f902b10c0929efb127fdb455a66ab4" + }, + { + "pair_index": 122, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-technical_explanation-38-unwatermarked", + "source_text_hash": "3e2b679d1e2212b9030cdcd91b7ce61c38830e9024b0805d91ec73681f3fe1ff", + "transformed_text_hash": "85f0c8c0da07ec26a6b57c0ab2854c5a9fd69ce1585a96f3192f6670b1d618ed", + "variant_hash": "af687c1247fa2b9aa7f6b5dc5ce6a5597417277e86d4ef5ebf36c8d450de0779" + }, + { + "pair_index": 123, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-technical_explanation-38-watermarked", + "source_text_hash": "357b1c2ed0969525e0cae41cbaa7088078f04dfae4e43d74dbbf35e46f9c7408", + "transformed_text_hash": "911dc8e239b776cc1ea96afa903775b5bb9ba2a611c26812913e508e91ab5ced", + "variant_hash": "a3f27bc5795783d60dea4240d49e6a2e7ce80407860890f96535e3b1f7c04780" + }, + { + "pair_index": 124, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-technical_explanation-39-unwatermarked", + "source_text_hash": "b4fe9932342201c1029deecce7db2826aee1d3571192a8994275bf1ff870090e", + "transformed_text_hash": "ee896095122ad888c4c852e4a0c5ffa126192e17920cae9f7a44e62e9fcd03f0", + "variant_hash": "a3356333bdcab518f3cbddd0874d47aa9012f552b28f03a132c8b091d57d35b7" + }, + { + "pair_index": 125, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-technical_explanation-39-watermarked", + "source_text_hash": "cfba3f9ecef0d7010e3e1c4e1ec951dce66abc3c1d085651cfcf7bf1f8b4698f", + "transformed_text_hash": "c81b1a32b0d49393c81132aec83eeed03d5b870f42b31b858c83817d030bf645", + "variant_hash": "691c0375cb38698e8671cce4f693cbc9f5bdbd84bd5cb9b1414a197f4ac1c3de" + }, + { + "pair_index": 126, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-technical_explanation-40-unwatermarked", + "source_text_hash": "3aebc2cc4f86b0a890776cc6524d14bcb05f10312d332fbc63f270613c6d0aee", + "transformed_text_hash": "75612102f54cd4746e2de89a286e2f9d7a613182921424b2f1a7c885047c7d1d", + "variant_hash": "52d6d409defc79409629f1385e09b53e21117058caabe150976e75a1f5b530bb" + }, + { + "pair_index": 127, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-technical_explanation-40-watermarked", + "source_text_hash": "2ef32ed901d397abf3f5fc6fdcc1ed7a89c365e275b87954720b7cc5d1cec976", + "transformed_text_hash": "bb0d656165a069013275dfddb05c768f2759799685ea89b6516a96e40ed1f94f", + "variant_hash": "0d6a0bb383ef277e4302acb5131f28d9f3e37c30cbd30862df8976b99508e756" + } + ], + "plan_hash": "5fd1146ba2f8c34bd3cce143154da7b144d0802ba5ebf8556737dc51c96efd77" +} From bc70e8ff72e8bb115c9caa226c89dca0299be874 Mon Sep 17 00:00:00 2001 From: Cyinh Date: Sat, 22 Aug 2026 22:38:23 -0400 Subject: [PATCH 15/30] Add measurement calibration corpus profile and generator --- fuckmark/corpus/measurement_calibration.py | 313 ++++++++++++++++++ .../tiny_dev_measurement_calibration_hf.py | 215 ++++++++++++ tests/test_measurement_calibration.py | 135 ++++++++ 3 files changed, 663 insertions(+) create mode 100644 fuckmark/corpus/measurement_calibration.py create mode 100644 fuckmark/tiny_dev_measurement_calibration_hf.py create mode 100644 tests/test_measurement_calibration.py diff --git a/fuckmark/corpus/measurement_calibration.py b/fuckmark/corpus/measurement_calibration.py new file mode 100644 index 00000000..16cb752d --- /dev/null +++ b/fuckmark/corpus/measurement_calibration.py @@ -0,0 +1,313 @@ +from __future__ import annotations + +from collections import defaultdict +from collections.abc import Sequence +from dataclasses import dataclass + +from .._validation import require_clean_string, require_int, require_sha256 +from ..hashing import sha256_json +from .prompt import PromptRecord +from .sample import CorpusSample +from .schema import CorpusDomain, CorpusSplit, KeySplit, WatermarkLabel +from .tiny_dev import TINY_DEV_DOMAINS, TINY_DEV_TARGET_LENGTH, TinyDevCorpusError + + +MEASUREMENT_CALIBRATION_CORPUS_VERSION = "measurement-calibration-corpus-v1" +MEASUREMENT_CALIBRATION_MANIFEST_VERSION = "measurement-calibration-manifest-v1" +MEASUREMENT_CALIBRATION_PROMPT_SOURCE_ID = "fuckmark-measurement-calibration-prompts-v1" +MEASUREMENT_CALIBRATION_TOPICS_PER_DOMAIN = 40 +MEASUREMENT_CALIBRATION_SEEDS_PER_PROMPT = 8 +MEASUREMENT_CALIBRATION_SAMPLE_COUNT = ( + MEASUREMENT_CALIBRATION_TOPICS_PER_DOMAIN + * len(TINY_DEV_DOMAINS) + * MEASUREMENT_CALIBRATION_SEEDS_PER_PROMPT +) +MEASUREMENT_CALIBRATION_NEGATIVE_COUNT = 1024 +MEASUREMENT_CALIBRATION_AUDIT_COUNT = ( + MEASUREMENT_CALIBRATION_SAMPLE_COUNT - MEASUREMENT_CALIBRATION_NEGATIVE_COUNT +) + + +_MEASUREMENT_CALIBRATION_TOPICS = ( + "artifact retention", + "audit trails", + "binary outcomes", + "blinding procedures", + "carryover effects", + "categorical drift", + "causal language", + "censoring rules", + "checklist discipline", + "codebook stability", + "cohort definitions", + "comparison validity", + "compute budgets", + "concept drift", + "consent records", + "contamination checks", + "data dictionaries", + "decision rules", + "definitional clarity", + "distribution shift", + "documentation debt", + "effect direction", + "endpoint choice", + "enumeration order", + "exclusion criteria", + "external validity", + "fair comparison", + "follow-up windows", + "group assignment", + "hypothesis registries", + "inclusion criteria", + "interrater agreement", + "label leakage", + "latent confounding", + "lookup tables", + "matching strategies", + "metadata quality", + "missingness patterns", + "operationalization", + "ownership boundaries", +) + + +def measurement_calibration_topics() -> tuple[str, ...]: + return _MEASUREMENT_CALIBRATION_TOPICS + + +@dataclass(frozen=True, slots=True) +class MeasurementCalibrationManifest: + algorithm_version: str + corpus_id: str + language: str + prompts: tuple[PromptRecord, ...] + samples: tuple[CorpusSample, ...] + prompt_manifest_hash: str + sample_manifest_hash: str + manifest_hash: str + + def __post_init__(self) -> None: + require_clean_string("algorithm_version", self.algorithm_version) + if self.algorithm_version != MEASUREMENT_CALIBRATION_MANIFEST_VERSION: + raise ValueError("unsupported measurement calibration manifest version") + require_clean_string("corpus_id", self.corpus_id) + require_clean_string("language", self.language) + if not isinstance(self.prompts, tuple) or not isinstance(self.samples, tuple): + raise TypeError("prompts and samples must be tuples") + _validate_measurement_calibration_manifest(self) + for name in ( + "prompt_manifest_hash", + "sample_manifest_hash", + "manifest_hash", + ): + require_sha256(name, getattr(self, name)) + payload = { + "algorithm_version": self.algorithm_version, + "corpus_id": self.corpus_id, + "language": self.language, + "prompt_manifest_hash": self.prompt_manifest_hash, + "sample_manifest_hash": self.sample_manifest_hash, + "sample_count": len(self.samples), + } + if self.manifest_hash != sha256_json(payload): + raise ValueError("manifest_hash does not match measurement calibration manifest") + + def payload(self) -> dict[str, object]: + return { + "algorithm_version": self.algorithm_version, + "corpus_id": self.corpus_id, + "language": self.language, + "prompts": self.prompts, + "samples": self.samples, + "prompt_manifest_hash": self.prompt_manifest_hash, + "sample_manifest_hash": self.sample_manifest_hash, + "manifest_hash": self.manifest_hash, + } + + +def build_measurement_calibration_manifest( + corpus_id: str, + prompts: Sequence[PromptRecord], + samples: Sequence[CorpusSample], + language: str = "en", +) -> MeasurementCalibrationManifest: + require_clean_string("corpus_id", corpus_id) + prompt_tuple = tuple(sorted(prompts, key=lambda value: value.prompt_id)) + sample_tuple = tuple(sorted(samples, key=lambda value: value.sample_id)) + prompt_manifest_hash = sha256_json(tuple(value.record_hash for value in prompt_tuple)) + sample_manifest_hash = sha256_json(tuple(value.record_hash for value in sample_tuple)) + payload = { + "algorithm_version": MEASUREMENT_CALIBRATION_MANIFEST_VERSION, + "corpus_id": corpus_id, + "language": language, + "prompt_manifest_hash": prompt_manifest_hash, + "sample_manifest_hash": sample_manifest_hash, + "sample_count": len(sample_tuple), + } + return MeasurementCalibrationManifest( + algorithm_version=MEASUREMENT_CALIBRATION_MANIFEST_VERSION, + corpus_id=corpus_id, + language=language, + prompts=prompt_tuple, + samples=sample_tuple, + prompt_manifest_hash=prompt_manifest_hash, + sample_manifest_hash=sample_manifest_hash, + manifest_hash=sha256_json(payload), + ) + + +@dataclass(frozen=True, slots=True) +class MeasurementCalibrationCorpus: + algorithm_version: str + manifest: MeasurementCalibrationManifest + target_length: int + topic_count_per_domain: int + seeds_per_prompt: int + sample_count: int + negative_count: int + audit_count: int + model_identity_hash: str + generation_matching_signature_hash: str + watermark_condition_hash: str + artifact_hash: str + + def __post_init__(self) -> None: + require_clean_string("algorithm_version", self.algorithm_version) + if self.algorithm_version != MEASUREMENT_CALIBRATION_CORPUS_VERSION: + raise ValueError("unsupported measurement calibration corpus version") + if not isinstance(self.manifest, MeasurementCalibrationManifest): + raise TypeError("manifest must be a MeasurementCalibrationManifest") + if self.target_length != TINY_DEV_TARGET_LENGTH: + raise ValueError("measurement calibration target length must be 64") + if self.topic_count_per_domain != MEASUREMENT_CALIBRATION_TOPICS_PER_DOMAIN: + raise ValueError("measurement calibration topic count is frozen") + if self.seeds_per_prompt != MEASUREMENT_CALIBRATION_SEEDS_PER_PROMPT: + raise ValueError("measurement calibration seed count is frozen") + if self.sample_count != MEASUREMENT_CALIBRATION_SAMPLE_COUNT: + raise ValueError("measurement calibration sample count is frozen") + if self.negative_count != MEASUREMENT_CALIBRATION_NEGATIVE_COUNT: + raise ValueError("measurement calibration negative count is frozen") + if self.audit_count != MEASUREMENT_CALIBRATION_AUDIT_COUNT: + raise ValueError("measurement calibration audit count is frozen") + for name in ( + "model_identity_hash", + "generation_matching_signature_hash", + "watermark_condition_hash", + "artifact_hash", + ): + require_sha256(name, getattr(self, name)) + _validate_measurement_calibration_manifest(self.manifest) + payload = self._payload() + if self.artifact_hash != sha256_json(payload): + raise ValueError("artifact_hash does not match measurement calibration corpus") + + def _payload(self) -> dict[str, object]: + return { + "algorithm_version": self.algorithm_version, + "manifest_hash": self.manifest.manifest_hash, + "target_length": self.target_length, + "topic_count_per_domain": self.topic_count_per_domain, + "seeds_per_prompt": self.seeds_per_prompt, + "sample_count": self.sample_count, + "negative_count": self.negative_count, + "audit_count": self.audit_count, + "model_identity_hash": self.model_identity_hash, + "generation_matching_signature_hash": self.generation_matching_signature_hash, + "watermark_condition_hash": self.watermark_condition_hash, + } + + def calibration_samples(self) -> tuple[CorpusSample, ...]: + ordered = tuple(sorted(self.manifest.samples, key=lambda sample: sample.sample_id)) + return ordered[: self.negative_count] + + def audit_samples(self) -> tuple[CorpusSample, ...]: + ordered = tuple(sorted(self.manifest.samples, key=lambda sample: sample.sample_id)) + return ordered[self.negative_count :] + + +def _validate_measurement_calibration_manifest(manifest: MeasurementCalibrationManifest) -> None: + prompts = manifest.prompts + samples = manifest.samples + if len(samples) != MEASUREMENT_CALIBRATION_SAMPLE_COUNT: + raise TinyDevCorpusError( + f"measurement calibration corpus must contain exactly {MEASUREMENT_CALIBRATION_SAMPLE_COUNT} samples" + ) + prompt_ids = tuple(value.prompt_id for value in prompts) + if len(set(prompt_ids)) != len(prompt_ids): + raise TinyDevCorpusError("measurement calibration prompt IDs must be unique") + prompt_texts = tuple(value.text for value in prompts) + if len(set(prompt_texts)) != len(prompt_texts): + raise TinyDevCorpusError("measurement calibration prompt texts must be unique") + family_splits: dict[str, set[CorpusSplit]] = defaultdict(set) + for prompt in prompts: + family_splits[prompt.prompt_family_id].add(prompt.split) + if any(len(splits) != 1 for splits in family_splits.values()): + raise TinyDevCorpusError("measurement calibration prompt families must not cross splits") + if any(sample.split is not CorpusSplit.THRESHOLD_CALIBRATION for sample in samples): + raise TinyDevCorpusError("measurement calibration samples must use the calibration split") + if any(sample.label is not WatermarkLabel.UNWATERMARKED for sample in samples): + raise TinyDevCorpusError("measurement calibration samples must be unwatermarked") + if any(sample.watermark.key_split is not KeySplit.DEV for sample in samples): + raise TinyDevCorpusError("measurement calibration samples must record DEV key split") + if any(sample.target_length != TINY_DEV_TARGET_LENGTH for sample in samples): + raise TinyDevCorpusError("measurement calibration samples must target 64 continuation tokens") + domains = {sample.domain for sample in samples} + if domains != set(TINY_DEV_DOMAINS): + raise TinyDevCorpusError("measurement calibration must cover the frozen four-domain profile") + if len({sample.sample_id for sample in samples}) != len(samples): + raise TinyDevCorpusError("measurement calibration sample IDs must be unique") + if len({sample.text_sha256 for sample in samples}) != len(samples): + raise TinyDevCorpusError("measurement calibration texts must be unique") + if len({sample.generation_tokens.continuation_token_hash for sample in samples}) != len(samples): + raise TinyDevCorpusError("measurement calibration continuations must be unique") + prompt_by_id = {value.prompt_id: value for value in prompts} + for sample in samples: + prompt = prompt_by_id.get(sample.prompt_id) + if prompt is None: + raise TinyDevCorpusError("measurement calibration sample references an unknown prompt") + if sample.prompt_family_id != prompt.prompt_family_id or sample.domain is not prompt.domain: + raise TinyDevCorpusError("measurement calibration sample metadata does not match its prompt") + if len(set(prompt_by_id) - {sample.prompt_id for sample in samples}): + raise TinyDevCorpusError("measurement calibration manifest contains unused prompts") + + +def build_measurement_calibration_corpus( + corpus_id: str, + prompts: Sequence[PromptRecord], + samples: Sequence[CorpusSample], +) -> MeasurementCalibrationCorpus: + require_clean_string("corpus_id", corpus_id) + manifest = build_measurement_calibration_manifest(corpus_id, prompts, samples) + model_hashes = {sample.model.identity_hash for sample in manifest.samples} + generation_hashes = {sample.generation.matching_signature_hash for sample in manifest.samples} + watermark_hashes = {sample.watermark.condition_hash for sample in manifest.samples} + if len(model_hashes) != 1 or len(generation_hashes) != 1 or len(watermark_hashes) != 1: + raise TinyDevCorpusError("measurement calibration must use exactly one identity triple") + payload = { + "algorithm_version": MEASUREMENT_CALIBRATION_CORPUS_VERSION, + "manifest_hash": manifest.manifest_hash, + "target_length": TINY_DEV_TARGET_LENGTH, + "topic_count_per_domain": MEASUREMENT_CALIBRATION_TOPICS_PER_DOMAIN, + "seeds_per_prompt": MEASUREMENT_CALIBRATION_SEEDS_PER_PROMPT, + "sample_count": len(manifest.samples), + "negative_count": MEASUREMENT_CALIBRATION_NEGATIVE_COUNT, + "audit_count": MEASUREMENT_CALIBRATION_AUDIT_COUNT, + "model_identity_hash": next(iter(model_hashes)), + "generation_matching_signature_hash": next(iter(generation_hashes)), + "watermark_condition_hash": next(iter(watermark_hashes)), + } + return MeasurementCalibrationCorpus( + algorithm_version=MEASUREMENT_CALIBRATION_CORPUS_VERSION, + manifest=manifest, + target_length=TINY_DEV_TARGET_LENGTH, + topic_count_per_domain=MEASUREMENT_CALIBRATION_TOPICS_PER_DOMAIN, + seeds_per_prompt=MEASUREMENT_CALIBRATION_SEEDS_PER_PROMPT, + sample_count=len(manifest.samples), + negative_count=MEASUREMENT_CALIBRATION_NEGATIVE_COUNT, + audit_count=MEASUREMENT_CALIBRATION_AUDIT_COUNT, + model_identity_hash=next(iter(model_hashes)), + generation_matching_signature_hash=next(iter(generation_hashes)), + watermark_condition_hash=next(iter(watermark_hashes)), + artifact_hash=sha256_json(payload), + ) diff --git a/fuckmark/tiny_dev_measurement_calibration_hf.py b/fuckmark/tiny_dev_measurement_calibration_hf.py new file mode 100644 index 00000000..486051cd --- /dev/null +++ b/fuckmark/tiny_dev_measurement_calibration_hf.py @@ -0,0 +1,215 @@ +from __future__ import annotations + +import argparse +import sys +from pathlib import Path + +from .config import canonical_json_text +from .corpus import WatermarkLabel +from .corpus.measurement_calibration import ( + MEASUREMENT_CALIBRATION_PROMPT_SOURCE_ID, + MEASUREMENT_CALIBRATION_SEEDS_PER_PROMPT, + MEASUREMENT_CALIBRATION_SAMPLE_COUNT, + MEASUREMENT_CALIBRATION_TOPICS_PER_DOMAIN, + build_measurement_calibration_corpus, + measurement_calibration_topics, +) +from .corpus.prompt import PromptRecord +from .corpus.sample import CorpusSample +from .corpus.schema import CorpusSplit, MAX_GENERATION_SEED +from .corpus.tokenization import GenerationTokenRecord, TextOnlyTokenRecord +from .corpus.tiny_dev import TINY_DEV_TARGET_LENGTH +from .corpus.tiny_dev_generation import ( + TINY_DEV_PROMPT_LICENSE_ID, + TINY_DEV_PROMPT_PROVENANCE, + TINY_DEV_SEED_POLICY_ID, + TinyDevGenerationError, + _TINY_DEV_TEMPLATES, +) +from .hashing import sha256_json, sha256_text +from .tiny_dev_corpus_hf import DEFAULT_MODEL_ID, DEFAULT_MODEL_REVISION, HuggingFaceTinyDevBackend + + +MEASUREMENT_CALIBRATION_GENERATION_VERSION = "measurement-calibration-generation-v1" +MEASUREMENT_CALIBRATION_SEED_STRIDE = 16 +MEASUREMENT_CALIBRATION_MAX_ATTEMPTS = 16 +MEASUREMENT_CALIBRATION_TOPIC_OFFSET = 100 + + +def build_measurement_calibration_prompt_records() -> tuple[PromptRecord, ...]: + topics = measurement_calibration_topics() + if len(topics) != MEASUREMENT_CALIBRATION_TOPICS_PER_DOMAIN: + raise ValueError("measurement calibration topic pool must contain 40 topics") + source_hash = sha256_json( + { + "algorithm_version": MEASUREMENT_CALIBRATION_GENERATION_VERSION, + "source_id": MEASUREMENT_CALIBRATION_PROMPT_SOURCE_ID, + "license_id": TINY_DEV_PROMPT_LICENSE_ID, + "provenance": TINY_DEV_PROMPT_PROVENANCE, + "topics": topics, + "templates": tuple( + (domain.value, _TINY_DEV_TEMPLATES[domain]) + for domain in _TINY_DEV_TEMPLATES + ), + } + ) + records: list[PromptRecord] = [] + for domain in _TINY_DEV_TEMPLATES: + template = _TINY_DEV_TEMPLATES[domain] + for local_index, topic in enumerate(topics): + topic_index = MEASUREMENT_CALIBRATION_TOPIC_OFFSET + local_index + prompt_id = f"measurement-calibration-{domain.value}-{topic_index:02d}" + records.append( + PromptRecord.create( + prompt_id=prompt_id, + prompt_family_id=f"measurement-calibration-family-{domain.value}-{topic_index:02d}", + domain=domain, + split=CorpusSplit.THRESHOLD_CALIBRATION, + source_id=MEASUREMENT_CALIBRATION_PROMPT_SOURCE_ID, + source_hash=source_hash, + license_id=TINY_DEV_PROMPT_LICENSE_ID, + provenance=TINY_DEV_PROMPT_PROVENANCE, + text=template.format(topic=topic), + ) + ) + return tuple(sorted(records, key=lambda value: value.prompt_id)) + + +def _build_slot_sample( + *, + prompt, + slot: int, + generated, + backend, + seed: int, +) -> CorpusSample: + model = backend.model_identity + generation = backend.generation_parameters(seed) + if generation.seed != seed: + raise TinyDevGenerationError("backend generation parameters do not bind the requested seed") + if generation.seed_policy_id != TINY_DEV_SEED_POLICY_ID: + raise TinyDevGenerationError("backend seed policy does not match the frozen tiny development policy") + generation_tokens = GenerationTokenRecord.create( + input_token_ids=generated.input_token_ids, + attention_mask=generated.attention_mask, + generated_sequence_ids=generated.input_token_ids + generated.continuation_token_ids, + continuation_start_index=len(generated.input_token_ids), + continuation_token_ids=generated.continuation_token_ids, + prompt_length_after_templating=sum(generated.attention_mask), + model_tokenizer_identity_hash=model.identity_hash, + ) + text_only = None + if generated.text_only_token_ids is not None: + text_only = TextOnlyTokenRecord.create( + source_text_sha256=sha256_text(generated.text), + token_ids=generated.text_only_token_ids, + model_tokenizer_identity_hash=model.identity_hash, + ) + sample_id = f"{prompt.prompt_id}-s{slot:02d}-{WatermarkLabel.UNWATERMARKED.value}" + return CorpusSample.create( + sample_id=sample_id, + match_id=f"match-{sample_id}", + prompt_id=prompt.prompt_id, + prompt_family_id=prompt.prompt_family_id, + domain=prompt.domain, + split=prompt.split, + label=WatermarkLabel.UNWATERMARKED, + text=generated.text, + model=model, + generation=generation, + watermark=backend.watermark_condition, + target_length=TINY_DEV_TARGET_LENGTH, + generation_tokens=generation_tokens, + text_only_tokens=text_only, + ) + + +def build_real_measurement_calibration_corpus( + backend: HuggingFaceTinyDevBackend, + *, + corpus_id: str = "fuckmark-measurement-calibration-v1", + seed_base: int = 500000, +) -> MeasurementCalibrationCorpus: + if type(seed_base) is not int or seed_base < 0: + raise ValueError("seed_base must be a non-negative integer") + prompts = build_measurement_calibration_prompt_records() + slot_count = len(prompts) * MEASUREMENT_CALIBRATION_SEEDS_PER_PROMPT + if seed_base + slot_count * MEASUREMENT_CALIBRATION_SEED_STRIDE + MEASUREMENT_CALIBRATION_MAX_ATTEMPTS > MAX_GENERATION_SEED: + raise ValueError("measurement calibration seed schedule exceeds the 64-bit range") + + samples: list[object] = [] + seen_text_hashes: set[str] = set() + seen_token_hashes: set[str] = set() + slot_index = 0 + for prompt in prompts: + for slot in range(MEASUREMENT_CALIBRATION_SEEDS_PER_PROMPT): + slot_seed_base = seed_base + slot_index * MEASUREMENT_CALIBRATION_SEED_STRIDE + slot_index += 1 + accepted = False + for attempt in range(MEASUREMENT_CALIBRATION_MAX_ATTEMPTS): + seed = slot_seed_base + attempt + generated = backend.generate(prompt.text, seed, watermarked=False) + if len(generated.continuation_token_ids) != TINY_DEV_TARGET_LENGTH: + continue + text_hash = sha256_text(generated.text) + token_hash = sha256_json(generated.continuation_token_ids) + if text_hash in seen_text_hashes or token_hash in seen_token_hashes: + continue + samples.append( + _build_slot_sample( + prompt=prompt, + slot=slot, + generated=generated, + backend=backend, + seed=seed, + ) + ) + seen_text_hashes.add(text_hash) + seen_token_hashes.add(token_hash) + accepted = True + break + if not accepted: + raise RuntimeError( + f"could not produce an exact unique 64-token negative for {prompt.prompt_id} slot {slot}" + ) + if len(samples) != MEASUREMENT_CALIBRATION_SAMPLE_COUNT: + raise RuntimeError("measurement calibration sample count does not match the frozen profile") + return build_measurement_calibration_corpus(corpus_id, prompts, samples) + + +def _parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser(prog="fuckmark-tiny-dev-measurement-calibration-hf") + parser.add_argument("--model", default=DEFAULT_MODEL_ID) + parser.add_argument("--model-revision", default=DEFAULT_MODEL_REVISION) + parser.add_argument("--device", choices=("auto", "cpu", "cuda"), default="auto") + parser.add_argument("--seed-base", type=int, default=500000) + parser.add_argument("--temperature", type=float, default=0.8) + parser.add_argument("--top-k", type=int, default=50) + parser.add_argument("--top-p", type=float, default=0.95) + parser.add_argument("--json", type=Path, required=True) + return parser + + +def main(argv: list[str] | None = None) -> int: + args = _parser().parse_args(argv) + backend = HuggingFaceTinyDevBackend( + args.model, + args.model_revision, + device=args.device, + temperature=args.temperature, + top_k=args.top_k, + top_p=args.top_p, + ) + artifact = build_real_measurement_calibration_corpus(backend, seed_base=args.seed_base) + args.json.parent.mkdir(parents=True, exist_ok=True) + args.json.write_text(canonical_json_text(artifact) + "\n", encoding="utf-8") + sys.stdout.write(f"artifact_hash={artifact.artifact_hash}\n") + sys.stdout.write(f"manifest_hash={artifact.manifest.manifest_hash}\n") + sys.stdout.write(f"sample_count={len(artifact.manifest.samples)}\n") + sys.stdout.write(f"model_identity_hash={artifact.model_identity_hash}\n") + sys.stdout.write(f"json={args.json.as_posix()}\n") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/test_measurement_calibration.py b/tests/test_measurement_calibration.py new file mode 100644 index 00000000..db1719e4 --- /dev/null +++ b/tests/test_measurement_calibration.py @@ -0,0 +1,135 @@ +from __future__ import annotations + +import pytest + +from fuckmark.config import canonical_json_text +from fuckmark.corpus import ( + CorpusDomain, + CorpusSplit, + GenerationParameters, + KeySplit, + ModelTokenizerIdentity, + PaddingSide, + WatermarkCondition, + WatermarkLabel, +) +from fuckmark.corpus.measurement_calibration import ( + MEASUREMENT_CALIBRATION_AUDIT_COUNT, + MEASUREMENT_CALIBRATION_NEGATIVE_COUNT, + MEASUREMENT_CALIBRATION_SAMPLE_COUNT, + build_measurement_calibration_corpus, +) +from fuckmark.corpus.tiny_dev_generation import ( + TINY_DEV_SEED_POLICY_ID, + TinyDevGeneratedContinuation, +) +from fuckmark.hashing import sha256_json, sha256_text +from fuckmark.tiny_dev_measurement_calibration_hf import ( + build_measurement_calibration_prompt_records, + build_real_measurement_calibration_corpus, +) + + +class FakeCalibrationBackend: + def __init__(self) -> None: + self._model = ModelTokenizerIdentity.create( + model_id="example/calibration-model", + model_revision="a" * 40, + tokenizer_id="example/calibration-tokenizer", + tokenizer_revision="b" * 40, + chat_template_present=False, + chat_template_hash=sha256_text(""), + special_token_map_hash=sha256_json({"eos_token": "", "pad_token": ""}), + padding_side=PaddingSide.LEFT, + bos_token_id=None, + eos_token_id=2, + pad_token_id=0, + add_bos_token=False, + add_eos_token=False, + ) + self._watermark = WatermarkCondition.create( + sha256_json({"ngram_len": 5, "keys": (1, 2, 3)}), + KeySplit.DEV, + "dev-test-key", + ) + + @property + def model_identity(self) -> ModelTokenizerIdentity: + return self._model + + @property + def watermark_condition(self) -> WatermarkCondition: + return self._watermark + + def generation_parameters(self, seed: int) -> GenerationParameters: + return GenerationParameters.create( + seed=seed, + seed_policy_id=TINY_DEV_SEED_POLICY_ID, + temperature=0.8, + top_k=50, + top_p=0.95, + max_new_tokens=64, + do_sample=True, + dtype="float32", + device="cpu", + backend_id="fake-calibration", + backend_version="1", + ) + + def generate(self, prompt: str, seed: int, *, watermarked: bool) -> TinyDevGeneratedContinuation: + assert watermarked is False + prompt_token = 10 + (sum(prompt.encode("utf-8")) % 1000) + continuation = tuple( + 200_000 + ((seed * 977 + index) % 90_000) + for index in range(64) + ) + text = f"calibration negative for {sha256_text(prompt)[:12]} seed {seed}" + text_only = tuple(300_000 + ((seed * 31 + index) % 80_000) for index in range(16)) + return TinyDevGeneratedContinuation( + text=text, + input_token_ids=(prompt_token, prompt_token + 1), + attention_mask=(1, 1), + continuation_token_ids=continuation, + text_only_token_ids=text_only, + ) + + +def test_measurement_calibration_prompt_records_are_frozen_and_disjoint() -> None: + prompts = build_measurement_calibration_prompt_records() + assert len(prompts) == 160 + assert len({prompt.prompt_id for prompt in prompts}) == 160 + counts: dict[CorpusDomain, int] = {} + for prompt in prompts: + counts[prompt.domain] = counts.get(prompt.domain, 0) + 1 + assert set(counts.values()) == {40} + from fuckmark.corpus.tiny_dev_v3_generation import build_tiny_dev_v3_prompt_records + + prior_texts = { + prompt.text + for prompt in build_tiny_dev_v3_prompt_records(16) + } + assert not any(prompt.text in prior_texts for prompt in prompts) + + +def test_measurement_calibration_corpus_builds_with_frozen_split() -> None: + backend = FakeCalibrationBackend() + artifact = build_real_measurement_calibration_corpus(backend, seed_base=6000) + assert len(artifact.manifest.samples) == MEASUREMENT_CALIBRATION_SAMPLE_COUNT == 1280 + assert artifact.negative_count == MEASUREMENT_CALIBRATION_NEGATIVE_COUNT == 1024 + assert artifact.audit_count == MEASUREMENT_CALIBRATION_AUDIT_COUNT == 256 + calibration = artifact.calibration_samples() + audit = artifact.audit_samples() + assert len(calibration) == 1024 + assert len(audit) == 256 + assert not {sample.sample_id for sample in calibration} & {sample.sample_id for sample in audit} + assert all(sample.split is CorpusSplit.THRESHOLD_CALIBRATION for sample in artifact.manifest.samples) + assert all(sample.label is WatermarkLabel.UNWATERMARKED for sample in artifact.manifest.samples) + text = canonical_json_text(artifact) + "\n" + assert len(text) > 0 + + +def test_measurement_calibration_rejects_wrong_sample_count() -> None: + backend = FakeCalibrationBackend() + prompts = build_measurement_calibration_prompt_records() + with pytest.raises(Exception): + build_measurement_calibration_corpus("bad-corpus", prompts, ()) From 86d12b9a6f7febd9834766e3a3d8dbbe3c3688fa Mon Sep 17 00:00:00 2001 From: Cyinh Date: Sat, 22 Aug 2026 22:43:39 -0400 Subject: [PATCH 16/30] Add general word spacing rule, content-region profile, and coverage-hole analysis --- fuckmark/experiments/coverage_holes.py | 337 ++++++++++++++++++ fuckmark/experiments/effectiveness_plan.py | 4 + fuckmark/tiny_dev_effectiveness_plan_hf.py | 2 + fuckmark/tiny_dev_effectiveness_score_hf.py | 1 + .../tiny_dev_measurement_calibration_hf.py | 7 +- fuckmark/transforms/__init__.py | 8 + fuckmark/transforms/effectiveness_profile.py | 56 +++ fuckmark/transforms/registry.py | 2 +- fuckmark/transforms/rules.py | 80 +++++ 9 files changed, 495 insertions(+), 2 deletions(-) create mode 100644 fuckmark/experiments/coverage_holes.py diff --git a/fuckmark/experiments/coverage_holes.py b/fuckmark/experiments/coverage_holes.py new file mode 100644 index 00000000..ef7ccad7 --- /dev/null +++ b/fuckmark/experiments/coverage_holes.py @@ -0,0 +1,337 @@ +from __future__ import annotations + +from collections import Counter +from typing import Any + +from ..hashing import sha256_json +from ..transforms.protected import ProtectedSpanExtractor + + +COVERAGE_HOLE_ANALYSIS_VERSION = "coverage-hole-structure-v1" +NGRAM_LEN = 5 + +_FUNCTION_WORDS = frozenset( + ( + "is", + "of", + "to", + "the", + "and", + "in", + "for", + "on", + "with", + "as", + "from", + "that", + "this", + "was", + "are", + "be", + "can", + "will", + "have", + "has", + "not", + "but", + "or", + "by", + "at", + "it", + "we", + "you", + "they", + "an", + "he", + "she", + "his", + "her", + "its", + "our", + "their", + "them", + "him", + "us", + "me", + "my", + "your", + "if", + "when", + "where", + "which", + "who", + "what", + "how", + "why", + "because", + "so", + "than", + "then", + "there", + "here", + "all", + "any", + "each", + "more", + "most", + "some", + "such", + "only", + "very", + "just", + "also", + "into", + "about", + "over", + "under", + "after", + "before", + "between", + "during", + "through", + "without", + "while", + "do", + "does", + "been", + "being", + "would", + "could", + "should", + "may", + "might", + "must", + "were", + "one", + "even", + "still", + "often", + ) +) + + +def classify_word(word: str) -> str: + stripped = word.strip(".,;:!?\"'()") + if not stripped: + return "punctuation_or_empty" + if stripped.isdigit(): + return "numeric" + lowered = stripped.lower() + if lowered in _FUNCTION_WORDS: + return "function_word" + if stripped[0].isupper(): + return "capitalized_content" + if stripped.isalpha(): + return "lowercase_content" + return "other" + + +def _rule_family(rule_id: str) -> str: + if rule_id.startswith("contract"): + return "contraction" + if rule_id.startswith("surface"): + return "surface_spacing" + if rule_id.startswith("lexical"): + return "lexical" + if rule_id.startswith("syntax"): + return "syntax" + return "other" + + +def build_coverage_hole_row( + text: str, + token_ids: tuple[int, ...], + offsets: tuple[tuple[int, int], ...], + enumeration, + geometry, + variant, +) -> dict[str, object]: + token_count = len(token_ids) + observation_count = max(0, token_count - NGRAM_LEN + 1) + coverage = geometry.coverage_mapping() + window_hits = Counter() + for candidate_id, intervals in coverage.items(): + for interval in intervals: + for window in range(interval.start, interval.end_exclusive): + window_hits[window] += 1 + uncovered = tuple(w for w in range(observation_count) if window_hits[w] == 0) + covered_windows = observation_count - len(uncovered) + longest_run = 0 + run = 0 + for window in range(observation_count): + if window_hits[window] == 0: + run += 1 + longest_run = max(longest_run, run) + else: + run = 0 + uncovered_spans = [] + cursor = None + for window in uncovered: + if cursor is None or window != cursor[1]: + if cursor is not None: + uncovered_spans.append(cursor) + cursor = [window, window + 1] + else: + cursor[1] = window + 1 + if cursor is not None: + uncovered_spans.append(cursor) + lexical_categories: Counter[str] = Counter() + uncovered_word_samples: list[str] = [] + protected = ProtectedSpanExtractor().extract(text) + protected_char_spans = tuple((span.start, span.end) for span in protected.spans) + protected_intersections = 0 + for start_window, end_window in uncovered_spans: + char_start = offsets[start_window][0] if start_window < len(offsets) else len(text) + char_end = offsets[min(end_window - 1, len(offsets) - 1)][1] if end_window - 1 < len(offsets) else len(text) + snippet = text[char_start:char_end] + for token in snippet.split(): + lexical_categories[classify_word(token)] += 1 + if len(uncovered_word_samples) < 5 and snippet.strip(): + uncovered_word_samples.append(snippet.strip()[:60]) + for span_start, span_end in protected_char_spans: + if span_start < char_end and char_start < span_end: + protected_intersections += 1 + break + selected_ids = set(variant["selected_candidate_ids"]) + realized_windows = sum( + 1 + for window in range(observation_count) + if any( + interval.start <= window < interval.end_exclusive + for candidate_id, intervals in coverage.items() + if candidate_id in selected_ids + for interval in intervals + ) + ) + candidate_by_family = Counter( + _rule_family(candidate.rule_id) for candidate in enumeration.candidates + ) + redundancy = ( + sum(window_hits.values()) / covered_windows if covered_windows else 0.0 + ) + return { + "source_sample_id": variant["source_sample_id"], + "source_label": variant["source_label"], + "domain": variant["domain"], + "token_count": token_count, + "observation_window_count": observation_count, + "candidate_count": len(enumeration.candidates), + "candidate_count_by_family": dict(candidate_by_family), + "candidate_density_per_100_tokens": round(100.0 * len(enumeration.candidates) / token_count, 4) if token_count else 0.0, + "zero_candidate_source": len(enumeration.candidates) == 0, + "achievable_coverage_fraction": covered_windows / observation_count if observation_count else 0.0, + "realized_coverage_fraction": realized_windows / observation_count if observation_count else 0.0, + "uncovered_window_count": len(uncovered), + "longest_uncovered_run": longest_run, + "uncovered_span_count": len(uncovered_spans), + "uncovered_lexical_categories": dict(lexical_categories), + "unprotected_word_samples": uncovered_word_samples, + "protected_span_intersections": protected_intersections, + "candidate_redundancy_mean_multiplicity": round(redundancy, 4), + } + + +def build_coverage_hole_report(corpus: Any, tokenizer: Any, plan: dict[str, object]) -> dict[str, object]: + from ..experiments.effectiveness_plan import _attack_samples, _encode_with_offsets + from ..transforms import build_candidate_tokenizer_geometry + + registry = _registry_for_plan(plan) + profile_budgets = plan["budgets"] + sources = {sample.sample_id: sample for sample in _attack_samples(corpus)} + rows = [] + for variant in plan["variants"]: + if variant["requested_budget"] != profile_budgets[0]: + continue + source = sources[variant["source_sample_id"]] + token_ids, offsets = _encode_with_offsets(tokenizer, source) + enumeration = registry.enumerate(source.text) + geometry = build_candidate_tokenizer_geometry( + source.text, + enumeration, + token_ids, + offsets, + tokenizer_identity_hash=source.model.identity_hash, + ngram_len=NGRAM_LEN, + ) + rows.append( + build_coverage_hole_row( + source.text, + token_ids, + offsets, + enumeration, + geometry, + variant, + ) + ) + row_tuple = tuple(rows) + watermarked = tuple(row for row in row_tuple if row["source_label"] == "watermarked") + aggregated: Counter[str] = Counter() + for row in watermarked: + for category, count in row["uncovered_lexical_categories"].items(): + aggregated[category] += count + payload = { + "algorithm_version": COVERAGE_HOLE_ANALYSIS_VERSION, + "analysis_scope": "structural text-geometry diagnosis; no detector information used", + "plan_hash": plan["plan_hash"], + "ruleset_hash": plan["ruleset_hash"], + "rows": row_tuple, + "watermarked_uncovered_lexical_totals": dict(aggregated), + } + return {**payload, "artifact_hash": sha256_json(payload)} + + +def _registry_for_plan(plan: dict[str, object]): + from ..transforms import ( + KEY_BLIND_COVERAGE_COMPLETION_PROFILE_ID, + CONTENT_REGION_COVERAGE_PROFILE_ID, + content_region_coverage_transform_registry, + key_blind_coverage_completion_transform_registry, + key_blind_high_coverage_transform_registry, + ) + + profile_id = plan["profile_id"] + if profile_id == KEY_BLIND_COVERAGE_COMPLETION_PROFILE_ID: + return key_blind_coverage_completion_transform_registry() + if profile_id == CONTENT_REGION_COVERAGE_PROFILE_ID: + return content_region_coverage_transform_registry() + return key_blind_high_coverage_transform_registry() + + +def main(argv: list[str] | None = None) -> int: + import argparse + import json + import sys + from pathlib import Path + + parser = argparse.ArgumentParser(prog="fuckmark-coverage-holes") + parser.add_argument("--corpus-json", type=Path, required=True) + parser.add_argument("--plan-json", type=Path, required=True) + parser.add_argument("--json", type=Path, required=True) + args = parser.parse_args(argv) + from ..corpus import load_tiny_dev_corpus_by_version_json + + corpus = load_tiny_dev_corpus_by_version_json(args.corpus_json) + plan = json.loads(args.plan_json.read_text(encoding="utf-8")) + from transformers import AutoTokenizer + + tokenizer = AutoTokenizer.from_pretrained( + "openai-community/gpt2", + revision="607a30d783dfa663caf39e06633721c8d4cfcd7e", + use_fast=True, + padding_side="left", + ) + report = build_coverage_hole_report(corpus, tokenizer, plan) + args.json.parent.mkdir(parents=True, exist_ok=True) + args.json.write_text( + json.dumps(report, indent=2, sort_keys=True) + "\n", encoding="utf-8" + ) + sys.stdout.write(f"artifact_hash={report['artifact_hash']}\n") + sys.stdout.write(f"row_count={len(report['rows'])}\n") + sys.stdout.write( + f"uncovered_lexical_totals={report['watermarked_uncovered_lexical_totals']}\n" + ) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/fuckmark/experiments/effectiveness_plan.py b/fuckmark/experiments/effectiveness_plan.py index f1e7d81f..6cefda6c 100644 --- a/fuckmark/experiments/effectiveness_plan.py +++ b/fuckmark/experiments/effectiveness_plan.py @@ -14,10 +14,12 @@ ScheduleGeometryMode, SchedulePolicy, build_candidate_tokenizer_geometry, + content_region_coverage_transform_registry, key_blind_coverage_completion_transform_registry, key_blind_high_coverage_transform_registry, validate_effectiveness_profile_registry, validate_hard_invariants, + CONTENT_REGION_COVERAGE_PROFILE_ID, KEY_BLIND_COVERAGE_COMPLETION_PROFILE_ID, ) @@ -156,6 +158,8 @@ def _schedule_seed( def _registry_for_profile(profile: EffectivenessTransformProfile): if profile.profile_id == KEY_BLIND_COVERAGE_COMPLETION_PROFILE_ID: return key_blind_coverage_completion_transform_registry() + if profile.profile_id == CONTENT_REGION_COVERAGE_PROFILE_ID: + return content_region_coverage_transform_registry() return key_blind_high_coverage_transform_registry() diff --git a/fuckmark/tiny_dev_effectiveness_plan_hf.py b/fuckmark/tiny_dev_effectiveness_plan_hf.py index d638310d..b70c5fdb 100644 --- a/fuckmark/tiny_dev_effectiveness_plan_hf.py +++ b/fuckmark/tiny_dev_effectiveness_plan_hf.py @@ -21,6 +21,7 @@ KEY_BLIND_HIGH_COVERAGE_PROFILE_ID, KEY_BLIND_FULL_POOL_COVERAGE_PROFILE_ID, KEY_BLIND_COVERAGE_COMPLETION_PROFILE_ID, + CONTENT_REGION_COVERAGE_PROFILE_ID, resolve_effectiveness_profile, ) @@ -89,6 +90,7 @@ def _parser() -> argparse.ArgumentParser: KEY_BLIND_HIGH_COVERAGE_PROFILE_ID, KEY_BLIND_FULL_POOL_COVERAGE_PROFILE_ID, KEY_BLIND_COVERAGE_COMPLETION_PROFILE_ID, + CONTENT_REGION_COVERAGE_PROFILE_ID, ), ) parser.add_argument("--budgets", default="") diff --git a/fuckmark/tiny_dev_effectiveness_score_hf.py b/fuckmark/tiny_dev_effectiveness_score_hf.py index 3e18a0fa..f65f20e6 100644 --- a/fuckmark/tiny_dev_effectiveness_score_hf.py +++ b/fuckmark/tiny_dev_effectiveness_score_hf.py @@ -231,6 +231,7 @@ def _parser() -> argparse.ArgumentParser: KEY_BLIND_HIGH_COVERAGE_PROFILE_ID, KEY_BLIND_FULL_POOL_COVERAGE_PROFILE_ID, KEY_BLIND_COVERAGE_COMPLETION_PROFILE_ID, + CONTENT_REGION_COVERAGE_PROFILE_ID, ), ) parser.add_argument("--budgets", default="") diff --git a/fuckmark/tiny_dev_measurement_calibration_hf.py b/fuckmark/tiny_dev_measurement_calibration_hf.py index 486051cd..673db01a 100644 --- a/fuckmark/tiny_dev_measurement_calibration_hf.py +++ b/fuckmark/tiny_dev_measurement_calibration_hf.py @@ -148,9 +148,14 @@ def build_real_measurement_calibration_corpus( accepted = False for attempt in range(MEASUREMENT_CALIBRATION_MAX_ATTEMPTS): seed = slot_seed_base + attempt - generated = backend.generate(prompt.text, seed, watermarked=False) + try: + generated = backend.generate(prompt.text, seed, watermarked=False) + except RuntimeError: + continue if len(generated.continuation_token_ids) != TINY_DEV_TARGET_LENGTH: continue + if not generated.text.strip(): + continue text_hash = sha256_text(generated.text) token_hash = sha256_json(generated.continuation_token_ids) if text_hash in seen_text_hashes or token_hash in seen_token_hashes: diff --git a/fuckmark/transforms/__init__.py b/fuckmark/transforms/__init__.py index 5906818b..6c29206e 100644 --- a/fuckmark/transforms/__init__.py +++ b/fuckmark/transforms/__init__.py @@ -1,5 +1,7 @@ from .candidate_artifacts import CandidateConflict, CandidateEnumeration, CandidateRejection, TransformCandidate from .effectiveness_profile import ( + CONTENT_REGION_COVERAGE_PROFILE_ID, + CONTENT_REGION_COVERAGE_SEED_BASE, EFFECTIVENESS_PROFILE_ALGORITHM_VERSION, KEY_BLIND_COVERAGE_COMPLETION_PROFILE_ID, KEY_BLIND_COVERAGE_COMPLETION_SEED_BASE, @@ -10,6 +12,8 @@ KEY_BLIND_HIGH_COVERAGE_PROFILE_ID, KEY_BLIND_HIGH_COVERAGE_SEED_BASE, EffectivenessTransformProfile, + content_region_coverage_profile, + content_region_coverage_transform_registry, key_blind_coverage_completion_profile, key_blind_coverage_completion_transform_registry, key_blind_full_pool_coverage_profile, @@ -218,6 +222,10 @@ "KEY_BLIND_FULL_POOL_COVERAGE_SEED_BASE", "KEY_BLIND_COVERAGE_COMPLETION_PROFILE_ID", "KEY_BLIND_COVERAGE_COMPLETION_SEED_BASE", + "CONTENT_REGION_COVERAGE_PROFILE_ID", + "CONTENT_REGION_COVERAGE_SEED_BASE", + "content_region_coverage_profile", + "content_region_coverage_transform_registry", "key_blind_coverage_completion_profile", "key_blind_coverage_completion_transform_registry", "LEXICAL_RETOKENIZATION_FIXTURE_ALGORITHM_VERSION", diff --git a/fuckmark/transforms/effectiveness_profile.py b/fuckmark/transforms/effectiveness_profile.py index c1fe54c4..89290139 100644 --- a/fuckmark/transforms/effectiveness_profile.py +++ b/fuckmark/transforms/effectiveness_profile.py @@ -13,6 +13,7 @@ TRANSFORM_REGISTRY_ALGORITHM_VERSION, TransformRegistry, ) +from .rules import GeneralWordSpacingRule from .scheduler import CANDIDATE_SCHEDULER_ALGORITHM_VERSION from .surface_rules import coverage_completion_surface_rules, development_surface_rules from .syntax_rules import development_syntax_rules @@ -27,6 +28,8 @@ KEY_BLIND_FULL_POOL_COVERAGE_SEED_BASE = 1_130_000 KEY_BLIND_COVERAGE_COMPLETION_PROFILE_ID = "key-blind-coverage-completion-v2" KEY_BLIND_COVERAGE_COMPLETION_SEED_BASE = 1_140_000 +CONTENT_REGION_COVERAGE_PROFILE_ID = "content-region-coverage-v1" +CONTENT_REGION_COVERAGE_SEED_BASE = 1_150_000 @dataclass(frozen=True, slots=True) @@ -210,9 +213,62 @@ def resolve_effectiveness_profile( if not budgets: raise ValueError("the coverage completion profile requires explicit budgets") return key_blind_coverage_completion_profile(budgets) + if profile_id == CONTENT_REGION_COVERAGE_PROFILE_ID: + if not budgets: + raise ValueError("the content region coverage profile requires explicit budgets") + return content_region_coverage_profile(budgets) raise ValueError("unknown effectiveness profile id") +CONTENT_REGION_GENERAL_SPACING_RULE_ID = "surface-space-after-any-word" + + +def content_region_surface_rules(): + punctuation = tuple( + rule + for rule in development_surface_rules() + if not rule.source.isalpha() + ) + general = GeneralWordSpacingRule.create(CONTENT_REGION_GENERAL_SPACING_RULE_ID) + return (*punctuation, general) + + +def content_region_coverage_transform_registry( + identifiers: Sequence[str] = (), +) -> TransformRegistry: + return TransformRegistry( + ( + *development_forward_contraction_rules(), + *content_region_surface_rules(), + *development_lexical_rules(), + *development_syntax_rules(), + ), + identifiers, + ) + + +def content_region_coverage_profile( + budgets: tuple[int, ...], +) -> EffectivenessTransformProfile: + if not isinstance(budgets, tuple): + raise TypeError("budgets must be a tuple") + return EffectivenessTransformProfile.create( + profile_id=CONTENT_REGION_COVERAGE_PROFILE_ID, + budgets=budgets, + budget_unit="operation", + schedule_policy_id="COVERAGE_GREEDY_KEY_BLIND", + schedule_seed_base=CONTENT_REGION_COVERAGE_SEED_BASE, + replicate_count=1, + ngram_len=5, + ruleset_hash=content_region_coverage_transform_registry().ruleset_hash, + scientific_scope=( + "Frozen detector-blind and key-blind content-region coverage profile pairing " + "contractions and punctuation spacing with a general any-word spacing rule; " + "exploratory effectiveness evidence only and not release authorization" + ), + ) + + def key_blind_coverage_completion_transform_registry( identifiers: Sequence[str] = (), ) -> TransformRegistry: diff --git a/fuckmark/transforms/registry.py b/fuckmark/transforms/registry.py index eeea8202..033f10a3 100644 --- a/fuckmark/transforms/registry.py +++ b/fuckmark/transforms/registry.py @@ -106,7 +106,7 @@ def enumerate(self, text: str, user_ranges: Sequence[UserProtectedRange] = ()) - if isinstance(rule, (LexicalTemplateRule, SyntaxTemplateRule)) and not rule.precondition(text, start, end): rejections.append(_make_rejection(input_hash, rule, start, end, source_text, CandidateRejectionReason.PRECONDITION_FAILED)) continue - replacement = rule.replacement + replacement = rule.replacement_for(source_text) if rule.preserve_simple_case: replacement = _simple_case_replacement(source_text, replacement) if replacement is None: diff --git a/fuckmark/transforms/rules.py b/fuckmark/transforms/rules.py index 6a037a5c..445d35bb 100644 --- a/fuckmark/transforms/rules.py +++ b/fuckmark/transforms/rules.py @@ -111,6 +111,86 @@ def pattern(self) -> re.Pattern[str]: pattern = rf"{pattern}(?!\w)" return re.compile(pattern) + def replacement_for(self, source_text: str) -> str: + return self.replacement + + +GENERAL_WORD_SPACING_RULE_VERSION = "general-word-space-after-v1" + + +@dataclass(frozen=True, slots=True) +class GeneralWordSpacingRule(LiteralTransformRule): + def __post_init__(self) -> None: + require_clean_string("rule_id", self.rule_id) + require_clean_string("version", self.version) + if self.version != GENERAL_WORD_SPACING_RULE_VERSION: + raise ValueError("unsupported general word spacing rule version") + if self.family is not TransformFamily.ORTHOGRAPHY: + raise ValueError("general word spacing rules must use the orthography family") + if self.tier is not TransformTier.SURFACE: + raise ValueError("general word spacing rules must use tier 1 surface") + if self.source != "word" or self.replacement != "word ": + raise ValueError("general word spacing rules use the fixed word sentinel contract") + if "\n" in self.source or "\r" in self.source or "\n" in self.replacement or "\r" in self.replacement: + raise ValueError("general word spacing rules must stay single-line") + require_bool("whole_word", self.whole_word) + require_bool("preserve_simple_case", self.preserve_simple_case) + require_bool("block_all_caps", self.block_all_caps) + if self.whole_word or self.preserve_simple_case or self.block_all_caps: + raise ValueError("general word spacing rules use an exact case-sensitive boundary contract") + require_sha256("rule_hash", self.rule_hash) + if self.rule_hash != sha256_json(self._payload()): + raise ValueError("rule_hash does not match general word spacing rule") + + def _payload(self) -> dict[str, object]: + return { + "algorithm_version": RULE_ALGORITHM_VERSION, + "rule_id": self.rule_id, + "version": self.version, + "family": self.family.value, + "tier": self.tier.value, + "source": self.source, + "replacement": self.replacement, + "whole_word": self.whole_word, + "preserve_simple_case": self.preserve_simple_case, + "block_all_caps": self.block_all_caps, + "general_pattern": GENERAL_WORD_SPACING_RULE_VERSION, + } + + @classmethod + def create(cls, rule_id: str) -> "GeneralWordSpacingRule": + hashed = { + "algorithm_version": RULE_ALGORITHM_VERSION, + "rule_id": rule_id, + "version": GENERAL_WORD_SPACING_RULE_VERSION, + "family": TransformFamily.ORTHOGRAPHY.value, + "tier": TransformTier.SURFACE.value, + "source": "word", + "replacement": "word ", + "whole_word": False, + "preserve_simple_case": False, + "block_all_caps": False, + "general_pattern": GENERAL_WORD_SPACING_RULE_VERSION, + } + return cls( + rule_id=rule_id, + version=GENERAL_WORD_SPACING_RULE_VERSION, + family=TransformFamily.ORTHOGRAPHY, + tier=TransformTier.SURFACE, + source="word", + replacement="word ", + whole_word=False, + preserve_simple_case=False, + block_all_caps=False, + rule_hash=sha256_json(hashed), + ) + + def pattern(self) -> re.Pattern[str]: + return re.compile(r"(? str: + return source_text + " " + class SurfaceSpacingRule(LiteralTransformRule): def __post_init__(self) -> None: From f437238ccdd32657d2235efe246949b183491889 Mon Sep 17 00:00:00 2001 From: Cyinh Date: Sat, 22 Aug 2026 22:47:03 -0400 Subject: [PATCH 17/30] Widen general spacing pattern and record structural hole diagnosis --- fuckmark/transforms/registry.py | 3 ++- fuckmark/transforms/rules.py | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/fuckmark/transforms/registry.py b/fuckmark/transforms/registry.py index 033f10a3..9f689cc5 100644 --- a/fuckmark/transforms/registry.py +++ b/fuckmark/transforms/registry.py @@ -106,7 +106,8 @@ def enumerate(self, text: str, user_ranges: Sequence[UserProtectedRange] = ()) - if isinstance(rule, (LexicalTemplateRule, SyntaxTemplateRule)) and not rule.precondition(text, start, end): rejections.append(_make_rejection(input_hash, rule, start, end, source_text, CandidateRejectionReason.PRECONDITION_FAILED)) continue - replacement = rule.replacement_for(source_text) + replacement_getter = getattr(rule, "replacement_for", None) + replacement = replacement_getter(source_text) if callable(replacement_getter) else rule.replacement if rule.preserve_simple_case: replacement = _simple_case_replacement(source_text, replacement) if replacement is None: diff --git a/fuckmark/transforms/rules.py b/fuckmark/transforms/rules.py index 445d35bb..b59eb274 100644 --- a/fuckmark/transforms/rules.py +++ b/fuckmark/transforms/rules.py @@ -186,7 +186,7 @@ def create(cls, rule_id: str) -> "GeneralWordSpacingRule": ) def pattern(self) -> re.Pattern[str]: - return re.compile(r"(? str: return source_text + " " From 93d8e9f856a6a9dffce9a3244981946510848ab2 Mon Sep 17 00:00:00 2001 From: Cyinh Date: Sat, 22 Aug 2026 22:53:09 -0400 Subject: [PATCH 18/30] Add fixed measurement threshold protocol with order-statistic freeze and audit --- fuckmark/corpus/measurement_calibration_io.py | 115 +++++++++++++ fuckmark/experiments/measurement_threshold.py | 96 +++++++++++ fuckmark/tiny_dev_effectiveness_score_hf.py | 49 +++++- fuckmark/tiny_dev_measurement_threshold_hf.py | 73 +++++++++ tests/test_content_region_profile.py | 154 ++++++++++++++++++ tests/test_measurement_threshold.py | 81 +++++++++ 6 files changed, 561 insertions(+), 7 deletions(-) create mode 100644 fuckmark/corpus/measurement_calibration_io.py create mode 100644 fuckmark/experiments/measurement_threshold.py create mode 100644 fuckmark/tiny_dev_measurement_threshold_hf.py create mode 100644 tests/test_content_region_profile.py create mode 100644 tests/test_measurement_threshold.py diff --git a/fuckmark/corpus/measurement_calibration_io.py b/fuckmark/corpus/measurement_calibration_io.py new file mode 100644 index 00000000..9dde2d2a --- /dev/null +++ b/fuckmark/corpus/measurement_calibration_io.py @@ -0,0 +1,115 @@ +from __future__ import annotations + +import json +from pathlib import Path + +from ..config import canonical_json_text +from .prompt import PromptRecord +from .sample import CorpusSample +from .tiny_dev_io import ( + TinyDevCorpusJsonError, + _array, + _mapping, + _prompt, + _reject_constant, + _sample, + _unique_object, +) +from .measurement_calibration import ( + MEASUREMENT_CALIBRATION_MANIFEST_VERSION, + MeasurementCalibrationCorpus, + MeasurementCalibrationManifest, +) + + +def _manifest(value: object) -> MeasurementCalibrationManifest: + data = _mapping( + "manifest", + value, + ( + "algorithm_version", + "corpus_id", + "language", + "prompts", + "samples", + "prompt_manifest_hash", + "sample_manifest_hash", + "manifest_hash", + ), + ) + return MeasurementCalibrationManifest( + algorithm_version=data["algorithm_version"], + corpus_id=data["corpus_id"], + language=data["language"], + prompts=tuple(_prompt(item) for item in _array("manifest.prompts", data["prompts"])), + samples=tuple(_sample(item) for item in _array("manifest.samples", data["samples"])), + prompt_manifest_hash=data["prompt_manifest_hash"], + sample_manifest_hash=data["sample_manifest_hash"], + manifest_hash=data["manifest_hash"], + ) + + +def parse_measurement_calibration_corpus_json( + text: str, + *, + require_canonical: bool = True, +) -> MeasurementCalibrationCorpus: + if not isinstance(text, str): + raise TypeError("text must be a string") + try: + decoded = json.loads( + text, + object_pairs_hook=_unique_object, + parse_constant=_reject_constant, + ) + except (json.JSONDecodeError, UnicodeError) as error: + raise TinyDevCorpusJsonError("measurement calibration JSON is not valid JSON") from error + data = _mapping( + "artifact", + decoded, + ( + "algorithm_version", + "manifest", + "target_length", + "topic_count_per_domain", + "seeds_per_prompt", + "sample_count", + "negative_count", + "audit_count", + "model_identity_hash", + "generation_matching_signature_hash", + "watermark_condition_hash", + "artifact_hash", + ), + ) + artifact = MeasurementCalibrationCorpus( + algorithm_version=data["algorithm_version"], + manifest=_manifest(data["manifest"]), + target_length=data["target_length"], + topic_count_per_domain=data["topic_count_per_domain"], + seeds_per_prompt=data["seeds_per_prompt"], + sample_count=data["sample_count"], + negative_count=data["negative_count"], + audit_count=data["audit_count"], + model_identity_hash=data["model_identity_hash"], + generation_matching_signature_hash=data["generation_matching_signature_hash"], + watermark_condition_hash=data["watermark_condition_hash"], + artifact_hash=data["artifact_hash"], + ) + if require_canonical: + canonical = canonical_json_text(artifact) + if text not in (canonical, canonical + "\n"): + raise TinyDevCorpusJsonError("measurement calibration JSON is not canonical") + return artifact + + +def load_measurement_calibration_corpus_json( + path: str | Path, + *, + require_canonical: bool = True, +) -> MeasurementCalibrationCorpus: + file_path = Path(path) + return parse_measurement_calibration_corpus_json( + file_path.read_text(encoding="utf-8"), + require_canonical=require_canonical, + ) diff --git a/fuckmark/experiments/measurement_threshold.py b/fuckmark/experiments/measurement_threshold.py new file mode 100644 index 00000000..c3de6c36 --- /dev/null +++ b/fuckmark/experiments/measurement_threshold.py @@ -0,0 +1,96 @@ +from __future__ import annotations + +import math + +from ..detectors import CalibrationScope, ComparisonOperator, calibrate_detector +from ..detectors.calibration_statistics import exact_binomial_interval +from ..hashing import sha256_json +from ..tiny_dev_transform_hf import ( + TEXT_ONLY_CALIBRATION_POPULATION_ID, + TEXT_ONLY_LENGTH_POLICY_ID, + TEXT_ONLY_PROMPT_BOUNDARY_MODE, + TEXT_ONLY_TOKEN_TRACK, + _text_only_weighted_evidence, +) + + +MEASUREMENT_STABILITY_VERSION = "open-detector-measurement-stability-v1" +MEASUREMENT_STABILITY_TARGET_FPR = 0.01 +MEASUREMENT_STABILITY_COMPARISON = ">=" + + +def build_fixed_threshold_artifact( + calibration_corpus, + adapter, + *, + frozen_at_utc: str, +) -> dict[str, object]: + negatives = calibration_corpus.calibration_samples() + audit = calibration_corpus.audit_samples() + evidence = tuple(_text_only_weighted_evidence(sample, adapter) for sample in negatives) + audit_evidence = tuple(_text_only_weighted_evidence(sample, adapter) for sample in audit) + scope = CalibrationScope.create( + corpus_id=calibration_corpus.manifest.corpus_id, + population_id=TEXT_ONLY_CALIBRATION_POPULATION_ID, + length_policy_id=TEXT_ONLY_LENGTH_POLICY_ID, + token_track=TEXT_ONLY_TOKEN_TRACK, + prompt_boundary_mode=TEXT_ONLY_PROMPT_BOUNDARY_MODE, + ) + bundle = calibrate_detector( + evidence, + scope, + target_fprs=(0.05, 0.01), + comparison_operator=ComparisonOperator.GREATER_THAN_OR_EQUAL, + confidence_level=0.95, + ) + scores = sorted(value.raw_score for value in evidence) + count = len(scores) + order_statistic = count - math.floor(MEASUREMENT_STABILITY_TARGET_FPR * count) + threshold = scores[order_statistic - 1] + calibration_exceedances = sum(1 for value in scores if value >= threshold) + audit_scores = tuple(value.raw_score for value in audit_evidence) + audit_exceedances = sum(1 for value in audit_scores if value >= threshold) + interval = exact_binomial_interval(audit_exceedances, len(audit_scores)) + payload = { + "algorithm_version": MEASUREMENT_STABILITY_VERSION, + "frozen_at_utc": frozen_at_utc, + "detector_identity_hash": bundle.detector_identity.identity_hash, + "calibration_bundle_hash": bundle.bundle_hash, + "calibration_corpus_artifact_hash": calibration_corpus.artifact_hash, + "calibration_negative_count": count, + "audit_negative_count": len(audit_scores), + "target_fpr": MEASUREMENT_STABILITY_TARGET_FPR, + "comparison_operator": MEASUREMENT_STABILITY_COMPARISON, + "threshold_definition": ( + "ascending order statistic n - floor(target_fpr * n) of the calibration " + "negative weighted-mean scores (the 1014th of 1024 for target FPR 0.01); " + "detection when score >= threshold; float64 scores as computed; no tie rounding" + ), + "threshold_order_statistic": order_statistic, + "threshold": threshold, + "calibration_exceedances": calibration_exceedances, + "audit_exceedances": audit_exceedances, + "audit_realized_fpr": audit_exceedances / len(audit_scores), + "audit_fpr_confidence_interval_95": [interval.lower, interval.upper], + } + return {**payload, "artifact_hash": sha256_json(payload)} + + +def validate_fixed_threshold_artifact(artifact, detector_identity_hash: str) -> float: + if not isinstance(artifact, dict): + raise TypeError("fixed threshold artifact must be a mapping") + if artifact.get("algorithm_version") != MEASUREMENT_STABILITY_VERSION: + raise ValueError("unsupported fixed threshold artifact version") + if artifact.get("detector_identity_hash") != detector_identity_hash: + raise ValueError("fixed threshold detector identity does not match the scoring detector") + if artifact.get("comparison_operator") != MEASUREMENT_STABILITY_COMPARISON: + raise ValueError("fixed threshold comparison operator does not match the scoring rule") + if artifact.get("target_fpr") != MEASUREMENT_STABILITY_TARGET_FPR: + raise ValueError("fixed threshold target FPR does not match the scoring target") + expected = sha256_json({key: value for key, value in artifact.items() if key != "artifact_hash"}) + if artifact.get("artifact_hash") != expected: + raise ValueError("fixed threshold artifact hash does not replay") + threshold = artifact.get("threshold") + if not isinstance(threshold, float) or not 0.0 <= threshold <= 1.0: + raise ValueError("fixed threshold must be a probability float") + return threshold diff --git a/fuckmark/tiny_dev_effectiveness_score_hf.py b/fuckmark/tiny_dev_effectiveness_score_hf.py index f65f20e6..ca768b47 100644 --- a/fuckmark/tiny_dev_effectiveness_score_hf.py +++ b/fuckmark/tiny_dev_effectiveness_score_hf.py @@ -96,13 +96,31 @@ def score_key_blind_high_coverage_plan( adapter: HuggingFaceSynthIDAdapter, *, profile: EffectivenessTransformProfile | None = None, + fixed_threshold: Mapping[str, object] | None = None, ) -> dict[str, object]: active_profile = profile if profile is not None else KEY_BLIND_HIGH_COVERAGE_PROFILE validate_key_blind_high_coverage_plan(plan, corpus, active_profile) if adapter.ngram_len != active_profile.ngram_len: raise ValueError("detector ngram length does not match the frozen effectiveness profile") calibration = _text_only_calibration(corpus, adapter) - threshold = _threshold(calibration, PRIMARY_TARGET_FPR) + if fixed_threshold is not None: + from .experiments.measurement_threshold import validate_fixed_threshold_artifact + + threshold_value = validate_fixed_threshold_artifact( + dict(fixed_threshold), + calibration.detector_identity.identity_hash, + ) + threshold_source = "open-detector-measurement-stability-v1" + fixed_threshold_artifact_hash = fixed_threshold["artifact_hash"] + threshold_hash_for_payload = fixed_threshold["artifact_hash"] + threshold_target_fpr = fixed_threshold["target_fpr"] + else: + threshold = _threshold(calibration, PRIMARY_TARGET_FPR) + threshold_value = threshold.value + threshold_source = "per-corpus-calibration" + fixed_threshold_artifact_hash = None + threshold_hash_for_payload = threshold.threshold_hash + threshold_target_fpr = threshold.target_fpr sources = {sample.sample_id: sample for sample in _attack_samples(corpus)} pristine = { sample_id: _text_only_weighted_evidence(sample, adapter) @@ -142,8 +160,8 @@ def score_key_blind_high_coverage_plan( "pristine_score": original.raw_score, "transformed_score": transformed.raw_score, "score_drop": original.raw_score - transformed.raw_score, - "pristine_detected": original.raw_score >= threshold.value, - "transformed_detected": transformed.raw_score >= threshold.value, + "pristine_detected": original.raw_score >= threshold_value, + "transformed_detected": transformed.raw_score >= threshold_value, } rows.append({**row, "row_hash": sha256_json(row)}) row_tuple = tuple(rows) @@ -161,9 +179,11 @@ def score_key_blind_high_coverage_plan( "ruleset_hash": active_profile.ruleset_hash, "detector_identity_hash": calibration.detector_identity.identity_hash, "calibration_bundle_hash": calibration.bundle_hash, - "threshold_hash": threshold.threshold_hash, - "threshold_target_fpr": threshold.target_fpr, - "threshold_value": threshold.value, + "threshold_hash": threshold_hash_for_payload, + "threshold_target_fpr": threshold_target_fpr, + "threshold_value": threshold_value, + "threshold_source": threshold_source, + "fixed_threshold_artifact_hash": fixed_threshold_artifact_hash, "selection_detector_access_observed": plan["detector_access_observed"], "selection_secret_access_observed": plan["secret_access_observed"], "rows": row_tuple, @@ -235,6 +255,11 @@ def _parser() -> argparse.ArgumentParser: ), ) parser.add_argument("--budgets", default="") + parser.add_argument( + "--fixed-threshold-json", + type=Path, + default=None, + ) parser.add_argument( "--json", type=Path, @@ -287,7 +312,17 @@ def main(argv: list[str] | None = None) -> int: device=args.device, ) started_at = _now() - evidence = score_key_blind_high_coverage_plan(corpus, tokenizer, plan, adapter, profile=profile) + fixed_threshold = None + if args.fixed_threshold_json is not None: + fixed_threshold = _load_json(args.fixed_threshold_json) + evidence = score_key_blind_high_coverage_plan( + corpus, + tokenizer, + plan, + adapter, + profile=profile, + fixed_threshold=fixed_threshold, + ) write_canonical_json_fsynced(args.json, evidence) fsynced_at = _now() provenance = _provenance( diff --git a/fuckmark/tiny_dev_measurement_threshold_hf.py b/fuckmark/tiny_dev_measurement_threshold_hf.py new file mode 100644 index 00000000..d7df93ca --- /dev/null +++ b/fuckmark/tiny_dev_measurement_threshold_hf.py @@ -0,0 +1,73 @@ +from __future__ import annotations + +import argparse +import json +import sys +from datetime import datetime, timezone +from pathlib import Path + +from .adapters import HuggingFaceSynthIDAdapter, HuggingFaceSynthIDConfig +from .corpus.measurement_calibration_io import load_measurement_calibration_corpus_json +from .durable_io import write_canonical_json_fsynced +from .experiments.measurement_threshold import build_fixed_threshold_artifact +from .tiny_dev_context_survival_plan_hf import ( + DEFAULT_MODEL_ID, + DEFAULT_MODEL_REVISION, + runtime_tokenizer_identity_public, +) +from .tiny_dev_detector_hf import default_watermark_payload + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(prog="fuckmark-tiny-dev-measurement-threshold-hf") + parser.add_argument("--calibration-corpus-json", type=Path, required=True) + parser.add_argument("--model", default=DEFAULT_MODEL_ID) + parser.add_argument("--model-revision", default=DEFAULT_MODEL_REVISION) + parser.add_argument("--device", choices=("cpu", "cuda"), default="cpu") + parser.add_argument("--json", type=Path, required=True) + args = parser.parse_args(argv) + try: + from transformers import AutoTokenizer + except ImportError as error: + raise RuntimeError("Install the pinned TinyDev Transformers dependencies first") from error + + corpus = load_measurement_calibration_corpus_json(args.calibration_corpus_json) + tokenizer = AutoTokenizer.from_pretrained( + args.model, + revision=args.model_revision, + use_fast=True, + padding_side="left", + ) + identity = runtime_tokenizer_identity_public(tokenizer, args.model, args.model_revision) + if identity.identity_hash != corpus.model_identity_hash: + raise RuntimeError("runtime tokenizer identity does not match the calibration corpus") + watermark_payload = default_watermark_payload() + adapter = HuggingFaceSynthIDAdapter.from_torch( + HuggingFaceSynthIDConfig( + ngram_len=watermark_payload["ngram_len"], + keys=watermark_payload["keys"], + context_history_size=watermark_payload["context_history_size"], + sampling_table_seed=watermark_payload["sampling_table_seed"], + sampling_table_size=watermark_payload["sampling_table_size"], + skip_first_ngram_calls=watermark_payload["skip_first_ngram_calls"], + debug_mode=watermark_payload["debug_mode"], + ), + device=args.device, + ) + artifact = build_fixed_threshold_artifact( + corpus, + adapter, + frozen_at_utc=datetime.now(timezone.utc).isoformat(), + ) + write_canonical_json_fsynced(args.json, artifact) + sys.stdout.write(f"artifact_hash={artifact['artifact_hash']}\n") + sys.stdout.write(f"threshold={artifact['threshold']!r}\n") + sys.stdout.write(f"calibration_exceedances={artifact['calibration_exceedances']}/{artifact['calibration_negative_count']}\n") + sys.stdout.write(f"audit_exceedances={artifact['audit_exceedances']}/{artifact['audit_negative_count']}\n") + sys.stdout.write(f"audit_realized_fpr={artifact['audit_realized_fpr']}\n") + sys.stdout.write(f"json={args.json.as_posix()}\n") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/test_content_region_profile.py b/tests/test_content_region_profile.py new file mode 100644 index 00000000..1c7fa221 --- /dev/null +++ b/tests/test_content_region_profile.py @@ -0,0 +1,154 @@ +from __future__ import annotations + +from types import SimpleNamespace + +import pytest + +from fuckmark.corpus import CorpusDomain, CorpusSplit, WatermarkLabel +from fuckmark.experiments.coverage_holes import classify_word +from fuckmark.experiments.effectiveness_plan import build_key_blind_high_coverage_plan +from fuckmark.hashing import sha256_text +from fuckmark.transforms import ( + CONTENT_REGION_COVERAGE_PROFILE_ID, + CONTENT_REGION_COVERAGE_SEED_BASE, + KEY_BLIND_HIGH_COVERAGE_PROFILE, + content_region_coverage_profile, + content_region_coverage_transform_registry, + resolve_effectiveness_profile, +) +from fuckmark.transforms.rules import GeneralWordSpacingRule + + +class FakeTokenizer: + def __call__( + self, + text: str, + *, + add_special_tokens: bool, + return_offsets_mapping: bool = False, + ) -> dict[str, object]: + assert add_special_tokens is False + ids = [index + 1 for index in range(len(text))] + value: dict[str, object] = {"input_ids": ids} + if return_offsets_mapping: + value["offset_mapping"] = [(index, index + 1) for index in range(len(text))] + return value + + +def _sample(sample_id: str, label: WatermarkLabel, text: str, identity_hash: str): + token_ids = tuple(index + 1 for index in range(len(text))) + return SimpleNamespace( + sample_id=sample_id, + split=CorpusSplit.ATTACK_DEVELOPMENT, + label=label, + prompt_family_id=f"prompt-{sample_id}", + domain=CorpusDomain.GENERAL_EXPLANATORY, + text=text, + text_sha256=sha256_text(text), + text_only_tokens=SimpleNamespace(token_ids=token_ids), + model=SimpleNamespace(identity_hash=identity_hash, eos_token_id=50256), + ) + + +def _corpus(): + identity_hash = sha256_text("fake-tokenizer") + text = ( + "Careful researchers documented measurement uncertainty thoroughly. " + "The auditor verified every reported number 42 before publishing results." + ) + samples = ( + _sample("negative-1", WatermarkLabel.UNWATERMARKED, text, identity_hash), + _sample("positive-1", WatermarkLabel.WATERMARKED, text, identity_hash), + ) + return SimpleNamespace( + artifact_hash=sha256_text("fake-corpus"), + model_identity_hash=identity_hash, + manifest=SimpleNamespace(samples=samples, manifest_hash=sha256_text("fake-manifest")), + ) + + +def test_general_word_spacing_rule_contract() -> None: + rule = GeneralWordSpacingRule.create("surface-space-after-any-word") + text = "Alpha beta gamma delta, epsilon zeta." + matches = [text[m.start() : m.end()] for m in rule.pattern().finditer(text)] + assert "Alpha" in matches + assert "beta" in matches + assert "gamma" in matches + assert "delta" not in matches + assert rule.replacement_for("beta") == "beta " + assert rule.replacement_for("Beta") == "Beta " + doubled = "beta gamma" + assert rule.pattern().search(doubled) is None + with pytest.raises(ValueError): + GeneralWordSpacingRule( + rule_id="bad", + version="general-word-space-after-v1", + family=rule.family, + tier=rule.tier, + source="word", + replacement="wrong ", + whole_word=False, + preserve_simple_case=False, + block_all_caps=False, + rule_hash=rule.rule_hash, + ) + + +def test_general_rule_enumerates_content_words_with_exact_replacement() -> None: + registry = content_region_coverage_transform_registry() + text = "Measurement uncertainty remained stubbornly high throughout replication attempts." + enumeration = registry.enumerate(text) + general = [ + candidate + for candidate in enumeration.candidates + if candidate.rule_id == "surface-space-after-any-word" + ] + assert any(candidate.source_text == "Measurement" for candidate in general) + assert all(candidate.replacement_text == candidate.source_text + " " for candidate in general) + for candidate in general: + transformed = text[: candidate.start] + candidate.replacement_text + text[candidate.end :] + assert transformed.count(candidate.source_text + " ") >= 1 + assert transformed.isascii() + + +def test_content_region_profile_resolves_and_builds_detector_blind_plans() -> None: + profile = resolve_effectiveness_profile(CONTENT_REGION_COVERAGE_PROFILE_ID, (16,)) + assert profile.profile_id == CONTENT_REGION_COVERAGE_PROFILE_ID + assert profile.schedule_seed_base == CONTENT_REGION_COVERAGE_SEED_BASE + assert content_region_coverage_profile((16,)) == profile + corpus = _corpus() + plan = build_key_blind_high_coverage_plan( + corpus, + FakeTokenizer(), + profile=profile, + source_code_commit="a" * 40, + ) + assert plan["ruleset_hash"] == profile.ruleset_hash + assert all(row["hard_invariant_status"] == "pass" for row in plan["variants"]) + assert all(row["detector_access_observed"] is False for row in plan["variants"]) + + +def test_content_region_registry_protects_numbers_and_keeps_b16_frozen() -> None: + registry = content_region_coverage_transform_registry() + text = "The reported value 3.14159 stayed fixed across runs." + enumeration = registry.enumerate(text) + for candidate in enumeration.candidates: + assert "3.14159" not in candidate.source_text + assert KEY_BLIND_HIGH_COVERAGE_PROFILE.profile_hash == ( + "6ad142262bfb11a714565d7bd43daa859657fe817bdafa9c8dcf0f4884c07512" + ) + + +def test_classify_word_categories() -> None: + assert classify_word("the") == "function_word" + assert classify_word("measurement") == "lowercase_content" + assert classify_word("Measurement") == "capitalized_content" + assert classify_word("42") == "numeric" + assert classify_word(",") == "punctuation_or_empty" + + +def test_resolve_rejects_invalid_content_region_requests() -> None: + with pytest.raises(ValueError): + resolve_effectiveness_profile(CONTENT_REGION_COVERAGE_PROFILE_ID) + with pytest.raises(ValueError): + content_region_coverage_profile((48, 16)) diff --git a/tests/test_measurement_threshold.py b/tests/test_measurement_threshold.py new file mode 100644 index 00000000..f997b57f --- /dev/null +++ b/tests/test_measurement_threshold.py @@ -0,0 +1,81 @@ +from __future__ import annotations + +from types import SimpleNamespace + +import pytest + +import fuckmark.experiments.measurement_threshold as mt +from fuckmark.experiments.measurement_threshold import ( + build_fixed_threshold_artifact, + validate_fixed_threshold_artifact, +) + + +class _FakeCalibrationCorpus: + def __init__(self, calibration_scores, audit_scores) -> None: + self.calibration_set = tuple( + SimpleNamespace(sample_id=f"cal-{index}", score=value) + for index, value in enumerate(calibration_scores) + ) + self.audit_set = tuple( + SimpleNamespace(sample_id=f"audit-{index}", score=value) + for index, value in enumerate(audit_scores) + ) + self.artifact_hash = "a" * 64 + self.manifest = SimpleNamespace(corpus_id="fake-calibration") + + def calibration_samples(self): + return self.calibration_set + + def audit_samples(self): + return self.audit_set + + +def _patch_scoring(monkeypatch, calibration_scores, audit_scores): + corpus = _FakeCalibrationCorpus(calibration_scores, audit_scores) + lookup = {sample.sample_id: sample.score for sample in (*corpus.calibration_set, *corpus.audit_set)} + + def fake_evidence(sample, adapter): + return SimpleNamespace(raw_score=lookup[sample.sample_id]) + + def fake_calibrate(evidence, scope, target_fprs, comparison_operator, confidence_level): + return SimpleNamespace( + detector_identity=SimpleNamespace(identity_hash="d" * 64), + bundle_hash="b" * 64, + ) + + monkeypatch.setattr(mt, "_text_only_weighted_evidence", fake_evidence) + monkeypatch.setattr(mt, "calibrate_detector", fake_calibrate) + return corpus + + +def test_fixed_threshold_uses_frozen_order_statistic(monkeypatch) -> None: + calibration = [index / 2000.0 for index in range(1024)] + audit = [0.0] * 250 + [0.9] * 6 + corpus = _patch_scoring(monkeypatch, calibration, audit) + artifact = build_fixed_threshold_artifact(corpus, None, frozen_at_utc="2026-08-23T00:00:00+00:00") + ordered = sorted(calibration) + assert artifact["threshold_order_statistic"] == 1014 + assert artifact["threshold"] == ordered[1013] + assert artifact["calibration_exceedances"] == 10 + assert artifact["audit_exceedances"] == 6 + assert artifact["audit_realized_fpr"] == 6 / 256 + assert artifact["detector_identity_hash"] == "d" * 64 + assert artifact["artifact_hash"] + + +def test_fixed_threshold_validation_rejects_mismatches(monkeypatch) -> None: + calibration = [index / 2000.0 for index in range(1024)] + corpus = _patch_scoring(monkeypatch, calibration, [0.0] * 256) + artifact = build_fixed_threshold_artifact(corpus, None, frozen_at_utc="2026-08-23T00:00:00+00:00") + assert validate_fixed_threshold_artifact(artifact, "d" * 64) == artifact["threshold"] + with pytest.raises(ValueError): + validate_fixed_threshold_artifact(artifact, "e" * 64) + tampered = dict(artifact) + tampered["threshold"] = 0.123456 + with pytest.raises(ValueError): + validate_fixed_threshold_artifact(tampered, "d" * 64) + bad_version = dict(artifact) + bad_version["algorithm_version"] = "other-v1" + with pytest.raises(ValueError): + validate_fixed_threshold_artifact(bad_version, "d" * 64) From e1f53693fb1d94e9473ca32a6f0c8b40ed01f25c Mon Sep 17 00:00:00 2001 From: Cyinh Date: Sat, 22 Aug 2026 22:53:35 -0400 Subject: [PATCH 19/30] Correct fixed-threshold order statistic to bound calibration exceedances at target FPR --- fuckmark/experiments/measurement_threshold.py | 9 +++++---- tests/test_measurement_threshold.py | 4 ++-- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/fuckmark/experiments/measurement_threshold.py b/fuckmark/experiments/measurement_threshold.py index c3de6c36..8231a49a 100644 --- a/fuckmark/experiments/measurement_threshold.py +++ b/fuckmark/experiments/measurement_threshold.py @@ -45,7 +45,7 @@ def build_fixed_threshold_artifact( ) scores = sorted(value.raw_score for value in evidence) count = len(scores) - order_statistic = count - math.floor(MEASUREMENT_STABILITY_TARGET_FPR * count) + order_statistic = count - math.floor(MEASUREMENT_STABILITY_TARGET_FPR * count) + 1 threshold = scores[order_statistic - 1] calibration_exceedances = sum(1 for value in scores if value >= threshold) audit_scores = tuple(value.raw_score for value in audit_evidence) @@ -62,9 +62,10 @@ def build_fixed_threshold_artifact( "target_fpr": MEASUREMENT_STABILITY_TARGET_FPR, "comparison_operator": MEASUREMENT_STABILITY_COMPARISON, "threshold_definition": ( - "ascending order statistic n - floor(target_fpr * n) of the calibration " - "negative weighted-mean scores (the 1014th of 1024 for target FPR 0.01); " - "detection when score >= threshold; float64 scores as computed; no tie rounding" + "ascending order statistic n - floor(target_fpr * n) + 1 of the calibration " + "negative weighted-mean scores (the 1015th of 1024 for target FPR 0.01, bounding " + "calibration exceedances at floor(target_fpr * n)); detection when score >= " + "threshold; float64 scores as computed; no tie rounding" ), "threshold_order_statistic": order_statistic, "threshold": threshold, diff --git a/tests/test_measurement_threshold.py b/tests/test_measurement_threshold.py index f997b57f..c7a1297c 100644 --- a/tests/test_measurement_threshold.py +++ b/tests/test_measurement_threshold.py @@ -55,8 +55,8 @@ def test_fixed_threshold_uses_frozen_order_statistic(monkeypatch) -> None: corpus = _patch_scoring(monkeypatch, calibration, audit) artifact = build_fixed_threshold_artifact(corpus, None, frozen_at_utc="2026-08-23T00:00:00+00:00") ordered = sorted(calibration) - assert artifact["threshold_order_statistic"] == 1014 - assert artifact["threshold"] == ordered[1013] + assert artifact["threshold_order_statistic"] == 1015 + assert artifact["threshold"] == ordered[1014] assert artifact["calibration_exceedances"] == 10 assert artifact["audit_exceedances"] == 6 assert artifact["audit_realized_fpr"] == 6 / 256 From 507df3827ec9abbfa35dbf09bb8ce6eeffa72a56 Mon Sep 17 00:00:00 2001 From: Cyinh Date: Sat, 22 Aug 2026 22:55:06 -0400 Subject: [PATCH 20/30] Freeze content-region coverage cycle-3 protocol and contract --- ...k-content-region-coverage-v3.contract.json | 85 ++++++++++++++ ...ark-content-region-coverage-v3.freeze.json | 9 ++ ...ark-content-region-coverage-v3.protocol.md | 110 ++++++++++++++++++ 3 files changed, 204 insertions(+) create mode 100644 specs/fuckmark-content-region-coverage-v3.contract.json create mode 100644 specs/fuckmark-content-region-coverage-v3.freeze.json create mode 100644 specs/fuckmark-content-region-coverage-v3.protocol.md diff --git a/specs/fuckmark-content-region-coverage-v3.contract.json b/specs/fuckmark-content-region-coverage-v3.contract.json new file mode 100644 index 00000000..a4804069 --- /dev/null +++ b/specs/fuckmark-content-region-coverage-v3.contract.json @@ -0,0 +1,85 @@ +{ + "algorithm_version": "content-region-coverage-contract-v3", + "base_main_commit": "16e153fdfa3739a88bca9b1559543a10ded2d859", + "blind_human_fidelity_evidence": "NOT_PROVIDED", + "candidate": { + "budget": 16, + "new_rule": "surface-space-after-any-word (general word spacing, no word list)", + "profile_hash": "5f4d417a13c2d222c68f70767aef8ed11d7a71c8a5ab9f9c0c21eb278977063c", + "profile_id": "content-region-coverage-v1", + "ruleset_hash": "82011e6dd7048a97e07918d32f7f8670e7723c66eb1b5b0ac859adb4e9cde8ca", + "schedule_policy": "COVERAGE_GREEDY_KEY_BLIND", + "schedule_seed_base": 1150000 + }, + "claim_boundary": { + "cross_platform_generation_reproducibility": "NOT_CLAIMED", + "level_b_product_admissible": false, + "release_authorized": false, + "status": "EXPLORATORY_FIXED_OPEN_DETECTOR_ONLY", + "undetectability_claim": false, + "watermark_removal_claim": false + }, + "confirmation_gates": { + "corpora": 3, + "fidelity_rows_required_per_arm": 128, + "pristine_watermarked_min": 60, + "stretch_target": "0/192 pooled transformed watermarked detections", + "transformed_unwatermarked_max": 0 + }, + "dev_arms": [ + "A_cycle2_baseline", + "B_cycle3_candidate", + "C_general_only", + "D_combined_ablation" + ], + "environment": { + "machine": "AMD64", + "model": "openai-community/gpt2@607a30d783dfa663caf39e06633721c8d4cfcd7e", + "os": "Windows-10-10.0.26220-SP0", + "processor": "AuthenticAMD Family 25 Model 68 Stepping 1", + "python": "3.11.15", + "torch": "2.13.0+cpu", + "transformers": "5.16.0.dev0 (pinned git revision a61d5f9e4fc184cff66938ff6c521cc358b5e024)" + }, + "measurement": { + "audit_negatives": 256, + "calibration_negatives": 1024, + "calibration_seed_base": 500000, + "comparison": ">=", + "identity": "open-detector-measurement-stability-v1", + "order_statistic": "n - floor(target_fpr*n) + 1 ascending", + "target_fpr": 0.01, + "threshold_frozen_before_confirmation_scoring": true + }, + "outcome_classes": [ + "STRONG_SUCCESS", + "PARTIAL_SUCCESS", + "NEUTRAL", + "REJECTED" + ], + "promotion_gate": { + "detector_tolerance_detected_difference": 2, + "low_coverage_mean_gain_required": 0.05, + "low_coverage_subset_size": 8, + "recorded_before_confirmation_scoring": true + }, + "protocol_id": "content-region-coverage-v3", + "recorded_date": "2026-08-23", + "research_branch_base": "4ddd84fa9bc184b1159946c56f8a370111d20f22", + "seed_ledger": { + "consumed": [ + "401000-404296", + "410000-414192", + "420000-425216", + "430000-434192", + "440000-445216", + "500000-520480" + ], + "cycle3_confirmations": [ + "460000", + "470000", + "480000" + ], + "cycle3_dev": "450000" + } +} diff --git a/specs/fuckmark-content-region-coverage-v3.freeze.json b/specs/fuckmark-content-region-coverage-v3.freeze.json new file mode 100644 index 00000000..1c04e752 --- /dev/null +++ b/specs/fuckmark-content-region-coverage-v3.freeze.json @@ -0,0 +1,9 @@ +{ + "appendices": {}, + "contract_sha256": "072c360e57abda8fabfafbc23654324a1dfb89efe585c367a0a745aa8474c7b3", + "freeze_record_version": "content-region-coverage-v3-freeze-v1", + "frozen_at_utc": "2026-08-23T02:55:06.234907+00:00", + "pre_score_amendments": [], + "protocol_id": "content-region-coverage-v3", + "protocol_sha256": "ea0de812869d60f7a5cc8e765eb0c5a68db0c61c8722b2347a64752e12fb24a2" +} diff --git a/specs/fuckmark-content-region-coverage-v3.protocol.md b/specs/fuckmark-content-region-coverage-v3.protocol.md new file mode 100644 index 00000000..08df9e41 --- /dev/null +++ b/specs/fuckmark-content-region-coverage-v3.protocol.md @@ -0,0 +1,110 @@ +# Frozen Research Protocol: Content-Region Coverage (Cycle 3) + +Protocol ID: `content-region-coverage-v3`. Frozen before any Cycle-3 detector scoring. +Freeze record: `specs/fuckmark-content-region-coverage-v3.freeze.json`. + +## 1. Objective and claim boundary + +Test whether a general any-word spacing candidate family (a) closes the content-region +candidate holes diagnosed in Cycles 1-2, (b) beats the Cycle-2 coverage-completion ruleset in +a paired design on fresh corpora, and (c) holds across three independent confirmation corpora +under a fixed corpus-independent threshold. Claim boundary unchanged: fixed open HF SynthID +weighted-mean detector, public development keys, uncleaned text-only path; no removal, +undetectability, unknown-key, proprietary-detector, or cleaning claims; Level B not pursued. + +## 2. Candidate mechanism + +Profile `content-region-coverage-v1` (ruleset hash recorded in artifacts): forward +contractions, the six punctuation spacing rules, ONE new `GeneralWordSpacingRule` +(`surface-space-after-any-word`, pattern `(?= +threshold; float64, no tie rounding. The threshold artifact freezes detector identity, +corpus hash, counts, realized audit FPR with exact 95% binomial interval, and environment +identity. ALL Cycle-3 confirmation scoring uses this fixed threshold; per-corpus calibration +thresholds are recorded alongside for comparison only. Threshold frozen before any +confirmation score. Dev scoring (both arms on the same corpus) may additionally report +per-corpus thresholds; paired comparisons use one threshold within each corpus. + +## 4. Seed-range ledger (recovered from artifacts) + +Consumed: 401000-404296, 410000-414192, 420000-425216, 430000-434192, 440000-445216, +calibration 500000-520480 (span 160 prompts x 8 slots x stride 16). Cycle 3 uses: dev +450000 (v3, 8 attack pairs/domain, spans about 450000-454224), confirmations 460000, +470000, 480000 (v3, 16 attack pairs/domain, each spanning about 5264 seeds). All ranges are +pairwise disjoint from every consumed range and from each other. + +## 5. Development design + +Fresh dev corpus 450000 (32 watermarked attack sources). Four plan arms per source, all +detector-blind before scoring: + +A. Cycle-2 coverage-completion ruleset (paired baseline) +B. Cycle-3 content-region profile +C. General-spacing-only arm (general rule + punctuation only, no contractions/lexical/syntax) +D. Combined arm (Cycle-2 word list + general rule) + +Primary structural endpoints (no detector data): candidate count, achievable coverage, +zero-candidate rate, longest uncovered run, uncovered window count, per-source paired +differences B minus A. Detector outcomes are evaluation-only. + +## 6. Promotion gate (all required before any confirmation scoring) + +1-8: historical hashes unchanged; suite green except documented environmental failures; hard +invariants pass on every dev row; protected-span violations 0; introduced hidden/control +codepoints 0; introduced normalization instability 0; Cf-strip identity holds; no source +becomes empty or corrupted. 9: B raises achievable coverage materially in the A-low-coverage +subset (mean paired gain >= 0.05 among the 8 lowest-A-coverage sources). 10: zero-candidate +rate not above baseline. 11: fidelity metrics not materially regressed (word edit rate, +protected spans, invariants). 12: paired detector result for B not clearly worse than A on +dev (detected count difference <= 2 on 32 sources). The confirmation decision (proceed or +reject) is recorded in the freeze record appendix before confirmation scoring. + +## 7. Confirmation design + +Three corpora frozen before the first confirmation score (hashes recorded in the appendix +after generation, before scoring): 460000, 470000, 480000; each 64 watermarked + 64 +unwatermarked attack sources. For each corpus, two frozen plans (A baseline, B candidate) +built detector-blind, then scored once each with the fixed threshold. Control gates per +corpus: pristine watermarked >= 60/64 (unconditional denominator primary; conditional count +reported); transformed unwatermarked == 0/64; fidelity/representation gates 128/128 per arm; +text-hash overlap 0 against every prior corpus with the pre-registered collision policy: +any naturally occurring collision is reported and followed by scoring the corpus as +generated; no regeneration or substitution after seeing outcomes. + +## 8. Outcome classes (pre-registered) + +STRONG_SUCCESS: all control gates pass on all three corpora AND pooled B residuals strictly +fewer than pooled A residuals AND B no worse than A on every corpus; stretch target 0/192. +PARTIAL_SUCCESS: B paired-better overall but residuals remain, or exactly one non-primary +gate fails. NEUTRAL: B not better than A pooled. REJECTED: B worse, false positives appear, +fidelity regresses, or the structural hypothesis fails on dev. Post-scoring margin analysis +(±0.005/±0.010/±0.025 bands) is diagnosis only. + +## 9. Fidelity and reporting + +All Cycle-1/2 gates retained. For the general-rule family report separately: number of +spacing insertions, number of sources receiving them, maximum and mean per source, exact +changed tokens, rule identities; word-level substitutions from other families reported as in +prior cycles. Blind human fidelity evidence: NOT_PROVIDED (no fabricated judgments). + +## 10. Reproducibility + +Environment manifest bound in the freeze record (Windows-10-10.0.26220 AMD64, Python +3.11.15, torch 2.13.0+cpu, transformers 5.16.0.dev0 at the pinned git revision, GPT-2 +revision 607a30d783dfa663caf39e06633721c8d4cfcd7e). Cross-platform corpus generation is NOT +claimed byte-reproducible (documented Cycle-1 finding); plans, scoring, and thresholds are +deterministic given texts and the pinned runtime. All artifacts content-addressed; amendments +only before affected scores, timestamped and hash-bound; local commits only, nothing pushed. From 4a528db63bbe5207e625e4a3332e7fcc36c8bea0 Mon Sep 17 00:00:00 2001 From: Cyinh Date: Sat, 22 Aug 2026 22:56:55 -0400 Subject: [PATCH 21/30] Add dev-only ablation profiles for general-only and combined arms --- fuckmark/experiments/effectiveness_plan.py | 8 ++ fuckmark/transforms/__init__.py | 12 +++ fuckmark/transforms/effectiveness_profile.py | 77 ++++++++++++++++++++ 3 files changed, 97 insertions(+) diff --git a/fuckmark/experiments/effectiveness_plan.py b/fuckmark/experiments/effectiveness_plan.py index 6cefda6c..bf42eb98 100644 --- a/fuckmark/experiments/effectiveness_plan.py +++ b/fuckmark/experiments/effectiveness_plan.py @@ -14,12 +14,16 @@ ScheduleGeometryMode, SchedulePolicy, build_candidate_tokenizer_geometry, + content_region_combined_transform_registry, content_region_coverage_transform_registry, + content_region_general_only_transform_registry, key_blind_coverage_completion_transform_registry, key_blind_high_coverage_transform_registry, validate_effectiveness_profile_registry, validate_hard_invariants, + CONTENT_REGION_COMBINED_PROFILE_ID, CONTENT_REGION_COVERAGE_PROFILE_ID, + CONTENT_REGION_GENERAL_ONLY_PROFILE_ID, KEY_BLIND_COVERAGE_COMPLETION_PROFILE_ID, ) @@ -160,6 +164,10 @@ def _registry_for_profile(profile: EffectivenessTransformProfile): return key_blind_coverage_completion_transform_registry() if profile.profile_id == CONTENT_REGION_COVERAGE_PROFILE_ID: return content_region_coverage_transform_registry() + if profile.profile_id == CONTENT_REGION_GENERAL_ONLY_PROFILE_ID: + return content_region_general_only_transform_registry() + if profile.profile_id == CONTENT_REGION_COMBINED_PROFILE_ID: + return content_region_combined_transform_registry() return key_blind_high_coverage_transform_registry() diff --git a/fuckmark/transforms/__init__.py b/fuckmark/transforms/__init__.py index 6c29206e..27f2fb99 100644 --- a/fuckmark/transforms/__init__.py +++ b/fuckmark/transforms/__init__.py @@ -1,7 +1,9 @@ from .candidate_artifacts import CandidateConflict, CandidateEnumeration, CandidateRejection, TransformCandidate from .effectiveness_profile import ( + CONTENT_REGION_COMBINED_PROFILE_ID, CONTENT_REGION_COVERAGE_PROFILE_ID, CONTENT_REGION_COVERAGE_SEED_BASE, + CONTENT_REGION_GENERAL_ONLY_PROFILE_ID, EFFECTIVENESS_PROFILE_ALGORITHM_VERSION, KEY_BLIND_COVERAGE_COMPLETION_PROFILE_ID, KEY_BLIND_COVERAGE_COMPLETION_SEED_BASE, @@ -12,8 +14,12 @@ KEY_BLIND_HIGH_COVERAGE_PROFILE_ID, KEY_BLIND_HIGH_COVERAGE_SEED_BASE, EffectivenessTransformProfile, + content_region_combined_profile, + content_region_combined_transform_registry, content_region_coverage_profile, content_region_coverage_transform_registry, + content_region_general_only_profile, + content_region_general_only_transform_registry, key_blind_coverage_completion_profile, key_blind_coverage_completion_transform_registry, key_blind_full_pool_coverage_profile, @@ -222,8 +228,14 @@ "KEY_BLIND_FULL_POOL_COVERAGE_SEED_BASE", "KEY_BLIND_COVERAGE_COMPLETION_PROFILE_ID", "KEY_BLIND_COVERAGE_COMPLETION_SEED_BASE", + "CONTENT_REGION_COMBINED_PROFILE_ID", "CONTENT_REGION_COVERAGE_PROFILE_ID", "CONTENT_REGION_COVERAGE_SEED_BASE", + "CONTENT_REGION_GENERAL_ONLY_PROFILE_ID", + "content_region_combined_profile", + "content_region_combined_transform_registry", + "content_region_general_only_profile", + "content_region_general_only_transform_registry", "content_region_coverage_profile", "content_region_coverage_transform_registry", "key_blind_coverage_completion_profile", diff --git a/fuckmark/transforms/effectiveness_profile.py b/fuckmark/transforms/effectiveness_profile.py index 89290139..67c0136c 100644 --- a/fuckmark/transforms/effectiveness_profile.py +++ b/fuckmark/transforms/effectiveness_profile.py @@ -30,6 +30,8 @@ KEY_BLIND_COVERAGE_COMPLETION_SEED_BASE = 1_140_000 CONTENT_REGION_COVERAGE_PROFILE_ID = "content-region-coverage-v1" CONTENT_REGION_COVERAGE_SEED_BASE = 1_150_000 +CONTENT_REGION_GENERAL_ONLY_PROFILE_ID = "content-region-general-only-dev-v1" +CONTENT_REGION_COMBINED_PROFILE_ID = "content-region-combined-dev-v1" @dataclass(frozen=True, slots=True) @@ -217,6 +219,14 @@ def resolve_effectiveness_profile( if not budgets: raise ValueError("the content region coverage profile requires explicit budgets") return content_region_coverage_profile(budgets) + if profile_id == CONTENT_REGION_GENERAL_ONLY_PROFILE_ID: + if not budgets: + raise ValueError("the general-only ablation profile requires explicit budgets") + return content_region_general_only_profile(budgets) + if profile_id == CONTENT_REGION_COMBINED_PROFILE_ID: + if not budgets: + raise ValueError("the combined ablation profile requires explicit budgets") + return content_region_combined_profile(budgets) raise ValueError("unknown effectiveness profile id") @@ -269,6 +279,73 @@ def content_region_coverage_profile( ) +def content_region_general_only_transform_registry( + identifiers: Sequence[str] = (), +) -> TransformRegistry: + return TransformRegistry(content_region_surface_rules(), identifiers) + + +def content_region_combined_transform_registry( + identifiers: Sequence[str] = (), +) -> TransformRegistry: + general = GeneralWordSpacingRule.create(CONTENT_REGION_GENERAL_SPACING_RULE_ID) + return TransformRegistry( + ( + *development_forward_contraction_rules(), + *coverage_completion_surface_rules(), + general, + *development_lexical_rules(), + *development_syntax_rules(), + ), + identifiers, + ) + + +def _dev_ablation_profile( + profile_id: str, + ruleset_hash: str, + budgets: tuple[int, ...], +) -> EffectivenessTransformProfile: + return EffectivenessTransformProfile.create( + profile_id=profile_id, + budgets=budgets, + budget_unit="operation", + schedule_policy_id="COVERAGE_GREEDY_KEY_BLIND", + schedule_seed_base=CONTENT_REGION_COVERAGE_SEED_BASE, + replicate_count=1, + ngram_len=5, + ruleset_hash=ruleset_hash, + scientific_scope=( + "Development-only ablation profile for the content-region cycle; exploratory " + "evidence only and not release authorization" + ), + ) + + +def content_region_general_only_profile( + budgets: tuple[int, ...], +) -> EffectivenessTransformProfile: + if not isinstance(budgets, tuple): + raise TypeError("budgets must be a tuple") + return _dev_ablation_profile( + CONTENT_REGION_GENERAL_ONLY_PROFILE_ID, + content_region_general_only_transform_registry().ruleset_hash, + budgets, + ) + + +def content_region_combined_profile( + budgets: tuple[int, ...], +) -> EffectivenessTransformProfile: + if not isinstance(budgets, tuple): + raise TypeError("budgets must be a tuple") + return _dev_ablation_profile( + CONTENT_REGION_COMBINED_PROFILE_ID, + content_region_combined_transform_registry().ruleset_hash, + budgets, + ) + + def key_blind_coverage_completion_transform_registry( identifiers: Sequence[str] = (), ) -> TransformRegistry: From a5c817848056ecf1b3647163ee4ee018015bba72 Mon Sep 17 00:00:00 2001 From: Cyinh Date: Sat, 22 Aug 2026 23:02:12 -0400 Subject: [PATCH 22/30] Extend runner profile choices with dev ablation profiles --- fuckmark/tiny_dev_effectiveness_plan_hf.py | 4 ++++ fuckmark/tiny_dev_effectiveness_score_hf.py | 2 ++ 2 files changed, 6 insertions(+) diff --git a/fuckmark/tiny_dev_effectiveness_plan_hf.py b/fuckmark/tiny_dev_effectiveness_plan_hf.py index b70c5fdb..e524c806 100644 --- a/fuckmark/tiny_dev_effectiveness_plan_hf.py +++ b/fuckmark/tiny_dev_effectiveness_plan_hf.py @@ -21,7 +21,9 @@ KEY_BLIND_HIGH_COVERAGE_PROFILE_ID, KEY_BLIND_FULL_POOL_COVERAGE_PROFILE_ID, KEY_BLIND_COVERAGE_COMPLETION_PROFILE_ID, + CONTENT_REGION_COMBINED_PROFILE_ID, CONTENT_REGION_COVERAGE_PROFILE_ID, + CONTENT_REGION_GENERAL_ONLY_PROFILE_ID, resolve_effectiveness_profile, ) @@ -91,6 +93,8 @@ def _parser() -> argparse.ArgumentParser: KEY_BLIND_FULL_POOL_COVERAGE_PROFILE_ID, KEY_BLIND_COVERAGE_COMPLETION_PROFILE_ID, CONTENT_REGION_COVERAGE_PROFILE_ID, + CONTENT_REGION_GENERAL_ONLY_PROFILE_ID, + CONTENT_REGION_COMBINED_PROFILE_ID, ), ) parser.add_argument("--budgets", default="") diff --git a/fuckmark/tiny_dev_effectiveness_score_hf.py b/fuckmark/tiny_dev_effectiveness_score_hf.py index ca768b47..01e6d3c0 100644 --- a/fuckmark/tiny_dev_effectiveness_score_hf.py +++ b/fuckmark/tiny_dev_effectiveness_score_hf.py @@ -252,6 +252,8 @@ def _parser() -> argparse.ArgumentParser: KEY_BLIND_FULL_POOL_COVERAGE_PROFILE_ID, KEY_BLIND_COVERAGE_COMPLETION_PROFILE_ID, CONTENT_REGION_COVERAGE_PROFILE_ID, + CONTENT_REGION_GENERAL_ONLY_PROFILE_ID, + CONTENT_REGION_COMBINED_PROFILE_ID, ), ) parser.add_argument("--budgets", default="") From f60f7ba2f1d7a9d1f1366fbb0e1f50e44407b796 Mon Sep 17 00:00:00 2001 From: Cyinh Date: Sat, 22 Aug 2026 23:04:04 -0400 Subject: [PATCH 23/30] Fix score runner ablation profile imports --- fuckmark/tiny_dev_effectiveness_score_hf.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/fuckmark/tiny_dev_effectiveness_score_hf.py b/fuckmark/tiny_dev_effectiveness_score_hf.py index 01e6d3c0..29f1241d 100644 --- a/fuckmark/tiny_dev_effectiveness_score_hf.py +++ b/fuckmark/tiny_dev_effectiveness_score_hf.py @@ -32,6 +32,9 @@ _word_edit_distance, ) from .transforms import ( + CONTENT_REGION_COMBINED_PROFILE_ID, + CONTENT_REGION_COVERAGE_PROFILE_ID, + CONTENT_REGION_GENERAL_ONLY_PROFILE_ID, KEY_BLIND_HIGH_COVERAGE_PROFILE, KEY_BLIND_HIGH_COVERAGE_PROFILE_ID, KEY_BLIND_FULL_POOL_COVERAGE_PROFILE_ID, From fd1660fa1fc89442557ad60a5582f72f40d1c1d7 Mon Sep 17 00:00:00 2001 From: Cyinh Date: Sat, 22 Aug 2026 23:08:57 -0400 Subject: [PATCH 24/30] Route coverage-hole registry selection through the plan profile resolver --- fuckmark/experiments/coverage_holes.py | 17 ++++------------- 1 file changed, 4 insertions(+), 13 deletions(-) diff --git a/fuckmark/experiments/coverage_holes.py b/fuckmark/experiments/coverage_holes.py index ef7ccad7..eae71e9c 100644 --- a/fuckmark/experiments/coverage_holes.py +++ b/fuckmark/experiments/coverage_holes.py @@ -281,20 +281,11 @@ def build_coverage_hole_report(corpus: Any, tokenizer: Any, plan: dict[str, obje def _registry_for_plan(plan: dict[str, object]): - from ..transforms import ( - KEY_BLIND_COVERAGE_COMPLETION_PROFILE_ID, - CONTENT_REGION_COVERAGE_PROFILE_ID, - content_region_coverage_transform_registry, - key_blind_coverage_completion_transform_registry, - key_blind_high_coverage_transform_registry, - ) + from .effectiveness_plan import _registry_for_profile + from ..transforms import EffectivenessTransformProfile, resolve_effectiveness_profile - profile_id = plan["profile_id"] - if profile_id == KEY_BLIND_COVERAGE_COMPLETION_PROFILE_ID: - return key_blind_coverage_completion_transform_registry() - if profile_id == CONTENT_REGION_COVERAGE_PROFILE_ID: - return content_region_coverage_transform_registry() - return key_blind_high_coverage_transform_registry() + profile = resolve_effectiveness_profile(plan["profile_id"], tuple(plan["budgets"])) + return _registry_for_profile(profile) def main(argv: list[str] | None = None) -> int: From 8979d67094e180a142217941e4b5e1a5dc3a3cb0 Mon Sep 17 00:00:00 2001 From: Cyinh Date: Sat, 22 Aug 2026 23:09:39 -0400 Subject: [PATCH 25/30] Record cycle-3 promotion decision before confirmation scoring --- ...ark-content-region-coverage-v3.freeze.json | 85 ++++++++++++++++++- 1 file changed, 84 insertions(+), 1 deletion(-) diff --git a/specs/fuckmark-content-region-coverage-v3.freeze.json b/specs/fuckmark-content-region-coverage-v3.freeze.json index 1c04e752..51cf87c5 100644 --- a/specs/fuckmark-content-region-coverage-v3.freeze.json +++ b/specs/fuckmark-content-region-coverage-v3.freeze.json @@ -1,5 +1,88 @@ { - "appendices": {}, + "appendices": { + "promotion_decision": { + "ablation_note": "arm C (general rule plus punctuation only) achieves the same mean achievable coverage as arm B; the cycle-2 word list in arm D adds candidates but no additional coverage; the general rule is the operative mechanism", + "confirmation_plan": { + "arms": [ + "A_cycle2_baseline", + "B_content_region" + ], + "corpora_seed_bases": [ + 460000, + 470000, + 480000 + ], + "threshold": "fixed open-detector-measurement-stability-v1 artifact, frozen before any confirmation score" + }, + "decision": "PROMOTE_CONTENT_REGION_CANDIDATE_TO_CONFIRMATION", + "dev_arm_summary": { + "evidence": { + "A": { + "artifact_hash": "9220f05c8d60000e7854ca967f3565ea966fe49824fa2852b6d647957551ae73", + "detected": 1, + "mean_score_drop": 0.10520081825713615 + }, + "B": { + "artifact_hash": "1fa95328246858bf49b6ae8a6ce0a4c5bdf97fc0c30dfdec6c89b7bea2930089", + "detected": 1, + "mean_score_drop": 0.1138549714227548 + }, + "C": { + "artifact_hash": "92b0e20199be75aa3a18fba21b0134764625ff95de1f99298ebf6d4332f26c90", + "detected": 1, + "mean_score_drop": 0.11398387642298335 + }, + "D": { + "artifact_hash": "a3a420a98ba552f5d8016465b3398a210b4fbb9eee5c3fd1ee7e76f4fe6b8d94", + "detected": 1, + "mean_score_drop": 0.11056383858328153 + } + }, + "structure": { + "A": { + "artifact_hash": "ab547b487cf37633eeaf0abdf151ed213bcd132ebe29912032b2d37b5e91bdc0", + "mean_achievable_coverage": 0.8405455508474576, + "mean_pool": 21.625, + "zero_candidate_sources": 0 + }, + "B": { + "artifact_hash": "308ccaa1153149b302593d33208db69968176c8948e7149db0e3a847f144feb4", + "mean_achievable_coverage": 0.9411458333333332, + "mean_pool": 45.0, + "zero_candidate_sources": 0 + }, + "C": { + "artifact_hash": "66c16e2a82b22a2e422d1e38a03d7c508b0a8111506ee272908fe67e17ceff8b", + "mean_achievable_coverage": 0.9411458333333332, + "mean_pool": 44.71875, + "zero_candidate_sources": 0 + }, + "D": { + "artifact_hash": "f533484077270016da9609ffe14d614f7249f83b0b19efb98814894fbe4ddd49", + "mean_achievable_coverage": 0.9411458333333332, + "mean_pool": 61.90625, + "zero_candidate_sources": 0 + } + } + }, + "gate_results": { + "cf_strip_identity": true, + "hard_invariants_all_rows": true, + "historical_hashes_unchanged": true, + "introduced_hidden_codepoints": 0, + "introduced_normalization_instability": 0, + "low_coverage_gate_required": 0.05, + "low_coverage_mean_gain": 0.2, + "max_word_edit_rate": 0.1509, + "paired_detector_B_minus_A_detected_difference": 0, + "protected_span_violations": 0, + "robustness_gate_pass_fraction": 1.0, + "tolerance": 2, + "zero_candidate_rate": 0 + }, + "recorded_at_utc": "2026-08-23T03:09:39.090311+00:00" + } + }, "contract_sha256": "072c360e57abda8fabfafbc23654324a1dfb89efe585c367a0a745aa8474c7b3", "freeze_record_version": "content-region-coverage-v3-freeze-v1", "frozen_at_utc": "2026-08-23T02:55:06.234907+00:00", From 4427f558dd717bc21fb48751a6e29ff282d8dcd6 Mon Sep 17 00:00:00 2001 From: Cyinh Date: Sat, 22 Aug 2026 23:20:39 -0400 Subject: [PATCH 26/30] Retry empty decodes in v3 corpus generation --- fuckmark/corpus/tiny_dev_v3_generation.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/fuckmark/corpus/tiny_dev_v3_generation.py b/fuckmark/corpus/tiny_dev_v3_generation.py index 5db28ed2..378837df 100644 --- a/fuckmark/corpus/tiny_dev_v3_generation.py +++ b/fuckmark/corpus/tiny_dev_v3_generation.py @@ -151,8 +151,13 @@ def build_real_tiny_dev_v3_corpus( accepted = False for attempt in range(max_attempts): seed = pair_seed_base + attempt - control = backend.generate(prompt.text, seed, watermarked=False) - watermarked = backend.generate(prompt.text, seed, watermarked=True) + try: + control = backend.generate(prompt.text, seed, watermarked=False) + watermarked = backend.generate(prompt.text, seed, watermarked=True) + except RuntimeError: + continue + if not control.text.strip() or not watermarked.text.strip(): + continue candidates = ( (WatermarkLabel.UNWATERMARKED, control), (WatermarkLabel.WATERMARKED, watermarked), From ca062cc7544527d0acb3ed0da11270412776444d Mon Sep 17 00:00:00 2001 From: Cyinh Date: Sat, 22 Aug 2026 23:51:31 -0400 Subject: [PATCH 27/30] Freeze the three cycle-3 confirmation corpora before scoring --- ...ark-content-region-coverage-v3.freeze.json | 33 +++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/specs/fuckmark-content-region-coverage-v3.freeze.json b/specs/fuckmark-content-region-coverage-v3.freeze.json index 51cf87c5..881c76b2 100644 --- a/specs/fuckmark-content-region-coverage-v3.freeze.json +++ b/specs/fuckmark-content-region-coverage-v3.freeze.json @@ -1,5 +1,38 @@ { "appendices": { + "confirmation_corpus_freeze": { + "corpora": { + "460000": { + "artifact_hash": "aa4b0fab8589e2830218b93a75a1f3cf6ca1ded01a505f7fb084d27843a57ab1", + "file_sha256": "ae7f847af1b4b22a0206483989480eb8925ad69a5533eed56b9b87107e87b55d", + "sample_count": 328, + "seed_span": [ + 460000, + 465216 + ] + }, + "470000": { + "artifact_hash": "381bb01ad2ce1a01a5f970bcded8c09fd22da784eba148fda208b9efe4e94974", + "file_sha256": "222c8eeb8ef88463a51c6802b4fd41c2b7bc7ffc8ff04b696fc3e4e036b9bff7", + "sample_count": 328, + "seed_span": [ + 470000, + 475216 + ] + }, + "480000": { + "artifact_hash": "60604bda9a6b7589a99b40d528ae967948a030f0d777d6554ef836bfab6cd701", + "file_sha256": "226185d0a73720189933612f2ddad3e8853f3cb89d3dfa5dcaf113351b45ef4e", + "sample_count": 328, + "seed_span": [ + 480000, + 485216 + ] + } + }, + "note": "all three confirmation corpora hashed and frozen before any cycle-3 confirmation detector score", + "recorded_at_utc": "2026-08-23T03:51:31.062376+00:00" + }, "promotion_decision": { "ablation_note": "arm C (general rule plus punctuation only) achieves the same mean achievable coverage as arm B; the cycle-2 word list in arm D adds candidates but no additional coverage; the general rule is the operative mechanism", "confirmation_plan": { From 0e9429d1fe17d9fc4c7f000812272a8e4e59d53f Mon Sep 17 00:00:00 2001 From: Cyinh Date: Sun, 23 Aug 2026 00:08:22 -0400 Subject: [PATCH 28/30] Make currency span extraction invariant to preceding whitespace --- fuckmark/transforms/protected_patterns.py | 3 +- ...t_protected_currency_context_invariance.py | 35 +++++++++++++++++++ 2 files changed, 37 insertions(+), 1 deletion(-) create mode 100644 tests/test_protected_currency_context_invariance.py diff --git a/fuckmark/transforms/protected_patterns.py b/fuckmark/transforms/protected_patterns.py index 45b8f58f..b3a2fd2f 100644 --- a/fuckmark/transforms/protected_patterns.py +++ b/fuckmark/transforms/protected_patterns.py @@ -20,7 +20,8 @@ _SLASH_DATE_RE = re.compile(r"(? list[str]: + return [ + text[span.start : span.end] + for span in ProtectedSpanExtractor().extract(text).spans + if "currency" in tuple(kind.value for kind in span.kinds) + ] + + +def test_currency_extraction_is_invariant_under_preceding_whitespace() -> None: + assert _currency_spans("for $100") == ["$100"] + assert _currency_spans("for $100") == ["$100"] + assert _currency_spans("a $5 and $100") == ["$5", "$100"] + assert _currency_spans("cost: $100,000 total") == ["$100,000"] + + +def test_signed_currency_forms_still_extract() -> None: + assert _currency_spans("- $5") == ["- $5"] + assert _currency_spans("+EUR 3.50") == ["+EUR 3.50"] + assert _currency_spans("USD 5") == ["USD 5"] + + +def test_spacing_before_currency_no_longer_breaks_hard_invariants() -> None: + source = "Apple has been selling its iPhones for $100, and buyers notice." + transformed = "Apple has been selling its iPhones for $100, and buyers notice." + report = validate_hard_invariants(source, transformed) + assert report.status.value == "pass" + assert [r.value for r in report.reasons] == [] From 77354e68a45aee9b93aa73afe2ddca0db0bffaa7 Mon Sep 17 00:00:00 2001 From: Cyinh Date: Sun, 23 Aug 2026 00:09:49 -0400 Subject: [PATCH 29/30] Record currency extractor fix as cycle-3 pre-score amendment --- specs/fuckmark-content-region-coverage-v3.freeze.json | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/specs/fuckmark-content-region-coverage-v3.freeze.json b/specs/fuckmark-content-region-coverage-v3.freeze.json index 881c76b2..c95ed1da 100644 --- a/specs/fuckmark-content-region-coverage-v3.freeze.json +++ b/specs/fuckmark-content-region-coverage-v3.freeze.json @@ -119,7 +119,16 @@ "contract_sha256": "072c360e57abda8fabfafbc23654324a1dfb89efe585c367a0a745aa8474c7b3", "freeze_record_version": "content-region-coverage-v3-freeze-v1", "frozen_at_utc": "2026-08-23T02:55:06.234907+00:00", - "pre_score_amendments": [], + "pre_score_amendments": [ + { + "amendment_id": "pre-score-amendment-1", + "commit": "0e9429d1fe17d9fc4c7f000812272a8e4e59d53f", + "finding": "plan A on corpus 480000 failed fail-closed with PROTECTED_CONTENT_CHANGED: the currency protected-span regex consumed preceding horizontal whitespace whenever a double space preceded a currency amount, so an extra space inserted before \"$100\" changed the extracted span text from \"$100\" to \" $100\"", + "fix": "currency extraction now requires a sign for the sign-whitespace branch, making span text invariant under preceding whitespace; signed forms still extract; regression tests added; no frozen ruleset or profile hash changed (extractor fix only); the fail-closed plan builder behaved correctly", + "kind": "mechanical_correction", + "recorded_at_utc": "2026-08-23T04:09:49.020302+00:00" + } + ], "protocol_id": "content-region-coverage-v3", "protocol_sha256": "ea0de812869d60f7a5cc8e765eb0c5a68db0c61c8722b2347a64752e12fb24a2" } From 0ec055be524b3a27ed973c4a886733ed4ab6571b Mon Sep 17 00:00:00 2001 From: Cyinh Date: Sun, 23 Aug 2026 00:20:20 -0400 Subject: [PATCH 30/30] Record cycle-3 content-region confirmation evidence and final report --- .../confirm3-binding-cross-check.json | 69 ++++++++++ .../cycle3-artifacts-SHA256SUMS.txt | 60 ++++++++ .../final-report.md | 129 ++++++++++++++++++ 3 files changed, 258 insertions(+) create mode 100644 evidence/content-region-coverage-2026-08-23/confirm3-binding-cross-check.json create mode 100644 evidence/content-region-coverage-2026-08-23/cycle3-artifacts-SHA256SUMS.txt create mode 100644 evidence/content-region-coverage-2026-08-23/final-report.md diff --git a/evidence/content-region-coverage-2026-08-23/confirm3-binding-cross-check.json b/evidence/content-region-coverage-2026-08-23/confirm3-binding-cross-check.json new file mode 100644 index 00000000..739d8189 --- /dev/null +++ b/evidence/content-region-coverage-2026-08-23/confirm3-binding-cross-check.json @@ -0,0 +1,69 @@ +{ + "460000-A": { + "blind": true, + "commit": true, + "corus": true, + "evidence": true, + "fixed_threshold": true, + "plan": true, + "rows": "128/128" + }, + "460000-B": { + "blind": true, + "commit": true, + "corus": true, + "evidence": true, + "fixed_threshold": true, + "plan": true, + "rows": "128/128" + }, + "460000-render": true, + "460000-robustness": true, + "470000-A": { + "blind": true, + "commit": true, + "corus": true, + "evidence": true, + "fixed_threshold": true, + "plan": true, + "rows": "128/128" + }, + "470000-B": { + "blind": true, + "commit": true, + "corus": true, + "evidence": true, + "fixed_threshold": true, + "plan": true, + "rows": "128/128" + }, + "470000-render": true, + "470000-robustness": true, + "480000-A": { + "blind": true, + "commit": true, + "corus": true, + "evidence": true, + "fixed_threshold": true, + "plan": true, + "rows": "128/128" + }, + "480000-B": { + "blind": true, + "commit": true, + "corus": true, + "evidence": true, + "fixed_threshold": true, + "plan": true, + "rows": "128/128" + }, + "480000-render": true, + "480000-robustness": true, + "calibration_replay": "7a12862c14ab16febc538c152b2c4d51ecda4b2648f9f8b3ec353ef42dc8ea93", + "threshold_audit": [ + 0, + 256 + ], + "threshold_replay": true, + "threshold_value": 0.5570987654320988 +} diff --git a/evidence/content-region-coverage-2026-08-23/cycle3-artifacts-SHA256SUMS.txt b/evidence/content-region-coverage-2026-08-23/cycle3-artifacts-SHA256SUMS.txt new file mode 100644 index 00000000..229a40d5 --- /dev/null +++ b/evidence/content-region-coverage-2026-08-23/cycle3-artifacts-SHA256SUMS.txt @@ -0,0 +1,60 @@ +8ad6cb1dd4e8b3a00a221735fd76b9174651b84c2d61957813a3a6e0b80088f7 *confirm3-binding-cross-check.json +cafc349e54c5ba7ebc83eb9d39d56969f416f3b2bb6392309d4cfc8081672611 *confirm3-corpus-460000.json +9ae5d941f594e5825d6dfbcfa181a26e2200b12f4bd8142fb6139f91d8523267 *confirm3-corpus-470000.json +60631ea2572042a6e18b2060751835bf65769c1b78c60a7a974517185d22bccd *confirm3-corpus-480000.json +05c2368d2b0cdca6eed2b259ca2bf277b3fa20bc7dcfe9ecb3afc268c034ba8e *confirm3-evidence-A-460000-provenance.json +a7f3bd18d2aa27a407fbdbc8f56be32cc03e731faa0356cb60dcf540a721c087 *confirm3-evidence-A-460000.json +23f2b87130012b5bfcedd5542bf3cee0a08765ec49f5b62d4dd34694db107241 *confirm3-evidence-A-470000-provenance.json +04c14296238e4bc8445310466d78be442d2edcf1e8c12f0f443d40127c4a1808 *confirm3-evidence-A-470000.json +c0e81029ab0a9a7f17f436bbd1b6b64500adb9aef6c242e0fb72bd12a3708d11 *confirm3-evidence-A-480000-provenance.json +a21c29fcb9446eb05a24b942754cb06de40ce74af23ec36396d251d159e75cd5 *confirm3-evidence-A-480000.json +bccaa46f0e13d08f3be8c2cc8dfed0513a3b8adc82866ee31ec7b21b2e35362c *confirm3-evidence-B-460000-provenance.json +36c3ca15202764aab5e58472984ca059f7a28d91b181322efe846d121787a395 *confirm3-evidence-B-460000.json +c33ce1c2f305802eb0f936c37d4c07f79c397c58c4370686263fb814bbbb556c *confirm3-evidence-B-470000-provenance.json +06c9834785b4cdceacfaa5ebb5182e316d4713df3d105d68bc751b8921dd8b46 *confirm3-evidence-B-470000.json +33f3e522923ba741986d41cc29922ab8a89448b26b474ba321d3af17d9adcdd2 *confirm3-evidence-B-480000-provenance.json +175803e559da351ea10a7c609a790b100f270559c084b3f5415b28df9cf03e97 *confirm3-evidence-B-480000.json +ea60b558d22467b5bb2793e871ab63a852f08ff7bd0ca24bf5c41e0a967561d7 *confirm3-plan-A-460000-provenance.json +bde3e082077584b288ce2622be7b6dca11fcee37916c32a27b6cc8be0a31f647 *confirm3-plan-A-460000.json +0ee54a3f28307cc567d676ab97b926959ff810239903eb4444dd8f3871b3c60f *confirm3-plan-A-470000-provenance.json +d08091e1259af5f9957ca678614d517333854a9af2f2138dec32d8fa9837b7ff *confirm3-plan-A-470000.json +f688ce34d5e1f0fabbd64bfd22351425acbd34535f69e50387d4d0f5b7e97028 *confirm3-plan-A-480000-provenance.json +023f138f13ef55858ea8f906596ea3918c8a68710b3167fb817785327a11cc9c *confirm3-plan-A-480000.json +9b94097882131daa8bf08b243332924bee40b9d30eb13c04974f0fffcfde30b7 *confirm3-plan-B-460000-provenance.json +6c311633ca839122d96ce3f2ad965ccd95a78c60b864e472cda69e6d6e549b54 *confirm3-plan-B-460000.json +0330e7a4b98e60e4188245bcfb82fc07f2d18ab4803a1a597df1b47baf4aa6c3 *confirm3-plan-B-470000-provenance.json +07a9688c4abe967c9ad0db055e9f5574134c6ca95dfb7f10f5f3120e2f8ecc47 *confirm3-plan-B-470000.json +26e08d091bb1291a9d77dcbd4cfef1328a7a01742061b3c1d4c2c5b8fba95acd *confirm3-plan-B-480000-provenance.json +f974c48e54bc08f8bfed88e71914ec12aed3fce3436c2b2e36b13af91d4ae50d *confirm3-plan-B-480000.json +e6f38f203666ac68864a9dd959bcf44c8388a7bf1f7d00519bfc59d612f3243e *confirm3-render-manifest-B-460000.json +ce2481fc83d57ef3134286b649d49bd201868f4894f9148b6b915bc3da233bdd *confirm3-render-manifest-B-470000.json +3a3573bad79bf5e10055fa3d3c425f8d72ef78bdcbc16ccc2ab13090e34f4721 *confirm3-render-manifest-B-480000.json +676a736975ba920dc2a22d423822c7e0476ad5a37066ad1433e372d7693c9500 *confirm3-robustness-460000.json +3404c6f7d52cf529fdaf0db340c6c11854155eec8c0e8296148791d0016ce050 *confirm3-robustness-470000.json +469448cd4a39e6c73a1919fdca3c232cd7a9f92528802b5c5528efa11c281b99 *confirm3-robustness-480000.json +5ec923cc0462af4d7e79e62b8fe6ffe3456f8c06c314d8acff3b6c71defee7bf *dev3-corpus-450000.json +e18c77c18cc33d1a1d5b9f90cc565dfe649b1827514d08c3d64f1b7ff8c4b2b4 *dev3-evidence-A-450000-provenance.json +d275295629ca9247241a723d50b67853c70fc2988f5521a7b8d21269db5e65b6 *dev3-evidence-A-450000.json +abe4597f04ab39c7166b0e031885fd7258bed9a604cf39e3f9fd65c3ec2c7b8f *dev3-evidence-B-450000-provenance.json +d2354dc8c4a2d13c7982557734c5d4086ea65153bd0172bfbbfb0d608f10e3dd *dev3-evidence-B-450000.json +425bada8bab9b5cb6fa336406754a120dd61acc4412f5fecd82270376b84e41d *dev3-evidence-C-450000-provenance.json +10f290f556acc85dcfceab1c94ba055eb09268b983d42d7703add81c156df780 *dev3-evidence-C-450000.json +9a58e96b7469b10e5366732e9158ff3c421120794549ce124488fcd71fe8d707 *dev3-evidence-D-450000-provenance.json +ab3017eeb7d08bbed3b4cdb7457dd23a4ba6adff22d0354af383ffd1222e167a *dev3-evidence-D-450000.json +e563703bdabdf866330b0f64cbe06bfbfa5d7b1fd0685a02e8eecf8fa0b55c2a *dev3-holes-A-450000.json +29f575bc5a74f7a2a32f815d5979650fa62c6843a9f7de0de591d4d3d55caf20 *dev3-holes-B-450000.json +ea100383c0ad31819d8224aa513b8de35630c5f4555589e10267baeb36f602de *dev3-holes-C-450000.json +a9e35e3c1afd395799ea05d110751daeedc048f210910784daa7178114d6904f *dev3-holes-D-450000.json +3ad9d5feabfe9958762cfa72ec1b7364d1775bb6d2296c708280e179eca3c27a *dev3-plan-A-450000-provenance.json +7c4257d4a09ece0cc53bc5c6529735caadf2fa4d064d21a2f4720f30d211f5ac *dev3-plan-A-450000.json +d6857024fb97e9c3ed4128f337ecab60a0c2bdb62db5c55db057fd3ca49d65cc *dev3-plan-B-450000-provenance.json +26f12b91f2d32ba1d3de077bfbccfac6629d2b647f009e348c9d91a956c8debe *dev3-plan-B-450000.json +1f8abdbe1db531847bbb4b7ab9c2dd81cec180972a005d4437048088f36d0e94 *dev3-plan-C-450000-provenance.json +60ddaeda97b5f55dff215eb4557dec5f680def2872c1395b7021ff2dc1ec23ad *dev3-plan-C-450000.json +64eb9d5622b0222566d454d16d24c22d5467dfa8ffcf504f46030e9364d7759f *dev3-plan-D-450000-provenance.json +0820650766b19917500cd7d94496bc7313b99837b60c9bf179ea7a9bfd21e1cf *dev3-plan-D-450000.json +99527adb63bcb282a9d64df29cbdae732499f947cd3f520bb2807e789dfad517 *dev3-robustness-450000.json +b775c7671740eaff8c6fdf608e89689025d232b44e770a9ab010d5e4f7fdcca0 *calibration-corpus-500000.json +0b1b9e6ead71caf567e1b21bb3996098298aa207ae86db15c07584faaae09f37 *fixed-threshold.json +1c04bb330363063995433a216c3cbc0306fcfd1535a537ffec0df97c79f4b15d *holes-cycle1-420000.json +43f09bc58f4e275b5d79f87942de1e72e615e01da9b38b3b3def8b9008a516be *holes-cycle2-440000.json diff --git a/evidence/content-region-coverage-2026-08-23/final-report.md b/evidence/content-region-coverage-2026-08-23/final-report.md new file mode 100644 index 00000000..a102d2bd --- /dev/null +++ b/evidence/content-region-coverage-2026-08-23/final-report.md @@ -0,0 +1,129 @@ +# Content-Region Coverage Cycle 3 - Final Research Report (2026-08-23) + +## Repository state + +- Base `origin/main`: `16e153fdfa3739a88bca9b1559543a10ded2d859` (unchanged upstream; no open PRs). +- Research branch: `research/content-region-coverage-v1`, child of Cycle-2 HEAD `4ddd84f`. +- Final HEAD: `77354e68a45aee9b93aa73afe2ddca0db0bffaa7` (confirmation-scoring commit). +- Nothing pushed at any point. Worktree clean after final commits. +- Frozen historical identities verified at takeover: B16 profile hash + `6ad142262bfb11a714565d7bd43daa859657fe817bdafa9c8dcf0f4884c07512` replays; Cycle-1/2 + artifacts intact; full suite 1436 passed / 8 skipped with only the pre-documented + Windows-only `spec.md` CRLF freeze failure (`git show HEAD:spec.md` hashes to the pin). + +## Measurement (Phase A) + +- New identity `open-detector-measurement-stability-v1`. Dedicated calibration corpus: 40 new + topics x 4 domains x 8 seeds = 1280 unwatermarked 64-token negatives (seed base 500000, + artifact `7a12862c...`), prompt/seed/text disjoint from every prior corpus. +- Frozen threshold: ascending order statistic n - floor(0.01*n) + 1 = 1015 of 1024 calibration + scores = **0.5570987654320988**; comparison `>=`; float64; no tie rounding; target FPR 1%. +- Audit: 0/256 exceedances on the independent audit split (exact 95% binomial interval + recorded in the artifact). Calibration exceedances 10/1024 (0.977%). Detector identity + bound and matched at every scoring invocation. +- This replaces per-corpus thresholds (Cycle 1: 0.5585; Cycle 2: 0.5803) with one + corpus-independent value; all six confirmation scorings used it. +- Environment manifest frozen in the contract (Windows-10-10.0.26220 AMD64, Python 3.11.15, + torch 2.13.0+cpu, transformers 5.16.0.dev0 pinned git revision, GPT-2 revision pinned). + Cross-platform corpus-generation byte-reproducibility is NOT claimed. + +## Candidate mechanism and diagnosis chain (Phases B/C) + +- Phase B (structural, on consumed Cycle-1/2 artifacts): uncovered observation windows are + dominated by lowercase content words (447 across both confirmations), then capitalized + content (99) and listed function words in unreachable positions (142); the worst texts are + protected numeric runs (correctly unreachable). New `coverage_holes` module reports token + counts, per-family candidate counts, achievable/realized coverage, longest uncovered run, + lexical categories, protected intersections, density, and redundancy per source. +- Phase C: one new rule, `GeneralWordSpacingRule` (`surface-space-after-any-word`): any + standalone alphabetic word followed by a single space and a non-space character gains one + trailing space. No word list, no hidden representation, case-sensitive, protected spans + rejected by the registry. Profile `content-region-coverage-v1` (hashes in the contract), + budget 16, unchanged COVERAGE_GREEDY_KEY_BLIND scheduler, seed base 1_150_000. +- Structural dev evidence: mean achievable coverage 0.841 (Cycle-2 ruleset) -> 0.941 + (candidate); in the 8 lowest-coverage sources the paired gain was +0.20 (gate >= 0.05); + pools 21.6 -> 45.0; zero-candidate rate 0. Ablation: the general rule alone reproduces the + full gain (arm C); adding the Cycle-2 word list (arm D) adds candidates but zero coverage - + the general rule is the operative mechanism. + +## Development (fresh corpus 450000, four detector-blind arms, per-corpus thresholds) + +All four arms: 1/32 transformed-watermarked detections (same source), 0/32 unwatermarked +false positives, fidelity gates 256/256 rows pass. Promotion gate: all twelve criteria +passed and recorded before confirmation scoring (freeze appendix). + +## Confirmation (three frozen corpora, paired A/B, fixed threshold, scored once each) + +| Corpus | Pristine WM | A (Cycle-2 ruleset) | B (Cycle-3 candidate) | FP A | FP B | +| --- | --- | --- | --- | --- | --- | +| 460000 | 63/64 | 4/64 | 3/64 | 1/64 | 0/64 | +| 470000 | 64/64 | 8/64 | 7/64 | 1/64 | 1/64 | +| 480000 | 63/64 | 10/64 | 2/64 | 0/64 | 0/64 | +| Pooled | 190/192 | 22/192 (11.5%) | 12/192 (6.3%) | 2/192 | 1/192 | + +B is paired-better on every corpus and pooled (relative reduction 45%). Mean watermarked +score drop: B 0.119 vs A 0.113. Margin bands (pooled, B): 9 within +-0.005, 21 within ++-0.010, 51 within +-0.025 of threshold - residual detection remains margin-driven. +Fidelity/representation gates: 256/256 rows per corpus across both arms. Cross-corpus +text-hash disjointness: pairwise zero among the three corpora; one naturally occurring +collision between 480000 and the consumed Cycle-2 corpus 440000 (single degenerate text, +policy followed: scored as generated, no regeneration). DOM text equality 384/384 pairs; +screenshot archived. + +## Adverse findings (all reported) + +1. Currency protected-span extractor bug: a double space before `$100` changed the extracted + span from "$100" to " $100" (`PROTECTED_CONTENT_CHANGED`), failing plan A on corpus 480000 + fail-closed before any scoring. Root cause fixed (whitespace branch now requires a sign), + regression tests added, recorded as pre-score amendment 1. The fail-closed builder worked + as designed. +2. Transformed-unwatermarked false positives: 3 events across 384 arm-corpus cells + (A: 460000 one, 470000 one; B: 470000 one). Two of the three texts were already above the + threshold when pristine (consistent with the fixed threshold's nominal 1% FPR; the 0/64 + gate was calibrated under the old higher per-corpus thresholds). Two genuine + transformation-induced crossings exist (A 0.5234->0.5611; B 0.4510->0.5589, a 0.1079 score + increase on an unwatermarked text). The pre-registered 0/64 FP gate FAILED for arm A on + 460000/470000 and arm B on 470000 and is reported as such. +3. Degenerate generation events: one all-EOS continuation crash path fixed with deterministic + retry in the v3 and calibration generators (pre-scoring); one cross-corpus degenerate text + collision (above). +4. Dev underpredicted absolute residual rates again (1/32 dev vs 12/192 pooled + confirmation for arm B). + +## Classification + +**PARTIAL_SUCCESS** (pre-registered class): the candidate is paired-better than the Cycle-2 +baseline on every confirmation corpus and pooled (22/192 -> 12/192), all structural +mechanism gates passed, all fidelity gates passed, and pristine-watermark control gates +passed on all three corpora (63, 64, 63 of 64) - but residual detections remain (12/192, not +the 0/192 stretch) and the transformed-unwatermarked 0/64 gate failed in three arm-corpus +cells under the fixed 1%-FPR threshold. + +## Permitted conclusion + +Under the fixed open Hugging Face SynthID weighted-mean detector with public development +keys, a frozen corpus-independent threshold at nominal 1% FPR (audit 0/256), and the +uncleaned text-only input path, a detector-blind key-blind general any-word spacing rule +materially increased legal candidate coverage in content-word regions and reduced pooled +transformed watermarked detection from 22/192 to 12/192 relative to the Cycle-2 ruleset in a +paired three-corpus design, while preserving every fidelity, representation, normalization, +and rendering gate. The evidence does not establish zero detection, watermark removal, +undetectability, unknown-key or proprietary-detector performance, cleaning durability, or +release readiness, and the fixed-threshold regime exhibits expected low-rate false positives +on unwatermarked text. + +## Not supported + +Watermark removal; undetectability; SynthID defeat; Gemini or proprietary-detector transfer; +unknown-key transfer; normalization/cleaning durability; human semantic equivalence (blind +human fidelity evidence: NOT_PROVIDED); production readiness or release authorization; +cross-platform byte-reproducible corpus generation. + +## Reproducibility + +Independent binding cross-check over all Cycle-3 artifacts: every corpus, calibration, +threshold, plan, evidence, robustness, and render-manifest hash replays and mutually binds; +128/128 row hashes in all six evidence files; all plans bind the scoring commit and record +detector_access_observed=false / secret_access_observed=false. Full suite: 1436 passed, +8 skipped, 1 pre-documented environmental failure. `git diff --check` clean. Artifact +SHA-256 manifest in this directory; local commits only, nothing pushed, main untouched.