diff --git a/evidence/budget-scaled-coverage-2026-08-22/artifacts-SHA256SUMS.txt b/evidence/budget-scaled-coverage-2026-08-22/artifacts-SHA256SUMS.txt new file mode 100644 index 00000000..1711638f --- /dev/null +++ b/evidence/budget-scaled-coverage-2026-08-22/artifacts-SHA256SUMS.txt @@ -0,0 +1,22 @@ +4a6744ae64b783f770dd426012cd4439c1e2d772416655107682ed38143b2067 *binding-cross-check.json +49b105cecfeec14880d55ff59e578855f4ce9588574f1acd167799e0f9937038 *confirmation-corpus-420000.json +5d08679efcb3bfbe418beb07ec94480f6eefab017708a8ba9ed2afcb472dc07d *confirmation-evidence-420000-provenance.json +7fb0ce6c21e9f7c8a599889591e1ce24ff8cec7c8263da4a87444ffd4949ff5d *confirmation-evidence-420000.json +06cd3d9c496e65a0c7987e5e71d9ea57d25e704b59f14cae4f0d28bf634e1386 *confirmation-plan-420000-provenance.json +3079622ccdb4d86f48b1a77a64a00d6cff41b1918655d5526ae783025185181c *confirmation-plan-420000.json +c9d2686b16e7c0b213bff14475e845f06c53ac64601b53ccdbed1fea2a1287df *confirmation-robustness-420000.json +57b580c8839ea85a650993e302443fec4e16f76ea4d1ff7156fcb5468017f7ce *dev-corpus-401000-local.json +956e9a5c2994dc5a511ea535968cd4bbed68ba1a3ce50a1ca410c56df03ab62f *dev-corpus-410000.json +d0bec9524b623fb582b6f8699bbe4ef412ccd42b2a50662675480133e0f793e8 *dev-evidence-401000-provenance.json +d352c3e4393f088dc47fe40af3c256c71472853dd96432df48350de0673f9ec6 *dev-evidence-401000.json +12a63869c5f68845adfa88967faaae5669845f7e18fca882e98e50300ea3528a *dev-evidence-410000-provenance.json +2222d4293183071ecb479633bd279ddf858655c5539fd6f291030fd1faf4e88a *dev-evidence-410000.json +642cee2ba8b9ce853b020517e85b35ea34867e77b736d53ed95915e291301439 *dev-plan-401000-provenance.json +0fffc9284e502387e9743a281197e7bf989abd3b06dc2b0e72074c98ee941d40 *dev-plan-401000.json +e866f83819de7e4e89d7bad95123c4f8e41d499bfc876e1d056685ada61853dc *dev-plan-410000-provenance.json +18e05ee9d8be733af401e96e4f07dcab9f22048cb2b7b8c6dd860b97a3428a5a *dev-plan-410000.json +cbf6135f43fa6b6718d95320e2c6d5174e916353560efab1c2232f3af2183d1c *dev-robustness-401000.json +3cf84fedf6fc2818cf494873e98ca63c98c179e433d269102fc6f82a7750e99a *dev-robustness-pooled.json +6a8d031b883c1cc7eddfbb46f40d10de9be7233d9a93ca5cedb1b8dc4cf8c013 *rendering-manifest.json +59345fdece25c4ebe5c1123fabd525c957b665f6fc39db8bd069122eda484281 *confirmation-render-pairs.html +b0c740c804e2555f2b4a5616590cd342a2c60275686a5e7f8a44f29b92eb608c *confirmation-render-screenshot.png diff --git a/evidence/budget-scaled-coverage-2026-08-22/binding-cross-check-2.json b/evidence/budget-scaled-coverage-2026-08-22/binding-cross-check-2.json new file mode 100644 index 00000000..3cef3ef7 --- /dev/null +++ b/evidence/budget-scaled-coverage-2026-08-22/binding-cross-check-2.json @@ -0,0 +1,14 @@ +{ + "corpus_artifact_hash_replay": "ccc9d5d6ffe5168f8059f7b2ab5f418050ed11b487e34e68ebf1634284a96760", + "evidence_binds_plan": true, + "evidence_hash_replay": true, + "evidence_profile": "key-blind-coverage-completion-v2", + "evidence_row_hash_replays": "128/128", + "plan_binds_corpus": true, + "plan_detector_access_false": true, + "plan_hash_replay": true, + "render_manifest_binds_plan": true, + "render_manifest_pair_count": 128, + "robustness_binds_plan": true, + "robustness_hash_replay": true +} diff --git a/evidence/budget-scaled-coverage-2026-08-22/binding-cross-check.json b/evidence/budget-scaled-coverage-2026-08-22/binding-cross-check.json new file mode 100644 index 00000000..e02b619e --- /dev/null +++ b/evidence/budget-scaled-coverage-2026-08-22/binding-cross-check.json @@ -0,0 +1,20 @@ +{ + "corpus_artifact_hash_replay": "85cdda3bef549c961caac07767126ec2924f360f299b5d5a6b43f4272aa7e2ba", + "evidence_binds_corpus": true, + "evidence_binds_plan": true, + "evidence_hash_replay": true, + "evidence_profile": "key-blind-full-pool-coverage-v1", + "evidence_row_hash_replays": "128/128", + "evidence_row_texts_match_plan": true, + "evidence_threshold": 0.5584848484848485, + "evidence_threshold_hash_present": true, + "plan_binds_commit": true, + "plan_binds_corpus": true, + "plan_binds_manifest": true, + "plan_detector_access_false": true, + "plan_hash_replay": true, + "render_manifest_binds_plan": true, + "render_manifest_pair_count": 128, + "robustness_binds_plan": true, + "robustness_hash_replay": true +} diff --git a/evidence/budget-scaled-coverage-2026-08-22/cycle-2-report.md b/evidence/budget-scaled-coverage-2026-08-22/cycle-2-report.md new file mode 100644 index 00000000..9c91d322 --- /dev/null +++ b/evidence/budget-scaled-coverage-2026-08-22/cycle-2-report.md @@ -0,0 +1,100 @@ +# Coverage-Completion Cycle 2 Report - 2026-08-23 + +Branch `research/ngram-window-coverage-v1` (local only, nothing pushed). Confirmation scored +once at commit `6d0ad2d2...` on corpus `ccc9d5d6...`. Protocol SHA-256 +`65856295a595acaa404392239ccd9db9fa43d3f45a86ae61ee0d88b0d1e0851a`, contract SHA-256 +`20ba6145793a89f0aa5cbc530305d9345731a60d60ad906199b323c74942d825`. + +## Executive summary + +On a new, source-disjoint 64-watermarked-source confirmation corpus, the frozen +detector-blind key-blind coverage-completion profile achieved **0/64 transformed watermarked +detections** (primary stretch target met for this condition) with **0/64 transformed +unwatermarked false positives** and all 128 fidelity/representation gates passing. One +pre-registered secondary gate FAILED and is reported as such: pristine watermarked detection +was 58/64, below the required 60, because this corpus's per-corpus calibration threshold rose +to 0.580261 (cycle 1: 0.558485), leaving six watermarked sources below threshold before any +transformation. Conditional on initial detectability, 0 of the 58 pristine-detected sources +remained detected after transformation. The permitted conclusion is unchanged in kind: +detector-specific detection reduction under the fixed open detector, public development keys, +and uncleaned text-only input path - here reaching the pre-registered narrow 0/64 target on +this corpus. It is not universal undetectability, watermark removal, or any unknown-key, +proprietary-detector, or cleaning-durability claim, and Level B (product admissibility) +remains unmet. + +## Mechanism and diagnosis chain + +Cycle 1 ended at 11/64 residual detection. The post-hoc coverage diagnosis (tool: +`tools/budget_scaled_coverage_diagnosis.py`) showed residual sources had mean achievable +n-gram coverage 0.682 vs 0.847 for escaped sources, with realized coverage equal to achievable +everywhere - the candidate pool, not the scheduler, was the bottleneck. Cycle 2 extended the +surface spacing word list by 65 frozen general-English function words (protocol section 2), +raising mean achievable coverage on confirmation watermarked sources to 0.827 (mechanistic +endpoint >= 0.80 met) and mean candidate pools to 21.9 per source. + +## Development phase (recorded before confirmation scoring; freeze appendix) + +| Corpus | Cycle-2 ruleset | Cycle-1 ruleset (paired) | +| --- | --- | --- | +| 410000 (repeat use, labeled) | 0/32 detected | 1/32 (cycle 1) | +| 430000 (fresh) | 5/32 detected | 4/32 (paired baseline) | +| Pooled dev | 5/64 | 5/64 | + +Dev showed NO pooled improvement; the extension grew pools where function words exist but +residual sources are content-word-dense with low candidate density (one had 5 candidates +total). Residual sets overlapped strongly across rulesets, tied to high pristine scores plus +sparse candidate regions. The confirmation proceeded as pre-registered and the outcome +exceeded the dev signal; corpus-to-corpus variance is high in both directions (cycle 1: +1/36 dev to 11/64 confirmation; cycle 2: 5/64 dev to 0/64 confirmation). This variance is +itself a finding: single-corpus dev counts are weak predictors. + +## Confirmation results (64 wm + 64 unwm, budget 16, threshold 0.580261 at 1% target FPR) + +| Condition | Detected | Rate | Mean score | +| --- | --- | --- | --- | +| Pristine watermarked | 58/64 | 90.6% | 0.6348 | +| Transformed watermarked | 0/64 | 0.0% | 0.5283 | +| Pristine unwatermarked | 0/64 | 0.0% | - | +| Transformed unwatermarked | 0/64 | 0.0% | - | + +Mean watermarked score drop 0.1064. Closest transformed margin to threshold: -0.0020 (one +source), then -0.0115. Realized edit cost mean 12.72 of mean pool 21.9; zero-candidate +sources: 0. Six pristine-watermarked sources below threshold (0.5590-0.5775) are listed in +the evidence and constitute the failed >= 60/64 control gate. + +## Gates, robustness, rendering, bindings + +- Section-6 gates: 128/128 confirmation rows pass (hard invariants, protected spans zero, + max word edit rate 0.1333 <= 0.30, no introduced non-ASCII or hidden-category codepoints, + no introduced normalization instability, Cf-strip identity on introduced characters). +- Cross-corpus text-hash disjointness: all pairwise overlaps zero among 410000, 430000, + 440000; also zero against cycle-1 corpora 401000 and 420000 (no repeat of the cycle-1 + degenerate collision). +- Fixed-browser rendering: DOM textContent equals plan texts on 128/128 pairs; screenshot + archived (`confirmation2-render-screenshot.png`). +- Independent cross-check: corpus/plan/evidence/robustness/render-manifest hashes all replay + and mutually bind; 128/128 evidence row hashes replay (`binding-cross-check-2.json`). + +## Adverse findings and caveats + +1. The >= 60/64 pristine-watermarked control gate failed at 58/64 (threshold sensitivity of + per-corpus calibration); the 0/64 primary result must be read together with this. +2. Dev predicted no improvement (5/64 both rulesets); the confirmation result therefore + carries corpus-sampling uncertainty. A single 0/64 does not establish a stable zero rate. +3. One transformed source sits 0.0020 below threshold - margin-level sensitivity persists. +4. Cycle-2 word extension did not reduce dev-phase residuals; residuals concentrate in + content-word-dense, low-candidate-density sources that word-list extension cannot reach. +5. Canonical-corpus cross-platform reproducibility limitation from cycle 1 still applies. + +## Classification + +- Product-safe code: extended surface ruleset (superset, base ruleset and frozen B16 profile + hash `6ad14226...` unchanged and test-pinned), coverage-completion profile + resolver + + registry selection, diagnosis tool, 8 new tests (total suite 1422 passing plus the known + Windows-only `spec.md` environmental failure). +- Candidate outcome: narrow pre-registered 0/64 target MET on this corpus and condition, + with the control-gate failure reported; exploratory only; NOT product/release admissible + (visible-text edits); no overclaim. +- Next research lever identified by diagnosis: candidate generation for content-word-dense + regions (rules that can place a visible-text-safe edit anywhere), which word-list + extension cannot solve. diff --git a/evidence/budget-scaled-coverage-2026-08-22/cycle2-artifacts-SHA256SUMS.txt b/evidence/budget-scaled-coverage-2026-08-22/cycle2-artifacts-SHA256SUMS.txt new file mode 100644 index 00000000..24e56987 --- /dev/null +++ b/evidence/budget-scaled-coverage-2026-08-22/cycle2-artifacts-SHA256SUMS.txt @@ -0,0 +1,24 @@ +59ae8aa407b4f9b93b93c8f50b968cfd154d84d0795b38360a9be9e54fa981cd *dev2-evidence-410000-provenance.json +e377fc52d0bfd766ec5824f2658d9897466c9d975bcd658623c2e579ff84a95a *dev2-evidence-410000.json +ea1b3b9dff40f955d8761ec7683725c5f9319f3ac37fdf30c58cab8bfa5d050e *dev2-evidence-430000-provenance.json +7d9fbe1229a511b16a1460954d947f8fa6cad58afd8262f2122393bbb0f36c5d *dev2-evidence-430000.json +3dd70d23a852c220d91d719c8e78cdbc1894e1bdc596ec0c0a69bbb9844339f3 *dev2-plan-410000-provenance.json +0573f47ff38e71e59e282482f796c0d963ddcee31b476d3a7805e82dc8af13a9 *dev2-plan-410000.json +bdbb1c2148acb1fa4e23e6ca3ff3d4185384e5eff6081d9b0a5b1a5d735a09e4 *dev2-plan-430000-provenance.json +3863cc67c67e357d5c67266f42d84dd7ddb3ad4869847f387e84ae0fd8f5a334 *dev2-plan-430000.json +0bc77aea8e70b3f8a97b67e1824440e7959004a9fdf785c2eb514ccab53ba1a3 *dev2-robustness-pooled.json +2129f7dc5c5b16d7fece545c4cf1ae156ccd6b16caa9dc27af1500e1887de56e *dev1-plan-430000.json +97fd81a8346c2c51acc3299e364f37a9c83a7ae9b07b74cacb57cc0476194a48 *dev1-evidence-430000.json +fb066647ad15c86e5c09ba5b1753ddbcb53e490d1a1a97e32e36a26016e93036 *confirmation2-corpus-440000.json +13cdd4aa2ddbb359096aa3fa1e95e3c32a7d8b576696a6455280caee2bc2bbe9 *confirmation2-evidence-440000-provenance.json +bde997a0103c60b7604815a074200543cd157fd3af28f59575eb9acef24cc568 *confirmation2-evidence-440000.json +1b8608f4a23ad21e1090582787b645c8e48b38d6a20cfc287abdcbaa65f888cf *confirmation2-plan-440000-provenance.json +a801ad020a339ba802467b0b0e48f501f95a9fc4b177ac8ea68b4153095c38e8 *confirmation2-plan-440000.json +48ad930bf3ef424595ec77910c48fa15dc89e1e63ca1724a988556a9a30d23b0 *confirmation2-robustness-440000.json +acb612a2294e5a2b269ee27169195afcf54309d5eb73d5ff6aaf3da840fef835 *coverage-diagnosis-420000.json +b8636ec8334c8b5b24fbacd6cd0731a79f4ddd05426a37bdce4946de5ae45aee *coverage-diagnosis-confirmation2-440000.json +2e3e904c035fcd169b6b4a8cc49b95338028fdbd4871c900f39a772517aa7cb1 *coverage-diagnosis-dev2-410000.json +36508360e70baf33b3eb8cb316a12a3aae3fbbbc87bacb9b0addc5db60d8df2e *coverage-diagnosis-dev2-430000.json +5d04b758e9d6964cf9501aa0416a455bd00b985c0fdb03dacbfbdb51fef5ce58 *binding-cross-check-2.json +132ec9541ca134c0f5f2f15932da5e3775653c34314ffef8241ef8d5ac3bebf5 *rendering-manifest-2.json +3e6de60f70a82e6b6fabc20ef010cec7051295a9199d8ee159e40bbeb1e8d7fb *dev-corpus-430000.json diff --git a/evidence/budget-scaled-coverage-2026-08-22/final-report.md b/evidence/budget-scaled-coverage-2026-08-22/final-report.md new file mode 100644 index 00000000..de3a8dd7 --- /dev/null +++ b/evidence/budget-scaled-coverage-2026-08-22/final-report.md @@ -0,0 +1,162 @@ +# Budget-Scaled Key-Blind Coverage Effectiveness - Research Cycle Report + +Cycle date: 2026-08-22/23. Research branch: `research/ngram-window-coverage-v1` (local only; +nothing pushed). Base commit: `16e153fdfa3739a88bca9b1559543a10ded2d859` (`origin/main`, which +already contains merged PR #80). + +## Executive summary + +The pre-registered stretch target was NOT met. On a new, source-disjoint confirmation corpus of +64 watermarked attack sources, the frozen detector-blind key-blind budget-16 coverage profile +reduced fixed-open-detector detection from 63/64 (pristine) to 11/64 (transformed) - an 82.5% +relative reduction in the detected count under the documented condition - but did not reach 0/64. +All fidelity and representation gates passed on every confirmation row. The evidence supports +only: detector-specific detection reduction under the fixed open detector, public development +keys, and uncleaned text-only input path. It does not support watermark removal, +undetectability, unknown-key, proprietary-detector, cleaning-durability, or release claims. +Level B (product/release admissibility) is explicitly NOT met and was not pursued: the +candidate intentionally edits visible text. + +## Protocol provenance + +- Protocol, contract, and hash-binding freeze record committed before any scoring + (commit `29d7a40`). +- Five timestamped pre-score amendments, all recorded before confirmation scoring: + 1. Seed bases moved to 410000 (dev) / 420000 (confirmation) - pair-stride overlap arithmetic. + 2. CI canonical corpus pin (`ee09527d...`) is not byte-reproducible on the local Windows CPU + runtime (locally regenerated 401000 corpus `42240f72...` / file `57b580c8...` used as + development data only; adverse reproducibility finding, reported not hidden). + 3. Word-edit-rate gate raised from 0.15 to 0.30 (contractions cost two word edits each; the + original gate was below the mechanical floor of the ruleset). + 4. Codepoint audit re-scoped to candidate-INTRODUCED characters (absolute ban flagged + pre-existing source newlines, category Cc). + 5. Normalization gate re-scoped to introduced instability (precomposed accented source + characters are NFD-unstable; the candidate introduces ASCII only). +- Final protocol SHA-256 `4fa68c7519a2064c2bf131393598c5b181b92f2ae5fd62f9fd1959b7ebea0367`, + contract SHA-256 `c878ff9fdbf4c412361f6058a72af1fa8c925e78997c89825d255c8d437f598f`. +- Budget selection rule applied exactly as frozen: no budget reached 0/36 pooled dev + transformed-watermarked detections (all budgets tied at 1/36 because n-gram-window coverage + saturates at 9-13 edits, below the smallest swept budget), so the fallback selected + the smallest budget, 16. Recorded in the freeze-record appendix before confirmation scoring. + +## Development results (iteration data; separate from confirmation) + +| Corpus (seed base) | Watermarked sources | Transformed detected | False positives (unwm) | Mean score drop | +| --- | --- | --- | --- | --- | +| 401000 local | 4 | 0/4 | 0/4 | 0.0980 | +| 410000 (v3, 8 attack pairs/domain) | 32 | 1/32 | 0/32 | 0.1052 | +| Pooled | 36 | 1/36 | 0/36 | - | + +Budget sweep {16, 24, 32, 48} produced identical detections at every budget on every dev source +(coverage saturation; realized edit costs 9-13). Dev robustness gates: 288/288 rows pass. + +## Confirmation results (scored once, at commit `46c220f`, profile `key-blind-full-pool-coverage-v1`, budget 16) + +Detector: HF SynthID weighted mean, ngram_len 5, public development keys, uncleaned text-only +re-tokenized path. Threshold 0.558485 calibrated at target FPR 0.01 on the confirmation corpus's +own 100 unwatermarked calibration negatives (>= comparison). + +| Condition | Detected | Rate | Mean score | +| --- | --- | --- | --- | +| Pristine watermarked | 63/64 | 98.4% | 0.6333 | +| Transformed watermarked | 11/64 | 17.2% | 0.5315 | +| Pristine unwatermarked | 1/64 | 1.6% | - | +| Transformed unwatermarked | 0/64 | 0.0% | 0.5029 | + +Mean per-source watermarked score drop: 0.1018. Residual detection by domain: +technical_explanation 5/16, conversational_prose 3/16, structured_instructional 2/16, +general_explanatory 1/16. Eight of eleven residuals sit within +0.024 of the threshold; four +non-detected sources sit within -0.012 - the residual is margin-driven. Realized edit costs: +mean 11.89 of a mean candidate pool of 18.6; only 2/64 sources exhausted the budget-16 cap; +one source enumerated zero candidates and passed through unchanged (still detected). + +Pre-registered secondary gates: pristine watermarked >= 60/64 PASSED (63; one weak-watermark +source below threshold at 0.5569 vs 0.5585, reported as an adverse observation). +Transformed-unwatermarked == 0/64 PASSED. Section-6 fidelity gates on 128/128 confirmation +rows PASSED (hard invariants, protected spans zero, word edit rate max 0.1333 <= 0.30, no +introduced non-ASCII or hidden-category codepoints, no introduced normalization instability, +Cf stripping is the identity on every introduced character by construction and per row). + +Primary endpoint (0/64): FAILED. + +## Representation, normalization, and rendering evidence + +- Codepoint audit: no introduced Cc/Cf/Mn/Mc/Me/Zl/Zp or non-space Zs characters on any of the + 128 rows; no introduced non-ASCII characters at all. Pre-existing source newlines and two + precomposed accented characters (NFD-unstable sources) are corpus properties, reported. +- Normalization: NFC/NFD/NFKC/NFKD stability preserved on all rows (sources stable under a form + imply transformed stable under it). The two NFD-unstable sources were unstable before + transformation. +- Fixed-browser rendering: all 128 source/transform pairs rendered in a fixed browser page; + DOM textContent equals the plan texts byte-for-byte on 128/128 pairs (0 source mismatches, + 0 transformed mismatches); screenshot archived + (`confirmation-render-screenshot.png`, SHA-256 `b0c740c8...`). +- Copy/DOM divergence: none possible by construction and none observed (no control or format + characters introduced). + +## Independent hash cross-check + +Reloaded artifacts from disk and recomputed every binding: corpus artifact hash replays; +plan hash replays and binds corpus artifact hash, manifest hash, code commit +`46c220f1c0dfe4c2b401cc32ecf4aaf70ac5ed7b`, detector_access_observed=False, +secret_access_observed=False; evidence artifact hash replays, binds the plan hash, corpus hash, +profile id `key-blind-full-pool-coverage-v1`, detector identity, calibration bundle, and +threshold; 128/128 row hashes replay; every scored transformed text hash matches the plan +variant's transformed text; robustness artifact hash replays and binds the plan; rendering +manifest binds the plan with 128 pairs. See `artifacts/budget-scaled/binding-cross-check.json` +(copied into this evidence directory). + +## Adverse and deviation findings (all reported, none suppressed) + +1. Canonical CI corpus pin is not byte-reproducible on this Windows CPU runtime despite pinned + versions; corpus generation is platform-sensitive. All within-cycle comparisons use one + runtime and per-corpus calibration, so the science is internally consistent, but + cross-platform regeneration of the same corpus artifact is not guaranteed. +2. Disjointness deviation: one confirmation attack text is byte-identical to one development + corpus (401000) watermarked calibration sample - a degenerate GPT-2 repetition under + different prompts and seeds. Calibration thresholds use only unwatermarked negatives, and + no development tuning ever read this text's detector score, so no optimization leak is + possible; the text-hash disjointness gate (all pairwise corpus overlaps zero) was violated + once (1 shared hash, dev-corpus-401000-local to confirmation corpus) and is reported as a + protocol deviation. +3. Development underpredicted confirmation residual detection (1/36 dev vs 11/64 + confirmation). The budget sweep showed saturation, so no frozen-rule budget choice could + have changed the outcome. +4. One pristine unwatermarked source scored above threshold (1.6% observed vs 1% nominal FPR; + consistent with binomial noise at n=64, P >= 1 at 1% is about 47%). +5. One pristine watermarked source scored below threshold (weak watermark signal). +6. One confirmation source enumerated zero transform candidates (no rule matched) and was + passed through unchanged. + +## Classification + +- Product-safe code delivered: corpus profile v3 (parameterized attack pairs, frozen + calibration), full-pool coverage profile factory with resolver, parameterized plan/score + runners (defaults preserve the frozen B16 CI behavior), robustness audit module with + introduced-codepoint and normalization-stability semantics, render-page tool, and 24 new + tests. +- Quarantined experiment status: unchanged; the U+200C visible-projection experiment was not + touched and remains excluded from default, development-release, and CLI paths. +- Candidate outcome: exploratory partial effectiveness under one fixed condition; primary + target failed; NOT product/release admissible (visible-text edits); no overclaim. +- Deviations: enumerated above; the corpus-generation platform sensitivity is the most + significant reproducibility limitation. + +## Reproducibility + +Full suite at final state: `python -m pytest -o addopts=` -> 1415 passed, 8 skipped, 1 failed +(`test_frozen_revision_2_spec_is_exact`, the pre-existing Windows CRLF-only `spec.md` freeze +assertion documented in PR #80; `git show HEAD:spec.md` hashes exactly to the pinned value). +`git diff --check` clean. Artifact SHA-256 manifest: `artifacts-SHA256SUMS.txt` in this +directory. Environment: Python 3.11.15, torch 2.13.0+cpu, transformers 5.16.0.dev0 at the +pinned git commit, Windows-10-10.0.26220. + +## Permitted conclusion, precisely + +Under the fixed Hugging Face SynthID weighted-mean open detector with public development keys +and the uncleaned text-only input path, at the pre-registered frozen threshold, the budget-16 +key-blind coverage profile reduced watermarked detection from 63/64 to 11/64 on a new, +source-disjoint 64-source confirmation corpus while passing all pre-registered fidelity, +representation, normalization, and rendering gates. It did not reach zero detection. Nothing in +this cycle demonstrates watermark removal, undetectability, robustness against input cleaning, +unknown keys, proprietary detectors, or release readiness. diff --git a/evidence/budget-scaled-coverage-2026-08-22/rendering-manifest-2.json b/evidence/budget-scaled-coverage-2026-08-22/rendering-manifest-2.json new file mode 100644 index 00000000..4821bb29 --- /dev/null +++ b/evidence/budget-scaled-coverage-2026-08-22/rendering-manifest-2.json @@ -0,0 +1,1030 @@ +{ + "pair_count": 128, + "pairs": [ + { + "pair_index": 0, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-conversational_prose-25-unwatermarked", + "source_text_hash": "27f8ba0dbcce150c40fe33527c222a7defee9c6b505c282a353cdb3f6e1b98ea", + "transformed_text_hash": "e1a0ad09e1835bdd68e4d4e8c1afb198938292cf924df5bc94ec448917eadc00", + "variant_hash": "f30f275162a5f0d65fcac8f6de949eef3ab98f412a5cb03c55a62d57088e8126" + }, + { + "pair_index": 1, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-conversational_prose-25-watermarked", + "source_text_hash": "2d0b08c069db8ffefad2a49d48ae1ce4fb2b7984ac535dfc639fabc00cda87f7", + "transformed_text_hash": "c4ca400a079c7da663718bcc076546b0835292c4a38adaf5a8df9342bf0edeb3", + "variant_hash": "a8b8194e53923943225868e4e16bcd2ff340be63c3afa65f2890b9f917415068" + }, + { + "pair_index": 2, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-conversational_prose-26-unwatermarked", + "source_text_hash": "fd861020eaacb6c8b20a6614c6f89f571b884aa4daf851c5be9ef21f38de39ae", + "transformed_text_hash": "a50078d6a071e29e39b66acf7d5fd76306194891c5817e6d88022bee0c1a1c4b", + "variant_hash": "ae1433cf011acc6039ce11ffb53f54e2996ad819783333ec270c0e6144e2e201" + }, + { + "pair_index": 3, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-conversational_prose-26-watermarked", + "source_text_hash": "e11ff30e17146ba71ab13b87113c1e5e02c06258001f91ef745f2ce82cea2632", + "transformed_text_hash": "25024f4f126fff75a102704dab378d875debe1d677281f5a67b63445d56347f7", + "variant_hash": "9bce3d49124167aac1e5ed18bcb963bb6ff43dece4432b214f5186844662589b" + }, + { + "pair_index": 4, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-conversational_prose-27-unwatermarked", + "source_text_hash": "b966102c29e33ea669c11a19c1cb936e0b9240de92ab4d927eb3a1fe01733b24", + "transformed_text_hash": "23bc52b91ec0e30f622b356954b814b4b699d32799a82ed8c3a6ed02df4ec3ca", + "variant_hash": "0e85e03929638a31f062dfdb7d4fee9bd0bc6c4cd965c1563a0aefb09711e14a" + }, + { + "pair_index": 5, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-conversational_prose-27-watermarked", + "source_text_hash": "c87df08809f5344784f7ee04b81c1ba6394f6a2bad6f779a26f9cb1e19bf969a", + "transformed_text_hash": "bf52c8dc94478da05ab037fa177005640071419d68ad6018a52e29422264681a", + "variant_hash": "6752a3a3f3379d7c92fe07c004df1c483b8ef9f757918060dd73e1cf7b544d69" + }, + { + "pair_index": 6, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-conversational_prose-28-unwatermarked", + "source_text_hash": "72ac69c200379ad6025858414a2e84bbf45ab91cb70135a4487beee1b5f86884", + "transformed_text_hash": "dc0ad761e1bf2e26aa018bf76812e728cdad166f5c6f387b9e31f899d9069d91", + "variant_hash": "b6375d95dc90731b45718a1c16a09a3f5f70ccc1442e31610e20bb82d05cad88" + }, + { + "pair_index": 7, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-conversational_prose-28-watermarked", + "source_text_hash": "1faea3a583e510b5d458a9044035fd2d10a34669acc5a63fa8c721cfd6d6c639", + "transformed_text_hash": "7049a4318d7e8c5995e6cf436c025d3ecc40bfa0c7d698d0da6c9a37915dcd60", + "variant_hash": "4311d8a7d3f822a4775fc3aef230132cbf59d193632fc4b9435883e21ab6c452" + }, + { + "pair_index": 8, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-conversational_prose-29-unwatermarked", + "source_text_hash": "8fb32ae0f622e56fd8670d15ef6ae922525f178800d9fd7fb3b12b7d420e9a78", + "transformed_text_hash": "007870db7404383d6a4c8234208ab4e652179c5b6db9cb460059a016d72656a1", + "variant_hash": "a9e778e9b903909c5305f2b93876661e469df5c5e3db8c60043b94571403daac" + }, + { + "pair_index": 9, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-conversational_prose-29-watermarked", + "source_text_hash": "ceb303e7722bbf5e4215cb309dd99cc67b8652155486e66432fa83023b68032e", + "transformed_text_hash": "3dab095166fcb0f2befd8b0299e52c8b7911e2935b467becb7ede3cb1918de56", + "variant_hash": "9cc8e6b68236c7923c3c6ce589cf4ea6e2a0c61d8c303164b29311806d7ff891" + }, + { + "pair_index": 10, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-conversational_prose-30-unwatermarked", + "source_text_hash": "f7efa61bcc45f3aaee42d618be9add3622fd5e71187060cb49b28f68c2ae74e5", + "transformed_text_hash": "808be4a549e1d7c02bc3a3032c3a50e9468d8a0c90a3eae46684dc7473c10391", + "variant_hash": "4a46572049851c82efebe06b90ba3f9eb9f79e5e48188b41503661ab65d0448b" + }, + { + "pair_index": 11, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-conversational_prose-30-watermarked", + "source_text_hash": "ca283ea301fb9c7536a9b489e69003074fb19cc3a21147e959078115e63e2fec", + "transformed_text_hash": "7c2de3cf637e56e94ace3bbb211b6d13e25d2d5fb946be538fc6c5f7af000d60", + "variant_hash": "1aa86d41fce41101293a50ef0b15ae4a50a01ef3a8b301fc539e5850cf282972" + }, + { + "pair_index": 12, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-conversational_prose-31-unwatermarked", + "source_text_hash": "1d322d153d43fe1943cbe21da3039e7e51ab738279fd87a9e630d112ccc86d5a", + "transformed_text_hash": "f846bbdf1efcd91b5865074187e3912900ec9541d8107e8bc13c74ec9a783dd8", + "variant_hash": "71dcf1f90268fa5ffb46f44a615fa6918d1b5b94e7a8c7ad8746556e294a4d5d" + }, + { + "pair_index": 13, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-conversational_prose-31-watermarked", + "source_text_hash": "57caa5c5f83673b2d39621b009482f075bde0c8b850aee23964829b9f2a61106", + "transformed_text_hash": "08bbb8b63974ad82d9e1060edd1047a36c7df550be4d7fbd30143972e5281233", + "variant_hash": "87ac1a170a2468c0df00ad05829764b7860ac238ebfc84a120d65941ef788d5a" + }, + { + "pair_index": 14, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-conversational_prose-32-unwatermarked", + "source_text_hash": "eb2187803c4ab790422393b22f4f3f0ac2847e7b40c7402363d22ceceb55e550", + "transformed_text_hash": "ef07cb61b9338e7d5c139c766a754b8c3ba3f37880b7b14505e82c62898e4b6e", + "variant_hash": "1334d911e33e7b016c211ca494a3a90032dbc541612ee4d519f78ff14a703a3e" + }, + { + "pair_index": 15, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-conversational_prose-32-watermarked", + "source_text_hash": "3673bb2d00c0c6fb331c3730ce9d4f53f0edcdc771f1e5e163f4c4a7d4cfd77a", + "transformed_text_hash": "7809e783d2a9ce6fefa846615b3950cc342a0d2504a9af2b02b7d0798999905b", + "variant_hash": "938c430fb5d4a21eb46ad0c03d5b263a33f52ec379f31879b4a390b8c4927df2" + }, + { + "pair_index": 16, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-conversational_prose-33-unwatermarked", + "source_text_hash": "507507a54c85af546ab81435c144622b9f8ad3f9161db63a1da535ba8bd6d108", + "transformed_text_hash": "4e6e50470de1454483d901a2c83de19719590371501e801c4d5962bad80773a0", + "variant_hash": "0cb9e6bdb3d9d1e729ff11e66256995a54ad1c9c5cc55720f9094c22e0d48a38" + }, + { + "pair_index": 17, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-conversational_prose-33-watermarked", + "source_text_hash": "3f0db812a783757f321c6f20893b4aed0ca926aa1948a625f3dab16d5d09dee4", + "transformed_text_hash": "31ea2f30cc4b26caca6a27fdabfe0348b6bb0966cdfdb78a81d98214608ca2aa", + "variant_hash": "1d3dbd96c449db4a5962f9f3ffca7b18327bbb0cc3b3e43a47907f8c7241e00d" + }, + { + "pair_index": 18, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-conversational_prose-34-unwatermarked", + "source_text_hash": "279966844e8bfc586f968bb0109e2a8520f7daf194755df67b827595cb5c174a", + "transformed_text_hash": "e5bbc32c5fb1438986f5cf3706bb16aeaaa4d6fabeccff833817d0865fe8bc9e", + "variant_hash": "87b740fc9990b5d2862dff25c617d760e9dbf15c1c64cde4fd1bf01ed5a451db" + }, + { + "pair_index": 19, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-conversational_prose-34-watermarked", + "source_text_hash": "fbc5e96d6d7ffa5ad1386319618620275700ff7a466c1a6e6cf2ed038a48c604", + "transformed_text_hash": "a303df732ebf31d0696a2c1e79d1f220e15298decd23de24a86c48ca1bb8918f", + "variant_hash": "cc55e0ec03951e1994d8e8f5d8a9d76cb696940a02ac39be1a0999690135e28a" + }, + { + "pair_index": 20, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-conversational_prose-35-unwatermarked", + "source_text_hash": "c15304545085decb50a67ef45be068b39c6136c031e0d616260d0dc8bbfa4647", + "transformed_text_hash": "7716ba3651596f791d0e8161cb9221767363b86967b2c84a102fd16534af27c0", + "variant_hash": "f578e1b260ce3ed1aafeeb7cf52dfd8eb1e076dc9ab1c3139b02b053ba17b3a0" + }, + { + "pair_index": 21, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-conversational_prose-35-watermarked", + "source_text_hash": "b4d4e1253dc6206e88cd8534f581b59d2255dda1b94c2f534cdf9b986d09f112", + "transformed_text_hash": "b2bd220c36ac86f65ba7355b5fe721ca888822d07ed144a3b939896452a2f1cd", + "variant_hash": "9b14753fac3e3d019023c2de8505e318147df6566da70ec10bff91e09f7046f3" + }, + { + "pair_index": 22, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-conversational_prose-36-unwatermarked", + "source_text_hash": "fdac0cd2fd8ebc7f4d2c4f21aace43d3ad6d9bf11d837e95d3fb6346f04561c2", + "transformed_text_hash": "09230c53357f5975bcb9ef3b84feffbb0fca8371c47af816e94b00b567762e4c", + "variant_hash": "a7d60b9025239cc51098967b545822bc02b4fc16ca01285ab24e59b3f70058d9" + }, + { + "pair_index": 23, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-conversational_prose-36-watermarked", + "source_text_hash": "1c711925c07461a390f3f6848bbed27848555f12092a17544ec97064953e3408", + "transformed_text_hash": "5bb3c2157098fea095ccf98011a54cb4908923174c83fe124627add3c8222d0d", + "variant_hash": "487367a761d46a89e04e1c77fa2c23cc6ef21d6f580dc0809bcc1ca4de8be5bc" + }, + { + "pair_index": 24, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-conversational_prose-37-unwatermarked", + "source_text_hash": "54f4dc986c86fdd126b8a4026997166be45dd5f67f31ed1b40c59704c9e04c45", + "transformed_text_hash": "914d0f213f0e2c5050ba1722a2bffb3dd9422f06b3a7746145c9a7b12421214e", + "variant_hash": "bc0710ab56e7bd7feaef35fb434e4cb93db947cf138832ba4114b8536dc6c416" + }, + { + "pair_index": 25, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-conversational_prose-37-watermarked", + "source_text_hash": "a4b79edc43e35ff68ea5b98bf0f2aa8cfe0c46b9e11abb4b7989c1c11db32601", + "transformed_text_hash": "bd2c89353617a5dbc9a26b931c4d9a5d3d7b9581443dade7a10e13635be4886f", + "variant_hash": "e20b9cd446c4f4c834c414d5124dd67e10a72f0dd73877250bbf5968de8bee24" + }, + { + "pair_index": 26, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-conversational_prose-38-unwatermarked", + "source_text_hash": "c5da8c3adf4cf67e5e65a8ac73c86b2ee5ad14fea3985dbea64a90cbd1716906", + "transformed_text_hash": "9f2b459c9cef0b32584008b66e4ec0d1ffa067ad52fa1210d2bfd7895dad6210", + "variant_hash": "d5edc2cdc64459f0316e4f2d01a861aea87d8f14e28a6a86c202c5bdf59676e8" + }, + { + "pair_index": 27, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-conversational_prose-38-watermarked", + "source_text_hash": "01cc3eb0a3f4234007ec2961ae4391d01bd7a0a322ba53427ea0a87aecc60c66", + "transformed_text_hash": "024173e995b2e51ca9311054fed1610c5787e1085b8ed87649e5e2ba8f5e61ff", + "variant_hash": "ff0dbbb5aa15ad315c679dc04b8f57c94128df3eddfe120bb0894e373c0fcd89" + }, + { + "pair_index": 28, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-conversational_prose-39-unwatermarked", + "source_text_hash": "3296acb17fac31c6486a6cbbe779cea9e321a5fdd21d93b83bfc93be2aecca83", + "transformed_text_hash": "ffb0a356f35679273bf9542da37a2def6a6c2dd13cc5f9eda5378cb92f971a63", + "variant_hash": "a5e0e4c1527eb9a6b09779c3491ec33ebe3be5a227f90c339846f5c8e499a391" + }, + { + "pair_index": 29, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-conversational_prose-39-watermarked", + "source_text_hash": "dfcd1303b5acb0410b50a005a5a43b0370f61e060715e71720bc3d26bf1d6ac4", + "transformed_text_hash": "b8b575549e963128f5ea4042d91a2bdd55a7d336cb279033e0decf5c164c42b3", + "variant_hash": "4c2cc73761a40c4a0f77b606109e8dc69ff61ea7127a50e6bc9d622489dc4201" + }, + { + "pair_index": 30, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-conversational_prose-40-unwatermarked", + "source_text_hash": "651810b18e48404c3e16d5c783dea4763647ab398dc1f9371a116eff9ccbdc2a", + "transformed_text_hash": "ba7c90cd2bf9a6dd16eac63ee8d7b3112e3d27276f0d08c7b3951c0c7405efbc", + "variant_hash": "765c053e85ae6e01b95f52f6a1db41f5ce2adc444605aaf29b738753c04e022c" + }, + { + "pair_index": 31, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-conversational_prose-40-watermarked", + "source_text_hash": "bca8b1bc83ddac15d0d4510292e10c20868ffeb9ae757c72b13b23a07d8c0902", + "transformed_text_hash": "400ed37ca6088e8a0bc3b792036d8918daa8e075298331fdf35dcfd5fa6a7ed4", + "variant_hash": "6bd293b9baa2fcfb1973458cf08d16fcdbe97c62e8dadb34aea41d9572811117" + }, + { + "pair_index": 32, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-general_explanatory-25-unwatermarked", + "source_text_hash": "64d4c84f0594b3f769e3608dd3592b62dd624eff3f8f6ba4fc044a67bc00ffb2", + "transformed_text_hash": "50207fd93e040bb5217affd6fe3472156cd7dc8001ae78c5a6f09a97b2fa0a45", + "variant_hash": "f16d1145f5f2202e26be6c253f19f51eb4a14cc3aaa30c12fd33605354990568" + }, + { + "pair_index": 33, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-general_explanatory-25-watermarked", + "source_text_hash": "4e6f13e90961b4684822da2837b680520f1e6f29d6e9dc37f5c35c8343e3b533", + "transformed_text_hash": "bc655b9fb2a19d8ecae5a8747d0015967f2cae6ecb00d78580bfc2d4c6e1a04d", + "variant_hash": "b306bef89eda2c3fb5b57151d74c8a48acd97edc9c6b9c2838b62055b081b4da" + }, + { + "pair_index": 34, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-general_explanatory-26-unwatermarked", + "source_text_hash": "1b59f0e235ed66350dd80f401d9668effe33a8ad1ca99fcf0de41c5773d9b283", + "transformed_text_hash": "4a27147fd5820482fea92862efec34a21f7da3f1ab9d88dcdcc014558780e553", + "variant_hash": "bf891ebf6b08dead01f337fc86790a5817334e73c7cc9755b8681f7104b87127" + }, + { + "pair_index": 35, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-general_explanatory-26-watermarked", + "source_text_hash": "1ec2baf65366438dab096c8e76ba0f44d6a06246ed38628d783520982cea25e2", + "transformed_text_hash": "ad12189310f4e819d5a629ab43692ea49ab960364f7706bca1574481979f7b88", + "variant_hash": "d4f9f7b586cfdb65028792462b0344a770423c84fe060348d0aec416ce438a7b" + }, + { + "pair_index": 36, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-general_explanatory-27-unwatermarked", + "source_text_hash": "0cdb401a36f5ba3d546c6ea80120e82d109ee2f9ffbe275c6c06b82c27cbca0a", + "transformed_text_hash": "2608d812ab29c7a6de5b5310d0d39983723efb0b3b4dfe82259c4df58aaa11f4", + "variant_hash": "d3940cc0eb45b2e770720715965db3d08e7e3209db11d382f83bbfba73356de7" + }, + { + "pair_index": 37, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-general_explanatory-27-watermarked", + "source_text_hash": "5ac906a0035a7581c7e599203408e23eeb789e96ed328e185a8547fc279cecaf", + "transformed_text_hash": "3c17de9099ffbe2b88f4550ba40d1c97b25a5a4a2c09346cd5ea95294d5157cf", + "variant_hash": "186d2d0c6e441ca27d18fa72ebc94ac5445f82c9459ef8b56c16741df2e67f86" + }, + { + "pair_index": 38, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-general_explanatory-28-unwatermarked", + "source_text_hash": "5f5c1958fc8fd7da24eec5d964bd3971ea5a227c76984829cd834842cd89792f", + "transformed_text_hash": "6af1734b5a7844286d1618eec9e3456f702d82e88b58a0d1563601ee5400a1b2", + "variant_hash": "296cd8250bb884f090deb3d003e116c495f906ac36ee7b58df29d0bc72a357bb" + }, + { + "pair_index": 39, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-general_explanatory-28-watermarked", + "source_text_hash": "ae531606f8dc91563b1eb72b11285b63b946c77580bbcfec51ef43bf527b218e", + "transformed_text_hash": "def9ced085beee4418db8d106577083e4ed5f04227312ba5586dad80d024cb47", + "variant_hash": "b575df1102e750272cc0dc60d518d04bc29a8836adac60d82967fc653ff6fabf" + }, + { + "pair_index": 40, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-general_explanatory-29-unwatermarked", + "source_text_hash": "f30a40a314e972eea4a8e3c3598ec5ed36f8d054d902d71a542e1b71d157e1f8", + "transformed_text_hash": "89052f64fea16f320ea1a0a47eebadf9c884f73351a6e9ef74ec5dc3a4646de0", + "variant_hash": "bd2eca7387e0733dc06c8acf7c345b632a3de34d6191d1badda9108537fff126" + }, + { + "pair_index": 41, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-general_explanatory-29-watermarked", + "source_text_hash": "6ae0c002c314555f07c733930aa59fded696c8a8a5a52734ebcdb04a8b9029f5", + "transformed_text_hash": "a2f92fbd0c7fa7dacb1b6df61fbbc2436bbb6ea1a8b64dc65b56c08059151c46", + "variant_hash": "ad995ed9c7a2081d7ad25818854acf23000b33a591b83e24d616dedfa655e4f1" + }, + { + "pair_index": 42, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-general_explanatory-30-unwatermarked", + "source_text_hash": "8a6ad16474327d7fcf33c22adb7b25854e17d196e37a59b120ffdbb95c72aded", + "transformed_text_hash": "8a6ad16474327d7fcf33c22adb7b25854e17d196e37a59b120ffdbb95c72aded", + "variant_hash": "c231c6e155815752f5c2939d27f3f9eb027a62e53399ac79c1a4cb03ae8b9f7b" + }, + { + "pair_index": 43, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-general_explanatory-30-watermarked", + "source_text_hash": "400b0eed4dffb52b8ca99d304a4032b83f6a3e5be8d42f299fde80dfe930e976", + "transformed_text_hash": "f8e4d1a4929fabd3197dac513bda669e15ea8b1feae8acb58fb194d62dca73e0", + "variant_hash": "7baf4df9ec6faa148b8fc63615e2c6e9335f78e51114aad1434a0835baef6980" + }, + { + "pair_index": 44, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-general_explanatory-31-unwatermarked", + "source_text_hash": "54496cefaa027f97af0732455f7fb4c962f719c2b89c4c8c1bd5a5841c102efc", + "transformed_text_hash": "0e950f725b8fe728236bad7d6c8466f2d1284ee7423995ffb5155950e0adbcda", + "variant_hash": "016cd11dbb5267b6c25204e7b0bf4e84da562d12907983bfc25e6a8979eb0e58" + }, + { + "pair_index": 45, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-general_explanatory-31-watermarked", + "source_text_hash": "c3b77d72c89575514a45f691a3374ddb07a8972a9c4fa07f011a7768ed8ccdf3", + "transformed_text_hash": "29340c2a00db97545bad8d833ec545e087189eaa036ebb6a92b4e4b245c5f5d0", + "variant_hash": "ee53839c17d26e69476b76d6aec1e49e0c55c2800b37f52e6c374dbe23c8ceab" + }, + { + "pair_index": 46, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-general_explanatory-32-unwatermarked", + "source_text_hash": "1b53b9dc18ed374a92992304a3a61740fe8d91b62a64f65695445e354070d768", + "transformed_text_hash": "5d3fdfaeac059449d67939126b7be653a5e23eb0494009f68538c71747afb54f", + "variant_hash": "07f7c6e82b1ed69b96fb4685a99ded89c96e143e85e921a58385fdc0277a2990" + }, + { + "pair_index": 47, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-general_explanatory-32-watermarked", + "source_text_hash": "79e949d224cce0d6b94e7f03595ca7a3e6a3baf9d5e7c43fbd249c42a30c911d", + "transformed_text_hash": "a0c18d5c512e5dad95f4efadd57b33ac4e320d428cc014e7b92015dd5d2fe2b5", + "variant_hash": "18efa2d538f9ec99e022c15adceec04ee589cf2f444b6d6de9cf53f3400556d4" + }, + { + "pair_index": 48, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-general_explanatory-33-unwatermarked", + "source_text_hash": "cc95dca9ee838fc79fae57e7d77212150ed21e8f287cee06a3d338ae51db6ecb", + "transformed_text_hash": "60116f36ee74e829ecea5d2f0c936fdf36cb9984d9365d9e189713d536a29777", + "variant_hash": "cbca89241095bf0b24528161a6e153bca3801b80a6006dc0b00ed0fc9f1287c0" + }, + { + "pair_index": 49, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-general_explanatory-33-watermarked", + "source_text_hash": "cafa503fbe6d77226ca63debacc1c0649b1b59a4d9f0425fd98e0ed91600305c", + "transformed_text_hash": "3863b7c5596772d81451bf745024305aa7eccc221caeb6275d43b0304e44353e", + "variant_hash": "e6fa4b204deae24f70b449098003ecd63928f40c69509ad09574a49056e228ce" + }, + { + "pair_index": 50, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-general_explanatory-34-unwatermarked", + "source_text_hash": "5cab5bf4fb05aac31ad15865fb99fc35c57992693896c8ed0e492ed64bcaeed5", + "transformed_text_hash": "2543bbff9f611b9a8303639399ab2305091a819f0fb6b5519586c494363ea8f6", + "variant_hash": "fc9e0d1025953737b93c52f2d7f6c4bab1b7d185d1747c910e94ddc23837092e" + }, + { + "pair_index": 51, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-general_explanatory-34-watermarked", + "source_text_hash": "928bc10e2d6c5a41d1f561bbf64d4ba765c024ddcf2dd21696743eb7c6c19c2e", + "transformed_text_hash": "e70be8ed37005556570c3b3fbcf50d532c0c47bea4a266018b95dd4674d80bfe", + "variant_hash": "ca29157c5d745375943f4203e451e1edfd012e39cb97ca06bbcf8d1e4dcff5bc" + }, + { + "pair_index": 52, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-general_explanatory-35-unwatermarked", + "source_text_hash": "9140098da50315235a8d1ec094885b4c51c27ae4f56ef062267dd4075028b335", + "transformed_text_hash": "c44119142a78e6330c3f4abfe0d05fb6b8c79eb50b976c8c413711c2639f8f23", + "variant_hash": "725ce569010e753bf70a77e1a8e59e8b55b58a76ac8891b68e7c006e83bc3e9d" + }, + { + "pair_index": 53, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-general_explanatory-35-watermarked", + "source_text_hash": "9c9df6d690837da7ff7903d2f6016e3adfb48b644cc6ed6fc99f28a8d413ddff", + "transformed_text_hash": "280248d6e85d80254d0f8084b66a0f6f58d58886b9ab63826ea82a58c9c38694", + "variant_hash": "28b91e57e3271b4729ce1d1cc2b48842eb7fb09a55043e0e13e9ccf4069b1e72" + }, + { + "pair_index": 54, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-general_explanatory-36-unwatermarked", + "source_text_hash": "3cfe4787b7773091d142f30ea2661345c71d82f8256ceddf758e4eed344da680", + "transformed_text_hash": "5620e50a96ee709cf2dd6eb4de33f386f39cea121179b79a0dcbd3ca0fe22b0b", + "variant_hash": "ed793f0d57e558535d9b8796d3921da0631c6e30cfd06a83bb9ab7f11c7d8bf7" + }, + { + "pair_index": 55, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-general_explanatory-36-watermarked", + "source_text_hash": "c9200bf5a1a801bd60934e04ce7f99dc6a5a1e6948fa27ad26cbe6f9099f548a", + "transformed_text_hash": "0290cd74b23b28cedd2abd043970d6024723436cfa31e3451fd9867c42dd3dc0", + "variant_hash": "2ce80a7da23e10b35499f1abdadc0c2ced2730832e7cc08c12f2912299212a35" + }, + { + "pair_index": 56, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-general_explanatory-37-unwatermarked", + "source_text_hash": "9cf92f6b6852ef462b22958a885a292a42546f59948878147c83cf84bcd3ea05", + "transformed_text_hash": "2d39dddf69d787ce225420434182e3917251f8f37e2ee331ee4f0c83d97f3138", + "variant_hash": "ff5b754012337b21320b6beb2a47e1785bcca543019ff354f3beed334a6959ed" + }, + { + "pair_index": 57, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-general_explanatory-37-watermarked", + "source_text_hash": "407d05e67bada2a54e52fe3eea2b78cdf5ef6f173b21246c02304c2e537c08e1", + "transformed_text_hash": "83770906e4d47e4da910572a1b2e52594ce22bc2f1f887fd46ea9fd47c138434", + "variant_hash": "faa90c8ca5ca2151d84768b27759df313df46ce4d65a906d9876356d4c5e617d" + }, + { + "pair_index": 58, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-general_explanatory-38-unwatermarked", + "source_text_hash": "144daf8835cedbab9c52ec0d8a8f45623194d30365751389ef367d702785e375", + "transformed_text_hash": "bbc42635a077fbcf69f52cdc0490d691eee0b98be5449f82f9d4879598d696e3", + "variant_hash": "853b93c14f568a02959d113b720f07764bd8a81f24dac725f3af3f79c9b8cd1f" + }, + { + "pair_index": 59, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-general_explanatory-38-watermarked", + "source_text_hash": "0f402676d7a30dab8744ccee6aa84766020eaff1aea4f8ea23a8f05a0e6da17e", + "transformed_text_hash": "fffd924a463e71566be8b31ab852030e21d01da0538e55310111b6d888c803cd", + "variant_hash": "cb645278241df8a09d89787106274562611d9182eeaa068a8525a06bc89c92f9" + }, + { + "pair_index": 60, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-general_explanatory-39-unwatermarked", + "source_text_hash": "3073de9328a5c9fd78ee05c6dc76b9bfde508190c95aeaecd8d157a7fd2534e5", + "transformed_text_hash": "6764b5f671a274196a9b8bb5254275b987a17d35650d57f0178a371c461e4482", + "variant_hash": "7dc11514fc89fdf1f5f01beb7981d6eff89fbc0e5ab5e01323e9dbcae86a04a5" + }, + { + "pair_index": 61, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-general_explanatory-39-watermarked", + "source_text_hash": "9da014e543010c6fa0ee2ab4198d5122949f0e801eaf0c96716846a5deb580ee", + "transformed_text_hash": "9c7d9afaac0149a70b5e3887300f366058e3f0b68b672df18bfd6e0b004dd588", + "variant_hash": "b2d9ea80061143310277f8298a5b6caccf77bd61a3224eb4e34ebbe765fb34b2" + }, + { + "pair_index": 62, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-general_explanatory-40-unwatermarked", + "source_text_hash": "31ea5a7c73690ffe0f2f589311ecdeb5cbecac1654f4a16c68a6c3c688b629bf", + "transformed_text_hash": "a5b36ae4e1eac579ad5e6b99b390e2a9fd71852f35b81dcd453646f284ceecd1", + "variant_hash": "411b8d1331e205d6d24d4aff75b85eb3c23440922130f85fd8fd5dd70e3f1b3a" + }, + { + "pair_index": 63, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-general_explanatory-40-watermarked", + "source_text_hash": "edc97fd1bb32d49384a48c7ebdff0ffbe97479a7308b5d2ab6c0389de3e28f05", + "transformed_text_hash": "1d771aca57968fb446b586455b538a574261f7b3ea4b9ea53fb3ab6a0ca9dbf8", + "variant_hash": "d2bdc1037cbab19bb9f67a2fc5af3539bc96e812505437a20da2c64b69b88206" + }, + { + "pair_index": 64, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-structured_instructional-25-unwatermarked", + "source_text_hash": "5bb4745b09af9bf63338822a5689b1ae8fcc27e1721f1d4d37530544c8dc4fef", + "transformed_text_hash": "6c0f671e1100abec01b4b39d6177b9a058e08ae79457c78479e3a49321e4395b", + "variant_hash": "6872300b27365a75ed11f29c0bf39f660382511e7132ed40f75ea0ae12536943" + }, + { + "pair_index": 65, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-structured_instructional-25-watermarked", + "source_text_hash": "aed5ad7a77dca6acd4d6378a28344e57ae06b0e6a6419bf8e6a2435740a01d1a", + "transformed_text_hash": "2b668f53abc034c37713ef25d666f497b5348c92b73e7ede462aff7719817232", + "variant_hash": "011999063cb51475ab8520b3bc90be6afff8f5314419770523a58b8f1a6b0834" + }, + { + "pair_index": 66, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-structured_instructional-26-unwatermarked", + "source_text_hash": "d1116a86430404f4bc1c3b596e7f51cd190073b68bd9e23b6fa4f6e6db64d59a", + "transformed_text_hash": "bb59d191e08819be8abd424eb247d49868328cc9b64891a533e59004ed4ff70d", + "variant_hash": "81dac4291c33931df4cdef2811c9154951b30153c80201de2a04994b9f8c0313" + }, + { + "pair_index": 67, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-structured_instructional-26-watermarked", + "source_text_hash": "384904d576e4a71c0c5c59684872d84d39bb804cc73e5a74b197556f197db9c3", + "transformed_text_hash": "175de2f48ea0f1c3ed75c40bfb118f6199760b005001b2842fd06ada18c84a3d", + "variant_hash": "bdd9c8967d2b1b810dd6e8d088cf0985832483ca390ceb11ff7edc885f17de16" + }, + { + "pair_index": 68, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-structured_instructional-27-unwatermarked", + "source_text_hash": "780d4ec4b4fee5cb5225d151b98cc16ab1d8a2341c4b750e8e59df3489268b34", + "transformed_text_hash": "6bbf930cfa73c477fa2b1a519961dbcffe8632599bdc8fcc8ac69b7e958d39d4", + "variant_hash": "3845ac3b2cd3c9445f29602c2bd618648ebca883601ee0875f3653d15a5680b4" + }, + { + "pair_index": 69, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-structured_instructional-27-watermarked", + "source_text_hash": "22c734e69ad9807003e7c22e7b26bcf829bc4e153f43952059db41509ac4015b", + "transformed_text_hash": "b9fd26e80a1381363b6ee752503f2d11ecacde7d910ebf27ce6e61431d16d1d9", + "variant_hash": "c3a4eb62f960b08005784001d5b1d6f3fcc3bc33abb1089249e2ed20657bb6ab" + }, + { + "pair_index": 70, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-structured_instructional-28-unwatermarked", + "source_text_hash": "bb1c9bc74a791f29cf45dd4662d763e4f53b1cf9fd4c63254c6e950e4ba83340", + "transformed_text_hash": "0520a858ee4db5292bded1b87b0073e3dd1fa90c9f25f337fe22555e63117bbf", + "variant_hash": "0240ff026ab0a011afd060c7697d4cef89e288b91e9186a41e45a002ecec9f0d" + }, + { + "pair_index": 71, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-structured_instructional-28-watermarked", + "source_text_hash": "0bc2d9d51e8647d381f6477dec495bba3f49dd9fa8f43c61a836d8f2381d31ba", + "transformed_text_hash": "752e910b14ad9fe2286ae0cd97408f9b5a8157c21362a8f27edd266f3abfd67a", + "variant_hash": "bb45a4504988755b7daabed9be62e3e6e30ea53c8aa521d5770ca45b19a81aa5" + }, + { + "pair_index": 72, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-structured_instructional-29-unwatermarked", + "source_text_hash": "4c1786c1847c73a9a031c3db02e3b48e3e4ef1c8b3c1e851d202e4e62379e526", + "transformed_text_hash": "4c1786c1847c73a9a031c3db02e3b48e3e4ef1c8b3c1e851d202e4e62379e526", + "variant_hash": "07eb67bd92cb0d915f43eaac7a0344804f294c3fb6efa04e81c6ff3fecc0194f" + }, + { + "pair_index": 73, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-structured_instructional-29-watermarked", + "source_text_hash": "fb5f9a9a4c6563df1cb26fa9f4eef2ab0f08ec10d25d8f54b4c17d2c53a3edf6", + "transformed_text_hash": "6270c695aff64675c80ee23e1de2c71c335350a5faf9858d544e37d27ff2bdfe", + "variant_hash": "08b4566319ec0ca23838e199cc88ff3ab7fe27f61fc671a25905be7f5e569132" + }, + { + "pair_index": 74, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-structured_instructional-30-unwatermarked", + "source_text_hash": "f9ab7744eab4c9d6f5eb9089ec9d8de486cea286d4206f40f9eeff7279554d2d", + "transformed_text_hash": "e803f90413aa18f49ab16d5f245dc5f17a790c923dca3bdc46e131815acaeaf4", + "variant_hash": "c8334aa4000a348586d69516b12cf65ae1ddae09e91ba32e85c86a6171c054dd" + }, + { + "pair_index": 75, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-structured_instructional-30-watermarked", + "source_text_hash": "6cded08a5270a56bd2255abfabba5c6cef36a2c18863e19048aacde2a6f6447f", + "transformed_text_hash": "28d9d88742e63ca9e4b5168b345987caac2a06cb9738b0313fe93c9104afcc40", + "variant_hash": "995c3323605dc3cd74a194b9ca0646f399e049e99a601d35ed4a88ba4fd7bb87" + }, + { + "pair_index": 76, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-structured_instructional-31-unwatermarked", + "source_text_hash": "c568eafe1b4e46d6b752120089b183a3578eb4784a7181eccd13dd545aa3b886", + "transformed_text_hash": "0c7123afee521c072eb5dc1b5ddb0c9559c13bd5cf8daf6a350bddeb783da5bf", + "variant_hash": "16d72fda07b382b545896de3a66e012bd7c55a82d80328210aabcf03790f156b" + }, + { + "pair_index": 77, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-structured_instructional-31-watermarked", + "source_text_hash": "c0c6611b034adf895d19a3959be3c8c89a56eec7e6c8691adf23e93064ecedd2", + "transformed_text_hash": "1163d4708154dd79b62925dea84251163734c5137749bb44e9de63de6d720709", + "variant_hash": "aee20f5d364a9430037662f91f70bc7d9a6347d82e8048868d1724ca0d7728cb" + }, + { + "pair_index": 78, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-structured_instructional-32-unwatermarked", + "source_text_hash": "98599963e22a674e4fce06f2ffb96e3a8a91986fbd1aea33265785cbb52efd53", + "transformed_text_hash": "a7b6d0643cd3a482d2106e214ed98fd7ef7f8198116c9e252ef6355ee2a06ad6", + "variant_hash": "3ebdc3bafd7795087ff032b901c78a50d01c5bd97dd348df28497d7ac7553f36" + }, + { + "pair_index": 79, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-structured_instructional-32-watermarked", + "source_text_hash": "90220632596b59b79311a5e66b8a12ce09833b1e004e09dd1cc363c15ce643a1", + "transformed_text_hash": "78d177606b19de94e532d35f2f1ce17ba9e2253b16370fc1288c1d3b88ed3c13", + "variant_hash": "52e615e5166840fab16b000a751a819abef26b5f9ced7a448714a1ddb134a15d" + }, + { + "pair_index": 80, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-structured_instructional-33-unwatermarked", + "source_text_hash": "7aebca0a135c5b4730034979c958f5427af2b513a6ad6ea68869abc01db28a48", + "transformed_text_hash": "9064e063cdd3784408b3dc30827e700a7ed28fa2bb707ff82712ead7fe7268b5", + "variant_hash": "57b26b52b80b797dfdd4bc88c2f150efc8d405f3de3ca9cf2bc1e1473c372f82" + }, + { + "pair_index": 81, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-structured_instructional-33-watermarked", + "source_text_hash": "b82736478591c828f943e9044326debc97e267f0850cc1dcf35e1fd23ff14233", + "transformed_text_hash": "51f1216a22fad29a682f2b65a494984ce2575621f4028c6b9e0563cad6735c1d", + "variant_hash": "fc233b169a152932847ff73937cf21dd56667b7044f99228f16640713e7637d1" + }, + { + "pair_index": 82, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-structured_instructional-34-unwatermarked", + "source_text_hash": "1cd7294e7a2f1575dc79db384a3eda1630cde1a9d8e963a8d6a8b5e4bdaedbb8", + "transformed_text_hash": "fa0a47954fc656fb4010bc1f705e76b85226044df49fd11aea288ed5a6fb0d91", + "variant_hash": "368a70dfdb12463c12b5c4e12027e08d4948bad0d622fd39e6b19e368d1ac69c" + }, + { + "pair_index": 83, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-structured_instructional-34-watermarked", + "source_text_hash": "346f243cf31760acf46d33c3e1d6499f08c8453b7b266baf04d44843962dee3e", + "transformed_text_hash": "a4a17dc944276aa5fecb97aab34cc42d6938ba307c32dfbafe7e6b218a13e894", + "variant_hash": "1544b0089f2ab1458e1f884654f315807186dfa1ddba252132331043e0a3b923" + }, + { + "pair_index": 84, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-structured_instructional-35-unwatermarked", + "source_text_hash": "616ca6891205a12645aee1377e57d4761e68a4085ecb5d79ce6e51e231c14820", + "transformed_text_hash": "35ce805479e7f4e71b38bfee4e5c7e18ec7747cc6a75b24543a8a103da7eb924", + "variant_hash": "0a91ba5dd48b375186e3f2f2349011a335a56e0a8316daac8defe9e787b63554" + }, + { + "pair_index": 85, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-structured_instructional-35-watermarked", + "source_text_hash": "39c01b143d77b384217262f7ef268d0af99128747b0968d2ec59bff6602451ac", + "transformed_text_hash": "4938376f8a2b3945d8c52401111552f0293e38b181d825dc72c4e4d8876aebb9", + "variant_hash": "4388dd0e9b61b397d8206b4ebdd414b142ca5cc058d87808a559f03f2b63e717" + }, + { + "pair_index": 86, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-structured_instructional-36-unwatermarked", + "source_text_hash": "87b32c5e6eeee104ae7615b0e394d8b59739642b33e91b9f9dead2f449d166e8", + "transformed_text_hash": "8d82a1faede58e5e7b4f7c4a278544601486b6f29dffc6f5092bc41c81193426", + "variant_hash": "4d20990cf8650d96ca87f6be6ad6353bc0d87d2a3f56db937ac1abf6c91433e3" + }, + { + "pair_index": 87, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-structured_instructional-36-watermarked", + "source_text_hash": "7055cc1d5e8e7f573aa2fb39a85727cc7c230165e7424652cd5c9b51b479004d", + "transformed_text_hash": "896de0c356b34c4d82444d64d80ddd3e9b9794736dda3ec8401cd4a8b4d85504", + "variant_hash": "04464e9c45bbae2d80976008f540029edf6ac532a0fb591c8fbe174e0c2cabbe" + }, + { + "pair_index": 88, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-structured_instructional-37-unwatermarked", + "source_text_hash": "492708f849be84ec13f86e072d85367f6a6bbb23b748d80f4336a9d0bd8e4dd8", + "transformed_text_hash": "4a094455390426aa52d63c6cd568f0cefeca9a5322b355bc8ed7869df06f6894", + "variant_hash": "8ff83965ddb1f349e0185118911f32b1adbddcb2ff0f39b8b9de9a7cf3f4c462" + }, + { + "pair_index": 89, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-structured_instructional-37-watermarked", + "source_text_hash": "7f97f4cb67b3e6be44ca6ab92093fc28c439e5f86a740c000a827800dea45848", + "transformed_text_hash": "2fc8cb1143d7cf522906b476da5d88fd1df2271ec2aa847a5e9d3ee089f4aed3", + "variant_hash": "81b89bd46c67a6fc68a87aa81a3b925cb37e17821651d3f2066a9fa6c2fbf792" + }, + { + "pair_index": 90, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-structured_instructional-38-unwatermarked", + "source_text_hash": "ec82880014f1772169c66e1710b5cbe8d8489ce826647d5b42533595e1624791", + "transformed_text_hash": "5880212659c686e2b1fc63f583e08fee2ec6e03065ee7819a236b31f5985a9cd", + "variant_hash": "5946863740842e31c7c7e48dc3a8e304c4472821c19889484f88ddb826d94fdd" + }, + { + "pair_index": 91, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-structured_instructional-38-watermarked", + "source_text_hash": "53546512fd56196dd6adb439ca7ae2c19d201444166b0d1ac1aa1aec03fedc71", + "transformed_text_hash": "08fa50bc0d0c977297209e4a7bd14b9dfab1b0a1cf098dff8316468a0beff1d9", + "variant_hash": "3408b7498c46a65d2aaa774967decb7bfe7d5e9591b6508acc150f24486937bb" + }, + { + "pair_index": 92, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-structured_instructional-39-unwatermarked", + "source_text_hash": "50831f201237267205f03267665fea3c53045d886482b688e08fbe044c1bda37", + "transformed_text_hash": "4c43c0a53a2b8d523c4acceaca6a40a63345702d8c34e81aa7b558aa5fbd4a64", + "variant_hash": "0432057ddccdad4f2063d8088a657255d8d7618753eef6a93fbcac1da3ab91dd" + }, + { + "pair_index": 93, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-structured_instructional-39-watermarked", + "source_text_hash": "3b75893d1bba9bfdc57c785305fa9e9f31603241adb81833c9f42a3d10adb47e", + "transformed_text_hash": "492b78bf6b28addccc579b2dced8589fac096f29333d9259568455cb013653ef", + "variant_hash": "c8d630e311a88692f2378de488605a9477d6d61e287f821fc8328b14c3965662" + }, + { + "pair_index": 94, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-structured_instructional-40-unwatermarked", + "source_text_hash": "916a39bcd1fc1ae4c7ef8db9c939c70dc94af86dc79777903c622575f6911c1d", + "transformed_text_hash": "55c3aa3b77dfcac5275f7d3cea3af491408fc177d86d16e54c5579ed568f0182", + "variant_hash": "1b32bb54fc095e90e5521cc3aa5034dd59f8ed444950471ec1fbd8b7d86b7c2a" + }, + { + "pair_index": 95, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-structured_instructional-40-watermarked", + "source_text_hash": "9e099e477443eccbf265b856c6c752b317710877277fa283b1107970bd005578", + "transformed_text_hash": "54cf379b5bdb32a201fa20480082265c43d4b68de9ffaf9cac9a53e82858500e", + "variant_hash": "c26ec27920c51f83e3f159b1e7b81acade353f37b0df59aab6374682317d5372" + }, + { + "pair_index": 96, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-technical_explanation-25-unwatermarked", + "source_text_hash": "693f010802429c5a391e636b64674554c9439333c59177c344cdfe34349c0f74", + "transformed_text_hash": "aaef6670e09bc69f64d62348a393df75288d4d47b59d2d0ed98e7c1b4edc9c5b", + "variant_hash": "fe1898db88ed2120a9a9ff1b50608554666a4289254bfa2797df67ec5225d593" + }, + { + "pair_index": 97, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-technical_explanation-25-watermarked", + "source_text_hash": "f87c451c1e5bb1e5fd0bbea9846e2aa5cb299ae1c05f281326e83ce01c036ff4", + "transformed_text_hash": "959f557639d722f4b525d3b58608df5c57ac42d1773e98a63fc183182bbf169a", + "variant_hash": "888d03f1cc5ecabae9a85c0a3f00ac15493ad0c94056546aae590990df3c6f41" + }, + { + "pair_index": 98, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-technical_explanation-26-unwatermarked", + "source_text_hash": "ae70a810b354f0bde1420f8001fb557ca029481730dfbafde19f53a5c35d99fc", + "transformed_text_hash": "4fa2f1a68d491ea4fcc043c5a031491c30cfaaa47282aaade7ee6c30dccc763b", + "variant_hash": "0edd197e9c67057ac7c53ec19af1bfad245d4ab421d82beb53301b60dcdd4f93" + }, + { + "pair_index": 99, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-technical_explanation-26-watermarked", + "source_text_hash": "d14b412f0fea10ce41459e8de3f86cc56409a655f1f1097e56de20b33ee3896c", + "transformed_text_hash": "397641946a36a3eccd89c7f14e4f24f8b1e8813732e54dc3479139d55abc19f2", + "variant_hash": "b4cd8a2c3a127484c12a531673bc540de7b809253257050eacfeee72fb8197ea" + }, + { + "pair_index": 100, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-technical_explanation-27-unwatermarked", + "source_text_hash": "02baf2d5bdff1c291b82ebf73778f8f8250fc03e194351e8974acc55c11b2cb6", + "transformed_text_hash": "9444826c237aa03c696343310f6865695f0a12d25a0fa0c65cd511f93f70ddd9", + "variant_hash": "73d51ee08b4ac5aa3bbdfa9d64c17af048eb362201655c3ad433d7dd44fc2140" + }, + { + "pair_index": 101, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-technical_explanation-27-watermarked", + "source_text_hash": "e58f4536dfdda95658b18e89c5af52c445bdf69eb6a3c8f8e85f4eb2491f38a6", + "transformed_text_hash": "1a3c5a2ec9fbd523501a9e6f23263935f3bdf177d98072750b7b69e34335e675", + "variant_hash": "0b9e979219188525a5a954ffb48dc8c8201e8de898049a8060282cd17b2b7a23" + }, + { + "pair_index": 102, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-technical_explanation-28-unwatermarked", + "source_text_hash": "188ced327bc1ad5840c2f6762b19421d68c21c0bc4d7e3aa39689b399883ab5f", + "transformed_text_hash": "499244c6351794071327e75d41bd90c3c21b1ad273a795aebf986464ec58624f", + "variant_hash": "e6db1e6902c15a806edd0fffb6c79c5febbf421508b43b28d9d3839a126c4809" + }, + { + "pair_index": 103, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-technical_explanation-28-watermarked", + "source_text_hash": "4be2a84a7c3887a5521578da9fd62129604e7cf720aa75032e139b3816303a5c", + "transformed_text_hash": "f6ead7823af5701780f87c5807cc46c4873a35ea537764c245af7ffb3d9da5ec", + "variant_hash": "20f750854949c29e1ef317c2a429cdfd0e260a21ba440a5a16e69c7b1c3dd1b8" + }, + { + "pair_index": 104, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-technical_explanation-29-unwatermarked", + "source_text_hash": "ea662d006f84898fdc34bfd3823f4df393af497201cbc450f2deb65992d4eea4", + "transformed_text_hash": "f61c4d1ec5cdb0a3ef2a7c37eae6ccc9e9f28738be64265845c1674f6baf899c", + "variant_hash": "a52e337b58a1f6d04f34cab6218172ac090e24907157dd6fea1485b6d4c352dd" + }, + { + "pair_index": 105, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-technical_explanation-29-watermarked", + "source_text_hash": "228c69e523de8eccd980812adabd554167f987b6abb214e6b40ed91d551adec3", + "transformed_text_hash": "081636737aec7087d647837bc64e87fb256833fc6d41253a7c3133889c04f148", + "variant_hash": "456ccca47a4229fee4496f5823df5496d909d842e2fdc039a761864b462b55e0" + }, + { + "pair_index": 106, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-technical_explanation-30-unwatermarked", + "source_text_hash": "3286e0c2bcd84d8134e7c5273c93095269f89094f363dceeb6d82ff42d1f7e1c", + "transformed_text_hash": "47b46fe53690c8adf7f1dc1f4d6cd7f5b261bd42b9c3405b1c24e7efce0368dd", + "variant_hash": "15f76efa2ad2cdee4275c352f2e7c0a34567cfd9963c287c5b5bd403d34784ab" + }, + { + "pair_index": 107, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-technical_explanation-30-watermarked", + "source_text_hash": "9cda66531dc58a651ecb8e78c96c153837a1de28d1ac542a59d5fe3fd61cecbd", + "transformed_text_hash": "e7d8d96faab7d8e25dde4188a3cbf5dd79272e61336c3c108c8cb8fb144c9a9b", + "variant_hash": "3276c61811d776993f6173e6d2eb473f25b01fefdebcf950916f9ec5cbd4d190" + }, + { + "pair_index": 108, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-technical_explanation-31-unwatermarked", + "source_text_hash": "c41537c178441c2538edbd08822e29876e913ade4859c390e75af53a55beef9f", + "transformed_text_hash": "2481d75afd75258382bfdaec27b3b5e66cb17bfe4be118e3c3f800709aea0947", + "variant_hash": "a0bdec7f6180d962aa779e5ae8636d8636632e92b5175a621942201416788a6b" + }, + { + "pair_index": 109, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-technical_explanation-31-watermarked", + "source_text_hash": "b31e060a90b80a18adaa3f0501c40d721d799fdf76904a4c1fc17cab665f881b", + "transformed_text_hash": "eb2ada38ba2c9aec3fa19b004e4219406569bff1acb07a082a744badb4de78b4", + "variant_hash": "397c70485121d4aeabf1474f17c319823c36d61bfa8ecccadce70d4281a2851b" + }, + { + "pair_index": 110, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-technical_explanation-32-unwatermarked", + "source_text_hash": "f833cdf060358c0f09302a8083369c8c3d4eb04260a126b4ebfb506566821b0a", + "transformed_text_hash": "6f23b7c3bbdf0b073fb3f4e6ca31e432d587391197ae7f966c2b524bacb64e6b", + "variant_hash": "9f0a1b945220a6c42ef8ccaa2a8ba89db8af51ea91d17511d325db3469759829" + }, + { + "pair_index": 111, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-technical_explanation-32-watermarked", + "source_text_hash": "1faa86cfebefea78b01ceaa69190983e135356e749a973197ea6375564712604", + "transformed_text_hash": "4152d424481435a8cd1a45e4a4d2dc73799c479afd40978d4a27f1fee5fec7ec", + "variant_hash": "bf4814068f60afee4e6b8c05a2899c0f6128d858f75c57bc37c64e8423a90537" + }, + { + "pair_index": 112, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-technical_explanation-33-unwatermarked", + "source_text_hash": "49ae9db1725a03387dcd66d63730725a14caf52e51755c5570905060b6a99261", + "transformed_text_hash": "34e0d47189cc27644a63ba44900000f7a485dcdfb60e26664549bf87e923a6d0", + "variant_hash": "09e5fbcf3847dbd1b9cd9c14571e77d990ab67772cd4d8e94d1be40f572f356d" + }, + { + "pair_index": 113, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-technical_explanation-33-watermarked", + "source_text_hash": "fa6f418c07b10618c0a06a0d2de7e42545cb8abcd25a4d03306e477bc04d5ac3", + "transformed_text_hash": "ff38758a226b18a02f88a7449b1ef32007c9ea0c0fc3ec8b0ae22bcfd5e43b6f", + "variant_hash": "54799da0fd837ba1d1be42c9c99966cc4b1b13d41c0f3e9ecc8df47cc80277a0" + }, + { + "pair_index": 114, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-technical_explanation-34-unwatermarked", + "source_text_hash": "d3e5788af180dd752d441df7622cf67a7eece306754395dd96a43ac281b8401c", + "transformed_text_hash": "a509ad063162e9d6ee88005d4445705407bfc4ed0a450c176821cc07c9ff451f", + "variant_hash": "8f7ade50cc99a1ab82e3b7abdae6d2feff2d3ae9827566cb3eb75b1934037d27" + }, + { + "pair_index": 115, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-technical_explanation-34-watermarked", + "source_text_hash": "c51081f9dc2f63d01d2175098348e6c15bad2df9fd72bc3e6a3d6d3583ca2c91", + "transformed_text_hash": "367dbefe2eee0e4829d055a74a4eb3be1888eed0e932443f15812f6e062848b7", + "variant_hash": "4002557d39567b84f48881c30574b7dec5031e98c9351ccf7bece266e4a6fbf1" + }, + { + "pair_index": 116, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-technical_explanation-35-unwatermarked", + "source_text_hash": "9f2d689d9bbe2b31eabe058ec8553869bd2fde6a7de2f542dc6e1b0f78abc11a", + "transformed_text_hash": "82fda0f4ef61b8de23de8fcebe8f9afc55ed3efa08c550a0646fd1da73deee62", + "variant_hash": "2443798d2c43a1683c2ede10c46dbf8fdf8de4589502ed09d177a975bf6ad8db" + }, + { + "pair_index": 117, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-technical_explanation-35-watermarked", + "source_text_hash": "abd419da92c1b2126d45ab71b4899a043d82ac97830adb1c3fa0f977ea21ea2e", + "transformed_text_hash": "58cfadd5fdfef177f70f71e9cec1dcfdbd45cb0c49db8a3f1bd0a1b32bb50822", + "variant_hash": "4f3f28fc7d5db623a1e97a39024d1d2c98ddd6965a54b9bee6d22197d636be49" + }, + { + "pair_index": 118, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-technical_explanation-36-unwatermarked", + "source_text_hash": "ca2aa16a76f084cd634af09eb3e24ea460c99ab0049074ef46076a318338002e", + "transformed_text_hash": "0c05ca66a89b7655b97bfac2a8faacf5ea3e0b22ca8f9c984080ca22191e6fca", + "variant_hash": "a9bf4e570588b7f5b50089180359a0e095da6271f2391c574cbf7d3a960d5bea" + }, + { + "pair_index": 119, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-technical_explanation-36-watermarked", + "source_text_hash": "6992b279f0cbf3f65a3e1901b62d1fa169aeefaa081727cd9d269fff3021703f", + "transformed_text_hash": "cf51a4e460de1e6026223d11dcc825fc2f8d9c1cba5206439ba333926414d0bb", + "variant_hash": "cc7b8f41c91c6a013cce81f4ffa3ad0a0ac7a8df9ea0f7e762d1298fe6f2ef4d" + }, + { + "pair_index": 120, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-technical_explanation-37-unwatermarked", + "source_text_hash": "d36bb6caba912e611280f86c8cae820e68abb5a60d3a5d56bebf9fe26cec8c8f", + "transformed_text_hash": "ad7500c4a6bb08e574cf4874c5dcd4113b5d84bb9cc9e30be5b8c156eb5babf5", + "variant_hash": "2641388b1372d7c8a6ce195f2810aee6be47c7f23f007edeb2008f141acd0b59" + }, + { + "pair_index": 121, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-technical_explanation-37-watermarked", + "source_text_hash": "39587cdb2aa7a627a7e225f21d489114690935132face2b29c8e9b3404726994", + "transformed_text_hash": "01f67e763dea7200fc8a1e028940bf96c278e95f838a0139bd4c5c24a6531d98", + "variant_hash": "fb7457d59c433b1c2cbf75e20a50c67de2f902b10c0929efb127fdb455a66ab4" + }, + { + "pair_index": 122, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-technical_explanation-38-unwatermarked", + "source_text_hash": "3e2b679d1e2212b9030cdcd91b7ce61c38830e9024b0805d91ec73681f3fe1ff", + "transformed_text_hash": "85f0c8c0da07ec26a6b57c0ab2854c5a9fd69ce1585a96f3192f6670b1d618ed", + "variant_hash": "af687c1247fa2b9aa7f6b5dc5ce6a5597417277e86d4ef5ebf36c8d450de0779" + }, + { + "pair_index": 123, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-technical_explanation-38-watermarked", + "source_text_hash": "357b1c2ed0969525e0cae41cbaa7088078f04dfae4e43d74dbbf35e46f9c7408", + "transformed_text_hash": "911dc8e239b776cc1ea96afa903775b5bb9ba2a611c26812913e508e91ab5ced", + "variant_hash": "a3f27bc5795783d60dea4240d49e6a2e7ce80407860890f96535e3b1f7c04780" + }, + { + "pair_index": 124, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-technical_explanation-39-unwatermarked", + "source_text_hash": "b4fe9932342201c1029deecce7db2826aee1d3571192a8994275bf1ff870090e", + "transformed_text_hash": "ee896095122ad888c4c852e4a0c5ffa126192e17920cae9f7a44e62e9fcd03f0", + "variant_hash": "a3356333bdcab518f3cbddd0874d47aa9012f552b28f03a132c8b091d57d35b7" + }, + { + "pair_index": 125, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-technical_explanation-39-watermarked", + "source_text_hash": "cfba3f9ecef0d7010e3e1c4e1ec951dce66abc3c1d085651cfcf7bf1f8b4698f", + "transformed_text_hash": "c81b1a32b0d49393c81132aec83eeed03d5b870f42b31b858c83817d030bf645", + "variant_hash": "691c0375cb38698e8671cce4f693cbc9f5bdbd84bd5cb9b1414a197f4ac1c3de" + }, + { + "pair_index": 126, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-technical_explanation-40-unwatermarked", + "source_text_hash": "3aebc2cc4f86b0a890776cc6524d14bcb05f10312d332fbc63f270613c6d0aee", + "transformed_text_hash": "75612102f54cd4746e2de89a286e2f9d7a613182921424b2f1a7c885047c7d1d", + "variant_hash": "52d6d409defc79409629f1385e09b53e21117058caabe150976e75a1f5b530bb" + }, + { + "pair_index": 127, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-technical_explanation-40-watermarked", + "source_text_hash": "2ef32ed901d397abf3f5fc6fdcc1ed7a89c365e275b87954720b7cc5d1cec976", + "transformed_text_hash": "bb0d656165a069013275dfddb05c768f2759799685ea89b6516a96e40ed1f94f", + "variant_hash": "0d6a0bb383ef277e4302acb5131f28d9f3e37c30cbd30862df8976b99508e756" + } + ], + "plan_hash": "5fd1146ba2f8c34bd3cce143154da7b144d0802ba5ebf8556737dc51c96efd77" +} diff --git a/evidence/budget-scaled-coverage-2026-08-22/rendering-manifest.json b/evidence/budget-scaled-coverage-2026-08-22/rendering-manifest.json new file mode 100644 index 00000000..cf53a892 --- /dev/null +++ b/evidence/budget-scaled-coverage-2026-08-22/rendering-manifest.json @@ -0,0 +1,1030 @@ +{ + "pair_count": 128, + "pairs": [ + { + "pair_index": 0, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-conversational_prose-25-unwatermarked", + "source_text_hash": "d6708285d65976877cd638c856e189cc2c8731a817539480434f2c4f46d2165f", + "transformed_text_hash": "2dedf2b5211b13505c17ed937b95fb6aeb7119db8833a52175c9dd74a03a26ee", + "variant_hash": "b0453702a822d2590421804f9af59b7caebb6f23b402f05353bb935014963df9" + }, + { + "pair_index": 1, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-conversational_prose-25-watermarked", + "source_text_hash": "9a286feb31c48e9ed0c9999789711b2579248fb083202063af1c669f37fecce4", + "transformed_text_hash": "ccb2c55f42aedd03f029ac83debc2edc3b873ff2b45dcfcc9b7c1c76250edc06", + "variant_hash": "9c08407ebff44ab776b64b61359cf13f44b956f72e9c7b9f44f10aec346af42b" + }, + { + "pair_index": 2, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-conversational_prose-26-unwatermarked", + "source_text_hash": "28a63d542a728713d0004d489be3895ec891f42df3ade3474381d3ab0e5f5ef0", + "transformed_text_hash": "a2b44c5407188a9969010a6e789828b06dc79ba16fd0080943fd385a76f91aba", + "variant_hash": "79be387834ba3bc142b78c77c9518559ab02fc500ee9e761f3883545492b5196" + }, + { + "pair_index": 3, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-conversational_prose-26-watermarked", + "source_text_hash": "1d936ad5aedeb613dc0b674eeae26f5cda6b24002664c061138008810fdf4de9", + "transformed_text_hash": "01a1e1aa606b9c9b6c584451302cce0d453c9ac1deeb543fc52ec56ef37d5ec7", + "variant_hash": "60197714bdbf8d5108b4b6663c138ebd8ae1406e39fe2c92278956b7946f6b3b" + }, + { + "pair_index": 4, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-conversational_prose-27-unwatermarked", + "source_text_hash": "e093007ba4fd439eecb20a9aff6088aa297f19447b26043c1bf8f753e2c829c4", + "transformed_text_hash": "07664b12c1af72e2f7be4829f11c97c008005de86faf15610cb9519abd303e07", + "variant_hash": "8403a8a97f960c31d0110a83b3aab6e262b6b464088e67ba2426f45965ac953e" + }, + { + "pair_index": 5, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-conversational_prose-27-watermarked", + "source_text_hash": "1997a2737eaa043176c546ab32aa604dcb6f2201bbc1454ab7965dcc3de57f7e", + "transformed_text_hash": "2e167e73821cb00702a32b3d3b620681fbb4c9bde0450c583e58dc372cbd8c4b", + "variant_hash": "6703efa875bbbabca3f65fe7c5fcb74d1eb06ce2b38a7bffe7c7d6dcd92871cf" + }, + { + "pair_index": 6, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-conversational_prose-28-unwatermarked", + "source_text_hash": "a671440b0822fcf6fdfd38074e7646e659fffe950db6c360b79eb35877b5240e", + "transformed_text_hash": "0959df19e6e37e313a8ed9112ac26880c7367f043fa8dec7d1f8015ec502be53", + "variant_hash": "f63a8709b060103b6f7bb5c629ac06a5ed4a023a3a1a44161e6d31daf742559a" + }, + { + "pair_index": 7, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-conversational_prose-28-watermarked", + "source_text_hash": "6a61fb0e5c1bd11ca922cf7549c3c08a05fc767c038fb3ffa5d90d0f43123f65", + "transformed_text_hash": "e424251a96f0c0e161ce276bd2163cad2ba2e80e01555ec368bdcacf0fd88ab6", + "variant_hash": "4a25eaca09e5299552edcb911d7408fc0699bae0ea24c32ea232721566ad4d1c" + }, + { + "pair_index": 8, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-conversational_prose-29-unwatermarked", + "source_text_hash": "62076d10fb04dc787d008b7cb4a77e9b5caf195029ecb5bed15a22519eaee486", + "transformed_text_hash": "658772dbb731fed08c020d90bac7dcf226cec94b2aef742a3f320bc992b5bfcc", + "variant_hash": "2745bb294cb9fe86777f26629332f204d718bfca8baf4fc8887f1c039cf204c9" + }, + { + "pair_index": 9, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-conversational_prose-29-watermarked", + "source_text_hash": "efe80422df6408b2e344bf490cc52fb67affc50f8626368df5d75d49ece9f003", + "transformed_text_hash": "a9ded56ce0f189d27f942931df5be047bca68d66574a14d75ab07ce4efde56cf", + "variant_hash": "b53d7dc17e715c5c465651235845acbf0ca9d3ec4d287a3fcb1ff13fe062f75b" + }, + { + "pair_index": 10, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-conversational_prose-30-unwatermarked", + "source_text_hash": "46110e06178e781b93329a52260164d0da26a9e55ed57a19900b913874b6901e", + "transformed_text_hash": "911b00f3456af46f3c0431dd35b1b45513c7144005bb83ad32be55dea7c8e131", + "variant_hash": "5acb8cabb0653736272cd6394170e436e58478cf2aec2172f4f3f6b6e0f11856" + }, + { + "pair_index": 11, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-conversational_prose-30-watermarked", + "source_text_hash": "f32ccdb871662f4113ff2adbb9eb6677ac0d8f6cc0d66afcc768093e68b1741c", + "transformed_text_hash": "810a16a234f5850f56dbcc92c3a0716a32646ce10d1283a8bbf4ee2693a2bd12", + "variant_hash": "3fb6693f7f40c4d2f6b141dcb0ba933e3ebc6b2eab3808c10f976af1df959dcf" + }, + { + "pair_index": 12, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-conversational_prose-31-unwatermarked", + "source_text_hash": "081245380d9b579084680a58e4e00faa4138a9e6593251d946505fdbae1b867c", + "transformed_text_hash": "0c07b8bf910d57a0ac858c83a23a57e192bead38a65bd3715e598d94e7caa468", + "variant_hash": "32dd000602ad7a62290b0537bc5deb6ae0da8dfa0ceee2837c81dc63e26ecb09" + }, + { + "pair_index": 13, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-conversational_prose-31-watermarked", + "source_text_hash": "84f33bd813a752f02a66f3d54d3a6795ee50f49ca0b17437187f62788ee52d10", + "transformed_text_hash": "67e7c3192f3a9c08529225b77eb59a7f3e9edd413c9b4b7ce3132c6984021d94", + "variant_hash": "a52bd68ac8affa345789476c33c6035ab3dac9f128cea6b19789909935243b49" + }, + { + "pair_index": 14, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-conversational_prose-32-unwatermarked", + "source_text_hash": "3258bc7eaee6dc1ec7cef4af96fc255c14b5b9ad018303678aa4d4c7fb66943b", + "transformed_text_hash": "c41ecdea48958aa641dcf27b4ab317913447f5ca8ed5303e6551d9b8ef7e3bd2", + "variant_hash": "8f89499f17f534c52164fe6700544838186284b806ac1034a75c6d9fb3593da2" + }, + { + "pair_index": 15, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-conversational_prose-32-watermarked", + "source_text_hash": "86366f2768d30b0be0407e38d772381c497dea2737135a99accb152c37a9c303", + "transformed_text_hash": "2903add455715008ede4664065862a1b8b968880ee7d962f449d5c56e3608aa8", + "variant_hash": "c0929a16ea739268db7732a513ceabc0b08b854e27214467ec079298522c581a" + }, + { + "pair_index": 16, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-conversational_prose-33-unwatermarked", + "source_text_hash": "4b0e4f32c2b572aa6fa7f42aef596bd0a468cd10f849562e1ba491e0354ab6bb", + "transformed_text_hash": "78416f6bf9af092afbdea548a9c2fe7f8c0f54028c2a511dc2e4f2bd75de3885", + "variant_hash": "33b98dceb17f76910e618d8d98f6ec004e3cf6f020addf25b8549d764dd04e1b" + }, + { + "pair_index": 17, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-conversational_prose-33-watermarked", + "source_text_hash": "d3b2df8701d5bfd247b4a1fe53e961d65c64048122c8ae2d4a9e6ba4e9538dc3", + "transformed_text_hash": "391d4624d6b6291d6f4a4ba6a5c1db90352979e1cb90975069298cce8007cec3", + "variant_hash": "b821af27ee2e7d84fb0af0d74a007c7461d80048cf06abc631c9b855fbf1ba1f" + }, + { + "pair_index": 18, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-conversational_prose-34-unwatermarked", + "source_text_hash": "7409ca9b2af0fa76a7c8d1641b78cfb24eca034777ae5a0b5779e5327186cd0b", + "transformed_text_hash": "09af6858fabeaafa6604fe1d831177382d904048cd8bb55a34e23c96a3fee93e", + "variant_hash": "ad8b3042f88c3a00e2b969d38f830da91eb8334f1a64098537bc1d37fd7a30fd" + }, + { + "pair_index": 19, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-conversational_prose-34-watermarked", + "source_text_hash": "2c951d293a0ef7aaa3cc150a5b57fd316b292701b344606aba0ebeb1bf39e2bd", + "transformed_text_hash": "2de55ce96a831afd1021efdff4b07909ad85e623c223c8166bfc61e41f6ab97d", + "variant_hash": "26e613e25b69c6940a2214265fefb7248b0dc6f085c5dbd488e7758dcedd660b" + }, + { + "pair_index": 20, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-conversational_prose-35-unwatermarked", + "source_text_hash": "b4b4eadc592ece2aa02c5cad703b16e32084f3ed6b8e377e60c7b3abe29d9b43", + "transformed_text_hash": "fa91884b4482f3494279dcbdc2713cec9e865d515978d6ed3a5781ef15fc79c1", + "variant_hash": "8adbf2c7fbd8bd01733424c40a5961978b333c02d8ab2cdc53679af619ad713c" + }, + { + "pair_index": 21, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-conversational_prose-35-watermarked", + "source_text_hash": "39f706f158f225d80afb4f92fe3f9c96edb1ec68a2d0046d3dc2453c1cebbe87", + "transformed_text_hash": "cbb85345f10b963dc32d4face2e70175949ead599b9f8f916ccda4cf4067812a", + "variant_hash": "621a9f3627902c5fb4e7f90bd8e38f15bc6c95c3c71a41ec180509bdfb78172e" + }, + { + "pair_index": 22, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-conversational_prose-36-unwatermarked", + "source_text_hash": "e8c5dc04c7d846973f1fb1eb16a61a630e3c54cd151af0fd3d241907a7018bd0", + "transformed_text_hash": "2bfa9f53db8927820e985fbbc0fb0554f1d56065ae1ac116154863a334f1d6ba", + "variant_hash": "551f08c82df5839e9ffa154f6bcd9f43ce7b5c3a571ed379ffd6f6cc4f525c7b" + }, + { + "pair_index": 23, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-conversational_prose-36-watermarked", + "source_text_hash": "71d5675b314dbc6683c6755efe4e4930a96816e85cd09d1d365b08443f2d8f6a", + "transformed_text_hash": "6597a4e811a00a590ed4f6ac07cf9c7c4326025f29c41e9aabd90944dc9f321d", + "variant_hash": "5e0a898cd8d29645733d5716fada331ee2f45bed3cf4570e4615e3ca69cd55c7" + }, + { + "pair_index": 24, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-conversational_prose-37-unwatermarked", + "source_text_hash": "36ba5db85f67575b54624c78f9b566063b0f58bfe410ae8e8f36517ac4811551", + "transformed_text_hash": "2cd123475ec3cd7c75d9696f382044f6f1f43eed09cc6842a8438c67fa166cf1", + "variant_hash": "61cfde568b39167933043f28e0bb87681de933cc540c5c6eca2dd848151166a3" + }, + { + "pair_index": 25, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-conversational_prose-37-watermarked", + "source_text_hash": "e2f26924c9d3b4f2bc024ba7e73e30f923485177423c59a518e3e40a3d8ed6ba", + "transformed_text_hash": "edeab024e8acb049c29d09330bfd53468223e454bdbbbd02a4a0a33a236605e6", + "variant_hash": "432c7f27ac9d2fbb00f938328b6de4d7370fb4c1197b2d1a0e275a555433f44b" + }, + { + "pair_index": 26, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-conversational_prose-38-unwatermarked", + "source_text_hash": "3bcca26dc68d190d03e98d96124babd18dac89a144583a0d1566c2ff588de1d3", + "transformed_text_hash": "8dce6a2ff68f1d07fc60482c67a9f7f5f891c4e1c0a3a3a0d4ba1ee9aee6d938", + "variant_hash": "aba89a6172474674623e5d783290c558916b5f0db55011c60fe774b8d5ea6482" + }, + { + "pair_index": 27, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-conversational_prose-38-watermarked", + "source_text_hash": "f31c7adbbca6d62a5726eeed8fca7d4aabfb9f9b4c8a36671d976c8789c0c5b4", + "transformed_text_hash": "07233fd02d3266a1cd910142e0c1aaf7b1f7ac579578c8b79a5eae364e733728", + "variant_hash": "7855fdd337c1865973232afda20fc01af6c76e6d1fa9c79db9b0524142fd3ad3" + }, + { + "pair_index": 28, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-conversational_prose-39-unwatermarked", + "source_text_hash": "59fbd805973ae2c003e7ca3ac05503290701665bf809a644a16dadff4be52291", + "transformed_text_hash": "bae50882f9377a2050be86a8808dea5839dd68ec9d1c114f9f5eec2abf02034e", + "variant_hash": "82d6672286411d0eaa7318b220e0095a4e7fe87dc3fc11b80fc45a7195c45b76" + }, + { + "pair_index": 29, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-conversational_prose-39-watermarked", + "source_text_hash": "df5150a41694687e719a836d2fa02c0e7200514fd8839a03a965584874d7bc77", + "transformed_text_hash": "b6381f2bff3b7499590bc2545ddbc599869a3dd3672239ecb8ae74ef7f4d1162", + "variant_hash": "20d19dfaee48c5a90ba8381249e014d917827e9cd6dd685dca27124c28de9f4e" + }, + { + "pair_index": 30, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-conversational_prose-40-unwatermarked", + "source_text_hash": "8c37969af9ad8ba0100a37ca6034167f671a2f3fab2662a96160c253846d3a03", + "transformed_text_hash": "cdf14cc71f5fe95995c1f94b0e35e4320c2731fdfbde63d0d3d95aaf8219d308", + "variant_hash": "4b99a27f0383ec93748124a8fed5a76dcfee3d55b12feb8eabe2570e6b34a758" + }, + { + "pair_index": 31, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-conversational_prose-40-watermarked", + "source_text_hash": "8d7c6ee86d885114b21f4d1d0701aa2b1172e3a72a7ffc55089f17d03e79a632", + "transformed_text_hash": "fb498ef4516bd6c69a1ac585c90f1ddade1e5efca68e1597a2741040c200d652", + "variant_hash": "87b82a74867f3da51f3bc11d469d012d204e7c952556e971c7a8f36e4c29a970" + }, + { + "pair_index": 32, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-general_explanatory-25-unwatermarked", + "source_text_hash": "fbb24992cd2c56529e2e67830c2f1b7e085cfbe8b9fa3165cdb155a8b0322efe", + "transformed_text_hash": "91d3c1a54ff9c2298f70b576662c9d4d03284edb78c7254482b29057c793fcf9", + "variant_hash": "40ee0209412c287928861d56be4cc1a9a678f69510b2b2b601cf00eec6a3547f" + }, + { + "pair_index": 33, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-general_explanatory-25-watermarked", + "source_text_hash": "92dd95e4efb92c47acf64fa85d32484d0cdf2abd6dc1f6134f25f523fd618d31", + "transformed_text_hash": "dead23c8aa8a8a183a1f6f481483b1cf49254dc35cad96c769db012dc7a06c36", + "variant_hash": "39cb74587a9fb1eabfb55325985e71b2eb92d40c57191409da123dee61fa0b47" + }, + { + "pair_index": 34, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-general_explanatory-26-unwatermarked", + "source_text_hash": "7ee55c3c1859567a1e62f0d2a5a5a907da34ca9cb506a733fd364d65a2e1a6a9", + "transformed_text_hash": "9558bb0e746d4039ede2704c9e7e51455dd8862fb420858d4cdfbfe5b07ed15f", + "variant_hash": "d2348c9ba55f78942612e8406bf510e42cd0785b4c127939a10962f6270faba4" + }, + { + "pair_index": 35, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-general_explanatory-26-watermarked", + "source_text_hash": "c6c4688cd77773ef7e81e66dd1e120657b79a936b3bc15316b7cc9f962b4c36a", + "transformed_text_hash": "18169915d727a1d811b6dce37ff12de3cdf1f7f435f5c8c21c599834818048f7", + "variant_hash": "e183c9a315734f620f3f0d16f7ffd10a4b535a12acecaa59b563a9b5c69727ed" + }, + { + "pair_index": 36, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-general_explanatory-27-unwatermarked", + "source_text_hash": "e22bec68fa599895b14f7a67e9fb25b8eb8ed8e2f19418a9cfc8dc166abcd5fd", + "transformed_text_hash": "c10300f3915f526defc1ad88c83246e0ea78a78216accd24b643148e93e0fed7", + "variant_hash": "a30c8504f1d215aad8fa100ab6d303aa9510d5b9c4d50ab445786c2de7aa7936" + }, + { + "pair_index": 37, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-general_explanatory-27-watermarked", + "source_text_hash": "3725849e088ec7704fcadd7de1e7b958a262e2161ca7438a14a8b8dedb492622", + "transformed_text_hash": "88089edf8f3be3d6c6a00c38af378ddc1158a74986abefaec2d3fa2ee4f5d59f", + "variant_hash": "bdef41ca87da0a48341cdbdf8ec7d16c66381739a73ea9dc2e08a0ec95de77c5" + }, + { + "pair_index": 38, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-general_explanatory-28-unwatermarked", + "source_text_hash": "76a08256c0f54cfd3f73dc2c6960b216cd95a02e729f998b33cb68af99879904", + "transformed_text_hash": "d60fffb793330229d29bfdeb198d308faa9837061585b28f04866e28a2fb72d7", + "variant_hash": "3de4e1b3628b84b6841c7e0e4faae6dade4d4dcb30f6b1b5bb55a18e5d9f6e3b" + }, + { + "pair_index": 39, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-general_explanatory-28-watermarked", + "source_text_hash": "afecfe7217a45d67e7dddf27f3d943670d1ebc4aff3f53dadc23f47441ecb97e", + "transformed_text_hash": "1bd61092e33d01d0aad2804b4186fdeed09b5c646159413f6a238f54f4c3a849", + "variant_hash": "447e8486345d0aa13749c38272125177a4577c26bd6a726e16539ccf23b72021" + }, + { + "pair_index": 40, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-general_explanatory-29-unwatermarked", + "source_text_hash": "a18560c164f77e6e4cdfbe07c5a0630476f9eb043e20fb97f4945c7523eb7fd3", + "transformed_text_hash": "60b2e43535824fd120c5c09e2f70ba7bfb18006973ab56c323fe6f0a87b8b5e0", + "variant_hash": "136eec6e4cb72623faba16c2206612609801a047155408294914a4a9d07037f5" + }, + { + "pair_index": 41, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-general_explanatory-29-watermarked", + "source_text_hash": "d34ffbeb248bb79fbe4461bfa05dcced87a608c01d2b72bf137285300ad022a7", + "transformed_text_hash": "e484ec1c9861ca847a4369c7a7e080fa325d7751cfde3a866af54b1da7cb54bf", + "variant_hash": "6d63eacc156c16b72e6bbe610dd8a9fa1f7feb1a34b1e5637a575b001604fd01" + }, + { + "pair_index": 42, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-general_explanatory-30-unwatermarked", + "source_text_hash": "63b695b0b3095c5999332e768f70374cf781414cf40d3c4dd645045538912fc4", + "transformed_text_hash": "5507061be4038001805837243516fe91d6d4f0505341dfa94ecf2f603ad9c756", + "variant_hash": "fabba121784618552cbbd7bb6a4e83eeb5354fa04eccaa225888731c5a1a4aa8" + }, + { + "pair_index": 43, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-general_explanatory-30-watermarked", + "source_text_hash": "ff476d79c406a3a7201e13f1efe45d15e8a82febbca45e28549c736329fd320c", + "transformed_text_hash": "68569314a56901a307636571791947b6686e1b7e9707d33db9f5631316187853", + "variant_hash": "e01da9ca178e8833154f80e3c1b0a5484b71e847eec5d0bc074ceca19d8d0387" + }, + { + "pair_index": 44, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-general_explanatory-31-unwatermarked", + "source_text_hash": "e233350d3c2830701e988fe9eb89e7b7c1fe7772e3e98a46e8a5c564ce5fba68", + "transformed_text_hash": "199b6bd6bf6ef33eee0e22c00f3f2935f15869bde16501a1508a526e2db938b6", + "variant_hash": "1fb60691ebed77a878169b6b84a744afaf03a2a66b6bb366486ec242bb8bca53" + }, + { + "pair_index": 45, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-general_explanatory-31-watermarked", + "source_text_hash": "1cd4dcdaad6b8682be53683493a326f80da5878f112437958c7de96aae23c588", + "transformed_text_hash": "8ddc87de99b05722531dc03d91c8c4abc0cb4e63f71ae5f93f5c482070bad468", + "variant_hash": "3853c2fcd01c45b2bee2d6ba6db7fb6c763dd09e643132ba4425bec5a0b57fad" + }, + { + "pair_index": 46, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-general_explanatory-32-unwatermarked", + "source_text_hash": "4f7e460f3f9baab5213a2b877339f83ed726b54973c9217c00e4f27d2987f409", + "transformed_text_hash": "a59558cc39d9dd8b6cd60ff5c2aa54938216926a3b543d6601a82da68f5b18d0", + "variant_hash": "02e4bc12dde4ac14e557088baf8eafa66474522c3bb01dbe81855988b503d0fb" + }, + { + "pair_index": 47, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-general_explanatory-32-watermarked", + "source_text_hash": "c8a32fcfcde6639a3d5f66aaafeeb9ba1883ebf19fe6aea7aad67d2806f5feb7", + "transformed_text_hash": "9fa24d9906ffb87d65b00c2905dc9a1f28defcd5c2d4f5ecb4353a8b041520e8", + "variant_hash": "ce32cc63d6bddcce7e015fcd1f7a9032e7f15e77459b8163353a61bb266cc9c9" + }, + { + "pair_index": 48, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-general_explanatory-33-unwatermarked", + "source_text_hash": "6d78c10093957ecff0e6ec0384c48a1460a2708d1790a450aaf7248e439f9943", + "transformed_text_hash": "6d78c10093957ecff0e6ec0384c48a1460a2708d1790a450aaf7248e439f9943", + "variant_hash": "b56435fe9583008fa031aed539a01b0121d857c0f976ddd51bd4749e93f9a67e" + }, + { + "pair_index": 49, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-general_explanatory-33-watermarked", + "source_text_hash": "dd09e6a66240dd1aa8859c9be952b0788f07bdbee4f4d486034e281e3e0b1a82", + "transformed_text_hash": "e247fb3ac95c5b6f1ac1b8ded1f13fd6e3af0ad39316b6c62e9b7d0aa1ac5025", + "variant_hash": "b262bafc81817c01401fdce749b5c1d89ca10f04df5ec180e3c4aee816b80279" + }, + { + "pair_index": 50, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-general_explanatory-34-unwatermarked", + "source_text_hash": "5b721d62dfa8ca30b711dc008912332182c4b43d0c9180ca295fc0a87a45755f", + "transformed_text_hash": "1f42c4509fc90f69d5007443dd56da99541c4af1bf5001259ead8552f032623f", + "variant_hash": "e63e0fee07a46a8e0cb5fabb7ed30ff43b4bf34b712b94515ba230ee420fd4bb" + }, + { + "pair_index": 51, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-general_explanatory-34-watermarked", + "source_text_hash": "193af4f5f4fcc333a30857b687a3aa199488bc118058dc0d403b11263f004a0c", + "transformed_text_hash": "2c9b77a7735ba1ba594ddb0944a81b0683e7033489d0db11a1a1f99e94f77efd", + "variant_hash": "f50321257fdbe3f15ba1b9745888d256306d63a6e92735c41194506325466c97" + }, + { + "pair_index": 52, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-general_explanatory-35-unwatermarked", + "source_text_hash": "9dc34c5dd7adf1e6bfccc838b803df7edc0b94bd0c1676600685f4a294ff0685", + "transformed_text_hash": "7d35791a32750bd62bf272e5ff1e6d3567edf0479c2167e3cdc42739f7a868ac", + "variant_hash": "29a49b12e03c2aa1b3d0f298c440e9c263da5f7b1da7f78a474fb4754daf4ca5" + }, + { + "pair_index": 53, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-general_explanatory-35-watermarked", + "source_text_hash": "e4b4d7897a0afee76dc29a7b91f09df662698f27433a2ec6c09128b6f389b1f5", + "transformed_text_hash": "2778e3f89efc5e0447bed25b6782fa02b5afcf5e5c60eea3b3ae3f849a02729d", + "variant_hash": "cdd6b8abe247ba6c236eea2022f0673665b7e8e56355251c27d183d44741a336" + }, + { + "pair_index": 54, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-general_explanatory-36-unwatermarked", + "source_text_hash": "190d2bbff1e708f7d11f156db266447b976baad7f5c825f0ae8c21e20bf9b4ec", + "transformed_text_hash": "a349dbbfb630fce308d29e69d1ebfeb688de91955bb37eab60cc96e5f4161017", + "variant_hash": "8890380b8d884bcff83115e8a63076398cdb4b47b112f76eb1c2237fee7b32e8" + }, + { + "pair_index": 55, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-general_explanatory-36-watermarked", + "source_text_hash": "2137ae71b3644ec82423ca4e75b545d8a3c5eb7a788dcb0182752a82305ee887", + "transformed_text_hash": "e5d80cd3eb50a7db3eae3cd2d0704a6a6ee3abc8085db5934049f5316636c7f4", + "variant_hash": "fc38182d2c0a63328b08db052af33bf4c841e15fd704349d138d25b9804d7a75" + }, + { + "pair_index": 56, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-general_explanatory-37-unwatermarked", + "source_text_hash": "d6813d5634c7657976a49e54e1ee3fb0c262743ba41d03186f3f47ae76f0950f", + "transformed_text_hash": "6c5922fb7facab8b2336feb3144d085a8d23d6b7121ff67e5fe678fe657802ce", + "variant_hash": "49a9836ab8ff2e6294f9d0b241f5b0eccefbce0cb4ca9450e05b4d5cf7305d8c" + }, + { + "pair_index": 57, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-general_explanatory-37-watermarked", + "source_text_hash": "c026a3b7a76822e5d5fb419554eb33859cb34f470b3f3fd022081958eb095df0", + "transformed_text_hash": "5470b665d53db233e8ed64e494338b60ee86e405f77a76d9cc96c6ba1dc9933f", + "variant_hash": "3274da063da5db8affb703e59b06f4bb6a8e5b0f4e6258037f9ff38be0ba34aa" + }, + { + "pair_index": 58, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-general_explanatory-38-unwatermarked", + "source_text_hash": "d945e74d3d6302e33d785c0f6aa31218e28de62d562d44b3b20d8139f297309b", + "transformed_text_hash": "631dd78c4f8e0f88af9e417255b8d56f11f935111176bc021ff6c68d4170e6b0", + "variant_hash": "52ffdb16e60b85637203ac06f95a0886302dd23ade638d5557ee9d34205b504d" + }, + { + "pair_index": 59, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-general_explanatory-38-watermarked", + "source_text_hash": "4863ba551a30470274c01823d63cf6b8da30fb125157a69bb2a313a2c67192ba", + "transformed_text_hash": "444297244587797a065cbce91ada0063fb5266e274c06ccba27738dbb29b623f", + "variant_hash": "ffae6e5c940f21c65d04d579e0cb844459bc72ccba3f6f11290dc7f173a56ee8" + }, + { + "pair_index": 60, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-general_explanatory-39-unwatermarked", + "source_text_hash": "f3cee919e65c87e7a778a3615d48e5f83461f050cefc4d1ef1220dc3a9347b0b", + "transformed_text_hash": "b9c520ff45f3729cb83ac8829562e2e182482d6a5da0a541dda14822b79ed91f", + "variant_hash": "665904e5852e2b1a03701312f2252e78af96483e732a1c17a943f11d97e02121" + }, + { + "pair_index": 61, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-general_explanatory-39-watermarked", + "source_text_hash": "58a323a4e75b4ff9a9f3ce1e154280754e04c6834903e20d5e62c2db8fcbef16", + "transformed_text_hash": "e9ce6f20bc711e3de9d05f6b851678847068c3bffb9fc3b738cb11e1becadd4f", + "variant_hash": "add68877a3c99b5f90334f467c2a72cbf7b3b6ff3b68c83879d63c8a4b2e95f6" + }, + { + "pair_index": 62, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-general_explanatory-40-unwatermarked", + "source_text_hash": "6bcfd4018abf123952fc2f97dc7c72cd3559e0c3cf6f12896093d85662acc9d8", + "transformed_text_hash": "1d50801f2efa974968106f09dc90944c5f2d77511ee705cdd038cf6f0753280d", + "variant_hash": "1605095faa8955ad813e822cf7dcd0c2e9da8b7df1afeaee5e3fd5ac572a1ca8" + }, + { + "pair_index": 63, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-general_explanatory-40-watermarked", + "source_text_hash": "4d92361e194b5afbbb52973066877852f23b8d8f884aed764ac1d26b78d3b718", + "transformed_text_hash": "186ecbea9be417d64340d31ffe8517dd37cdbe0ff5808353a452a3547a8cedbc", + "variant_hash": "0f32e7df37649cb4df387bf78e8a37dd059c41815cdc22eb0ae23b194c92c177" + }, + { + "pair_index": 64, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-structured_instructional-25-unwatermarked", + "source_text_hash": "025cb365b51494a2bdb43d1a2716adb08bb4f381510b14326ba914c03c553220", + "transformed_text_hash": "2b81386abec26cfca7f2416a2e625f1d9afb48a07d80870d3aec81bf0a739e04", + "variant_hash": "ae4be47fecf6dedc26a38fdeea3bb4feb0ecec819d06ce2a9d4807a0465b216a" + }, + { + "pair_index": 65, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-structured_instructional-25-watermarked", + "source_text_hash": "2b62c7cc0ade63a8eee3e94785107948f74d8876b02966bc4f1c29f5bf4ec80c", + "transformed_text_hash": "7beff0b797a693836a34ba440003fe5455a88d40456b3ee91363c9a9f7252483", + "variant_hash": "7d112173b225d8bd408f4e91ffaa265699139a550ce3190ada5ed5479e64eee7" + }, + { + "pair_index": 66, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-structured_instructional-26-unwatermarked", + "source_text_hash": "4efb4170d38a7c8d0b0aba17f018f1ec6deb003b21e4267d02b0d380c0d83395", + "transformed_text_hash": "2b195d06ba3cb0430145970d0fe2df00d26b636ab170cc063ab9098db6a32e44", + "variant_hash": "b9518e71c504d3445f1461deaca2ed121ecf8e0c2f991ce7d0f7ae2417f9028d" + }, + { + "pair_index": 67, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-structured_instructional-26-watermarked", + "source_text_hash": "28bda76ec6e3b385896853cf26d0bd038f9b4fd1448990da55dcb3c795b10ddb", + "transformed_text_hash": "f34c47a3a8f747ab6d5b8b0c8df6b6edbe043f14802ce1c546790f666d18efee", + "variant_hash": "74bc2667e7f1e6f12af4a6c93df28dcfe7abf2b9b45b5734d1f26f871a325a17" + }, + { + "pair_index": 68, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-structured_instructional-27-unwatermarked", + "source_text_hash": "97ab427fd1c2a9c4a7b0b4f578528e95b26f914838ace8fa1ca397f2e2ce0011", + "transformed_text_hash": "5397a002107d780a168dedbde9eb4f25dc5f348f78510c1cb45cec8c61393f8f", + "variant_hash": "c574dae9cad08d58df7bf0447b28dcf187d15ec7e507b189d716ffa8193b10ff" + }, + { + "pair_index": 69, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-structured_instructional-27-watermarked", + "source_text_hash": "1be7efc24e0caf1b1fb4d64b3426ec98b5a29ced2570568d60c78f22357eb9b1", + "transformed_text_hash": "b8297d63941cde67ffa3c72363f1d900a6ec3e2ee6736b0878b334bf38b434d3", + "variant_hash": "4d4c1b67698fa63cbd4815eba8bff9e6e74f683f48a43132c8f465928eb61d4a" + }, + { + "pair_index": 70, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-structured_instructional-28-unwatermarked", + "source_text_hash": "ec7e8b2891ea5a91b2d8b93208f815ea35ac1e134b7f14e66b81857f897133cc", + "transformed_text_hash": "2f8ad2074f959c76f29d46e006fd65bdc9b98b56805c80ae27b76ce4d04e3904", + "variant_hash": "a337da48e9edebd0e704cdaa4937742b1bf559268de0014da46efb94b1d10b6e" + }, + { + "pair_index": 71, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-structured_instructional-28-watermarked", + "source_text_hash": "f8ab3216d6a5fc3973aa44cd1f7dbd1496453e12cc44ab9a711d037dfcdda2f0", + "transformed_text_hash": "cc9fd1b35102fe884339bcb8bd962b56aa5066138c8499e5f45d16f8b236a8b6", + "variant_hash": "c2024adad04cf780ca99b1d82133e5daf30e280cfc41b97891d32b2a47d52956" + }, + { + "pair_index": 72, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-structured_instructional-29-unwatermarked", + "source_text_hash": "b57b98a14c7fbf36bef83c9df141069355982d7bb94502fcefec7702c6c61538", + "transformed_text_hash": "8ee7eeaa3e68582bff11d7d76edd7b6354d3dff7fdfc5ea74f7464ad560edf0a", + "variant_hash": "289e59beafb5ae374633a5bb185890f9cefd9d3b64c7d7c891f333e2fe855bf3" + }, + { + "pair_index": 73, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-structured_instructional-29-watermarked", + "source_text_hash": "12c51d46529466ebb0a2420afb0aa277eddab8f323e4420066d0f30ca4616185", + "transformed_text_hash": "e6e22770d7d62c9fce1afdd40936fb07dd457dcd0c89ce01395e4dbb9f7cebd3", + "variant_hash": "fb9a511fe33d9049b92cbffd4b85ef7135069f1f7bcd8cbd3fe98e07d7a6e5e3" + }, + { + "pair_index": 74, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-structured_instructional-30-unwatermarked", + "source_text_hash": "b1eca7f8c9b995751ccb280ab6ea6365f6d6d80bfd21684a511fc7759d42e4b2", + "transformed_text_hash": "88252014a95fa840d3d2dc85eefa6ebb93c5dc8637c07316b84fe7e2829cf869", + "variant_hash": "7d52382dc5621551b86cd4d76f3d955fa384a72832e152d5fab9a36f7a5348a0" + }, + { + "pair_index": 75, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-structured_instructional-30-watermarked", + "source_text_hash": "7943ff3211ce0255b57fcada8a7d80e3954a891b42998ca163a07a086e676eba", + "transformed_text_hash": "1cd6a59b4b6dca583da2397e7ea0322ef961058d7800e66637466f7d503c5ae9", + "variant_hash": "c293cb3d9243249f8374a6842cbb3bfc6efe996bf476008f5c832ebe8089c2c8" + }, + { + "pair_index": 76, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-structured_instructional-31-unwatermarked", + "source_text_hash": "b13fef25d155ae159a1957e0877d3de50129a7d2c3699d2ddd1a7b84cac67c7c", + "transformed_text_hash": "4c94e399423997263863e5f9b935ae5a02627dc7bbcf55aca4ac0b841cd982d7", + "variant_hash": "831b48dd7ba0c4e9f4c1058bf2da69b09b21ef8311e81f2ea721430e89e46cf5" + }, + { + "pair_index": 77, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-structured_instructional-31-watermarked", + "source_text_hash": "599477f09b19c1c0c01d8022013f5a0bb9bdf0ea4095a78d3de0be476c1eb591", + "transformed_text_hash": "8a0dcac7ba78eca5333732e2d09f03b3d85099be3d239335d763d383c6636a9f", + "variant_hash": "b648543b2aa780ffe2da3ea0f9a4126cf168c3e85b817af4a76a174b35014d17" + }, + { + "pair_index": 78, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-structured_instructional-32-unwatermarked", + "source_text_hash": "a216b9ae97e02f6dc4c52b1cbb4afc7ca6f445c38d8bc0c49f72af7351ff50e8", + "transformed_text_hash": "d7b23bf24f464fbcc5121758712f47149651688969c5128f9f638f7090f5a1e0", + "variant_hash": "609331274f1dc1d9abf8e5bfce75587531cf223f9c131e1e44c9989dc06b8551" + }, + { + "pair_index": 79, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-structured_instructional-32-watermarked", + "source_text_hash": "535192d753ecb53f4aa4ce28ff9589ccb0ee271f1ffef76c24d6e6c7d4635f29", + "transformed_text_hash": "b071dff4be47bcad25ebdfb8b5bab2f40ff4f1a85e1ae37ae6ed0414d575deeb", + "variant_hash": "a05a0819cc0e2fe2d4e4ac29db2b0dfc433cf92ab693aa7a52eac96b60019dbb" + }, + { + "pair_index": 80, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-structured_instructional-33-unwatermarked", + "source_text_hash": "41bac0e0e10b3b70ac7044cd2bd06b3ec638cb9469f1ecd7f9c985ff9060d748", + "transformed_text_hash": "c4342e1245d0c9f85fdc505b6fb29f360c7c52718cf5ed01ad46f46a4b56cf1a", + "variant_hash": "6d8dd46c2c7b8c4bed606142e3cc48dc75ab75e9038c08a105c17725cef616ea" + }, + { + "pair_index": 81, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-structured_instructional-33-watermarked", + "source_text_hash": "8f4d92c4f2d69e444775fc921d1088fe519fff017ee6bc55b39a0fa8e4b36ef3", + "transformed_text_hash": "59dae99693e5310c7e885a08576daa2683f5f96d9bac50c3016cdb32fea10d3a", + "variant_hash": "b1a1787c6f93f55b5dd167a86920f660f2142c28248af13401dc369810a750b2" + }, + { + "pair_index": 82, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-structured_instructional-34-unwatermarked", + "source_text_hash": "d73c7437a3670cb5a2a5027681cbec9f105af66233f025ab8d757b3c13b6ec4e", + "transformed_text_hash": "c8f1d0e6b46a8f9d0a8eb46cc0ac35abf6d7657043e92596bffe437a317ab30a", + "variant_hash": "585a4ec39c03123d065ea6662c37e93ebab3877807e4c0b440da8cd21765e210" + }, + { + "pair_index": 83, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-structured_instructional-34-watermarked", + "source_text_hash": "f260b4814fb7944dc7d68aa88290ce8a0f5669af4b07adf85ee0e0c27a2957e7", + "transformed_text_hash": "2d980c23ae252bc2b7003e70e4bc8d2542bce20adaa385bac509fd8ff914617a", + "variant_hash": "9cd1180c34ede01a4800993231ae7574369870fdf82c594e6b6c80c1ad61a73b" + }, + { + "pair_index": 84, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-structured_instructional-35-unwatermarked", + "source_text_hash": "66fc607dde40bc3279c1188a5f918afb7805d47ceff11ad49f02d4480c887da2", + "transformed_text_hash": "aa5f467276273194989745b298d195ab45ed4bea26b681dc1e96cea489182a45", + "variant_hash": "fb5f7b4cb5ca88b0800883856100957ff374bac34db0db257e58c0564df6db13" + }, + { + "pair_index": 85, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-structured_instructional-35-watermarked", + "source_text_hash": "6ce7960a22fffef0b10c622c7df3c4733d005e81047a4ca592adbd8d44dd79bc", + "transformed_text_hash": "8d3e8b257a1252c69a7470b49f1030004361445a3e4443e00945b03e1e248251", + "variant_hash": "6f608f138b59772821880aafaeb09036491f17e8892e62c1437171b08bbaddff" + }, + { + "pair_index": 86, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-structured_instructional-36-unwatermarked", + "source_text_hash": "b9fae73db16fdb07bd7d09c8cd7d1453fe6718a375749b7d6e13f9320511e8ac", + "transformed_text_hash": "88dda196e468dab97fecae1d7850418d9f764a41e5d4c0ad7f1a91ee600ac3da", + "variant_hash": "be549314a745443ea23a13a23cf7a02f5d17e7780d5015c24ac000744d377636" + }, + { + "pair_index": 87, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-structured_instructional-36-watermarked", + "source_text_hash": "6937c1fc3e5eabaf18ceceb43b016e09d2d25120448a96c4db6ccbd47b411ba5", + "transformed_text_hash": "6937c1fc3e5eabaf18ceceb43b016e09d2d25120448a96c4db6ccbd47b411ba5", + "variant_hash": "e9c4c6e5cee1eaa1ae1c345a701030e56c2e611ed5676713d0085990886e0b08" + }, + { + "pair_index": 88, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-structured_instructional-37-unwatermarked", + "source_text_hash": "d971912c4e81ad68c7660fc6a33227b9ebf2e04ccb877df22ca2470edac8c5c9", + "transformed_text_hash": "e7e1a9ba890702e3f482543ccd51244d2401618eca3ebc4c800c6e757ce69a29", + "variant_hash": "041b1e76b08a22151466f16457b0524c9574c251f4143b7cc763a0370e2d2218" + }, + { + "pair_index": 89, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-structured_instructional-37-watermarked", + "source_text_hash": "81e85ca4adb64ff68d692f9617155415217c1c14b87e0918b579d741ff16e691", + "transformed_text_hash": "80ad71d5e0b77b765e012cbc5df0ce91c8367a1b7bbc23000cf652c3786b16a4", + "variant_hash": "2853fc7a000ce089795b26d5953a9b041492154760b650b269cf13b2e4531efb" + }, + { + "pair_index": 90, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-structured_instructional-38-unwatermarked", + "source_text_hash": "af61bfaa7335bde2b4eb156421d8fa7263ba9b042f91f35b0d64d64084ec89fe", + "transformed_text_hash": "ddccf6482ec4a8d48b1870130a0429ccf2e3f7e51b729bc4cdabb2756e0fbc56", + "variant_hash": "a64666bb99a293af8649d8d2ea1e3eaec82951576156683e4eee13b49779552c" + }, + { + "pair_index": 91, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-structured_instructional-38-watermarked", + "source_text_hash": "d1cb44f88b79e088131054890b7b05cb53bbd7c92d4df6efea28f205ad03abc1", + "transformed_text_hash": "6bca44635dc944364c27708ad36be643c68407c8c0cabf787f85de74749d7614", + "variant_hash": "2d1519a50547889603a0552179be0ecfad676964c41b0ab1806451f53282132d" + }, + { + "pair_index": 92, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-structured_instructional-39-unwatermarked", + "source_text_hash": "34eef3b23ba205782c5828edc4c645d0276cde52e40a175894edd0f50223791e", + "transformed_text_hash": "b18ff457a5ef82be912e1d96fbb7d2393e44ef7d5b3a005195a6fef357462a9f", + "variant_hash": "258f0b091e09980b23ab864d80a6e2c801d02a7daff74c833443f835801bf133" + }, + { + "pair_index": 93, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-structured_instructional-39-watermarked", + "source_text_hash": "92be15a5321024b93ad2fb0cef2f5bbf29f9b485a6382e0341459bed04b40664", + "transformed_text_hash": "8a9c0b478abebd27403dd97a36b0bc54d9709b2cdac4f1a546e948258d61ac30", + "variant_hash": "d9c84e6c4a64cbdf44c4ebd826bf983937d10b246accf23f1ef025b531f36300" + }, + { + "pair_index": 94, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-structured_instructional-40-unwatermarked", + "source_text_hash": "7d769536562ef15be541999941cb806911f884f36da1b7b759eafceb82723b2d", + "transformed_text_hash": "513f65a1969b891e0cba62ddbaf3079332a2fc27653ce8955886da94ab594fe9", + "variant_hash": "cdfc4fa923862c2cb448a70bec44eb3fab2897983fba3c02f2dcfe1312a2feb3" + }, + { + "pair_index": 95, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-structured_instructional-40-watermarked", + "source_text_hash": "44ae6a7ec136ae7e66e55bd2ba0721862ce70ad1862dcd773875ad601fbe9ffc", + "transformed_text_hash": "a86f84dec6c0149e7384aff7569e2eeefdbc40bff8c99fd8db4d3374189e20d7", + "variant_hash": "cf723d83e28180d60454c0e1f44b89405e19f06a6a95e126fb5312e3f3274d77" + }, + { + "pair_index": 96, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-technical_explanation-25-unwatermarked", + "source_text_hash": "f464fe5dd77e49d97663110b42028017254ddff2bf703aa830c70a491d1afe1d", + "transformed_text_hash": "c834109fabed0d78ee188336a2274eb9fb3f10fd3cde794468fbfacb390d7f5f", + "variant_hash": "7079195b32f233204c6c0cf0718bc8f64810b75e772fc3e34496dbba6c4991f3" + }, + { + "pair_index": 97, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-technical_explanation-25-watermarked", + "source_text_hash": "b92ab33eda4c23ad09d46982047a047ac7519db101054bccdd8692a66f873a80", + "transformed_text_hash": "ceab1fefb5abb2e8f13ff08a0182b172a4b0900fd304a3919aa6f14c8a6f2a27", + "variant_hash": "b21bf72e5e534a9e0a45483a717a3d6ce36b767d6f2403e5e7b24fc7858ef395" + }, + { + "pair_index": 98, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-technical_explanation-26-unwatermarked", + "source_text_hash": "638b518cb906138ae63dbb6f054db29361d4b453f09f3ed0cb94f54eaea2ea7b", + "transformed_text_hash": "936c59e657595a50caef360777a611f0a156eb60d39f9e83f8ae847f1fc92317", + "variant_hash": "9485089156ff14456c831902be3cabb24edafa8c69a173392fd5b7dea1423019" + }, + { + "pair_index": 99, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-technical_explanation-26-watermarked", + "source_text_hash": "bdc4461bce35c07a1927dbf598b5809e2d50071bde32b9ac8ae5c18e6b32519b", + "transformed_text_hash": "f6329d37338f0c3e49e54a834ec6411b69ea6069d588e51eefd6801ceceb924a", + "variant_hash": "71d51c7fa965207398e419319835981fe19b62ded1b9b2ae746c141fedc9cd20" + }, + { + "pair_index": 100, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-technical_explanation-27-unwatermarked", + "source_text_hash": "ef6b15b678fc3448315915f6296fd4e0d0e58d2413bf637015a11e23c2282354", + "transformed_text_hash": "bad14c2f83880468f4326f972cfc26818fed495a85c4cf591bdf890496c43704", + "variant_hash": "203e1e06eb9fc71351229bc5ad25948906704cdcff9acb405ff05ce082bce296" + }, + { + "pair_index": 101, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-technical_explanation-27-watermarked", + "source_text_hash": "a6ba58ebe6cd82e56ec0a2b20ef0cff8204f8401ef697b989b08132bf34314ce", + "transformed_text_hash": "3b899135d9c844e876de686a3acab79c10ce2be9e851219affd491a1e4e1ce16", + "variant_hash": "78e2fb373786cf778ed94664926b9e147273440ec31426551cf3602831975cd0" + }, + { + "pair_index": 102, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-technical_explanation-28-unwatermarked", + "source_text_hash": "c9e67b9c40119311fbb5b562ad9afb0bd43252edd286705b9241fafefff2389d", + "transformed_text_hash": "445f9ae528fca119e4e36944631999110fe1944f4ac92367d65b6f54937f33ec", + "variant_hash": "8ede6ccaa5eebe866e1add38abd37984406457a0f4cb28bb1b5e6645143d9178" + }, + { + "pair_index": 103, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-technical_explanation-28-watermarked", + "source_text_hash": "4001ba60d8426872b3e703ab1c6128133564b26f60de106f5680a0f90ca6059f", + "transformed_text_hash": "a394992f13eaf7b42e956e093df019fe448d244a7cc7e5a3aa55aec8af86428f", + "variant_hash": "101edf0542389a3632279541dd410e5ff11d87bcb4d1f40c00af9a72856499b0" + }, + { + "pair_index": 104, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-technical_explanation-29-unwatermarked", + "source_text_hash": "af70907cafd4785f441751a7bf4a7f28d120225e13183426715167a8124aacf1", + "transformed_text_hash": "2b658e6a925ae3d136eaf7548a1bcc85b4f1a0d5ca577480800d5afd64c301ad", + "variant_hash": "5aa0130cbc61d32d3705d675f8ad3a9d69c492bd69f3d692c567e9bb54ca5bf8" + }, + { + "pair_index": 105, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-technical_explanation-29-watermarked", + "source_text_hash": "c7f6fbd4a3a0ff9b172c2457c58d03ed10e7685cf05dbe2bdf9e875313b64fa9", + "transformed_text_hash": "60d3de4ab47e84a6da8f5facc55089dfd8c9fe52c6dca7c7e55195547e98a260", + "variant_hash": "76d9d13a1d571e24dac6e90b1afde35bac45c34424c41d742b67f26dedd13511" + }, + { + "pair_index": 106, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-technical_explanation-30-unwatermarked", + "source_text_hash": "094c4c617e17470fcaf797ccee0bcb48948d0d2ba0e76d7cfec918dc9faf3509", + "transformed_text_hash": "15ad814dca73485a43954fb79516fb79afc6619bae65f2551f7838a3ee1d3d94", + "variant_hash": "fc2369ac1c4a922d17c0ee1f85d57eee7644c182ceeefeb1630762f33422eb4a" + }, + { + "pair_index": 107, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-technical_explanation-30-watermarked", + "source_text_hash": "d65c37c8c61d01f8cf570d56c0caaae2edae2e88c3462332d6462b49e8003f9a", + "transformed_text_hash": "bed2060dba326f55e33077cb220fb16db0e29d6d3378fa232d42e9638dd1c117", + "variant_hash": "81e080adfcfb45dd69cfc9f32ca291ac72cdd2bfc26ef2a2fd686a6354b78427" + }, + { + "pair_index": 108, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-technical_explanation-31-unwatermarked", + "source_text_hash": "f87342601a1f3b7d8883725c6d07dcd9a482b46eda7839e464fb7842561bddf6", + "transformed_text_hash": "625296697efb81d18cf4f09117f19616e3287c94e0609e0cf2fb8d00f1230af4", + "variant_hash": "4728cf5cf89c08015dc022ff4d112f1831dc7887bba208780ae10bbe7d9209eb" + }, + { + "pair_index": 109, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-technical_explanation-31-watermarked", + "source_text_hash": "005d6e251ea9cebf868c324dfe2a2c2cf7f11825579aaab0451f8e7b98bbc98b", + "transformed_text_hash": "005d6e251ea9cebf868c324dfe2a2c2cf7f11825579aaab0451f8e7b98bbc98b", + "variant_hash": "7fc1e764877aedcd81a4fc2d8f9dd188447246baf7c269f05ad3776673f68fc5" + }, + { + "pair_index": 110, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-technical_explanation-32-unwatermarked", + "source_text_hash": "00c2abda02876c7d1571e18044b3a5a43221be9a127655b7741ed82911011585", + "transformed_text_hash": "da725dbfa5490d0f037b5f2e92e61d5e407c6951ebb00b99c766e8632cbe9fa0", + "variant_hash": "0e205bc0450566c7499f2ae50f6e0a62f7a57f9b1594f3c372527e02d9f0a248" + }, + { + "pair_index": 111, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-technical_explanation-32-watermarked", + "source_text_hash": "e53e90ee0a99b0064a3e47f2e08ba23cc187d92467309b8cd05df6f8965606bc", + "transformed_text_hash": "eb04e7f9f46744eeb14402f7c5533befdb02bc2c15bb09e5ec6351561c2c10f6", + "variant_hash": "5b6242917e8d9db7e14e095124cc02417edc633c1b437b9e68bf7476670021b3" + }, + { + "pair_index": 112, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-technical_explanation-33-unwatermarked", + "source_text_hash": "0e18166a22e16d906c5da852a06f3b5e3908fa7d5fc95701b854ff7502fe4b8e", + "transformed_text_hash": "ad0add174f7f566c0b5eeb7f661a1f0e771c620aa52a115c8c64babd0a396f8c", + "variant_hash": "1b94a7a18f204935787f784f4f318f9551c1c1fbecf709756c06a3c64b8d1773" + }, + { + "pair_index": 113, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-technical_explanation-33-watermarked", + "source_text_hash": "25a761f2aea706cfd8e5381b8786f42c422724304cf893209cb44ad6a35ffd8d", + "transformed_text_hash": "3d2d5457644383a9c6e990ed5791c2ffae915cbd2c827e66be08e3b043f36745", + "variant_hash": "08ab5edf49599d169e6f70a82bb3f2296560458c36442b642f115711bb4c1eb5" + }, + { + "pair_index": 114, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-technical_explanation-34-unwatermarked", + "source_text_hash": "d5da2b31912adb1247d8d5b71530808a672a2714234d3f17f876ca40353353d9", + "transformed_text_hash": "fb86dee23e7c44227d89c0bf112abc5f2b7aff2120a5b0a14878f890eef7fc7d", + "variant_hash": "07893994f7ce917655496e315eacb7c4cc48033dad62ef0683cba01d1eea3dbe" + }, + { + "pair_index": 115, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-technical_explanation-34-watermarked", + "source_text_hash": "1db050472cb804d5bcb93a9c2cba244ffbcc1e7a3ab61e189c063ccf44e897de", + "transformed_text_hash": "cac31cf2a605e3efe5bc38c0df6ef9f34aec4651d74827839d85f4b3da14a84b", + "variant_hash": "137e708f493af406fa334ff5451c3dd4a95b29a54ae55a33eb30354cca0d8e23" + }, + { + "pair_index": 116, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-technical_explanation-35-unwatermarked", + "source_text_hash": "ea88cf9308284c3682c879505e1bc1fbbae7f53e9f6c1ccfb3302bfc971f76e0", + "transformed_text_hash": "358ca37fd9c40ce020701ed2eedbb85c980abc9c0ea21308f61e3fea5484535a", + "variant_hash": "e937a3829e7ad5d84c2453629c271a0eec617eaf50758ac7be5e3d0cd224360e" + }, + { + "pair_index": 117, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-technical_explanation-35-watermarked", + "source_text_hash": "4ca94603c26cae02a9e23dace0a4daa5879337937f9724765f8d0585a7b83835", + "transformed_text_hash": "9dd21d3b346d4e5e3755d4c700df1b56acc1ffbd816cabb47a60050681c8d036", + "variant_hash": "ac0959e59b01f8c47d6cea45020f23c0e20c6a1bc0948a2a79ada8a9691eaa4c" + }, + { + "pair_index": 118, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-technical_explanation-36-unwatermarked", + "source_text_hash": "110953fbe47df723a63c6ffd3ef36e58932d09fb78dcdd731e5d3b65fffada11", + "transformed_text_hash": "e935bfdda819854aec69abc28d5481e45e5c9a83f042e487153b36a6fb13e918", + "variant_hash": "9d2215739c2050d7ac5475d3b656b5e24cedd1a8edf79ab763caedd639cc864c" + }, + { + "pair_index": 119, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-technical_explanation-36-watermarked", + "source_text_hash": "90c59b226b947891c69066d1f1fdbc69e33fc4d3fcd43592f6293d2cdbaf6339", + "transformed_text_hash": "383d545b2547abf1ffa6f77f8bb68a711a094b6b833e564f942f0dcb96cec01b", + "variant_hash": "b45660a5e42d53c994e726847f919e5782329d74b59ea1be137a2c96d90154bc" + }, + { + "pair_index": 120, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-technical_explanation-37-unwatermarked", + "source_text_hash": "36bc0fcae3d2b596a144dbc7fca15fb6ec75a4c8e48fa83c4aaa95b90969894b", + "transformed_text_hash": "5d24b1e22028ec3057edd9383a6e251b8b28eb8a7e119d5bc2e87062ea6d3524", + "variant_hash": "c618d24cd56aa86c227fc09fa7eeca12a6728f24f854628efe2ba619386d9595" + }, + { + "pair_index": 121, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-technical_explanation-37-watermarked", + "source_text_hash": "a52f86968e94b15c8bbce9cd1b7a4cc5b8f1c6035b8a6e69fbb73c6dec2928cc", + "transformed_text_hash": "e81fbf6b76b77c30d5bcbd05f44f23318653ae876229815bd4441fe6f2356ee6", + "variant_hash": "6e40c9325747fab7f7e22a6e83331e33d7c11cda1805b37501dca63aae89ae3d" + }, + { + "pair_index": 122, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-technical_explanation-38-unwatermarked", + "source_text_hash": "7619dacb1f52101c1572eed45418ef28b31d5ec9f69335429e0d76424a7940cb", + "transformed_text_hash": "57701bb6849780319f4cd928ba4be15e06351c23c6e00c147e51a27bc34053ad", + "variant_hash": "0d5e7b500afc636ef61a6a123fc40ee38231190f5d71767af5e2c41974968005" + }, + { + "pair_index": 123, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-technical_explanation-38-watermarked", + "source_text_hash": "f0c3446335e6e0a8a18b61dcc1e5f23536f67bffb8366188babe08b6570f004a", + "transformed_text_hash": "a939179ae219bb3ecb63658742509b035001eb609f9b479a9db0e33c146e0c72", + "variant_hash": "7265f38fc927641972f017d475e41a2e05d85d627a447913cf037a4d69fbae4d" + }, + { + "pair_index": 124, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-technical_explanation-39-unwatermarked", + "source_text_hash": "0cef400b372e42544f6460d2f9c654d60d2fc21787032c936c1b63e514871817", + "transformed_text_hash": "cedd91591392eded86d4899846a6fb0f119504c2c3cecc47dc3a9bff19f24f84", + "variant_hash": "b391c6b95a6104f1b7db930e8215dbb62b6cc3cfc9cbe045d4ac5ef8e11c9329" + }, + { + "pair_index": 125, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-technical_explanation-39-watermarked", + "source_text_hash": "2ea43120ab9a29e13e6a28b2d4d425a4b4004487c8736d14e612f9844d8d30b4", + "transformed_text_hash": "d1e744f85b98b70c2b61b7b1b877f05926989fc06c083d6907093fc006917b71", + "variant_hash": "84d3b8bfc6abb22b000c4d0aea31a5c300662c5913103b8d4e6cee2024856831" + }, + { + "pair_index": 126, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-technical_explanation-40-unwatermarked", + "source_text_hash": "8bf20401901d858b677f85fe7cda63ae176833821aaf2293f7af849019104058", + "transformed_text_hash": "d3e26a7c91922b0cc0f78e88551d2bf65026e39d72bbd4bf1527f9f61343f5ba", + "variant_hash": "ef0a18812a4c9e7c8df488057ca29dc175282acffe75c66c8e88d6feb902f91c" + }, + { + "pair_index": 127, + "requested_budget": 16, + "source_sample_id": "tiny-dev-attack_development-technical_explanation-40-watermarked", + "source_text_hash": "b3d351fbb6710b347298cfe6848d1aa16861865e4577a6f05de46e92c825515d", + "transformed_text_hash": "1bf4b9649fc65c6b4a558d408b02486ef7fd2a7a70428321028cb22f538ad4ff", + "variant_hash": "671bf1aab76e4d27b4069f2d19c9dc425c13b022eb311d84398ac2bea0181da7" + } + ], + "plan_hash": "0cce41214c3e55ee88cb1ca665d0ebf6c6376bea0d0113a3f4ea481049a5199d" +} diff --git a/evidence/content-region-coverage-2026-08-23/confirm3-binding-cross-check.json b/evidence/content-region-coverage-2026-08-23/confirm3-binding-cross-check.json new file mode 100644 index 00000000..739d8189 --- /dev/null +++ b/evidence/content-region-coverage-2026-08-23/confirm3-binding-cross-check.json @@ -0,0 +1,69 @@ +{ + "460000-A": { + "blind": true, + "commit": true, + "corus": true, + "evidence": true, + "fixed_threshold": true, + "plan": true, + "rows": "128/128" + }, + "460000-B": { + "blind": true, + "commit": true, + "corus": true, + "evidence": true, + "fixed_threshold": true, + "plan": true, + "rows": "128/128" + }, + "460000-render": true, + "460000-robustness": true, + "470000-A": { + "blind": true, + "commit": true, + "corus": true, + "evidence": true, + "fixed_threshold": true, + "plan": true, + "rows": "128/128" + }, + "470000-B": { + "blind": true, + "commit": true, + "corus": true, + "evidence": true, + "fixed_threshold": true, + "plan": true, + "rows": "128/128" + }, + "470000-render": true, + "470000-robustness": true, + "480000-A": { + "blind": true, + "commit": true, + "corus": true, + "evidence": true, + "fixed_threshold": true, + "plan": true, + "rows": "128/128" + }, + "480000-B": { + "blind": true, + "commit": true, + "corus": true, + "evidence": true, + "fixed_threshold": true, + "plan": true, + "rows": "128/128" + }, + "480000-render": true, + "480000-robustness": true, + "calibration_replay": "7a12862c14ab16febc538c152b2c4d51ecda4b2648f9f8b3ec353ef42dc8ea93", + "threshold_audit": [ + 0, + 256 + ], + "threshold_replay": true, + "threshold_value": 0.5570987654320988 +} diff --git a/evidence/content-region-coverage-2026-08-23/cycle3-artifacts-SHA256SUMS.txt b/evidence/content-region-coverage-2026-08-23/cycle3-artifacts-SHA256SUMS.txt new file mode 100644 index 00000000..229a40d5 --- /dev/null +++ b/evidence/content-region-coverage-2026-08-23/cycle3-artifacts-SHA256SUMS.txt @@ -0,0 +1,60 @@ +8ad6cb1dd4e8b3a00a221735fd76b9174651b84c2d61957813a3a6e0b80088f7 *confirm3-binding-cross-check.json +cafc349e54c5ba7ebc83eb9d39d56969f416f3b2bb6392309d4cfc8081672611 *confirm3-corpus-460000.json +9ae5d941f594e5825d6dfbcfa181a26e2200b12f4bd8142fb6139f91d8523267 *confirm3-corpus-470000.json +60631ea2572042a6e18b2060751835bf65769c1b78c60a7a974517185d22bccd *confirm3-corpus-480000.json +05c2368d2b0cdca6eed2b259ca2bf277b3fa20bc7dcfe9ecb3afc268c034ba8e *confirm3-evidence-A-460000-provenance.json +a7f3bd18d2aa27a407fbdbc8f56be32cc03e731faa0356cb60dcf540a721c087 *confirm3-evidence-A-460000.json +23f2b87130012b5bfcedd5542bf3cee0a08765ec49f5b62d4dd34694db107241 *confirm3-evidence-A-470000-provenance.json +04c14296238e4bc8445310466d78be442d2edcf1e8c12f0f443d40127c4a1808 *confirm3-evidence-A-470000.json +c0e81029ab0a9a7f17f436bbd1b6b64500adb9aef6c242e0fb72bd12a3708d11 *confirm3-evidence-A-480000-provenance.json +a21c29fcb9446eb05a24b942754cb06de40ce74af23ec36396d251d159e75cd5 *confirm3-evidence-A-480000.json +bccaa46f0e13d08f3be8c2cc8dfed0513a3b8adc82866ee31ec7b21b2e35362c *confirm3-evidence-B-460000-provenance.json +36c3ca15202764aab5e58472984ca059f7a28d91b181322efe846d121787a395 *confirm3-evidence-B-460000.json +c33ce1c2f305802eb0f936c37d4c07f79c397c58c4370686263fb814bbbb556c *confirm3-evidence-B-470000-provenance.json +06c9834785b4cdceacfaa5ebb5182e316d4713df3d105d68bc751b8921dd8b46 *confirm3-evidence-B-470000.json +33f3e522923ba741986d41cc29922ab8a89448b26b474ba321d3af17d9adcdd2 *confirm3-evidence-B-480000-provenance.json +175803e559da351ea10a7c609a790b100f270559c084b3f5415b28df9cf03e97 *confirm3-evidence-B-480000.json +ea60b558d22467b5bb2793e871ab63a852f08ff7bd0ca24bf5c41e0a967561d7 *confirm3-plan-A-460000-provenance.json +bde3e082077584b288ce2622be7b6dca11fcee37916c32a27b6cc8be0a31f647 *confirm3-plan-A-460000.json +0ee54a3f28307cc567d676ab97b926959ff810239903eb4444dd8f3871b3c60f *confirm3-plan-A-470000-provenance.json +d08091e1259af5f9957ca678614d517333854a9af2f2138dec32d8fa9837b7ff *confirm3-plan-A-470000.json +f688ce34d5e1f0fabbd64bfd22351425acbd34535f69e50387d4d0f5b7e97028 *confirm3-plan-A-480000-provenance.json +023f138f13ef55858ea8f906596ea3918c8a68710b3167fb817785327a11cc9c *confirm3-plan-A-480000.json +9b94097882131daa8bf08b243332924bee40b9d30eb13c04974f0fffcfde30b7 *confirm3-plan-B-460000-provenance.json +6c311633ca839122d96ce3f2ad965ccd95a78c60b864e472cda69e6d6e549b54 *confirm3-plan-B-460000.json +0330e7a4b98e60e4188245bcfb82fc07f2d18ab4803a1a597df1b47baf4aa6c3 *confirm3-plan-B-470000-provenance.json +07a9688c4abe967c9ad0db055e9f5574134c6ca95dfb7f10f5f3120e2f8ecc47 *confirm3-plan-B-470000.json +26e08d091bb1291a9d77dcbd4cfef1328a7a01742061b3c1d4c2c5b8fba95acd *confirm3-plan-B-480000-provenance.json +f974c48e54bc08f8bfed88e71914ec12aed3fce3436c2b2e36b13af91d4ae50d *confirm3-plan-B-480000.json +e6f38f203666ac68864a9dd959bcf44c8388a7bf1f7d00519bfc59d612f3243e *confirm3-render-manifest-B-460000.json +ce2481fc83d57ef3134286b649d49bd201868f4894f9148b6b915bc3da233bdd *confirm3-render-manifest-B-470000.json +3a3573bad79bf5e10055fa3d3c425f8d72ef78bdcbc16ccc2ab13090e34f4721 *confirm3-render-manifest-B-480000.json +676a736975ba920dc2a22d423822c7e0476ad5a37066ad1433e372d7693c9500 *confirm3-robustness-460000.json +3404c6f7d52cf529fdaf0db340c6c11854155eec8c0e8296148791d0016ce050 *confirm3-robustness-470000.json +469448cd4a39e6c73a1919fdca3c232cd7a9f92528802b5c5528efa11c281b99 *confirm3-robustness-480000.json +5ec923cc0462af4d7e79e62b8fe6ffe3456f8c06c314d8acff3b6c71defee7bf *dev3-corpus-450000.json +e18c77c18cc33d1a1d5b9f90cc565dfe649b1827514d08c3d64f1b7ff8c4b2b4 *dev3-evidence-A-450000-provenance.json +d275295629ca9247241a723d50b67853c70fc2988f5521a7b8d21269db5e65b6 *dev3-evidence-A-450000.json +abe4597f04ab39c7166b0e031885fd7258bed9a604cf39e3f9fd65c3ec2c7b8f *dev3-evidence-B-450000-provenance.json +d2354dc8c4a2d13c7982557734c5d4086ea65153bd0172bfbbfb0d608f10e3dd *dev3-evidence-B-450000.json +425bada8bab9b5cb6fa336406754a120dd61acc4412f5fecd82270376b84e41d *dev3-evidence-C-450000-provenance.json +10f290f556acc85dcfceab1c94ba055eb09268b983d42d7703add81c156df780 *dev3-evidence-C-450000.json +9a58e96b7469b10e5366732e9158ff3c421120794549ce124488fcd71fe8d707 *dev3-evidence-D-450000-provenance.json +ab3017eeb7d08bbed3b4cdb7457dd23a4ba6adff22d0354af383ffd1222e167a *dev3-evidence-D-450000.json +e563703bdabdf866330b0f64cbe06bfbfa5d7b1fd0685a02e8eecf8fa0b55c2a *dev3-holes-A-450000.json +29f575bc5a74f7a2a32f815d5979650fa62c6843a9f7de0de591d4d3d55caf20 *dev3-holes-B-450000.json +ea100383c0ad31819d8224aa513b8de35630c5f4555589e10267baeb36f602de *dev3-holes-C-450000.json +a9e35e3c1afd395799ea05d110751daeedc048f210910784daa7178114d6904f *dev3-holes-D-450000.json +3ad9d5feabfe9958762cfa72ec1b7364d1775bb6d2296c708280e179eca3c27a *dev3-plan-A-450000-provenance.json +7c4257d4a09ece0cc53bc5c6529735caadf2fa4d064d21a2f4720f30d211f5ac *dev3-plan-A-450000.json +d6857024fb97e9c3ed4128f337ecab60a0c2bdb62db5c55db057fd3ca49d65cc *dev3-plan-B-450000-provenance.json +26f12b91f2d32ba1d3de077bfbccfac6629d2b647f009e348c9d91a956c8debe *dev3-plan-B-450000.json +1f8abdbe1db531847bbb4b7ab9c2dd81cec180972a005d4437048088f36d0e94 *dev3-plan-C-450000-provenance.json +60ddaeda97b5f55dff215eb4557dec5f680def2872c1395b7021ff2dc1ec23ad *dev3-plan-C-450000.json +64eb9d5622b0222566d454d16d24c22d5467dfa8ffcf504f46030e9364d7759f *dev3-plan-D-450000-provenance.json +0820650766b19917500cd7d94496bc7313b99837b60c9bf179ea7a9bfd21e1cf *dev3-plan-D-450000.json +99527adb63bcb282a9d64df29cbdae732499f947cd3f520bb2807e789dfad517 *dev3-robustness-450000.json +b775c7671740eaff8c6fdf608e89689025d232b44e770a9ab010d5e4f7fdcca0 *calibration-corpus-500000.json +0b1b9e6ead71caf567e1b21bb3996098298aa207ae86db15c07584faaae09f37 *fixed-threshold.json +1c04bb330363063995433a216c3cbc0306fcfd1535a537ffec0df97c79f4b15d *holes-cycle1-420000.json +43f09bc58f4e275b5d79f87942de1e72e615e01da9b38b3b3def8b9008a516be *holes-cycle2-440000.json diff --git a/evidence/content-region-coverage-2026-08-23/final-report.md b/evidence/content-region-coverage-2026-08-23/final-report.md new file mode 100644 index 00000000..a102d2bd --- /dev/null +++ b/evidence/content-region-coverage-2026-08-23/final-report.md @@ -0,0 +1,129 @@ +# Content-Region Coverage Cycle 3 - Final Research Report (2026-08-23) + +## Repository state + +- Base `origin/main`: `16e153fdfa3739a88bca9b1559543a10ded2d859` (unchanged upstream; no open PRs). +- Research branch: `research/content-region-coverage-v1`, child of Cycle-2 HEAD `4ddd84f`. +- Final HEAD: `77354e68a45aee9b93aa73afe2ddca0db0bffaa7` (confirmation-scoring commit). +- Nothing pushed at any point. Worktree clean after final commits. +- Frozen historical identities verified at takeover: B16 profile hash + `6ad142262bfb11a714565d7bd43daa859657fe817bdafa9c8dcf0f4884c07512` replays; Cycle-1/2 + artifacts intact; full suite 1436 passed / 8 skipped with only the pre-documented + Windows-only `spec.md` CRLF freeze failure (`git show HEAD:spec.md` hashes to the pin). + +## Measurement (Phase A) + +- New identity `open-detector-measurement-stability-v1`. Dedicated calibration corpus: 40 new + topics x 4 domains x 8 seeds = 1280 unwatermarked 64-token negatives (seed base 500000, + artifact `7a12862c...`), prompt/seed/text disjoint from every prior corpus. +- Frozen threshold: ascending order statistic n - floor(0.01*n) + 1 = 1015 of 1024 calibration + scores = **0.5570987654320988**; comparison `>=`; float64; no tie rounding; target FPR 1%. +- Audit: 0/256 exceedances on the independent audit split (exact 95% binomial interval + recorded in the artifact). Calibration exceedances 10/1024 (0.977%). Detector identity + bound and matched at every scoring invocation. +- This replaces per-corpus thresholds (Cycle 1: 0.5585; Cycle 2: 0.5803) with one + corpus-independent value; all six confirmation scorings used it. +- Environment manifest frozen in the contract (Windows-10-10.0.26220 AMD64, Python 3.11.15, + torch 2.13.0+cpu, transformers 5.16.0.dev0 pinned git revision, GPT-2 revision pinned). + Cross-platform corpus-generation byte-reproducibility is NOT claimed. + +## Candidate mechanism and diagnosis chain (Phases B/C) + +- Phase B (structural, on consumed Cycle-1/2 artifacts): uncovered observation windows are + dominated by lowercase content words (447 across both confirmations), then capitalized + content (99) and listed function words in unreachable positions (142); the worst texts are + protected numeric runs (correctly unreachable). New `coverage_holes` module reports token + counts, per-family candidate counts, achievable/realized coverage, longest uncovered run, + lexical categories, protected intersections, density, and redundancy per source. +- Phase C: one new rule, `GeneralWordSpacingRule` (`surface-space-after-any-word`): any + standalone alphabetic word followed by a single space and a non-space character gains one + trailing space. No word list, no hidden representation, case-sensitive, protected spans + rejected by the registry. Profile `content-region-coverage-v1` (hashes in the contract), + budget 16, unchanged COVERAGE_GREEDY_KEY_BLIND scheduler, seed base 1_150_000. +- Structural dev evidence: mean achievable coverage 0.841 (Cycle-2 ruleset) -> 0.941 + (candidate); in the 8 lowest-coverage sources the paired gain was +0.20 (gate >= 0.05); + pools 21.6 -> 45.0; zero-candidate rate 0. Ablation: the general rule alone reproduces the + full gain (arm C); adding the Cycle-2 word list (arm D) adds candidates but zero coverage - + the general rule is the operative mechanism. + +## Development (fresh corpus 450000, four detector-blind arms, per-corpus thresholds) + +All four arms: 1/32 transformed-watermarked detections (same source), 0/32 unwatermarked +false positives, fidelity gates 256/256 rows pass. Promotion gate: all twelve criteria +passed and recorded before confirmation scoring (freeze appendix). + +## Confirmation (three frozen corpora, paired A/B, fixed threshold, scored once each) + +| Corpus | Pristine WM | A (Cycle-2 ruleset) | B (Cycle-3 candidate) | FP A | FP B | +| --- | --- | --- | --- | --- | --- | +| 460000 | 63/64 | 4/64 | 3/64 | 1/64 | 0/64 | +| 470000 | 64/64 | 8/64 | 7/64 | 1/64 | 1/64 | +| 480000 | 63/64 | 10/64 | 2/64 | 0/64 | 0/64 | +| Pooled | 190/192 | 22/192 (11.5%) | 12/192 (6.3%) | 2/192 | 1/192 | + +B is paired-better on every corpus and pooled (relative reduction 45%). Mean watermarked +score drop: B 0.119 vs A 0.113. Margin bands (pooled, B): 9 within +-0.005, 21 within ++-0.010, 51 within +-0.025 of threshold - residual detection remains margin-driven. +Fidelity/representation gates: 256/256 rows per corpus across both arms. Cross-corpus +text-hash disjointness: pairwise zero among the three corpora; one naturally occurring +collision between 480000 and the consumed Cycle-2 corpus 440000 (single degenerate text, +policy followed: scored as generated, no regeneration). DOM text equality 384/384 pairs; +screenshot archived. + +## Adverse findings (all reported) + +1. Currency protected-span extractor bug: a double space before `$100` changed the extracted + span from "$100" to " $100" (`PROTECTED_CONTENT_CHANGED`), failing plan A on corpus 480000 + fail-closed before any scoring. Root cause fixed (whitespace branch now requires a sign), + regression tests added, recorded as pre-score amendment 1. The fail-closed builder worked + as designed. +2. Transformed-unwatermarked false positives: 3 events across 384 arm-corpus cells + (A: 460000 one, 470000 one; B: 470000 one). Two of the three texts were already above the + threshold when pristine (consistent with the fixed threshold's nominal 1% FPR; the 0/64 + gate was calibrated under the old higher per-corpus thresholds). Two genuine + transformation-induced crossings exist (A 0.5234->0.5611; B 0.4510->0.5589, a 0.1079 score + increase on an unwatermarked text). The pre-registered 0/64 FP gate FAILED for arm A on + 460000/470000 and arm B on 470000 and is reported as such. +3. Degenerate generation events: one all-EOS continuation crash path fixed with deterministic + retry in the v3 and calibration generators (pre-scoring); one cross-corpus degenerate text + collision (above). +4. Dev underpredicted absolute residual rates again (1/32 dev vs 12/192 pooled + confirmation for arm B). + +## Classification + +**PARTIAL_SUCCESS** (pre-registered class): the candidate is paired-better than the Cycle-2 +baseline on every confirmation corpus and pooled (22/192 -> 12/192), all structural +mechanism gates passed, all fidelity gates passed, and pristine-watermark control gates +passed on all three corpora (63, 64, 63 of 64) - but residual detections remain (12/192, not +the 0/192 stretch) and the transformed-unwatermarked 0/64 gate failed in three arm-corpus +cells under the fixed 1%-FPR threshold. + +## Permitted conclusion + +Under the fixed open Hugging Face SynthID weighted-mean detector with public development +keys, a frozen corpus-independent threshold at nominal 1% FPR (audit 0/256), and the +uncleaned text-only input path, a detector-blind key-blind general any-word spacing rule +materially increased legal candidate coverage in content-word regions and reduced pooled +transformed watermarked detection from 22/192 to 12/192 relative to the Cycle-2 ruleset in a +paired three-corpus design, while preserving every fidelity, representation, normalization, +and rendering gate. The evidence does not establish zero detection, watermark removal, +undetectability, unknown-key or proprietary-detector performance, cleaning durability, or +release readiness, and the fixed-threshold regime exhibits expected low-rate false positives +on unwatermarked text. + +## Not supported + +Watermark removal; undetectability; SynthID defeat; Gemini or proprietary-detector transfer; +unknown-key transfer; normalization/cleaning durability; human semantic equivalence (blind +human fidelity evidence: NOT_PROVIDED); production readiness or release authorization; +cross-platform byte-reproducible corpus generation. + +## Reproducibility + +Independent binding cross-check over all Cycle-3 artifacts: every corpus, calibration, +threshold, plan, evidence, robustness, and render-manifest hash replays and mutually binds; +128/128 row hashes in all six evidence files; all plans bind the scoring commit and record +detector_access_observed=false / secret_access_observed=false. Full suite: 1436 passed, +8 skipped, 1 pre-documented environmental failure. `git diff --check` clean. Artifact +SHA-256 manifest in this directory; local commits only, nothing pushed, main untouched. diff --git a/fuckmark/corpus/__init__.py b/fuckmark/corpus/__init__.py index 1166bd81..73ebc2b3 100644 --- a/fuckmark/corpus/__init__.py +++ b/fuckmark/corpus/__init__.py @@ -38,6 +38,21 @@ load_tiny_dev_corpus_json, parse_tiny_dev_corpus_json, ) +from .tiny_dev_v3 import ( + TINY_DEV_V3_CORPUS_ALGORITHM_VERSION, + TINY_DEV_V3_MAX_ATTACK_PAIRS_PER_DOMAIN, + TinyDevV3CorpusArtifact, + build_tiny_dev_v3_corpus, +) +from .tiny_dev_v3_generation import ( + build_real_tiny_dev_v3_corpus, + build_tiny_dev_v3_prompt_records, +) +from .tiny_dev_io_v3 import ( + load_tiny_dev_corpus_by_version_json, + load_tiny_dev_v3_corpus_json, + parse_tiny_dev_v3_corpus_json, +) from .tokenization import GenerationTokenRecord, TextOnlyTokenRecord, TokenTrack from .mid_dev_calibration_independence_v3 import ( MID_DEV_CALIBRATION_INDEPENDENCE_V3_SELECTION_RULE, @@ -78,6 +93,8 @@ "TINY_DEV_JSON_MAX_BYTES", "TINY_DEV_SPLITS", "TINY_DEV_TARGET_LENGTH", + "TINY_DEV_V3_CORPUS_ALGORITHM_VERSION", + "TINY_DEV_V3_MAX_ATTACK_PAIRS_PER_DOMAIN", "CorpusDomain", "CorpusIntegrityError", "CorpusLeakageError", @@ -107,14 +124,20 @@ "TinyDevCorpusCell", "TinyDevCorpusError", "TinyDevCorpusJsonError", + "TinyDevV3CorpusArtifact", "TokenTrack", "WatermarkCondition", "WatermarkLabel", "build_corpus_manifest", "build_calibration_independence_v3", "build_calibration_independence_v3_from_samples", + "build_real_tiny_dev_v3_corpus", "build_tiny_dev_corpus", + "build_tiny_dev_v3_corpus", + "build_tiny_dev_v3_prompt_records", "load_tiny_dev_corpus_json", + "load_tiny_dev_corpus_by_version_json", + "load_tiny_dev_v3_corpus_json", "load_mid_dev_calibration_independence_v3_artifact_json", "load_mid_dev_calibration_independence_v3_exclusion_json", "load_mid_dev_calibration_independence_v3_manifest_json", @@ -122,4 +145,5 @@ "parse_mid_dev_calibration_independence_v3_exclusion_json", "parse_mid_dev_calibration_independence_v3_manifest_json", "parse_tiny_dev_corpus_json", + "parse_tiny_dev_v3_corpus_json", ] diff --git a/fuckmark/corpus/measurement_calibration.py b/fuckmark/corpus/measurement_calibration.py new file mode 100644 index 00000000..16cb752d --- /dev/null +++ b/fuckmark/corpus/measurement_calibration.py @@ -0,0 +1,313 @@ +from __future__ import annotations + +from collections import defaultdict +from collections.abc import Sequence +from dataclasses import dataclass + +from .._validation import require_clean_string, require_int, require_sha256 +from ..hashing import sha256_json +from .prompt import PromptRecord +from .sample import CorpusSample +from .schema import CorpusDomain, CorpusSplit, KeySplit, WatermarkLabel +from .tiny_dev import TINY_DEV_DOMAINS, TINY_DEV_TARGET_LENGTH, TinyDevCorpusError + + +MEASUREMENT_CALIBRATION_CORPUS_VERSION = "measurement-calibration-corpus-v1" +MEASUREMENT_CALIBRATION_MANIFEST_VERSION = "measurement-calibration-manifest-v1" +MEASUREMENT_CALIBRATION_PROMPT_SOURCE_ID = "fuckmark-measurement-calibration-prompts-v1" +MEASUREMENT_CALIBRATION_TOPICS_PER_DOMAIN = 40 +MEASUREMENT_CALIBRATION_SEEDS_PER_PROMPT = 8 +MEASUREMENT_CALIBRATION_SAMPLE_COUNT = ( + MEASUREMENT_CALIBRATION_TOPICS_PER_DOMAIN + * len(TINY_DEV_DOMAINS) + * MEASUREMENT_CALIBRATION_SEEDS_PER_PROMPT +) +MEASUREMENT_CALIBRATION_NEGATIVE_COUNT = 1024 +MEASUREMENT_CALIBRATION_AUDIT_COUNT = ( + MEASUREMENT_CALIBRATION_SAMPLE_COUNT - MEASUREMENT_CALIBRATION_NEGATIVE_COUNT +) + + +_MEASUREMENT_CALIBRATION_TOPICS = ( + "artifact retention", + "audit trails", + "binary outcomes", + "blinding procedures", + "carryover effects", + "categorical drift", + "causal language", + "censoring rules", + "checklist discipline", + "codebook stability", + "cohort definitions", + "comparison validity", + "compute budgets", + "concept drift", + "consent records", + "contamination checks", + "data dictionaries", + "decision rules", + "definitional clarity", + "distribution shift", + "documentation debt", + "effect direction", + "endpoint choice", + "enumeration order", + "exclusion criteria", + "external validity", + "fair comparison", + "follow-up windows", + "group assignment", + "hypothesis registries", + "inclusion criteria", + "interrater agreement", + "label leakage", + "latent confounding", + "lookup tables", + "matching strategies", + "metadata quality", + "missingness patterns", + "operationalization", + "ownership boundaries", +) + + +def measurement_calibration_topics() -> tuple[str, ...]: + return _MEASUREMENT_CALIBRATION_TOPICS + + +@dataclass(frozen=True, slots=True) +class MeasurementCalibrationManifest: + algorithm_version: str + corpus_id: str + language: str + prompts: tuple[PromptRecord, ...] + samples: tuple[CorpusSample, ...] + prompt_manifest_hash: str + sample_manifest_hash: str + manifest_hash: str + + def __post_init__(self) -> None: + require_clean_string("algorithm_version", self.algorithm_version) + if self.algorithm_version != MEASUREMENT_CALIBRATION_MANIFEST_VERSION: + raise ValueError("unsupported measurement calibration manifest version") + require_clean_string("corpus_id", self.corpus_id) + require_clean_string("language", self.language) + if not isinstance(self.prompts, tuple) or not isinstance(self.samples, tuple): + raise TypeError("prompts and samples must be tuples") + _validate_measurement_calibration_manifest(self) + for name in ( + "prompt_manifest_hash", + "sample_manifest_hash", + "manifest_hash", + ): + require_sha256(name, getattr(self, name)) + payload = { + "algorithm_version": self.algorithm_version, + "corpus_id": self.corpus_id, + "language": self.language, + "prompt_manifest_hash": self.prompt_manifest_hash, + "sample_manifest_hash": self.sample_manifest_hash, + "sample_count": len(self.samples), + } + if self.manifest_hash != sha256_json(payload): + raise ValueError("manifest_hash does not match measurement calibration manifest") + + def payload(self) -> dict[str, object]: + return { + "algorithm_version": self.algorithm_version, + "corpus_id": self.corpus_id, + "language": self.language, + "prompts": self.prompts, + "samples": self.samples, + "prompt_manifest_hash": self.prompt_manifest_hash, + "sample_manifest_hash": self.sample_manifest_hash, + "manifest_hash": self.manifest_hash, + } + + +def build_measurement_calibration_manifest( + corpus_id: str, + prompts: Sequence[PromptRecord], + samples: Sequence[CorpusSample], + language: str = "en", +) -> MeasurementCalibrationManifest: + require_clean_string("corpus_id", corpus_id) + prompt_tuple = tuple(sorted(prompts, key=lambda value: value.prompt_id)) + sample_tuple = tuple(sorted(samples, key=lambda value: value.sample_id)) + prompt_manifest_hash = sha256_json(tuple(value.record_hash for value in prompt_tuple)) + sample_manifest_hash = sha256_json(tuple(value.record_hash for value in sample_tuple)) + payload = { + "algorithm_version": MEASUREMENT_CALIBRATION_MANIFEST_VERSION, + "corpus_id": corpus_id, + "language": language, + "prompt_manifest_hash": prompt_manifest_hash, + "sample_manifest_hash": sample_manifest_hash, + "sample_count": len(sample_tuple), + } + return MeasurementCalibrationManifest( + algorithm_version=MEASUREMENT_CALIBRATION_MANIFEST_VERSION, + corpus_id=corpus_id, + language=language, + prompts=prompt_tuple, + samples=sample_tuple, + prompt_manifest_hash=prompt_manifest_hash, + sample_manifest_hash=sample_manifest_hash, + manifest_hash=sha256_json(payload), + ) + + +@dataclass(frozen=True, slots=True) +class MeasurementCalibrationCorpus: + algorithm_version: str + manifest: MeasurementCalibrationManifest + target_length: int + topic_count_per_domain: int + seeds_per_prompt: int + sample_count: int + negative_count: int + audit_count: int + model_identity_hash: str + generation_matching_signature_hash: str + watermark_condition_hash: str + artifact_hash: str + + def __post_init__(self) -> None: + require_clean_string("algorithm_version", self.algorithm_version) + if self.algorithm_version != MEASUREMENT_CALIBRATION_CORPUS_VERSION: + raise ValueError("unsupported measurement calibration corpus version") + if not isinstance(self.manifest, MeasurementCalibrationManifest): + raise TypeError("manifest must be a MeasurementCalibrationManifest") + if self.target_length != TINY_DEV_TARGET_LENGTH: + raise ValueError("measurement calibration target length must be 64") + if self.topic_count_per_domain != MEASUREMENT_CALIBRATION_TOPICS_PER_DOMAIN: + raise ValueError("measurement calibration topic count is frozen") + if self.seeds_per_prompt != MEASUREMENT_CALIBRATION_SEEDS_PER_PROMPT: + raise ValueError("measurement calibration seed count is frozen") + if self.sample_count != MEASUREMENT_CALIBRATION_SAMPLE_COUNT: + raise ValueError("measurement calibration sample count is frozen") + if self.negative_count != MEASUREMENT_CALIBRATION_NEGATIVE_COUNT: + raise ValueError("measurement calibration negative count is frozen") + if self.audit_count != MEASUREMENT_CALIBRATION_AUDIT_COUNT: + raise ValueError("measurement calibration audit count is frozen") + for name in ( + "model_identity_hash", + "generation_matching_signature_hash", + "watermark_condition_hash", + "artifact_hash", + ): + require_sha256(name, getattr(self, name)) + _validate_measurement_calibration_manifest(self.manifest) + payload = self._payload() + if self.artifact_hash != sha256_json(payload): + raise ValueError("artifact_hash does not match measurement calibration corpus") + + def _payload(self) -> dict[str, object]: + return { + "algorithm_version": self.algorithm_version, + "manifest_hash": self.manifest.manifest_hash, + "target_length": self.target_length, + "topic_count_per_domain": self.topic_count_per_domain, + "seeds_per_prompt": self.seeds_per_prompt, + "sample_count": self.sample_count, + "negative_count": self.negative_count, + "audit_count": self.audit_count, + "model_identity_hash": self.model_identity_hash, + "generation_matching_signature_hash": self.generation_matching_signature_hash, + "watermark_condition_hash": self.watermark_condition_hash, + } + + def calibration_samples(self) -> tuple[CorpusSample, ...]: + ordered = tuple(sorted(self.manifest.samples, key=lambda sample: sample.sample_id)) + return ordered[: self.negative_count] + + def audit_samples(self) -> tuple[CorpusSample, ...]: + ordered = tuple(sorted(self.manifest.samples, key=lambda sample: sample.sample_id)) + return ordered[self.negative_count :] + + +def _validate_measurement_calibration_manifest(manifest: MeasurementCalibrationManifest) -> None: + prompts = manifest.prompts + samples = manifest.samples + if len(samples) != MEASUREMENT_CALIBRATION_SAMPLE_COUNT: + raise TinyDevCorpusError( + f"measurement calibration corpus must contain exactly {MEASUREMENT_CALIBRATION_SAMPLE_COUNT} samples" + ) + prompt_ids = tuple(value.prompt_id for value in prompts) + if len(set(prompt_ids)) != len(prompt_ids): + raise TinyDevCorpusError("measurement calibration prompt IDs must be unique") + prompt_texts = tuple(value.text for value in prompts) + if len(set(prompt_texts)) != len(prompt_texts): + raise TinyDevCorpusError("measurement calibration prompt texts must be unique") + family_splits: dict[str, set[CorpusSplit]] = defaultdict(set) + for prompt in prompts: + family_splits[prompt.prompt_family_id].add(prompt.split) + if any(len(splits) != 1 for splits in family_splits.values()): + raise TinyDevCorpusError("measurement calibration prompt families must not cross splits") + if any(sample.split is not CorpusSplit.THRESHOLD_CALIBRATION for sample in samples): + raise TinyDevCorpusError("measurement calibration samples must use the calibration split") + if any(sample.label is not WatermarkLabel.UNWATERMARKED for sample in samples): + raise TinyDevCorpusError("measurement calibration samples must be unwatermarked") + if any(sample.watermark.key_split is not KeySplit.DEV for sample in samples): + raise TinyDevCorpusError("measurement calibration samples must record DEV key split") + if any(sample.target_length != TINY_DEV_TARGET_LENGTH for sample in samples): + raise TinyDevCorpusError("measurement calibration samples must target 64 continuation tokens") + domains = {sample.domain for sample in samples} + if domains != set(TINY_DEV_DOMAINS): + raise TinyDevCorpusError("measurement calibration must cover the frozen four-domain profile") + if len({sample.sample_id for sample in samples}) != len(samples): + raise TinyDevCorpusError("measurement calibration sample IDs must be unique") + if len({sample.text_sha256 for sample in samples}) != len(samples): + raise TinyDevCorpusError("measurement calibration texts must be unique") + if len({sample.generation_tokens.continuation_token_hash for sample in samples}) != len(samples): + raise TinyDevCorpusError("measurement calibration continuations must be unique") + prompt_by_id = {value.prompt_id: value for value in prompts} + for sample in samples: + prompt = prompt_by_id.get(sample.prompt_id) + if prompt is None: + raise TinyDevCorpusError("measurement calibration sample references an unknown prompt") + if sample.prompt_family_id != prompt.prompt_family_id or sample.domain is not prompt.domain: + raise TinyDevCorpusError("measurement calibration sample metadata does not match its prompt") + if len(set(prompt_by_id) - {sample.prompt_id for sample in samples}): + raise TinyDevCorpusError("measurement calibration manifest contains unused prompts") + + +def build_measurement_calibration_corpus( + corpus_id: str, + prompts: Sequence[PromptRecord], + samples: Sequence[CorpusSample], +) -> MeasurementCalibrationCorpus: + require_clean_string("corpus_id", corpus_id) + manifest = build_measurement_calibration_manifest(corpus_id, prompts, samples) + model_hashes = {sample.model.identity_hash for sample in manifest.samples} + generation_hashes = {sample.generation.matching_signature_hash for sample in manifest.samples} + watermark_hashes = {sample.watermark.condition_hash for sample in manifest.samples} + if len(model_hashes) != 1 or len(generation_hashes) != 1 or len(watermark_hashes) != 1: + raise TinyDevCorpusError("measurement calibration must use exactly one identity triple") + payload = { + "algorithm_version": MEASUREMENT_CALIBRATION_CORPUS_VERSION, + "manifest_hash": manifest.manifest_hash, + "target_length": TINY_DEV_TARGET_LENGTH, + "topic_count_per_domain": MEASUREMENT_CALIBRATION_TOPICS_PER_DOMAIN, + "seeds_per_prompt": MEASUREMENT_CALIBRATION_SEEDS_PER_PROMPT, + "sample_count": len(manifest.samples), + "negative_count": MEASUREMENT_CALIBRATION_NEGATIVE_COUNT, + "audit_count": MEASUREMENT_CALIBRATION_AUDIT_COUNT, + "model_identity_hash": next(iter(model_hashes)), + "generation_matching_signature_hash": next(iter(generation_hashes)), + "watermark_condition_hash": next(iter(watermark_hashes)), + } + return MeasurementCalibrationCorpus( + algorithm_version=MEASUREMENT_CALIBRATION_CORPUS_VERSION, + manifest=manifest, + target_length=TINY_DEV_TARGET_LENGTH, + topic_count_per_domain=MEASUREMENT_CALIBRATION_TOPICS_PER_DOMAIN, + seeds_per_prompt=MEASUREMENT_CALIBRATION_SEEDS_PER_PROMPT, + sample_count=len(manifest.samples), + negative_count=MEASUREMENT_CALIBRATION_NEGATIVE_COUNT, + audit_count=MEASUREMENT_CALIBRATION_AUDIT_COUNT, + model_identity_hash=next(iter(model_hashes)), + generation_matching_signature_hash=next(iter(generation_hashes)), + watermark_condition_hash=next(iter(watermark_hashes)), + artifact_hash=sha256_json(payload), + ) diff --git a/fuckmark/corpus/measurement_calibration_io.py b/fuckmark/corpus/measurement_calibration_io.py new file mode 100644 index 00000000..9dde2d2a --- /dev/null +++ b/fuckmark/corpus/measurement_calibration_io.py @@ -0,0 +1,115 @@ +from __future__ import annotations + +import json +from pathlib import Path + +from ..config import canonical_json_text +from .prompt import PromptRecord +from .sample import CorpusSample +from .tiny_dev_io import ( + TinyDevCorpusJsonError, + _array, + _mapping, + _prompt, + _reject_constant, + _sample, + _unique_object, +) +from .measurement_calibration import ( + MEASUREMENT_CALIBRATION_MANIFEST_VERSION, + MeasurementCalibrationCorpus, + MeasurementCalibrationManifest, +) + + +def _manifest(value: object) -> MeasurementCalibrationManifest: + data = _mapping( + "manifest", + value, + ( + "algorithm_version", + "corpus_id", + "language", + "prompts", + "samples", + "prompt_manifest_hash", + "sample_manifest_hash", + "manifest_hash", + ), + ) + return MeasurementCalibrationManifest( + algorithm_version=data["algorithm_version"], + corpus_id=data["corpus_id"], + language=data["language"], + prompts=tuple(_prompt(item) for item in _array("manifest.prompts", data["prompts"])), + samples=tuple(_sample(item) for item in _array("manifest.samples", data["samples"])), + prompt_manifest_hash=data["prompt_manifest_hash"], + sample_manifest_hash=data["sample_manifest_hash"], + manifest_hash=data["manifest_hash"], + ) + + +def parse_measurement_calibration_corpus_json( + text: str, + *, + require_canonical: bool = True, +) -> MeasurementCalibrationCorpus: + if not isinstance(text, str): + raise TypeError("text must be a string") + try: + decoded = json.loads( + text, + object_pairs_hook=_unique_object, + parse_constant=_reject_constant, + ) + except (json.JSONDecodeError, UnicodeError) as error: + raise TinyDevCorpusJsonError("measurement calibration JSON is not valid JSON") from error + data = _mapping( + "artifact", + decoded, + ( + "algorithm_version", + "manifest", + "target_length", + "topic_count_per_domain", + "seeds_per_prompt", + "sample_count", + "negative_count", + "audit_count", + "model_identity_hash", + "generation_matching_signature_hash", + "watermark_condition_hash", + "artifact_hash", + ), + ) + artifact = MeasurementCalibrationCorpus( + algorithm_version=data["algorithm_version"], + manifest=_manifest(data["manifest"]), + target_length=data["target_length"], + topic_count_per_domain=data["topic_count_per_domain"], + seeds_per_prompt=data["seeds_per_prompt"], + sample_count=data["sample_count"], + negative_count=data["negative_count"], + audit_count=data["audit_count"], + model_identity_hash=data["model_identity_hash"], + generation_matching_signature_hash=data["generation_matching_signature_hash"], + watermark_condition_hash=data["watermark_condition_hash"], + artifact_hash=data["artifact_hash"], + ) + if require_canonical: + canonical = canonical_json_text(artifact) + if text not in (canonical, canonical + "\n"): + raise TinyDevCorpusJsonError("measurement calibration JSON is not canonical") + return artifact + + +def load_measurement_calibration_corpus_json( + path: str | Path, + *, + require_canonical: bool = True, +) -> MeasurementCalibrationCorpus: + file_path = Path(path) + return parse_measurement_calibration_corpus_json( + file_path.read_text(encoding="utf-8"), + require_canonical=require_canonical, + ) diff --git a/fuckmark/corpus/tiny_dev_io_v3.py b/fuckmark/corpus/tiny_dev_io_v3.py new file mode 100644 index 00000000..59c0306c --- /dev/null +++ b/fuckmark/corpus/tiny_dev_io_v3.py @@ -0,0 +1,132 @@ +from __future__ import annotations + +import json +from pathlib import Path + +from ..config import canonical_json_text +from .schema import CorpusDomain, CorpusSplit +from .tiny_dev_io import ( + TINY_DEV_JSON_MAX_BYTES, + TinyDevCorpusJsonError, + _array, + _manifest, + _mapping, + _reject_constant, + _unique_object, +) +from .tiny_dev import TinyDevCorpusError +from .tiny_dev_v3 import ( + TINY_DEV_V3_CORPUS_ALGORITHM_VERSION, + TINY_DEV_V3_MAX_ATTACK_PAIRS_PER_DOMAIN, + TinyDevV3CorpusArtifact, + TinyDevV3CorpusCell, +) + + +def _cell(value: object) -> TinyDevV3CorpusCell: + data = _mapping("cell", value, ("split", "domain", "pair_count")) + return TinyDevV3CorpusCell( + split=CorpusSplit(data["split"]), + domain=CorpusDomain(data["domain"]), + pair_count=data["pair_count"], + ) + + +def _artifact_v3(value: object) -> TinyDevV3CorpusArtifact: + data = _mapping( + "artifact", + value, + ( + "algorithm_version", + "manifest", + "target_length", + "calibration_pairs_per_domain", + "attack_pairs_per_domain", + "required_splits", + "required_domains", + "model_identity_hash", + "generation_matching_signature_hash", + "watermark_condition_hash", + "cells", + "artifact_hash", + ), + ) + return TinyDevV3CorpusArtifact( + algorithm_version=data["algorithm_version"], + manifest=_manifest(data["manifest"]), + target_length=data["target_length"], + calibration_pairs_per_domain=data["calibration_pairs_per_domain"], + attack_pairs_per_domain=data["attack_pairs_per_domain"], + required_splits=tuple( + CorpusSplit(item) for item in _array("artifact.required_splits", data["required_splits"]) + ), + required_domains=tuple( + CorpusDomain(item) for item in _array("artifact.required_domains", data["required_domains"]) + ), + model_identity_hash=data["model_identity_hash"], + generation_matching_signature_hash=data["generation_matching_signature_hash"], + watermark_condition_hash=data["watermark_condition_hash"], + cells=tuple(_cell(item) for item in _array("artifact.cells", data["cells"])), + artifact_hash=data["artifact_hash"], + ) + + +def parse_tiny_dev_v3_corpus_json(text: str, *, require_canonical: bool = True) -> TinyDevV3CorpusArtifact: + if not isinstance(text, str): + raise TypeError("text must be a string") + if len(text.encode("utf-8")) > TINY_DEV_JSON_MAX_BYTES: + raise TinyDevCorpusJsonError("tiny-dev v3 JSON exceeds the size limit") + try: + decoded = json.loads( + text, + object_pairs_hook=_unique_object, + parse_constant=_reject_constant, + ) + except TinyDevCorpusJsonError: + raise + except (json.JSONDecodeError, UnicodeError) as error: + raise TinyDevCorpusJsonError("tiny-dev v3 JSON is not valid JSON") from error + try: + artifact = _artifact_v3(decoded) + except (KeyError, TypeError, ValueError) as error: + if isinstance(error, TinyDevCorpusJsonError): + raise + raise TinyDevCorpusJsonError("tiny-dev v3 JSON failed artifact validation") from error + if require_canonical: + canonical = canonical_json_text(artifact) + if text not in (canonical, canonical + "\n"): + raise TinyDevCorpusJsonError("tiny-dev v3 JSON is not in canonical serialized form") + return artifact + + +def load_tiny_dev_v3_corpus_json(path: str | Path, *, require_canonical: bool = True) -> TinyDevV3CorpusArtifact: + if not isinstance(path, (str, Path)): + raise TypeError("path must be a string or Path") + file_path = Path(path) + size = file_path.stat().st_size + if size > TINY_DEV_JSON_MAX_BYTES: + raise TinyDevCorpusJsonError("tiny-dev v3 JSON exceeds the size limit") + try: + text = file_path.read_text(encoding="utf-8") + except UnicodeDecodeError as error: + raise TinyDevCorpusJsonError("tiny-dev v3 JSON must be UTF-8") from error + return parse_tiny_dev_v3_corpus_json(text, require_canonical=require_canonical) + + +def load_tiny_dev_corpus_by_version_json(path: str | Path): + if not isinstance(path, (str, Path)): + raise TypeError("path must be a string or Path") + file_path = Path(path) + text = file_path.read_text(encoding="utf-8") + try: + decoded = json.loads(text) + except json.JSONDecodeError as error: + raise TinyDevCorpusJsonError("tiny-dev corpus JSON is not valid JSON") from error + if not isinstance(decoded, dict): + raise TinyDevCorpusJsonError("tiny-dev corpus JSON must be an object") + version = decoded.get("algorithm_version") + if version == TINY_DEV_V3_CORPUS_ALGORITHM_VERSION: + return parse_tiny_dev_v3_corpus_json(text) + from .tiny_dev_io import parse_tiny_dev_corpus_json + + return parse_tiny_dev_corpus_json(text) diff --git a/fuckmark/corpus/tiny_dev_v3.py b/fuckmark/corpus/tiny_dev_v3.py new file mode 100644 index 00000000..dca17b38 --- /dev/null +++ b/fuckmark/corpus/tiny_dev_v3.py @@ -0,0 +1,265 @@ +from __future__ import annotations + +from collections import Counter +from collections.abc import Sequence +from dataclasses import dataclass + +from .._validation import require_clean_string, require_int, require_sha256 +from ..hashing import sha256_json +from .manifest import CorpusManifest, build_corpus_manifest +from .prompt import PromptRecord +from .sample import CorpusSample +from .schema import CorpusDomain, CorpusSplit, KeySplit, TARGET_LENGTHS +from .tiny_dev import ( + TINY_DEV_CALIBRATION_PAIRS_PER_DOMAIN, + TINY_DEV_SPLITS, + TINY_DEV_TARGET_LENGTH, + TINY_DEV_DOMAINS, + TinyDevCorpusError, +) + + +TINY_DEV_V3_CORPUS_ALGORITHM_VERSION = "tiny-dev-corpus-v3" +TINY_DEV_V3_MAX_ATTACK_PAIRS_PER_DOMAIN = 16 + + +def _validate_attack_pairs_per_domain(attack_pairs_per_domain: int) -> None: + require_int("attack_pairs_per_domain", attack_pairs_per_domain) + if attack_pairs_per_domain < 1 or attack_pairs_per_domain > TINY_DEV_V3_MAX_ATTACK_PAIRS_PER_DOMAIN: + raise TinyDevCorpusError( + "v3 attack pairs per domain must lie between 1 and " + f"{TINY_DEV_V3_MAX_ATTACK_PAIRS_PER_DOMAIN}" + ) + + +@dataclass(frozen=True, slots=True) +class TinyDevV3CorpusCell: + split: CorpusSplit + domain: CorpusDomain + pair_count: int + + def __post_init__(self) -> None: + if not isinstance(self.split, CorpusSplit): + raise TypeError("split must be a CorpusSplit") + if not isinstance(self.domain, CorpusDomain): + raise TypeError("domain must be a CorpusDomain") + require_int("pair_count", self.pair_count) + if self.pair_count < 1: + raise ValueError("tiny development v3 corpus cell pair count must be positive") + + +@dataclass(frozen=True, slots=True) +class TinyDevV3CorpusArtifact: + algorithm_version: str + manifest: CorpusManifest + target_length: int + calibration_pairs_per_domain: int + attack_pairs_per_domain: int + required_splits: tuple[CorpusSplit, ...] + required_domains: tuple[CorpusDomain, ...] + model_identity_hash: str + generation_matching_signature_hash: str + watermark_condition_hash: str + cells: tuple[TinyDevV3CorpusCell, ...] + artifact_hash: str + + def __post_init__(self) -> None: + require_clean_string("algorithm_version", self.algorithm_version) + if self.algorithm_version != TINY_DEV_V3_CORPUS_ALGORITHM_VERSION: + raise ValueError("unsupported tiny development v3 corpus algorithm version") + if not isinstance(self.manifest, CorpusManifest): + raise TypeError("manifest must be a CorpusManifest") + require_int("target_length", self.target_length) + if self.target_length != TINY_DEV_TARGET_LENGTH or self.target_length not in TARGET_LENGTHS: + raise ValueError("tiny development v3 corpus target length must be 64") + require_int("calibration_pairs_per_domain", self.calibration_pairs_per_domain) + if self.calibration_pairs_per_domain != TINY_DEV_CALIBRATION_PAIRS_PER_DOMAIN: + raise ValueError("v3 calibration pair count must match the frozen calibration profile") + _validate_attack_pairs_per_domain(self.attack_pairs_per_domain) + if self.required_splits != TINY_DEV_SPLITS: + raise ValueError("tiny development v3 corpus split profile does not match the frozen profile") + if self.required_domains != TINY_DEV_DOMAINS: + raise ValueError("tiny development v3 corpus domain profile does not match the frozen profile") + require_sha256("model_identity_hash", self.model_identity_hash) + require_sha256("generation_matching_signature_hash", self.generation_matching_signature_hash) + require_sha256("watermark_condition_hash", self.watermark_condition_hash) + if not isinstance(self.cells, tuple): + raise TypeError("cells must be a tuple") + if any(not isinstance(value, TinyDevV3CorpusCell) for value in self.cells): + raise TypeError("cells must contain TinyDevV3CorpusCell values") + expected_cells = _expected_v3_cells(self.attack_pairs_per_domain) + if self.cells != expected_cells: + raise ValueError("tiny development v3 corpus cells do not match the parameterized matrix") + _validate_v3_manifest( + self.manifest, + self.model_identity_hash, + self.generation_matching_signature_hash, + self.watermark_condition_hash, + self.attack_pairs_per_domain, + ) + require_sha256("artifact_hash", self.artifact_hash) + if self.artifact_hash != sha256_json(self._payload()): + raise ValueError("artifact_hash does not match tiny development v3 corpus artifact") + + def _payload(self) -> dict[str, object]: + return { + "algorithm_version": self.algorithm_version, + "manifest_hash": self.manifest.manifest_hash, + "target_length": self.target_length, + "calibration_pairs_per_domain": self.calibration_pairs_per_domain, + "attack_pairs_per_domain": self.attack_pairs_per_domain, + "required_splits": tuple(value.value for value in self.required_splits), + "required_domains": tuple(value.value for value in self.required_domains), + "model_identity_hash": self.model_identity_hash, + "generation_matching_signature_hash": self.generation_matching_signature_hash, + "watermark_condition_hash": self.watermark_condition_hash, + "cells": self.cells, + } + + +def _expected_v3_cells(attack_pairs_per_domain: int) -> tuple[TinyDevV3CorpusCell, ...]: + return tuple( + TinyDevV3CorpusCell(split, domain, _v3_pairs_for_split(split, attack_pairs_per_domain)) + for split in TINY_DEV_SPLITS + for domain in TINY_DEV_DOMAINS + ) + + +def _v3_pairs_for_split(split: CorpusSplit, attack_pairs_per_domain: int) -> int: + if split is CorpusSplit.THRESHOLD_CALIBRATION: + return TINY_DEV_CALIBRATION_PAIRS_PER_DOMAIN + if split is CorpusSplit.ATTACK_DEVELOPMENT: + return attack_pairs_per_domain + raise TinyDevCorpusError("split is outside the frozen tiny development profile") + + +def _validate_v3_manifest( + manifest: CorpusManifest, + model_identity_hash: str, + generation_matching_signature_hash: str, + watermark_condition_hash: str, + attack_pairs_per_domain: int, +) -> None: + prompts = manifest.prompts + samples = manifest.samples + expected_pair_count = sum( + _v3_pairs_for_split(split, attack_pairs_per_domain) * len(TINY_DEV_DOMAINS) + for split in TINY_DEV_SPLITS + ) + expected_sample_count = expected_pair_count * 2 + if len(samples) != expected_sample_count: + raise TinyDevCorpusError( + f"tiny development v3 corpus must contain exactly {expected_sample_count} samples" + ) + if len({sample.match_id for sample in samples}) != expected_pair_count: + raise TinyDevCorpusError( + f"tiny development v3 corpus must contain exactly {expected_pair_count} matched pairs" + ) + if len(prompts) != expected_pair_count: + raise TinyDevCorpusError("tiny development v3 corpus must contain exactly one prompt per matched pair") + allowed_splits = set(TINY_DEV_SPLITS) + allowed_domains = set(TINY_DEV_DOMAINS) + if any(prompt.split not in allowed_splits for prompt in prompts): + raise TinyDevCorpusError("tiny development v3 prompts may use only calibration or attack-development splits") + if any(prompt.domain not in allowed_domains for prompt in prompts): + raise TinyDevCorpusError("tiny development v3 prompts must use the frozen four-domain profile") + if any(sample.split not in allowed_splits for sample in samples): + raise TinyDevCorpusError("tiny development v3 samples may use only calibration or attack-development splits") + if any(sample.domain not in allowed_domains for sample in samples): + raise TinyDevCorpusError("tiny development v3 samples must use the frozen four-domain profile") + if any(sample.target_length != TINY_DEV_TARGET_LENGTH for sample in samples): + raise TinyDevCorpusError("tiny development v3 samples must target 64 continuation tokens") + if any(sample.watermark.key_split is not KeySplit.DEV for sample in samples): + raise TinyDevCorpusError("tiny development v3 samples must use DEV_KEYS only") + if {sample.model.identity_hash for sample in samples} != {model_identity_hash}: + raise TinyDevCorpusError("tiny development v3 corpus must use one frozen model/tokenizer identity") + if {sample.generation.matching_signature_hash for sample in samples} != { + generation_matching_signature_hash + }: + raise TinyDevCorpusError("tiny development v3 corpus must use one matched generation-parameter signature") + if {sample.watermark.condition_hash for sample in samples} != {watermark_condition_hash}: + raise TinyDevCorpusError("tiny development v3 corpus must use one frozen DEV watermark condition") + prompt_counts = Counter(sample.prompt_id for sample in samples) + if any(prompt_counts[prompt.prompt_id] != 2 for prompt in prompts): + raise TinyDevCorpusError("each tiny development v3 prompt must bind exactly one matched on/off pair") + cell_pairs: Counter[tuple[CorpusSplit, CorpusDomain]] = Counter() + seen_match_ids: set[str] = set() + for sample in samples: + if sample.match_id in seen_match_ids: + continue + seen_match_ids.add(sample.match_id) + cell_pairs[(sample.split, sample.domain)] += 1 + expected_cells = { + (split, domain): _v3_pairs_for_split(split, attack_pairs_per_domain) + for split in TINY_DEV_SPLITS + for domain in TINY_DEV_DOMAINS + } + if dict(cell_pairs) != expected_cells: + raise TinyDevCorpusError("tiny development v3 corpus split/domain pair counts do not match the profile") + calibration_negatives = sum( + 1 + for sample in samples + if sample.split is CorpusSplit.THRESHOLD_CALIBRATION + and sample.label.value == "unwatermarked" + ) + if calibration_negatives < 100: + raise TinyDevCorpusError("tiny development v3 corpus must contain at least 100 calibration negatives") + + +def build_tiny_dev_v3_corpus( + corpus_id: str, + prompts: Sequence[PromptRecord], + samples: Sequence[CorpusSample], + *, + attack_pairs_per_domain: int, +) -> TinyDevV3CorpusArtifact: + require_clean_string("corpus_id", corpus_id) + _validate_attack_pairs_per_domain(attack_pairs_per_domain) + manifest = build_corpus_manifest(corpus_id, prompts, samples) + model_hashes = {sample.model.identity_hash for sample in manifest.samples} + generation_hashes = {sample.generation.matching_signature_hash for sample in manifest.samples} + watermark_hashes = {sample.watermark.condition_hash for sample in manifest.samples} + if len(model_hashes) != 1: + raise TinyDevCorpusError("tiny development v3 corpus must use exactly one model/tokenizer identity") + if len(generation_hashes) != 1: + raise TinyDevCorpusError("tiny development v3 corpus must use exactly one generation matching signature") + if len(watermark_hashes) != 1: + raise TinyDevCorpusError("tiny development v3 corpus must use exactly one watermark condition") + model_identity_hash = next(iter(model_hashes)) + generation_matching_signature_hash = next(iter(generation_hashes)) + watermark_condition_hash = next(iter(watermark_hashes)) + _validate_v3_manifest( + manifest, + model_identity_hash, + generation_matching_signature_hash, + watermark_condition_hash, + attack_pairs_per_domain, + ) + cells = _expected_v3_cells(attack_pairs_per_domain) + payload = { + "algorithm_version": TINY_DEV_V3_CORPUS_ALGORITHM_VERSION, + "manifest_hash": manifest.manifest_hash, + "target_length": TINY_DEV_TARGET_LENGTH, + "calibration_pairs_per_domain": TINY_DEV_CALIBRATION_PAIRS_PER_DOMAIN, + "attack_pairs_per_domain": attack_pairs_per_domain, + "required_splits": tuple(value.value for value in TINY_DEV_SPLITS), + "required_domains": tuple(value.value for value in TINY_DEV_DOMAINS), + "model_identity_hash": model_identity_hash, + "generation_matching_signature_hash": generation_matching_signature_hash, + "watermark_condition_hash": watermark_condition_hash, + "cells": cells, + } + return TinyDevV3CorpusArtifact( + TINY_DEV_V3_CORPUS_ALGORITHM_VERSION, + manifest, + TINY_DEV_TARGET_LENGTH, + TINY_DEV_CALIBRATION_PAIRS_PER_DOMAIN, + attack_pairs_per_domain, + TINY_DEV_SPLITS, + TINY_DEV_DOMAINS, + model_identity_hash, + generation_matching_signature_hash, + watermark_condition_hash, + cells, + sha256_json(payload), + ) diff --git a/fuckmark/corpus/tiny_dev_v3_generation.py b/fuckmark/corpus/tiny_dev_v3_generation.py new file mode 100644 index 00000000..378837df --- /dev/null +++ b/fuckmark/corpus/tiny_dev_v3_generation.py @@ -0,0 +1,216 @@ +from __future__ import annotations + +from ..hashing import sha256_json, sha256_text +from .prompt import PromptRecord +from .schema import CorpusDomain, CorpusSplit, KeySplit, MAX_GENERATION_SEED, WatermarkLabel +from .tiny_dev import ( + TINY_DEV_CALIBRATION_PAIRS_PER_DOMAIN, + TINY_DEV_SPLITS, + TINY_DEV_TARGET_LENGTH, + TinyDevCorpusError, +) +from .tiny_dev_generation import ( + TINY_DEV_DEFAULT_MAX_ATTEMPTS, + TINY_DEV_PAIR_SEED_STRIDE, + TINY_DEV_PROMPT_LICENSE_ID, + TINY_DEV_PROMPT_PROVENANCE, + TINY_DEV_PROMPT_SOURCE_ID, + TINY_DEV_SEED_POLICY_ID, + TinyDevGenerationBackend, + TinyDevGenerationError, + _TINY_DEV_TEMPLATES, + _TINY_DEV_TOPICS, + _build_sample, +) +from .tiny_dev_v3 import ( + TINY_DEV_V3_MAX_ATTACK_PAIRS_PER_DOMAIN, + TinyDevV3CorpusArtifact, + build_tiny_dev_v3_corpus, +) + + +TINY_DEV_V3_GENERATION_ALGORITHM_VERSION = "tiny-dev-v3-generation-v1" +TINY_DEV_V3_PROMPT_SOURCE_ID = "fuckmark-tiny-dev-v3-prompts-v1" +TINY_DEV_V3_DOMAINS = tuple(CorpusDomain) + +_TINY_DEV_V3_ATTACK_TOPICS = ( + "research claims", + "effect size honesty", + "preregistration discipline", + "randomization integrity", + "baseline drift", + "reporting bias", + "measurement error", + "replication crisis", + "statistical power", + "selection effects", + "confound control", + "outcome switching", + "sample size planning", + "analysis robustness", + "evidence strength", + "uncertainty reporting", +) + + +def _v3_topics() -> tuple[str, ...]: + if _TINY_DEV_TOPICS[25] != _TINY_DEV_V3_ATTACK_TOPICS[0]: + raise TinyDevCorpusError("v3 attack topic pool must extend the v2 attack topic") + return (*_TINY_DEV_TOPICS[:25], *_TINY_DEV_V3_ATTACK_TOPICS) + + +def _v3_prompt_source_hash(attack_pairs_per_domain: int) -> str: + return sha256_json( + { + "algorithm_version": TINY_DEV_V3_GENERATION_ALGORITHM_VERSION, + "v2_source_id": TINY_DEV_PROMPT_SOURCE_ID, + "source_id": TINY_DEV_V3_PROMPT_SOURCE_ID, + "license_id": TINY_DEV_PROMPT_LICENSE_ID, + "provenance": TINY_DEV_PROMPT_PROVENANCE, + "attack_pairs_per_domain": attack_pairs_per_domain, + "calibration_topics": _TINY_DEV_TOPICS[:25], + "attack_topics": _TINY_DEV_V3_ATTACK_TOPICS, + "templates": tuple( + (domain.value, _TINY_DEV_TEMPLATES[domain]) + for domain in TINY_DEV_V3_DOMAINS + ), + } + ) + + +def build_tiny_dev_v3_prompt_records(attack_pairs_per_domain: int) -> tuple[PromptRecord, ...]: + if type(attack_pairs_per_domain) is not int or attack_pairs_per_domain < 1: + raise ValueError("attack_pairs_per_domain must be a positive integer") + if attack_pairs_per_domain > TINY_DEV_V3_MAX_ATTACK_PAIRS_PER_DOMAIN: + raise TinyDevCorpusError( + "v3 attack pairs per domain exceeds the frozen maximum of " + f"{TINY_DEV_V3_MAX_ATTACK_PAIRS_PER_DOMAIN}" + ) + topics = _v3_topics() + source_hash = _v3_prompt_source_hash(attack_pairs_per_domain) + records: list[PromptRecord] = [] + for split in TINY_DEV_SPLITS: + pair_count = ( + TINY_DEV_CALIBRATION_PAIRS_PER_DOMAIN + if split is CorpusSplit.THRESHOLD_CALIBRATION + else attack_pairs_per_domain + ) + topic_offset = ( + 0 + if split is CorpusSplit.THRESHOLD_CALIBRATION + else TINY_DEV_CALIBRATION_PAIRS_PER_DOMAIN + ) + for domain in TINY_DEV_V3_DOMAINS: + template = _TINY_DEV_TEMPLATES[domain] + for local_index in range(pair_count): + topic_index = topic_offset + local_index + topic = topics[topic_index] + prompt_id = f"tiny-dev-{split.value}-{domain.value}-{topic_index:02d}" + records.append( + PromptRecord.create( + prompt_id=prompt_id, + prompt_family_id=f"tiny-dev-family-{domain.value}-{topic_index:02d}", + domain=domain, + split=split, + source_id=TINY_DEV_V3_PROMPT_SOURCE_ID, + source_hash=source_hash, + license_id=TINY_DEV_PROMPT_LICENSE_ID, + provenance=TINY_DEV_PROMPT_PROVENANCE, + text=template.format(topic=topic), + ) + ) + return tuple(sorted(records, key=lambda value: value.prompt_id)) + + +def build_real_tiny_dev_v3_corpus( + backend: TinyDevGenerationBackend, + *, + corpus_id: str = "fuckmark-tiny-dev-real-v3", + seed_base: int = 402000, + attack_pairs_per_domain: int = 1, + max_attempts: int = TINY_DEV_DEFAULT_MAX_ATTEMPTS, +) -> TinyDevV3CorpusArtifact: + if type(seed_base) is not int or seed_base < 0: + raise ValueError("seed_base must be a non-negative integer") + if type(max_attempts) is not int or max_attempts <= 0: + raise ValueError("max_attempts must be a positive integer") + if type(attack_pairs_per_domain) is not int or attack_pairs_per_domain < 1: + raise ValueError("attack_pairs_per_domain must be a positive integer") + if attack_pairs_per_domain > TINY_DEV_V3_MAX_ATTACK_PAIRS_PER_DOMAIN: + raise ValueError("attack_pairs_per_domain exceeds the frozen v3 maximum") + prompts = build_tiny_dev_v3_prompt_records(attack_pairs_per_domain) + if seed_base + len(prompts) * TINY_DEV_PAIR_SEED_STRIDE + max_attempts > MAX_GENERATION_SEED: + raise ValueError("seed schedule exceeds the 64-bit generation-seed range") + + samples: list[object] = [] + seen_text_hashes: set[str] = set() + seen_token_hashes: set[str] = set() + + for pair_index, prompt in enumerate(prompts): + pair_seed_base = seed_base + pair_index * TINY_DEV_PAIR_SEED_STRIDE + accepted = False + for attempt in range(max_attempts): + seed = pair_seed_base + attempt + try: + control = backend.generate(prompt.text, seed, watermarked=False) + watermarked = backend.generate(prompt.text, seed, watermarked=True) + except RuntimeError: + continue + if not control.text.strip() or not watermarked.text.strip(): + continue + candidates = ( + (WatermarkLabel.UNWATERMARKED, control), + (WatermarkLabel.WATERMARKED, watermarked), + ) + if any( + len(value.continuation_token_ids) != TINY_DEV_TARGET_LENGTH + for _, value in candidates + ): + continue + text_hashes = tuple(sha256_text(value.text) for _, value in candidates) + token_hashes = tuple( + sha256_json(value.continuation_token_ids) + for _, value in candidates + ) + if len(set(text_hashes)) != len(text_hashes) or len(set(token_hashes)) != len(token_hashes): + continue + if any(value in seen_text_hashes for value in text_hashes): + continue + if any(value in seen_token_hashes for value in token_hashes): + continue + pair_samples = tuple( + _build_sample( + prompt=prompt, + label=label, + generated=value, + backend=backend, + seed=seed, + ) + for label, value in candidates + ) + samples.extend(pair_samples) + seen_text_hashes.update(text_hashes) + seen_token_hashes.update(token_hashes) + accepted = True + break + if not accepted: + raise TinyDevGenerationError( + f"could not produce an exact, unique 64-token pair for {prompt.prompt_id} " + f"within {max_attempts} deterministic attempts" + ) + + artifact = build_tiny_dev_v3_corpus( + corpus_id, + prompts, + samples, + attack_pairs_per_domain=attack_pairs_per_domain, + ) + if any(sample.watermark.key_split is not KeySplit.DEV for sample in artifact.manifest.samples): + raise TinyDevGenerationError("real tiny development v3 generation must use DEV_KEYS") + for match_id in sorted({sample.match_id for sample in artifact.manifest.samples}): + pair = tuple(sample for sample in artifact.manifest.samples if sample.match_id == match_id) + if len({sample.generation.seed for sample in pair}) != 1: + raise TinyDevGenerationError("real tiny development v3 runner requires one shared seed per on/off pair") + if any(sample.generation.seed_policy_id != TINY_DEV_SEED_POLICY_ID for sample in pair): + raise TinyDevGenerationError("real tiny development v3 pair seed policy drifted") + return artifact diff --git a/fuckmark/experiments/coverage_holes.py b/fuckmark/experiments/coverage_holes.py new file mode 100644 index 00000000..eae71e9c --- /dev/null +++ b/fuckmark/experiments/coverage_holes.py @@ -0,0 +1,328 @@ +from __future__ import annotations + +from collections import Counter +from typing import Any + +from ..hashing import sha256_json +from ..transforms.protected import ProtectedSpanExtractor + + +COVERAGE_HOLE_ANALYSIS_VERSION = "coverage-hole-structure-v1" +NGRAM_LEN = 5 + +_FUNCTION_WORDS = frozenset( + ( + "is", + "of", + "to", + "the", + "and", + "in", + "for", + "on", + "with", + "as", + "from", + "that", + "this", + "was", + "are", + "be", + "can", + "will", + "have", + "has", + "not", + "but", + "or", + "by", + "at", + "it", + "we", + "you", + "they", + "an", + "he", + "she", + "his", + "her", + "its", + "our", + "their", + "them", + "him", + "us", + "me", + "my", + "your", + "if", + "when", + "where", + "which", + "who", + "what", + "how", + "why", + "because", + "so", + "than", + "then", + "there", + "here", + "all", + "any", + "each", + "more", + "most", + "some", + "such", + "only", + "very", + "just", + "also", + "into", + "about", + "over", + "under", + "after", + "before", + "between", + "during", + "through", + "without", + "while", + "do", + "does", + "been", + "being", + "would", + "could", + "should", + "may", + "might", + "must", + "were", + "one", + "even", + "still", + "often", + ) +) + + +def classify_word(word: str) -> str: + stripped = word.strip(".,;:!?\"'()") + if not stripped: + return "punctuation_or_empty" + if stripped.isdigit(): + return "numeric" + lowered = stripped.lower() + if lowered in _FUNCTION_WORDS: + return "function_word" + if stripped[0].isupper(): + return "capitalized_content" + if stripped.isalpha(): + return "lowercase_content" + return "other" + + +def _rule_family(rule_id: str) -> str: + if rule_id.startswith("contract"): + return "contraction" + if rule_id.startswith("surface"): + return "surface_spacing" + if rule_id.startswith("lexical"): + return "lexical" + if rule_id.startswith("syntax"): + return "syntax" + return "other" + + +def build_coverage_hole_row( + text: str, + token_ids: tuple[int, ...], + offsets: tuple[tuple[int, int], ...], + enumeration, + geometry, + variant, +) -> dict[str, object]: + token_count = len(token_ids) + observation_count = max(0, token_count - NGRAM_LEN + 1) + coverage = geometry.coverage_mapping() + window_hits = Counter() + for candidate_id, intervals in coverage.items(): + for interval in intervals: + for window in range(interval.start, interval.end_exclusive): + window_hits[window] += 1 + uncovered = tuple(w for w in range(observation_count) if window_hits[w] == 0) + covered_windows = observation_count - len(uncovered) + longest_run = 0 + run = 0 + for window in range(observation_count): + if window_hits[window] == 0: + run += 1 + longest_run = max(longest_run, run) + else: + run = 0 + uncovered_spans = [] + cursor = None + for window in uncovered: + if cursor is None or window != cursor[1]: + if cursor is not None: + uncovered_spans.append(cursor) + cursor = [window, window + 1] + else: + cursor[1] = window + 1 + if cursor is not None: + uncovered_spans.append(cursor) + lexical_categories: Counter[str] = Counter() + uncovered_word_samples: list[str] = [] + protected = ProtectedSpanExtractor().extract(text) + protected_char_spans = tuple((span.start, span.end) for span in protected.spans) + protected_intersections = 0 + for start_window, end_window in uncovered_spans: + char_start = offsets[start_window][0] if start_window < len(offsets) else len(text) + char_end = offsets[min(end_window - 1, len(offsets) - 1)][1] if end_window - 1 < len(offsets) else len(text) + snippet = text[char_start:char_end] + for token in snippet.split(): + lexical_categories[classify_word(token)] += 1 + if len(uncovered_word_samples) < 5 and snippet.strip(): + uncovered_word_samples.append(snippet.strip()[:60]) + for span_start, span_end in protected_char_spans: + if span_start < char_end and char_start < span_end: + protected_intersections += 1 + break + selected_ids = set(variant["selected_candidate_ids"]) + realized_windows = sum( + 1 + for window in range(observation_count) + if any( + interval.start <= window < interval.end_exclusive + for candidate_id, intervals in coverage.items() + if candidate_id in selected_ids + for interval in intervals + ) + ) + candidate_by_family = Counter( + _rule_family(candidate.rule_id) for candidate in enumeration.candidates + ) + redundancy = ( + sum(window_hits.values()) / covered_windows if covered_windows else 0.0 + ) + return { + "source_sample_id": variant["source_sample_id"], + "source_label": variant["source_label"], + "domain": variant["domain"], + "token_count": token_count, + "observation_window_count": observation_count, + "candidate_count": len(enumeration.candidates), + "candidate_count_by_family": dict(candidate_by_family), + "candidate_density_per_100_tokens": round(100.0 * len(enumeration.candidates) / token_count, 4) if token_count else 0.0, + "zero_candidate_source": len(enumeration.candidates) == 0, + "achievable_coverage_fraction": covered_windows / observation_count if observation_count else 0.0, + "realized_coverage_fraction": realized_windows / observation_count if observation_count else 0.0, + "uncovered_window_count": len(uncovered), + "longest_uncovered_run": longest_run, + "uncovered_span_count": len(uncovered_spans), + "uncovered_lexical_categories": dict(lexical_categories), + "unprotected_word_samples": uncovered_word_samples, + "protected_span_intersections": protected_intersections, + "candidate_redundancy_mean_multiplicity": round(redundancy, 4), + } + + +def build_coverage_hole_report(corpus: Any, tokenizer: Any, plan: dict[str, object]) -> dict[str, object]: + from ..experiments.effectiveness_plan import _attack_samples, _encode_with_offsets + from ..transforms import build_candidate_tokenizer_geometry + + registry = _registry_for_plan(plan) + profile_budgets = plan["budgets"] + sources = {sample.sample_id: sample for sample in _attack_samples(corpus)} + rows = [] + for variant in plan["variants"]: + if variant["requested_budget"] != profile_budgets[0]: + continue + source = sources[variant["source_sample_id"]] + token_ids, offsets = _encode_with_offsets(tokenizer, source) + enumeration = registry.enumerate(source.text) + geometry = build_candidate_tokenizer_geometry( + source.text, + enumeration, + token_ids, + offsets, + tokenizer_identity_hash=source.model.identity_hash, + ngram_len=NGRAM_LEN, + ) + rows.append( + build_coverage_hole_row( + source.text, + token_ids, + offsets, + enumeration, + geometry, + variant, + ) + ) + row_tuple = tuple(rows) + watermarked = tuple(row for row in row_tuple if row["source_label"] == "watermarked") + aggregated: Counter[str] = Counter() + for row in watermarked: + for category, count in row["uncovered_lexical_categories"].items(): + aggregated[category] += count + payload = { + "algorithm_version": COVERAGE_HOLE_ANALYSIS_VERSION, + "analysis_scope": "structural text-geometry diagnosis; no detector information used", + "plan_hash": plan["plan_hash"], + "ruleset_hash": plan["ruleset_hash"], + "rows": row_tuple, + "watermarked_uncovered_lexical_totals": dict(aggregated), + } + return {**payload, "artifact_hash": sha256_json(payload)} + + +def _registry_for_plan(plan: dict[str, object]): + from .effectiveness_plan import _registry_for_profile + from ..transforms import EffectivenessTransformProfile, resolve_effectiveness_profile + + profile = resolve_effectiveness_profile(plan["profile_id"], tuple(plan["budgets"])) + return _registry_for_profile(profile) + + +def main(argv: list[str] | None = None) -> int: + import argparse + import json + import sys + from pathlib import Path + + parser = argparse.ArgumentParser(prog="fuckmark-coverage-holes") + parser.add_argument("--corpus-json", type=Path, required=True) + parser.add_argument("--plan-json", type=Path, required=True) + parser.add_argument("--json", type=Path, required=True) + args = parser.parse_args(argv) + from ..corpus import load_tiny_dev_corpus_by_version_json + + corpus = load_tiny_dev_corpus_by_version_json(args.corpus_json) + plan = json.loads(args.plan_json.read_text(encoding="utf-8")) + from transformers import AutoTokenizer + + tokenizer = AutoTokenizer.from_pretrained( + "openai-community/gpt2", + revision="607a30d783dfa663caf39e06633721c8d4cfcd7e", + use_fast=True, + padding_side="left", + ) + report = build_coverage_hole_report(corpus, tokenizer, plan) + args.json.parent.mkdir(parents=True, exist_ok=True) + args.json.write_text( + json.dumps(report, indent=2, sort_keys=True) + "\n", encoding="utf-8" + ) + sys.stdout.write(f"artifact_hash={report['artifact_hash']}\n") + sys.stdout.write(f"row_count={len(report['rows'])}\n") + sys.stdout.write( + f"uncovered_lexical_totals={report['watermarked_uncovered_lexical_totals']}\n" + ) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/fuckmark/experiments/effectiveness_plan.py b/fuckmark/experiments/effectiveness_plan.py index 7fc10183..bf42eb98 100644 --- a/fuckmark/experiments/effectiveness_plan.py +++ b/fuckmark/experiments/effectiveness_plan.py @@ -14,9 +14,17 @@ ScheduleGeometryMode, SchedulePolicy, build_candidate_tokenizer_geometry, + content_region_combined_transform_registry, + content_region_coverage_transform_registry, + content_region_general_only_transform_registry, + key_blind_coverage_completion_transform_registry, key_blind_high_coverage_transform_registry, validate_effectiveness_profile_registry, validate_hard_invariants, + CONTENT_REGION_COMBINED_PROFILE_ID, + CONTENT_REGION_COVERAGE_PROFILE_ID, + CONTENT_REGION_GENERAL_ONLY_PROFILE_ID, + KEY_BLIND_COVERAGE_COMPLETION_PROFILE_ID, ) @@ -151,6 +159,18 @@ def _schedule_seed( return seed +def _registry_for_profile(profile: EffectivenessTransformProfile): + if profile.profile_id == KEY_BLIND_COVERAGE_COMPLETION_PROFILE_ID: + return key_blind_coverage_completion_transform_registry() + if profile.profile_id == CONTENT_REGION_COVERAGE_PROFILE_ID: + return content_region_coverage_transform_registry() + if profile.profile_id == CONTENT_REGION_GENERAL_ONLY_PROFILE_ID: + return content_region_general_only_transform_registry() + if profile.profile_id == CONTENT_REGION_COMBINED_PROFILE_ID: + return content_region_combined_transform_registry() + return key_blind_high_coverage_transform_registry() + + def build_key_blind_high_coverage_plan( corpus: Any, tokenizer: Any, @@ -159,7 +179,7 @@ def build_key_blind_high_coverage_plan( source_code_commit: str, ) -> dict[str, object]: _require_git_object_id("source_code_commit", source_code_commit) - registry = key_blind_high_coverage_transform_registry() + registry = _registry_for_profile(profile) validate_effectiveness_profile_registry(profile, registry) policy = SchedulePolicy(profile.schedule_policy_id) if policy is not SchedulePolicy.COVERAGE_GREEDY_KEY_BLIND: diff --git a/fuckmark/experiments/measurement_threshold.py b/fuckmark/experiments/measurement_threshold.py new file mode 100644 index 00000000..8231a49a --- /dev/null +++ b/fuckmark/experiments/measurement_threshold.py @@ -0,0 +1,97 @@ +from __future__ import annotations + +import math + +from ..detectors import CalibrationScope, ComparisonOperator, calibrate_detector +from ..detectors.calibration_statistics import exact_binomial_interval +from ..hashing import sha256_json +from ..tiny_dev_transform_hf import ( + TEXT_ONLY_CALIBRATION_POPULATION_ID, + TEXT_ONLY_LENGTH_POLICY_ID, + TEXT_ONLY_PROMPT_BOUNDARY_MODE, + TEXT_ONLY_TOKEN_TRACK, + _text_only_weighted_evidence, +) + + +MEASUREMENT_STABILITY_VERSION = "open-detector-measurement-stability-v1" +MEASUREMENT_STABILITY_TARGET_FPR = 0.01 +MEASUREMENT_STABILITY_COMPARISON = ">=" + + +def build_fixed_threshold_artifact( + calibration_corpus, + adapter, + *, + frozen_at_utc: str, +) -> dict[str, object]: + negatives = calibration_corpus.calibration_samples() + audit = calibration_corpus.audit_samples() + evidence = tuple(_text_only_weighted_evidence(sample, adapter) for sample in negatives) + audit_evidence = tuple(_text_only_weighted_evidence(sample, adapter) for sample in audit) + scope = CalibrationScope.create( + corpus_id=calibration_corpus.manifest.corpus_id, + population_id=TEXT_ONLY_CALIBRATION_POPULATION_ID, + length_policy_id=TEXT_ONLY_LENGTH_POLICY_ID, + token_track=TEXT_ONLY_TOKEN_TRACK, + prompt_boundary_mode=TEXT_ONLY_PROMPT_BOUNDARY_MODE, + ) + bundle = calibrate_detector( + evidence, + scope, + target_fprs=(0.05, 0.01), + comparison_operator=ComparisonOperator.GREATER_THAN_OR_EQUAL, + confidence_level=0.95, + ) + scores = sorted(value.raw_score for value in evidence) + count = len(scores) + order_statistic = count - math.floor(MEASUREMENT_STABILITY_TARGET_FPR * count) + 1 + threshold = scores[order_statistic - 1] + calibration_exceedances = sum(1 for value in scores if value >= threshold) + audit_scores = tuple(value.raw_score for value in audit_evidence) + audit_exceedances = sum(1 for value in audit_scores if value >= threshold) + interval = exact_binomial_interval(audit_exceedances, len(audit_scores)) + payload = { + "algorithm_version": MEASUREMENT_STABILITY_VERSION, + "frozen_at_utc": frozen_at_utc, + "detector_identity_hash": bundle.detector_identity.identity_hash, + "calibration_bundle_hash": bundle.bundle_hash, + "calibration_corpus_artifact_hash": calibration_corpus.artifact_hash, + "calibration_negative_count": count, + "audit_negative_count": len(audit_scores), + "target_fpr": MEASUREMENT_STABILITY_TARGET_FPR, + "comparison_operator": MEASUREMENT_STABILITY_COMPARISON, + "threshold_definition": ( + "ascending order statistic n - floor(target_fpr * n) + 1 of the calibration " + "negative weighted-mean scores (the 1015th of 1024 for target FPR 0.01, bounding " + "calibration exceedances at floor(target_fpr * n)); detection when score >= " + "threshold; float64 scores as computed; no tie rounding" + ), + "threshold_order_statistic": order_statistic, + "threshold": threshold, + "calibration_exceedances": calibration_exceedances, + "audit_exceedances": audit_exceedances, + "audit_realized_fpr": audit_exceedances / len(audit_scores), + "audit_fpr_confidence_interval_95": [interval.lower, interval.upper], + } + return {**payload, "artifact_hash": sha256_json(payload)} + + +def validate_fixed_threshold_artifact(artifact, detector_identity_hash: str) -> float: + if not isinstance(artifact, dict): + raise TypeError("fixed threshold artifact must be a mapping") + if artifact.get("algorithm_version") != MEASUREMENT_STABILITY_VERSION: + raise ValueError("unsupported fixed threshold artifact version") + if artifact.get("detector_identity_hash") != detector_identity_hash: + raise ValueError("fixed threshold detector identity does not match the scoring detector") + if artifact.get("comparison_operator") != MEASUREMENT_STABILITY_COMPARISON: + raise ValueError("fixed threshold comparison operator does not match the scoring rule") + if artifact.get("target_fpr") != MEASUREMENT_STABILITY_TARGET_FPR: + raise ValueError("fixed threshold target FPR does not match the scoring target") + expected = sha256_json({key: value for key, value in artifact.items() if key != "artifact_hash"}) + if artifact.get("artifact_hash") != expected: + raise ValueError("fixed threshold artifact hash does not replay") + threshold = artifact.get("threshold") + if not isinstance(threshold, float) or not 0.0 <= threshold <= 1.0: + raise ValueError("fixed threshold must be a probability float") + return threshold diff --git a/fuckmark/tiny_dev_budget_scaled_robustness.py b/fuckmark/tiny_dev_budget_scaled_robustness.py new file mode 100644 index 00000000..1e8f19cc --- /dev/null +++ b/fuckmark/tiny_dev_budget_scaled_robustness.py @@ -0,0 +1,208 @@ +from __future__ import annotations + +import argparse +import json +import sys +import unicodedata +from pathlib import Path + +from .corpus import load_tiny_dev_corpus_by_version_json +from .experiments.mid_dev_quality import protected_span_violation_count, word_edit_rate +from .hashing import sha256_json, sha256_text +from .transforms import validate_hard_invariants, InvariantStatus + + +BUDGET_SCALED_ROBUSTNESS_VERSION = "budget-scaled-robustness-v1" +FORBIDDEN_CATEGORIES = ("Cc", "Cf", "Mn", "Mc", "Me", "Zl", "Zp") +WORD_EDIT_RATE_MAX = 0.30 + + +def _forbidden_codepoints(text: str) -> tuple[str, ...]: + return tuple( + f"U+{ord(character):04X}({unicodedata.category(character)})" + for character in sorted(set(text)) + if unicodedata.category(character) in FORBIDDEN_CATEGORIES + or (unicodedata.category(character) == "Zs" and character != " ") + ) + + +def _introduced_forbidden_codepoints(source_text: str, transformed_text: str) -> tuple[str, ...]: + source_alphabet = set(source_text) + return tuple( + f"U+{ord(character):04X}({unicodedata.category(character)})" + for character in sorted(set(transformed_text) - source_alphabet) + if unicodedata.category(character) in FORBIDDEN_CATEGORIES + or (unicodedata.category(character) == "Zs" and character != " ") + ) + + +def _introduced_non_ascii(source_text: str, transformed_text: str) -> tuple[str, ...]: + source_alphabet = set(source_text) + return tuple( + f"U+{ord(character):04X}" + for character in sorted(set(transformed_text) - source_alphabet) + if not character.isascii() + ) + + +def _cf_stripped(text: str) -> str: + return "".join( + character + for character in text + if unicodedata.category(character) != "Cf" + ) + + +def _robustness_row(source_text: str, transformed_text: str) -> dict[str, object]: + normalization_noops = { + "nfc": unicodedata.normalize("NFC", transformed_text) == transformed_text, + "nfd": unicodedata.normalize("NFD", transformed_text) == transformed_text, + "nfkc": unicodedata.normalize("NFKC", transformed_text) == transformed_text, + "nfkd": unicodedata.normalize("NFKD", transformed_text) == transformed_text, + } + source_normalization_noops = { + "nfc": unicodedata.normalize("NFC", source_text) == source_text, + "nfd": unicodedata.normalize("NFD", source_text) == source_text, + "nfkc": unicodedata.normalize("NFKC", source_text) == source_text, + "nfkd": unicodedata.normalize("NFKD", source_text) == source_text, + } + introduced_instability = tuple( + form + for form in normalization_noops + if source_normalization_noops[form] and not normalization_noops[form] + ) + invariant_status = validate_hard_invariants(source_text, transformed_text).status + row = { + "source_text_hash": sha256_text(source_text), + "transformed_text_hash": sha256_text(transformed_text), + "transformed_is_ascii": transformed_text.isascii(), + "preexisting_forbidden_codepoints": _forbidden_codepoints(transformed_text), + "introduced_forbidden_codepoints": _introduced_forbidden_codepoints(source_text, transformed_text), + "introduced_non_ascii_codepoints": _introduced_non_ascii(source_text, transformed_text), + "normalization_noops": normalization_noops, + "source_normalization_noops": source_normalization_noops, + "introduced_normalization_instability": introduced_instability, + "cf_strip_noop": _cf_stripped(transformed_text) == transformed_text, + "word_edit_rate": word_edit_rate(source_text, transformed_text), + "protected_span_violation_count": protected_span_violation_count(source_text, transformed_text), + "hard_invariant_status": invariant_status.value, + } + gates_pass = ( + not row["introduced_forbidden_codepoints"] + and not row["introduced_non_ascii_codepoints"] + and not row["introduced_normalization_instability"] + and row["cf_strip_noop"] + and row["word_edit_rate"] <= WORD_EDIT_RATE_MAX + and row["protected_span_violation_count"] == 0 + and invariant_status is InvariantStatus.PASS + ) + return {**row, "gates_pass": gates_pass} + + +def build_budget_scaled_robustness_report( + corpora: tuple[object, ...], + plans: tuple[dict[str, object], ...], +) -> dict[str, object]: + corpus_by_artifact: dict[str, object] = {} + for corpus in corpora: + corpus_by_artifact[corpus.artifact_hash] = corpus + corpus_text_hashes: list[set[str]] = [ + {sample.text_sha256 for sample in corpus.manifest.samples} for corpus in corpora + ] + pairwise_overlaps = {} + for left in range(len(corpora)): + for right in range(left + 1, len(corpora)): + pairwise_overlaps[f"{left}-{right}"] = len( + corpus_text_hashes[left] & corpus_text_hashes[right] + ) + rows: list[dict[str, object]] = [] + for plan in plans: + corpus = corpus_by_artifact.get(plan["tiny_dev_artifact_hash"]) + if corpus is None: + raise ValueError("robustness report received a plan whose corpus was not supplied") + sources = {sample.sample_id: sample for sample in corpus.manifest.samples} + for variant in plan["variants"]: + sample = sources[variant["source_sample_id"]] + rows.append( + { + "plan_hash": plan["plan_hash"], + "corpus_artifact_hash": corpus.artifact_hash, + "source_sample_id": variant["source_sample_id"], + "requested_budget": variant["requested_budget"], + **_robustness_row(sample.text, variant["transformed_text"]), + } + ) + row_tuple = tuple(rows) + summary = { + "row_count": len(row_tuple), + "gate_pass_fraction": sum(1 for row in row_tuple if row["gates_pass"]) / len(row_tuple) if row_tuple else 1.0, + "all_introduced_codepoints_clean": all( + not row["introduced_forbidden_codepoints"] and not row["introduced_non_ascii_codepoints"] + for row in row_tuple + ), + "all_normalization_stability_preserved": all( + not row["introduced_normalization_instability"] for row in row_tuple + ), + "rows_with_preexisting_normalization_instability": sum( + 1 + for row in row_tuple + if not all(row["normalization_noops"].values()) + ), + "all_cf_strip_noop": all(row["cf_strip_noop"] for row in row_tuple), + "max_word_edit_rate": max((row["word_edit_rate"] for row in row_tuple), default=0.0), + "total_protected_span_violations": sum( + row["protected_span_violation_count"] for row in row_tuple + ), + "all_hard_invariants_pass": all( + row["hard_invariant_status"] == InvariantStatus.PASS.value for row in row_tuple + ), + "corpus_pairwise_text_hash_overlaps": pairwise_overlaps, + } + payload = { + "algorithm_version": BUDGET_SCALED_ROBUSTNESS_VERSION, + "word_edit_rate_max": WORD_EDIT_RATE_MAX, + "forbidden_categories": FORBIDDEN_CATEGORIES, + "corpus_artifact_hashes": tuple(corpus.artifact_hash for corpus in corpora), + "plan_hashes": tuple(plan["plan_hash"] for plan in plans), + "rows": row_tuple, + "summary": summary, + } + return {**payload, "artifact_hash": sha256_json(payload)} + + +def _load_plan(path: Path) -> dict[str, object]: + value = json.loads(path.read_text(encoding="utf-8")) + if not isinstance(value, dict): + raise ValueError("expected a JSON object") + return value + + +def _parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser(prog="fuckmark-tiny-dev-budget-scaled-robustness") + parser.add_argument("--corpus-json", type=Path, action="append", required=True) + parser.add_argument("--plan-json", type=Path, action="append", required=True) + parser.add_argument("--json", type=Path, required=True) + return parser + + +def main(argv: list[str] | None = None) -> int: + args = _parser().parse_args(argv) + corpora = tuple( + load_tiny_dev_corpus_by_version_json(path) for path in args.corpus_json + ) + plans = tuple(_load_plan(path) for path in args.plan_json) + report = build_budget_scaled_robustness_report(corpora, plans) + args.json.parent.mkdir(parents=True, exist_ok=True) + args.json.write_text( + json.dumps(report, indent=2, sort_keys=True, ensure_ascii=False) + "\n", + encoding="utf-8", + ) + sys.stdout.write(f"artifact_hash={report['artifact_hash']}\n") + sys.stdout.write(f"row_count={report['summary']['row_count']}\n") + sys.stdout.write(f"gate_pass_fraction={report['summary']['gate_pass_fraction']}\n") + sys.stdout.write(f"json={args.json.as_posix()}\n") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/fuckmark/tiny_dev_corpus_hf.py b/fuckmark/tiny_dev_corpus_hf.py index f37b5f82..8a1c001e 100644 --- a/fuckmark/tiny_dev_corpus_hf.py +++ b/fuckmark/tiny_dev_corpus_hf.py @@ -11,6 +11,9 @@ ModelTokenizerIdentity, PaddingSide, WatermarkCondition, + build_real_tiny_dev_v3_corpus, + load_tiny_dev_corpus_json, + load_tiny_dev_v3_corpus_json, ) from .corpus.tiny_dev import TINY_DEV_TARGET_LENGTH from .corpus.tiny_dev_generation import ( @@ -215,12 +218,26 @@ def _parser() -> argparse.ArgumentParser: parser.add_argument("--temperature", type=float, default=0.8) parser.add_argument("--top-k", type=int, default=50) parser.add_argument("--top-p", type=float, default=0.95) + parser.add_argument( + "--profile", + choices=("v2", "v3"), + default="v2", + ) + parser.add_argument( + "--attack-pairs-per-domain", + type=int, + default=1, + ) parser.add_argument("--json", type=Path, default=Path("artifacts/tiny-dev-corpus.json")) return parser def main(argv: list[str] | None = None) -> int: args = _parser().parse_args(argv) + if args.profile == "v3" and args.attack_pairs_per_domain < 1: + raise ValueError("the v3 profile requires at least one attack pair per domain") + if args.profile == "v2" and args.attack_pairs_per_domain != 1: + raise ValueError("the v2 profile is frozen at one attack pair per domain") backend = HuggingFaceTinyDevBackend( args.model, args.model_revision, @@ -229,13 +246,28 @@ def main(argv: list[str] | None = None) -> int: top_k=args.top_k, top_p=args.top_p, ) - artifact = build_real_tiny_dev_corpus( - backend, - seed_base=args.seed_base, - max_attempts=args.max_attempts, - ) + if args.profile == "v3": + artifact = build_real_tiny_dev_v3_corpus( + backend, + seed_base=args.seed_base, + attack_pairs_per_domain=args.attack_pairs_per_domain, + max_attempts=args.max_attempts, + ) + else: + artifact = build_real_tiny_dev_corpus( + backend, + seed_base=args.seed_base, + max_attempts=args.max_attempts, + ) args.json.parent.mkdir(parents=True, exist_ok=True) args.json.write_text(canonical_json_text(artifact) + "\n", encoding="utf-8") + loaded = ( + load_tiny_dev_v3_corpus_json(args.json) + if args.profile == "v3" + else load_tiny_dev_corpus_json(args.json) + ) + if loaded.artifact_hash != artifact.artifact_hash: + raise RuntimeError("canonical corpus JSON did not replay the generated artifact hash") sys.stdout.write(f"artifact_hash={artifact.artifact_hash}\n") sys.stdout.write(f"manifest_hash={artifact.manifest.manifest_hash}\n") sys.stdout.write(f"prompt_count={len(artifact.manifest.prompts)}\n") @@ -245,6 +277,9 @@ def main(argv: list[str] | None = None) -> int: f"generation_signature_hash={artifact.generation_matching_signature_hash}\n" ) sys.stdout.write(f"watermark_condition_hash={artifact.watermark_condition_hash}\n") + sys.stdout.write(f"profile={args.profile}\n") + if args.profile == "v3": + sys.stdout.write(f"attack_pairs_per_domain={args.attack_pairs_per_domain}\n") sys.stdout.write(f"json={args.json.as_posix()}\n") return 0 diff --git a/fuckmark/tiny_dev_effectiveness_plan_hf.py b/fuckmark/tiny_dev_effectiveness_plan_hf.py index ca3cbd56..e524c806 100644 --- a/fuckmark/tiny_dev_effectiveness_plan_hf.py +++ b/fuckmark/tiny_dev_effectiveness_plan_hf.py @@ -6,7 +6,7 @@ from datetime import datetime, timezone from pathlib import Path -from .corpus import load_tiny_dev_corpus_json +from .corpus import load_tiny_dev_corpus_by_version_json from .durable_io import write_canonical_json_fsynced from .experiments.effectiveness_plan import ( build_key_blind_high_coverage_plan, @@ -17,7 +17,15 @@ DEFAULT_MODEL_REVISION, runtime_tokenizer_identity_public, ) -from .transforms import KEY_BLIND_HIGH_COVERAGE_PROFILE +from .transforms import ( + KEY_BLIND_HIGH_COVERAGE_PROFILE_ID, + KEY_BLIND_FULL_POOL_COVERAGE_PROFILE_ID, + KEY_BLIND_COVERAGE_COMPLETION_PROFILE_ID, + CONTENT_REGION_COMBINED_PROFILE_ID, + CONTENT_REGION_COVERAGE_PROFILE_ID, + CONTENT_REGION_GENERAL_ONLY_PROFILE_ID, + resolve_effectiveness_profile, +) EFFECTIVENESS_PLAN_PROVENANCE_VERSION = "effectiveness-plan-provenance-v1" @@ -53,12 +61,43 @@ def _provenance( return {**payload, "provenance_hash": sha256_json(payload)} +def _parse_budgets(raw: str) -> tuple[int, ...]: + if not raw: + return () + values: list[int] = [] + for chunk in raw.split(","): + stripped = chunk.strip() + if not stripped: + raise ValueError("budget list contains an empty entry") + value = int(stripped) + if value <= 0: + raise ValueError("budgets must be positive integers") + values.append(value) + budgets = tuple(sorted(set(values))) + if budgets != tuple(values): + raise ValueError("budgets must be provided in ascending order without duplicates") + return budgets + + def _parser() -> argparse.ArgumentParser: parser = argparse.ArgumentParser(prog="fuckmark-tiny-dev-effectiveness-plan-hf") parser.add_argument("--corpus-json", type=Path, required=True) parser.add_argument("--model", default=DEFAULT_MODEL_ID) parser.add_argument("--model-revision", default=DEFAULT_MODEL_REVISION) parser.add_argument("--source-code-commit", required=True) + parser.add_argument( + "--profile-id", + default=KEY_BLIND_HIGH_COVERAGE_PROFILE_ID, + choices=( + KEY_BLIND_HIGH_COVERAGE_PROFILE_ID, + KEY_BLIND_FULL_POOL_COVERAGE_PROFILE_ID, + KEY_BLIND_COVERAGE_COMPLETION_PROFILE_ID, + CONTENT_REGION_COVERAGE_PROFILE_ID, + CONTENT_REGION_GENERAL_ONLY_PROFILE_ID, + CONTENT_REGION_COMBINED_PROFILE_ID, + ), + ) + parser.add_argument("--budgets", default="") parser.add_argument( "--plan-json", type=Path, @@ -79,7 +118,8 @@ def main(argv: list[str] | None = None) -> int: except ImportError as error: raise RuntimeError("Install the pinned TinyDev Transformers dependencies first") from error - corpus = load_tiny_dev_corpus_json(args.corpus_json) + profile = resolve_effectiveness_profile(args.profile_id, _parse_budgets(args.budgets)) + corpus = load_tiny_dev_corpus_by_version_json(args.corpus_json) tokenizer = AutoTokenizer.from_pretrained( args.model, revision=args.model_revision, @@ -96,7 +136,7 @@ def main(argv: list[str] | None = None) -> int: plan = build_key_blind_high_coverage_plan( corpus, tokenizer, - profile=KEY_BLIND_HIGH_COVERAGE_PROFILE, + profile=profile, source_code_commit=args.source_code_commit, ) write_canonical_json_fsynced(args.plan_json, plan) @@ -104,14 +144,14 @@ def main(argv: list[str] | None = None) -> int: provenance = _provenance( source_code_commit=args.source_code_commit, plan_hash=plan["plan_hash"], - profile_hash=KEY_BLIND_HIGH_COVERAGE_PROFILE.profile_hash, + profile_hash=profile.profile_hash, plan_started_at=started_at, plan_fsynced_at=fsynced_at, ) write_canonical_json_fsynced(args.provenance_json, provenance) - sys.stdout.write(f"profile_id={KEY_BLIND_HIGH_COVERAGE_PROFILE.profile_id}\n") - sys.stdout.write(f"profile_hash={KEY_BLIND_HIGH_COVERAGE_PROFILE.profile_hash}\n") + sys.stdout.write(f"profile_id={profile.profile_id}\n") + sys.stdout.write(f"profile_hash={profile.profile_hash}\n") sys.stdout.write(f"plan_hash={plan['plan_hash']}\n") sys.stdout.write(f"variant_count={len(plan['variants'])}\n") sys.stdout.write(f"plan_json={args.plan_json.as_posix()}\n") diff --git a/fuckmark/tiny_dev_effectiveness_score_hf.py b/fuckmark/tiny_dev_effectiveness_score_hf.py index 1ac2f5b6..29f1241d 100644 --- a/fuckmark/tiny_dev_effectiveness_score_hf.py +++ b/fuckmark/tiny_dev_effectiveness_score_hf.py @@ -10,7 +10,7 @@ from typing import Any from .adapters import HuggingFaceSynthIDAdapter, HuggingFaceSynthIDConfig -from .corpus import load_tiny_dev_corpus_json +from .corpus import load_tiny_dev_corpus_by_version_json from .detectors import weighted_mean_evidence from .durable_io import write_canonical_json_fsynced from .experiments.effectiveness_plan import validate_key_blind_high_coverage_plan @@ -31,7 +31,17 @@ _threshold, _word_edit_distance, ) -from .transforms import KEY_BLIND_HIGH_COVERAGE_PROFILE +from .transforms import ( + CONTENT_REGION_COMBINED_PROFILE_ID, + CONTENT_REGION_COVERAGE_PROFILE_ID, + CONTENT_REGION_GENERAL_ONLY_PROFILE_ID, + KEY_BLIND_HIGH_COVERAGE_PROFILE, + KEY_BLIND_HIGH_COVERAGE_PROFILE_ID, + KEY_BLIND_FULL_POOL_COVERAGE_PROFILE_ID, + KEY_BLIND_COVERAGE_COMPLETION_PROFILE_ID, + EffectivenessTransformProfile, + resolve_effectiveness_profile, +) KEY_BLIND_HIGH_COVERAGE_EVIDENCE_VERSION = "key-blind-high-coverage-evidence-v1" @@ -87,12 +97,33 @@ def score_key_blind_high_coverage_plan( tokenizer: Any, plan: Mapping[str, object], adapter: HuggingFaceSynthIDAdapter, + *, + profile: EffectivenessTransformProfile | None = None, + fixed_threshold: Mapping[str, object] | None = None, ) -> dict[str, object]: - validate_key_blind_high_coverage_plan(plan, corpus, KEY_BLIND_HIGH_COVERAGE_PROFILE) - if adapter.ngram_len != KEY_BLIND_HIGH_COVERAGE_PROFILE.ngram_len: + active_profile = profile if profile is not None else KEY_BLIND_HIGH_COVERAGE_PROFILE + validate_key_blind_high_coverage_plan(plan, corpus, active_profile) + if adapter.ngram_len != active_profile.ngram_len: raise ValueError("detector ngram length does not match the frozen effectiveness profile") calibration = _text_only_calibration(corpus, adapter) - threshold = _threshold(calibration, PRIMARY_TARGET_FPR) + if fixed_threshold is not None: + from .experiments.measurement_threshold import validate_fixed_threshold_artifact + + threshold_value = validate_fixed_threshold_artifact( + dict(fixed_threshold), + calibration.detector_identity.identity_hash, + ) + threshold_source = "open-detector-measurement-stability-v1" + fixed_threshold_artifact_hash = fixed_threshold["artifact_hash"] + threshold_hash_for_payload = fixed_threshold["artifact_hash"] + threshold_target_fpr = fixed_threshold["target_fpr"] + else: + threshold = _threshold(calibration, PRIMARY_TARGET_FPR) + threshold_value = threshold.value + threshold_source = "per-corpus-calibration" + fixed_threshold_artifact_hash = None + threshold_hash_for_payload = threshold.threshold_hash + threshold_target_fpr = threshold.target_fpr sources = {sample.sample_id: sample for sample in _attack_samples(corpus)} pristine = { sample_id: _text_only_weighted_evidence(sample, adapter) @@ -132,8 +163,8 @@ def score_key_blind_high_coverage_plan( "pristine_score": original.raw_score, "transformed_score": transformed.raw_score, "score_drop": original.raw_score - transformed.raw_score, - "pristine_detected": original.raw_score >= threshold.value, - "transformed_detected": transformed.raw_score >= threshold.value, + "pristine_detected": original.raw_score >= threshold_value, + "transformed_detected": transformed.raw_score >= threshold_value, } rows.append({**row, "row_hash": sha256_json(row)}) row_tuple = tuple(rows) @@ -146,14 +177,16 @@ def score_key_blind_high_coverage_plan( "tiny_dev_artifact_hash": corpus.artifact_hash, "corpus_manifest_hash": corpus.manifest.manifest_hash, "plan_hash": plan["plan_hash"], - "profile_id": KEY_BLIND_HIGH_COVERAGE_PROFILE.profile_id, - "profile_hash": KEY_BLIND_HIGH_COVERAGE_PROFILE.profile_hash, - "ruleset_hash": KEY_BLIND_HIGH_COVERAGE_PROFILE.ruleset_hash, + "profile_id": active_profile.profile_id, + "profile_hash": active_profile.profile_hash, + "ruleset_hash": active_profile.ruleset_hash, "detector_identity_hash": calibration.detector_identity.identity_hash, "calibration_bundle_hash": calibration.bundle_hash, - "threshold_hash": threshold.threshold_hash, - "threshold_target_fpr": threshold.target_fpr, - "threshold_value": threshold.value, + "threshold_hash": threshold_hash_for_payload, + "threshold_target_fpr": threshold_target_fpr, + "threshold_value": threshold_value, + "threshold_source": threshold_source, + "fixed_threshold_artifact_hash": fixed_threshold_artifact_hash, "selection_detector_access_observed": plan["detector_access_observed"], "selection_secret_access_observed": plan["secret_access_observed"], "rows": row_tuple, @@ -188,6 +221,24 @@ def _provenance( return {**payload, "provenance_hash": sha256_json(payload)} +def _parse_budgets(raw: str) -> tuple[int, ...]: + if not raw: + return () + values: list[int] = [] + for chunk in raw.split(","): + stripped = chunk.strip() + if not stripped: + raise ValueError("budget list contains an empty entry") + value = int(stripped) + if value <= 0: + raise ValueError("budgets must be positive integers") + values.append(value) + budgets = tuple(sorted(set(values))) + if budgets != tuple(values): + raise ValueError("budgets must be provided in ascending order without duplicates") + return budgets + + def _parser() -> argparse.ArgumentParser: parser = argparse.ArgumentParser(prog="fuckmark-tiny-dev-effectiveness-score-hf") parser.add_argument("--corpus-json", type=Path, required=True) @@ -196,6 +247,24 @@ def _parser() -> argparse.ArgumentParser: parser.add_argument("--model-revision", default=DEFAULT_MODEL_REVISION) parser.add_argument("--device", choices=("cpu", "cuda"), default="cpu") parser.add_argument("--source-code-commit", required=True) + parser.add_argument( + "--profile-id", + default=KEY_BLIND_HIGH_COVERAGE_PROFILE_ID, + choices=( + KEY_BLIND_HIGH_COVERAGE_PROFILE_ID, + KEY_BLIND_FULL_POOL_COVERAGE_PROFILE_ID, + KEY_BLIND_COVERAGE_COMPLETION_PROFILE_ID, + CONTENT_REGION_COVERAGE_PROFILE_ID, + CONTENT_REGION_GENERAL_ONLY_PROFILE_ID, + CONTENT_REGION_COMBINED_PROFILE_ID, + ), + ) + parser.add_argument("--budgets", default="") + parser.add_argument( + "--fixed-threshold-json", + type=Path, + default=None, + ) parser.add_argument( "--json", type=Path, @@ -216,9 +285,10 @@ def main(argv: list[str] | None = None) -> int: except ImportError as error: raise RuntimeError("Install the pinned TinyDev Transformers dependencies first") from error - corpus = load_tiny_dev_corpus_json(args.corpus_json) + profile = resolve_effectiveness_profile(args.profile_id, _parse_budgets(args.budgets)) + corpus = load_tiny_dev_corpus_by_version_json(args.corpus_json) plan = _load_json(args.plan_json) - validate_key_blind_high_coverage_plan(plan, corpus, KEY_BLIND_HIGH_COVERAGE_PROFILE) + validate_key_blind_high_coverage_plan(plan, corpus, profile) if plan["source_code_commit"] != args.source_code_commit: raise ValueError("score invocation commit does not match the frozen plan") tokenizer = AutoTokenizer.from_pretrained( @@ -247,7 +317,17 @@ def main(argv: list[str] | None = None) -> int: device=args.device, ) started_at = _now() - evidence = score_key_blind_high_coverage_plan(corpus, tokenizer, plan, adapter) + fixed_threshold = None + if args.fixed_threshold_json is not None: + fixed_threshold = _load_json(args.fixed_threshold_json) + evidence = score_key_blind_high_coverage_plan( + corpus, + tokenizer, + plan, + adapter, + profile=profile, + fixed_threshold=fixed_threshold, + ) write_canonical_json_fsynced(args.json, evidence) fsynced_at = _now() provenance = _provenance( @@ -259,7 +339,7 @@ def main(argv: list[str] | None = None) -> int: ) write_canonical_json_fsynced(args.provenance_json, provenance) - sys.stdout.write(f"profile_id={KEY_BLIND_HIGH_COVERAGE_PROFILE.profile_id}\n") + sys.stdout.write(f"profile_id={profile.profile_id}\n") sys.stdout.write(f"plan_hash={plan['plan_hash']}\n") sys.stdout.write(f"artifact_hash={evidence['artifact_hash']}\n") for summary in evidence["summaries"]: diff --git a/fuckmark/tiny_dev_measurement_calibration_hf.py b/fuckmark/tiny_dev_measurement_calibration_hf.py new file mode 100644 index 00000000..673db01a --- /dev/null +++ b/fuckmark/tiny_dev_measurement_calibration_hf.py @@ -0,0 +1,220 @@ +from __future__ import annotations + +import argparse +import sys +from pathlib import Path + +from .config import canonical_json_text +from .corpus import WatermarkLabel +from .corpus.measurement_calibration import ( + MEASUREMENT_CALIBRATION_PROMPT_SOURCE_ID, + MEASUREMENT_CALIBRATION_SEEDS_PER_PROMPT, + MEASUREMENT_CALIBRATION_SAMPLE_COUNT, + MEASUREMENT_CALIBRATION_TOPICS_PER_DOMAIN, + build_measurement_calibration_corpus, + measurement_calibration_topics, +) +from .corpus.prompt import PromptRecord +from .corpus.sample import CorpusSample +from .corpus.schema import CorpusSplit, MAX_GENERATION_SEED +from .corpus.tokenization import GenerationTokenRecord, TextOnlyTokenRecord +from .corpus.tiny_dev import TINY_DEV_TARGET_LENGTH +from .corpus.tiny_dev_generation import ( + TINY_DEV_PROMPT_LICENSE_ID, + TINY_DEV_PROMPT_PROVENANCE, + TINY_DEV_SEED_POLICY_ID, + TinyDevGenerationError, + _TINY_DEV_TEMPLATES, +) +from .hashing import sha256_json, sha256_text +from .tiny_dev_corpus_hf import DEFAULT_MODEL_ID, DEFAULT_MODEL_REVISION, HuggingFaceTinyDevBackend + + +MEASUREMENT_CALIBRATION_GENERATION_VERSION = "measurement-calibration-generation-v1" +MEASUREMENT_CALIBRATION_SEED_STRIDE = 16 +MEASUREMENT_CALIBRATION_MAX_ATTEMPTS = 16 +MEASUREMENT_CALIBRATION_TOPIC_OFFSET = 100 + + +def build_measurement_calibration_prompt_records() -> tuple[PromptRecord, ...]: + topics = measurement_calibration_topics() + if len(topics) != MEASUREMENT_CALIBRATION_TOPICS_PER_DOMAIN: + raise ValueError("measurement calibration topic pool must contain 40 topics") + source_hash = sha256_json( + { + "algorithm_version": MEASUREMENT_CALIBRATION_GENERATION_VERSION, + "source_id": MEASUREMENT_CALIBRATION_PROMPT_SOURCE_ID, + "license_id": TINY_DEV_PROMPT_LICENSE_ID, + "provenance": TINY_DEV_PROMPT_PROVENANCE, + "topics": topics, + "templates": tuple( + (domain.value, _TINY_DEV_TEMPLATES[domain]) + for domain in _TINY_DEV_TEMPLATES + ), + } + ) + records: list[PromptRecord] = [] + for domain in _TINY_DEV_TEMPLATES: + template = _TINY_DEV_TEMPLATES[domain] + for local_index, topic in enumerate(topics): + topic_index = MEASUREMENT_CALIBRATION_TOPIC_OFFSET + local_index + prompt_id = f"measurement-calibration-{domain.value}-{topic_index:02d}" + records.append( + PromptRecord.create( + prompt_id=prompt_id, + prompt_family_id=f"measurement-calibration-family-{domain.value}-{topic_index:02d}", + domain=domain, + split=CorpusSplit.THRESHOLD_CALIBRATION, + source_id=MEASUREMENT_CALIBRATION_PROMPT_SOURCE_ID, + source_hash=source_hash, + license_id=TINY_DEV_PROMPT_LICENSE_ID, + provenance=TINY_DEV_PROMPT_PROVENANCE, + text=template.format(topic=topic), + ) + ) + return tuple(sorted(records, key=lambda value: value.prompt_id)) + + +def _build_slot_sample( + *, + prompt, + slot: int, + generated, + backend, + seed: int, +) -> CorpusSample: + model = backend.model_identity + generation = backend.generation_parameters(seed) + if generation.seed != seed: + raise TinyDevGenerationError("backend generation parameters do not bind the requested seed") + if generation.seed_policy_id != TINY_DEV_SEED_POLICY_ID: + raise TinyDevGenerationError("backend seed policy does not match the frozen tiny development policy") + generation_tokens = GenerationTokenRecord.create( + input_token_ids=generated.input_token_ids, + attention_mask=generated.attention_mask, + generated_sequence_ids=generated.input_token_ids + generated.continuation_token_ids, + continuation_start_index=len(generated.input_token_ids), + continuation_token_ids=generated.continuation_token_ids, + prompt_length_after_templating=sum(generated.attention_mask), + model_tokenizer_identity_hash=model.identity_hash, + ) + text_only = None + if generated.text_only_token_ids is not None: + text_only = TextOnlyTokenRecord.create( + source_text_sha256=sha256_text(generated.text), + token_ids=generated.text_only_token_ids, + model_tokenizer_identity_hash=model.identity_hash, + ) + sample_id = f"{prompt.prompt_id}-s{slot:02d}-{WatermarkLabel.UNWATERMARKED.value}" + return CorpusSample.create( + sample_id=sample_id, + match_id=f"match-{sample_id}", + prompt_id=prompt.prompt_id, + prompt_family_id=prompt.prompt_family_id, + domain=prompt.domain, + split=prompt.split, + label=WatermarkLabel.UNWATERMARKED, + text=generated.text, + model=model, + generation=generation, + watermark=backend.watermark_condition, + target_length=TINY_DEV_TARGET_LENGTH, + generation_tokens=generation_tokens, + text_only_tokens=text_only, + ) + + +def build_real_measurement_calibration_corpus( + backend: HuggingFaceTinyDevBackend, + *, + corpus_id: str = "fuckmark-measurement-calibration-v1", + seed_base: int = 500000, +) -> MeasurementCalibrationCorpus: + if type(seed_base) is not int or seed_base < 0: + raise ValueError("seed_base must be a non-negative integer") + prompts = build_measurement_calibration_prompt_records() + slot_count = len(prompts) * MEASUREMENT_CALIBRATION_SEEDS_PER_PROMPT + if seed_base + slot_count * MEASUREMENT_CALIBRATION_SEED_STRIDE + MEASUREMENT_CALIBRATION_MAX_ATTEMPTS > MAX_GENERATION_SEED: + raise ValueError("measurement calibration seed schedule exceeds the 64-bit range") + + samples: list[object] = [] + seen_text_hashes: set[str] = set() + seen_token_hashes: set[str] = set() + slot_index = 0 + for prompt in prompts: + for slot in range(MEASUREMENT_CALIBRATION_SEEDS_PER_PROMPT): + slot_seed_base = seed_base + slot_index * MEASUREMENT_CALIBRATION_SEED_STRIDE + slot_index += 1 + accepted = False + for attempt in range(MEASUREMENT_CALIBRATION_MAX_ATTEMPTS): + seed = slot_seed_base + attempt + try: + generated = backend.generate(prompt.text, seed, watermarked=False) + except RuntimeError: + continue + if len(generated.continuation_token_ids) != TINY_DEV_TARGET_LENGTH: + continue + if not generated.text.strip(): + continue + text_hash = sha256_text(generated.text) + token_hash = sha256_json(generated.continuation_token_ids) + if text_hash in seen_text_hashes or token_hash in seen_token_hashes: + continue + samples.append( + _build_slot_sample( + prompt=prompt, + slot=slot, + generated=generated, + backend=backend, + seed=seed, + ) + ) + seen_text_hashes.add(text_hash) + seen_token_hashes.add(token_hash) + accepted = True + break + if not accepted: + raise RuntimeError( + f"could not produce an exact unique 64-token negative for {prompt.prompt_id} slot {slot}" + ) + if len(samples) != MEASUREMENT_CALIBRATION_SAMPLE_COUNT: + raise RuntimeError("measurement calibration sample count does not match the frozen profile") + return build_measurement_calibration_corpus(corpus_id, prompts, samples) + + +def _parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser(prog="fuckmark-tiny-dev-measurement-calibration-hf") + parser.add_argument("--model", default=DEFAULT_MODEL_ID) + parser.add_argument("--model-revision", default=DEFAULT_MODEL_REVISION) + parser.add_argument("--device", choices=("auto", "cpu", "cuda"), default="auto") + parser.add_argument("--seed-base", type=int, default=500000) + parser.add_argument("--temperature", type=float, default=0.8) + parser.add_argument("--top-k", type=int, default=50) + parser.add_argument("--top-p", type=float, default=0.95) + parser.add_argument("--json", type=Path, required=True) + return parser + + +def main(argv: list[str] | None = None) -> int: + args = _parser().parse_args(argv) + backend = HuggingFaceTinyDevBackend( + args.model, + args.model_revision, + device=args.device, + temperature=args.temperature, + top_k=args.top_k, + top_p=args.top_p, + ) + artifact = build_real_measurement_calibration_corpus(backend, seed_base=args.seed_base) + args.json.parent.mkdir(parents=True, exist_ok=True) + args.json.write_text(canonical_json_text(artifact) + "\n", encoding="utf-8") + sys.stdout.write(f"artifact_hash={artifact.artifact_hash}\n") + sys.stdout.write(f"manifest_hash={artifact.manifest.manifest_hash}\n") + sys.stdout.write(f"sample_count={len(artifact.manifest.samples)}\n") + sys.stdout.write(f"model_identity_hash={artifact.model_identity_hash}\n") + sys.stdout.write(f"json={args.json.as_posix()}\n") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/fuckmark/tiny_dev_measurement_threshold_hf.py b/fuckmark/tiny_dev_measurement_threshold_hf.py new file mode 100644 index 00000000..d7df93ca --- /dev/null +++ b/fuckmark/tiny_dev_measurement_threshold_hf.py @@ -0,0 +1,73 @@ +from __future__ import annotations + +import argparse +import json +import sys +from datetime import datetime, timezone +from pathlib import Path + +from .adapters import HuggingFaceSynthIDAdapter, HuggingFaceSynthIDConfig +from .corpus.measurement_calibration_io import load_measurement_calibration_corpus_json +from .durable_io import write_canonical_json_fsynced +from .experiments.measurement_threshold import build_fixed_threshold_artifact +from .tiny_dev_context_survival_plan_hf import ( + DEFAULT_MODEL_ID, + DEFAULT_MODEL_REVISION, + runtime_tokenizer_identity_public, +) +from .tiny_dev_detector_hf import default_watermark_payload + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(prog="fuckmark-tiny-dev-measurement-threshold-hf") + parser.add_argument("--calibration-corpus-json", type=Path, required=True) + parser.add_argument("--model", default=DEFAULT_MODEL_ID) + parser.add_argument("--model-revision", default=DEFAULT_MODEL_REVISION) + parser.add_argument("--device", choices=("cpu", "cuda"), default="cpu") + parser.add_argument("--json", type=Path, required=True) + args = parser.parse_args(argv) + try: + from transformers import AutoTokenizer + except ImportError as error: + raise RuntimeError("Install the pinned TinyDev Transformers dependencies first") from error + + corpus = load_measurement_calibration_corpus_json(args.calibration_corpus_json) + tokenizer = AutoTokenizer.from_pretrained( + args.model, + revision=args.model_revision, + use_fast=True, + padding_side="left", + ) + identity = runtime_tokenizer_identity_public(tokenizer, args.model, args.model_revision) + if identity.identity_hash != corpus.model_identity_hash: + raise RuntimeError("runtime tokenizer identity does not match the calibration corpus") + watermark_payload = default_watermark_payload() + adapter = HuggingFaceSynthIDAdapter.from_torch( + HuggingFaceSynthIDConfig( + ngram_len=watermark_payload["ngram_len"], + keys=watermark_payload["keys"], + context_history_size=watermark_payload["context_history_size"], + sampling_table_seed=watermark_payload["sampling_table_seed"], + sampling_table_size=watermark_payload["sampling_table_size"], + skip_first_ngram_calls=watermark_payload["skip_first_ngram_calls"], + debug_mode=watermark_payload["debug_mode"], + ), + device=args.device, + ) + artifact = build_fixed_threshold_artifact( + corpus, + adapter, + frozen_at_utc=datetime.now(timezone.utc).isoformat(), + ) + write_canonical_json_fsynced(args.json, artifact) + sys.stdout.write(f"artifact_hash={artifact['artifact_hash']}\n") + sys.stdout.write(f"threshold={artifact['threshold']!r}\n") + sys.stdout.write(f"calibration_exceedances={artifact['calibration_exceedances']}/{artifact['calibration_negative_count']}\n") + sys.stdout.write(f"audit_exceedances={artifact['audit_exceedances']}/{artifact['audit_negative_count']}\n") + sys.stdout.write(f"audit_realized_fpr={artifact['audit_realized_fpr']}\n") + sys.stdout.write(f"json={args.json.as_posix()}\n") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/fuckmark/transforms/__init__.py b/fuckmark/transforms/__init__.py index cca9ca93..27f2fb99 100644 --- a/fuckmark/transforms/__init__.py +++ b/fuckmark/transforms/__init__.py @@ -1,12 +1,30 @@ from .candidate_artifacts import CandidateConflict, CandidateEnumeration, CandidateRejection, TransformCandidate from .effectiveness_profile import ( + CONTENT_REGION_COMBINED_PROFILE_ID, + CONTENT_REGION_COVERAGE_PROFILE_ID, + CONTENT_REGION_COVERAGE_SEED_BASE, + CONTENT_REGION_GENERAL_ONLY_PROFILE_ID, EFFECTIVENESS_PROFILE_ALGORITHM_VERSION, + KEY_BLIND_COVERAGE_COMPLETION_PROFILE_ID, + KEY_BLIND_COVERAGE_COMPLETION_SEED_BASE, + KEY_BLIND_FULL_POOL_COVERAGE_PROFILE_ID, + KEY_BLIND_FULL_POOL_COVERAGE_SEED_BASE, KEY_BLIND_HIGH_COVERAGE_BUDGETS, KEY_BLIND_HIGH_COVERAGE_PROFILE, KEY_BLIND_HIGH_COVERAGE_PROFILE_ID, KEY_BLIND_HIGH_COVERAGE_SEED_BASE, EffectivenessTransformProfile, + content_region_combined_profile, + content_region_combined_transform_registry, + content_region_coverage_profile, + content_region_coverage_transform_registry, + content_region_general_only_profile, + content_region_general_only_transform_registry, + key_blind_coverage_completion_profile, + key_blind_coverage_completion_transform_registry, + key_blind_full_pool_coverage_profile, key_blind_high_coverage_transform_registry, + resolve_effectiveness_profile, validate_effectiveness_profile_registry, ) from .fidelity_readiness import ( @@ -196,6 +214,8 @@ "HardInvariantSignature", "hard_invariant_signature", "key_blind_high_coverage_transform_registry", + "key_blind_full_pool_coverage_profile", + "resolve_effectiveness_profile", "validate_hard_invariants", "InvariantDifference", "InvariantStatus", @@ -204,6 +224,22 @@ "KEY_BLIND_HIGH_COVERAGE_PROFILE", "KEY_BLIND_HIGH_COVERAGE_PROFILE_ID", "KEY_BLIND_HIGH_COVERAGE_SEED_BASE", + "KEY_BLIND_FULL_POOL_COVERAGE_PROFILE_ID", + "KEY_BLIND_FULL_POOL_COVERAGE_SEED_BASE", + "KEY_BLIND_COVERAGE_COMPLETION_PROFILE_ID", + "KEY_BLIND_COVERAGE_COMPLETION_SEED_BASE", + "CONTENT_REGION_COMBINED_PROFILE_ID", + "CONTENT_REGION_COVERAGE_PROFILE_ID", + "CONTENT_REGION_COVERAGE_SEED_BASE", + "CONTENT_REGION_GENERAL_ONLY_PROFILE_ID", + "content_region_combined_profile", + "content_region_combined_transform_registry", + "content_region_general_only_profile", + "content_region_general_only_transform_registry", + "content_region_coverage_profile", + "content_region_coverage_transform_registry", + "key_blind_coverage_completion_profile", + "key_blind_coverage_completion_transform_registry", "LEXICAL_RETOKENIZATION_FIXTURE_ALGORITHM_VERSION", "LEXICAL_RULE_AUDIT_ALGORITHM_VERSION", "LEXICAL_TEMPLATE_RULE_ALGORITHM_VERSION", diff --git a/fuckmark/transforms/effectiveness_profile.py b/fuckmark/transforms/effectiveness_profile.py index 06c724e7..67c0136c 100644 --- a/fuckmark/transforms/effectiveness_profile.py +++ b/fuckmark/transforms/effectiveness_profile.py @@ -13,8 +13,9 @@ TRANSFORM_REGISTRY_ALGORITHM_VERSION, TransformRegistry, ) +from .rules import GeneralWordSpacingRule from .scheduler import CANDIDATE_SCHEDULER_ALGORITHM_VERSION -from .surface_rules import development_surface_rules +from .surface_rules import coverage_completion_surface_rules, development_surface_rules from .syntax_rules import development_syntax_rules from .tokenizer_geometry import TOKENIZER_GEOMETRY_ALGORITHM_VERSION @@ -23,6 +24,14 @@ KEY_BLIND_HIGH_COVERAGE_PROFILE_ID = "key-blind-high-coverage-v1" KEY_BLIND_HIGH_COVERAGE_BUDGETS = (16,) KEY_BLIND_HIGH_COVERAGE_SEED_BASE = 1_120_000 +KEY_BLIND_FULL_POOL_COVERAGE_PROFILE_ID = "key-blind-full-pool-coverage-v1" +KEY_BLIND_FULL_POOL_COVERAGE_SEED_BASE = 1_130_000 +KEY_BLIND_COVERAGE_COMPLETION_PROFILE_ID = "key-blind-coverage-completion-v2" +KEY_BLIND_COVERAGE_COMPLETION_SEED_BASE = 1_140_000 +CONTENT_REGION_COVERAGE_PROFILE_ID = "content-region-coverage-v1" +CONTENT_REGION_COVERAGE_SEED_BASE = 1_150_000 +CONTENT_REGION_GENERAL_ONLY_PROFILE_ID = "content-region-general-only-dev-v1" +CONTENT_REGION_COMBINED_PROFILE_ID = "content-region-combined-dev-v1" @dataclass(frozen=True, slots=True) @@ -167,6 +176,212 @@ def key_blind_high_coverage_transform_registry( ) +def key_blind_full_pool_coverage_profile( + budgets: tuple[int, ...], +) -> EffectivenessTransformProfile: + if not isinstance(budgets, tuple): + raise TypeError("budgets must be a tuple") + return EffectivenessTransformProfile.create( + profile_id=KEY_BLIND_FULL_POOL_COVERAGE_PROFILE_ID, + budgets=budgets, + budget_unit="operation", + schedule_policy_id="COVERAGE_GREEDY_KEY_BLIND", + schedule_seed_base=KEY_BLIND_FULL_POOL_COVERAGE_SEED_BASE, + replicate_count=1, + ngram_len=5, + ruleset_hash=key_blind_high_coverage_transform_registry().ruleset_hash, + scientific_scope=( + "Frozen detector-blind and key-blind budget-scaled coverage profile over the " + "development ruleset with one fixed source-index seed per sorted source; " + "exploratory effectiveness evidence only and not release authorization" + ), + ) + + +def resolve_effectiveness_profile( + profile_id: str, + budgets: tuple[int, ...] = (), +) -> EffectivenessTransformProfile: + require_clean_string("profile_id", profile_id) + if profile_id == KEY_BLIND_HIGH_COVERAGE_PROFILE_ID: + if budgets: + raise ValueError("the frozen B16 profile does not accept custom budgets") + return KEY_BLIND_HIGH_COVERAGE_PROFILE + if profile_id == KEY_BLIND_FULL_POOL_COVERAGE_PROFILE_ID: + if not budgets: + raise ValueError("the full-pool coverage profile requires explicit budgets") + return key_blind_full_pool_coverage_profile(budgets) + if profile_id == KEY_BLIND_COVERAGE_COMPLETION_PROFILE_ID: + if not budgets: + raise ValueError("the coverage completion profile requires explicit budgets") + return key_blind_coverage_completion_profile(budgets) + if profile_id == CONTENT_REGION_COVERAGE_PROFILE_ID: + if not budgets: + raise ValueError("the content region coverage profile requires explicit budgets") + return content_region_coverage_profile(budgets) + if profile_id == CONTENT_REGION_GENERAL_ONLY_PROFILE_ID: + if not budgets: + raise ValueError("the general-only ablation profile requires explicit budgets") + return content_region_general_only_profile(budgets) + if profile_id == CONTENT_REGION_COMBINED_PROFILE_ID: + if not budgets: + raise ValueError("the combined ablation profile requires explicit budgets") + return content_region_combined_profile(budgets) + raise ValueError("unknown effectiveness profile id") + + +CONTENT_REGION_GENERAL_SPACING_RULE_ID = "surface-space-after-any-word" + + +def content_region_surface_rules(): + punctuation = tuple( + rule + for rule in development_surface_rules() + if not rule.source.isalpha() + ) + general = GeneralWordSpacingRule.create(CONTENT_REGION_GENERAL_SPACING_RULE_ID) + return (*punctuation, general) + + +def content_region_coverage_transform_registry( + identifiers: Sequence[str] = (), +) -> TransformRegistry: + return TransformRegistry( + ( + *development_forward_contraction_rules(), + *content_region_surface_rules(), + *development_lexical_rules(), + *development_syntax_rules(), + ), + identifiers, + ) + + +def content_region_coverage_profile( + budgets: tuple[int, ...], +) -> EffectivenessTransformProfile: + if not isinstance(budgets, tuple): + raise TypeError("budgets must be a tuple") + return EffectivenessTransformProfile.create( + profile_id=CONTENT_REGION_COVERAGE_PROFILE_ID, + budgets=budgets, + budget_unit="operation", + schedule_policy_id="COVERAGE_GREEDY_KEY_BLIND", + schedule_seed_base=CONTENT_REGION_COVERAGE_SEED_BASE, + replicate_count=1, + ngram_len=5, + ruleset_hash=content_region_coverage_transform_registry().ruleset_hash, + scientific_scope=( + "Frozen detector-blind and key-blind content-region coverage profile pairing " + "contractions and punctuation spacing with a general any-word spacing rule; " + "exploratory effectiveness evidence only and not release authorization" + ), + ) + + +def content_region_general_only_transform_registry( + identifiers: Sequence[str] = (), +) -> TransformRegistry: + return TransformRegistry(content_region_surface_rules(), identifiers) + + +def content_region_combined_transform_registry( + identifiers: Sequence[str] = (), +) -> TransformRegistry: + general = GeneralWordSpacingRule.create(CONTENT_REGION_GENERAL_SPACING_RULE_ID) + return TransformRegistry( + ( + *development_forward_contraction_rules(), + *coverage_completion_surface_rules(), + general, + *development_lexical_rules(), + *development_syntax_rules(), + ), + identifiers, + ) + + +def _dev_ablation_profile( + profile_id: str, + ruleset_hash: str, + budgets: tuple[int, ...], +) -> EffectivenessTransformProfile: + return EffectivenessTransformProfile.create( + profile_id=profile_id, + budgets=budgets, + budget_unit="operation", + schedule_policy_id="COVERAGE_GREEDY_KEY_BLIND", + schedule_seed_base=CONTENT_REGION_COVERAGE_SEED_BASE, + replicate_count=1, + ngram_len=5, + ruleset_hash=ruleset_hash, + scientific_scope=( + "Development-only ablation profile for the content-region cycle; exploratory " + "evidence only and not release authorization" + ), + ) + + +def content_region_general_only_profile( + budgets: tuple[int, ...], +) -> EffectivenessTransformProfile: + if not isinstance(budgets, tuple): + raise TypeError("budgets must be a tuple") + return _dev_ablation_profile( + CONTENT_REGION_GENERAL_ONLY_PROFILE_ID, + content_region_general_only_transform_registry().ruleset_hash, + budgets, + ) + + +def content_region_combined_profile( + budgets: tuple[int, ...], +) -> EffectivenessTransformProfile: + if not isinstance(budgets, tuple): + raise TypeError("budgets must be a tuple") + return _dev_ablation_profile( + CONTENT_REGION_COMBINED_PROFILE_ID, + content_region_combined_transform_registry().ruleset_hash, + budgets, + ) + + +def key_blind_coverage_completion_transform_registry( + identifiers: Sequence[str] = (), +) -> TransformRegistry: + return TransformRegistry( + ( + *development_forward_contraction_rules(), + *coverage_completion_surface_rules(), + *development_lexical_rules(), + *development_syntax_rules(), + ), + identifiers, + ) + + +def key_blind_coverage_completion_profile( + budgets: tuple[int, ...], +) -> EffectivenessTransformProfile: + if not isinstance(budgets, tuple): + raise TypeError("budgets must be a tuple") + return EffectivenessTransformProfile.create( + profile_id=KEY_BLIND_COVERAGE_COMPLETION_PROFILE_ID, + budgets=budgets, + budget_unit="operation", + schedule_policy_id="COVERAGE_GREEDY_KEY_BLIND", + schedule_seed_base=KEY_BLIND_COVERAGE_COMPLETION_SEED_BASE, + replicate_count=1, + ngram_len=5, + ruleset_hash=key_blind_coverage_completion_transform_registry().ruleset_hash, + scientific_scope=( + "Frozen detector-blind and key-blind coverage-completion profile extending the " + "development ruleset surface list; exploratory effectiveness evidence only and " + "not release authorization" + ), + ) + + def validate_effectiveness_profile_registry( profile: EffectivenessTransformProfile, registry: TransformRegistry, diff --git a/fuckmark/transforms/protected_patterns.py b/fuckmark/transforms/protected_patterns.py index 45b8f58f..b3a2fd2f 100644 --- a/fuckmark/transforms/protected_patterns.py +++ b/fuckmark/transforms/protected_patterns.py @@ -20,7 +20,8 @@ _SLASH_DATE_RE = re.compile(r"(? re.Pattern[str]: pattern = rf"{pattern}(?!\w)" return re.compile(pattern) + def replacement_for(self, source_text: str) -> str: + return self.replacement + + +GENERAL_WORD_SPACING_RULE_VERSION = "general-word-space-after-v1" + + +@dataclass(frozen=True, slots=True) +class GeneralWordSpacingRule(LiteralTransformRule): + def __post_init__(self) -> None: + require_clean_string("rule_id", self.rule_id) + require_clean_string("version", self.version) + if self.version != GENERAL_WORD_SPACING_RULE_VERSION: + raise ValueError("unsupported general word spacing rule version") + if self.family is not TransformFamily.ORTHOGRAPHY: + raise ValueError("general word spacing rules must use the orthography family") + if self.tier is not TransformTier.SURFACE: + raise ValueError("general word spacing rules must use tier 1 surface") + if self.source != "word" or self.replacement != "word ": + raise ValueError("general word spacing rules use the fixed word sentinel contract") + if "\n" in self.source or "\r" in self.source or "\n" in self.replacement or "\r" in self.replacement: + raise ValueError("general word spacing rules must stay single-line") + require_bool("whole_word", self.whole_word) + require_bool("preserve_simple_case", self.preserve_simple_case) + require_bool("block_all_caps", self.block_all_caps) + if self.whole_word or self.preserve_simple_case or self.block_all_caps: + raise ValueError("general word spacing rules use an exact case-sensitive boundary contract") + require_sha256("rule_hash", self.rule_hash) + if self.rule_hash != sha256_json(self._payload()): + raise ValueError("rule_hash does not match general word spacing rule") + + def _payload(self) -> dict[str, object]: + return { + "algorithm_version": RULE_ALGORITHM_VERSION, + "rule_id": self.rule_id, + "version": self.version, + "family": self.family.value, + "tier": self.tier.value, + "source": self.source, + "replacement": self.replacement, + "whole_word": self.whole_word, + "preserve_simple_case": self.preserve_simple_case, + "block_all_caps": self.block_all_caps, + "general_pattern": GENERAL_WORD_SPACING_RULE_VERSION, + } + + @classmethod + def create(cls, rule_id: str) -> "GeneralWordSpacingRule": + hashed = { + "algorithm_version": RULE_ALGORITHM_VERSION, + "rule_id": rule_id, + "version": GENERAL_WORD_SPACING_RULE_VERSION, + "family": TransformFamily.ORTHOGRAPHY.value, + "tier": TransformTier.SURFACE.value, + "source": "word", + "replacement": "word ", + "whole_word": False, + "preserve_simple_case": False, + "block_all_caps": False, + "general_pattern": GENERAL_WORD_SPACING_RULE_VERSION, + } + return cls( + rule_id=rule_id, + version=GENERAL_WORD_SPACING_RULE_VERSION, + family=TransformFamily.ORTHOGRAPHY, + tier=TransformTier.SURFACE, + source="word", + replacement="word ", + whole_word=False, + preserve_simple_case=False, + block_all_caps=False, + rule_hash=sha256_json(hashed), + ) + + def pattern(self) -> re.Pattern[str]: + return re.compile(r"(? str: + return source_text + " " + class SurfaceSpacingRule(LiteralTransformRule): def __post_init__(self) -> None: diff --git a/fuckmark/transforms/surface_rules.py b/fuckmark/transforms/surface_rules.py index 8e822752..45d61f90 100644 --- a/fuckmark/transforms/surface_rules.py +++ b/fuckmark/transforms/surface_rules.py @@ -65,3 +65,101 @@ def development_surface_rules() -> tuple[SurfaceSpacingRule, ...]: for name, source in punctuation ) return (*word_rules, *punctuation_rules) + + +COVERAGE_COMPLETION_EXTENSION_ALGORITHM_VERSION = "coverage-completion-extension-words-v1" + + +def coverage_completion_extension_words() -> tuple[str, ...]: + return ( + "an", + "he", + "she", + "his", + "her", + "its", + "our", + "their", + "them", + "him", + "us", + "me", + "my", + "your", + "if", + "when", + "where", + "which", + "who", + "what", + "how", + "why", + "because", + "so", + "than", + "then", + "there", + "here", + "all", + "any", + "each", + "more", + "most", + "some", + "such", + "only", + "very", + "just", + "also", + "into", + "about", + "over", + "under", + "after", + "before", + "between", + "during", + "through", + "without", + "while", + "do", + "does", + "been", + "being", + "would", + "could", + "should", + "may", + "might", + "must", + "were", + "one", + "even", + "still", + "often", + ) + + +def coverage_completion_surface_rules() -> tuple[SurfaceSpacingRule, ...]: + base = development_surface_rules() + base_ids = {rule.rule_id for rule in base} + base_words = {rule.source for rule in base if rule.source.isalpha()} + extension_words = coverage_completion_extension_words() + if len(set(extension_words)) != len(extension_words): + raise ValueError("coverage completion extension words must be unique") + if any(word in base_words for word in extension_words): + raise ValueError("coverage completion extension words must not duplicate the base surface list") + if any(not word.isalpha() or word != word.lower() or len(word) < 2 for word in extension_words): + raise ValueError("coverage completion extension words must be lowercase alphabetic words of length >= 2") + extension_rules = tuple( + SurfaceSpacingRule.create( + rule_id=f"surface-space-after-{word}", + version="v1", + source=word, + replacement=word + " ", + ) + for word in extension_words + ) + if any(rule.rule_id in base_ids for rule in extension_rules): + raise ValueError("coverage completion extension rule IDs must not collide with base rules") + return (*base, *extension_rules) diff --git a/specs/fuckmark-budget-scaled-coverage-effectiveness-v1.contract.json b/specs/fuckmark-budget-scaled-coverage-effectiveness-v1.contract.json new file mode 100644 index 00000000..c4922288 --- /dev/null +++ b/specs/fuckmark-budget-scaled-coverage-effectiveness-v1.contract.json @@ -0,0 +1,114 @@ +{ + "protocol_id": "budget-scaled-coverage-effectiveness-v1", + "algorithm_version": "budget-scaled-coverage-effectiveness-contract-v1", + "recorded_date": "2026-08-22", + "base_main_commit": "16e153fdfa3739a88bca9b1559543a10ded2d859", + "claim_boundary": { + "status": "EXPLORATORY_FIXED_OPEN_DETECTOR_ONLY", + "watermark_removal_claim": false, + "undetectability_claim": false, + "unknown_key_claim": false, + "proprietary_detector_claim": false, + "cleaning_durability_claim": false, + "release_authorized": false, + "level_b_product_admissible": false, + "level_b_failure_reason": "candidate intentionally edits visible text (spacing and contractions)" + }, + "candidate_profile": { + "profile_id": "key-blind-full-pool-coverage-v1", + "ruleset": "development forward contractions + surface + lexical + syntax (B16 line ruleset)", + "schedule_policy": "COVERAGE_GREEDY_KEY_BLIND", + "geometry_mode": "TOKENIZER_AWARE_PUBLIC", + "ngram_len": 5, + "replicate_count": 1, + "budget_selection": "frozen rule in protocol section 7", + "development_sweep_budgets": [16, 24, 32, 48], + "selection_access": { + "detector_scores": false, + "detector_decisions": false, + "watermark_secret_keys": false, + "confirmation_results": false, + "public_tokenizer_geometry": true, + "public_detector_ngram_len": true + } + }, + "corpora": { + "model_id": "openai-community/gpt2", + "model_revision": "607a30d783dfa663caf39e06633721c8d4cfcd7e", + "generation": { + "device": "cpu", + "temperature": 0.8, + "top_k": 50, + "top_p": 0.95, + "target_length": 64, + "max_attempts": 16 + }, + "watermark_keys_public_dev": [654, 400, 836, 123, 340, 443, 597, 160, 57], + "development": [ + {"seed_base": 401000, "profile": "tiny-dev-v2-canonical", "attack_pairs_per_domain": 1}, + {"seed_base": 410000, "profile": "tiny-dev-v3-parameterized", "attack_pairs_per_domain": 8} + ], + "confirmation": { + "seed_base": 420000, + "profile": "tiny-dev-v3-parameterized", + "attack_pairs_per_domain": 16, + "watermarked_attack_sources": 64, + "unwatermarked_attack_controls": 64, + "calibration_pairs_per_domain": 25 + }, + "disjointness": "pairwise-distinct text sha256 across corpora; confirmation disjoint from all development texts; asserted at scoring time" + }, + "detector": { + "family": "Hugging Face SynthID Weighted Mean", + "ngram_len": 5, + "input_path": "uncleaned text-only re-tokenized track", + "calibration": "100 unwatermarked calibration negatives of the same corpus", + "target_fpr": 0.01, + "comparison_operator": ">=", + "threshold_scope": "recomputed per corpus from its own calibration split" + }, + "fidelity_gates": { + "hard_invariants": "PASS required for every row", + "protected_span_violations": 0, + "word_edit_rate_max": 0.3, + "introduced_codepoints_must_be_ascii": true, + "introduced_forbidden_codepoint_categories": ["Cc", "Cf", "Mn", "Mc", "Me", "Zl", "Zp"], + "introduced_allowed_space": "U+0020 only", + "preexisting_source_codepoints": "reported, not gated", + "normalization_noop": ["NFC", "NFD", "NFKC", "NFKD"], + "cf_stripping_noop": true + }, + "controls": [ + "pristine_watermarked", + "transformed_watermarked", + "pristine_unwatermarked", + "transformed_unwatermarked" + ], + "budget_selection_rule": { + "eligibility": "development transformed-watermarked detected count == 0 AND all fidelity gates pass on all development rows", + "selection": "smallest eligible budget from [16, 24, 32, 48]", + "fallback": "smallest development transformed-watermarked detected count, ties toward smaller budget", + "freeze": "selected budget recorded in freeze-record appendix before any confirmation scoring" + }, + "primary_endpoint": { + "target": "confirmation transformed watermarked detected count == 0 out of 64", + "interpretation_if_met": "narrow exploratory detector-specific target met for this detector, keys, and input path only" + }, + "secondary_endpoints": { + "pristine_watermarked_detected_min": 60, + "transformed_unwatermarked_detected_max": 0, + "gate_pass_fraction_required": 1.0 + }, + "artifacts": [ + "specs/fuckmark-budget-scaled-coverage-effectiveness-v1.freeze.json", + "artifacts/budget-scaled/dev-corpus-410000.json", + "artifacts/budget-scaled/dev-plan-{budget}.json", + "artifacts/budget-scaled/dev-evidence-{budget}.json", + "artifacts/budget-scaled/confirmation-corpus-420000.json", + "artifacts/budget-scaled/confirmation-plan.json", + "artifacts/budget-scaled/confirmation-evidence.json", + "artifacts/budget-scaled/robustness-report.json", + "artifacts/budget-scaled/rendering-manifest.json", + "artifacts/budget-scaled/final-report.md" + ] +} diff --git a/specs/fuckmark-budget-scaled-coverage-effectiveness-v1.freeze.json b/specs/fuckmark-budget-scaled-coverage-effectiveness-v1.freeze.json new file mode 100644 index 00000000..f512a898 --- /dev/null +++ b/specs/fuckmark-budget-scaled-coverage-effectiveness-v1.freeze.json @@ -0,0 +1,99 @@ +{ + "appendix_confirmation_budget": { + "development_gate_pass_fraction": 1.0, + "evidence_hashes": { + "dev_evidence_401000": "3ab52d1adf9ffaa934ad088dd5b425042e3b5ad440b24eacf16f60574adf59c2", + "dev_evidence_410000": "2d9776f05ccea7e113449dde08394a498b630e61e14f354487b979b5395e8511", + "dev_robustness_pooled": "eacd39a761997d4f66eb214dac3a86a17df9776a189854714dfcb41d9083005a" + }, + "pooled_development_watermarked_sources": 36, + "recorded_at_utc": "2026-08-23T00:30:07.496640+00:00", + "rule_input": { + "16": { + "detected_401000": 0, + "detected_410000": 1, + "pooled": 1 + }, + "24": { + "detected_401000": 0, + "detected_410000": 1, + "pooled": 1 + }, + "32": { + "detected_401000": 0, + "detected_410000": 1, + "pooled": 1 + }, + "48": { + "detected_401000": 0, + "detected_410000": 1, + "pooled": 1 + } + }, + "rule_path": "fallback rule (no eligible budget)", + "selected_confirmation_budget": 16 + }, + "base_main_commit": "16e153fdfa3739a88bca9b1559543a10ded2d859", + "contract_sha256": "c878ff9fdbf4c412361f6058a72af1fa8c925e78997c89825d255c8d437f598f", + "freeze_record_version": "budget-scaled-coverage-freeze-v1", + "frozen_at_utc": "2026-08-23T00:05:32.588787+00:00", + "pre_score_amendments": [ + { + "amendment_id": "pre-score-amendment-1", + "change": { + "confirmation_corpus_seed_base": { + "from": 403000, + "to": 420000 + }, + "development_second_corpus_seed_base": { + "from": 402000, + "to": 410000 + } + }, + "justification": "pair seed stride is 32; corpora at 402000/403000 would have overlapping seed ranges (dev spans ~4200 seeds, confirmation ~5250), and shared calibration prompt texts at equal seeds could reproduce identical texts across corpora; bases 410000 and 420000 are disjoint from each other and from the canonical 401000 corpus (which ends near 404300)", + "reason": "seed-range overlap arithmetic hazard found during implementation, before any scoring", + "recorded_at_utc": "2026-08-23T00:12:24.926060+00:00" + }, + { + "amendment_id": "pre-score-amendment-2", + "change": { + "development_canonical_corpus": { + "from": "CI-pinned ee0952 artifact", + "to": "locally generated seed-401000 corpus artifact 42240f72 as development data only" + } + }, + "finding": "generating the canonical seed-401000 corpus with the pinned model revision, pinned torch 2.13.0+cpu, and the pinned transformers git commit produced artifact_hash 42240f72e17df1a048f44197f216a0b5f30128c26e2da93afbe92820946e4561 and file sha256 57b580c8839ea85a650993e302443fec4e16f76ea4d1ff7156fcb5468017f7ce instead of the CI pin ee09527d06abc58e612f7157b69908d5abe06101865714ce13a1287d1f0d7664", + "interpretation": "generation is platform-sensitive (CPU float nondeterminism across operating systems); detector scoring arithmetic over fixed token sequences is platform-stable; thresholds are computed per corpus from that same corpus calibration split, so all within-cycle comparisons remain internally consistent", + "reason": "canonical corpus pin is not byte-reproducible on the local Windows CPU runtime", + "recorded_at_utc": "2026-08-23T00:14:34.573438+00:00" + }, + { + "amendment_id": "pre-score-amendment-3", + "change": { + "word_edit_rate_max": { + "from": 0.15, + "to": 0.3 + } + }, + "finding": "a contraction rewrite such as \"are not\" to \"aren't\" costs two word-level edit operations (substitute plus delete), so even a semantically faithful, rules-only transformation of a short source reaches a word edit rate near 0.27; the original 0.15 gate would fail rows for reasons unrelated to the research question", + "reason": "word edit rate gate was set below the mechanical floor of the ruleset before any scoring", + "recorded_at_utc": "2026-08-23T00:16:07.299567+00:00" + }, + { + "amendment_id": "pre-score-amendment-4", + "change": "gates now bind to characters INTRODUCED by the transformation (present in transformed, absent from source alphabet): introduced characters must be ASCII and must include no Cc, Cf, Mn, Mc, Me, Zl, Zp, or non-space Zs codepoint; pre-existing source codepoints are reported without gating", + "finding": "the audit gated on absolute presence of category Cc characters in transformed text, which flags ordinary newlines (U+000A) that the GPT-2 source generation already contains; 28 of 32 development rows failed a gate that measures the corpus, not the candidate", + "reason": "codepoint audit scope error found on development robustness evidence before confirmation scoring", + "recorded_at_utc": "2026-08-23T00:24:31.440654+00:00" + }, + { + "amendment_id": "pre-score-amendment-5", + "change": "normalization gates now bind to introduced instability only: for each normalization form, source stability implies transformed stability; pre-existing source instability is reported without gating", + "finding": "2 of 128 confirmation robustness rows failed NFD/NFKD no-op checks because the GPT-2 source text itself contains precomposed accented characters that decompose under NFD; the candidate introduced only ASCII characters and cannot have caused the instability", + "reason": "normalization gate inherited the same absolute-text scope error corrected for codepoints in amendment 4", + "recorded_at_utc": "2026-08-23T00:49:12.978531+00:00" + } + ], + "protocol_id": "budget-scaled-coverage-effectiveness-v1", + "protocol_sha256": "4fa68c7519a2064c2bf131393598c5b181b92f2ae5fd62f9fd1959b7ebea0367" +} diff --git a/specs/fuckmark-budget-scaled-coverage-effectiveness-v1.protocol.md b/specs/fuckmark-budget-scaled-coverage-effectiveness-v1.protocol.md new file mode 100644 index 00000000..59ee92fd --- /dev/null +++ b/specs/fuckmark-budget-scaled-coverage-effectiveness-v1.protocol.md @@ -0,0 +1,140 @@ +# Frozen Research Protocol: Budget-Scaled Key-Blind Coverage Effectiveness (v1) + +Protocol ID: `budget-scaled-coverage-effectiveness-v1` +Frozen before any detector scoring for this cycle. See the machine-readable contract in +`specs/fuckmark-budget-scaled-coverage-effectiveness-v1.contract.json` and the hash-binding +freeze record in `specs/fuckmark-budget-scaled-coverage-effectiveness-v1.freeze.json`. + +## 1. Objective and claim boundary + +Level A (this protocol): for the fixed, documented open detector and the existing uncleaned +text-only input path, measure whether a budget-scaled, detector-blind, key-blind transform +profile lowers watermarked held-out detection on a new source-disjoint confirmation corpus while +preserving all pre-registered fidelity gates. The stretch exploratory target is 0 detected +watermarked sources out of at least 64. A 0/64 result is a narrow detector-specific measurement +under the documented fixed input path; it is not watermark removal, universal undetectability, +immunity to input cleaning, or safety against unknown keys or proprietary detectors. + +Level B (product/release admissibility): this candidate family intentionally edits visible text +(spacing and contractions), so it is NOT a product candidate this cycle regardless of Level A +outcome. No candidate output will be added to default, development-release, or CLI registries. + +## 2. Candidate mechanism + +Profile `key-blind-full-pool-coverage-v1`: the frozen development ruleset already used by the +B16 line (ruleset hash bound in artifacts; forward contractions, surface spacing, lexical, and +syntax rules), the COVERAGE_GREEDY_KEY_BLIND scheduler over public-tokenizer n-gram disruption +geometry (ngram_len 5), one replicate, and schedule seeds derived from the frozen seed base plus +sorted source index. The only new degree of freedom is the requested budget, swept on +development data and then frozen by the pre-registered rule in section 7. Selection reads the +public tokenizer and public detector n-gram length only; it never reads detector scores, +decisions, keys beyond the public development keys, or confirmation results. + +## 3. Data splits and source disjointness + +- Development: the canonical pinned TinyDev corpus (seed base 401000, corpus artifact hash + recorded at generation time) plus one fresh development corpus at seed base 410000 with 8 + attack pairs per domain (32 watermarked attack sources). Pooled development watermarked + attack sources: 36. +- Confirmation: one fresh corpus at seed base 420000 with 16 attack pairs per domain + (64 watermarked attack sources plus 64 unwatermarked attack controls), generated only after + the protocol, contract, and freeze record exist, and scored exactly once. +- Every corpus keeps the frozen 25 threshold-calibration pairs per domain (100 negatives). +- Disjointness gates: no text SHA-256 may appear in more than one corpus used by this protocol; + confirmation text hashes must be disjoint from all development text hashes; recorded + automatically at scoring time. Neither corpus reuses any example from the archived B10 or + B16 holdout corpora; disjointness from those corpora is asserted by artifact-hash inequality + and text-hash disjointness against any locally available recorded holdout text hashes. + +## 4. Generation and detector configuration + +- Model: `openai-community/gpt2` at revision `607a30d783dfa663caf39e06633721c8d4cfcd7e`, + CPU, temperature 0.8, top-k 50, top-p 0.95, 64 target continuation tokens, max 16 attempts + per pair, seeds derived from the corpus seed base by the existing frozen seed policy. +- Watermarking: Hugging Face SynthID text integration with the public development keys + (654, 400, 836, 123, 340, 443, 597, 160, 57), ngram_len 5, context history 1024, sampling + table seed 0, size 65536, debug off. +- Detector: the Hugging Face SynthID weighted-mean open detector over the text-only + re-tokenized track, ngram_len 5, public development keys (the detector is the open research + harness detector; the keys are public development keys, not provider secrets). +- Threshold: text-only calibration on the 100 unwatermarked calibration negatives of the SAME + corpus being scored, target FPR 0.01, decision rule score >= threshold. Threshold value and + identity hashes are bound inside every scoring artifact. The confirmation threshold is + computed only from the confirmation corpus calibration split. + +## 5. Controls + +All four paired controls are scored for every source: pristine watermarked, transformed +watermarked, pristine unwatermarked, transformed unwatermarked. Summaries report raw counts, +rates, mean scores, mean per-source score change, and false positives, overall and per domain. + +## 6. Fidelity, representation, and robustness gates + +Gates evaluated on every plan row, development and confirmation: + +1. Hard invariants: status PASS (enforced at plan build; any failure aborts). +2. Protected spans: zero violations under the symmetric protected-span counter. +3. Word edit rate: word-level edit distance divided by source word count must be <= 0.30. +4. Introduced-codepoint gate: every character the transformation introduces (present in the + transformed text but absent from the source alphabet) must be ASCII; pre-existing source + characters such as newlines are corpus properties and are reported, not gated. +5. Codepoint audit: the transformation must introduce no Unicode Cf, Cc, Mn, Mc, Me, Zl, Zp, + or Zs characters other than U+0020. Pre-existing forbidden-category characters in the + source are reported separately and do not gate the candidate. +6. Normalization stability: the transformation must not introduce normalization instability; + for each of NFC, NFD, NFKC, NFKD, if the source text is stable under the form then the + transformed text must be stable under it too. Sources that are themselves normalization + unstable (for example precomposed accented characters under NFD) are corpus properties, + reported without gating the candidate. Unicode Cf stripping must not alter any + candidate-introduced character; candidates introduce ASCII only, so this holds by + construction and is verified per row. +7. Copy/DOM: transformed text must contain no control or format characters, so DOM + textContent equals the transformed string; verified by the codepoint audit and archived + fixed-browser renders. +8. Fixed-browser rendering: every confirmation watermarked source/transform pair is rendered + side by side in a fixed browser and screenshotted; the rendered DOM text must equal the + plan transformed text. Screenshots are archived; the deterministic gates above carry the + decision weight. + +## 7. Pre-registered budget selection rule (frozen) + +Development sweep budgets: 16, 24, 32, 48. For each budget, compute on the pooled 36 +development watermarked attack sources: transformed detected count at the development +thresholds, and gate status for every development row. A budget is eligible when its +development transformed-watermarked detected count is 0 and all gates pass on all development +rows at that budget. The confirmation budget is the smallest eligible budget. If no budget is +eligible, the confirmation budget is the budget with the smallest development +transformed-watermarked detected count (ties broken toward the smaller budget); in that case +the primary target is expected to fail and the confirmation run proceeds and reports honestly. +The rule is fully determined before any confirmation scoring; no post-hoc budget change is +permitted. No confirmation score may be computed before the confirmation budget is recorded in +the freeze record appendix. + +## 8. Success and falsification criteria + +Primary (Level A stretch target): transformed watermarked detected count == 0 out of 64 +confirmation watermarked sources at the frozen confirmation threshold. + +Mandatory secondary gates on the confirmation corpus: pristine watermarked detected >= 60/64; +transformed unwatermarked detected == 0/64; every gate in section 6 passes on 100% of +confirmation rows; all four controls' artifacts replay and hash-bind. + +Adverse results that must be reported, never hidden: any residual transformed-watermarked +detection; any gate failure; any normalization or Cf-stripping non-identity; any pristine +watermarked source below threshold (weak watermark); any false positive; per-domain or +per-length heterogeneity. + +Permitted conclusion language is exactly: detector-specific detection reduction under the +documented fixed open detector, public development keys, and uncleaned text-only input path. +Prohibited: watermark removal, undetectability, untraceability, robustness against cleaning, +unknown-key or proprietary-detector claims, release readiness. + +## 9. Reproducibility and artifacts + +Canonical JSON artifacts with SHA-256 hashes: corpora (+ provenance), plans (+ provenance), +scoring evidence (+ provenance), robustness report, rendering archive manifest, final report, +and the freeze record appendix recording the rule-selected confirmation budget before +confirmation scoring. All artifacts bind the source code commit, corpus artifact hash, +plan hash, profile hash, ruleset hash, detector identity hash, calibration bundle hash, and +threshold hash. Full test suite, `git diff --check`, and `git status --short --branch` run at +the end. No pushes, PRs, merges, or releases in this cycle. diff --git a/specs/fuckmark-content-region-coverage-v3.contract.json b/specs/fuckmark-content-region-coverage-v3.contract.json new file mode 100644 index 00000000..a4804069 --- /dev/null +++ b/specs/fuckmark-content-region-coverage-v3.contract.json @@ -0,0 +1,85 @@ +{ + "algorithm_version": "content-region-coverage-contract-v3", + "base_main_commit": "16e153fdfa3739a88bca9b1559543a10ded2d859", + "blind_human_fidelity_evidence": "NOT_PROVIDED", + "candidate": { + "budget": 16, + "new_rule": "surface-space-after-any-word (general word spacing, no word list)", + "profile_hash": "5f4d417a13c2d222c68f70767aef8ed11d7a71c8a5ab9f9c0c21eb278977063c", + "profile_id": "content-region-coverage-v1", + "ruleset_hash": "82011e6dd7048a97e07918d32f7f8670e7723c66eb1b5b0ac859adb4e9cde8ca", + "schedule_policy": "COVERAGE_GREEDY_KEY_BLIND", + "schedule_seed_base": 1150000 + }, + "claim_boundary": { + "cross_platform_generation_reproducibility": "NOT_CLAIMED", + "level_b_product_admissible": false, + "release_authorized": false, + "status": "EXPLORATORY_FIXED_OPEN_DETECTOR_ONLY", + "undetectability_claim": false, + "watermark_removal_claim": false + }, + "confirmation_gates": { + "corpora": 3, + "fidelity_rows_required_per_arm": 128, + "pristine_watermarked_min": 60, + "stretch_target": "0/192 pooled transformed watermarked detections", + "transformed_unwatermarked_max": 0 + }, + "dev_arms": [ + "A_cycle2_baseline", + "B_cycle3_candidate", + "C_general_only", + "D_combined_ablation" + ], + "environment": { + "machine": "AMD64", + "model": "openai-community/gpt2@607a30d783dfa663caf39e06633721c8d4cfcd7e", + "os": "Windows-10-10.0.26220-SP0", + "processor": "AuthenticAMD Family 25 Model 68 Stepping 1", + "python": "3.11.15", + "torch": "2.13.0+cpu", + "transformers": "5.16.0.dev0 (pinned git revision a61d5f9e4fc184cff66938ff6c521cc358b5e024)" + }, + "measurement": { + "audit_negatives": 256, + "calibration_negatives": 1024, + "calibration_seed_base": 500000, + "comparison": ">=", + "identity": "open-detector-measurement-stability-v1", + "order_statistic": "n - floor(target_fpr*n) + 1 ascending", + "target_fpr": 0.01, + "threshold_frozen_before_confirmation_scoring": true + }, + "outcome_classes": [ + "STRONG_SUCCESS", + "PARTIAL_SUCCESS", + "NEUTRAL", + "REJECTED" + ], + "promotion_gate": { + "detector_tolerance_detected_difference": 2, + "low_coverage_mean_gain_required": 0.05, + "low_coverage_subset_size": 8, + "recorded_before_confirmation_scoring": true + }, + "protocol_id": "content-region-coverage-v3", + "recorded_date": "2026-08-23", + "research_branch_base": "4ddd84fa9bc184b1159946c56f8a370111d20f22", + "seed_ledger": { + "consumed": [ + "401000-404296", + "410000-414192", + "420000-425216", + "430000-434192", + "440000-445216", + "500000-520480" + ], + "cycle3_confirmations": [ + "460000", + "470000", + "480000" + ], + "cycle3_dev": "450000" + } +} diff --git a/specs/fuckmark-content-region-coverage-v3.freeze.json b/specs/fuckmark-content-region-coverage-v3.freeze.json new file mode 100644 index 00000000..c95ed1da --- /dev/null +++ b/specs/fuckmark-content-region-coverage-v3.freeze.json @@ -0,0 +1,134 @@ +{ + "appendices": { + "confirmation_corpus_freeze": { + "corpora": { + "460000": { + "artifact_hash": "aa4b0fab8589e2830218b93a75a1f3cf6ca1ded01a505f7fb084d27843a57ab1", + "file_sha256": "ae7f847af1b4b22a0206483989480eb8925ad69a5533eed56b9b87107e87b55d", + "sample_count": 328, + "seed_span": [ + 460000, + 465216 + ] + }, + "470000": { + "artifact_hash": "381bb01ad2ce1a01a5f970bcded8c09fd22da784eba148fda208b9efe4e94974", + "file_sha256": "222c8eeb8ef88463a51c6802b4fd41c2b7bc7ffc8ff04b696fc3e4e036b9bff7", + "sample_count": 328, + "seed_span": [ + 470000, + 475216 + ] + }, + "480000": { + "artifact_hash": "60604bda9a6b7589a99b40d528ae967948a030f0d777d6554ef836bfab6cd701", + "file_sha256": "226185d0a73720189933612f2ddad3e8853f3cb89d3dfa5dcaf113351b45ef4e", + "sample_count": 328, + "seed_span": [ + 480000, + 485216 + ] + } + }, + "note": "all three confirmation corpora hashed and frozen before any cycle-3 confirmation detector score", + "recorded_at_utc": "2026-08-23T03:51:31.062376+00:00" + }, + "promotion_decision": { + "ablation_note": "arm C (general rule plus punctuation only) achieves the same mean achievable coverage as arm B; the cycle-2 word list in arm D adds candidates but no additional coverage; the general rule is the operative mechanism", + "confirmation_plan": { + "arms": [ + "A_cycle2_baseline", + "B_content_region" + ], + "corpora_seed_bases": [ + 460000, + 470000, + 480000 + ], + "threshold": "fixed open-detector-measurement-stability-v1 artifact, frozen before any confirmation score" + }, + "decision": "PROMOTE_CONTENT_REGION_CANDIDATE_TO_CONFIRMATION", + "dev_arm_summary": { + "evidence": { + "A": { + "artifact_hash": "9220f05c8d60000e7854ca967f3565ea966fe49824fa2852b6d647957551ae73", + "detected": 1, + "mean_score_drop": 0.10520081825713615 + }, + "B": { + "artifact_hash": "1fa95328246858bf49b6ae8a6ce0a4c5bdf97fc0c30dfdec6c89b7bea2930089", + "detected": 1, + "mean_score_drop": 0.1138549714227548 + }, + "C": { + "artifact_hash": "92b0e20199be75aa3a18fba21b0134764625ff95de1f99298ebf6d4332f26c90", + "detected": 1, + "mean_score_drop": 0.11398387642298335 + }, + "D": { + "artifact_hash": "a3a420a98ba552f5d8016465b3398a210b4fbb9eee5c3fd1ee7e76f4fe6b8d94", + "detected": 1, + "mean_score_drop": 0.11056383858328153 + } + }, + "structure": { + "A": { + "artifact_hash": "ab547b487cf37633eeaf0abdf151ed213bcd132ebe29912032b2d37b5e91bdc0", + "mean_achievable_coverage": 0.8405455508474576, + "mean_pool": 21.625, + "zero_candidate_sources": 0 + }, + "B": { + "artifact_hash": "308ccaa1153149b302593d33208db69968176c8948e7149db0e3a847f144feb4", + "mean_achievable_coverage": 0.9411458333333332, + "mean_pool": 45.0, + "zero_candidate_sources": 0 + }, + "C": { + "artifact_hash": "66c16e2a82b22a2e422d1e38a03d7c508b0a8111506ee272908fe67e17ceff8b", + "mean_achievable_coverage": 0.9411458333333332, + "mean_pool": 44.71875, + "zero_candidate_sources": 0 + }, + "D": { + "artifact_hash": "f533484077270016da9609ffe14d614f7249f83b0b19efb98814894fbe4ddd49", + "mean_achievable_coverage": 0.9411458333333332, + "mean_pool": 61.90625, + "zero_candidate_sources": 0 + } + } + }, + "gate_results": { + "cf_strip_identity": true, + "hard_invariants_all_rows": true, + "historical_hashes_unchanged": true, + "introduced_hidden_codepoints": 0, + "introduced_normalization_instability": 0, + "low_coverage_gate_required": 0.05, + "low_coverage_mean_gain": 0.2, + "max_word_edit_rate": 0.1509, + "paired_detector_B_minus_A_detected_difference": 0, + "protected_span_violations": 0, + "robustness_gate_pass_fraction": 1.0, + "tolerance": 2, + "zero_candidate_rate": 0 + }, + "recorded_at_utc": "2026-08-23T03:09:39.090311+00:00" + } + }, + "contract_sha256": "072c360e57abda8fabfafbc23654324a1dfb89efe585c367a0a745aa8474c7b3", + "freeze_record_version": "content-region-coverage-v3-freeze-v1", + "frozen_at_utc": "2026-08-23T02:55:06.234907+00:00", + "pre_score_amendments": [ + { + "amendment_id": "pre-score-amendment-1", + "commit": "0e9429d1fe17d9fc4c7f000812272a8e4e59d53f", + "finding": "plan A on corpus 480000 failed fail-closed with PROTECTED_CONTENT_CHANGED: the currency protected-span regex consumed preceding horizontal whitespace whenever a double space preceded a currency amount, so an extra space inserted before \"$100\" changed the extracted span text from \"$100\" to \" $100\"", + "fix": "currency extraction now requires a sign for the sign-whitespace branch, making span text invariant under preceding whitespace; signed forms still extract; regression tests added; no frozen ruleset or profile hash changed (extractor fix only); the fail-closed plan builder behaved correctly", + "kind": "mechanical_correction", + "recorded_at_utc": "2026-08-23T04:09:49.020302+00:00" + } + ], + "protocol_id": "content-region-coverage-v3", + "protocol_sha256": "ea0de812869d60f7a5cc8e765eb0c5a68db0c61c8722b2347a64752e12fb24a2" +} diff --git a/specs/fuckmark-content-region-coverage-v3.protocol.md b/specs/fuckmark-content-region-coverage-v3.protocol.md new file mode 100644 index 00000000..08df9e41 --- /dev/null +++ b/specs/fuckmark-content-region-coverage-v3.protocol.md @@ -0,0 +1,110 @@ +# Frozen Research Protocol: Content-Region Coverage (Cycle 3) + +Protocol ID: `content-region-coverage-v3`. Frozen before any Cycle-3 detector scoring. +Freeze record: `specs/fuckmark-content-region-coverage-v3.freeze.json`. + +## 1. Objective and claim boundary + +Test whether a general any-word spacing candidate family (a) closes the content-region +candidate holes diagnosed in Cycles 1-2, (b) beats the Cycle-2 coverage-completion ruleset in +a paired design on fresh corpora, and (c) holds across three independent confirmation corpora +under a fixed corpus-independent threshold. Claim boundary unchanged: fixed open HF SynthID +weighted-mean detector, public development keys, uncleaned text-only path; no removal, +undetectability, unknown-key, proprietary-detector, or cleaning claims; Level B not pursued. + +## 2. Candidate mechanism + +Profile `content-region-coverage-v1` (ruleset hash recorded in artifacts): forward +contractions, the six punctuation spacing rules, ONE new `GeneralWordSpacingRule` +(`surface-space-after-any-word`, pattern `(?= +threshold; float64, no tie rounding. The threshold artifact freezes detector identity, +corpus hash, counts, realized audit FPR with exact 95% binomial interval, and environment +identity. ALL Cycle-3 confirmation scoring uses this fixed threshold; per-corpus calibration +thresholds are recorded alongside for comparison only. Threshold frozen before any +confirmation score. Dev scoring (both arms on the same corpus) may additionally report +per-corpus thresholds; paired comparisons use one threshold within each corpus. + +## 4. Seed-range ledger (recovered from artifacts) + +Consumed: 401000-404296, 410000-414192, 420000-425216, 430000-434192, 440000-445216, +calibration 500000-520480 (span 160 prompts x 8 slots x stride 16). Cycle 3 uses: dev +450000 (v3, 8 attack pairs/domain, spans about 450000-454224), confirmations 460000, +470000, 480000 (v3, 16 attack pairs/domain, each spanning about 5264 seeds). All ranges are +pairwise disjoint from every consumed range and from each other. + +## 5. Development design + +Fresh dev corpus 450000 (32 watermarked attack sources). Four plan arms per source, all +detector-blind before scoring: + +A. Cycle-2 coverage-completion ruleset (paired baseline) +B. Cycle-3 content-region profile +C. General-spacing-only arm (general rule + punctuation only, no contractions/lexical/syntax) +D. Combined arm (Cycle-2 word list + general rule) + +Primary structural endpoints (no detector data): candidate count, achievable coverage, +zero-candidate rate, longest uncovered run, uncovered window count, per-source paired +differences B minus A. Detector outcomes are evaluation-only. + +## 6. Promotion gate (all required before any confirmation scoring) + +1-8: historical hashes unchanged; suite green except documented environmental failures; hard +invariants pass on every dev row; protected-span violations 0; introduced hidden/control +codepoints 0; introduced normalization instability 0; Cf-strip identity holds; no source +becomes empty or corrupted. 9: B raises achievable coverage materially in the A-low-coverage +subset (mean paired gain >= 0.05 among the 8 lowest-A-coverage sources). 10: zero-candidate +rate not above baseline. 11: fidelity metrics not materially regressed (word edit rate, +protected spans, invariants). 12: paired detector result for B not clearly worse than A on +dev (detected count difference <= 2 on 32 sources). The confirmation decision (proceed or +reject) is recorded in the freeze record appendix before confirmation scoring. + +## 7. Confirmation design + +Three corpora frozen before the first confirmation score (hashes recorded in the appendix +after generation, before scoring): 460000, 470000, 480000; each 64 watermarked + 64 +unwatermarked attack sources. For each corpus, two frozen plans (A baseline, B candidate) +built detector-blind, then scored once each with the fixed threshold. Control gates per +corpus: pristine watermarked >= 60/64 (unconditional denominator primary; conditional count +reported); transformed unwatermarked == 0/64; fidelity/representation gates 128/128 per arm; +text-hash overlap 0 against every prior corpus with the pre-registered collision policy: +any naturally occurring collision is reported and followed by scoring the corpus as +generated; no regeneration or substitution after seeing outcomes. + +## 8. Outcome classes (pre-registered) + +STRONG_SUCCESS: all control gates pass on all three corpora AND pooled B residuals strictly +fewer than pooled A residuals AND B no worse than A on every corpus; stretch target 0/192. +PARTIAL_SUCCESS: B paired-better overall but residuals remain, or exactly one non-primary +gate fails. NEUTRAL: B not better than A pooled. REJECTED: B worse, false positives appear, +fidelity regresses, or the structural hypothesis fails on dev. Post-scoring margin analysis +(±0.005/±0.010/±0.025 bands) is diagnosis only. + +## 9. Fidelity and reporting + +All Cycle-1/2 gates retained. For the general-rule family report separately: number of +spacing insertions, number of sources receiving them, maximum and mean per source, exact +changed tokens, rule identities; word-level substitutions from other families reported as in +prior cycles. Blind human fidelity evidence: NOT_PROVIDED (no fabricated judgments). + +## 10. Reproducibility + +Environment manifest bound in the freeze record (Windows-10-10.0.26220 AMD64, Python +3.11.15, torch 2.13.0+cpu, transformers 5.16.0.dev0 at the pinned git revision, GPT-2 +revision 607a30d783dfa663caf39e06633721c8d4cfcd7e). Cross-platform corpus generation is NOT +claimed byte-reproducible (documented Cycle-1 finding); plans, scoring, and thresholds are +deterministic given texts and the pinned runtime. All artifacts content-addressed; amendments +only before affected scores, timestamped and hash-bound; local commits only, nothing pushed. diff --git a/specs/fuckmark-coverage-completion-v2.contract.json b/specs/fuckmark-coverage-completion-v2.contract.json new file mode 100644 index 00000000..500689f4 --- /dev/null +++ b/specs/fuckmark-coverage-completion-v2.contract.json @@ -0,0 +1,136 @@ +{ + "algorithm_version": "coverage-completion-contract-v2", + "base_main_commit": "16e153fdfa3739a88bca9b1559543a10ded2d859", + "candidate_profile": { + "budget": 16, + "budget_justification": "cycle-1 coverage saturation below 16 on every source; no sweep needed", + "geometry_mode": "TOKENIZER_AWARE_PUBLIC", + "ngram_len": 5, + "profile_id": "key-blind-coverage-completion-v2", + "replicate_count": 1, + "ruleset": "cycle-1 development ruleset plus 65 frozen extension surface words", + "schedule_policy": "COVERAGE_GREEDY_KEY_BLIND", + "schedule_seed_base": 1140000 + }, + "claim_boundary": { + "level_b_product_admissible": false, + "release_authorized": false, + "status": "EXPLORATORY_FIXED_OPEN_DETECTOR_ONLY", + "undetectability_claim": false, + "watermark_removal_claim": false + }, + "corpora": { + "confirmation": { + "attack_pairs_per_domain": 16, + "profile": "tiny-dev-v3", + "seed_base": 440000 + }, + "development": [ + { + "attack_pairs_per_domain": 8, + "profile": "tiny-dev-v3", + "seed_base": 430000 + }, + { + "attack_pairs_per_domain": 8, + "note": "repeat use from cycle 1, labeled", + "profile": "tiny-dev-v3", + "seed_base": 410000 + } + ], + "forbidden": "cycle-1 confirmation corpus at seed base 420000 may not be scored in cycle 2" + }, + "cycle1_reference": { + "confirmation_residual_detected": 11, + "confirmation_watermarked_sources": 64, + "mean_achievable_coverage_all": 0.8187, + "mean_achievable_coverage_detected": 0.6818, + "mean_achievable_coverage_escaped": 0.8469 + }, + "detector": { + "comparison_operator": ">=", + "family": "Hugging Face SynthID Weighted Mean", + "keys": "public development keys", + "ngram_len": 5, + "target_fpr": 0.01 + }, + "endpoints": { + "controls": { + "gate_pass_fraction_required": 1.0, + "pristine_watermarked_detected_min": 60, + "transformed_unwatermarked_detected_max": 0 + }, + "improvement": "detected count < 11", + "mechanistic": "mean achievable coverage >= 0.80 on confirmation watermarked sources", + "primary": "confirmation transformed watermarked detected == 0/64" + }, + "extension_words": [ + "an", + "he", + "she", + "his", + "her", + "its", + "our", + "their", + "them", + "him", + "us", + "me", + "my", + "your", + "if", + "when", + "where", + "which", + "who", + "what", + "how", + "why", + "because", + "so", + "than", + "then", + "there", + "here", + "all", + "any", + "each", + "more", + "most", + "some", + "such", + "only", + "very", + "just", + "also", + "into", + "about", + "over", + "under", + "after", + "before", + "between", + "during", + "through", + "without", + "while", + "do", + "does", + "been", + "being", + "would", + "could", + "should", + "may", + "might", + "must", + "were", + "one", + "even", + "still", + "often" + ], + "protocol_id": "coverage-completion-v2", + "recorded_date": "2026-08-23" +} diff --git a/specs/fuckmark-coverage-completion-v2.freeze.json b/specs/fuckmark-coverage-completion-v2.freeze.json new file mode 100644 index 00000000..e06ddbb6 --- /dev/null +++ b/specs/fuckmark-coverage-completion-v2.freeze.json @@ -0,0 +1,26 @@ +{ + "appendix_confirmation_budget": 16, + "appendix_development_outcome": { + "cycle1_ruleset_paired_baseline_on_430000": 4, + "cycle2_dev_detected": { + "410000": 0, + "430000": 5, + "pooled_of_64": 5 + }, + "evidence_hashes": { + "dev1_paired_baseline_430000": "7c7f9624e2a95dfb16fac5b17dac7778bbe7d87be6af988c2412f09aadb26e9e", + "dev2_evidence_410000": "2eda8a0d1c3599e35df38d6779f0ed395e56a13fc958c31f7cf3d689a6b736db", + "dev2_evidence_430000": "cf46b933ce3b7ae17a893432d4230db9b7b4f04bae8905844e30f74ad2f9a49a" + }, + "interpretation": "pooled dev detection identical between rulesets (5/64 each); the extension grew candidate pools but residuals concentrate in sources with high pristine scores and content-word-dense low-candidate regions; confirmation proceeds as pre-registered and may return a neutral result", + "phase": "development outcome recorded before confirmation scoring", + "recorded_at_utc": "2026-08-23T01:39:54.423362+00:00" + }, + "base_main_commit": "16e153fdfa3739a88bca9b1559543a10ded2d859", + "contract_sha256": "20ba6145793a89f0aa5cbc530305d9345731a60d60ad906199b323c74942d825", + "freeze_record_version": "coverage-completion-v2-freeze-v1", + "frozen_at_utc": "2026-08-23T01:24:49.558549+00:00", + "pre_score_amendments": [], + "protocol_id": "coverage-completion-v2", + "protocol_sha256": "65856295a595acaa404392239ccd9db9fa43d3f45a86ae61ee0d88b0d1e0851a" +} diff --git a/specs/fuckmark-coverage-completion-v2.protocol.md b/specs/fuckmark-coverage-completion-v2.protocol.md new file mode 100644 index 00000000..142c1cd0 --- /dev/null +++ b/specs/fuckmark-coverage-completion-v2.protocol.md @@ -0,0 +1,85 @@ +# Frozen Research Protocol: Key-Blind Coverage Completion (v2) + +Protocol ID: `coverage-completion-v2` +Cycle 2 of the budget-scaled effectiveness line. Frozen before any cycle-2 scoring. Hash-binding +freeze record: `specs/fuckmark-coverage-completion-v2.freeze.json`. + +## 1. Objective and claim boundary + +Level A: for the same fixed open detector and uncleaned text-only input path as cycle 1, test +whether a coverage-completion ruleset - the cycle-1 development ruleset plus an extended +surface spacing word list chosen from standard general-English function words - lowers +watermarked held-out detection on a new source-disjoint 64-source confirmation corpus relative +to cycle 1's 11/64 residual, with stretch target 0/64. Mechanistic endpoint: mean achievable +detector-n-gram coverage of the confirmation watermarked sources must be at least 0.80 (cycle 1 +measured 0.819 overall with 0.682 among residual sources). The claim boundary is identical to +cycle 1: detector-specific detection reduction under the documented fixed condition only; no +removal, undetectability, unknown-key, proprietary-detector, cleaning-durability, or release +claims. Level B is not pursued; the candidate edits visible text. + +## 2. Candidate mechanism + +Profile `key-blind-coverage-completion-v2`: the cycle-1 registry (forward contractions, surface +spacing, lexical, syntax) EXTENDED by 65 additional surface spacing words, frozen below; +COVERAGE_GREEDY_KEY_BLIND scheduling over public-tokenizer n-gram disruption geometry +(ngram_len 5); replicate count 1; schedule seed base 1_140_000 plus sorted source index; +budget fixed at 16 operations (cycle 1 demonstrated coverage saturation below 16 on every +source, so a budget sweep carries no information; this is pre-registered, not tuned). +Selection accesses only public tokenizer geometry and the public detector n-gram length. + +Frozen extension words (65, lowercase alphabetic, length >= 2, none already in the cycle-1 +surface list, chosen from standard general-English function-word rankings, not mined from any +project corpus): + +an, he, she, his, her, its, our, their, them, him, us, me, my, your, if, when, where, which, +who, what, how, why, because, so, than, then, there, here, all, any, each, more, most, some, +such, only, very, just, also, into, about, over, under, after, before, between, during, +through, without, while, do, does, been, being, would, could, should, may, might, must, were, +one, even, still, often + +## 3. Data splits and source disjointness + +- Development: fresh corpus at seed base 430000 (v3, 8 attack pairs per domain, 32 watermarked + attack sources) plus the cycle-1 development corpus at seed base 410000 (repeat use, + labeled; overfitting caveat noted) and the locally regenerated 401000 corpus. No cycle-2 + scoring may read the cycle-1 confirmation corpus at 420000. +- Confirmation: fresh corpus at seed base 440000 (v3, 16 attack pairs per domain, 64 + watermarked attack sources plus 64 unwatermarked controls), generated after this protocol is + frozen and scored exactly once. +- Disjointness: pairwise text-hash overlap zero across 410000, 430000, and 440000 corpora; + degenerate-generation collisions (cycle 1 recorded one against 401000) are checked and + reported; any collision is a reportable deviation. + +## 4. Generation and detector configuration + +Identical to cycle 1 (pinned GPT-2 revision, CPU, temperature 0.8, top-k 50, top-p 0.95, +64 tokens, 16 attempts; HF SynthID weighted mean, ngram_len 5, public development keys, +per-corpus text-only calibration at target FPR 0.01, decision score >= threshold). + +## 5. Controls, gates, robustness + +All four paired controls scored. All cycle-1 section-6 gates apply unchanged (hard invariants, +protected spans zero, word edit rate <= 0.30, no introduced non-ASCII or hidden-category +codepoints, no introduced normalization instability, Cf-strip identity on introduced +characters, fixed-browser DOM text equality with archived screenshot). + +## 6. Pre-registered endpoints and success criteria + +Primary: confirmation transformed-watermarked detected count, target 0/64. +Improvement criterion (secondary, must hold for the cycle to count as progress): detected +count strictly below 11 (cycle 1 residual on an equally constructed corpus). +Mechanistic criterion: mean achievable coverage >= 0.80 on confirmation watermarked sources. +Control criteria: pristine watermarked >= 60/64 detected; transformed unwatermarked == 0/64; +all fidelity gates pass on 100% of rows. +Adverse results (residual detection, any gate failure, any cross-corpus text collision, +pristine-unwatermarked exceedances, zero-candidate sources) are reportable, never hidden. +The frozen budget (16) is recorded in the freeze-record appendix before confirmation scoring; +no post-hoc change. + +## 7. Reproducibility + +Canonical JSON artifacts with SHA-256 hashes; plan binds corpus, manifest, code commit, and +explicit detector_access_observed=False / secret_access_observed=False; evidence binds plan, +profile, ruleset, detector identity, calibration bundle, threshold; robustness report binds +plans and corpora; rendering manifest binds the plan; independent hash cross-check at the end; +full test suite, `git diff --check`, and clean status; local commits only, no pushes. diff --git a/tests/test_budget_scaled_robustness.py b/tests/test_budget_scaled_robustness.py new file mode 100644 index 00000000..8d6b3675 --- /dev/null +++ b/tests/test_budget_scaled_robustness.py @@ -0,0 +1,168 @@ +from __future__ import annotations + +from types import SimpleNamespace + +from fuckmark.corpus import CorpusDomain, CorpusSplit, WatermarkLabel +from fuckmark.experiments.effectiveness_plan import build_key_blind_high_coverage_plan +from fuckmark.hashing import sha256_text +from fuckmark.tiny_dev_budget_scaled_robustness import ( + _cf_stripped, + _forbidden_codepoints, + build_budget_scaled_robustness_report, +) +from fuckmark.transforms import key_blind_full_pool_coverage_profile + + +class FakeTokenizer: + def __call__( + self, + text: str, + *, + add_special_tokens: bool, + return_offsets_mapping: bool = False, + ) -> dict[str, object]: + assert add_special_tokens is False + ids = [index + 1 for index in range(len(text))] + value: dict[str, object] = {"input_ids": ids} + if return_offsets_mapping: + value["offset_mapping"] = [(index, index + 1) for index in range(len(text))] + return value + + +def _sample(sample_id: str, label: WatermarkLabel, text: str, identity_hash: str): + token_ids = tuple(index + 1 for index in range(len(text))) + return SimpleNamespace( + sample_id=sample_id, + split=CorpusSplit.ATTACK_DEVELOPMENT, + label=label, + prompt_family_id=f"prompt-{sample_id}", + domain=CorpusDomain.GENERAL_EXPLANATORY, + text=text, + text_sha256=sha256_text(text), + text_only_tokens=SimpleNamespace(token_ids=token_ids), + model=SimpleNamespace(identity_hash=identity_hash, eos_token_id=50256), + ) + + +def _corpus(text: str, salt: str): + identity_hash = sha256_text("fake-tokenizer") + samples = ( + _sample(f"negative-{salt}", WatermarkLabel.UNWATERMARKED, text, identity_hash), + _sample(f"positive-{salt}", WatermarkLabel.WATERMARKED, text, identity_hash), + ) + return SimpleNamespace( + artifact_hash=sha256_text(f"fake-corpus-{salt}"), + model_identity_hash=identity_hash, + manifest=SimpleNamespace(samples=samples, manifest_hash=sha256_text("fake-manifest")), + ) + + +def test_forbidden_codepoint_audit_detects_hidden_representation() -> None: + clean = "plain ascii text with spaces only" + assert _forbidden_codepoints(clean) == () + zero_width = "hidden" + chr(0x200C) + "join" + combining = "combining" + chr(0x0301) + nbsp = "nbsp" + chr(0x00A0) + soft_hyphen = "soft" + chr(0x00AD) + assert _forbidden_codepoints(zero_width) == ("U+200C(Cf)",) + assert _forbidden_codepoints(combining) == ("U+0301(Mn)",) + assert _forbidden_codepoints(nbsp) == ("U+00A0(Zs)",) + assert _forbidden_codepoints(soft_hyphen) == ("U+00AD(Cf)",) + assert _cf_stripped(zero_width) == "hiddenjoin" + assert _cf_stripped(clean) == clean + + +def test_introduced_codepoint_gate_ignores_preexisting_source_characters() -> None: + from fuckmark.tiny_dev_budget_scaled_robustness import ( + _introduced_forbidden_codepoints, + _introduced_non_ascii, + _robustness_row, + ) + + source = "source with\nnewline and " + chr(0x2019) + "curly quote" + transformed_same = source + " and one more plain ascii word" + assert _introduced_forbidden_codepoints(source, transformed_same) == () + assert _introduced_non_ascii(source, transformed_same) == () + transformed_hidden = transformed_same + chr(0x200B) + assert _introduced_forbidden_codepoints(source, transformed_hidden) == ("U+200B(Cf)",) + transformed_visible_unicode = transformed_same + chr(0x00E9) + assert _introduced_non_ascii(source, transformed_visible_unicode) == ("U+00E9",) + + +def test_normalization_gate_ignores_source_instability_but_flags_introduced_instability() -> None: + from fuckmark.tiny_dev_budget_scaled_robustness import _robustness_row + + stable_source = "plain stable ascii source text" + stable_row = _robustness_row(stable_source, stable_source + " edited") + assert stable_row["gates_pass"] is True + assert stable_row["introduced_normalization_instability"] == () + decomposed_source = "caf" + chr(0x0065) + chr(0x0301) + " source text with several more words here" + inherited_decomposed = _robustness_row(decomposed_source, decomposed_source + " edited") + assert inherited_decomposed["normalization_noops"]["nfc"] is False + assert inherited_decomposed["introduced_normalization_instability"] == () + assert inherited_decomposed["gates_pass"] is True + introduced_unstable = _robustness_row(stable_source, stable_source + " caf" + chr(0x0065) + chr(0x0301)) + assert introduced_unstable["introduced_normalization_instability"] == ("nfc", "nfkc") + assert introduced_unstable["gates_pass"] is False + unstable_source = "caf" + chr(0x00E9) + " source text with several more words here" + inherited_row = _robustness_row(unstable_source, unstable_source + " edited") + assert inherited_row["normalization_noops"]["nfd"] is False + assert inherited_row["introduced_normalization_instability"] == () + assert inherited_row["gates_pass"] is True + + +def test_robustness_report_gates_pass_on_rule_transforms() -> None: + text = "You are not ready, and we do not stop when the system is in use." + corpus = _corpus(text, "1") + plan = build_key_blind_high_coverage_plan( + corpus, + FakeTokenizer(), + profile=key_blind_full_pool_coverage_profile((32,)), + source_code_commit="a" * 40, + ) + report = build_budget_scaled_robustness_report((corpus,), (plan,)) + assert report["summary"]["row_count"] == 2 + assert report["summary"]["gate_pass_fraction"] == 1.0 + assert report["summary"]["all_normalization_stability_preserved"] is True + assert report["summary"]["all_cf_strip_noop"] is True + assert report["summary"]["all_introduced_codepoints_clean"] is True + assert report["summary"]["corpus_pairwise_text_hash_overlaps"] == {} + assert all(row["gates_pass"] for row in report["rows"]) + assert report["artifact_hash"] + + +def test_robustness_report_joins_plans_to_their_own_corpus_by_artifact_hash() -> None: + identity_hash = sha256_text("fake-tokenizer") + first_text = "You are not ready, and we do not stop when the system is in use." + second_text = "A second corpus shares sample IDs but holds different source texts here." + first = _corpus(first_text, "1") + second = _corpus(second_text, "1") + first.artifact_hash = sha256_text("corpus-a") + second.artifact_hash = sha256_text("corpus-b") + for corpus, text in ((first, first_text), (second, second_text)): + for sample in corpus.manifest.samples: + sample.text = text + sample.text_sha256 = sha256_text(text) + plan = build_key_blind_high_coverage_plan( + first, + FakeTokenizer(), + profile=key_blind_full_pool_coverage_profile((16,)), + source_code_commit="a" * 40, + ) + report = build_budget_scaled_robustness_report((first, second), (plan,)) + assert all(row["corpus_artifact_hash"] == sha256_text("corpus-a") for row in report["rows"]) + assert report["summary"]["gate_pass_fraction"] == 1.0 + + +def test_robustness_report_detects_cross_corpus_text_overlap() -> None: + text = "Shared generation text that appears in two corpora at once." + first = _corpus(text, "1") + second = _corpus(text, "2") + plan = build_key_blind_high_coverage_plan( + first, + FakeTokenizer(), + profile=key_blind_full_pool_coverage_profile((16,)), + source_code_commit="a" * 40, + ) + report = build_budget_scaled_robustness_report((first, second), (plan,)) + assert report["summary"]["corpus_pairwise_text_hash_overlaps"] == {"0-1": 1} diff --git a/tests/test_content_region_profile.py b/tests/test_content_region_profile.py new file mode 100644 index 00000000..1c7fa221 --- /dev/null +++ b/tests/test_content_region_profile.py @@ -0,0 +1,154 @@ +from __future__ import annotations + +from types import SimpleNamespace + +import pytest + +from fuckmark.corpus import CorpusDomain, CorpusSplit, WatermarkLabel +from fuckmark.experiments.coverage_holes import classify_word +from fuckmark.experiments.effectiveness_plan import build_key_blind_high_coverage_plan +from fuckmark.hashing import sha256_text +from fuckmark.transforms import ( + CONTENT_REGION_COVERAGE_PROFILE_ID, + CONTENT_REGION_COVERAGE_SEED_BASE, + KEY_BLIND_HIGH_COVERAGE_PROFILE, + content_region_coverage_profile, + content_region_coverage_transform_registry, + resolve_effectiveness_profile, +) +from fuckmark.transforms.rules import GeneralWordSpacingRule + + +class FakeTokenizer: + def __call__( + self, + text: str, + *, + add_special_tokens: bool, + return_offsets_mapping: bool = False, + ) -> dict[str, object]: + assert add_special_tokens is False + ids = [index + 1 for index in range(len(text))] + value: dict[str, object] = {"input_ids": ids} + if return_offsets_mapping: + value["offset_mapping"] = [(index, index + 1) for index in range(len(text))] + return value + + +def _sample(sample_id: str, label: WatermarkLabel, text: str, identity_hash: str): + token_ids = tuple(index + 1 for index in range(len(text))) + return SimpleNamespace( + sample_id=sample_id, + split=CorpusSplit.ATTACK_DEVELOPMENT, + label=label, + prompt_family_id=f"prompt-{sample_id}", + domain=CorpusDomain.GENERAL_EXPLANATORY, + text=text, + text_sha256=sha256_text(text), + text_only_tokens=SimpleNamespace(token_ids=token_ids), + model=SimpleNamespace(identity_hash=identity_hash, eos_token_id=50256), + ) + + +def _corpus(): + identity_hash = sha256_text("fake-tokenizer") + text = ( + "Careful researchers documented measurement uncertainty thoroughly. " + "The auditor verified every reported number 42 before publishing results." + ) + samples = ( + _sample("negative-1", WatermarkLabel.UNWATERMARKED, text, identity_hash), + _sample("positive-1", WatermarkLabel.WATERMARKED, text, identity_hash), + ) + return SimpleNamespace( + artifact_hash=sha256_text("fake-corpus"), + model_identity_hash=identity_hash, + manifest=SimpleNamespace(samples=samples, manifest_hash=sha256_text("fake-manifest")), + ) + + +def test_general_word_spacing_rule_contract() -> None: + rule = GeneralWordSpacingRule.create("surface-space-after-any-word") + text = "Alpha beta gamma delta, epsilon zeta." + matches = [text[m.start() : m.end()] for m in rule.pattern().finditer(text)] + assert "Alpha" in matches + assert "beta" in matches + assert "gamma" in matches + assert "delta" not in matches + assert rule.replacement_for("beta") == "beta " + assert rule.replacement_for("Beta") == "Beta " + doubled = "beta gamma" + assert rule.pattern().search(doubled) is None + with pytest.raises(ValueError): + GeneralWordSpacingRule( + rule_id="bad", + version="general-word-space-after-v1", + family=rule.family, + tier=rule.tier, + source="word", + replacement="wrong ", + whole_word=False, + preserve_simple_case=False, + block_all_caps=False, + rule_hash=rule.rule_hash, + ) + + +def test_general_rule_enumerates_content_words_with_exact_replacement() -> None: + registry = content_region_coverage_transform_registry() + text = "Measurement uncertainty remained stubbornly high throughout replication attempts." + enumeration = registry.enumerate(text) + general = [ + candidate + for candidate in enumeration.candidates + if candidate.rule_id == "surface-space-after-any-word" + ] + assert any(candidate.source_text == "Measurement" for candidate in general) + assert all(candidate.replacement_text == candidate.source_text + " " for candidate in general) + for candidate in general: + transformed = text[: candidate.start] + candidate.replacement_text + text[candidate.end :] + assert transformed.count(candidate.source_text + " ") >= 1 + assert transformed.isascii() + + +def test_content_region_profile_resolves_and_builds_detector_blind_plans() -> None: + profile = resolve_effectiveness_profile(CONTENT_REGION_COVERAGE_PROFILE_ID, (16,)) + assert profile.profile_id == CONTENT_REGION_COVERAGE_PROFILE_ID + assert profile.schedule_seed_base == CONTENT_REGION_COVERAGE_SEED_BASE + assert content_region_coverage_profile((16,)) == profile + corpus = _corpus() + plan = build_key_blind_high_coverage_plan( + corpus, + FakeTokenizer(), + profile=profile, + source_code_commit="a" * 40, + ) + assert plan["ruleset_hash"] == profile.ruleset_hash + assert all(row["hard_invariant_status"] == "pass" for row in plan["variants"]) + assert all(row["detector_access_observed"] is False for row in plan["variants"]) + + +def test_content_region_registry_protects_numbers_and_keeps_b16_frozen() -> None: + registry = content_region_coverage_transform_registry() + text = "The reported value 3.14159 stayed fixed across runs." + enumeration = registry.enumerate(text) + for candidate in enumeration.candidates: + assert "3.14159" not in candidate.source_text + assert KEY_BLIND_HIGH_COVERAGE_PROFILE.profile_hash == ( + "6ad142262bfb11a714565d7bd43daa859657fe817bdafa9c8dcf0f4884c07512" + ) + + +def test_classify_word_categories() -> None: + assert classify_word("the") == "function_word" + assert classify_word("measurement") == "lowercase_content" + assert classify_word("Measurement") == "capitalized_content" + assert classify_word("42") == "numeric" + assert classify_word(",") == "punctuation_or_empty" + + +def test_resolve_rejects_invalid_content_region_requests() -> None: + with pytest.raises(ValueError): + resolve_effectiveness_profile(CONTENT_REGION_COVERAGE_PROFILE_ID) + with pytest.raises(ValueError): + content_region_coverage_profile((48, 16)) diff --git a/tests/test_coverage_completion_profile.py b/tests/test_coverage_completion_profile.py new file mode 100644 index 00000000..fc55a725 --- /dev/null +++ b/tests/test_coverage_completion_profile.py @@ -0,0 +1,139 @@ +from __future__ import annotations + +from types import SimpleNamespace + +import pytest + +from fuckmark.corpus import CorpusDomain, CorpusSplit, WatermarkLabel +from fuckmark.experiments.effectiveness_plan import build_key_blind_high_coverage_plan +from fuckmark.hashing import sha256_text +from fuckmark.transforms import ( + KEY_BLIND_COVERAGE_COMPLETION_PROFILE_ID, + KEY_BLIND_COVERAGE_COMPLETION_SEED_BASE, + KEY_BLIND_HIGH_COVERAGE_PROFILE, + key_blind_coverage_completion_profile, + key_blind_coverage_completion_transform_registry, + key_blind_high_coverage_transform_registry, + resolve_effectiveness_profile, +) +from fuckmark.transforms.surface_rules import ( + coverage_completion_extension_words, + coverage_completion_surface_rules, + development_surface_rules, +) + + +class FakeTokenizer: + def __call__( + self, + text: str, + *, + add_special_tokens: bool, + return_offsets_mapping: bool = False, + ) -> dict[str, object]: + assert add_special_tokens is False + ids = [index + 1 for index in range(len(text))] + value: dict[str, object] = {"input_ids": ids} + if return_offsets_mapping: + value["offset_mapping"] = [(index, index + 1) for index in range(len(text))] + return value + + +def _sample(sample_id: str, label: WatermarkLabel, text: str, identity_hash: str): + token_ids = tuple(index + 1 for index in range(len(text))) + return SimpleNamespace( + sample_id=sample_id, + split=CorpusSplit.ATTACK_DEVELOPMENT, + label=label, + prompt_family_id=f"prompt-{sample_id}", + domain=CorpusDomain.GENERAL_EXPLANATORY, + text=text, + text_sha256=sha256_text(text), + text_only_tokens=SimpleNamespace(token_ids=token_ids), + model=SimpleNamespace(identity_hash=identity_hash, eos_token_id=50256), + ) + + +def _corpus(): + identity_hash = sha256_text("fake-tokenizer") + text = "He said they were not sure, because more work was needed before one can decide." + samples = ( + _sample("negative-1", WatermarkLabel.UNWATERMARKED, text, identity_hash), + _sample("positive-1", WatermarkLabel.WATERMARKED, text, identity_hash), + ) + return SimpleNamespace( + artifact_hash=sha256_text("fake-corpus"), + model_identity_hash=identity_hash, + manifest=SimpleNamespace(samples=samples, manifest_hash=sha256_text("fake-manifest")), + ) + + +def test_extension_words_obey_the_frozen_surface_contract() -> None: + words = coverage_completion_extension_words() + assert len(words) == 65 + assert len(set(words)) == len(words) + base_words = {rule.source for rule in development_surface_rules() if rule.source.isalpha()} + assert not (set(words) & base_words) + assert all(word.isalpha() and word == word.lower() and len(word) >= 2 for word in words) + rules = coverage_completion_surface_rules() + base = development_surface_rules() + assert len(rules) == len(base) + 65 + assert set(rule.rule_id for rule in base) < set(rule.rule_id for rule in rules) + + +def test_extension_rules_do_not_split_words_mid_token() -> None: + registry = key_blind_coverage_completion_transform_registry() + text = "The weather, whether he measured then, was often more random than before." + enumeration = registry.enumerate(text) + replaced_words = set() + for candidate in enumeration.candidates: + replaced_words.add(text[candidate.start : candidate.end].strip()) + assert "he" in replaced_words or "then" in replaced_words + joined = "".join( + candidate.replacement_text + for candidate in enumeration.candidates[:1] + ) + assert "random" not in joined or joined == "random" + transformed_spans = [text[c.start : c.end] for c in enumeration.candidates] + assert all(span.strip().isalpha() or span in (". ", ", ", "; ", ": ", "? ", "! ") for span in transformed_spans) + + +def test_coverage_completion_registry_extends_and_rebinds_ruleset() -> None: + base_registry = key_blind_high_coverage_transform_registry() + registry = key_blind_coverage_completion_transform_registry() + assert registry.ruleset_hash != base_registry.ruleset_hash + base_ids = {rule.rule_id for rule in base_registry.rules} + extended_ids = {rule.rule_id for rule in registry.rules} + assert base_ids < extended_ids + + +def test_frozen_b16_profile_hash_is_unchanged() -> None: + assert ( + KEY_BLIND_HIGH_COVERAGE_PROFILE.profile_hash + == "6ad142262bfb11a714565d7bd43daa859657fe817bdafa9c8dcf0f4884c07512" + ) + + +def test_coverage_completion_profile_resolves_and_builds_plans() -> None: + profile = resolve_effectiveness_profile(KEY_BLIND_COVERAGE_COMPLETION_PROFILE_ID, (16,)) + assert profile.profile_id == KEY_BLIND_COVERAGE_COMPLETION_PROFILE_ID + assert profile.schedule_seed_base == KEY_BLIND_COVERAGE_COMPLETION_SEED_BASE + assert key_blind_coverage_completion_profile((16,)) == profile + corpus = _corpus() + plan = build_key_blind_high_coverage_plan( + corpus, + FakeTokenizer(), + profile=profile, + source_code_commit="a" * 40, + ) + assert plan["ruleset_hash"] == profile.ruleset_hash + assert len(plan["variants"]) == 2 + assert all(row["hard_invariant_status"] == "pass" for row in plan["variants"]) + assert all(row["detector_access_observed"] is False for row in plan["variants"]) + + +def test_resolve_rejects_invalid_coverage_completion_requests() -> None: + with pytest.raises(ValueError): + resolve_effectiveness_profile(KEY_BLIND_COVERAGE_COMPLETION_PROFILE_ID) + with pytest.raises(ValueError): + key_blind_coverage_completion_profile((48, 16)) diff --git a/tests/test_effectiveness_full_pool_profile.py b/tests/test_effectiveness_full_pool_profile.py new file mode 100644 index 00000000..a3fa96c9 --- /dev/null +++ b/tests/test_effectiveness_full_pool_profile.py @@ -0,0 +1,152 @@ +from __future__ import annotations + +from types import SimpleNamespace + +import pytest + +from fuckmark.corpus import CorpusDomain, CorpusSplit, WatermarkLabel +from fuckmark.experiments.effectiveness_plan import ( + build_key_blind_high_coverage_plan, + validate_key_blind_high_coverage_plan, +) +from fuckmark.hashing import sha256_text +from fuckmark.transforms import ( + KEY_BLIND_FULL_POOL_COVERAGE_PROFILE_ID, + KEY_BLIND_FULL_POOL_COVERAGE_SEED_BASE, + KEY_BLIND_HIGH_COVERAGE_PROFILE_ID, + key_blind_full_pool_coverage_profile, + resolve_effectiveness_profile, +) +from fuckmark.tiny_dev_effectiveness_plan_hf import _parse_budgets +from fuckmark.tiny_dev_effectiveness_score_hf import _parse_budgets as _parse_budgets_score + + +class FakeTokenizer: + def __call__( + self, + text: str, + *, + add_special_tokens: bool, + return_offsets_mapping: bool = False, + ) -> dict[str, object]: + assert add_special_tokens is False + ids = [index + 1 for index in range(len(text))] + value: dict[str, object] = {"input_ids": ids} + if return_offsets_mapping: + value["offset_mapping"] = [(index, index + 1) for index in range(len(text))] + return value + + +def _sample(sample_id: str, label: WatermarkLabel, text: str, identity_hash: str): + token_ids = tuple(index + 1 for index in range(len(text))) + return SimpleNamespace( + sample_id=sample_id, + split=CorpusSplit.ATTACK_DEVELOPMENT, + label=label, + prompt_family_id=f"prompt-{sample_id}", + domain=CorpusDomain.GENERAL_EXPLANATORY, + text=text, + text_sha256=sha256_text(text), + text_only_tokens=SimpleNamespace(token_ids=token_ids), + model=SimpleNamespace(identity_hash=identity_hash, eos_token_id=50256), + ) + + +def _corpus(): + identity_hash = sha256_text("fake-tokenizer") + text = "You are not ready, and we do not stop when the system is in use." + samples = ( + _sample("negative-1", WatermarkLabel.UNWATERMARKED, text, identity_hash), + _sample("positive-1", WatermarkLabel.WATERMARKED, text, identity_hash), + ) + return SimpleNamespace( + artifact_hash=sha256_text("fake-corpus"), + model_identity_hash=identity_hash, + manifest=SimpleNamespace(samples=samples, manifest_hash=sha256_text("fake-manifest")), + ) + + +def test_full_pool_profile_is_deterministic_and_binds_budgets() -> None: + first = key_blind_full_pool_coverage_profile((16, 24, 32, 48)) + second = key_blind_full_pool_coverage_profile((16, 24, 32, 48)) + assert first == second + assert first.profile_id == KEY_BLIND_FULL_POOL_COVERAGE_PROFILE_ID + assert first.budgets == (16, 24, 32, 48) + assert first.schedule_seed_base == KEY_BLIND_FULL_POOL_COVERAGE_SEED_BASE + assert first.ruleset_hash == resolve_effectiveness_profile( + KEY_BLIND_HIGH_COVERAGE_PROFILE_ID + ).ruleset_hash + other = key_blind_full_pool_coverage_profile((32,)) + assert other.profile_hash != first.profile_hash + + +def test_resolve_effectiveness_profile_rejects_invalid_requests() -> None: + with pytest.raises(ValueError): + resolve_effectiveness_profile("unknown-profile") + with pytest.raises(ValueError): + resolve_effectiveness_profile(KEY_BLIND_HIGH_COVERAGE_PROFILE_ID, (16,)) + with pytest.raises(ValueError): + resolve_effectiveness_profile(KEY_BLIND_FULL_POOL_COVERAGE_PROFILE_ID) + with pytest.raises(ValueError): + key_blind_full_pool_coverage_profile((48, 16)) + + +def test_full_pool_plan_covers_every_budget_with_derived_seeds() -> None: + corpus = _corpus() + profile = key_blind_full_pool_coverage_profile((24, 32)) + plan = build_key_blind_high_coverage_plan( + corpus, + FakeTokenizer(), + profile=profile, + source_code_commit="a" * 40, + ) + validate_key_blind_high_coverage_plan(plan, corpus, profile) + assert plan["budgets"] == (24, 32) + assert len(plan["variants"]) == 4 + assert {row["requested_budget"] for row in plan["variants"]} == {24, 32} + assert [row["schedule_seed"] for row in plan["variants"]] == [ + KEY_BLIND_FULL_POOL_COVERAGE_SEED_BASE, + KEY_BLIND_FULL_POOL_COVERAGE_SEED_BASE, + KEY_BLIND_FULL_POOL_COVERAGE_SEED_BASE + 1, + KEY_BLIND_FULL_POOL_COVERAGE_SEED_BASE + 1, + ] + assert all(row["detector_access_observed"] is False for row in plan["variants"]) + assert all(row["secret_access_observed"] is False for row in plan["variants"]) + + +def test_full_pool_plan_rejects_mismatched_profile_on_validation() -> None: + corpus = _corpus() + profile = key_blind_full_pool_coverage_profile((24,)) + plan = build_key_blind_high_coverage_plan( + corpus, + FakeTokenizer(), + profile=profile, + source_code_commit="a" * 40, + ) + with pytest.raises(ValueError): + validate_key_blind_high_coverage_plan( + plan, + corpus, + key_blind_full_pool_coverage_profile((32,)), + ) + + +@pytest.mark.parametrize( + ("raw", "expected"), + ( + ("", ()), + ("16", (16,)), + ("16,24,32,48", (16, 24, 32, 48)), + ), +) +def test_budget_parsing_accepts_valid_lists(raw: str, expected: tuple[int, ...]) -> None: + assert _parse_budgets(raw) == expected + assert _parse_budgets_score(raw) == expected + + +@pytest.mark.parametrize("raw", ("16,,24", "0", "-4", "24,16", "16,16")) +def test_budget_parsing_rejects_invalid_lists(raw: str) -> None: + with pytest.raises(ValueError): + _parse_budgets(raw) + with pytest.raises(ValueError): + _parse_budgets_score(raw) diff --git a/tests/test_measurement_calibration.py b/tests/test_measurement_calibration.py new file mode 100644 index 00000000..db1719e4 --- /dev/null +++ b/tests/test_measurement_calibration.py @@ -0,0 +1,135 @@ +from __future__ import annotations + +import pytest + +from fuckmark.config import canonical_json_text +from fuckmark.corpus import ( + CorpusDomain, + CorpusSplit, + GenerationParameters, + KeySplit, + ModelTokenizerIdentity, + PaddingSide, + WatermarkCondition, + WatermarkLabel, +) +from fuckmark.corpus.measurement_calibration import ( + MEASUREMENT_CALIBRATION_AUDIT_COUNT, + MEASUREMENT_CALIBRATION_NEGATIVE_COUNT, + MEASUREMENT_CALIBRATION_SAMPLE_COUNT, + build_measurement_calibration_corpus, +) +from fuckmark.corpus.tiny_dev_generation import ( + TINY_DEV_SEED_POLICY_ID, + TinyDevGeneratedContinuation, +) +from fuckmark.hashing import sha256_json, sha256_text +from fuckmark.tiny_dev_measurement_calibration_hf import ( + build_measurement_calibration_prompt_records, + build_real_measurement_calibration_corpus, +) + + +class FakeCalibrationBackend: + def __init__(self) -> None: + self._model = ModelTokenizerIdentity.create( + model_id="example/calibration-model", + model_revision="a" * 40, + tokenizer_id="example/calibration-tokenizer", + tokenizer_revision="b" * 40, + chat_template_present=False, + chat_template_hash=sha256_text(""), + special_token_map_hash=sha256_json({"eos_token": "", "pad_token": ""}), + padding_side=PaddingSide.LEFT, + bos_token_id=None, + eos_token_id=2, + pad_token_id=0, + add_bos_token=False, + add_eos_token=False, + ) + self._watermark = WatermarkCondition.create( + sha256_json({"ngram_len": 5, "keys": (1, 2, 3)}), + KeySplit.DEV, + "dev-test-key", + ) + + @property + def model_identity(self) -> ModelTokenizerIdentity: + return self._model + + @property + def watermark_condition(self) -> WatermarkCondition: + return self._watermark + + def generation_parameters(self, seed: int) -> GenerationParameters: + return GenerationParameters.create( + seed=seed, + seed_policy_id=TINY_DEV_SEED_POLICY_ID, + temperature=0.8, + top_k=50, + top_p=0.95, + max_new_tokens=64, + do_sample=True, + dtype="float32", + device="cpu", + backend_id="fake-calibration", + backend_version="1", + ) + + def generate(self, prompt: str, seed: int, *, watermarked: bool) -> TinyDevGeneratedContinuation: + assert watermarked is False + prompt_token = 10 + (sum(prompt.encode("utf-8")) % 1000) + continuation = tuple( + 200_000 + ((seed * 977 + index) % 90_000) + for index in range(64) + ) + text = f"calibration negative for {sha256_text(prompt)[:12]} seed {seed}" + text_only = tuple(300_000 + ((seed * 31 + index) % 80_000) for index in range(16)) + return TinyDevGeneratedContinuation( + text=text, + input_token_ids=(prompt_token, prompt_token + 1), + attention_mask=(1, 1), + continuation_token_ids=continuation, + text_only_token_ids=text_only, + ) + + +def test_measurement_calibration_prompt_records_are_frozen_and_disjoint() -> None: + prompts = build_measurement_calibration_prompt_records() + assert len(prompts) == 160 + assert len({prompt.prompt_id for prompt in prompts}) == 160 + counts: dict[CorpusDomain, int] = {} + for prompt in prompts: + counts[prompt.domain] = counts.get(prompt.domain, 0) + 1 + assert set(counts.values()) == {40} + from fuckmark.corpus.tiny_dev_v3_generation import build_tiny_dev_v3_prompt_records + + prior_texts = { + prompt.text + for prompt in build_tiny_dev_v3_prompt_records(16) + } + assert not any(prompt.text in prior_texts for prompt in prompts) + + +def test_measurement_calibration_corpus_builds_with_frozen_split() -> None: + backend = FakeCalibrationBackend() + artifact = build_real_measurement_calibration_corpus(backend, seed_base=6000) + assert len(artifact.manifest.samples) == MEASUREMENT_CALIBRATION_SAMPLE_COUNT == 1280 + assert artifact.negative_count == MEASUREMENT_CALIBRATION_NEGATIVE_COUNT == 1024 + assert artifact.audit_count == MEASUREMENT_CALIBRATION_AUDIT_COUNT == 256 + calibration = artifact.calibration_samples() + audit = artifact.audit_samples() + assert len(calibration) == 1024 + assert len(audit) == 256 + assert not {sample.sample_id for sample in calibration} & {sample.sample_id for sample in audit} + assert all(sample.split is CorpusSplit.THRESHOLD_CALIBRATION for sample in artifact.manifest.samples) + assert all(sample.label is WatermarkLabel.UNWATERMARKED for sample in artifact.manifest.samples) + text = canonical_json_text(artifact) + "\n" + assert len(text) > 0 + + +def test_measurement_calibration_rejects_wrong_sample_count() -> None: + backend = FakeCalibrationBackend() + prompts = build_measurement_calibration_prompt_records() + with pytest.raises(Exception): + build_measurement_calibration_corpus("bad-corpus", prompts, ()) diff --git a/tests/test_measurement_threshold.py b/tests/test_measurement_threshold.py new file mode 100644 index 00000000..c7a1297c --- /dev/null +++ b/tests/test_measurement_threshold.py @@ -0,0 +1,81 @@ +from __future__ import annotations + +from types import SimpleNamespace + +import pytest + +import fuckmark.experiments.measurement_threshold as mt +from fuckmark.experiments.measurement_threshold import ( + build_fixed_threshold_artifact, + validate_fixed_threshold_artifact, +) + + +class _FakeCalibrationCorpus: + def __init__(self, calibration_scores, audit_scores) -> None: + self.calibration_set = tuple( + SimpleNamespace(sample_id=f"cal-{index}", score=value) + for index, value in enumerate(calibration_scores) + ) + self.audit_set = tuple( + SimpleNamespace(sample_id=f"audit-{index}", score=value) + for index, value in enumerate(audit_scores) + ) + self.artifact_hash = "a" * 64 + self.manifest = SimpleNamespace(corpus_id="fake-calibration") + + def calibration_samples(self): + return self.calibration_set + + def audit_samples(self): + return self.audit_set + + +def _patch_scoring(monkeypatch, calibration_scores, audit_scores): + corpus = _FakeCalibrationCorpus(calibration_scores, audit_scores) + lookup = {sample.sample_id: sample.score for sample in (*corpus.calibration_set, *corpus.audit_set)} + + def fake_evidence(sample, adapter): + return SimpleNamespace(raw_score=lookup[sample.sample_id]) + + def fake_calibrate(evidence, scope, target_fprs, comparison_operator, confidence_level): + return SimpleNamespace( + detector_identity=SimpleNamespace(identity_hash="d" * 64), + bundle_hash="b" * 64, + ) + + monkeypatch.setattr(mt, "_text_only_weighted_evidence", fake_evidence) + monkeypatch.setattr(mt, "calibrate_detector", fake_calibrate) + return corpus + + +def test_fixed_threshold_uses_frozen_order_statistic(monkeypatch) -> None: + calibration = [index / 2000.0 for index in range(1024)] + audit = [0.0] * 250 + [0.9] * 6 + corpus = _patch_scoring(monkeypatch, calibration, audit) + artifact = build_fixed_threshold_artifact(corpus, None, frozen_at_utc="2026-08-23T00:00:00+00:00") + ordered = sorted(calibration) + assert artifact["threshold_order_statistic"] == 1015 + assert artifact["threshold"] == ordered[1014] + assert artifact["calibration_exceedances"] == 10 + assert artifact["audit_exceedances"] == 6 + assert artifact["audit_realized_fpr"] == 6 / 256 + assert artifact["detector_identity_hash"] == "d" * 64 + assert artifact["artifact_hash"] + + +def test_fixed_threshold_validation_rejects_mismatches(monkeypatch) -> None: + calibration = [index / 2000.0 for index in range(1024)] + corpus = _patch_scoring(monkeypatch, calibration, [0.0] * 256) + artifact = build_fixed_threshold_artifact(corpus, None, frozen_at_utc="2026-08-23T00:00:00+00:00") + assert validate_fixed_threshold_artifact(artifact, "d" * 64) == artifact["threshold"] + with pytest.raises(ValueError): + validate_fixed_threshold_artifact(artifact, "e" * 64) + tampered = dict(artifact) + tampered["threshold"] = 0.123456 + with pytest.raises(ValueError): + validate_fixed_threshold_artifact(tampered, "d" * 64) + bad_version = dict(artifact) + bad_version["algorithm_version"] = "other-v1" + with pytest.raises(ValueError): + validate_fixed_threshold_artifact(bad_version, "d" * 64) diff --git a/tests/test_protected_currency_context_invariance.py b/tests/test_protected_currency_context_invariance.py new file mode 100644 index 00000000..8cafde79 --- /dev/null +++ b/tests/test_protected_currency_context_invariance.py @@ -0,0 +1,35 @@ +from __future__ import annotations + +import pytest + +from fuckmark.transforms import validate_hard_invariants +from fuckmark.transforms.protected import ProtectedSpanExtractor + + +def _currency_spans(text: str) -> list[str]: + return [ + text[span.start : span.end] + for span in ProtectedSpanExtractor().extract(text).spans + if "currency" in tuple(kind.value for kind in span.kinds) + ] + + +def test_currency_extraction_is_invariant_under_preceding_whitespace() -> None: + assert _currency_spans("for $100") == ["$100"] + assert _currency_spans("for $100") == ["$100"] + assert _currency_spans("a $5 and $100") == ["$5", "$100"] + assert _currency_spans("cost: $100,000 total") == ["$100,000"] + + +def test_signed_currency_forms_still_extract() -> None: + assert _currency_spans("- $5") == ["- $5"] + assert _currency_spans("+EUR 3.50") == ["+EUR 3.50"] + assert _currency_spans("USD 5") == ["USD 5"] + + +def test_spacing_before_currency_no_longer_breaks_hard_invariants() -> None: + source = "Apple has been selling its iPhones for $100, and buyers notice." + transformed = "Apple has been selling its iPhones for $100, and buyers notice." + report = validate_hard_invariants(source, transformed) + assert report.status.value == "pass" + assert [r.value for r in report.reasons] == [] diff --git a/tests/test_tiny_dev_corpus_v3.py b/tests/test_tiny_dev_corpus_v3.py new file mode 100644 index 00000000..343f57be --- /dev/null +++ b/tests/test_tiny_dev_corpus_v3.py @@ -0,0 +1,179 @@ +from __future__ import annotations + +from collections import defaultdict + +import pytest + +from fuckmark.config import canonical_json_text +from fuckmark.corpus import ( + CorpusDomain, + CorpusSplit, + GenerationParameters, + KeySplit, + ModelTokenizerIdentity, + PaddingSide, + WatermarkCondition, + WatermarkLabel, + TINY_DEV_V3_CORPUS_ALGORITHM_VERSION, + TINY_DEV_V3_MAX_ATTACK_PAIRS_PER_DOMAIN, + build_tiny_dev_v3_prompt_records, + build_real_tiny_dev_v3_corpus, + parse_tiny_dev_v3_corpus_json, +) +from fuckmark.corpus.tiny_dev_generation import ( + TINY_DEV_SEED_POLICY_ID, + TinyDevGeneratedContinuation, + build_tiny_dev_prompt_records, +) +from fuckmark.corpus.tiny_dev_io_v3 import load_tiny_dev_corpus_by_version_json +from fuckmark.hashing import sha256_json, sha256_text + + +class FakeTinyDevV3Backend: + def __init__(self) -> None: + self._model = ModelTokenizerIdentity.create( + model_id="example/tiny-dev-model", + model_revision="a" * 40, + tokenizer_id="example/tiny-dev-tokenizer", + tokenizer_revision="b" * 40, + chat_template_present=False, + chat_template_hash=sha256_text(""), + special_token_map_hash=sha256_json({"eos_token": "", "pad_token": ""}), + padding_side=PaddingSide.LEFT, + bos_token_id=None, + eos_token_id=2, + pad_token_id=0, + add_bos_token=False, + add_eos_token=False, + ) + self._watermark = WatermarkCondition.create( + sha256_json({"ngram_len": 5, "keys": (1, 2, 3)}), + KeySplit.DEV, + "dev-test-key", + ) + + @property + def model_identity(self) -> ModelTokenizerIdentity: + return self._model + + @property + def watermark_condition(self) -> WatermarkCondition: + return self._watermark + + def generation_parameters(self, seed: int) -> GenerationParameters: + return GenerationParameters.create( + seed=seed, + seed_policy_id=TINY_DEV_SEED_POLICY_ID, + temperature=0.8, + top_k=50, + top_p=0.95, + max_new_tokens=64, + do_sample=True, + dtype="float32", + device="cpu", + backend_id="fake-tiny-dev", + backend_version="1", + ) + + def generate(self, prompt: str, seed: int, *, watermarked: bool) -> TinyDevGeneratedContinuation: + prompt_token = 10 + (sum(prompt.encode("utf-8")) % 1000) + label_offset = 100_000 if watermarked else 200_000 + continuation = tuple( + label_offset + ((seed * 131 + index) % 90_000) + for index in range(64) + ) + text = ( + f"{'watermarked' if watermarked else 'control'} output for " + f"{sha256_text(prompt)[:12]} seed {seed}" + ) + text_only = tuple(300_000 + ((seed * 17 + index) % 80_000) for index in range(16)) + return TinyDevGeneratedContinuation( + text=text, + input_token_ids=(prompt_token, prompt_token + 1), + attention_mask=(1, 1), + continuation_token_ids=continuation, + text_only_token_ids=text_only, + ) + + +def test_v3_prompt_records_preserve_v2_calibration_and_extend_attack() -> None: + v2 = build_tiny_dev_prompt_records() + v3 = build_tiny_dev_v3_prompt_records(16) + v2_calibration = {prompt.text for prompt in v2 if prompt.split is CorpusSplit.THRESHOLD_CALIBRATION} + v3_calibration = {prompt.text for prompt in v3 if prompt.split is CorpusSplit.THRESHOLD_CALIBRATION} + assert v2_calibration == v3_calibration + v2_attack = {prompt.text for prompt in v2 if prompt.split is CorpusSplit.ATTACK_DEVELOPMENT} + v3_one = build_tiny_dev_v3_prompt_records(1) + v3_one_attack = {prompt.text for prompt in v3_one if prompt.split is CorpusSplit.ATTACK_DEVELOPMENT} + assert v2_attack == v3_one_attack + counts: defaultdict[tuple[CorpusSplit, CorpusDomain], int] = defaultdict(int) + for prompt in v3: + counts[(prompt.split, prompt.domain)] += 1 + for domain in CorpusDomain: + assert counts[(CorpusSplit.THRESHOLD_CALIBRATION, domain)] == 25 + assert counts[(CorpusSplit.ATTACK_DEVELOPMENT, domain)] == 16 + assert len({prompt.prompt_id for prompt in v3}) == 164 + assert len({prompt.prompt_family_id for prompt in v3}) == 164 + + +@pytest.mark.parametrize("count", (0, -1, TINY_DEV_V3_MAX_ATTACK_PAIRS_PER_DOMAIN + 1)) +def test_v3_prompt_records_reject_out_of_range_attack_pairs(count: int) -> None: + with pytest.raises((ValueError, Exception)): + build_tiny_dev_v3_prompt_records(count) + + +def test_real_v3_runner_builds_parameterized_paired_corpus() -> None: + backend = FakeTinyDevV3Backend() + artifact = build_real_tiny_dev_v3_corpus( + backend, + seed_base=5000, + attack_pairs_per_domain=8, + ) + assert artifact.algorithm_version == TINY_DEV_V3_CORPUS_ALGORITHM_VERSION + assert artifact.attack_pairs_per_domain == 8 + assert artifact.calibration_pairs_per_domain == 25 + assert len(artifact.manifest.prompts) == 132 + assert len(artifact.manifest.samples) == 264 + attack_watermarked = [ + sample + for sample in artifact.manifest.samples + if sample.split is CorpusSplit.ATTACK_DEVELOPMENT + and sample.label is WatermarkLabel.WATERMARKED + ] + assert len(attack_watermarked) == 32 + assert all(sample.watermark.key_split is KeySplit.DEV for sample in artifact.manifest.samples) + assert len({sample.text_sha256 for sample in artifact.manifest.samples}) == 264 + + +def test_v3_corpus_json_replays_canonically_and_version_dispatch_loads() -> None: + backend = FakeTinyDevV3Backend() + artifact = build_real_tiny_dev_v3_corpus( + backend, + seed_base=9000, + attack_pairs_per_domain=2, + ) + text = canonical_json_text(artifact) + "\n" + replay = parse_tiny_dev_v3_corpus_json(text) + assert replay == artifact + assert replay.artifact_hash == artifact.artifact_hash + + +def test_v3_builder_rejects_mismatched_prompt_profile() -> None: + from fuckmark.corpus import build_tiny_dev_v3_corpus + + backend = FakeTinyDevV3Backend() + prompts = build_tiny_dev_v3_prompt_records(4) + with pytest.raises(Exception): + build_tiny_dev_v3_corpus( + "mismatch-corpus", + prompts, + (), + attack_pairs_per_domain=8, + ) + + +def test_version_dispatch_loader_rejects_unknown_version(tmp_path) -> None: + bad = tmp_path / "corpus.json" + bad.write_text('{"algorithm_version": "tiny-dev-corpus-v9"}', encoding="utf-8") + with pytest.raises(Exception): + load_tiny_dev_corpus_by_version_json(bad) diff --git a/tools/budget_scaled_coverage_diagnosis.py b/tools/budget_scaled_coverage_diagnosis.py new file mode 100644 index 00000000..e73dd5f2 --- /dev/null +++ b/tools/budget_scaled_coverage_diagnosis.py @@ -0,0 +1,157 @@ +from __future__ import annotations + +import argparse +import json +from pathlib import Path + +from fuckmark.corpus import load_tiny_dev_corpus_by_version_json +from fuckmark.experiments.effectiveness_plan import _attack_samples, _encode_with_offsets +from fuckmark.hashing import sha256_json +from fuckmark.transforms import ( + SchedulePolicy, + CandidateScheduler, + KeyBlindScheduleInput, + ScheduleGeometryMode, + build_candidate_tokenizer_geometry, + key_blind_high_coverage_transform_registry, +) +from fuckmark.coverage import union_size, merge_intervals + + +DIAGNOSIS_VERSION = "budget-scaled-coverage-diagnosis-v1" + + +def diagnose_sources(corpus, tokenizer, plan, evidence) -> dict[str, object]: + registry = key_blind_high_coverage_transform_registry() + scheduler = CandidateScheduler() + evidence_rows = { + row["variant_hash"]: row for row in evidence["rows"] + } + rows = [] + for variant in plan["variants"]: + if variant["requested_budget"] != plan["budgets"][0]: + continue + row_evidence = evidence_rows[variant["variant_hash"]] + source = next( + sample + for sample in corpus.manifest.samples + if sample.sample_id == variant["source_sample_id"] + ) + token_ids, offsets = _encode_with_offsets(tokenizer, source) + enumeration = registry.enumerate(source.text) + geometry = build_candidate_tokenizer_geometry( + source.text, + enumeration, + token_ids, + offsets, + tokenizer_identity_hash=source.model.identity_hash, + ngram_len=5, + ) + coverage = geometry.coverage_mapping() + observation_count = max(0, len(token_ids) - 5 + 1) + achievable = union_size( + interval + for intervals in coverage.values() + for interval in intervals + ) if coverage else 0 + selected_ids = set(variant["selected_candidate_ids"]) + realized = union_size( + interval + for candidate_id, intervals in coverage.items() + if candidate_id in selected_ids + for interval in intervals + ) if coverage else 0 + hole_intervals = [] + covered = merge_intervals( + interval + for intervals in coverage.values() + for interval in intervals + ) if coverage else () + cursor = 0 + for interval in covered: + start, end = interval.start, interval.end_exclusive + if start > cursor: + hole_intervals.append((cursor, start)) + cursor = max(cursor, end) + if cursor < observation_count: + hole_intervals.append((cursor, observation_count)) + rows.append( + { + "source_sample_id": variant["source_sample_id"], + "source_label": variant["source_label"], + "domain": variant["domain"], + "observation_count": observation_count, + "candidate_count": variant["candidate_count"], + "realized_edit_cost": variant["realized_edit_cost"], + "achievable_coverage_fraction": achievable / observation_count if observation_count else 0.0, + "realized_coverage_fraction": realized / observation_count if observation_count else 0.0, + "hole_count": len(hole_intervals), + "hole_token_total": sum(end - start for start, end in hole_intervals), + "pristine_score": row_evidence["pristine_score"], + "transformed_score": row_evidence["transformed_score"], + "transformed_detected": row_evidence["transformed_detected"], + "score_drop": row_evidence["score_drop"], + } + ) + watermarked = [row for row in rows if row["source_label"] == "watermarked"] + detected = [row for row in watermarked if row["transformed_detected"]] + escaped = [row for row in watermarked if not row["transformed_detected"]] + + def _mean(values): + return sum(values) / len(values) if values else None + + payload = { + "algorithm_version": DIAGNOSIS_VERSION, + "analysis_scope": "post-hoc explanatory analysis of the completed budget-scaled confirmation cycle; not used to tune any candidate against this corpus", + "plan_hash": plan["plan_hash"], + "evidence_hash": evidence["artifact_hash"], + "corpus_hash": corpus.artifact_hash, + "rows": tuple(rows), + "watermarked_summary": { + "total": len(watermarked), + "detected": len(detected), + "mean_achievable_coverage_detected": _mean([r["achievable_coverage_fraction"] for r in detected]), + "mean_achievable_coverage_escaped": _mean([r["achievable_coverage_fraction"] for r in escaped]), + "mean_realized_coverage_detected": _mean([r["realized_coverage_fraction"] for r in detected]), + "mean_realized_coverage_escaped": _mean([r["realized_coverage_fraction"] for r in escaped]), + "mean_hole_token_total_detected": _mean([r["hole_token_total"] for r in detected]), + "mean_hole_token_total_escaped": _mean([r["hole_token_total"] for r in escaped]), + "achievable_below_70pct_detected": sum(1 for r in detected if r["achievable_coverage_fraction"] < 0.7), + "achievable_below_70pct_escaped": sum(1 for r in escaped if r["achievable_coverage_fraction"] < 0.7), + }, + } + return {**payload, "artifact_hash": sha256_json(payload)} + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(prog="budget-scaled-coverage-diagnosis") + parser.add_argument("--corpus-json", type=Path, required=True) + parser.add_argument("--plan-json", type=Path, required=True) + parser.add_argument("--evidence-json", type=Path, required=True) + parser.add_argument("--json", type=Path, required=True) + args = parser.parse_args(argv) + from transformers import AutoTokenizer + + corpus = load_tiny_dev_corpus_by_version_json(args.corpus_json) + plan = json.loads(args.plan_json.read_text(encoding="utf-8")) + evidence = json.loads(args.evidence_json.read_text(encoding="utf-8")) + tokenizer = AutoTokenizer.from_pretrained( + "openai-community/gpt2", + revision="607a30d783dfa663caf39e06633721c8d4cfcd7e", + use_fast=True, + padding_side="left", + ) + report = diagnose_sources(corpus, tokenizer, plan, evidence) + args.json.parent.mkdir(parents=True, exist_ok=True) + args.json.write_text( + json.dumps(report, indent=2, sort_keys=True) + "\n", encoding="utf-8" + ) + summary = report["watermarked_summary"] + print(f"artifact_hash={report['artifact_hash']}") + for key, value in summary.items(): + print(f"{key}={value}") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tools/budget_scaled_render_page.py b/tools/budget_scaled_render_page.py new file mode 100644 index 00000000..9e0fbd7b --- /dev/null +++ b/tools/budget_scaled_render_page.py @@ -0,0 +1,88 @@ +from __future__ import annotations + +import argparse +import html +import json +from pathlib import Path + + +def _load(path: Path) -> dict: + value = json.loads(path.read_text(encoding="utf-8")) + if not isinstance(value, dict): + raise ValueError("expected a JSON object") + return value + + +def _load_corpus_texts(corpus_path: Path) -> dict[str, str]: + corpus = _load(corpus_path) + return {sample["sample_id"]: sample["text"] for sample in corpus["manifest"]["samples"]} + + +def build_render_page(plan: dict, corpus_texts: dict[str, str]) -> tuple[str, list[dict]]: + blocks: list[str] = [] + manifest: list[dict] = [] + for index, variant in enumerate(plan["variants"]): + source_id = variant["source_sample_id"] + source_text = corpus_texts[source_id] + transformed_text = variant["transformed_text"] + blocks.append( + "
" + "

Pair {index}: {source_id} (budget {budget})

" + "
{source}
" + "
{transformed}
" + "
".format( + index=index, + source_id=html.escape(source_id), + budget=variant["requested_budget"], + source=html.escape(source_text), + transformed=html.escape(transformed_text), + ) + ) + manifest.append( + { + "pair_index": index, + "source_sample_id": source_id, + "requested_budget": variant["requested_budget"], + "source_text_hash": variant["source_text_hash"], + "transformed_text_hash": variant["transformed_text_hash"], + "variant_hash": variant["variant_hash"], + } + ) + page = ( + "" + "FuckMark budget-scaled render check" + "" + + "".join(blocks) + + "" + ) + return page, manifest + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(prog="budget-scaled-render-page") + parser.add_argument("--corpus-json", type=Path, required=True) + parser.add_argument("--plan-json", type=Path, required=True) + parser.add_argument("--html", type=Path, required=True) + parser.add_argument("--manifest-json", type=Path, required=True) + args = parser.parse_args(argv) + plan = _load(args.plan_json) + corpus_texts = _load_corpus_texts(args.corpus_json) + page, manifest = build_render_page(plan, corpus_texts) + args.html.parent.mkdir(parents=True, exist_ok=True) + args.html.write_text(page, encoding="utf-8") + args.manifest_json.write_text( + json.dumps({"pair_count": len(manifest), "plan_hash": plan["plan_hash"], "pairs": manifest}, indent=2, sort_keys=True) + "\n", + encoding="utf-8", + ) + print(f"pair_count={len(manifest)}") + print(f"html={args.html.as_posix()}") + print(f"manifest_json={args.manifest_json.as_posix()}") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main())