Preserve and reproduce real TinyDev evidence

.github/workflows/tiny-dev-corpus.yml

@@ -37,7 +37,7 @@ jobs:
       - name: Install FuckMark
         run: python -m pip install -e .
       - name: Install CPU Torch
-        run: python -m pip install "torch>=2.4,<3" --index-url https://download.pytorch.org/whl/cpu
+        run: python -m pip install "torch==2.13.0" --index-url https://download.pytorch.org/whl/cpu
       - name: Install maintained Transformers SynthID dependencies
         run: python -m pip install -r requirements-smoke.txt
       - name: Generate real TinyDev corpus

evidence/tiny_dev/real-hf-gpt2-2026-08-18.json

@@ -0,0 +1,95 @@
+{
+  "algorithm_version": "real-tiny-dev-evidence-v1",
+  "evidence_kind": "development-corpus-generation",
+  "scientific_scope": "DEV_KEYS-only TinyDev generation prerequisite; not M6 readiness or a confirmatory result",
+  "source": {
+    "repository": "byte271/FuckMark",
+    "generator_head_commit": "9b829b3e594eeb1335d2c0509124deae6e7c0057",
+    "merged_main_commit": "33b4a8c10b9382dcc1068baacb99b30e67113a92",
+    "workflow_run_id": 32084697518,
+    "workflow_run_number": 1,
+    "workflow_name": "Real TinyDev Corpus",
+    "workflow_conclusion": "success",
+    "artifact_id": 9306497613,
+    "artifact_name": "real-tiny-dev-corpus",
+    "artifact_zip_sha256": "3fc78ee29929e9a394fa0615db80422ffc6a55ae967b989e522f23175bec6c0f",
+    "artifact_json_sha256": "ee09527d06abc58e612f7157b69908d5abe06101865714ce13a1287d1f0d7664"
+  },
+  "corpus": {
+    "algorithm_version": "tiny-dev-corpus-v2",
+    "artifact_hash": "42240f72e17df1a048f44197f216a0b5f30128c26e2da93afbe92820946e4561",
+    "manifest_hash": "693788e76bceea43e72425b98f326bc09e4840d069eb47dacb18784ec7c2e388",
+    "prompt_manifest_hash": "6456a6f2fbcbc58c8905133b1981f84b500c06c9f653dc979a1d962ab0fc862f",
+    "sample_manifest_hash": "350aa801cbb12c0908686b39b2ffb77bc4419f3b71e0210c42d7735dbd3259be",
+    "prompt_count": 104,
+    "sample_count": 208,
+    "matched_pair_count": 104,
+    "target_length": 64,
+    "calibration_pairs_per_domain": 25,
+    "attack_pairs_per_domain": 1,
+    "required_domains": [
+      "general_explanatory",
+      "technical_explanation",
+      "conversational_prose",
+      "structured_instructional"
+    ],
+    "required_splits": [
+      "threshold_calibration",
+      "attack_development"
+    ],
+    "language": "en",
+    "deduplication_policy": "exact_utf8_v1",
+    "model_identity_hash": "1be91dc38048d8f69fc45d5fb8175b0edb7c6ec807af1f6b85aa657343dbb95e",
+    "generation_matching_signature_hash": "8de16173bb6be768141be143ffd01fdf28b623c64b461fca6d77c3abdeb5df95",
+    "watermark_condition_hash": "4cdef888bf3ef50029b6acef203d3efbbc2ae3f29cad5f1075bcb61202f89d59"
+  },
+  "runtime": {
+    "model_id": "openai-community/gpt2",
+    "model_revision": "607a30d783dfa663caf39e06633721c8d4cfcd7e",
+    "tokenizer_id": "openai-community/gpt2",
+    "tokenizer_revision": "607a30d783dfa663caf39e06633721c8d4cfcd7e",
+    "device": "cpu",
+    "dtype": "float32",
+    "backend_id": "huggingface-transformers-synthid-tiny-dev",
+    "backend_version": "transformers=5.16.0.dev0;torch=2.13.0+cpu;device=cpu;revision=607a30d783dfa663caf39e06633721c8d4cfcd7e",
+    "seed_policy_id": "tiny-dev-paired-seed-v1",
+    "seed_base": 401000,
+    "temperature": 0.8,
+    "top_k": 50,
+    "top_p": 0.95,
+    "max_new_tokens": 64,
+    "watermark_key_split": "DEV_KEYS",
+    "watermark_key_id": "dev-huggingface-synthid-default-v1",
+    "watermark_config_hash": "d5918a5b92bd723b78f393d986b812f786b2855c767eb7ab60192e63d2a34086"
+  },
+  "execution": {
+    "pair_attempt_offset_counts": {
+      "0": 99,
+      "1": 4,
+      "2": 1
+    },
+    "maximum_pair_attempt_offset": 2,
+    "exact_generation_token_track_count": 208,
+    "text_only_track_count": 208,
+    "text_only_exact_token_match_count": 197,
+    "text_only_retokenized_length_min": 63,
+    "text_only_retokenized_length_max": 65
+  },
+  "verification": {
+    "canonical_nested_hashes_recomputed": true,
+    "artifact_hash_recomputed": true,
+    "manifest_hash_recomputed": true,
+    "prompt_ids_unique": true,
+    "sample_ids_unique": true,
+    "sample_text_hashes_unique": true,
+    "generation_continuation_hashes_unique": true,
+    "all_generation_realized_lengths_exactly_64": true,
+    "all_samples_use_dev_keys": true,
+    "all_pairs_have_one_watermarked_and_one_unwatermarked_sample": true,
+    "all_pairs_share_one_actual_seed": true,
+    "all_samples_share_one_model_identity": true,
+    "all_samples_share_one_generation_matching_signature": true,
+    "all_samples_share_one_watermark_condition": true
+  },
+  "evidence_hash": "c093a248be705437e812a3554c4e38468a010822415565dd0235e9ffac6d78cd"
+}

tests/test_real_tiny_dev_evidence.py

@@ -0,0 +1,79 @@
+import json
+from pathlib import Path
+
+from fuckmark.hashing import sha256_json
+
+
+_EVIDENCE_PATH = Path("evidence/tiny_dev/real-hf-gpt2-2026-08-18.json")
+
+
+def _evidence() -> dict[str, object]:
+    return json.loads(_EVIDENCE_PATH.read_text(encoding="utf-8"))
+
+
+def test_real_tiny_dev_evidence_is_self_hashed_and_scoped() -> None:
+    evidence = _evidence()
+    evidence_hash = evidence.pop("evidence_hash")
+    assert evidence_hash == sha256_json(evidence)
+    assert evidence["algorithm_version"] == "real-tiny-dev-evidence-v1"
+    assert evidence["evidence_kind"] == "development-corpus-generation"
+    assert evidence["scientific_scope"] == (
+        "DEV_KEYS-only TinyDev generation prerequisite; not M6 readiness or a confirmatory result"
+    )
+
+
+def test_real_tiny_dev_evidence_binds_successful_source_run() -> None:
+    source = _evidence()["source"]
+    assert source["generator_head_commit"] == "9b829b3e594eeb1335d2c0509124deae6e7c0057"
+    assert source["merged_main_commit"] == "33b4a8c10b9382dcc1068baacb99b30e67113a92"
+    assert source["workflow_run_id"] == 32084697518
+    assert source["workflow_conclusion"] == "success"
+    assert source["artifact_zip_sha256"] == (
+        "3fc78ee29929e9a394fa0615db80422ffc6a55ae967b989e522f23175bec6c0f"
+    )
+    assert source["artifact_json_sha256"] == (
+        "ee09527d06abc58e612f7157b69908d5abe06101865714ce13a1287d1f0d7664"
+    )
+
+
+def test_real_tiny_dev_evidence_records_frozen_corpus_shape() -> None:
+    corpus = _evidence()["corpus"]
+    assert corpus["algorithm_version"] == "tiny-dev-corpus-v2"
+    assert corpus["prompt_count"] == 104
+    assert corpus["sample_count"] == 208
+    assert corpus["matched_pair_count"] == 104
+    assert corpus["target_length"] == 64
+    assert corpus["calibration_pairs_per_domain"] == 25
+    assert corpus["attack_pairs_per_domain"] == 1
+    assert corpus["required_domains"] == [
+        "general_explanatory",
+        "technical_explanation",
+        "conversational_prose",
+        "structured_instructional",
+    ]
+    assert corpus["required_splits"] == ["threshold_calibration", "attack_development"]
+
+
+def test_real_tiny_dev_evidence_records_pinned_runtime_identity() -> None:
+    runtime = _evidence()["runtime"]
+    assert runtime["model_id"] == "openai-community/gpt2"
+    assert runtime["model_revision"] == "607a30d783dfa663caf39e06633721c8d4cfcd7e"
+    assert runtime["tokenizer_revision"] == "607a30d783dfa663caf39e06633721c8d4cfcd7e"
+    assert runtime["seed_policy_id"] == "tiny-dev-paired-seed-v1"
+    assert runtime["seed_base"] == 401000
+    assert runtime["watermark_key_split"] == "DEV_KEYS"
+    assert runtime["backend_version"] == (
+        "transformers=5.16.0.dev0;torch=2.13.0+cpu;device=cpu;"
+        "revision=607a30d783dfa663caf39e06633721c8d4cfcd7e"
+    )
+
+
+def test_real_tiny_dev_evidence_records_independent_integrity_checks() -> None:
+    execution = _evidence()["execution"]
+    verification = _evidence()["verification"]
+    assert execution["pair_attempt_offset_counts"] == {"0": 99, "1": 4, "2": 1}
+    assert execution["maximum_pair_attempt_offset"] == 2
+    assert execution["exact_generation_token_track_count"] == 208
+    assert execution["text_only_track_count"] == 208
+    assert execution["text_only_exact_token_match_count"] == 197
+    assert all(verification.values())