diff --git a/.github/workflows/tiny-dev-corpus.yml b/.github/workflows/tiny-dev-corpus.yml index e5481dcc..fab8a567 100644 --- a/.github/workflows/tiny-dev-corpus.yml +++ b/.github/workflows/tiny-dev-corpus.yml @@ -37,7 +37,7 @@ jobs: - name: Install FuckMark run: python -m pip install -e . - name: Install CPU Torch - run: python -m pip install "torch>=2.4,<3" --index-url https://download.pytorch.org/whl/cpu + run: python -m pip install "torch==2.13.0" --index-url https://download.pytorch.org/whl/cpu - name: Install maintained Transformers SynthID dependencies run: python -m pip install -r requirements-smoke.txt - name: Generate real TinyDev corpus diff --git a/evidence/tiny_dev/real-hf-gpt2-2026-08-18.json b/evidence/tiny_dev/real-hf-gpt2-2026-08-18.json new file mode 100644 index 00000000..31970254 --- /dev/null +++ b/evidence/tiny_dev/real-hf-gpt2-2026-08-18.json @@ -0,0 +1,95 @@ +{ + "algorithm_version": "real-tiny-dev-evidence-v1", + "evidence_kind": "development-corpus-generation", + "scientific_scope": "DEV_KEYS-only TinyDev generation prerequisite; not M6 readiness or a confirmatory result", + "source": { + "repository": "byte271/FuckMark", + "generator_head_commit": "9b829b3e594eeb1335d2c0509124deae6e7c0057", + "merged_main_commit": "33b4a8c10b9382dcc1068baacb99b30e67113a92", + "workflow_run_id": 32084697518, + "workflow_run_number": 1, + "workflow_name": "Real TinyDev Corpus", + "workflow_conclusion": "success", + "artifact_id": 9306497613, + "artifact_name": "real-tiny-dev-corpus", + "artifact_zip_sha256": "3fc78ee29929e9a394fa0615db80422ffc6a55ae967b989e522f23175bec6c0f", + "artifact_json_sha256": "ee09527d06abc58e612f7157b69908d5abe06101865714ce13a1287d1f0d7664" + }, + "corpus": { + "algorithm_version": "tiny-dev-corpus-v2", + "artifact_hash": "42240f72e17df1a048f44197f216a0b5f30128c26e2da93afbe92820946e4561", + "manifest_hash": "693788e76bceea43e72425b98f326bc09e4840d069eb47dacb18784ec7c2e388", + "prompt_manifest_hash": "6456a6f2fbcbc58c8905133b1981f84b500c06c9f653dc979a1d962ab0fc862f", + "sample_manifest_hash": "350aa801cbb12c0908686b39b2ffb77bc4419f3b71e0210c42d7735dbd3259be", + "prompt_count": 104, + "sample_count": 208, + "matched_pair_count": 104, + "target_length": 64, + "calibration_pairs_per_domain": 25, + "attack_pairs_per_domain": 1, + "required_domains": [ + "general_explanatory", + "technical_explanation", + "conversational_prose", + "structured_instructional" + ], + "required_splits": [ + "threshold_calibration", + "attack_development" + ], + "language": "en", + "deduplication_policy": "exact_utf8_v1", + "model_identity_hash": "1be91dc38048d8f69fc45d5fb8175b0edb7c6ec807af1f6b85aa657343dbb95e", + "generation_matching_signature_hash": "8de16173bb6be768141be143ffd01fdf28b623c64b461fca6d77c3abdeb5df95", + "watermark_condition_hash": "4cdef888bf3ef50029b6acef203d3efbbc2ae3f29cad5f1075bcb61202f89d59" + }, + "runtime": { + "model_id": "openai-community/gpt2", + "model_revision": "607a30d783dfa663caf39e06633721c8d4cfcd7e", + "tokenizer_id": "openai-community/gpt2", + "tokenizer_revision": "607a30d783dfa663caf39e06633721c8d4cfcd7e", + "device": "cpu", + "dtype": "float32", + "backend_id": "huggingface-transformers-synthid-tiny-dev", + "backend_version": "transformers=5.16.0.dev0;torch=2.13.0+cpu;device=cpu;revision=607a30d783dfa663caf39e06633721c8d4cfcd7e", + "seed_policy_id": "tiny-dev-paired-seed-v1", + "seed_base": 401000, + "temperature": 0.8, + "top_k": 50, + "top_p": 0.95, + "max_new_tokens": 64, + "watermark_key_split": "DEV_KEYS", + "watermark_key_id": "dev-huggingface-synthid-default-v1", + "watermark_config_hash": "d5918a5b92bd723b78f393d986b812f786b2855c767eb7ab60192e63d2a34086" + }, + "execution": { + "pair_attempt_offset_counts": { + "0": 99, + "1": 4, + "2": 1 + }, + "maximum_pair_attempt_offset": 2, + "exact_generation_token_track_count": 208, + "text_only_track_count": 208, + "text_only_exact_token_match_count": 197, + "text_only_retokenized_length_min": 63, + "text_only_retokenized_length_max": 65 + }, + "verification": { + "canonical_nested_hashes_recomputed": true, + "artifact_hash_recomputed": true, + "manifest_hash_recomputed": true, + "prompt_ids_unique": true, + "sample_ids_unique": true, + "sample_text_hashes_unique": true, + "generation_continuation_hashes_unique": true, + "all_generation_realized_lengths_exactly_64": true, + "all_samples_use_dev_keys": true, + "all_pairs_have_one_watermarked_and_one_unwatermarked_sample": true, + "all_pairs_share_one_actual_seed": true, + "all_samples_share_one_model_identity": true, + "all_samples_share_one_generation_matching_signature": true, + "all_samples_share_one_watermark_condition": true + }, + "evidence_hash": "c093a248be705437e812a3554c4e38468a010822415565dd0235e9ffac6d78cd" +} diff --git a/tests/test_real_tiny_dev_evidence.py b/tests/test_real_tiny_dev_evidence.py new file mode 100644 index 00000000..702392cd --- /dev/null +++ b/tests/test_real_tiny_dev_evidence.py @@ -0,0 +1,79 @@ +import json +from pathlib import Path + +from fuckmark.hashing import sha256_json + + +_EVIDENCE_PATH = Path("evidence/tiny_dev/real-hf-gpt2-2026-08-18.json") + + +def _evidence() -> dict[str, object]: + return json.loads(_EVIDENCE_PATH.read_text(encoding="utf-8")) + + +def test_real_tiny_dev_evidence_is_self_hashed_and_scoped() -> None: + evidence = _evidence() + evidence_hash = evidence.pop("evidence_hash") + assert evidence_hash == sha256_json(evidence) + assert evidence["algorithm_version"] == "real-tiny-dev-evidence-v1" + assert evidence["evidence_kind"] == "development-corpus-generation" + assert evidence["scientific_scope"] == ( + "DEV_KEYS-only TinyDev generation prerequisite; not M6 readiness or a confirmatory result" + ) + + +def test_real_tiny_dev_evidence_binds_successful_source_run() -> None: + source = _evidence()["source"] + assert source["generator_head_commit"] == "9b829b3e594eeb1335d2c0509124deae6e7c0057" + assert source["merged_main_commit"] == "33b4a8c10b9382dcc1068baacb99b30e67113a92" + assert source["workflow_run_id"] == 32084697518 + assert source["workflow_conclusion"] == "success" + assert source["artifact_zip_sha256"] == ( + "3fc78ee29929e9a394fa0615db80422ffc6a55ae967b989e522f23175bec6c0f" + ) + assert source["artifact_json_sha256"] == ( + "ee09527d06abc58e612f7157b69908d5abe06101865714ce13a1287d1f0d7664" + ) + + +def test_real_tiny_dev_evidence_records_frozen_corpus_shape() -> None: + corpus = _evidence()["corpus"] + assert corpus["algorithm_version"] == "tiny-dev-corpus-v2" + assert corpus["prompt_count"] == 104 + assert corpus["sample_count"] == 208 + assert corpus["matched_pair_count"] == 104 + assert corpus["target_length"] == 64 + assert corpus["calibration_pairs_per_domain"] == 25 + assert corpus["attack_pairs_per_domain"] == 1 + assert corpus["required_domains"] == [ + "general_explanatory", + "technical_explanation", + "conversational_prose", + "structured_instructional", + ] + assert corpus["required_splits"] == ["threshold_calibration", "attack_development"] + + +def test_real_tiny_dev_evidence_records_pinned_runtime_identity() -> None: + runtime = _evidence()["runtime"] + assert runtime["model_id"] == "openai-community/gpt2" + assert runtime["model_revision"] == "607a30d783dfa663caf39e06633721c8d4cfcd7e" + assert runtime["tokenizer_revision"] == "607a30d783dfa663caf39e06633721c8d4cfcd7e" + assert runtime["seed_policy_id"] == "tiny-dev-paired-seed-v1" + assert runtime["seed_base"] == 401000 + assert runtime["watermark_key_split"] == "DEV_KEYS" + assert runtime["backend_version"] == ( + "transformers=5.16.0.dev0;torch=2.13.0+cpu;device=cpu;" + "revision=607a30d783dfa663caf39e06633721c8d4cfcd7e" + ) + + +def test_real_tiny_dev_evidence_records_independent_integrity_checks() -> None: + execution = _evidence()["execution"] + verification = _evidence()["verification"] + assert execution["pair_attempt_offset_counts"] == {"0": 99, "1": 4, "2": 1} + assert execution["maximum_pair_attempt_offset"] == 2 + assert execution["exact_generation_token_track_count"] == 208 + assert execution["text_only_track_count"] == 208 + assert execution["text_only_exact_token_match_count"] == 197 + assert all(verification.values())