From 4f2929da167bfd16cb3617215f6cd685bd40f68e Mon Sep 17 00:00:00 2001 From: Byte271 Date: Mon, 17 Aug 2026 20:28:08 -0400 Subject: [PATCH 1/4] Add deterministic real TinyDev corpus builder --- fuckmark/corpus/tiny_dev_generation.py | 335 +++++++++++++++++++++++++ 1 file changed, 335 insertions(+) create mode 100644 fuckmark/corpus/tiny_dev_generation.py diff --git a/fuckmark/corpus/tiny_dev_generation.py b/fuckmark/corpus/tiny_dev_generation.py new file mode 100644 index 00000000..6ec4a4e8 --- /dev/null +++ b/fuckmark/corpus/tiny_dev_generation.py @@ -0,0 +1,335 @@ +from __future__ import annotations + +from dataclasses import dataclass +from typing import Protocol + +from ..hashing import sha256_json, sha256_text +from .generation import GenerationParameters, WatermarkCondition +from .identity import ModelTokenizerIdentity +from .prompt import PromptRecord +from .sample import CorpusSample +from .schema import CorpusDomain, CorpusSplit, KeySplit, MAX_GENERATION_SEED, WatermarkLabel +from .tiny_dev import ( + TINY_DEV_ATTACK_PAIRS_PER_DOMAIN, + TINY_DEV_CALIBRATION_PAIRS_PER_DOMAIN, + TINY_DEV_DOMAINS, + TINY_DEV_SPLITS, + TINY_DEV_TARGET_LENGTH, + TinyDevCorpusArtifact, + build_tiny_dev_corpus, +) +from .tokenization import GenerationTokenRecord, TextOnlyTokenRecord + + +TINY_DEV_GENERATION_ALGORITHM_VERSION = "tiny-dev-generation-v1" +TINY_DEV_SEED_POLICY_ID = "tiny-dev-paired-seed-v1" +TINY_DEV_PROMPT_SOURCE_ID = "fuckmark-tiny-dev-prompts-v1" +TINY_DEV_PROMPT_LICENSE_ID = "LicenseRef-FuckMark-Unspecified" +TINY_DEV_PROMPT_PROVENANCE = "fuckmark/corpus/tiny_dev_generation.py" +TINY_DEV_DEFAULT_MAX_ATTEMPTS = 16 +TINY_DEV_PAIR_SEED_STRIDE = 32 + +_TINY_DEV_TOPICS = ( + "reproducibility", + "control groups", + "measurement uncertainty", + "data provenance", + "source licensing", + "fixed thresholds", + "held-out evaluation", + "random baselines", + "failure logging", + "tokenization changes", + "prompt boundaries", + "calibration drift", + "independent replication", + "protected spans", + "semantic fidelity", + "edit budgets", + "deterministic replay", + "multiple testing", + "confidence intervals", + "model revision pinning", + "key separation", + "negative controls", + "ablation studies", + "domain shift", + "null results", + "research claims", +) + +_TINY_DEV_TEMPLATES = { + CorpusDomain.GENERAL_EXPLANATORY: ( + "Explain in plain English why {topic} matters in careful scientific work. " + "Write one coherent paragraph without a list." + ), + CorpusDomain.TECHNICAL_EXPLANATION: ( + "Give a technical explanation of {topic} in software experiments. " + "Define the main idea, one failure mode, and one validation check." + ), + CorpusDomain.CONVERSATIONAL_PROSE: ( + "Answer a colleague who asks why {topic} matters. " + "Use natural conversational prose while keeping the explanation precise." + ), + CorpusDomain.STRUCTURED_INSTRUCTIONAL: ( + "Write a short three-step instruction for applying {topic} in an experiment. " + "Use complete sentences and keep each step concrete." + ), +} + + +class TinyDevGenerationError(RuntimeError): + pass + + +@dataclass(frozen=True, slots=True) +class TinyDevGeneratedContinuation: + text: str + input_token_ids: tuple[int, ...] + attention_mask: tuple[int, ...] + continuation_token_ids: tuple[int, ...] + text_only_token_ids: tuple[int, ...] | None + + def __post_init__(self) -> None: + if not isinstance(self.text, str) or not self.text: + raise ValueError("generated continuation text must be a non-empty string") + for name, value in ( + ("input_token_ids", self.input_token_ids), + ("attention_mask", self.attention_mask), + ("continuation_token_ids", self.continuation_token_ids), + ): + if not isinstance(value, tuple): + raise TypeError(f"{name} must be a tuple") + if not self.input_token_ids: + raise ValueError("input_token_ids must not be empty") + if len(self.attention_mask) != len(self.input_token_ids): + raise ValueError("attention_mask length must match input_token_ids") + if any(type(value) is not int or value < 0 for value in self.input_token_ids): + raise ValueError("input_token_ids must contain non-negative integers") + if any(type(value) is not int or value not in (0, 1) for value in self.attention_mask): + raise ValueError("attention_mask must contain binary integers") + if not self.continuation_token_ids: + raise ValueError("continuation_token_ids must not be empty") + if any(type(value) is not int or value < 0 for value in self.continuation_token_ids): + raise ValueError("continuation_token_ids must contain non-negative integers") + if self.text_only_token_ids is not None: + if not isinstance(self.text_only_token_ids, tuple): + raise TypeError("text_only_token_ids must be a tuple or None") + if not self.text_only_token_ids: + raise ValueError("text_only_token_ids must not be empty when present") + if any(type(value) is not int or value < 0 for value in self.text_only_token_ids): + raise ValueError("text_only_token_ids must contain non-negative integers") + + +class TinyDevGenerationBackend(Protocol): + @property + def model_identity(self) -> ModelTokenizerIdentity: ... + + @property + def watermark_condition(self) -> WatermarkCondition: ... + + def generation_parameters(self, seed: int) -> GenerationParameters: ... + + def generate( + self, + prompt: str, + seed: int, + *, + watermarked: bool, + ) -> TinyDevGeneratedContinuation: ... + + +def _pairs_for_split(split: CorpusSplit) -> int: + if split is CorpusSplit.THRESHOLD_CALIBRATION: + return TINY_DEV_CALIBRATION_PAIRS_PER_DOMAIN + if split is CorpusSplit.ATTACK_DEVELOPMENT: + return TINY_DEV_ATTACK_PAIRS_PER_DOMAIN + raise ValueError("split is outside the frozen tiny development profile") + + +def _prompt_source_hash() -> str: + return sha256_json( + { + "algorithm_version": TINY_DEV_GENERATION_ALGORITHM_VERSION, + "source_id": TINY_DEV_PROMPT_SOURCE_ID, + "license_id": TINY_DEV_PROMPT_LICENSE_ID, + "provenance": TINY_DEV_PROMPT_PROVENANCE, + "topics": _TINY_DEV_TOPICS, + "templates": tuple( + (domain.value, _TINY_DEV_TEMPLATES[domain]) + for domain in TINY_DEV_DOMAINS + ), + } + ) + + +def build_tiny_dev_prompt_records() -> tuple[PromptRecord, ...]: + if len(_TINY_DEV_TOPICS) != ( + TINY_DEV_CALIBRATION_PAIRS_PER_DOMAIN + TINY_DEV_ATTACK_PAIRS_PER_DOMAIN + ): + raise RuntimeError("tiny development prompt topics do not match the frozen pair profile") + source_hash = _prompt_source_hash() + records: list[PromptRecord] = [] + for split in TINY_DEV_SPLITS: + pair_count = _pairs_for_split(split) + topic_offset = ( + 0 + if split is CorpusSplit.THRESHOLD_CALIBRATION + else TINY_DEV_CALIBRATION_PAIRS_PER_DOMAIN + ) + for domain in TINY_DEV_DOMAINS: + template = _TINY_DEV_TEMPLATES[domain] + for local_index in range(pair_count): + topic_index = topic_offset + local_index + topic = _TINY_DEV_TOPICS[topic_index] + prompt_id = f"tiny-dev-{split.value}-{domain.value}-{topic_index:02d}" + records.append( + PromptRecord.create( + prompt_id=prompt_id, + prompt_family_id=f"tiny-dev-family-{domain.value}-{topic_index:02d}", + domain=domain, + split=split, + source_id=TINY_DEV_PROMPT_SOURCE_ID, + source_hash=source_hash, + license_id=TINY_DEV_PROMPT_LICENSE_ID, + provenance=TINY_DEV_PROMPT_PROVENANCE, + text=template.format(topic=topic), + ) + ) + return tuple(sorted(records, key=lambda value: value.prompt_id)) + + +def _sample_id(prompt: PromptRecord, label: WatermarkLabel) -> str: + return f"{prompt.prompt_id}-{label.value}" + + +def _match_id(prompt: PromptRecord) -> str: + return f"match-{prompt.prompt_id}" + + +def _build_sample( + *, + prompt: PromptRecord, + label: WatermarkLabel, + generated: TinyDevGeneratedContinuation, + backend: TinyDevGenerationBackend, + seed: int, +) -> CorpusSample: + model = backend.model_identity + generation = backend.generation_parameters(seed) + if generation.seed != seed: + raise TinyDevGenerationError("backend generation parameters do not bind the requested seed") + if generation.seed_policy_id != TINY_DEV_SEED_POLICY_ID: + raise TinyDevGenerationError("backend seed policy does not match the frozen tiny development policy") + generation_tokens = GenerationTokenRecord.create( + input_token_ids=generated.input_token_ids, + attention_mask=generated.attention_mask, + generated_sequence_ids=generated.input_token_ids + generated.continuation_token_ids, + continuation_start_index=len(generated.input_token_ids), + continuation_token_ids=generated.continuation_token_ids, + prompt_length_after_templating=sum(generated.attention_mask), + model_tokenizer_identity_hash=model.identity_hash, + ) + text_only = None + if generated.text_only_token_ids is not None: + text_only = TextOnlyTokenRecord.create( + source_text_sha256=sha256_text(generated.text), + token_ids=generated.text_only_token_ids, + model_tokenizer_identity_hash=model.identity_hash, + ) + return CorpusSample.create( + sample_id=_sample_id(prompt, label), + match_id=_match_id(prompt), + prompt_id=prompt.prompt_id, + prompt_family_id=prompt.prompt_family_id, + domain=prompt.domain, + split=prompt.split, + label=label, + text=generated.text, + model=model, + generation=generation, + watermark=backend.watermark_condition, + target_length=TINY_DEV_TARGET_LENGTH, + generation_tokens=generation_tokens, + text_only_tokens=text_only, + ) + + +def build_real_tiny_dev_corpus( + backend: TinyDevGenerationBackend, + *, + corpus_id: str = "fuckmark-tiny-dev-real-v1", + seed_base: int = 401000, + max_attempts: int = TINY_DEV_DEFAULT_MAX_ATTEMPTS, +) -> TinyDevCorpusArtifact: + if type(seed_base) is not int or seed_base < 0: + raise ValueError("seed_base must be a non-negative integer") + if type(max_attempts) is not int or max_attempts <= 0: + raise ValueError("max_attempts must be a positive integer") + prompts = build_tiny_dev_prompt_records() + if seed_base + len(prompts) * TINY_DEV_PAIR_SEED_STRIDE + max_attempts > MAX_GENERATION_SEED: + raise ValueError("seed schedule exceeds the 64-bit generation-seed range") + + samples: list[CorpusSample] = [] + seen_text_hashes: set[str] = set() + seen_token_hashes: set[str] = set() + + for pair_index, prompt in enumerate(prompts): + pair_seed_base = seed_base + pair_index * TINY_DEV_PAIR_SEED_STRIDE + accepted = False + for attempt in range(max_attempts): + seed = pair_seed_base + attempt + control = backend.generate(prompt.text, seed, watermarked=False) + watermarked = backend.generate(prompt.text, seed, watermarked=True) + candidates = ( + (WatermarkLabel.UNWATERMARKED, control), + (WatermarkLabel.WATERMARKED, watermarked), + ) + if any( + len(value.continuation_token_ids) != TINY_DEV_TARGET_LENGTH + for _, value in candidates + ): + continue + text_hashes = tuple(sha256_text(value.text) for _, value in candidates) + token_hashes = tuple( + sha256_json(value.continuation_token_ids) + for _, value in candidates + ) + if len(set(text_hashes)) != len(text_hashes) or len(set(token_hashes)) != len(token_hashes): + continue + if any(value in seen_text_hashes for value in text_hashes): + continue + if any(value in seen_token_hashes for value in token_hashes): + continue + + pair_samples = tuple( + _build_sample( + prompt=prompt, + label=label, + generated=value, + backend=backend, + seed=seed, + ) + for label, value in candidates + ) + samples.extend(pair_samples) + seen_text_hashes.update(text_hashes) + seen_token_hashes.update(token_hashes) + accepted = True + break + if not accepted: + raise TinyDevGenerationError( + f"could not produce an exact, unique 64-token pair for {prompt.prompt_id} " + f"within {max_attempts} deterministic attempts" + ) + + artifact = build_tiny_dev_corpus(corpus_id, prompts, samples) + if any(sample.watermark.key_split is not KeySplit.DEV for sample in artifact.manifest.samples): + raise TinyDevGenerationError("real tiny development generation must use DEV_KEYS") + for match_id in sorted({sample.match_id for sample in artifact.manifest.samples}): + pair = tuple(sample for sample in artifact.manifest.samples if sample.match_id == match_id) + if len({sample.generation.seed for sample in pair}) != 1: + raise TinyDevGenerationError("real tiny development runner requires one shared seed per on/off pair") + if any(sample.generation.seed_policy_id != TINY_DEV_SEED_POLICY_ID for sample in pair): + raise TinyDevGenerationError("real tiny development pair seed policy drifted") + return artifact From 5547c7ce8fe0bbea8440d0563f70fc0488bee903 Mon Sep 17 00:00:00 2001 From: Byte271 Date: Mon, 17 Aug 2026 20:28:28 -0400 Subject: [PATCH 2/4] Test real TinyDev generation contracts --- tests/test_tiny_dev_generation.py | 177 ++++++++++++++++++++++++++++++ 1 file changed, 177 insertions(+) create mode 100644 tests/test_tiny_dev_generation.py diff --git a/tests/test_tiny_dev_generation.py b/tests/test_tiny_dev_generation.py new file mode 100644 index 00000000..9b30b878 --- /dev/null +++ b/tests/test_tiny_dev_generation.py @@ -0,0 +1,177 @@ +from __future__ import annotations + +from collections import defaultdict + +import pytest + +from fuckmark.corpus import ( + CorpusDomain, + CorpusSplit, + GenerationParameters, + KeySplit, + ModelTokenizerIdentity, + PaddingSide, + WatermarkCondition, + WatermarkLabel, +) +from fuckmark.corpus.tiny_dev_generation import ( + TINY_DEV_GENERATION_ALGORITHM_VERSION, + TINY_DEV_PROMPT_LICENSE_ID, + TINY_DEV_PROMPT_SOURCE_ID, + TINY_DEV_SEED_POLICY_ID, + TinyDevGeneratedContinuation, + TinyDevGenerationError, + build_real_tiny_dev_corpus, + build_tiny_dev_prompt_records, +) +from fuckmark.hashing import sha256_json, sha256_text + + +class FakeTinyDevBackend: + def __init__(self, *, short_first_attempt: bool = False, always_short: bool = False) -> None: + self._model = ModelTokenizerIdentity.create( + model_id="example/tiny-dev-model", + model_revision="a" * 40, + tokenizer_id="example/tiny-dev-tokenizer", + tokenizer_revision="b" * 40, + chat_template_present=False, + chat_template_hash=sha256_text(""), + special_token_map_hash=sha256_json({"eos_token": "", "pad_token": ""}), + padding_side=PaddingSide.LEFT, + bos_token_id=None, + eos_token_id=2, + pad_token_id=0, + add_bos_token=False, + add_eos_token=False, + ) + self._watermark = WatermarkCondition.create( + sha256_json({"ngram_len": 5, "keys": (1, 2, 3)}), + KeySplit.DEV, + "dev-test-key", + ) + self._short_first_attempt = short_first_attempt + self._always_short = always_short + self.calls: defaultdict[tuple[str, bool], list[int]] = defaultdict(list) + + @property + def model_identity(self) -> ModelTokenizerIdentity: + return self._model + + @property + def watermark_condition(self) -> WatermarkCondition: + return self._watermark + + def generation_parameters(self, seed: int) -> GenerationParameters: + return GenerationParameters.create( + seed=seed, + seed_policy_id=TINY_DEV_SEED_POLICY_ID, + temperature=0.8, + top_k=50, + top_p=0.95, + max_new_tokens=64, + do_sample=True, + dtype="float32", + device="cpu", + backend_id="fake-tiny-dev", + backend_version="1", + ) + + def generate(self, prompt: str, seed: int, *, watermarked: bool) -> TinyDevGeneratedContinuation: + self.calls[(prompt, watermarked)].append(seed) + first_seed = self.calls[(prompt, watermarked)][0] + short = self._always_short or ( + self._short_first_attempt and seed == first_seed + ) + continuation_length = 63 if short else 64 + prompt_token = 10 + (sum(prompt.encode("utf-8")) % 1000) + label_offset = 100_000 if watermarked else 200_000 + continuation = tuple( + label_offset + ((seed * 131 + index) % 90_000) + for index in range(continuation_length) + ) + text = ( + f"{'watermarked' if watermarked else 'control'} output for " + f"{sha256_text(prompt)[:12]} seed {seed}" + ) + text_only = tuple(300_000 + ((seed * 17 + index) % 80_000) for index in range(16)) + return TinyDevGeneratedContinuation( + text=text, + input_token_ids=(prompt_token, prompt_token + 1), + attention_mask=(1, 1), + continuation_token_ids=continuation, + text_only_token_ids=text_only, + ) + + +def test_prompt_suite_matches_frozen_tiny_dev_matrix() -> None: + prompts = build_tiny_dev_prompt_records() + assert len(prompts) == 104 + assert len({prompt.prompt_id for prompt in prompts}) == 104 + assert len({prompt.prompt_family_id for prompt in prompts}) == 104 + assert {prompt.source_id for prompt in prompts} == {TINY_DEV_PROMPT_SOURCE_ID} + assert {prompt.license_id for prompt in prompts} == {TINY_DEV_PROMPT_LICENSE_ID} + assert len({prompt.source_hash for prompt in prompts}) == 1 + counts = defaultdict(int) + for prompt in prompts: + counts[(prompt.split, prompt.domain)] += 1 + for domain in CorpusDomain: + assert counts[(CorpusSplit.THRESHOLD_CALIBRATION, domain)] == 25 + assert counts[(CorpusSplit.ATTACK_DEVELOPMENT, domain)] == 1 + + +def test_real_tiny_dev_runner_builds_exact_paired_corpus() -> None: + backend = FakeTinyDevBackend() + artifact = build_real_tiny_dev_corpus(backend, seed_base=5000) + assert artifact.algorithm_version == "tiny-dev-corpus-v2" + assert len(artifact.manifest.prompts) == 104 + assert len(artifact.manifest.samples) == 208 + assert all(sample.generation_realized_length == 64 for sample in artifact.manifest.samples) + assert all(sample.watermark.key_split is KeySplit.DEV for sample in artifact.manifest.samples) + assert all( + sample.generation.seed_policy_id == TINY_DEV_SEED_POLICY_ID + for sample in artifact.manifest.samples + ) + by_match = defaultdict(list) + for sample in artifact.manifest.samples: + by_match[sample.match_id].append(sample) + assert len(by_match) == 104 + for pair in by_match.values(): + assert {sample.label for sample in pair} == { + WatermarkLabel.UNWATERMARKED, + WatermarkLabel.WATERMARKED, + } + assert len({sample.generation.seed for sample in pair}) == 1 + assert len({sample.generation.matching_signature_hash for sample in pair}) == 1 + assert all(sample.text_only_tokens is not None for sample in pair) + + +def test_real_tiny_dev_runner_retries_whole_pair_after_early_eos() -> None: + backend = FakeTinyDevBackend(short_first_attempt=True) + artifact = build_real_tiny_dev_corpus(backend, seed_base=7000, max_attempts=2) + first_prompt = build_tiny_dev_prompt_records()[0] + first_pair = tuple( + sample for sample in artifact.manifest.samples + if sample.prompt_id == first_prompt.prompt_id + ) + assert len(first_pair) == 2 + assert {sample.generation.seed for sample in first_pair} == {7001} + assert backend.calls[(first_prompt.text, False)] == [7000, 7001] + assert backend.calls[(first_prompt.text, True)] == [7000, 7001] + + +def test_real_tiny_dev_runner_fails_closed_when_exact_length_cannot_be_reached() -> None: + backend = FakeTinyDevBackend(always_short=True) + with pytest.raises(TinyDevGenerationError, match="within 2 deterministic attempts"): + build_real_tiny_dev_corpus(backend, seed_base=9000, max_attempts=2) + + +def test_generation_contract_rejects_invalid_continuation_shapes() -> None: + with pytest.raises(ValueError, match="attention_mask length"): + TinyDevGeneratedContinuation( + text="output", + input_token_ids=(1, 2), + attention_mask=(1,), + continuation_token_ids=(3,), + text_only_token_ids=(3,), + ) + assert TINY_DEV_GENERATION_ALGORITHM_VERSION == "tiny-dev-generation-v1" From b7b87dbfd8443873cd518855405cec78e65e98cf Mon Sep 17 00:00:00 2001 From: Byte271 Date: Mon, 17 Aug 2026 20:28:51 -0400 Subject: [PATCH 3/4] Add pinned Hugging Face TinyDev generator --- fuckmark/tiny_dev_corpus_hf.py | 253 +++++++++++++++++++++++++++++++++ 1 file changed, 253 insertions(+) create mode 100644 fuckmark/tiny_dev_corpus_hf.py diff --git a/fuckmark/tiny_dev_corpus_hf.py b/fuckmark/tiny_dev_corpus_hf.py new file mode 100644 index 00000000..f37b5f82 --- /dev/null +++ b/fuckmark/tiny_dev_corpus_hf.py @@ -0,0 +1,253 @@ +from __future__ import annotations + +import argparse +import sys +from pathlib import Path + +from .config import canonical_json_text +from .corpus import ( + GenerationParameters, + KeySplit, + ModelTokenizerIdentity, + PaddingSide, + WatermarkCondition, +) +from .corpus.tiny_dev import TINY_DEV_TARGET_LENGTH +from .corpus.tiny_dev_generation import ( + TINY_DEV_SEED_POLICY_ID, + TinyDevGeneratedContinuation, + build_real_tiny_dev_corpus, +) +from .hashing import sha256_json, sha256_text + + +DEFAULT_MODEL_ID = "openai-community/gpt2" +DEFAULT_MODEL_REVISION = "607a30d783dfa663caf39e06633721c8d4cfcd7e" +DEFAULT_KEYS = (654, 400, 836, 123, 340, 443, 597, 160, 57) +DEFAULT_NGRAM_LEN = 5 + + +class HuggingFaceTinyDevBackend: + def __init__( + self, + model_id: str, + model_revision: str, + *, + device: str, + temperature: float, + top_k: int, + top_p: float, + ngram_len: int = DEFAULT_NGRAM_LEN, + keys: tuple[int, ...] = DEFAULT_KEYS, + ) -> None: + try: + import torch + import transformers + from transformers import AutoModelForCausalLM, AutoTokenizer, SynthIDTextWatermarkingConfig + except ImportError as error: + raise RuntimeError( + "Install the smoke dependencies before running the real TinyDev corpus generator" + ) from error + if len(model_revision) not in (40, 64) or any( + character not in "0123456789abcdef" for character in model_revision + ): + raise ValueError("model_revision must be an immutable lowercase hexadecimal revision") + if device == "auto": + device = "cuda" if torch.cuda.is_available() else "cpu" + if device not in {"cpu", "cuda"}: + raise ValueError("device must be auto, cpu, or cuda") + if device == "cuda" and not torch.cuda.is_available(): + raise RuntimeError("CUDA was requested but is not available") + + self._torch = torch + self._transformers = transformers + self._model_id = model_id + self._model_revision = model_revision + self._device = device + self._temperature = temperature + self._top_k = top_k + self._top_p = top_p + + self._tokenizer = AutoTokenizer.from_pretrained( + model_id, + revision=model_revision, + padding_side="left", + ) + if self._tokenizer.eos_token_id is None: + raise RuntimeError("the selected tokenizer must define eos_token_id") + if self._tokenizer.pad_token_id is None: + self._tokenizer.pad_token = self._tokenizer.eos_token + self._model = AutoModelForCausalLM.from_pretrained( + model_id, + revision=model_revision, + ) + self._model.to(device) + self._model.eval() + + self._watermark_config = SynthIDTextWatermarkingConfig( + ngram_len=ngram_len, + keys=list(keys), + ) + watermark_payload = { + "ngram_len": ngram_len, + "keys": keys, + "context_history_size": 1024, + "sampling_table_seed": 0, + "sampling_table_size": 65536, + "skip_first_ngram_calls": False, + "debug_mode": False, + } + self._watermark_condition = WatermarkCondition.create( + sha256_json(watermark_payload), + KeySplit.DEV, + "dev-huggingface-synthid-default-v1", + ) + + chat_template = getattr(self._tokenizer, "chat_template", None) + if chat_template is not None and not isinstance(chat_template, str): + raise RuntimeError("tokenizer chat_template must be a string when present") + self._model_identity = ModelTokenizerIdentity.create( + model_id=model_id, + model_revision=model_revision, + tokenizer_id=model_id, + tokenizer_revision=model_revision, + chat_template_present=bool(chat_template), + chat_template_hash=sha256_text(chat_template or ""), + special_token_map_hash=sha256_json(self._tokenizer.special_tokens_map), + padding_side=PaddingSide.LEFT, + bos_token_id=self._tokenizer.bos_token_id, + eos_token_id=self._tokenizer.eos_token_id, + pad_token_id=self._tokenizer.pad_token_id, + add_bos_token=bool(getattr(self._tokenizer, "add_bos_token", False)), + add_eos_token=bool(getattr(self._tokenizer, "add_eos_token", False)), + ) + first_parameter = next(self._model.parameters()) + self._dtype = str(first_parameter.dtype).removeprefix("torch.") + self._backend_version = ( + f"transformers={transformers.__version__};" + f"torch={torch.__version__};device={device};revision={model_revision}" + ) + + @property + def model_identity(self) -> ModelTokenizerIdentity: + return self._model_identity + + @property + def watermark_condition(self) -> WatermarkCondition: + return self._watermark_condition + + def generation_parameters(self, seed: int) -> GenerationParameters: + return GenerationParameters.create( + seed=seed, + seed_policy_id=TINY_DEV_SEED_POLICY_ID, + temperature=self._temperature, + top_k=self._top_k, + top_p=self._top_p, + max_new_tokens=TINY_DEV_TARGET_LENGTH, + do_sample=True, + dtype=self._dtype, + device=self._device, + backend_id="huggingface-transformers-synthid-tiny-dev", + backend_version=self._backend_version, + ) + + def generate( + self, + prompt: str, + seed: int, + *, + watermarked: bool, + ) -> TinyDevGeneratedContinuation: + torch = self._torch + torch.manual_seed(seed) + if self._device == "cuda": + torch.cuda.manual_seed_all(seed) + + encoded = self._tokenizer(prompt, return_tensors="pt") + encoded = {key: value.to(self._device) for key, value in encoded.items()} + kwargs = { + "do_sample": True, + "temperature": self._temperature, + "top_k": self._top_k, + "top_p": self._top_p, + "max_new_tokens": TINY_DEV_TARGET_LENGTH, + "pad_token_id": self._tokenizer.pad_token_id, + } + if watermarked: + kwargs["watermarking_config"] = self._watermark_config + + with torch.inference_mode(): + output = self._model.generate(**encoded, **kwargs) + input_ids = tuple( + int(value) for value in encoded["input_ids"][0].detach().cpu().tolist() + ) + attention_mask = tuple( + int(value) for value in encoded["attention_mask"][0].detach().cpu().tolist() + ) + continuation = tuple( + int(value) + for value in output[0, len(input_ids) :].detach().cpu().tolist() + ) + text = self._tokenizer.decode(continuation, skip_special_tokens=True) + if not text: + raise RuntimeError("generation produced an empty decoded continuation") + text_only = tuple( + int(value) for value in self._tokenizer.encode(text, add_special_tokens=False) + ) + if not text_only: + raise RuntimeError("text-only re-encoding produced no tokens") + return TinyDevGeneratedContinuation( + text=text, + input_token_ids=input_ids, + attention_mask=attention_mask, + continuation_token_ids=continuation, + text_only_token_ids=text_only, + ) + + +def _parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser(prog="fuckmark-tiny-dev-corpus-hf") + parser.add_argument("--model", default=DEFAULT_MODEL_ID) + parser.add_argument("--model-revision", default=DEFAULT_MODEL_REVISION) + parser.add_argument("--device", choices=("auto", "cpu", "cuda"), default="auto") + parser.add_argument("--seed-base", type=int, default=401000) + parser.add_argument("--max-attempts", type=int, default=16) + parser.add_argument("--temperature", type=float, default=0.8) + parser.add_argument("--top-k", type=int, default=50) + parser.add_argument("--top-p", type=float, default=0.95) + parser.add_argument("--json", type=Path, default=Path("artifacts/tiny-dev-corpus.json")) + return parser + + +def main(argv: list[str] | None = None) -> int: + args = _parser().parse_args(argv) + backend = HuggingFaceTinyDevBackend( + args.model, + args.model_revision, + device=args.device, + temperature=args.temperature, + top_k=args.top_k, + top_p=args.top_p, + ) + artifact = build_real_tiny_dev_corpus( + backend, + seed_base=args.seed_base, + max_attempts=args.max_attempts, + ) + args.json.parent.mkdir(parents=True, exist_ok=True) + args.json.write_text(canonical_json_text(artifact) + "\n", encoding="utf-8") + sys.stdout.write(f"artifact_hash={artifact.artifact_hash}\n") + sys.stdout.write(f"manifest_hash={artifact.manifest.manifest_hash}\n") + sys.stdout.write(f"prompt_count={len(artifact.manifest.prompts)}\n") + sys.stdout.write(f"sample_count={len(artifact.manifest.samples)}\n") + sys.stdout.write(f"model_identity_hash={artifact.model_identity_hash}\n") + sys.stdout.write( + f"generation_signature_hash={artifact.generation_matching_signature_hash}\n" + ) + sys.stdout.write(f"watermark_condition_hash={artifact.watermark_condition_hash}\n") + sys.stdout.write(f"json={args.json.as_posix()}\n") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) From 9b829b3e594eeb1335d2c0509124deae6e7c0057 Mon Sep 17 00:00:00 2001 From: Byte271 Date: Mon, 17 Aug 2026 20:29:02 -0400 Subject: [PATCH 4/4] Run real TinyDev corpus generation in Actions --- .github/workflows/tiny-dev-corpus.yml | 60 +++++++++++++++++++++++++++ 1 file changed, 60 insertions(+) create mode 100644 .github/workflows/tiny-dev-corpus.yml diff --git a/.github/workflows/tiny-dev-corpus.yml b/.github/workflows/tiny-dev-corpus.yml new file mode 100644 index 00000000..e5481dcc --- /dev/null +++ b/.github/workflows/tiny-dev-corpus.yml @@ -0,0 +1,60 @@ +name: Real TinyDev Corpus + +on: + workflow_dispatch: + inputs: + seed_base: + description: First deterministic paired generation seed + required: true + default: "401000" + type: string + max_attempts: + description: Deterministic retry budget for exact 64-token unique pairs + required: true + default: "16" + type: string + pull_request: + paths: + - ".github/workflows/tiny-dev-corpus.yml" + - "fuckmark/corpus/tiny_dev_generation.py" + - "fuckmark/tiny_dev_corpus_hf.py" + - "tests/test_tiny_dev_generation.py" + - "requirements-smoke.txt" + +permissions: + contents: read + +jobs: + real-tiny-dev: + runs-on: ubuntu-24.04 + timeout-minutes: 180 + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: "3.12" + cache: pip + - name: Install FuckMark + run: python -m pip install -e . + - name: Install CPU Torch + run: python -m pip install "torch>=2.4,<3" --index-url https://download.pytorch.org/whl/cpu + - name: Install maintained Transformers SynthID dependencies + run: python -m pip install -r requirements-smoke.txt + - name: Generate real TinyDev corpus + run: >- + python -m fuckmark.tiny_dev_corpus_hf + --model openai-community/gpt2 + --model-revision 607a30d783dfa663caf39e06633721c8d4cfcd7e + --device cpu + --seed-base "${{ inputs.seed_base || '401000' }}" + --max-attempts "${{ inputs.max_attempts || '16' }}" + --temperature 0.8 + --top-k 50 + --top-p 0.95 + --json artifacts/tiny-dev-corpus.json + - uses: actions/upload-artifact@v4 + if: always() + with: + name: real-tiny-dev-corpus + path: artifacts/ + if-no-files-found: error