FuckMark/.github/workflows/tiny-dev-effectiveness.yml at main · byte271/FuckMark · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
name: Frozen TinyDev Effectiveness Profile

on:
  pull_request:
    paths:
      - ".github/workflows/tiny-dev-effectiveness.yml"
      - "requirements-smoke.txt"
      - "fuckmark/corpus/**"
      - "fuckmark/experiments/effectiveness_plan.py"
      - "fuckmark/tiny_dev_corpus_hf.py"
      - "fuckmark/tiny_dev_effectiveness_plan_hf.py"
      - "fuckmark/tiny_dev_effectiveness_score_hf.py"
      - "fuckmark/transforms/contractions.py"
      - "fuckmark/transforms/effectiveness_profile.py"
      - "fuckmark/transforms/hard_invariants.py"
      - "fuckmark/transforms/lexical_rules.py"
      - "fuckmark/transforms/registry.py"
      - "fuckmark/transforms/scheduler.py"
      - "fuckmark/transforms/surface_rules.py"
      - "fuckmark/transforms/syntax_rules.py"
      - "fuckmark/transforms/tokenizer_geometry.py"
  workflow_dispatch:

permissions:
  contents: read

jobs:
  frozen-effectiveness:
    runs-on: ubuntu-24.04
    timeout-minutes: 180
    steps:
      - uses: actions/checkout@v4
        with:
          ref: ${{ github.event.pull_request.head.sha || github.sha }}
      - uses: actions/setup-python@v5
        with:
          python-version: "3.12"
          cache: pip
      - name: Install FuckMark
        run: python -m pip install -e .
      - name: Install exact CPU Torch runtime
        run: python -m pip install "torch==2.13.0" --index-url https://download.pytorch.org/whl/cpu
      - name: Install maintained Transformers SynthID dependencies
        run: python -m pip install -r requirements-smoke.txt
      - name: Reproduce canonical real TinyDev corpus
        run: >-
          python -m fuckmark.tiny_dev_corpus_hf
          --model openai-community/gpt2
          --model-revision 607a30d783dfa663caf39e06633721c8d4cfcd7e
          --device cpu
          --seed-base 401000
          --max-attempts 16
          --temperature 0.8
          --top-k 50
          --top-p 0.95
          --json artifacts/tiny-dev-corpus.json
      - name: Verify canonical corpus SHA-256
        shell: bash
        run: |
          actual="$(sha256sum artifacts/tiny-dev-corpus.json | cut -d' ' -f1)"
          test "$actual" = "ee09527d06abc58e612f7157b69908d5abe06101865714ce13a1287d1f0d7664"
      - name: Freeze detector-blind selections
        run: >-
          python -m fuckmark.tiny_dev_effectiveness_plan_hf
          --corpus-json artifacts/tiny-dev-corpus.json
          --model openai-community/gpt2
          --model-revision 607a30d783dfa663caf39e06633721c8d4cfcd7e
          --source-code-commit "$(git rev-parse HEAD)"
          --plan-json artifacts/tiny-dev-effectiveness-plan.json
          --provenance-json artifacts/tiny-dev-effectiveness-plan-provenance.json
      - name: Score the frozen plan in a separate process
        run: >-
          python -m fuckmark.tiny_dev_effectiveness_score_hf
          --corpus-json artifacts/tiny-dev-corpus.json
          --plan-json artifacts/tiny-dev-effectiveness-plan.json
          --model openai-community/gpt2
          --model-revision 607a30d783dfa663caf39e06633721c8d4cfcd7e
          --device cpu
          --source-code-commit "$(git rev-parse HEAD)"
          --json artifacts/tiny-dev-effectiveness-evidence.json
          --provenance-json artifacts/tiny-dev-effectiveness-score-provenance.json
      - uses: actions/upload-artifact@v4
        if: always()
        with:
          name: frozen-tiny-dev-effectiveness-profile
          path: artifacts/
          if-no-files-found: error