Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 5 additions & 2 deletions .github/workflows/cycle8-carrier-detector.yml
Original file line number Diff line number Diff line change
Expand Up @@ -43,10 +43,13 @@ jobs:
- name: Install maintained Transformers SynthID dependencies
run: python -m pip install -r requirements-smoke.txt
- name: Run Cycle 8 exploratory detector compare on seed 890000
env:
INPUT_DEVICE: ${{ inputs.device || 'cpu' }}
INPUT_MAX_ATTEMPTS: ${{ inputs.max_attempts || '64' }}
run: >-
python -m fuckmark.cycle8_hf
--device "${{ inputs.device || 'cpu' }}"
--max-attempts "${{ inputs.max_attempts || '64' }}"
--device "$INPUT_DEVICE"
--max-attempts "$INPUT_MAX_ATTEMPTS"
--detector-json artifacts/cycle8-exploratory-890000/detector-compare.json
- uses: actions/upload-artifact@v4
if: always()
Expand Down
32 changes: 22 additions & 10 deletions .github/workflows/e26-open-adapter-transfer.yml
Original file line number Diff line number Diff line change
Expand Up @@ -49,17 +49,23 @@ jobs:
- name: Install maintained Transformers SynthID dependencies
run: python -m pip install -r requirements-smoke.txt
- name: Run maintained Transformers native smoke
env:
INPUT_MODEL: ${{ inputs.model }}
INPUT_PROMPT_LIMIT: ${{ inputs.prompt_limit }}
INPUT_SEED_BASE: ${{ inputs.seed_base }}
INPUT_MAX_NEW_TOKENS: ${{ inputs.max_new_tokens }}
INPUT_TARGET_FPR: ${{ inputs.target_fpr }}
run: >-
python -m fuckmark.synthid_smoke_hf
--model "${{ inputs.model }}"
--model "$INPUT_MODEL"
--device cpu
--prompt-limit "${{ inputs.prompt_limit }}"
--seed-base "${{ inputs.seed_base }}"
--max-new-tokens "${{ inputs.max_new_tokens }}"
--prompt-limit "$INPUT_PROMPT_LIMIT"
--seed-base "$INPUT_SEED_BASE"
--max-new-tokens "$INPUT_MAX_NEW_TOKENS"
--temperature 0.8
--top-k 50
--top-p 0.95
--target-fpr "${{ inputs.target_fpr }}"
--target-fpr "$INPUT_TARGET_FPR"
--json artifacts/huggingface.json
--csv artifacts/huggingface.csv
- uses: actions/upload-artifact@v4
Expand All @@ -86,17 +92,23 @@ jobs:
python -m pip install "numpy==1.26.0" "immutabledict==4.2.0" "transformers==4.43.3"
python -m pip install --no-deps "git+https://github.com/google-deepmind/synthid-text.git@addb4a158143c7c6851a1308f78b89fceed59683"
- name: Run pinned DeepMind reference native smoke
env:
INPUT_MODEL: ${{ inputs.model }}
INPUT_PROMPT_LIMIT: ${{ inputs.prompt_limit }}
INPUT_SEED_BASE: ${{ inputs.seed_base }}
INPUT_MAX_NEW_TOKENS: ${{ inputs.max_new_tokens }}
INPUT_TARGET_FPR: ${{ inputs.target_fpr }}
run: >-
python -m fuckmark.synthid_smoke_deepmind
--model "${{ inputs.model }}"
--model "$INPUT_MODEL"
--device cpu
--prompt-limit "${{ inputs.prompt_limit }}"
--seed-base "${{ inputs.seed_base }}"
--max-new-tokens "${{ inputs.max_new_tokens }}"
--prompt-limit "$INPUT_PROMPT_LIMIT"
--seed-base "$INPUT_SEED_BASE"
--max-new-tokens "$INPUT_MAX_NEW_TOKENS"
--temperature 0.8
--top-k 50
--top-p 0.95
--target-fpr "${{ inputs.target_fpr }}"
--target-fpr "$INPUT_TARGET_FPR"
--json artifacts/deepmind.json
--csv artifacts/deepmind.csv
- uses: actions/upload-artifact@v4
Expand Down
58 changes: 17 additions & 41 deletions .github/workflows/release-engineering.yml
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,11 @@ on:
- "docs/**"
- "LICENSE"
workflow_dispatch:
inputs:
publish_github_release:
description: Publish a GitHub Release for the existing v<pyproject version> tag. This job never creates tags and never deletes branches.
type: boolean
default: false

permissions:
contents: read
Expand Down Expand Up @@ -62,27 +67,25 @@ jobs:
if-no-files-found: error

release:
name: Ensure GitHub Release
if: github.event_name == 'push' && github.ref == 'refs/heads/main'
name: Publish GitHub Release
if: github.event_name == 'workflow_dispatch' && inputs.publish_github_release == true
needs: package-e2e
runs-on: ubuntu-latest
permissions:
contents: write
outputs:
published: ${{ steps.state.outputs.publish }}
steps:
- name: Checkout release commit
uses: actions/checkout@v7
with:
fetch-depth: 0
persist-credentials: true
persist-credentials: false

- name: Set up Python
uses: actions/setup-python@v7
with:
python-version: "3.12"

- name: Resolve release state
- name: Require existing version tag
id: state
env:
GH_TOKEN: ${{ github.token }}
Expand All @@ -93,27 +96,21 @@ jobs:
release_tag="v${package_version}"
echo "tag=$release_tag" >> "$GITHUB_OUTPUT"
git fetch --tags --force
if git rev-parse --verify --quiet "refs/tags/$release_tag" >/dev/null; then
echo "tag_exists=true" >> "$GITHUB_OUTPUT"
else
echo "tag_exists=false" >> "$GITHUB_OUTPUT"
if ! git rev-parse --verify --quiet "refs/tags/$release_tag" >/dev/null; then
printf '%s\n' "Tag $release_tag does not exist. Create and push that tag before publishing."
exit 1
fi
tagged_sha="$(git rev-parse "refs/tags/$release_tag^{commit}")"
if [ "$tagged_sha" != "$GITHUB_SHA" ]; then
printf '%s\n' "Tag $release_tag points at $tagged_sha, not this commit $GITHUB_SHA."
exit 1
fi
if gh release view "$release_tag" --repo "$GITHUB_REPOSITORY" >/dev/null 2>&1; then
echo "publish=false" >> "$GITHUB_OUTPUT"
else
echo "publish=true" >> "$GITHUB_OUTPUT"
fi

- name: Create immutable release tag
if: steps.state.outputs.tag_exists == 'false'
shell: bash
run: |
set -euo pipefail
git config user.name "github-actions[bot]"
git config user.email "41898282+github-actions[bot]@users.noreply.github.com"
git tag -a "${{ steps.state.outputs.tag }}" "$GITHUB_SHA" -m "${{ steps.state.outputs.tag }}"
git push origin "refs/tags/${{ steps.state.outputs.tag }}"

- name: Download verified distributions
if: steps.state.outputs.publish == 'true'
uses: actions/download-artifact@v8
Expand All @@ -130,24 +127,3 @@ jobs:
run: |
set -euo pipefail
gh release create "$RELEASE_TAG" release-dist/* --verify-tag --title "FuckMark $RELEASE_TAG" --generate-notes

cleanup-merged-branches:
name: Remove merged branches
if: needs.release.outputs.published == 'true'
needs: release
runs-on: ubuntu-latest
permissions:
contents: write
steps:
- name: Delete merged pull-request branches
env:
GH_TOKEN: ${{ github.token }}
shell: bash
run: |
set -euo pipefail
branches="$(gh api --paginate --slurp "repos/$GITHUB_REPOSITORY/pulls?state=closed&per_page=100" | jq -r --arg repo "$GITHUB_REPOSITORY" '.[][] | select(.merged_at != null and .head.repo.full_name == $repo) | .head.ref' | sort -u)"
while IFS= read -r branch; do
[[ -n "$branch" ]] || continue
encoded="$(jq -rn --arg value "$branch" '$value | @uri')"
gh api --method DELETE "repos/$GITHUB_REPOSITORY/git/refs/heads/$encoded" || true
done <<< "$branches"
16 changes: 11 additions & 5 deletions .github/workflows/synthid-eligible-geometry.yml
Original file line number Diff line number Diff line change
Expand Up @@ -53,14 +53,20 @@ jobs:
- name: Install SynthID experiment dependencies
run: python -m pip install -r requirements-smoke.txt
- name: Run public-eligibility geometry pilot
env:
INPUT_MODEL: ${{ inputs.model }}
INPUT_PROMPT_LIMIT: ${{ inputs.prompt_limit }}
INPUT_MAX_NEW_TOKENS: ${{ inputs.max_new_tokens }}
INPUT_REGISTRY: ${{ inputs.registry }}
INPUT_BUDGETS: ${{ inputs.budgets }}
run: >-
python -m fuckmark.synthid_eligible_geometry_hf
--model "${{ inputs.model }}"
--model "$INPUT_MODEL"
--device cpu
--prompt-limit "${{ inputs.prompt_limit }}"
--max-new-tokens "${{ inputs.max_new_tokens }}"
--registry "${{ inputs.registry }}"
--budgets "${{ inputs.budgets }}"
--prompt-limit "$INPUT_PROMPT_LIMIT"
--max-new-tokens "$INPUT_MAX_NEW_TOKENS"
--registry "$INPUT_REGISTRY"
--budgets "$INPUT_BUDGETS"
- uses: actions/upload-artifact@v4
if: always()
with:
Expand Down
16 changes: 11 additions & 5 deletions .github/workflows/synthid-geometry-headroom.yml
Original file line number Diff line number Diff line change
Expand Up @@ -53,14 +53,20 @@ jobs:
- name: Install SynthID experiment dependencies
run: python -m pip install -r requirements-smoke.txt
- name: Run detector-free geometry headroom diagnostic
env:
INPUT_MODEL: ${{ inputs.model }}
INPUT_PROMPT_LIMIT: ${{ inputs.prompt_limit }}
INPUT_MAX_NEW_TOKENS: ${{ inputs.max_new_tokens }}
INPUT_REGISTRY: ${{ inputs.registry }}
INPUT_BUDGETS: ${{ inputs.budgets }}
run: >-
python -m fuckmark.synthid_geometry_headroom_hf
--model "${{ inputs.model }}"
--model "$INPUT_MODEL"
--device cpu
--prompt-limit "${{ inputs.prompt_limit }}"
--max-new-tokens "${{ inputs.max_new_tokens }}"
--registry "${{ inputs.registry }}"
--budgets "${{ inputs.budgets }}"
--prompt-limit "$INPUT_PROMPT_LIMIT"
--max-new-tokens "$INPUT_MAX_NEW_TOKENS"
--registry "$INPUT_REGISTRY"
--budgets "$INPUT_BUDGETS"
- uses: actions/upload-artifact@v4
if: always()
with:
Expand Down
19 changes: 13 additions & 6 deletions .github/workflows/synthid-geometry.yml
Original file line number Diff line number Diff line change
Expand Up @@ -58,15 +58,22 @@ jobs:
- name: Install SynthID experiment dependencies
run: python -m pip install -r requirements-smoke.txt
- name: Run matched random-vs-greedy geometry pilot
env:
INPUT_MODEL: ${{ inputs.model }}
INPUT_PROMPT_LIMIT: ${{ inputs.prompt_limit }}
INPUT_MAX_NEW_TOKENS: ${{ inputs.max_new_tokens }}
INPUT_REGISTRY: ${{ inputs.registry }}
INPUT_BUDGETS: ${{ inputs.budgets }}
INPUT_RANDOM_SEED_COUNT: ${{ inputs.random_seed_count }}
run: >-
python -m fuckmark.synthid_geometry_hf
--model "${{ inputs.model }}"
--model "$INPUT_MODEL"
--device cpu
--prompt-limit "${{ inputs.prompt_limit }}"
--max-new-tokens "${{ inputs.max_new_tokens }}"
--registry "${{ inputs.registry }}"
--budgets "${{ inputs.budgets }}"
--random-seed-count "${{ inputs.random_seed_count }}"
--prompt-limit "$INPUT_PROMPT_LIMIT"
--max-new-tokens "$INPUT_MAX_NEW_TOKENS"
--registry "$INPUT_REGISTRY"
--budgets "$INPUT_BUDGETS"
--random-seed-count "$INPUT_RANDOM_SEED_COUNT"
- uses: actions/upload-artifact@v4
if: always()
with:
Expand Down
16 changes: 11 additions & 5 deletions .github/workflows/synthid-high-repetition-detector.yml
Original file line number Diff line number Diff line change
Expand Up @@ -53,14 +53,20 @@ jobs:
- name: Install SynthID experiment dependencies
run: python -m pip install -r requirements-smoke.txt
- name: Run frozen high-repetition detector pilot
env:
INPUT_MODEL: ${{ inputs.model }}
INPUT_PROMPT_LIMIT: ${{ inputs.prompt_limit }}
INPUT_MAX_NEW_TOKENS: ${{ inputs.max_new_tokens }}
INPUT_REGISTRY: ${{ inputs.registry }}
INPUT_BUDGETS: ${{ inputs.budgets }}
run: >-
python -m fuckmark.synthid_high_repetition_detector_hf
--model "${{ inputs.model }}"
--model "$INPUT_MODEL"
--device cpu
--prompt-limit "${{ inputs.prompt_limit }}"
--max-new-tokens "${{ inputs.max_new_tokens }}"
--registry "${{ inputs.registry }}"
--budgets "${{ inputs.budgets }}"
--prompt-limit "$INPUT_PROMPT_LIMIT"
--max-new-tokens "$INPUT_MAX_NEW_TOKENS"
--registry "$INPUT_REGISTRY"
--budgets "$INPUT_BUDGETS"
- uses: actions/upload-artifact@v4
if: always()
with:
Expand Down
19 changes: 13 additions & 6 deletions .github/workflows/synthid-postselection.yml
Original file line number Diff line number Diff line change
Expand Up @@ -58,15 +58,22 @@ jobs:
- name: Install SynthID experiment dependencies
run: python -m pip install -r requirements-smoke.txt
- name: Run post-selection descriptive audit
env:
INPUT_MODEL: ${{ inputs.model }}
INPUT_PROMPT_LIMIT: ${{ inputs.prompt_limit }}
INPUT_MAX_NEW_TOKENS: ${{ inputs.max_new_tokens }}
INPUT_REGISTRY: ${{ inputs.registry }}
INPUT_BUDGETS: ${{ inputs.budgets }}
INPUT_RANDOM_SEED_COUNT: ${{ inputs.random_seed_count }}
run: >-
python -m fuckmark.synthid_postselection_hf
--model "${{ inputs.model }}"
--model "$INPUT_MODEL"
--device cpu
--prompt-limit "${{ inputs.prompt_limit }}"
--max-new-tokens "${{ inputs.max_new_tokens }}"
--registry "${{ inputs.registry }}"
--budgets "${{ inputs.budgets }}"
--random-seed-count "${{ inputs.random_seed_count }}"
--prompt-limit "$INPUT_PROMPT_LIMIT"
--max-new-tokens "$INPUT_MAX_NEW_TOKENS"
--registry "$INPUT_REGISTRY"
--budgets "$INPUT_BUDGETS"
--random-seed-count "$INPUT_RANDOM_SEED_COUNT"
- uses: actions/upload-artifact@v4
if: always()
with:
Expand Down
16 changes: 11 additions & 5 deletions .github/workflows/synthid-repetition-strata.yml
Original file line number Diff line number Diff line change
Expand Up @@ -53,14 +53,20 @@ jobs:
- name: Install SynthID experiment dependencies
run: python -m pip install -r requirements-smoke.txt
- name: Run repetition-stratified detector-free headroom analysis
env:
INPUT_MODEL: ${{ inputs.model }}
INPUT_PROMPT_LIMIT: ${{ inputs.prompt_limit }}
INPUT_MAX_NEW_TOKENS: ${{ inputs.max_new_tokens }}
INPUT_REGISTRY: ${{ inputs.registry }}
INPUT_BUDGETS: ${{ inputs.budgets }}
run: >-
python -m fuckmark.synthid_repetition_strata_hf
--model "${{ inputs.model }}"
--model "$INPUT_MODEL"
--device cpu
--prompt-limit "${{ inputs.prompt_limit }}"
--max-new-tokens "${{ inputs.max_new_tokens }}"
--registry "${{ inputs.registry }}"
--budgets "${{ inputs.budgets }}"
--prompt-limit "$INPUT_PROMPT_LIMIT"
--max-new-tokens "$INPUT_MAX_NEW_TOKENS"
--registry "$INPUT_REGISTRY"
--budgets "$INPUT_BUDGETS"
- uses: actions/upload-artifact@v4
if: always()
with:
Expand Down
19 changes: 13 additions & 6 deletions .github/workflows/synthid-schedule-stress.yml
Original file line number Diff line number Diff line change
Expand Up @@ -58,15 +58,22 @@ jobs:
- name: Install SynthID experiment dependencies
run: python -m pip install -r requirements-smoke.txt
- name: Run public-geometry headroom stress analysis
env:
INPUT_MODEL: ${{ inputs.model }}
INPUT_PROMPT_LIMIT: ${{ inputs.prompt_limit }}
INPUT_MAX_NEW_TOKENS: ${{ inputs.max_new_tokens }}
INPUT_REGISTRY: ${{ inputs.registry }}
INPUT_BUDGETS: ${{ inputs.budgets }}
INPUT_RANDOM_SEED_COUNT: ${{ inputs.random_seed_count }}
run: >-
python -m fuckmark.synthid_schedule_stress_hf
--model "${{ inputs.model }}"
--model "$INPUT_MODEL"
--device cpu
--prompt-limit "${{ inputs.prompt_limit }}"
--max-new-tokens "${{ inputs.max_new_tokens }}"
--registry "${{ inputs.registry }}"
--budgets "${{ inputs.budgets }}"
--random-seed-count "${{ inputs.random_seed_count }}"
--prompt-limit "$INPUT_PROMPT_LIMIT"
--max-new-tokens "$INPUT_MAX_NEW_TOKENS"
--registry "$INPUT_REGISTRY"
--budgets "$INPUT_BUDGETS"
--random-seed-count "$INPUT_RANDOM_SEED_COUNT"
- uses: actions/upload-artifact@v4
if: always()
with:
Expand Down
6 changes: 5 additions & 1 deletion .github/workflows/synthid-smoke.yml
Original file line number Diff line number Diff line change
Expand Up @@ -39,7 +39,11 @@ jobs:
- name: Install smoke dependencies
run: python -m pip install -r requirements-smoke.txt
- name: Run open SynthID smoke
run: python -m fuckmark.synthid_smoke_hf --model "${{ inputs.model }}" --device cpu --prompt-limit "${{ inputs.prompt_limit }}" --max-new-tokens "${{ inputs.max_new_tokens }}"
env:
INPUT_MODEL: ${{ inputs.model }}
INPUT_PROMPT_LIMIT: ${{ inputs.prompt_limit }}
INPUT_MAX_NEW_TOKENS: ${{ inputs.max_new_tokens }}
run: python -m fuckmark.synthid_smoke_hf --model "$INPUT_MODEL" --device cpu --prompt-limit "$INPUT_PROMPT_LIMIT" --max-new-tokens "$INPUT_MAX_NEW_TOKENS"
- uses: actions/upload-artifact@v4
if: always()
with:
Expand Down
Loading
Loading