tinybpe/.github/workflows/python-package.yml at main · neluca/tinybpe · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
name: build

on:
  push:
    branches: [ main ]
    paths:
      - "src/**"
      - "tinybpe/**"
      - "tests/**"
      - "setup.py"
      - "pyproject.toml"
      - "requirements_dev.txt"
      - "MANIFEST.in"
      - ".github/workflows/python-package.yml"
  pull_request:
    paths:
      - "src/**"
      - "tinybpe/**"
      - "tests/**"
      - "setup.py"
      - "pyproject.toml"
      - "requirements_dev.txt"
      - "MANIFEST.in"
      - ".github/workflows/python-package.yml"
  schedule:
    - cron: '0 10 * * *'

concurrency:
  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
  cancel-in-progress: true

jobs:
  build:
    strategy:
      matrix:
        python-version: [ '3.9', '3.10', '3.11', '3.12', '3.13' ]
        os: [ ubuntu-latest, macos-latest, windows-latest ]
        exclude:
          - python-version: '3.9'
            os: macos-latest
    runs-on: ${{ matrix.os }}
    timeout-minutes: 30
    steps:
      - uses: actions/checkout@v6
        with:
          ref: ${{ github.event.pull_request.head.sha }}
      - name: Set up Python ${{ matrix.python-version }}
        uses: actions/setup-python@v6
        with:
          python-version: ${{ matrix.python-version }}
      - name: Install dependencies
        run: |
          python -m pip install --upgrade pip
          pip install -r requirements_dev.txt
      - name: Build and install
        run: |
          pip install -e .
      - name: Test with pytest
        run: |
          python -m pytest --cov=tinybpe --cov-branch --cov-report=xml --cov-fail-under=90
      - name: Upload coverage reports to Codecov
        uses: codecov/codecov-action@v7
        with:
          token: ${{ secrets.CODECOV_TOKEN }}
          slug: neluca/tinybpe

  build-sdist:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v6
      - name: Set up Python
        uses: actions/setup-python@v6
        with:
          python-version: '3.11'
      - name: Install dependencies
        run: |
          python -m pip install --upgrade pip
          pip install build
      - name: Build sdist
        run: python -m build --sdist
      - name: Install from sdist
        run: |
          pip install dist/*.tar.gz
      - name: Verify import and model loading
        run: |
          cd /tmp
          python -c "import tinybpe; print(tinybpe.__version__); print(tinybpe.list_models()); from tinybpe import Tokenizer; tok = Tokenizer.from_pretrained('cl100k_base'); print(len(tok.encode('hello world')))"