Speed up cold long-context BPE tokenization

.github/workflows/build-maturin.yml

@@ -75,7 +75,7 @@ jobs:
         if: ${{ matrix.platform.ft }}
         with:
           target: ${{ matrix.platform.target }}
-          args: --release --out dist -i python3.13t
+          args: --release --out dist -i python3.14t
           sccache: "false"
           manylinux: auto
           before-script-linux: |
@@ -122,7 +122,7 @@ jobs:
         uses: PyO3/maturin-action@v1
         with:
           target: ${{ matrix.platform.target }}
-          args: --release --out dist -i python3.13t
+          args: --release --out dist -i python3.14t
           sccache: "false"
           manylinux: musllinux_1_2
       - name: Upload wheels
@@ -175,7 +175,7 @@ jobs:
         uses: PyO3/maturin-action@v1
         with:
           target: ${{ matrix.platform.target }}
-          args: --release --out dist -i python3.13t
+          args: --release --out dist -i python3.14t
           sccache: "false"
       - name: Upload wheels
         uses: actions/upload-artifact@v4

.github/workflows/build-release-fastokens.yml

@@ -48,9 +48,9 @@ jobs:
       - uses: actions/checkout@v6
       - uses: dtolnay/rust-toolchain@stable
       - name: Verify package
-        run: cargo publish -p fastokens --dry-run --locked
+        run: cargo publish -p fastokens-b10 --dry-run --locked
       - name: Publish
         if: ${{ github.event_name == 'workflow_dispatch' && fromJSON(github.event.inputs.publish) }}
         env:
           CARGO_REGISTRY_TOKEN: ${{ secrets.CARGO_REGISTRY_TOKEN }}
-        run: cargo publish -p fastokens --locked
+        run: cargo publish -p fastokens-b10 --locked

.github/workflows/ci.yml

@@ -5,9 +5,11 @@ on:
     branches: [main]
   pull_request:
     branches: [main]
+  workflow_dispatch:
 
 env:
   CARGO_TERM_COLOR: always
+  HF_TOKEN: ${{ secrets.HF_TOKEN }}
 
 jobs:
   test:
@@ -49,6 +51,9 @@ jobs:
       - name: Build and install wheel
         run: maturin build --release --out dist && pip install dist/*.whl
 
+      - name: Install Python test dependencies
+        run: pip install pytest tiktoken
+
       - name: Smoke test
         run: |
           python - <<'EOF'
@@ -61,6 +66,9 @@ jobs:
           print("smoke test passed")
           EOF
 
+      - name: Run Python tests
+        run: pytest python/tests -v
+
   patch-transformers:
     runs-on: ubuntu-latest
     strategy:
@@ -85,7 +93,7 @@ jobs:
         run: maturin build --release --out dist && pip install dist/*.whl
 
       - name: Install transformers ${{ matrix.transformers }} and pytest
-        run: pip install transformers==${{ matrix.transformers }} pytest
+        run: pip install transformers==${{ matrix.transformers }} pytest jinja2
 
       - name: Test patch_transformers
         run: pytest python/tests/test_patch_transformers.py -v

.github/workflows/python-ci.yml

@@ -0,0 +1,62 @@
+name: Python CI
+
+on:
+  push:
+    branches: [main]
+  pull_request:
+    branches: [main]
+  workflow_dispatch:
+
+env:
+  CARGO_TERM_COLOR: always
+  HF_TOKEN: ${{ secrets.HF_TOKEN }}
+
+permissions:
+  contents: read
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v6
+
+      - uses: dtolnay/rust-toolchain@stable
+
+      - uses: Swatinem/rust-cache@v2
+
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "3.12"
+          cache: pip
+
+      - name: Cache Hugging Face tokenizers
+        uses: actions/cache@v4
+        with:
+          path: |
+            ~/.cache/huggingface/hub
+            ~/.cache/huggingface/xet
+            ~/.cache/tokenizers
+          key: ${{ runner.os }}-py3.12-hf-tokenizers-${{ hashFiles('pyproject.toml', 'python/Cargo.toml', 'Cargo.lock', '.github/workflows/python-ci.yml') }}
+          restore-keys: |
+            ${{ runner.os }}-py3.12-hf-tokenizers-
+            ${{ runner.os }}-hf-tokenizers-
+
+      - name: Configure Hugging Face token
+        run: |
+          if [ -n "${HF_TOKEN:-}" ]; then
+            mkdir -p ~/.cache/huggingface
+            printf '%s' "$HF_TOKEN" > ~/.cache/huggingface/token
+          fi
+
+      - name: Install Python test dependencies
+        run: python -m pip install --upgrade pip maturin pytest transformers==4.57.1 jinja2
+
+      - name: Build and install fastokens
+        run: maturin build --release --out dist && pip install dist/*.whl
+
+      - name: Run Python tests
+        run: pytest python/tests -v
+
+      - name: Remove Hugging Face token before cache save
+        if: always()
+        run: rm -f ~/.cache/huggingface/token

.gitignore

@@ -4,4 +4,5 @@ __pycache__/
 *.pyd
 
 dist/
-target/
+target/
+vendored_tokenizers/private_*