diff --git a/.github/workflows/node-release.yml b/.github/workflows/node-release.yml index 8ef097d5b..53a7948d8 100644 --- a/.github/workflows/node-release.yml +++ b/.github/workflows/node-release.yml @@ -46,16 +46,16 @@ jobs: strip *.node - host: ubuntu-latest target: x86_64-unknown-linux-musl - build: yarn build --target x86_64-unknown-linux-musl --zig + build: yarn build --target x86_64-unknown-linux-musl --cross-compile - host: ubuntu-latest target: aarch64-unknown-linux-gnu - build: yarn build --target aarch64-unknown-linux-gnu --zig + build: yarn build --target aarch64-unknown-linux-gnu --cross-compile - host: ubuntu-latest target: aarch64-unknown-linux-musl - build: yarn build --target aarch64-unknown-linux-musl --zig + build: yarn build --target aarch64-unknown-linux-musl --cross-compile - host: ubuntu-latest target: armv7-unknown-linux-gnueabihf - build: yarn build --target armv7-unknown-linux-gnueabihf --zig + build: yarn build --target armv7-unknown-linux-gnueabihf --cross-compile - host: ubuntu-latest target: aarch64-linux-android build: yarn build --target aarch64-linux-android @@ -88,7 +88,7 @@ jobs: run: yarn install --immutable - name: Setup zig - if: ${{ !matrix.settings.docker && contains(matrix.settings.build, '--zig') }} + if: ${{ !matrix.settings.docker && contains(matrix.settings.build, '--cross-compile') }} uses: mlugg/setup-zig@d1434d08867e3ee9daa34448df10607b98908d29 # v2.2.1 with: # Pin — without a version the action installs zig master (nightly) @@ -98,7 +98,7 @@ jobs: version: 0.16.0 - name: Verify zig - if: ${{ !matrix.settings.docker && contains(matrix.settings.build, '--zig') }} + if: ${{ !matrix.settings.docker && contains(matrix.settings.build, '--cross-compile') }} run: zig version && which zig - name: Build native (host) @@ -155,7 +155,14 @@ jobs: path: ./bindings/node/artifacts - name: Combine into universal binary working-directory: ./bindings/node - run: npx napi universal --dir ./artifacts + # napi-rs/cli v3 renamed `napi universal` -> `napi universalize` + # and dropped `--dir`; it now expects the npm// layout + # populated by `napi artifacts`. Cheaper to lipo directly — that's + # exactly what the wrapper does internally. + run: | + lipo -create -output ${{ env.APP_NAME }}.darwin-universal.node \ + artifacts/${{ env.APP_NAME }}.darwin-x64.node \ + artifacts/${{ env.APP_NAME }}.darwin-arm64.node - name: Upload universal artifact uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7 with: @@ -164,8 +171,16 @@ jobs: if-no-files-found: error publish: + # Skip the npm publish step on rc tags — `napi prepublish` would try + # to create a GitHub release and call `npm publish` for each + # per-platform sub-package, both of which need stable-version / + # write-permission setup we only want on real releases. rc tags still + # exercise the build + universal-macOS jobs (so the matrix is tested + # end-to-end), they just don't push to the registry. + if: ${{ !contains(github.ref, 'rc') }} permissions: - contents: read + # `napi prepublish` creates a GitHub Release; needs contents=write. + contents: write id-token: write name: Publish runs-on: ubuntu-latest @@ -188,6 +203,15 @@ jobs: uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8 with: path: ./bindings/node/artifacts + - name: Create missing npm/ dirs + # `yarn artifacts` (napi artifacts) writes each `.node` file into + # `npm//`. Some triples (e.g. darwin-universal) have no + # checked-in template dir, so without this step the artifacts + # command fails with ENOENT. `create-npm-dirs` populates anything + # missing from `napi.targets` in package.json. + working-directory: ./bindings/node + run: npx napi create-npm-dirs + - name: Move artifacts working-directory: ./bindings/node run: yarn artifacts diff --git a/.github/workflows/python-release.yml b/.github/workflows/python-release.yml index 772e857c1..6a8996e36 100644 --- a/.github/workflows/python-release.yml +++ b/.github/workflows/python-release.yml @@ -19,7 +19,7 @@ jobs: working-directory: ./bindings/python build: - name: build on ${{ matrix.platform || matrix.os }} (${{ matrix.target }} - ${{ matrix.manylinux || 'auto' }}) + name: build on ${{ matrix.platform || matrix.os }} (${{ matrix.target }} - ${{ matrix.manylinux || 'auto' }} - ${{ matrix.flavor == 'ft' && '3.14t' || matrix.interpreter || '3.14' }}) # only run on push to main and on release needs: [lock_exists] if: startsWith(github.ref, 'refs/tags/') || github.ref == 'refs/heads/main' || contains(github.event.pull_request.labels.*.name, 'Full Build') @@ -29,43 +29,46 @@ jobs: os: [ubuntu, macos, windows] target: [x86_64, aarch64] manylinux: [auto] + # `flavor` discriminates regular abi3 wheels from free-threaded + # (3.14t, non-abi3) wheels so the include: entries below for + # 3.14t create new matrix cells instead of merging with the abi3 + # combos. + flavor: [abi3] include: - os: ubuntu platform: linux - os: windows ls: dir - interpreter: 3.14 3.14t + interpreter: "3.14" - os: windows ls: dir target: x86_64 python-architecture: x64 - python-install: | - 3.14 - 3.14t - interpreter: 3.14 3.14t + python-install: "3.14" + interpreter: "3.14" - os: windows ls: dir target: i686 python-architecture: x86 - python-install: | - 3.14 - 3.14t - interpreter: 3.14 3.14t + python-install: "3.14" + interpreter: "3.14" - os: windows-11-arm ls: dir target: aarch64 python-architecture: arm64 - python-install: | - 3.14 - 3.14t - interpreter: 3.14 3.14t + # 3.14t arm64-freethreaded is currently broken upstream: + # actions/python-versions ships a 0-byte python.exe so pip + # install fails with `ModuleNotFoundError: encodings`. Drop + # 3.14t here until the upstream package is fixed. + python-install: "3.14" + interpreter: "3.14" # - os: windows # ls: dir # target: aarch64 # interpreter: 3.11 3.12 - os: macos target: aarch64 - interpreter: 3.14 3.14t + interpreter: "3.14" - os: ubuntu platform: linux target: i686 @@ -76,7 +79,7 @@ jobs: - os: ubuntu platform: linux target: armv7 - interpreter: 3.14 3.14t + interpreter: "3.14" # musllinux - os: ubuntu platform: linux @@ -89,11 +92,28 @@ jobs: - os: ubuntu platform: linux target: ppc64le - interpreter: 3.14 3.14t + interpreter: "3.14" - os: ubuntu platform: linux target: s390x - interpreter: 3.14 3.14t + interpreter: "3.14" + + # --- Free-threaded Python 3.14t wheels ------------------------- + # `flavor: ft` switches the build to non-abi3 (`--no-default-features + # --features ext-module`) and `--interpreter 3.14t`, derived from + # `flavor` in the maturin invocation below. linux container builds + # (manylinux/musllinux) get 3.14t from the docker image, so + # `python-install` is unset for those; macOS/windows host builds + # need it set explicitly so setup-python actually installs 3.14t. + # windows-11-arm 3.14t is intentionally absent — see the abi3 + # windows-11-arm entry above (upstream-broken package). + - { os: ubuntu, platform: linux, target: x86_64, manylinux: auto, flavor: ft } + - { os: ubuntu, platform: linux, target: aarch64, manylinux: auto, flavor: ft } + - { os: ubuntu, platform: linux, target: x86_64, manylinux: musllinux_1_1, flavor: ft } + - { os: ubuntu, platform: linux, target: aarch64, manylinux: musllinux_1_1, flavor: ft } + - { os: macos, target: x86_64, manylinux: auto, flavor: ft, python-install: "3.14t" } + - { os: macos, target: aarch64, manylinux: auto, flavor: ft, python-install: "3.14t" } + - { os: windows, ls: dir, target: x86_64, manylinux: auto, python-architecture: x64, python-install: "3.14t", flavor: ft } exclude: - os: windows target: aarch64 @@ -128,7 +148,13 @@ jobs: working-directory: ./bindings/python manylinux: ${{ matrix.manylinux || 'auto' }} container: ${{ matrix.container }} - args: --release --out dist --interpreter ${{ matrix.interpreter || '3.14 3.14t' }} ${{ matrix.extra-build-args }} + # `flavor=ft` builds drop the abi3 cargo feature so the resulting + # wheel is non-abi3 (free-threaded Python can't load limited-API + # extensions). `flavor=abi3` builds use defaults. + args: >- + --release --out dist + --interpreter ${{ matrix.flavor == 'ft' && '3.14t' || matrix.interpreter || '3.14' }} + ${{ matrix.flavor == 'ft' && '--no-default-features --features ext-module' || '' }} rust-toolchain: stable sccache: false docker-options: -e CI @@ -141,7 +167,7 @@ jobs: - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: - name: pypi_files-${{ matrix.os }}-${{ matrix.target }}-${{ matrix.manylinux }} + name: pypi_files-${{ matrix.os }}-${{ matrix.target }}-${{ matrix.manylinux }}-${{ matrix.flavor || 'abi3' }} path: ./bindings/python/dist build-sdist: name: build sdist diff --git a/bindings/python/pyproject.toml b/bindings/python/pyproject.toml index c4354c9e4..e21434a02 100644 --- a/bindings/python/pyproject.toml +++ b/bindings/python/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "tokenizers" -requires-python = ">=3.9" +requires-python = ">=3.10" authors = [ { name = "Nicolas Patry", email = "patry.nicolas@protonmail.com" }, { name = "Anthony Moi", email = "anthony@huggingface.co" }, @@ -13,11 +13,11 @@ classifiers = [ "License :: OSI Approved :: Apache Software License", "Operating System :: OS Independent", "Programming Language :: Python :: 3", - "Programming Language :: Python :: 3.9", "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", "Programming Language :: Python :: 3.12", "Programming Language :: Python :: 3.13", + "Programming Language :: Python :: 3.14", "Programming Language :: Python :: 3 :: Only", "Topic :: Scientific/Engineering :: Artificial Intelligence", ]