From 7f1623b90b5adfb9bc327d4c3468d2f70bbce262 Mon Sep 17 00:00:00 2001 From: Arthur Date: Mon, 27 Apr 2026 22:21:25 +0900 Subject: [PATCH 1/3] Bump version to 0.23.1 --- bindings/node/Cargo.toml | 2 +- bindings/node/package.json | 2 +- bindings/python/Cargo.lock | 1565 ++++++++++++++++++++++++++++++++++++ bindings/python/Cargo.toml | 2 +- tokenizers/Cargo.toml | 2 +- 5 files changed, 1569 insertions(+), 4 deletions(-) create mode 100644 bindings/python/Cargo.lock diff --git a/bindings/node/Cargo.toml b/bindings/node/Cargo.toml index 296ad71d4..576959822 100644 --- a/bindings/node/Cargo.toml +++ b/bindings/node/Cargo.toml @@ -2,7 +2,7 @@ authors = ["Nicolas Patry "] edition = "2021" name = "node" -version = "0.22.3-dev.0" +version = "0.23.1" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html diff --git a/bindings/node/package.json b/bindings/node/package.json index d1bfed5bd..cbfd7400b 100644 --- a/bindings/node/package.json +++ b/bindings/node/package.json @@ -1,6 +1,6 @@ { "name": "tokenizers", - "version": "0.15.3-dev0", + "version": "0.23.1", "repository": { "type": "git", "url": "git+https://github.com/huggingface/tokenizers.git" diff --git a/bindings/python/Cargo.lock b/bindings/python/Cargo.lock new file mode 100644 index 000000000..a82919bde --- /dev/null +++ b/bindings/python/Cargo.lock @@ -0,0 +1,1565 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "ahash" +version = "0.8.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a15f179cd60c4584b8a8c596927aadc462e27f2ca70c04e0071964a73ba7a75" +dependencies = [ + "cfg-if", + "getrandom 0.3.4", + "once_cell", + "serde", + "version_check", + "zerocopy", +] + +[[package]] +name = "aho-corasick" +version = "1.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ddd31a130427c27518df266943a5308ed92d4b226cc639f5a8f1002816174301" +dependencies = [ + "memchr", +] + +[[package]] +name = "anstream" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "824a212faf96e9acacdbd09febd34438f8f711fb84e09a8916013cd7815ca28d" +dependencies = [ + "anstyle", + "anstyle-parse", + "anstyle-query", + "anstyle-wincon", + "colorchoice", + "is_terminal_polyfill", + "utf8parse", +] + +[[package]] +name = "anstyle" +version = "1.0.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "940b3a0ca603d1eade50a4846a2afffd5ef57a9feac2c0e2ec2e14f9ead76000" + +[[package]] +name = "anstyle-parse" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52ce7f38b242319f7cabaa6813055467063ecdc9d355bbb4ce0c68908cd8130e" +dependencies = [ + "utf8parse", +] + +[[package]] +name = "anstyle-query" +version = "1.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "40c48f72fd53cd289104fc64099abca73db4166ad86ea0b4341abe65af83dadc" +dependencies = [ + "windows-sys", +] + +[[package]] +name = "anstyle-wincon" +version = "3.0.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "291e6a250ff86cd4a820112fb8898808a366d8f9f58ce16d1f538353ad55747d" +dependencies = [ + "anstyle", + "once_cell_polyfill", + "windows-sys", +] + +[[package]] +name = "anyhow" +version = "1.0.102" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f202df86484c868dbad7eaa557ef785d5c66295e41b460ef922eca0723b842c" + +[[package]] +name = "autocfg" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c08606f8c3cbf4ce6ec8e28fb0014a2c086708fe954eaa885384a6165172e7e8" + +[[package]] +name = "base64" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9e1b586273c5702936fe7b7d6896644d8be71e6314cfe09d3167c95f712589e8" + +[[package]] +name = "bitflags" +version = "2.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c4512299f36f043ab09a583e57bceb5a5aab7a73db1805848e8fef3c9e8c78b3" + +[[package]] +name = "bumpalo" +version = "3.20.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5d20789868f4b01b2f2caec9f5c4e0213b41e3e5702a50157d699ae31ced2fcb" + +[[package]] +name = "castaway" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dec551ab6e7578819132c713a93c022a05d60159dc86e7a7050223577484c55a" +dependencies = [ + "rustversion", +] + +[[package]] +name = "cc" +version = "1.2.61" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d16d90359e986641506914ba71350897565610e87ce0ad9e6f28569db3dd5c6d" +dependencies = [ + "find-msvc-tools", + "shlex", +] + +[[package]] +name = "cfg-if" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" + +[[package]] +name = "colorchoice" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d07550c9036bf2ae0c684c4297d503f838287c83c53686d05370d0e139ae570" + +[[package]] +name = "compact_str" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3fdb1325a1cece981e8a296ab8f0f9b63ae357bd0784a9faaf548cc7b480707a" +dependencies = [ + "castaway", + "cfg-if", + "itoa", + "rustversion", + "ryu", + "serde", + "static_assertions", +] + +[[package]] +name = "console" +version = "0.16.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d64e8af5551369d19cf50138de61f1c42074ab970f74e99be916646777f8fc87" +dependencies = [ + "encode_unicode", + "libc", + "unicode-width", + "windows-sys", +] + +[[package]] +name = "crossbeam-deque" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9dd111b7b7f7d55b72c0a6ae361660ee5853c9af73f70c3c2ef6858b950e2e51" +dependencies = [ + "crossbeam-epoch", + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-epoch" +version = "0.9.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b82ac4a3c2ca9c3460964f020e1402edd5753411d7737aa39c3714ad1b5420e" +dependencies = [ + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-utils" +version = "0.8.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d0a5c400df2834b80a4c3327b3aad3a4c4cd4de0629063962b03235697506a28" + +[[package]] +name = "daachorse" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6f55d7153ba3b507595872a3874803f07a8a81d1e888abed8e5db7da0597d6e2" + +[[package]] +name = "darling" +version = "0.20.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc7f46116c46ff9ab3eb1597a45688b6715c6e628b5c133e288e709a29bcb4ee" +dependencies = [ + "darling_core", + "darling_macro", +] + +[[package]] +name = "darling_core" +version = "0.20.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0d00b9596d185e565c2207a0b01f8bd1a135483d02d9b7b0a54b11da8d53412e" +dependencies = [ + "fnv", + "ident_case", + "proc-macro2", + "quote", + "strsim", + "syn", +] + +[[package]] +name = "darling_macro" +version = "0.20.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc34b93ccb385b40dc71c6fceac4b2ad23662c7eeb248cf10d529b7e055b6ead" +dependencies = [ + "darling_core", + "quote", + "syn", +] + +[[package]] +name = "dary_heap" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b1e3a325bc115f096c8b77bbf027a7c2592230e70be2d985be950d3d5e60ebe" +dependencies = [ + "serde", +] + +[[package]] +name = "derive_builder" +version = "0.20.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "507dfb09ea8b7fa618fcf76e953f4f5e192547945816d5358edffe39f6f94947" +dependencies = [ + "derive_builder_macro", +] + +[[package]] +name = "derive_builder_core" +version = "0.20.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2d5bcf7b024d6835cfb3d473887cd966994907effbe9227e8c8219824d06c4e8" +dependencies = [ + "darling", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "derive_builder_macro" +version = "0.20.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ab63b0e2bf4d5928aff72e83a7dace85d7bba5fe12dcc3c5a572d78caffd3f3c" +dependencies = [ + "derive_builder_core", + "syn", +] + +[[package]] +name = "either" +version = "1.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "48c757948c5ede0e46177b7add2e67155f70e33c07fea8284df6576da70b3719" + +[[package]] +name = "encode_unicode" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "34aa73646ffb006b8f5147f3dc182bd4bcb190227ce861fc4a4844bf8e3cb2c0" + +[[package]] +name = "env_filter" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32e90c2accc4b07a8456ea0debdc2e7587bdd890680d71173a15d4ae604f6eef" +dependencies = [ + "log", + "regex", +] + +[[package]] +name = "env_logger" +version = "0.11.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0621c04f2196ac3f488dd583365b9c09be011a4ab8b9f37248ffcc8f6198b56a" +dependencies = [ + "anstream", + "anstyle", + "env_filter", + "jiff", + "log", +] + +[[package]] +name = "equivalent" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f" + +[[package]] +name = "errno" +version = "0.3.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" +dependencies = [ + "libc", + "windows-sys", +] + +[[package]] +name = "esaxx-rs" +version = "0.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d817e038c30374a4bcb22f94d0a8a0e216958d4c3dcde369b1439fec4bdda6e6" +dependencies = [ + "cc", +] + +[[package]] +name = "fastrand" +version = "2.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f1f227452a390804cdb637b74a86990f2a7d7ba4b7d5693aac9b4dd6defd8d6" + +[[package]] +name = "find-msvc-tools" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5baebc0774151f905a1a2cc41989300b1e6fbb29aff0ceffa1064fdd3088d582" + +[[package]] +name = "fnv" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" + +[[package]] +name = "foldhash" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d9c4f5dac5e15c24eb999c26181a6ca40b39fe946cbe4c263c7209467bc83af2" + +[[package]] +name = "futures-channel" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "07bbe89c50d7a535e539b8c17bc0b49bdb77747034daa8087407d655f3f7cc1d" +dependencies = [ + "futures-core", +] + +[[package]] +name = "futures-core" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7e3450815272ef58cec6d564423f6e755e25379b217b0bc688e295ba24df6b1d" + +[[package]] +name = "futures-macro" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e835b70203e41293343137df5c0664546da5745f82ec9b84d40be8336958447b" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "futures-task" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "037711b3d59c33004d3856fbdc83b99d4ff37a24768fa1be9ce3538a1cde4393" + +[[package]] +name = "futures-util" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "389ca41296e6190b48053de0321d02a77f32f8a5d2461dd38762c0593805c6d6" +dependencies = [ + "futures-core", + "futures-macro", + "futures-task", + "pin-project-lite", + "slab", +] + +[[package]] +name = "getrandom" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "899def5c37c4fd7b2664648c28120ecec138e4d395b459e5ca34f9cce2dd77fd" +dependencies = [ + "cfg-if", + "libc", + "r-efi 5.3.0", + "wasip2", +] + +[[package]] +name = "getrandom" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0de51e6874e94e7bf76d726fc5d13ba782deca734ff60d5bb2fb2607c7406555" +dependencies = [ + "cfg-if", + "libc", + "r-efi 6.0.0", + "wasip2", + "wasip3", +] + +[[package]] +name = "hashbrown" +version = "0.15.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9229cfe53dfd69f0609a49f65461bd93001ea1ef889cd5529dd176593f5338a1" +dependencies = [ + "foldhash", +] + +[[package]] +name = "hashbrown" +version = "0.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4f467dd6dccf739c208452f8014c75c18bb8301b050ad1cfb27153803edb0f51" + +[[package]] +name = "heck" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" + +[[package]] +name = "id-arena" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3d3067d79b975e8844ca9eb072e16b31c3c1c36928edf9c6789548c524d0d954" + +[[package]] +name = "ident_case" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9e0384b61958566e926dc50660321d12159025e767c18e043daf26b70104c39" + +[[package]] +name = "indexmap" +version = "2.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d466e9454f08e4a911e14806c24e16fba1b4c121d1ea474396f396069cf949d9" +dependencies = [ + "equivalent", + "hashbrown 0.17.0", + "serde", + "serde_core", +] + +[[package]] +name = "indicatif" +version = "0.18.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "25470f23803092da7d239834776d653104d551bc4d7eacaf31e6837854b8e9eb" +dependencies = [ + "console", + "portable-atomic", + "unicode-width", + "unit-prefix", + "web-time", +] + +[[package]] +name = "is_terminal_polyfill" +version = "1.70.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6cb138bb79a146c1bd460005623e142ef0181e3d0219cb493e02f7d08a35695" + +[[package]] +name = "itertools" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b192c782037fadd9cfa75548310488aabdbf3d2da73885b31bd0abd03351285" +dependencies = [ + "either", +] + +[[package]] +name = "itoa" +version = "1.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" + +[[package]] +name = "jiff" +version = "0.2.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f00b5dbd620d61dfdcb6007c9c1f6054ebd75319f163d886a9055cec1155073d" +dependencies = [ + "jiff-static", + "log", + "portable-atomic", + "portable-atomic-util", + "serde_core", +] + +[[package]] +name = "jiff-static" +version = "0.2.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e000de030ff8022ea1da3f466fbb0f3a809f5e51ed31f6dd931c35181ad8e6d7" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "js-sys" +version = "0.3.95" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2964e92d1d9dc3364cae4d718d93f227e3abb088e747d92e0395bfdedf1c12ca" +dependencies = [ + "once_cell", + "wasm-bindgen", +] + +[[package]] +name = "leb128fmt" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09edd9e8b54e49e587e4f6295a7d29c3ea94d469cb40ab8ca70b288248a81db2" + +[[package]] +name = "libc" +version = "0.2.186" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68ab91017fe16c622486840e4c83c9a37afeff978bd239b5293d61ece587de66" + +[[package]] +name = "linux-raw-sys" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a66949e030da00e8c7d4434b251670a91556f4144941d37452769c25d58a53" + +[[package]] +name = "log" +version = "0.4.29" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5e5032e24019045c762d3c0f28f5b6b8bbf38563a65908389bf7978758920897" + +[[package]] +name = "macro_rules_attribute" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "65049d7923698040cd0b1ddcced9b0eb14dd22c5f86ae59c3740eab64a676520" +dependencies = [ + "macro_rules_attribute-proc_macro", + "paste", +] + +[[package]] +name = "macro_rules_attribute-proc_macro" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "670fdfda89751bc4a84ac13eaa63e205cf0fd22b4c9a5fbfa085b63c1f1d3a30" + +[[package]] +name = "matrixmultiply" +version = "0.3.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a06de3016e9fae57a36fd14dba131fccf49f74b40b7fbdb472f96e361ec71a08" +dependencies = [ + "autocfg", + "rawpointer", +] + +[[package]] +name = "memchr" +version = "2.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8ca58f447f06ed17d5fc4043ce1b10dd205e060fb3ce5b979b8ed8e59ff3f79" + +[[package]] +name = "minimal-lexical" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" + +[[package]] +name = "mio" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "50b7e5b27aa02a74bac8c3f23f448f8d87ff11f92d3aac1a6ed369ee08cc56c1" +dependencies = [ + "libc", + "wasi", + "windows-sys", +] + +[[package]] +name = "monostate" +version = "0.1.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3341a273f6c9d5bef1908f17b7267bbab0e95c9bf69a0d4dcf8e9e1b2c76ef67" +dependencies = [ + "monostate-impl", + "serde", + "serde_core", +] + +[[package]] +name = "monostate-impl" +version = "0.1.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e4db6d5580af57bf992f59068d4ea26fd518574ff48d7639b255a36f9de6e7e9" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "ndarray" +version = "0.16.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "882ed72dce9365842bf196bdeedf5055305f11fc8c03dee7bb0194a6cad34841" +dependencies = [ + "matrixmultiply", + "num-complex", + "num-integer", + "num-traits", + "portable-atomic", + "portable-atomic-util", + "rawpointer", +] + +[[package]] +name = "ndarray" +version = "0.17.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "520080814a7a6b4a6e9070823bb24b4531daac8c4627e08ba5de8c5ef2f2752d" +dependencies = [ + "matrixmultiply", + "num-complex", + "num-integer", + "num-traits", + "portable-atomic", + "portable-atomic-util", + "rawpointer", +] + +[[package]] +name = "nom" +version = "7.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d273983c5a657a70a3e8f2a01329822f3b8c8172b73826411a55751e404a0a4a" +dependencies = [ + "memchr", + "minimal-lexical", +] + +[[package]] +name = "num-complex" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "73f88a1307638156682bada9d7604135552957b7818057dcef22705b4d509495" +dependencies = [ + "num-traits", +] + +[[package]] +name = "num-integer" +version = "0.1.46" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7969661fd2958a5cb096e56c8e1ad0444ac2bbcd0061bd28660485a44879858f" +dependencies = [ + "num-traits", +] + +[[package]] +name = "num-traits" +version = "0.2.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" +dependencies = [ + "autocfg", +] + +[[package]] +name = "numpy" +version = "0.28.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "778da78c64ddc928ebf5ad9df5edf0789410ff3bdbf3619aed51cd789a6af1e2" +dependencies = [ + "libc", + "ndarray 0.17.2", + "num-complex", + "num-integer", + "num-traits", + "pyo3", + "pyo3-build-config", + "rustc-hash", +] + +[[package]] +name = "once_cell" +version = "1.21.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" + +[[package]] +name = "once_cell_polyfill" +version = "1.70.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "384b8ab6d37215f3c5301a95a4accb5d64aa607f1fcb26a11b5303878451b4fe" + +[[package]] +name = "onig" +version = "6.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0cc3cbf698f9438986c11a880c90a6d04b9de27575afd28bbf45b154b6c709e2" +dependencies = [ + "bitflags", + "libc", + "once_cell", + "onig_sys", +] + +[[package]] +name = "onig_sys" +version = "69.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e68317604e77e53b85896388e1a803c1d21b74c899ec9e5e1112db90735edd7" +dependencies = [ + "cc", + "pkg-config", +] + +[[package]] +name = "paste" +version = "1.0.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "57c0d7b74b563b49d38dae00a0c37d4d6de9b432382b2892f0574ddcae73fd0a" + +[[package]] +name = "pin-project-lite" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a89322df9ebe1c1578d689c92318e070967d1042b512afbe49518723f4e6d5cd" + +[[package]] +name = "pkg-config" +version = "0.3.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "19f132c84eca552bf34cab8ec81f1c1dcc229b811638f9d283dceabe58c5569e" + +[[package]] +name = "portable-atomic" +version = "1.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c33a9471896f1c69cecef8d20cbe2f7accd12527ce60845ff44c153bb2a21b49" + +[[package]] +name = "portable-atomic-util" +version = "0.2.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c2a106d1259c23fac8e543272398ae0e3c0b8d33c88ed73d0cc71b0f1d902618" +dependencies = [ + "portable-atomic", +] + +[[package]] +name = "ppv-lite86" +version = "0.2.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85eae3c4ed2f50dcfe72643da4befc30deadb458a9b590d720cde2f2b1e97da9" +dependencies = [ + "zerocopy", +] + +[[package]] +name = "prettyplease" +version = "0.2.37" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "479ca8adacdd7ce8f1fb39ce9ecccbfe93a3f1344b3d0d97f20bc0196208f62b" +dependencies = [ + "proc-macro2", + "syn", +] + +[[package]] +name = "proc-macro2" +version = "1.0.106" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8fd00f0bb2e90d81d1044c2b32617f68fcb9fa3bb7640c23e9c748e53fb30934" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "pyo3" +version = "0.28.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf85e27e86080aafd5a22eae58a162e133a589551542b3e5cee4beb27e54f8e1" +dependencies = [ + "libc", + "once_cell", + "portable-atomic", + "pyo3-build-config", + "pyo3-ffi", + "pyo3-macros", +] + +[[package]] +name = "pyo3-async-runtimes" +version = "0.28.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9e7364a95bf00e8377bbf9b0f09d7ff9715a29d8fcf93b47d1a967363b973178" +dependencies = [ + "futures-channel", + "futures-util", + "once_cell", + "pin-project-lite", + "pyo3", + "tokio", +] + +[[package]] +name = "pyo3-build-config" +version = "0.28.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8bf94ee265674bf76c09fa430b0e99c26e319c945d96ca0d5a8215f31bf81cf7" +dependencies = [ + "target-lexicon", +] + +[[package]] +name = "pyo3-ffi" +version = "0.28.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "491aa5fc66d8059dd44a75f4580a2962c1862a1c2945359db36f6c2818b748dc" +dependencies = [ + "libc", + "pyo3-build-config", +] + +[[package]] +name = "pyo3-macros" +version = "0.28.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f5d671734e9d7a43449f8480f8b38115df67bef8d21f76837fa75ee7aaa5e52e" +dependencies = [ + "proc-macro2", + "pyo3-macros-backend", + "quote", + "syn", +] + +[[package]] +name = "pyo3-macros-backend" +version = "0.28.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22faaa1ce6c430a1f71658760497291065e6450d7b5dc2bcf254d49f66ee700a" +dependencies = [ + "heck", + "proc-macro2", + "pyo3-build-config", + "quote", + "syn", +] + +[[package]] +name = "quote" +version = "1.0.45" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41f2619966050689382d2b44f664f4bc593e129785a36d6ee376ddf37259b924" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "r-efi" +version = "5.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "69cdb34c158ceb288df11e18b4bd39de994f6657d83847bdffdbd7f346754b0f" + +[[package]] +name = "r-efi" +version = "6.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" + +[[package]] +name = "rand" +version = "0.9.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "44c5af06bb1b7d3216d91932aed5265164bf384dc89cd6ba05cf59a35f5f76ea" +dependencies = [ + "rand_chacha", + "rand_core", +] + +[[package]] +name = "rand_chacha" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3022b5f1df60f26e1ffddd6c66e8aa15de382ae63b3a0c1bfc0e4d3e3f325cb" +dependencies = [ + "ppv-lite86", + "rand_core", +] + +[[package]] +name = "rand_core" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "76afc826de14238e6e8c374ddcc1fa19e374fd8dd986b0d2af0d02377261d83c" +dependencies = [ + "getrandom 0.3.4", +] + +[[package]] +name = "rawpointer" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "60a357793950651c4ed0f3f52338f53b2f809f32d83a07f72909fa13e4c6c1e3" + +[[package]] +name = "rayon" +version = "1.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fb39b166781f92d482534ef4b4b1b2568f42613b53e5b6c160e24cfbfa30926d" +dependencies = [ + "either", + "rayon-core", +] + +[[package]] +name = "rayon-cond" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2964d0cf57a3e7a06e8183d14a8b527195c706b7983549cd5462d5aa3747438f" +dependencies = [ + "either", + "itertools", + "rayon", +] + +[[package]] +name = "rayon-core" +version = "1.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22e18b0f0062d30d4230b2e85ff77fdfe4326feb054b9783a3460d8435c8ab91" +dependencies = [ + "crossbeam-deque", + "crossbeam-utils", +] + +[[package]] +name = "regex" +version = "1.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e10754a14b9137dd7b1e3e5b0493cc9171fdd105e0ab477f51b72e7f3ac0e276" +dependencies = [ + "aho-corasick", + "memchr", + "regex-automata", + "regex-syntax", +] + +[[package]] +name = "regex-automata" +version = "0.4.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e1dd4122fc1595e8162618945476892eefca7b88c52820e74af6262213cae8f" +dependencies = [ + "aho-corasick", + "memchr", + "regex-syntax", +] + +[[package]] +name = "regex-syntax" +version = "0.8.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc897dd8d9e8bd1ed8cdad82b5966c3e0ecae09fb1907d58efaa013543185d0a" + +[[package]] +name = "rustc-hash" +version = "2.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94300abf3f1ae2e2b8ffb7b58043de3d399c73fa6f4b73826402a5c457614dbe" + +[[package]] +name = "rustix" +version = "1.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6fe4565b9518b83ef4f91bb47ce29620ca828bd32cb7e408f0062e9930ba190" +dependencies = [ + "bitflags", + "errno", + "libc", + "linux-raw-sys", + "windows-sys", +] + +[[package]] +name = "rustversion" +version = "1.0.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b39cdef0fa800fc44525c84ccb54a029961a8215f9619753635a9c0d2538d46d" + +[[package]] +name = "ryu" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9774ba4a74de5f7b1c1451ed6cd5285a32eddb5cccb8cc655a4e50009e06477f" + +[[package]] +name = "semver" +version = "1.0.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a7852d02fc848982e0c167ef163aaff9cd91dc640ba85e263cb1ce46fae51cd" + +[[package]] +name = "serde" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a8e94ea7f378bd32cbbd37198a4a91436180c5bb472411e48b5ec2e2124ae9e" +dependencies = [ + "serde_core", + "serde_derive", +] + +[[package]] +name = "serde_core" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41d385c7d4ca58e59fc732af25c3983b67ac852c1a25000afe1175de458b67ad" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "serde_json" +version = "1.0.149" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "83fc039473c5595ace860d8c4fafa220ff474b3fc6bfdb4293327f1a37e94d86" +dependencies = [ + "itoa", + "memchr", + "serde", + "serde_core", + "zmij", +] + +[[package]] +name = "shlex" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" + +[[package]] +name = "signal-hook-registry" +version = "1.4.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c4db69cba1110affc0e9f7bcd48bbf87b3f4fc7c61fc9155afd4c469eb3d6c1b" +dependencies = [ + "errno", + "libc", +] + +[[package]] +name = "slab" +version = "0.4.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c790de23124f9ab44544d7ac05d60440adc586479ce501c1d6d7da3cd8c9cf5" + +[[package]] +name = "smallvec" +version = "1.15.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67b1b7a3b5fe4f1376887184045fcf45c69e92af734b7aaddc05fb777b6fbd03" + +[[package]] +name = "spm_precompiled" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5851699c4033c63636f7ea4cf7b7c1f1bf06d0cc03cfb42e711de5a5c46cf326" +dependencies = [ + "base64", + "nom", + "serde", + "unicode-segmentation", +] + +[[package]] +name = "static_assertions" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f" + +[[package]] +name = "strsim" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" + +[[package]] +name = "syn" +version = "2.0.117" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e665b8803e7b1d2a727f4023456bbbbe74da67099c585258af0ad9c5013b9b99" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "target-lexicon" +version = "0.13.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "adb6935a6f5c20170eeceb1a3835a49e12e19d792f6dd344ccc76a985ca5a6ca" + +[[package]] +name = "tempfile" +version = "3.27.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd" +dependencies = [ + "fastrand", + "getrandom 0.4.2", + "once_cell", + "rustix", + "windows-sys", +] + +[[package]] +name = "thiserror" +version = "2.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4288b5bcbc7920c07a1149a35cf9590a2aa808e0bc1eafaade0b80947865fbc4" +dependencies = [ + "thiserror-impl", +] + +[[package]] +name = "thiserror-impl" +version = "2.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ebc4ee7f67670e9b64d05fa4253e753e016c6c95ff35b89b7941d6b856dec1d5" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "tokenizers" +version = "0.23.1" +dependencies = [ + "ahash", + "compact_str", + "daachorse", + "dary_heap", + "derive_builder", + "esaxx-rs", + "getrandom 0.3.4", + "indicatif", + "itertools", + "log", + "macro_rules_attribute", + "monostate", + "onig", + "paste", + "rand", + "rayon", + "rayon-cond", + "regex", + "regex-syntax", + "serde", + "serde_json", + "spm_precompiled", + "thiserror", + "unicode-normalization-alignments", + "unicode-segmentation", + "unicode_categories", +] + +[[package]] +name = "tokenizers-python" +version = "0.23.1" +dependencies = [ + "ahash", + "env_logger", + "itertools", + "libc", + "ndarray 0.16.1", + "numpy", + "once_cell", + "pyo3", + "pyo3-async-runtimes", + "pyo3-build-config", + "pyo3-ffi", + "rayon", + "serde", + "serde_json", + "tempfile", + "tokenizers", + "tokio", +] + +[[package]] +name = "tokio" +version = "1.52.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b67dee974fe86fd92cc45b7a95fdd2f99a36a6d7b0d431a231178d3d670bbcc6" +dependencies = [ + "libc", + "mio", + "pin-project-lite", + "signal-hook-registry", + "tokio-macros", + "windows-sys", +] + +[[package]] +name = "tokio-macros" +version = "2.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "385a6cb71ab9ab790c5fe8d67f1645e6c450a7ce006a33de03daa956cf70a496" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "unicode-ident" +version = "1.0.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" + +[[package]] +name = "unicode-normalization-alignments" +version = "0.1.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "43f613e4fa046e69818dd287fdc4bc78175ff20331479dab6e1b0f98d57062de" +dependencies = [ + "smallvec", +] + +[[package]] +name = "unicode-segmentation" +version = "1.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9629274872b2bfaf8d66f5f15725007f635594914870f65218920345aa11aa8c" + +[[package]] +name = "unicode-width" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b4ac048d71ede7ee76d585517add45da530660ef4390e49b098733c6e897f254" + +[[package]] +name = "unicode-xid" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ebc1c04c71510c7f702b52b7c350734c9ff1295c464a03335b00bb84fc54f853" + +[[package]] +name = "unicode_categories" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "39ec24b3121d976906ece63c9daad25b85969647682eee313cb5779fdd69e14e" + +[[package]] +name = "unit-prefix" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "81e544489bf3d8ef66c953931f56617f423cd4b5494be343d9b9d3dda037b9a3" + +[[package]] +name = "utf8parse" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" + +[[package]] +name = "version_check" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" + +[[package]] +name = "wasi" +version = "0.11.1+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b" + +[[package]] +name = "wasip2" +version = "1.0.3+wasi-0.2.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "20064672db26d7cdc89c7798c48a0fdfac8213434a1186e5ef29fd560ae223d6" +dependencies = [ + "wit-bindgen 0.57.1", +] + +[[package]] +name = "wasip3" +version = "0.4.0+wasi-0.3.0-rc-2026-01-06" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5428f8bf88ea5ddc08faddef2ac4a67e390b88186c703ce6dbd955e1c145aca5" +dependencies = [ + "wit-bindgen 0.51.0", +] + +[[package]] +name = "wasm-bindgen" +version = "0.2.118" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0bf938a0bacb0469e83c1e148908bd7d5a6010354cf4fb73279b7447422e3a89" +dependencies = [ + "cfg-if", + "once_cell", + "rustversion", + "wasm-bindgen-macro", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-macro" +version = "0.2.118" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eeff24f84126c0ec2db7a449f0c2ec963c6a49efe0698c4242929da037ca28ed" +dependencies = [ + "quote", + "wasm-bindgen-macro-support", +] + +[[package]] +name = "wasm-bindgen-macro-support" +version = "0.2.118" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9d08065faf983b2b80a79fd87d8254c409281cf7de75fc4b773019824196c904" +dependencies = [ + "bumpalo", + "proc-macro2", + "quote", + "syn", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-shared" +version = "0.2.118" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5fd04d9e306f1907bd13c6361b5c6bfc7b3b3c095ed3f8a9246390f8dbdee129" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "wasm-encoder" +version = "0.244.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "990065f2fe63003fe337b932cfb5e3b80e0b4d0f5ff650e6985b1048f62c8319" +dependencies = [ + "leb128fmt", + "wasmparser", +] + +[[package]] +name = "wasm-metadata" +version = "0.244.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bb0e353e6a2fbdc176932bbaab493762eb1255a7900fe0fea1a2f96c296cc909" +dependencies = [ + "anyhow", + "indexmap", + "wasm-encoder", + "wasmparser", +] + +[[package]] +name = "wasmparser" +version = "0.244.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "47b807c72e1bac69382b3a6fb3dbe8ea4c0ed87ff5629b8685ae6b9a611028fe" +dependencies = [ + "bitflags", + "hashbrown 0.15.5", + "indexmap", + "semver", +] + +[[package]] +name = "web-time" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a6580f308b1fad9207618087a65c04e7a10bc77e02c8e84e9b00dd4b12fa0bb" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "windows-link" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" + +[[package]] +name = "windows-sys" +version = "0.61.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae137229bcbd6cdf0f7b80a31df61766145077ddf49416a728b02cb3921ff3fc" +dependencies = [ + "windows-link", +] + +[[package]] +name = "wit-bindgen" +version = "0.51.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d7249219f66ced02969388cf2bb044a09756a083d0fab1e566056b04d9fbcaa5" +dependencies = [ + "wit-bindgen-rust-macro", +] + +[[package]] +name = "wit-bindgen" +version = "0.57.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ebf944e87a7c253233ad6766e082e3cd714b5d03812acc24c318f549614536e" + +[[package]] +name = "wit-bindgen-core" +version = "0.51.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ea61de684c3ea68cb082b7a88508a8b27fcc8b797d738bfc99a82facf1d752dc" +dependencies = [ + "anyhow", + "heck", + "wit-parser", +] + +[[package]] +name = "wit-bindgen-rust" +version = "0.51.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b7c566e0f4b284dd6561c786d9cb0142da491f46a9fbed79ea69cdad5db17f21" +dependencies = [ + "anyhow", + "heck", + "indexmap", + "prettyplease", + "syn", + "wasm-metadata", + "wit-bindgen-core", + "wit-component", +] + +[[package]] +name = "wit-bindgen-rust-macro" +version = "0.51.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c0f9bfd77e6a48eccf51359e3ae77140a7f50b1e2ebfe62422d8afdaffab17a" +dependencies = [ + "anyhow", + "prettyplease", + "proc-macro2", + "quote", + "syn", + "wit-bindgen-core", + "wit-bindgen-rust", +] + +[[package]] +name = "wit-component" +version = "0.244.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9d66ea20e9553b30172b5e831994e35fbde2d165325bec84fc43dbf6f4eb9cb2" +dependencies = [ + "anyhow", + "bitflags", + "indexmap", + "log", + "serde", + "serde_derive", + "serde_json", + "wasm-encoder", + "wasm-metadata", + "wasmparser", + "wit-parser", +] + +[[package]] +name = "wit-parser" +version = "0.244.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ecc8ac4bc1dc3381b7f59c34f00b67e18f910c2c0f50015669dde7def656a736" +dependencies = [ + "anyhow", + "id-arena", + "indexmap", + "log", + "semver", + "serde", + "serde_derive", + "serde_json", + "unicode-xid", + "wasmparser", +] + +[[package]] +name = "zerocopy" +version = "0.8.48" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eed437bf9d6692032087e337407a86f04cd8d6a16a37199ed57949d415bd68e9" +dependencies = [ + "zerocopy-derive", +] + +[[package]] +name = "zerocopy-derive" +version = "0.8.48" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "70e3cd084b1788766f53af483dd21f93881ff30d7320490ec3ef7526d203bad4" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "zmij" +version = "1.0.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8848ee67ecc8aedbaf3e4122217aff892639231befc6a1b58d29fff4c2cabaa" diff --git a/bindings/python/Cargo.toml b/bindings/python/Cargo.toml index 37f6a3ef4..ce082975b 100644 --- a/bindings/python/Cargo.toml +++ b/bindings/python/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "tokenizers-python" -version = "0.22.3-dev.0" +version = "0.23.1" authors = ["Anthony MOI "] edition = "2021" diff --git a/tokenizers/Cargo.toml b/tokenizers/Cargo.toml index 0e937f3cc..d4c291963 100644 --- a/tokenizers/Cargo.toml +++ b/tokenizers/Cargo.toml @@ -5,7 +5,7 @@ authors = [ ] edition = "2018" name = "tokenizers" -version = "0.22.3-dev.0" +version = "0.23.1" homepage = "https://github.com/huggingface/tokenizers" repository = "https://github.com/huggingface/tokenizers" documentation = "https://docs.rs/tokenizers/" From 618b38e57e7ea8fcc27989a4c14952d2faa24956 Mon Sep 17 00:00:00 2001 From: Arthur Date: Tue, 28 Apr 2026 00:26:43 +0900 Subject: [PATCH 2/3] ci: trigger node release on `node-v*` tag, skip GH release re-create Lets us re-cut the npm publish independently of the python/rust release workflows (which own `v*` and already created the GitHub Release for 0.23.1, causing `napi prepublish` to 422 on retry). - trigger: `v*` -> `node-v*` - split publish into two steps: - `napi prepublish -t npm --no-gh-release` does the per-platform `npm publish` from `npm//`, skipping the `POST /releases` call. - `npm publish --ignore-scripts` publishes the parent package without re-running the `prepublishOnly` hook. Co-Authored-By: Claude Opus 4.7 (1M context) --- .github/workflows/node-release.yml | 26 +++++++++++++++++++++++--- 1 file changed, 23 insertions(+), 3 deletions(-) diff --git a/.github/workflows/node-release.yml b/.github/workflows/node-release.yml index 53a7948d8..d24b2983d 100644 --- a/.github/workflows/node-release.yml +++ b/.github/workflows/node-release.yml @@ -10,7 +10,12 @@ permissions: on: push: tags: - - v* + # Node releases use a dedicated `node-v*` tag (e.g. `node-v0.23.1`) + # so we can re-cut the npm publish without colliding with the + # `v*` tag already consumed by the python / rust release workflows + # (which create a GitHub Release that `napi prepublish` would + # otherwise try to recreate -> 422 already_exists). + - node-v* jobs: build: @@ -219,9 +224,24 @@ jobs: working-directory: ./bindings/node run: ls -R ./npm shell: bash - - name: Publish + - name: Publish per-platform packages + # napi prepublish stamps each `npm//package.json` with the + # parent version + writes `optionalDependencies` on the parent, + # then runs `npm publish` from each `npm//` dir. + # `--no-gh-release` skips the `POST /repos/.../releases` call — + # we trigger this workflow on `node-v*` tags, but the package + # version may correspond to a `v*` tag that the python/rust + # release already published, so re-creating it would 422. working-directory: ./bindings/node - run: npm publish --access public --provenance + run: npx napi prepublish -t npm --no-gh-release + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }} + - name: Publish parent package + # `--ignore-scripts` skips the `prepublishOnly` hook — it would + # re-run `napi prepublish` which we just executed above. + working-directory: ./bindings/node + run: npm publish --ignore-scripts --access public --provenance env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }} \ No newline at end of file From 6b6137ab204ecdfe9330dda8cbe759a84f4c3bdf Mon Sep 17 00:00:00 2001 From: Arthur Date: Tue, 28 Apr 2026 00:45:37 +0900 Subject: [PATCH 3/3] node: scope platform packages under @huggingface/ MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Set napi.packageName so per-platform sub-packages publish as `@huggingface/tokenizers-` (e.g. @huggingface/tokenizers-darwin-x64) instead of unscoped `tokenizers-`. The unscoped names on npm are owned by `n1t0` (Anthony Moi) — the org CI npm token has no publish access there, which is why the 0.23.1 release attempt 404'd on the very first sub-package. The unscoped sub-packages were never successfully published from CI; the only versions on the registry (0.13.4-rc1/rc2) were hand-published by n1t0 and the 0.13.3 stable parent shipped with optionalDependencies that 404 at install (npm tolerates it for optionalDependencies). Parent package stays `tokenizers` — public-facing import name is unchanged. napi regenerates the runtime loader to require the scoped names. Also delete the 13 checked-in `bindings/node/npm//` template files. They hardcoded the old unscoped name and version 0.13.4-rc1 and were misleading as source. CI regenerates them via `napi create-npm-dirs` (already in node-release.yml line 213). Co-Authored-By: Claude Opus 4.7 (1M context) --- bindings/node/npm/android-arm-eabi/README.md | 3 -- .../node/npm/android-arm-eabi/package.json | 32 ----------------- bindings/node/npm/android-arm64/README.md | 3 -- bindings/node/npm/android-arm64/package.json | 32 ----------------- bindings/node/npm/darwin-arm64/README.md | 3 -- bindings/node/npm/darwin-arm64/package.json | 32 ----------------- bindings/node/npm/darwin-x64/README.md | 3 -- bindings/node/npm/darwin-x64/package.json | 32 ----------------- bindings/node/npm/freebsd-x64/README.md | 3 -- bindings/node/npm/freebsd-x64/package.json | 32 ----------------- .../node/npm/linux-arm-gnueabihf/README.md | 3 -- .../node/npm/linux-arm-gnueabihf/package.json | 32 ----------------- bindings/node/npm/linux-arm64-gnu/README.md | 3 -- .../node/npm/linux-arm64-gnu/package.json | 35 ------------------- bindings/node/npm/linux-arm64-musl/README.md | 3 -- .../node/npm/linux-arm64-musl/package.json | 35 ------------------- bindings/node/npm/linux-x64-gnu/README.md | 3 -- bindings/node/npm/linux-x64-gnu/package.json | 35 ------------------- bindings/node/npm/linux-x64-musl/README.md | 3 -- bindings/node/npm/linux-x64-musl/package.json | 35 ------------------- bindings/node/npm/win32-arm64-msvc/README.md | 3 -- .../node/npm/win32-arm64-msvc/package.json | 32 ----------------- bindings/node/npm/win32-ia32-msvc/README.md | 3 -- .../node/npm/win32-ia32-msvc/package.json | 32 ----------------- bindings/node/npm/win32-x64-msvc/README.md | 3 -- bindings/node/npm/win32-x64-msvc/package.json | 32 ----------------- bindings/node/package.json | 1 + 27 files changed, 1 insertion(+), 467 deletions(-) delete mode 100644 bindings/node/npm/android-arm-eabi/README.md delete mode 100644 bindings/node/npm/android-arm-eabi/package.json delete mode 100644 bindings/node/npm/android-arm64/README.md delete mode 100644 bindings/node/npm/android-arm64/package.json delete mode 100644 bindings/node/npm/darwin-arm64/README.md delete mode 100644 bindings/node/npm/darwin-arm64/package.json delete mode 100644 bindings/node/npm/darwin-x64/README.md delete mode 100644 bindings/node/npm/darwin-x64/package.json delete mode 100644 bindings/node/npm/freebsd-x64/README.md delete mode 100644 bindings/node/npm/freebsd-x64/package.json delete mode 100644 bindings/node/npm/linux-arm-gnueabihf/README.md delete mode 100644 bindings/node/npm/linux-arm-gnueabihf/package.json delete mode 100644 bindings/node/npm/linux-arm64-gnu/README.md delete mode 100644 bindings/node/npm/linux-arm64-gnu/package.json delete mode 100644 bindings/node/npm/linux-arm64-musl/README.md delete mode 100644 bindings/node/npm/linux-arm64-musl/package.json delete mode 100644 bindings/node/npm/linux-x64-gnu/README.md delete mode 100644 bindings/node/npm/linux-x64-gnu/package.json delete mode 100644 bindings/node/npm/linux-x64-musl/README.md delete mode 100644 bindings/node/npm/linux-x64-musl/package.json delete mode 100644 bindings/node/npm/win32-arm64-msvc/README.md delete mode 100644 bindings/node/npm/win32-arm64-msvc/package.json delete mode 100644 bindings/node/npm/win32-ia32-msvc/README.md delete mode 100644 bindings/node/npm/win32-ia32-msvc/package.json delete mode 100644 bindings/node/npm/win32-x64-msvc/README.md delete mode 100644 bindings/node/npm/win32-x64-msvc/package.json diff --git a/bindings/node/npm/android-arm-eabi/README.md b/bindings/node/npm/android-arm-eabi/README.md deleted file mode 100644 index e02d09a5b..000000000 --- a/bindings/node/npm/android-arm-eabi/README.md +++ /dev/null @@ -1,3 +0,0 @@ -# `tokenizers-android-arm-eabi` - -This is the **armv7-linux-androideabi** binary for `tokenizers` diff --git a/bindings/node/npm/android-arm-eabi/package.json b/bindings/node/npm/android-arm-eabi/package.json deleted file mode 100644 index ec3afb6d9..000000000 --- a/bindings/node/npm/android-arm-eabi/package.json +++ /dev/null @@ -1,32 +0,0 @@ -{ - "name": "tokenizers-android-arm-eabi", - "version": "0.13.4-rc1", - "os": [ - "android" - ], - "cpu": [ - "arm" - ], - "main": "tokenizers.android-arm-eabi.node", - "files": [ - "tokenizers.android-arm-eabi.node" - ], - "description": "Tokenizers platform specific bindings", - "keywords": [ - "napi-rs", - "NAPI", - "N-API", - "Rust", - "node-addon", - "node-addon-api" - ], - "license": "MIT", - "engines": { - "node": ">= 10" - }, - "publishConfig": { - "registry": "https://registry.npmjs.org/", - "access": "public" - }, - "repository": "tokenizers" -} \ No newline at end of file diff --git a/bindings/node/npm/android-arm64/README.md b/bindings/node/npm/android-arm64/README.md deleted file mode 100644 index 9ffabcf38..000000000 --- a/bindings/node/npm/android-arm64/README.md +++ /dev/null @@ -1,3 +0,0 @@ -# `tokenizers-android-arm64` - -This is the **aarch64-linux-android** binary for `tokenizers` diff --git a/bindings/node/npm/android-arm64/package.json b/bindings/node/npm/android-arm64/package.json deleted file mode 100644 index d1caf2cd2..000000000 --- a/bindings/node/npm/android-arm64/package.json +++ /dev/null @@ -1,32 +0,0 @@ -{ - "name": "tokenizers-android-arm64", - "version": "0.13.4-rc1", - "os": [ - "android" - ], - "cpu": [ - "arm64" - ], - "main": "tokenizers.android-arm64.node", - "files": [ - "tokenizers.android-arm64.node" - ], - "description": "Tokenizers platform specific bindings", - "keywords": [ - "napi-rs", - "NAPI", - "N-API", - "Rust", - "node-addon", - "node-addon-api" - ], - "license": "MIT", - "engines": { - "node": ">= 10" - }, - "publishConfig": { - "registry": "https://registry.npmjs.org/", - "access": "public" - }, - "repository": "tokenizers" -} \ No newline at end of file diff --git a/bindings/node/npm/darwin-arm64/README.md b/bindings/node/npm/darwin-arm64/README.md deleted file mode 100644 index 8e6cefceb..000000000 --- a/bindings/node/npm/darwin-arm64/README.md +++ /dev/null @@ -1,3 +0,0 @@ -# `tokenizers-darwin-arm64` - -This is the **aarch64-apple-darwin** binary for `tokenizers` diff --git a/bindings/node/npm/darwin-arm64/package.json b/bindings/node/npm/darwin-arm64/package.json deleted file mode 100644 index ced185a17..000000000 --- a/bindings/node/npm/darwin-arm64/package.json +++ /dev/null @@ -1,32 +0,0 @@ -{ - "name": "tokenizers-darwin-arm64", - "version": "0.13.4-rc1", - "os": [ - "darwin" - ], - "cpu": [ - "arm64" - ], - "main": "tokenizers.darwin-arm64.node", - "files": [ - "tokenizers.darwin-arm64.node" - ], - "description": "Tokenizers platform specific bindings", - "keywords": [ - "napi-rs", - "NAPI", - "N-API", - "Rust", - "node-addon", - "node-addon-api" - ], - "license": "MIT", - "engines": { - "node": ">= 10" - }, - "publishConfig": { - "registry": "https://registry.npmjs.org/", - "access": "public" - }, - "repository": "tokenizers" -} \ No newline at end of file diff --git a/bindings/node/npm/darwin-x64/README.md b/bindings/node/npm/darwin-x64/README.md deleted file mode 100644 index 097359000..000000000 --- a/bindings/node/npm/darwin-x64/README.md +++ /dev/null @@ -1,3 +0,0 @@ -# `tokenizers-darwin-x64` - -This is the **x86_64-apple-darwin** binary for `tokenizers` diff --git a/bindings/node/npm/darwin-x64/package.json b/bindings/node/npm/darwin-x64/package.json deleted file mode 100644 index 43674028c..000000000 --- a/bindings/node/npm/darwin-x64/package.json +++ /dev/null @@ -1,32 +0,0 @@ -{ - "name": "tokenizers-darwin-x64", - "version": "0.13.4-rc1", - "os": [ - "darwin" - ], - "cpu": [ - "x64" - ], - "main": "tokenizers.darwin-x64.node", - "files": [ - "tokenizers.darwin-x64.node" - ], - "description": "Tokenizers platform specific bindings", - "keywords": [ - "napi-rs", - "NAPI", - "N-API", - "Rust", - "node-addon", - "node-addon-api" - ], - "license": "MIT", - "engines": { - "node": ">= 10" - }, - "publishConfig": { - "registry": "https://registry.npmjs.org/", - "access": "public" - }, - "repository": "tokenizers" -} \ No newline at end of file diff --git a/bindings/node/npm/freebsd-x64/README.md b/bindings/node/npm/freebsd-x64/README.md deleted file mode 100644 index 612df808c..000000000 --- a/bindings/node/npm/freebsd-x64/README.md +++ /dev/null @@ -1,3 +0,0 @@ -# `tokenizers-freebsd-x64` - -This is the **x86_64-unknown-freebsd** binary for `tokenizers` diff --git a/bindings/node/npm/freebsd-x64/package.json b/bindings/node/npm/freebsd-x64/package.json deleted file mode 100644 index 876c8f217..000000000 --- a/bindings/node/npm/freebsd-x64/package.json +++ /dev/null @@ -1,32 +0,0 @@ -{ - "name": "tokenizers-freebsd-x64", - "version": "0.13.4-rc1", - "os": [ - "freebsd" - ], - "cpu": [ - "x64" - ], - "main": "tokenizers.freebsd-x64.node", - "files": [ - "tokenizers.freebsd-x64.node" - ], - "description": "Tokenizers platform specific bindings", - "keywords": [ - "napi-rs", - "NAPI", - "N-API", - "Rust", - "node-addon", - "node-addon-api" - ], - "license": "MIT", - "engines": { - "node": ">= 10" - }, - "publishConfig": { - "registry": "https://registry.npmjs.org/", - "access": "public" - }, - "repository": "tokenizers" -} \ No newline at end of file diff --git a/bindings/node/npm/linux-arm-gnueabihf/README.md b/bindings/node/npm/linux-arm-gnueabihf/README.md deleted file mode 100644 index 95709a847..000000000 --- a/bindings/node/npm/linux-arm-gnueabihf/README.md +++ /dev/null @@ -1,3 +0,0 @@ -# `tokenizers-linux-arm-gnueabihf` - -This is the **armv7-unknown-linux-gnueabihf** binary for `tokenizers` diff --git a/bindings/node/npm/linux-arm-gnueabihf/package.json b/bindings/node/npm/linux-arm-gnueabihf/package.json deleted file mode 100644 index 2c98ec6d4..000000000 --- a/bindings/node/npm/linux-arm-gnueabihf/package.json +++ /dev/null @@ -1,32 +0,0 @@ -{ - "name": "tokenizers-linux-arm-gnueabihf", - "version": "0.13.4-rc1", - "os": [ - "linux" - ], - "cpu": [ - "arm" - ], - "main": "tokenizers.linux-arm-gnueabihf.node", - "files": [ - "tokenizers.linux-arm-gnueabihf.node" - ], - "description": "Tokenizers platform specific bindings", - "keywords": [ - "napi-rs", - "NAPI", - "N-API", - "Rust", - "node-addon", - "node-addon-api" - ], - "license": "MIT", - "engines": { - "node": ">= 10" - }, - "publishConfig": { - "registry": "https://registry.npmjs.org/", - "access": "public" - }, - "repository": "tokenizers" -} \ No newline at end of file diff --git a/bindings/node/npm/linux-arm64-gnu/README.md b/bindings/node/npm/linux-arm64-gnu/README.md deleted file mode 100644 index 0925e15eb..000000000 --- a/bindings/node/npm/linux-arm64-gnu/README.md +++ /dev/null @@ -1,3 +0,0 @@ -# `tokenizers-linux-arm64-gnu` - -This is the **aarch64-unknown-linux-gnu** binary for `tokenizers` diff --git a/bindings/node/npm/linux-arm64-gnu/package.json b/bindings/node/npm/linux-arm64-gnu/package.json deleted file mode 100644 index ff448db71..000000000 --- a/bindings/node/npm/linux-arm64-gnu/package.json +++ /dev/null @@ -1,35 +0,0 @@ -{ - "name": "tokenizers-linux-arm64-gnu", - "version": "0.13.4-rc1", - "os": [ - "linux" - ], - "cpu": [ - "arm64" - ], - "main": "tokenizers.linux-arm64-gnu.node", - "files": [ - "tokenizers.linux-arm64-gnu.node" - ], - "description": "Tokenizers platform specific bindings", - "keywords": [ - "napi-rs", - "NAPI", - "N-API", - "Rust", - "node-addon", - "node-addon-api" - ], - "license": "MIT", - "engines": { - "node": ">= 10" - }, - "publishConfig": { - "registry": "https://registry.npmjs.org/", - "access": "public" - }, - "repository": "tokenizers", - "libc": [ - "glibc" - ] -} \ No newline at end of file diff --git a/bindings/node/npm/linux-arm64-musl/README.md b/bindings/node/npm/linux-arm64-musl/README.md deleted file mode 100644 index a53a870df..000000000 --- a/bindings/node/npm/linux-arm64-musl/README.md +++ /dev/null @@ -1,3 +0,0 @@ -# `tokenizers-linux-arm64-musl` - -This is the **aarch64-unknown-linux-musl** binary for `tokenizers` diff --git a/bindings/node/npm/linux-arm64-musl/package.json b/bindings/node/npm/linux-arm64-musl/package.json deleted file mode 100644 index 34e52295a..000000000 --- a/bindings/node/npm/linux-arm64-musl/package.json +++ /dev/null @@ -1,35 +0,0 @@ -{ - "name": "tokenizers-linux-arm64-musl", - "version": "0.13.4-rc1", - "os": [ - "linux" - ], - "cpu": [ - "arm64" - ], - "main": "tokenizers.linux-arm64-musl.node", - "files": [ - "tokenizers.linux-arm64-musl.node" - ], - "description": "Tokenizers platform specific bindings", - "keywords": [ - "napi-rs", - "NAPI", - "N-API", - "Rust", - "node-addon", - "node-addon-api" - ], - "license": "MIT", - "engines": { - "node": ">= 10" - }, - "publishConfig": { - "registry": "https://registry.npmjs.org/", - "access": "public" - }, - "repository": "tokenizers", - "libc": [ - "musl" - ] -} \ No newline at end of file diff --git a/bindings/node/npm/linux-x64-gnu/README.md b/bindings/node/npm/linux-x64-gnu/README.md deleted file mode 100644 index c35a19a2d..000000000 --- a/bindings/node/npm/linux-x64-gnu/README.md +++ /dev/null @@ -1,3 +0,0 @@ -# `tokenizers-linux-x64-gnu` - -This is the **x86_64-unknown-linux-gnu** binary for `tokenizers` diff --git a/bindings/node/npm/linux-x64-gnu/package.json b/bindings/node/npm/linux-x64-gnu/package.json deleted file mode 100644 index 459de7a78..000000000 --- a/bindings/node/npm/linux-x64-gnu/package.json +++ /dev/null @@ -1,35 +0,0 @@ -{ - "name": "tokenizers-linux-x64-gnu", - "version": "0.13.4-rc1", - "os": [ - "linux" - ], - "cpu": [ - "x64" - ], - "main": "tokenizers.linux-x64-gnu.node", - "files": [ - "tokenizers.linux-x64-gnu.node" - ], - "description": "Tokenizers platform specific bindings", - "keywords": [ - "napi-rs", - "NAPI", - "N-API", - "Rust", - "node-addon", - "node-addon-api" - ], - "license": "MIT", - "engines": { - "node": ">= 10" - }, - "publishConfig": { - "registry": "https://registry.npmjs.org/", - "access": "public" - }, - "repository": "tokenizers", - "libc": [ - "glibc" - ] -} \ No newline at end of file diff --git a/bindings/node/npm/linux-x64-musl/README.md b/bindings/node/npm/linux-x64-musl/README.md deleted file mode 100644 index 7be6d2345..000000000 --- a/bindings/node/npm/linux-x64-musl/README.md +++ /dev/null @@ -1,3 +0,0 @@ -# `tokenizers-linux-x64-musl` - -This is the **x86_64-unknown-linux-musl** binary for `tokenizers` diff --git a/bindings/node/npm/linux-x64-musl/package.json b/bindings/node/npm/linux-x64-musl/package.json deleted file mode 100644 index 886b3c028..000000000 --- a/bindings/node/npm/linux-x64-musl/package.json +++ /dev/null @@ -1,35 +0,0 @@ -{ - "name": "tokenizers-linux-x64-musl", - "version": "0.13.4-rc1", - "os": [ - "linux" - ], - "cpu": [ - "x64" - ], - "main": "tokenizers.linux-x64-musl.node", - "files": [ - "tokenizers.linux-x64-musl.node" - ], - "description": "Tokenizers platform specific bindings", - "keywords": [ - "napi-rs", - "NAPI", - "N-API", - "Rust", - "node-addon", - "node-addon-api" - ], - "license": "MIT", - "engines": { - "node": ">= 10" - }, - "publishConfig": { - "registry": "https://registry.npmjs.org/", - "access": "public" - }, - "repository": "tokenizers", - "libc": [ - "musl" - ] -} \ No newline at end of file diff --git a/bindings/node/npm/win32-arm64-msvc/README.md b/bindings/node/npm/win32-arm64-msvc/README.md deleted file mode 100644 index fcb70d6ef..000000000 --- a/bindings/node/npm/win32-arm64-msvc/README.md +++ /dev/null @@ -1,3 +0,0 @@ -# `tokenizers-win32-arm64-msvc` - -This is the **aarch64-pc-windows-msvc** binary for `tokenizers` diff --git a/bindings/node/npm/win32-arm64-msvc/package.json b/bindings/node/npm/win32-arm64-msvc/package.json deleted file mode 100644 index 100713956..000000000 --- a/bindings/node/npm/win32-arm64-msvc/package.json +++ /dev/null @@ -1,32 +0,0 @@ -{ - "name": "tokenizers-win32-arm64-msvc", - "version": "0.13.4-rc1", - "os": [ - "win32" - ], - "cpu": [ - "arm64" - ], - "main": "tokenizers.win32-arm64-msvc.node", - "files": [ - "tokenizers.win32-arm64-msvc.node" - ], - "description": "Tokenizers platform specific bindings", - "keywords": [ - "napi-rs", - "NAPI", - "N-API", - "Rust", - "node-addon", - "node-addon-api" - ], - "license": "MIT", - "engines": { - "node": ">= 10" - }, - "publishConfig": { - "registry": "https://registry.npmjs.org/", - "access": "public" - }, - "repository": "tokenizers" -} \ No newline at end of file diff --git a/bindings/node/npm/win32-ia32-msvc/README.md b/bindings/node/npm/win32-ia32-msvc/README.md deleted file mode 100644 index dc8eb878e..000000000 --- a/bindings/node/npm/win32-ia32-msvc/README.md +++ /dev/null @@ -1,3 +0,0 @@ -# `tokenizers-win32-ia32-msvc` - -This is the **i686-pc-windows-msvc** binary for `tokenizers` diff --git a/bindings/node/npm/win32-ia32-msvc/package.json b/bindings/node/npm/win32-ia32-msvc/package.json deleted file mode 100644 index e7e06ef6e..000000000 --- a/bindings/node/npm/win32-ia32-msvc/package.json +++ /dev/null @@ -1,32 +0,0 @@ -{ - "name": "tokenizers-win32-ia32-msvc", - "version": "0.13.4-rc1", - "os": [ - "win32" - ], - "cpu": [ - "ia32" - ], - "main": "tokenizers.win32-ia32-msvc.node", - "files": [ - "tokenizers.win32-ia32-msvc.node" - ], - "description": "Tokenizers platform specific bindings", - "keywords": [ - "napi-rs", - "NAPI", - "N-API", - "Rust", - "node-addon", - "node-addon-api" - ], - "license": "MIT", - "engines": { - "node": ">= 10" - }, - "publishConfig": { - "registry": "https://registry.npmjs.org/", - "access": "public" - }, - "repository": "tokenizers" -} \ No newline at end of file diff --git a/bindings/node/npm/win32-x64-msvc/README.md b/bindings/node/npm/win32-x64-msvc/README.md deleted file mode 100644 index b640b69e4..000000000 --- a/bindings/node/npm/win32-x64-msvc/README.md +++ /dev/null @@ -1,3 +0,0 @@ -# `tokenizers-win32-x64-msvc` - -This is the **x86_64-pc-windows-msvc** binary for `tokenizers` diff --git a/bindings/node/npm/win32-x64-msvc/package.json b/bindings/node/npm/win32-x64-msvc/package.json deleted file mode 100644 index a78d2472d..000000000 --- a/bindings/node/npm/win32-x64-msvc/package.json +++ /dev/null @@ -1,32 +0,0 @@ -{ - "name": "tokenizers-win32-x64-msvc", - "version": "0.13.4-rc1", - "os": [ - "win32" - ], - "cpu": [ - "x64" - ], - "main": "tokenizers.win32-x64-msvc.node", - "files": [ - "tokenizers.win32-x64-msvc.node" - ], - "description": "Tokenizers platform specific bindings", - "keywords": [ - "napi-rs", - "NAPI", - "N-API", - "Rust", - "node-addon", - "node-addon-api" - ], - "license": "MIT", - "engines": { - "node": ">= 10" - }, - "publishConfig": { - "registry": "https://registry.npmjs.org/", - "access": "public" - }, - "repository": "tokenizers" -} \ No newline at end of file diff --git a/bindings/node/package.json b/bindings/node/package.json index cbfd7400b..fb254afb6 100644 --- a/bindings/node/package.json +++ b/bindings/node/package.json @@ -18,6 +18,7 @@ ], "napi": { "binaryName": "tokenizers", + "packageName": "@huggingface/tokenizers", "targets": [ "x86_64-apple-darwin", "x86_64-pc-windows-msvc",