release: v1.0.0-rc.1

Author: SBrandeis

bindings/node/Cargo.lock

@@ -2,7 +2,7 @@
 authors = ["Nicolas Patry <nicolas@huggingface.co>"]
 edition = "2024"
 name    = "node"
-version = "1.0.0-dev.0"
+version = "1.0.0-rc.1"
 
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 

bindings/node/Cargo.toml

@@ -1,6 +1,6 @@
 {
   "name": "tokenizers",
-  "version": "1.0.0-dev.0",
+  "version": "1.0.0-rc.1",
   "repository": {
     "type": "git",
     "url": "git+https://github.com/huggingface/tokenizers.git"

bindings/node/package.json

@@ -1,6 +1,6 @@
 [package]
 name = "tokenizers-python"
-version = "1.0.0-dev.0"
+version = "1.0.0-rc.1"
 edition = "2024"
 publish = false
 authors = [

bindings/python/Cargo.lock

@@ -13,7 +13,7 @@ authors = [
 ]
 edition = "2024"
 name = "tokenizers"
-version = "1.0.0-dev.0"
+version = "1.0.0-rc.1"
 homepage = "https://github.com/huggingface/tokenizers"
 repository = "https://github.com/huggingface/tokenizers"
 documentation = "https://docs.rs/tokenizers/"
@@ -44,7 +44,7 @@ path = "src/lib.rs"
 bench = false
 
 [dependencies]
-tk-encode = { path = "tk-encode", version = "0.1.0-dev.0", default-features = false, features = [
+tk-encode = { path = "tk-encode", version = "0.1.0-rc.1", default-features = false, features = [
   "parallelism",
   "normalizers",
   "bpe",
@@ -53,10 +53,10 @@ tk-encode = { path = "tk-encode", version = "0.1.0-dev.0", default-features = fa
   "wordlevel",
   "unicode-scripts",
 ] }
-tk-convert = { path = "tk-convert", version = "0.1.0-dev.0", default-features = false }
+tk-convert = { path = "tk-convert", version = "0.1.0-rc.1", default-features = false }
 # The reader and the writer. Same component features as `tk-encode` above: the reader has one match
 # arm per component, so a component compiled out of the runtime cannot be read into it either.
-tk-serialize = { path = "tk-serialize", version = "0.1.0-dev.0", default-features = false, features = [
+tk-serialize = { path = "tk-serialize", version = "0.1.0-rc.1", default-features = false, features = [
   "deserialize",
   "serialize",
   "parallelism",

bindings/python/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "bitcannon"
-version = "0.1.0-dev.0"
+version = "0.1.0-rc.1"
 edition = "2024"
 rust-version = "1.89" # AVX-512 VBMI intrinsics (simd_avx_classify) are stable only since 1.89
 authors = [

tokenizers/Cargo.lock

@@ -1,6 +1,6 @@
 [package]
 name = "tk-convert"
-version = "0.1.0-dev.0"
+version = "0.1.0-rc.1"
 edition = "2024"
 authors = [
   "Arthur Zucker <arthur@huggingface.co>",
@@ -31,7 +31,7 @@ thiserror = "2"
 # The GPT-2 pattern a `ByteLevel` lowers to. `bitcannon::regexes` is the single source of truth for
 # it -- tk-encode's runtime recognizer reads the same const -- so a copy here could drift and cost
 # the FSM fast path without changing any test.
-bitcannon = { path = "../bitcannon", version = "0.1.0-dev.0" }
+bitcannon = { path = "../bitcannon", version = "0.1.0-rc.1" }
 
 # Latest released tokenizers, the independent id/decode oracle `tests/oracle.rs` compares against.
 tokenizers-release = { package = "tokenizers", version = "=0.23.2", optional = true }

tokenizers/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "tk-encode"
-version = "0.1.0-dev.0"
+version = "0.1.0-rc.1"
 edition = "2024"
 authors = [
   "Anthony MOI <m.anthony.moi@gmail.com>",
@@ -29,7 +29,7 @@ path = "src/lib.rs"
 
 [dependencies]
 ############ Required dependencies ###############################
-bitcannon = { path = "../bitcannon", version = "0.1.0-dev.0" }
+bitcannon = { path = "../bitcannon", version = "0.1.0-rc.1" }
 rand = "0.10"
 ptr_hash = { version = "2.0.2", default-features = false }
 unicode_categories = "0.1"