Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion .github/workflows/CI.yml
Original file line number Diff line number Diff line change
Expand Up @@ -197,7 +197,7 @@ jobs:
steps:
- uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
- name: Generate artifact attestation
uses: actions/attest-build-provenance@ef244123eb79f2f7a7e75d99086184180e6d0018 # v1
uses: actions/attest-build-provenance@ef244123eb79f2f7a7e75d99086184180e6d0018 # v1.4.4
with:
subject-path: 'wheels-*/*'
- name: Publish to PyPI
Expand Down
16 changes: 12 additions & 4 deletions .github/workflows/bitsplit.yml
Original file line number Diff line number Diff line change
Expand Up @@ -10,12 +10,16 @@ defaults:
run:
working-directory: tokenizers

permissions: {}

jobs:
# x86_64 runner → runs the SSE4.1/AVX classify SIMD paths NATIVELY, plus fmt/clippy + the drift guard.
native:
permissions:
contents: read
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
- uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0
- uses: dtolnay/rust-toolchain@stable
with: { components: rustfmt, clippy }
- run: cargo fmt -p bitsplit -p bitmap_gen -- --check
Expand All @@ -27,9 +31,11 @@ jobs:
# AVX-512 classify path — GitHub runners often lack AVX-512, so emulate it with Intel SDE and run the
# SAME byte-exact test binaries under it.
avx512-sde:
permissions:
contents: read
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
- uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0
- uses: dtolnay/rust-toolchain@stable
- name: Install Intel SDE
run: |
Expand All @@ -48,12 +54,14 @@ jobs:

# wasm32 SIMD128 classify + class-runs boundary extractor — run the test suite under wasmtime.
wasm:
permissions:
contents: read
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
- uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0
- uses: dtolnay/rust-toolchain@stable
with: { targets: wasm32-wasip1 }
- uses: bytecodealliance/actions/wasmtime/setup@v1
- uses: bytecodealliance/actions/wasmtime/setup@9152e710e9f7182e4c29ad218e4f335a7b203613 # v1.1.3
- name: Test under wasmtime (simd128)
env:
RUSTFLAGS: "-C target-feature=+simd128"
Expand Down
4 changes: 4 additions & 0 deletions .github/workflows/docs-check.yml
Original file line number Diff line number Diff line change
Expand Up @@ -12,8 +12,12 @@ on:
branches-ignore:
- feat/train_encode_split

permissions: {}

jobs:
build:
permissions:
contents: read
runs-on: ubuntu-latest
steps:
- name: Checkout repository
Expand Down
16 changes: 12 additions & 4 deletions .github/workflows/node.yml
Original file line number Diff line number Diff line change
Expand Up @@ -9,8 +9,12 @@ on:
paths-ignore:
- bindings/python/**

permissions: {}

jobs:
build_and_test:
permissions:
contents: read
name: Check everything builds & tests
runs-on: ubuntu-latest
defaults:
Expand All @@ -30,7 +34,7 @@ jobs:
shared-key: node

- name: Install Node
uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4
uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4.4.0
with:
node-version: latest

Expand All @@ -43,7 +47,7 @@ jobs:
# Test fixtures are immutable — cache them so CI doesn't re-download
# from the Hub (and get rate-limited) on every run.
- name: Cache HF test data
uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4
uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0
with:
path: bindings/node/data
key: hf-test-data-node-${{ hashFiles('bindings/node/Makefile') }}
Expand All @@ -52,14 +56,16 @@ jobs:
# Provides `uvx`, used to run the huggingface_hub CLI that downloads
# test fixtures (see the Makefile) without a separate install step.
- name: Install uv
uses: astral-sh/setup-uv@v6
uses: astral-sh/setup-uv@d0cc045d04ccac9d8b7881df0226f9e82c39688e # v6.8.0

- name: Run JS tests
env:
HF_TOKEN: ${{ secrets.HF_TOKEN }}
run: make test HF="uvx --from huggingface_hub hf"

quality:
permissions:
contents: read
name: Lint & format
runs-on: ubuntu-latest
defaults:
Expand Down Expand Up @@ -87,7 +93,7 @@ jobs:
run: cargo clippy --all-targets --all-features -- -D warnings

- name: Install Node
uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4
uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4.4.0
with:
node-version: latest

Expand All @@ -98,6 +104,8 @@ jobs:
run: yarn lint

audit:
permissions:
contents: read
name: Audit dependencies
runs-on: ubuntu-latest
defaults:
Expand Down
6 changes: 3 additions & 3 deletions .github/workflows/python.yml
Original file line number Diff line number Diff line change
Expand Up @@ -73,7 +73,7 @@ jobs:
# Also provides `uvx`, which runs the huggingface_hub CLI that fetches the test fixtures
# (see the Makefile) without a separate install step.
- name: Install uv
uses: astral-sh/setup-uv@v6
uses: astral-sh/setup-uv@d0cc045d04ccac9d8b7881df0226f9e82c39688e # v6.8.0
with:
python-version: ${{ matrix.python }}

Expand All @@ -83,7 +83,7 @@ jobs:

# Test fixtures are immutable, so cache them: the Hub rate-limits a fresh download per run.
- name: Cache HF test data
uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4
uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0
with:
path: bindings/python/data
key: hf-test-data-python-${{ hashFiles('bindings/python/Makefile') }}
Expand Down Expand Up @@ -125,7 +125,7 @@ jobs:
run: cargo clippy --all-targets --all-features -- -D warnings

- name: Install uv
uses: astral-sh/setup-uv@v6
uses: astral-sh/setup-uv@d0cc045d04ccac9d8b7881df0226f9e82c39688e # v6.8.0
with:
python-version: "3.14"

Expand Down
4 changes: 4 additions & 0 deletions .github/workflows/rust-release.yml
Original file line number Diff line number Diff line change
Expand Up @@ -8,8 +8,12 @@ on:
tags:
- v*

permissions: {}

jobs:
rust_publish:
permissions:
contents: read
runs-on: ubuntu-latest
steps:
- name: Checkout repository
Expand Down
24 changes: 20 additions & 4 deletions .github/workflows/rust.yml
Original file line number Diff line number Diff line change
Expand Up @@ -6,8 +6,12 @@ on:
- main
pull_request:

permissions: {}

jobs:
build_and_test:
permissions:
contents: read
name: Build & test
runs-on: ${{ matrix.os }}
env:
Expand Down Expand Up @@ -44,7 +48,7 @@ jobs:
# from the Hub (and get rate-limited) on every run.
- name: Cache HF test data
if: matrix.os != 'windows-latest'
uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4
uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0
with:
path: tokenizers/data
key: hf-test-data-${{ hashFiles('tokenizers/Makefile') }}
Expand All @@ -53,7 +57,7 @@ jobs:
# Provides `uvx`, used to run the huggingface_hub CLI that downloads
# test fixtures (see the Makefile) without a separate install step.
- name: Install uv
uses: astral-sh/setup-uv@v6
uses: astral-sh/setup-uv@d0cc045d04ccac9d8b7881df0226f9e82c39688e # v6.8.0

- name: Run tests
if: matrix.os != 'windows-latest'
Expand Down Expand Up @@ -99,6 +103,8 @@ jobs:

# Matrixed so platform-specific cfg gates and target-specific lints get coverage.
clippy:
permissions:
contents: read
name: Clippy
runs-on: ${{ matrix.os }}
strategy:
Expand Down Expand Up @@ -126,6 +132,8 @@ jobs:
run: cargo clippy --workspace --all-targets --all-features -- -D warnings

format:
permissions:
contents: read
name: Format
runs-on: ubuntu-latest
defaults:
Expand All @@ -143,6 +151,8 @@ jobs:
run: cargo fmt --all -- --check

readme:
permissions:
contents: read
name: READMEs up to date
runs-on: ubuntu-latest
defaults:
Expand Down Expand Up @@ -174,6 +184,8 @@ jobs:
done

audit:
permissions:
contents: read
name: Audit dependencies
runs-on: ubuntu-latest
defaults:
Expand All @@ -198,6 +210,8 @@ jobs:
run: cargo audit -D warnings --ignore RUSTSEC-2024-0436 --ignore RUSTSEC-2025-0014 --ignore RUSTSEC-2025-0119 --ignore RUSTSEC-2026-0204 --ignore RUSTSEC-2025-0057

semver:
permissions:
contents: read
name: Semver checks
runs-on: ubuntu-latest
defaults:
Expand Down Expand Up @@ -231,6 +245,8 @@ jobs:
# `use Model as _` import needed by two of the three model gates and therefore
# unused in the third. So every combination gets its own clippy, and every feature its own tests.
feature_matrix:
permissions:
contents: read
name: Feature matrix
runs-on: ubuntu-latest
defaults:
Expand All @@ -255,7 +271,7 @@ jobs:
# Test fixtures are immutable — cache them so CI doesn't re-download
# from the Hub (and get rate-limited) on every run.
- name: Cache HF test data
uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4
uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0
with:
path: tokenizers/data
key: hf-test-data-${{ hashFiles('tokenizers/Makefile') }}
Expand All @@ -264,7 +280,7 @@ jobs:
# Provides `uvx`, used to run the huggingface_hub CLI that downloads
# test fixtures (see the Makefile) without a separate install step.
- name: Install uv
uses: astral-sh/setup-uv@v6
uses: astral-sh/setup-uv@d0cc045d04ccac9d8b7881df0226f9e82c39688e # v6.8.0

# The same two fixtures the Windows leg fetches, and for the same reason:
# tk-encode's lib tests unwrap these outright. Every other fixture is
Expand Down
Loading