From fb114b62d9bc80280df0ceffbdcfa9be976f535b Mon Sep 17 00:00:00 2001 From: Jacob Szwejbka Date: Fri, 16 Oct 2026 13:02:25 -0700 Subject: [PATCH 1/2] Build Python 3.15 Windows tokenizers before wheel smoke installs No cp315 Windows pytorch-tokenizers wheel is published, so pip falls back to the old 1.0.1 source distribution in the plain smoke shell and cannot identify ClangCL. Build the pinned source first through the Visual Studio environment helper using the already installed build dependencies. Authored with OpenAI Codex. --- .ci/scripts/wheel/pre_build_script.sh | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/.ci/scripts/wheel/pre_build_script.sh b/.ci/scripts/wheel/pre_build_script.sh index 3b83194241e..b38b787f7c6 100755 --- a/.ci/scripts/wheel/pre_build_script.sh +++ b/.ci/scripts/wheel/pre_build_script.sh @@ -164,6 +164,16 @@ fi "${GITHUB_WORKSPACE}/${REPOSITORY}/install_requirements.sh" --example +if [[ $UNAME_S == *"MINGW"* || $UNAME_S == *"MSYS"* ]]; then + PYTHON_BIN=$(command -v python || command -v python3) + if "${PYTHON_BIN}" -c 'import sys; sys.exit(sys.version_info < (3, 15))'; then + # No cp315 tokenizer wheel is published. Build the pinned source with the + # existing build dependencies and Visual Studio environment before smoke install. + cmd //c '.ci\scripts\wheel\vc_env_helper.bat' python -m pip install \ + --no-build-isolation extension/llm/tokenizers + fi +fi + # Enable VGF in pybind wheel builds when the platform-specific build input is # available from pip. if [[ "$UNAME_S" == "Linux" || "$UNAME_S" == "Darwin" ]]; then From 55cdbdc402703a10f9256e2666bd8e1c6aa78528 Mon Sep 17 00:00:00 2001 From: Jacob Szwejbka Date: Fri, 16 Oct 2026 13:17:35 -0700 Subject: [PATCH 2/2] Keep Windows tokenizer compiler probes below the path limit Build the pinned tokenizer source from a temporary short path. MSBuild file tracking in CMake compiler probes fails on the nested source build path even inside the Visual Studio environment. Authored with OpenAI Codex. --- .ci/scripts/wheel/pre_build_script.sh | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.ci/scripts/wheel/pre_build_script.sh b/.ci/scripts/wheel/pre_build_script.sh index b38b787f7c6..92dd431adc3 100755 --- a/.ci/scripts/wheel/pre_build_script.sh +++ b/.ci/scripts/wheel/pre_build_script.sh @@ -169,8 +169,12 @@ if [[ $UNAME_S == *"MINGW"* || $UNAME_S == *"MSYS"* ]]; then if "${PYTHON_BIN}" -c 'import sys; sys.exit(sys.version_info < (3, 15))'; then # No cp315 tokenizer wheel is published. Build the pinned source with the # existing build dependencies and Visual Studio environment before smoke install. + # MSBuild's file tracker needs a short path even during CMake compiler checks. + tokenizer_source_dir=$(mktemp -d /c/et-tok.XXXXXX) + cp -R extension/llm/tokenizers/. "${tokenizer_source_dir}" cmd //c '.ci\scripts\wheel\vc_env_helper.bat' python -m pip install \ - --no-build-isolation extension/llm/tokenizers + --no-build-isolation "$(cygpath -w "${tokenizer_source_dir}")" + rm -rf "${tokenizer_source_dir}" fi fi