Skip to content

Commit ac9857b

Browse files
authored
Catch a CUDA row built against the wrong minor version (pytorch#21991)
The wheel build refuses a row whose train disagrees with the installed toolkit, so a wheel cannot declare one CUDA runtime and be compiled by another. The check compared major versions only, and two of the published rows share a major: cu130 and cu132 both reduce to 13. A cu132 row built on a 13.0 toolkit passed. That combination is worth catching. The declared runtime dependency is per major, so it stays correct, but the device code and the version recorded in the wheel's local label both come from the row. The result installs, resolves its dependency, and claims a toolkit it was not compiled by. Read both numbers from the toolkit and compare the pair. The existing major-only comparison stays, because it reports a different and blunter problem, and the new one runs after it so its message can name both versions. The detection helper keeps returning whatever the toolkit reports rather than reading through the supported-version validator. That was already deliberate: going through the validator made the guard blind to any unlisted minor, which is how a mismatched wheel got built in the first place, and narrowing it now would reopen that hole. Verified against every combination the published rows allow: a cu132 row on a 13.0 toolkit is now refused and names both versions; cu126 on 12.6, cu130 on 13.0 and cu132 on 13.2 are silent; a cu130 row on a 12.6 toolkit still fails on the major; and a build with no toolkit installed is unaffected.
1 parent 5030bdd commit ac9857b

2 files changed

Lines changed: 44 additions & 9 deletions

File tree

‎install_utils.py‎

Lines changed: 15 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -13,7 +13,7 @@
1313
import shutil
1414
import subprocess
1515
import sys
16-
from typing import List, Optional
16+
from typing import List, Optional, Tuple
1717

1818
# Supported CUDA versions - modify this to add/remove supported versions
1919
# Format: tuple of (major, minor) version numbers
@@ -209,23 +209,31 @@ def _cmake_args_from_env() -> List[str]:
209209

210210

211211
@functools.lru_cache(maxsize=1)
212-
def _detected_cuda_major() -> Optional[int]:
213-
"""The CUDA major version of the installed toolkit, or None if none is installed.
212+
def _detected_cuda_version() -> Optional[Tuple[int, int]]:
213+
"""The CUDA version of the installed toolkit, or None if none is installed.
214214
215215
Kept separate from `_get_cuda_version` because the mismatch guard in the wheel build
216-
needs the major regardless of whether the exact (major, minor) is listed in
216+
needs whatever the toolkit reports regardless of whether that exact pair is listed in
217217
SUPPORTED_CUDA_VERSIONS. Reading through the validator caused the guard to see an
218218
empty detection for any unlisted minor (say 12.8), so a cu130 row built on a CUDA 12
219-
toolkit produced a wheel with no error.
219+
toolkit produced a wheel with no error. Both numbers are returned for the same
220+
reason: a row names a train down to the minor, so comparing only majors let a cu132
221+
row build against a 13.0 toolkit and declare a train it was not compiled for.
220222
"""
221223
try:
222224
result = subprocess.run(
223225
_selected_nvcc(), capture_output=True, text=True, check=True
224226
)
225227
except (subprocess.CalledProcessError, OSError):
226228
return None
227-
match = re.search(r"release (\d+)\.\d+", result.stdout)
228-
return int(match.group(1)) if match else None
229+
match = re.search(r"release (\d+)\.(\d+)", result.stdout)
230+
return (int(match.group(1)), int(match.group(2))) if match else None
231+
232+
233+
def _detected_cuda_major() -> Optional[int]:
234+
"""The CUDA major version of the installed toolkit, or None if none is installed."""
235+
version = _detected_cuda_version()
236+
return version[0] if version is not None else None
229237

230238

231239
def _extract_cmake_define(args: List[str], name: str) -> Optional[str]:

‎setup.py‎

Lines changed: 29 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -333,15 +333,22 @@ def _cuda_train() -> str:
333333
# row spelled with an unsupported minor (say cu125) was classified CPU by the shell and
334334
# CUDA 12 here, and the wheel then declared CUDA runtime packages for a CPU build.
335335
digits = re.sub(r"[^0-9]", "", raw)
336+
# The same rows as `trains` below, keeping both numbers rather than reducing to the
337+
# major, so the guard can tell two rows of one major apart.
338+
_CUDA_ROW_MINORS = {
339+
f"{major}{minor}": (major, minor)
340+
for major, minor in install_utils.SUPPORTED_CUDA_VERSIONS
341+
}
336342
trains = {
337343
f"{major}{minor}": str(major)
338344
for major, minor in install_utils.SUPPORTED_CUDA_VERSIONS
339345
}
340346
requested = trains.get(digits, "")
341347

342-
# Read the toolkit major directly, without the (major, minor) validator, so the guard
348+
# Read the toolkit version directly, without the (major, minor) validator, so the guard
343349
# below fires on any mismatch rather than only on the three listed pairs.
344-
detected_major = install_utils._detected_cuda_major()
350+
detected_version = install_utils._detected_cuda_version()
351+
detected_major = detected_version[0] if detected_version is not None else None
345352
detected = (
346353
str(detected_major)
347354
if detected_major is not None and str(detected_major) in _CUDA_RUNTIME_PACKAGES
@@ -363,6 +370,26 @@ def _cuda_train() -> str:
363370
"wheel would install and then fail to load. Install a matching toolkit "
364371
"or build the row that matches this one."
365372
)
373+
# The check above compares majors, which two rows of the same major share. A row
374+
# names its train down to the minor, so cu130 and cu132 both reduce to 13 and a
375+
# cu132 row built against a 13.0 toolkit passed. The device code and the version
376+
# in the wheel's local label both come from the row, so that wheel claims a
377+
# toolkit it was not compiled by. Compared here rather than folded above so the
378+
# message can name both numbers.
379+
if detected_version is not None and digits in _CUDA_ROW_MINORS:
380+
requested_pair = _CUDA_ROW_MINORS[digits]
381+
if detected_version != requested_pair:
382+
requested_text = f"{requested_pair[0]}.{requested_pair[1]}"
383+
detected_text = f"{detected_version[0]}.{detected_version[1]}"
384+
raise RuntimeError(
385+
f"this build targets CUDA {requested_text} (from "
386+
f"{'CU_VERSION' if os.environ.get('CU_VERSION') else 'DESIRED_CUDA'}="
387+
f"{raw!r}) but the installed toolkit is CUDA {detected_text}. Those "
388+
"share a major version, so the runtime dependency this wheel declares "
389+
"is correct while the device code and the version recorded in its "
390+
"local label are not. Install a matching toolkit or build the row "
391+
"that matches this one."
392+
)
366393
return requested
367394

368395
if raw and not requested:

0 commit comments

Comments
 (0)