Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
35 commits
Select commit Hold shift + click to select a range
41329b9
fix: patch cuda hardcodes for plugin compatibility (complete debug di…
lxd-cumt Sep 14, 2026
c8feb1a
fix: remove explicit import to avoid black parsing error
lxd-cumt Sep 14, 2026
e8bf31c
fix: correct malformed imports that prevent module loading
lxd-cumt Sep 14, 2026
100803e
fix: add missing te_device_type import to files using it
lxd-cumt Sep 14, 2026
9e0b0e2
fix: correct malformed imports in additional files
lxd-cumt Sep 14, 2026
d7b7fe9
fix: use routing_map device instead of hardcoded device type
lxd-cumt Sep 14, 2026
6180739
revert: remove te_platform() replacements, keep te_device_type()
lxd-cumt Sep 14, 2026
a0f9487
fix: remove remaining te_platform() calls in utils, fused_adam, and d…
lxd-cumt Sep 14, 2026
77f644d
Merge remote-tracking branch 'origin/main' into cuda_patch_main_latest
lxd-cumt Sep 22, 2026
1bad7ee
keep sync with torch.cuda api usage
lxd-cumt Sep 22, 2026
a323d3c
merge latest main and fix conflicts
lxd-cumt Sep 23, 2026
b946091
merge latest main and fix conflicts
lxd-cumt Oct 8, 2026
2d63707
format
lxd-cumt Oct 8, 2026
8fbfcfe
merge latest main and fix conflicts
lxd-cumt Oct 9, 2026
90d6d32
Fix missing te_device_type imports and import ordering
lxd-cumt Oct 19, 2026
6166d2b
Merge latest main into cuda_patch_main_latest
lxd-cumt Oct 19, 2026
011d005
Query autocast state and dtype on the same device
lxd-cumt Oct 19, 2026
a2baa20
Gate per-document THD partitioning on CUDA
lxd-cumt Oct 19, 2026
171148f
Drop incidental changes from the device-type patch
lxd-cumt Oct 19, 2026
748ed35
Cover device-type checks missed in recently merged code
lxd-cumt Oct 19, 2026
3f1e95c
Merge latest main into cuda_patch_main_latest
lxd-cumt Oct 21, 2026
5f8b277
Merge branch 'main' into cuda_patch_main_latest
lxd-cumt Oct 21, 2026
0c7cf39
Apply plugin device patches after the _nv alias is registered
lxd-cumt Oct 21, 2026
a3f9403
Merge latest main into cuda_patch_main_latest
lxd-cumt Oct 21, 2026
87ff4f2
Accept the configured device type in the shared forward input check
lxd-cumt Oct 21, 2026
a99fd28
Merge remote-tracking branch 'lxd_origin/cuda_patch_main_latest'
lxd-cumt Oct 21, 2026
15d3502
Restore the original spacing in the top-level package
lxd-cumt Oct 21, 2026
d92f20c
Skip patch discovery for single-file plugins
lxd-cumt Oct 21, 2026
32fa15a
Merge remote-tracking branch 'origin/main' into cuda_patch_main_latest
lxd-cumt Oct 26, 2026
83f8e84
Use the configured device type in GroupedTensor record_stream
lxd-cumt Oct 26, 2026
a627286
Accept the configured device type in the attention input checks
lxd-cumt Oct 26, 2026
ad5b363
Keep the paged KV cache and EP buffer on CUDA
lxd-cumt Oct 26, 2026
4cc584f
Merge remote-tracking branch 'origin/main' into cuda_patch_main_latest
lxd-cumt Oct 27, 2026
be5a062
Apply plugin device patches after the plugin finishes loading
lxd-cumt Oct 27, 2026
a417941
Fix regex in test_cross_entropy
ptrendx Oct 27, 2026
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion tests/pytorch/test_cross_entropy.py
Original file line number Diff line number Diff line change
Expand Up @@ -417,7 +417,7 @@ def test_parallel_cross_entropy_validation_errors():
target.transpose(0, 1),
overwrite_input=True,
)
with pytest.raises(ValueError, match="CUDA"):
with pytest.raises(ValueError, match="cuda"):
parallel_cross_entropy(logits.cpu(), target.cpu())


Expand Down
10 changes: 10 additions & 0 deletions transformer_engine/common/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -232,6 +232,16 @@ def load_framework_extension(framework: str) -> None:
sys.modules[module_name + "_nv"] = solib
_plugin = importlib.import_module(_nvte_plugin)
_plugin.load_plugins()
# Apply device patches last: they rewrite process-global state in
# place and cannot be rolled back, so load_plugins() runs first and
# must not depend on the patched state. Still ahead of the PyTorch
# submodules that capture torch.cuda entry points. The submodule is
# optional: a CUDA-like plugin may omit it, and a single-file plugin
# has no submodules to search.
if hasattr(_plugin, "__path__") and (
importlib.util.find_spec(f"{_nvte_plugin}.patches") is not None
):
importlib.import_module(f"{_nvte_plugin}.patches").apply_patches()
except Exception as e: # pylint: disable=broad-exception-caught
# Rollback to pre-plugin state if plugin failed to fully initialize
sys.modules.pop(module_name + "_nv", None)
Expand Down
5 changes: 4 additions & 1 deletion transformer_engine/debug/features/fake_quant.py
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@
from nvdlfw_inspect.utils import append_parent_docstring


from transformer_engine import te_device_type
from transformer_engine.debug.features.api import TEConfigAPIMapper
from transformer_engine.common.recipe import Format
from transformer_engine.pytorch import DType
Expand All @@ -30,7 +31,9 @@ def fake_quantize(tensor: torch.Tensor, fp8_format: str, out=None):
torch.float16,
torch.bfloat16,
), "[NVTORCH INSPECT ERROR] Unsupported tensor type."
assert tensor.is_cuda, "[NVTORCH INSPECT ERROR] Must be a GPU tensor."
assert (
tensor.device.type == te_device_type()
), f"[NVTORCH INSPECT ERROR] Must be a {te_device_type()} tensor."
assert fp8_format in {
"FP8E4M3",
"FP8E5M2",
Expand Down
7 changes: 6 additions & 1 deletion transformer_engine/debug/features/log_fp8_tensor_stats.py
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,8 @@
from transformer_engine.debug.features.utils.stats_buffer import STATS_BUFFERS
from transformer_engine.debug.features.utils import get_reduction_params, next_enabled_iter
from transformer_engine.pytorch.tensor import Quantizer, QuantizedTensor

from transformer_engine import te_device_type
from transformer_engine.pytorch.tensor.float8_tensor import (
Float8Quantizer,
Float8CurrentScalingQuantizer,
Expand Down Expand Up @@ -57,7 +59,10 @@ def _get_new_quantizer(recipe_name, fp8_dtype):
return Float8BlockQuantizer(fp8_dtype=fp8_dtype, rowwise=True, columnwise=True)
if recipe_name == "fp8_current_scaling":
return Float8CurrentScalingQuantizer(
fp8_dtype=fp8_dtype, device=torch.device("cuda"), rowwise=True, columnwise=True
fp8_dtype=fp8_dtype,
device=torch.device(te_device_type()),
rowwise=True,
columnwise=True,
)
if recipe_name == "mxfp8":
return MXFP8Quantizer(fp8_dtype=fp8_dtype, rowwise=True, columnwise=True)
Expand Down
5 changes: 4 additions & 1 deletion transformer_engine/debug/features/per_tensor_scaling.py
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@
import nvdlfw_inspect.api as debug_api
from nvdlfw_inspect.registry import Registry, api_method

from transformer_engine import te_device_type
from transformer_engine.pytorch import DType
from transformer_engine.pytorch.tensor import Quantizer
from transformer_engine.pytorch.tensor.float8_tensor import (
Expand All @@ -33,7 +34,9 @@ def per_tensor_cast(
torch.float16,
torch.bfloat16,
), "[NVTORCH INSPECT ERROR] Unsupported tensor type for per tensor current scaling"
assert tensor.is_cuda, "[NVTORCH INSPECT ERROR] Must be a GPU tensor."
assert (
tensor.device.type == te_device_type()
), f"[NVTORCH INSPECT ERROR] Must be a {te_device_type()} tensor."
assert fp8_dtype in {
DType.kFloat8E4M3,
DType.kFloat8E5M2,
Expand Down
5 changes: 3 additions & 2 deletions transformer_engine/debug/features/utils/stats_buffer.py
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@
from nvdlfw_inspect.utils import gather_along_first_dim
from nvdlfw_inspect.logging import MetricLogger

from transformer_engine import te_device_type
from transformer_engine.debug.features.utils.stats_computation import (
STATS,
DEPENDENCIES,
Expand All @@ -41,14 +42,14 @@ def __init__(self, layer_name, tensor_name, stats, reduction_group, reduce_withi
for stat in stats:
self.stats_to_compute = self.stats_to_compute | DEPENDENCIES[stat]

self._buffer = torch.zeros(len(STATS), dtype=torch.float32).cuda()
self._buffer = torch.zeros(len(STATS), dtype=torch.float32).to(te_device_type())
self._new_buffer = self._buffer.clone()
self._tmp_buffer = self._buffer.clone()

# in case of data parallelism it is possible that layer will not be run on one node
# modified is set to True if node is run
# we do not take not run nodes into account
self.modified = torch.tensor([False], dtype=torch.bool).cuda()
self.modified = torch.tensor([False], dtype=torch.bool).to(te_device_type())
self.iteration = None
self.skip_reduction = False

Expand Down
21 changes: 21 additions & 0 deletions transformer_engine/pytorch/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -9,13 +9,34 @@
import functools

import torch
import transformer_engine

from transformer_engine.common import load_framework_extension
from transformer_engine.pytorch.torch_version import torch_version

assert torch_version() >= (2, 1), f"Minimum torch version 2.1 required. Found {torch_version()}."

load_framework_extension("torch")

# Device type and platform globals — live here because torch cannot be imported
# at the top-level transformer_engine package (shared with JAX).
# Use plugin-set values if available, otherwise fall back to CUDA defaults.
transformer_engine.TE_DEVICE_TYPE = getattr(transformer_engine, "TE_DEVICE_TYPE", "cuda")
transformer_engine.TE_PLATFORM = getattr(transformer_engine, "TE_PLATFORM", torch.cuda)


def te_device_type() -> str:
"""Torch device type used by Transformer Engine, e.g. `cuda`."""
return transformer_engine.TE_DEVICE_TYPE


def te_platform():
"""Torch device module used by Transformer Engine, e.g. `torch.cuda`."""
return transformer_engine.TE_PLATFORM


transformer_engine.te_device_type = te_device_type
transformer_engine.te_platform = te_platform
from transformer_engine.pytorch import constants
from transformer_engine.pytorch.constants import CPLoadBalancingStrategy, DType
from transformer_engine.pytorch.module import LayerNormLinear
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,8 @@

import torch
import torch.nn.functional as F

from transformer_engine import te_device_type
from transformer_engine.pytorch.utils import (
get_device_compute_capability,
split_tensor_along_dim,
Expand Down Expand Up @@ -663,10 +665,10 @@ def _forward(
# S/dP are forced to use DS quantizers in DPA.init_fp8_metadata; revert them here for true CS emulation
if fp8_recipe.float8_current_scaling():
S_quantizer = Float8CurrentScalingQuantizer(
fp8_dtype=S_quantizer.dtype, device="cuda"
fp8_dtype=S_quantizer.dtype, device=te_device_type()
)
dP_quantizer = Float8CurrentScalingQuantizer(
fp8_dtype=dP_quantizer.dtype, device="cuda"
fp8_dtype=dP_quantizer.dtype, device=te_device_type()
)
# disable swizzle for MXFP8Quantizer
for quantizer in [
Expand Down Expand Up @@ -1038,9 +1040,9 @@ def forward(
x.dtype in [torch.float16, torch.bfloat16] or isinstance(x, Float8Tensor)
for x in [query_layer, key_layer, value_layer]
), "FlashAttention only supports FP16 and BF16 data types, or Float8Tensors."
assert (
query_layer.is_cuda and key_layer.is_cuda and value_layer.is_cuda
), "FlashAttention currently only supports CUDA tensors."
assert all(
x.device.type == te_device_type() for x in (query_layer, key_layer, value_layer)
), f"FlashAttention currently only supports {te_device_type()} tensors."
assert (
qkv_layout in QKVLayouts
), f"FlashAttention does not support qkv_layout = {qkv_layout}!"
Expand Down Expand Up @@ -2718,9 +2720,9 @@ def forward(
x.dtype in [torch.float16, torch.bfloat16] or isinstance(x, QuantizedTensorStorage)
for x in [query_layer, key_layer, value_layer]
), "FusedAttention only supports FP16 and BF16 data types, or QuantizedTensors."
assert (
query_layer.is_cuda and key_layer.is_cuda and value_layer.is_cuda
), "FusedAttention only supports CUDA tensors."
assert all(
x.device.type == te_device_type() for x in (query_layer, key_layer, value_layer)
), f"FusedAttention only supports {te_device_type()} tensors."
assert (
qkv_layout in QKVLayouts
), f"FusedAttention does not support qkv_layout = {qkv_layout}!"
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@
import torch
import transformer_engine_torch as tex

from transformer_engine import te_device_type
from transformer_engine.pytorch.utils import (
get_cudnn_version,
nvtx_range_pop,
Expand Down Expand Up @@ -299,7 +300,9 @@ def get_thd_partitioned_indices(
cp_size,
)
target_device = torch.device(device if device is not None else cu_seqlens_padded.device)
target_dtype = torch.int32 if target_device.type == "cuda" else cu_seqlens_padded.dtype
target_dtype = (
torch.int32 if target_device.type == te_device_type() else cu_seqlens_padded.dtype
)
chunk_size = total_tokens // cp_size
return torch.arange(
cp_rank * chunk_size,
Expand All @@ -308,9 +311,9 @@ def get_thd_partitioned_indices(
device=target_device,
)
cu_seqlens_padded = _get_thd_partition_cu_seqlens(cu_seqlens_padded, device)
assert cu_seqlens_padded.is_cuda, (
"Per-document THD partitioning requires CUDA cu_seqlens; pass device='cuda' "
"when the source metadata is on CPU."
assert cu_seqlens_padded.device.type == te_device_type(), (
f"Per-document THD partitioning requires {te_device_type()} cu_seqlens; pass"
f" device='{te_device_type()}' when the source metadata is on CPU."
)
if cu_seqlens_padded.dtype != torch.int32:
cu_seqlens_padded = cu_seqlens_padded.to(torch.int32)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@
from torch.fx.experimental.symbolic_shapes import guard_scalar
from torch.nn.parameter import Parameter

from transformer_engine import te_device_type
from transformer_engine.common.recipe import (
Format,
Recipe,
Expand Down Expand Up @@ -962,12 +963,14 @@ def __init__(
self.softmax_offset = None
if self.softmax_type == "off-by-one":
self.softmax_offset = torch.zeros(
self.num_attention_heads // self.tp_size, device="cuda"
self.num_attention_heads // self.tp_size, device=te_device_type()
)
Comment thread
greptile-apps[bot] marked this conversation as resolved.
Resolved
if self.softmax_type == "learnable":
self.register_parameter(
"softmax_offset",
Parameter(torch.zeros(self.num_attention_heads // self.tp_size, device="cuda")),
Parameter(
torch.zeros(self.num_attention_heads // self.tp_size, device=te_device_type())
),
get_rng_state_tracker=get_rng_state_tracker,
)

Expand Down Expand Up @@ -2353,9 +2356,9 @@ def forward(
fp8_output = False

# checks for q/k/v shapes
assert (
query_layer.is_cuda and key_layer.is_cuda and value_layer.is_cuda
), "DotProductAttention only supports CUDA tensors."
assert all(
x.device.type == te_device_type() for x in (query_layer, key_layer, value_layer)
), f"DotProductAttention only supports {te_device_type()} tensors."
assert (
query_layer.dtype == key_layer.dtype and query_layer.dtype == value_layer.dtype
), "Queries, keys and values must have the same data type!"
Expand Down Expand Up @@ -2516,7 +2519,7 @@ def forward(
"Mixed THD policy sequence_ids must be one-dimensional int32 or "
"int64 tensors."
)
if not sequence_ids.is_cuda or sequence_ids.device != query_layer.device:
if sequence_ids.device != query_layer.device:
raise ValueError(
"Mixed THD policy sequence_ids must be on the Q/K/V device."
)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@
from typing import Any, Callable, Dict, Optional, Tuple

import torch
from transformer_engine import te_device_type

_cudnn_score_mod_handles: Dict[torch.device, Any] = {}
_cudnn_score_mod_graph_cache: Dict[Tuple[Any, ...], Any] = {}
Expand Down Expand Up @@ -140,7 +141,7 @@ def _score_mod_callback_cache_key(callback: Optional[Callable]) -> Any:

def _score_mod_device_key(device: torch.device) -> Tuple[Any, ...]:
"""Normalize a tensor device for graph cache keys."""
if device.type == "cuda":
if device.type == te_device_type():
index = device.index
if index is None:
index = torch.cuda.current_device()
Expand Down Expand Up @@ -194,10 +195,10 @@ def _wrapped_score_mod(sdpa_graph, score_tensor):

def _get_cudnn_current_stream_handle(cudnn, device: torch.device):
"""Return a cuDNN handle for device, bound to PyTorch's current stream."""
if device.type != "cuda":
if device.type != te_device_type():
raise ValueError(f"Flex Attention only supports CUDA tensors, got device {device}.")
if device.index is None:
device = torch.device("cuda", torch.cuda.current_device())
device = torch.device(te_device_type(), torch.cuda.current_device())

handle = _cudnn_score_mod_handles.get(device)
with torch.cuda.device(device):
Expand Down Expand Up @@ -287,8 +288,8 @@ def _execute_cudnn_graph(
"""Execute a built cuDNN frontend Python graph."""
cudnn = _import_cudnn_frontend()

if device.type == "cuda" and device.index is None:
device = torch.device("cuda", torch.cuda.current_device())
if device.type == te_device_type() and device.index is None:
device = torch.device(te_device_type(), torch.cuda.current_device())
workspace = torch.empty(
workspace_size,
device=device,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@
from torch import nn
import transformer_engine_torch as tex
from transformer_engine.pytorch.export import is_in_onnx_export_mode
from transformer_engine import te_device_type

THREADS_PER_WARP = 32
THREADS_PER_BLOCK = 128
Expand Down Expand Up @@ -236,7 +237,7 @@ def _get_default_causal_mask(mask_type: str, sq: int, sk: int) -> torch.Tensor:
def _get_mask():
diagonal_offset = sk - sq + 1 if "bottom_right" in mask_type else 1
return torch.triu(
torch.ones(sq, sk, dtype=torch.bool, device="cuda"), diagonal=diagonal_offset
torch.ones(sq, sk, dtype=torch.bool, device=te_device_type()), diagonal=diagonal_offset
)

if is_in_onnx_export_mode():
Expand Down
Loading
Loading