Skip to content

Commit e998be0

Browse files
balasaajaybalasaajay
authored
build: update Transformer Engine to release 2.20 tip (#7683)
Signed-off-by: Ajay Balasa <abalasa@nvidia.com>
1 parent c16f739 commit e998be0

30 files changed

Lines changed: 5072 additions & 4910 deletions

File tree

‎megatron/training/determinism.py‎

Lines changed: 11 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -39,6 +39,10 @@
3939
# Env-var defaults required for bit-exact reproducibility.
4040
DETERMINISM_ENV_VAR_DEFAULTS: dict[str, str] = {
4141
"NCCL_ALGO": "Ring",
42+
# NCCL EP's count-exchange path assigns expert slots with atomic counters.
43+
# Scan mode preserves token order, and therefore expert wgrad reduction order.
44+
# NCCL EP snapshots this setting when its group is created.
45+
"NCCL_EP_HT_EM_AG_SCAN_MODE": "1",
4246
"NVTE_ALLOW_NONDETERMINISTIC_ALGO": "0",
4347
"CUBLAS_WORKSPACE_CONFIG": ":4096:8",
4448
# TRITON_CACHE_AUTOTUNING is deliberately absent: unset is already deterministic, so
@@ -76,7 +80,10 @@
7680
# other truthy spelling ("true", "yes") would silently read as opted out.
7781
# Both settings are deterministic, so both are accepted and neither is
7882
# defaulted -- see :func:`apply_determinism_env` for the pairing rule.
83+
# - ``NCCL_EP_HT_EM_AG_SCAN_MODE``: require the canonical enabled value to
84+
# preserve expert token order instead of using atomic slot assignment.
7985
ACCEPTED_ENV_VAR_VALUES: dict[str, frozenset[str]] = {
86+
"NCCL_EP_HT_EM_AG_SCAN_MODE": frozenset({"1"}),
8087
"NVTE_ALLOW_NONDETERMINISTIC_ALGO": frozenset({"0"}),
8188
"CUBLAS_WORKSPACE_CONFIG": frozenset({":4096:8", ":16:8"}),
8289
"TRITON_CACHE_AUTOTUNING": frozenset({"0", "1"}),
@@ -90,7 +97,8 @@ def apply_determinism_env(env: MutableMapping[str, str]) -> None:
9097
9198
* ``NCCL_ALGO`` — if set, each comma-separated token must be in
9299
:data:`ACCEPTED_NCCL_ALGO_TOKENS`.
93-
* ``NVTE_ALLOW_NONDETERMINISTIC_ALGO`` / ``CUBLAS_WORKSPACE_CONFIG`` —
100+
* ``NCCL_EP_HT_EM_AG_SCAN_MODE`` / ``NVTE_ALLOW_NONDETERMINISTIC_ALGO`` /
101+
``CUBLAS_WORKSPACE_CONFIG`` —
94102
if set, must be in :data:`ACCEPTED_ENV_VAR_VALUES`.
95103
* ``MAMBA_DETERMINISTIC`` / ``CAUSAL_CONV1D_DETERMINISTIC`` — if set
96104
(non-empty), must start with ``'1'``; unset auto-follows
@@ -168,7 +176,8 @@ def apply_determinism_to_args(args) -> None:
168176
mutates ``args``.
169177
2. Calls :func:`apply_determinism_env` on ``os.environ`` — validates
170178
every determinism-relevant env var (``NCCL_ALGO``,
171-
``NVTE_ALLOW_NONDETERMINISTIC_ALGO``, ``CUBLAS_WORKSPACE_CONFIG``,
179+
``NCCL_EP_HT_EM_AG_SCAN_MODE``, ``NVTE_ALLOW_NONDETERMINISTIC_ALGO``,
180+
``CUBLAS_WORKSPACE_CONFIG``,
172181
``MAMBA_DETERMINISTIC``, ``CAUSAL_CONV1D_DETERMINISTIC``,
173182
``TRITON_CACHE_AUTOTUNING`` and its required ``TRITON_CACHE_DIR``) and
174183
setdefaults the canonical values.

‎pyproject.toml‎

Lines changed: 7 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -104,7 +104,7 @@ dev = [
104104
"av",
105105
"Pillow",
106106
"flashinfer-python>=0.5.0,<0.7.0",
107-
"nvidia-cudnn-frontend[cutedsl]==1.28.0",
107+
"nvidia-cudnn-frontend[cutedsl]==1.29.0",
108108
"wget",
109109
"onnxscript",
110110
"fastapi~=0.50", # Forcing a little bit more recent version of fastapi to be compatible with pydantic 2.0
@@ -178,7 +178,7 @@ build = [
178178
"pybind11",
179179
"Cython>=3.0.0",
180180
"torch",
181-
"nvidia-cudnn-frontend>=1.28.0", # for TE
181+
"nvidia-cudnn-frontend>=1.29.0", # for TE
182182
"nvidia-mathdx", # for TE
183183
]
184184
linting = [
@@ -195,6 +195,8 @@ no_pypi_wheels = ["flash_mla", "emerging_optimizers", "deep_gemm"]
195195
[tool.uv]
196196
managed = true
197197
default-groups = ["linting", "build", "test"]
198+
# CUTLASS 4.8 moved CuTe APIs used by older Quack releases.
199+
constraint-dependencies = ["quack-kernels>=0.6.5"]
198200
no-build-isolation-package = [
199201
"causal-conv1d",
200202
"flash_mla",
@@ -227,7 +229,7 @@ requires-dist = []
227229

228230
[[tool.uv.dependency-metadata]]
229231
name = "transformer-engine"
230-
version = "2.19.0+5e52befd"
232+
version = "2.20.0+131157b5"
231233
requires-dist = [
232234
# Cap below 2.14: pydantic 2.14 breaks langchain_core's module-level
233235
# RunnablePassthrough() instantiation, which is imported transitively in
@@ -240,7 +242,7 @@ requires-dist = [
240242
"onnxscript",
241243
"onnx",
242244
"nvdlfw-inspect",
243-
"nvidia-cudnn-frontend>=1.28.0",
245+
"nvidia-cudnn-frontend>=1.29.0",
244246
]
245247

246248
[[tool.uv.dependency-metadata]]
@@ -254,7 +256,7 @@ flash_mla = [
254256
{ git = "https://github.com/deepseek-ai/FlashMLA", rev = "nv_dev" },
255257
]
256258
deep_gemm = { git = "https://github.com/deepseek-ai/DeepGEMM.git", rev = "714dd1a4a980f7937a74343d19a8eba4fe321480" }
257-
transformer-engine = { git = "https://github.com/NVIDIA/TransformerEngine.git", rev = "5e52befd5262c06289106338c308079d6adb391f" }
259+
transformer-engine = { git = "https://github.com/NVIDIA/TransformerEngine.git", rev = "131157b5750b8578cb8efb78554f6fe6a671cc53" }
258260
nemo-run = { git = "https://github.com/NVIDIA-NeMo/Run.git", rev = "e3935393a290aed1822af52139b4b8ee270fed1f" }
259261
emerging_optimizers = { git = "https://github.com/NVIDIA-NeMo/Emerging-Optimizers.git", rev = "v0.3.0" }
260262
fast-hadamard-transform = { git = "https://github.com/Dao-AILab/fast-hadamard-transform.git", rev = "f134af63deb2df17e1171a9ec1ea4a7d8604d5ca" }

‎tests/functional_tests/test_cases/gpt/gpt3_mcore_te_mfsdp_v2_cp2/golden_values_dev_dgx_h100.json‎

Lines changed: 50 additions & 50 deletions
Original file line numberDiff line numberDiff line change
@@ -5,56 +5,56 @@
55
"step_interval": 1,
66
"value_precision": "full",
77
"values": {
8-
"1": 10.939653396606445,
9-
"2": 10.930423736572266,
10-
"3": 10.922745704650879,
11-
"4": 10.933757781982422,
12-
"5": 10.933038711547852,
13-
"6": 10.926679611206055,
14-
"7": 10.920910835266113,
15-
"8": 10.919511795043945,
16-
"9": 10.917047500610352,
17-
"10": 10.924277305603027,
18-
"11": 10.908919334411621,
19-
"12": 10.922995567321777,
20-
"13": 10.889339447021484,
21-
"14": 10.891376495361328,
22-
"15": 10.878791809082031,
23-
"16": 10.862626075744629,
24-
"17": 10.856717109680176,
25-
"18": 10.856390953063965,
26-
"19": 10.83975601196289,
27-
"20": 10.78012466430664,
28-
"21": 10.757363319396973,
29-
"22": 10.734782218933105,
30-
"23": 10.74186897277832,
31-
"24": 10.719328880310059,
32-
"25": 10.718711853027344,
33-
"26": 10.686211585998535,
34-
"27": 10.645018577575684,
35-
"28": 10.572410583496094,
36-
"29": 10.55211067199707,
37-
"30": 10.510461807250977,
38-
"31": 10.521717071533203,
39-
"32": 10.499227523803711,
40-
"33": 10.469740867614746,
41-
"34": 10.437095642089844,
42-
"35": 10.43850326538086,
43-
"36": 10.420669555664062,
44-
"37": 10.380794525146484,
45-
"38": 10.380276679992676,
46-
"39": 10.346519470214844,
47-
"40": 10.327482223510742,
48-
"41": 10.301814079284668,
49-
"42": 10.297276496887207,
50-
"43": 10.26720142364502,
51-
"44": 10.2523775100708,
52-
"45": 10.257597923278809,
53-
"46": 10.216726303100586,
54-
"47": 10.204315185546875,
55-
"48": 10.164474487304688,
56-
"49": 10.15493392944336,
57-
"50": 10.166776657104492
8+
"1": 10.939658164978027,
9+
"2": 10.930441856384277,
10+
"3": 10.92272663116455,
11+
"4": 10.933773040771484,
12+
"5": 10.933093070983887,
13+
"6": 10.926690101623535,
14+
"7": 10.92093276977539,
15+
"8": 10.91954231262207,
16+
"9": 10.917055130004883,
17+
"10": 10.924273490905762,
18+
"11": 10.908937454223633,
19+
"12": 10.923017501831055,
20+
"13": 10.889325141906738,
21+
"14": 10.891388893127441,
22+
"15": 10.878785133361816,
23+
"16": 10.862615585327148,
24+
"17": 10.85673713684082,
25+
"18": 10.856353759765625,
26+
"19": 10.839736938476562,
27+
"20": 10.780092239379883,
28+
"21": 10.757366180419922,
29+
"22": 10.734825134277344,
30+
"23": 10.74189281463623,
31+
"24": 10.719334602355957,
32+
"25": 10.71875286102295,
33+
"26": 10.686202049255371,
34+
"27": 10.645021438598633,
35+
"28": 10.572418212890625,
36+
"29": 10.552096366882324,
37+
"30": 10.510473251342773,
38+
"31": 10.521665573120117,
39+
"32": 10.499201774597168,
40+
"33": 10.469749450683594,
41+
"34": 10.437087059020996,
42+
"35": 10.438556671142578,
43+
"36": 10.420646667480469,
44+
"37": 10.380769729614258,
45+
"38": 10.380247116088867,
46+
"39": 10.346480369567871,
47+
"40": 10.32747745513916,
48+
"41": 10.301830291748047,
49+
"42": 10.297258377075195,
50+
"43": 10.267192840576172,
51+
"44": 10.252351760864258,
52+
"45": 10.257586479187012,
53+
"46": 10.216672897338867,
54+
"47": 10.204336166381836,
55+
"48": 10.16448974609375,
56+
"49": 10.154945373535156,
57+
"50": 10.166790008544922
5858
}
5959
},
6060
"num-zeros": {

0 commit comments

Comments
 (0)