feat(dependencies): add transformer-engine as a git dependency and update metadata

- Included transformer-engine from GitHub to ensure the latest features and fixes are available.
- Updated dependency metadata in both pyproject.toml and uv.lock to reflect the new source and version.
- Enhanced validation in the Docker dependency script to ensure proper locking of the transformer-engine source revision.

Signed-off-by: Ajay Balasa <abalasa@nvidia.com>

Author: balasaajay

.github/scripts/test_docker_dependency_layers.sh

@@ -232,9 +232,26 @@ if "$temporary_dir/revision-validator" "$revision_repo" not-a-full-sha; then
   exit 1
 fi
 
-if grep -q 'transformer-engine @ git+https://github.com/NVIDIA/TransformerEngine.git@' pyproject.toml || \
-  grep -q '^name = "transformer-engine"$' pyproject.toml; then
-  echo "Bridge must inherit the TransformerEngine source and metadata from the selected MCore ref" >&2
+te_override=$(grep -Eo \
+  'transformer-engine @ git\+https://github.com/NVIDIA/TransformerEngine\.git@[0-9a-f]{40}' \
+  pyproject.toml || true)
+te_revision=${te_override##*@}
+if [[ ! "$te_revision" =~ ^[0-9a-f]{40}$ ]]; then
+  echo "Bridge must pin TransformerEngine to a full source revision" >&2
+  exit 1
+fi
+if ! grep -q '^name = "transformer-engine"$' pyproject.toml; then
+  echo "Bridge must provide lock metadata for the pinned TransformerEngine source" >&2
+  exit 1
+fi
+if ! grep -Fq \
+  "source = { git = \"https://github.com/NVIDIA/TransformerEngine.git?rev=${te_revision}#${te_revision}\" }" \
+  uv.lock; then
+  echo "TransformerEngine must be locked to the validated source revision" >&2
+  exit 1
+fi
+if grep -qE '^name = "transformer-engine-(cu13|torch)"$' uv.lock; then
+  echo "TransformerEngine must be built from source instead of split binary packages" >&2
   exit 1
 fi
 

pyproject.toml

@@ -130,6 +130,7 @@ override-dependencies = [
     "torch; sys_platform == 'never'",
     "torchvision; sys_platform == 'never'",
     "triton; sys_platform == 'never'",
+    "transformer-engine @ git+https://github.com/NVIDIA/TransformerEngine.git@4329ff84bfbdaa778a33cba02a15fb0807c64689",
     "mlflow>=3.15.1",                                       # To address CVE-2025-15031
     "cachetools>=5.0.0",
     "cryptography>=43.0.0,<47",
@@ -157,6 +158,25 @@ name = "flash-mla"
 version = "1.0.0+b7643bd"
 requires-dist = []
 
+# transformer-engine is a git dependency marked no-build-isolation, so uv would build it
+# from source just to read its metadata while locking. Keep the metadata synchronized with
+# the pinned TE commit so lock generation stays fast while container installation still
+# compiles TE against the target CUDA and cuDNN runtime.
+[[tool.uv.dependency-metadata]]
+name = "transformer-engine"
+version = "2.17.1+4329ff84"
+requires-dist = [
+    "einops",
+    "importlib-metadata>=1.0",
+    "nvdlfw-inspect",
+    "onnx",
+    "onnxscript",
+    "packaging",
+    "pydantic",
+    "torch>=2.1",
+]
+provides-extras = ["core-cu13", "pytorch"]
+
 [tool.uv.sources]
 megatron-core = { path = "3rdparty/Megatron-LM/", editable = true }
 nvidia-modelopt = { index = "nvidia" }
@@ -217,7 +237,6 @@ build = [
     "numpy",
     "ninja",
     "nvidia-mathdx",
-    "nvidia-cudnn-frontend>=1.25.0",
 ]
 # CVE-carrying WAN diffusion codecs, needed only by WAN video caching
 # (src/megatron/bridge/diffusion/models/wan/inference/utils.py). All three are suppressed