diff --git a/CHANGELOG.rst b/CHANGELOG.rst
index e32358de351..4a6116df670 100755
--- a/CHANGELOG.rst
+++ b/CHANGELOG.rst
@@ -40,6 +40,7 @@ Changelog
- Add an end-to-end W4A4 NVFP4 PTQ and QAD tutorial for Qwen3.6-35B-A3B also covering evaluation and vLLM throughput benchmarking. See `examples/megatron_bridge/tutorials/Qwen3.6-35B-A3B/README.md `_ for details.
- Add ``--mlflow `` to the ``examples/megatron_bridge`` scripts that write a checkpoint -- ``prune_minitron.py``, ``quantize.py``, ``distill.py``, ``export_quantized_megatron_to_hf.py`` and ``export_distilled_megatron_to_hf.py`` (MLflow's own ``MLFLOW_TRACKING_URI`` is honoured too). Each run records the invocation, its arguments as searchable params and its log -- a distillation records its training metrics instead of the rank-0 log -- and writes ``.experiment.json`` into the checkpoint it produced, so a pruning, a quantization, the distillation that refines its checkpoint and the export that deploys it can be traced to one another; uploading the checkpoints themselves stays off unless ``--mlflow_log_checkpoints`` is passed.
- Add ``--ep_size`` to ``examples/megatron_bridge/export_quantized_megatron_to_hf.py`` so large MoE models with grouped-GEMM experts can be exported with their experts sharded across GPUs. Checkpoints built with ``--no_moe_grouped_gemm`` must still be exported at ``--ep_size 1``.
+- Add unified HF export of GLM-5 / GLM-5.2 (``glm_moe_dsa``) checkpoints quantized with ``examples/megatron_bridge``. The MTP layer, which Megatron-Bridge does not build for these models, is copied through unquantized from the source checkpoint. DSA sparse-attention models also gain FP8 KV-cache quantization on Megatron.
*Misc*
diff --git a/modelopt/torch/distill/plugins/megatron.py b/modelopt/torch/distill/plugins/megatron.py
index 1bb9a3db791..5f6a1de4fb7 100644
--- a/modelopt/torch/distill/plugins/megatron.py
+++ b/modelopt/torch/distill/plugins/megatron.py
@@ -40,6 +40,7 @@
import modelopt.torch.distill as mtd
from modelopt.torch.distill.config import Criterion
from modelopt.torch.quantization.nn import TensorQuantizer
+from modelopt.torch.utils import warn_rank_0
if TYPE_CHECKING:
from megatron.core.dist_checkpointing.mapping import ShardedStateDict
@@ -592,9 +593,11 @@ def _sharded_state_dict(self, *args, **kwargs) -> "ShardedStateDict":
skip_mtp_loss = _mtp_excluded_from_quantization(model)
if distill_cfg.skip_lm_loss and skip_mtp_loss:
# Freeze the untrained MTP head: DDP's overlapped grad reduce asserts on params with no grad.
- for name, param in model.named_parameters():
- if "mtp" in name.split("."):
- param.requires_grad_(False)
+ warn_rank_0("MTP head is outside quantization and its loss is skipped: freezing it.")
+ with model.hide_teacher_model():
+ for name, param in model.named_parameters():
+ if "mtp" in name.split("."):
+ param.requires_grad_(False)
def _compute_student_lm_loss(self, labels, logits) -> Tensor:
self._lm_loss_call_count += 1
diff --git a/modelopt/torch/export/plugins/mcore_common.py b/modelopt/torch/export/plugins/mcore_common.py
index d275aee4c0a..b74a4fc6aaa 100644
--- a/modelopt/torch/export/plugins/mcore_common.py
+++ b/modelopt/torch/export/plugins/mcore_common.py
@@ -18,6 +18,7 @@
from typing import Any
from .mcore_deepseek import deepseek_causal_lm_export, deepseek_causal_lm_import
+from .mcore_glm import glm_moe_dsa_causal_lm_export
from .mcore_gptoss import gptoss_causal_lm_export, gptoss_causal_lm_import
from .mcore_llama import (
eagle3_deep_llama_causal_lm_export,
@@ -63,6 +64,7 @@
"Qwen3VLForConditionalGeneration": qwen3vl_causal_lm_export,
"Qwen3_5ForConditionalGeneration": qwen3_5_vl_causal_lm_export,
"Qwen3_5MoeForConditionalGeneration": qwen3_5_vl_causal_lm_export,
+ "GlmMoeDsaForCausalLM": glm_moe_dsa_causal_lm_export,
}
# VLM architectures whose Megatron export covers the language model only: the vision tower is copied
diff --git a/modelopt/torch/export/plugins/mcore_glm.py b/modelopt/torch/export/plugins/mcore_glm.py
new file mode 100644
index 00000000000..25a967860ac
--- /dev/null
+++ b/modelopt/torch/export/plugins/mcore_glm.py
@@ -0,0 +1,41 @@
+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Custom mappings from Megatron Core models to GLM-5.x Hugging Face models.
+
+GLM-5 / GLM-5.2 (``glm_moe_dsa``) is DeepSeek-V3-style MLA with a DSA indexer. It keeps the MTP
+layer at HF index ``num_hidden_layers`` under the decoder's names; Megatron-Bridge does not build
+it, so the exporter copies it from the source checkpoint.
+"""
+
+from .mcore_custom import (
+ GroupedGatedMLPSlicing,
+ GroupedMLPSlicing,
+ NameRemapping,
+ SelfAttentionScaling,
+)
+from .mcore_deepseek import deepseek_causal_lm_export
+
+glm_moe_dsa_causal_lm_export: dict = {
+ **deepseek_causal_lm_export,
+ "mtp_in_decoder_layers": True,
+ "core_attention": SelfAttentionScaling("model.layers.{}.self_attn."),
+ "indexer.linear_wq_b": NameRemapping("model.layers.{}.self_attn.indexer.wq_b."),
+ "indexer.linear_wk": NameRemapping("model.layers.{}.self_attn.indexer.wk."),
+ "indexer.k_norm": NameRemapping("model.layers.{}.self_attn.indexer.k_norm."),
+ "indexer.linear_weights_proj": NameRemapping("model.layers.{}.self_attn.indexer.weights_proj."),
+ "experts.linear_fc1": GroupedGatedMLPSlicing("model.layers.{}.mlp.experts.{{}}"),
+ "experts.linear_fc2": GroupedMLPSlicing("model.layers.{}.mlp.experts.{{}}.down_proj"),
+}
diff --git a/modelopt/torch/export/unified_export_megatron.py b/modelopt/torch/export/unified_export_megatron.py
index 8cf2e7b0001..f4b52c56bb2 100644
--- a/modelopt/torch/export/unified_export_megatron.py
+++ b/modelopt/torch/export/unified_export_megatron.py
@@ -29,7 +29,7 @@
import torch
import torch.distributed
-from huggingface_hub import get_safetensors_metadata, hf_hub_download
+from huggingface_hub import get_safetensors_metadata, hf_hub_download, snapshot_download
from huggingface_hub.errors import EntryNotFoundError
from safetensors import safe_open
from safetensors.torch import save_file
@@ -175,6 +175,7 @@ def __init__(
self._hf_text_config = getattr(self._hf_config, "text_config", self._hf_config)
# Update hf_config
+ self._src_num_hidden_layers = self._hf_text_config.num_hidden_layers
self._hf_text_config.num_hidden_layers = language_model.config.num_layers
self._hf_text_config.hidden_size = language_model.config.hidden_size
# MLA's kv_channels is the V head dim, not HF's head_dim (e.g. glm5_next derives it from RoPE).
@@ -203,6 +204,12 @@ def __init__(
del self._hf_config.quantization_config
self.all_rules = self._populate_rule_book()
self.rules = self.all_rules[self.arch]
+ self.all_mcore_mappings = all_mcore_hf_export_mapping[self.arch]
+ if self.rules.get("mtp_in_decoder_layers", False) and language_model.config.mtp_num_layers:
+ # Only the copy from the source checkpoint is supported (the bridge builds no MTP).
+ raise NotImplementedError(
+ f"Exporting a Megatron-built MTP for {self.arch} is not supported yet."
+ )
# The vision tower is copied through unquantized, so deployments must not treat it as such.
self.exclude_modules = [
prefix.removesuffix(".") + "*" for prefix in self.vision_passthrough_prefixes or ()
@@ -351,9 +358,11 @@ def save_pretrained(
"Export at expert parallel size > 1 needs grouped-GEMM experts; "
"export SequentialMLP (--no_moe_grouped_gemm) checkpoints at EP=1."
)
- # One writer per pipeline stage: other TP / DP / EP ranks hold the same layers (EP>1 ranks
- # hold no gathered experts at all), and writing them too would race on the same files.
- writes_layers = (
+ # One layer-shard writer per pipeline stage: other TP / DP / EP ranks hold the same layers
+ # (EP>1 ranks hold no gathered experts at all), and writing them too would race on the same
+ # files. Unlike is_writer_rank (one global writer of save_directory metadata), this has no
+ # PP term on purpose: each stage owns a disjoint slice of the layer shards.
+ is_stage_layer_writer = (
tp_rank == 0 and get_data_parallel_rank() == 0 and get_expert_model_parallel_rank() == 0
)
@@ -421,7 +430,7 @@ def save_pretrained(
# The live MTP export runs EP collectives, so every last-stage main rank joins it; the
# collective-free copy from the source checkpoint only runs on the writer.
- mtp_state_dict = self._get_mtp_state_dict(copy_from_pretrained=writes_layers)
+ mtp_state_dict = self._get_mtp_state_dict(copy_from_pretrained=is_stage_layer_writer)
if len(mtp_state_dict) > 0:
layer_state_dicts[self.model.config.num_layers].update(mtp_state_dict)
print(f"Successfully loaded {len(mtp_state_dict)} MTP tensors")
@@ -459,7 +468,7 @@ def save_pretrained(
# Add multimodal components to state_dict. Since only support decoder model quantization,
# no changes will be made to the multimodal components. We copy the multimodal components
# from the pretrained model directly to the state_dict to avoid implementing the export logic.
- if is_first_stage_main_rank and writes_layers:
+ if is_first_stage_main_rank and is_stage_layer_writer:
# layer_state_dicts is keyed by layer_number (1-indexed), so the first
# decoder layer on this (first) PP stage is the smallest key, not 0.
# Merge the multimodal components into that shard so they land in a file
@@ -488,7 +497,7 @@ def save_pretrained(
torch.distributed.barrier()
save_safetensors_by_layer_index(
- layer_state_dicts=layer_state_dicts if writes_layers else {},
+ layer_state_dicts=layer_state_dicts if is_stage_layer_writer else {},
total_layers=self.model.config.num_layers,
save_directory=save_directory,
name_template="model-{:05d}-of-{:05d}",
@@ -550,7 +559,11 @@ def _verify_exported_keys(self, save_directory, pretrained_model_name_or_path) -
# Narrow on purpose: compare module prefixes, not tensor names, since a quantized source
# carries extras with no export counterpart, and only inside decoder layers, whose naming
# is stable. A dropped decoder module is the case that loads fine and produces garbage.
- num_layers = self.model.config.num_layers
+ # HF decoder depth of this export: Megatron may build several physical layers per HF layer.
+ hf_depth = self._hf_text_config.num_hidden_layers
+ num_mtp = 0
+ if self.rules.get("mtp_in_decoder_layers", False):
+ num_mtp = getattr(self._hf_text_config, "num_nextn_predict_layers", 0) or 0
# Ancestors too: an export may expand one source module into several (Qwen3.5 packs
# routed experts; the quantized export writes them per expert). Expansion is not a drop.
exported_modules = set()
@@ -562,12 +575,18 @@ def _verify_exported_keys(self, save_directory, pretrained_model_name_or_path) -
break
exported_modules.add(prefix)
missing = set()
- for key in source - exported:
+ for key in source:
layer = re.search(r"\.layers\.(\d+)\.", key)
if layer is None:
continue # see the note above: decoder layers only
- if int(layer.group(1)) >= num_layers:
- continue # depth-pruned model: the source has layers this export does not
+ if int(layer.group(1)) >= hf_depth:
+ mtp_id = int(layer.group(1)) - self._src_num_hidden_layers
+ if not 0 <= mtp_id < num_mtp:
+ continue # depth-pruned model: the source has layers this export does not
+ # An MTP stored as extra decoder layers follows the exported decoder layers.
+ key = f"{key[: layer.start(1)]}{hf_depth + mtp_id}{key[layer.end(1) :]}"
+ if key in exported:
+ continue
if key.rsplit(".", 1)[0] in exported_modules:
continue # module is exported; this name is a source-side quantization artifact
if "rotary_emb" in key:
@@ -693,6 +712,12 @@ def _get_transformer_layer_state_dict(self, layer, layer_id, is_mtp=False):
layer.self_attention.linear_kv_up_proj, layer_id, is_mtp=is_mtp
)
self.rules["linear_proj"](layer.self_attention.linear_proj, layer_id, is_mtp=is_mtp)
+ core_attention = getattr(layer.self_attention, "core_attention", None)
+ if core_attention is not None and "core_attention" in self.rules:
+ self.rules["core_attention"](core_attention, layer_id, is_mtp=is_mtp)
+ indexer = getattr(core_attention, "indexer", None)
+ if indexer is not None:
+ self._get_dsa_indexer_state_dict(indexer, layer_id, is_mtp)
elif "linear_attn" in self.rules and hasattr(layer.self_attention, "in_proj"):
# GatedDeltaNet (Qwen3.5 linear attention): no q/k layernorm, no core_attention.
self._get_gated_delta_net_state_dict(layer, layer_id, is_mtp=is_mtp)
@@ -857,6 +882,8 @@ def _copy_mtp_state_dict_from_pretrained(self) -> dict[str, torch.Tensor]:
mtp_state_dict = {}
if not self._hf_pretrained_model_name:
return mtp_state_dict
+ if self.rules.get("mtp_in_decoder_layers", False):
+ return self._copy_decoder_mtp_layers_from_pretrained()
mtp_exists = False
@@ -910,6 +937,55 @@ def _copy_mtp_state_dict_from_pretrained(self) -> dict[str, torch.Tensor]:
self.exclude_modules.append("mtp*")
return mtp_state_dict
+ def _copy_decoder_mtp_layers_from_pretrained(self) -> dict[str, torch.Tensor]:
+ """Copy MTP layers stored as extra decoder layers (GLM-5.x) from the source, dequantized.
+
+ Used when Megatron did not build the MTP (e.g. Megatron-Bridge's GLM-5 bridge); the copies
+ stay BF16 and are excluded from quantization, like the released NVFP4 checkpoints.
+ """
+ num_mtp = getattr(self._hf_text_config, "num_nextn_predict_layers", 0) or 0
+ source = self._hf_pretrained_model_name
+ if num_mtp == 0 or source is None:
+ return {}
+ layers_prefix = self.all_mcore_mappings["input_layernorm"].target_name_or_prefix
+ layers_prefix = layers_prefix.split("{}")[0] # e.g. "model.layers."
+ src_prefixes = [
+ f"{layers_prefix}{self._src_num_hidden_layers + i}." for i in range(num_mtp)
+ ]
+ if not os.path.isdir(source):
+ source = self._download_hub_shards(str(source), tuple(src_prefixes))
+ keys = _read_checkpoint_keys(source)
+ mtp_state_dict = {}
+ for i in range(num_mtp):
+ src = src_prefixes[i]
+ dst = f"{layers_prefix}{self._hf_text_config.num_hidden_layers + i}."
+ for key in sorted(
+ k for k in keys if k.startswith(src) and not k.endswith("_scale_inv")
+ ):
+ mtp_state_dict[dst + key[len(src) :]] = get_safetensor(
+ str(source), key, dequantize=True
+ )
+ self.exclude_modules.append(dst + "*")
+ if mtp_state_dict:
+ print(f"Copied {len(mtp_state_dict)} MTP tensors from {source}")
+ else:
+ warn_rank_0(f"No MTP tensors under {src_prefixes} in {source}; exporting without MTP.")
+ return mtp_state_dict
+
+ @staticmethod
+ def _download_hub_shards(repo_id: str, key_prefixes: tuple[str, ...]) -> str:
+ """Download only the Hub shards holding tensors under ``key_prefixes``; return the local dir."""
+ try:
+ index_file = hf_hub_download(repo_id, "model.safetensors.index.json")
+ except EntryNotFoundError: # unsharded checkpoint
+ return snapshot_download(repo_id, allow_patterns=["model.safetensors"])
+ with open(index_file) as f:
+ weight_map = json.load(f)["weight_map"]
+ shards = sorted(
+ {shard for key, shard in weight_map.items() if key.startswith(key_prefixes)}
+ )
+ return snapshot_download(repo_id, allow_patterns=["model.safetensors.index.json", *shards])
+
def _get_gated_delta_net_state_dict(self, layer, layer_id, is_mtp=False):
"""Export a GatedDeltaNet (Qwen3.5 linear-attention) layer's ``self_attention``."""
gdn = layer.self_attention
@@ -920,6 +996,13 @@ def _get_gated_delta_net_state_dict(self, layer, layer_id, is_mtp=False):
self.rules["linear_attn.out_norm"](gdn.out_norm, layer_id, is_mtp=is_mtp)
self.rules["linear_attn.out_proj"](gdn.out_proj, layer_id, is_mtp=is_mtp)
+ def _get_dsa_indexer_state_dict(self, indexer, layer_id, is_mtp=False):
+ """Export the DSA indexer of a sparse MLA layer."""
+ if "indexer.linear_wq_b" not in self.rules:
+ raise NotImplementedError(f"No export rule for the DSA indexer of {self.arch}.")
+ for name in ("linear_wq_b", "linear_wk", "k_norm", "linear_weights_proj"):
+ self.rules[f"indexer.{name}"](getattr(indexer, name), layer_id, is_mtp=is_mtp)
+
def _get_mamba_layer_state_dict(self, layer, layer_id, is_mtp=False):
if not isinstance(layer.norm, IdentityOp):
self.rules["norm"](layer.norm, layer_id, is_mtp=is_mtp)
diff --git a/modelopt/torch/quantization/plugins/megatron.py b/modelopt/torch/quantization/plugins/megatron.py
index aaacf198a12..5e422235e14 100644
--- a/modelopt/torch/quantization/plugins/megatron.py
+++ b/modelopt/torch/quantization/plugins/megatron.py
@@ -30,6 +30,15 @@
import megatron.core.transformer.moe.experts as megatron_moe
import torch
from megatron.core.dist_checkpointing.utils import replace_prefix_for_sharding
+from megatron.core.extensions.transformer_engine import (
+ TEColumnParallelGroupedLinear,
+ TEColumnParallelLinear,
+ TEDotProductAttention,
+ TELayerNormColumnParallelLinear,
+ TELinear,
+ TERowParallelGroupedLinear,
+ TERowParallelLinear,
+)
from megatron.core.models.gpt import GPTModel
from megatron.core.parallel_state import get_data_parallel_group
from megatron.core.tensor_parallel.mappings import gather_from_sequence_parallel_region
@@ -62,23 +71,14 @@
from ..utils.layerwise_calib import LayerActivationCollector
from .custom import CUSTOM_MODEL_PLUGINS, _ParallelLinear
from .gated_delta_net import GatedDeltaNetStateQuantMixin
+from .transformer_engine import _QuantTEGroupedLinear, _QuantTELayerNormLinear, _QuantTELinear
try:
- from megatron.core.extensions.transformer_engine import (
- TEColumnParallelGroupedLinear,
- TEColumnParallelLinear,
- TEDotProductAttention,
- TELayerNormColumnParallelLinear,
- TELinear,
- TERowParallelGroupedLinear,
- TERowParallelLinear,
- )
-
- from .transformer_engine import _QuantTEGroupedLinear, _QuantTELayerNormLinear, _QuantTELinear
-
- HAS_TE = True
+ from megatron.core.transformer.experimental_attention_variant.dsa import DSAttention
+
+ HAS_DSA = True
except ImportError:
- HAS_TE = False
+ HAS_DSA = False
try:
from megatron.core.ssm.gated_delta_net import GatedDeltaNet
@@ -835,251 +835,275 @@ def sharded_state_dict(self, prefix="", sharded_offsets=(), metadata=None):
return sharded_state_dict
-if HAS_TE:
+@QuantModuleRegistry.register({TERowParallelLinear: "te_mcore_RowParallelLinear"})
+class _QuantTEMCoreRowParallelLinear(_QuantTELinear, _MegatronRowParallelLinear):
+ pass
- @QuantModuleRegistry.register({TERowParallelLinear: "te_mcore_RowParallelLinear"})
- class _QuantTEMCoreRowParallelLinear(_QuantTELinear, _MegatronRowParallelLinear):
- pass
- @QuantModuleRegistry.register({TEColumnParallelLinear: "te_mcore_ColumnParallelLinear"})
- class _QuantTEMCoreColumnParallelLinear(_QuantTELinear, _MegatronColumnParallelLinear):
- pass
+@QuantModuleRegistry.register({TEColumnParallelLinear: "te_mcore_ColumnParallelLinear"})
+class _QuantTEMCoreColumnParallelLinear(_QuantTELinear, _MegatronColumnParallelLinear):
+ pass
- @QuantModuleRegistry.register({TELinear: "te_mcore_Linear"})
- class _QuantTEMCoreLinear(_QuantTELinear):
- pass
- @QuantModuleRegistry.register(
- {TELayerNormColumnParallelLinear: "te_mcore_LayerNormColumnParallelLinear"}
- )
- class _QuantTELayerNormColumnParallelLinear(
- _QuantTELayerNormLinear, _MegatronColumnParallelLinear
- ):
- pass
-
- # Quantized subclasses to support TEGroupedLinear quantization
- class _QuantMegatronTEGroupedLinear(_QuantTEGroupedLinear, _MegatronParallelLinear):
- def modelopt_post_load_extra_state(self):
- _initialize_grouped_weight_quantizer_state(self)
-
- def _load_from_state_dict(self, state_dict, prefix, *args, **kwargs):
- # _sharded_state_dict_grouped adds _extra_state{gemm_idx} for gemm_idx:[1, num_gemms] in
- # sharded_state_dict which is same as _extra_state. The _extra_state{gemm_idx} is used for
- # TE Fp8 checkpoint, we need to remove the _extra_state{gemm_idx} for gemm_idx:[1, num_gemms]
- # for modelopt checkpoint restore
- filtered_state_dict = {
- k: v
- for k, v in state_dict.items()
- if not any(k.endswith(f"_extra_state{num}") for num in range(1, self.num_gemms))
- }
- return super()._load_from_state_dict(filtered_state_dict, prefix, *args, **kwargs)
+@QuantModuleRegistry.register({TELinear: "te_mcore_Linear"})
+class _QuantTEMCoreLinear(_QuantTELinear):
+ pass
+
+
+@QuantModuleRegistry.register(
+ {TELayerNormColumnParallelLinear: "te_mcore_LayerNormColumnParallelLinear"}
+)
+class _QuantTELayerNormColumnParallelLinear(_QuantTELayerNormLinear, _MegatronColumnParallelLinear):
+ pass
+
+
+# Quantized subclasses to support TEGroupedLinear quantization
+class _QuantMegatronTEGroupedLinear(_QuantTEGroupedLinear, _MegatronParallelLinear):
+ def modelopt_post_load_extra_state(self):
+ _initialize_grouped_weight_quantizer_state(self)
+
+ def _load_from_state_dict(self, state_dict, prefix, *args, **kwargs):
+ # _sharded_state_dict_grouped adds _extra_state{gemm_idx} for gemm_idx:[1, num_gemms] in
+ # sharded_state_dict which is same as _extra_state. The _extra_state{gemm_idx} is used for
+ # TE Fp8 checkpoint, we need to remove the _extra_state{gemm_idx} for gemm_idx:[1, num_gemms]
+ # for modelopt checkpoint restore
+ filtered_state_dict = {
+ k: v
+ for k, v in state_dict.items()
+ if not any(k.endswith(f"_extra_state{num}") for num in range(1, self.num_gemms))
+ }
+ return super()._load_from_state_dict(filtered_state_dict, prefix, *args, **kwargs)
+
+ def _process_quantizer_amax(self, k, v, quantizer_state_dict):
+ # Per-expert quantizers have independent checkpoint keys. Preserve their native
+ # scalar, channel, or block shape instead of flattening them through the legacy
+ # single-quantizer path.
+ if re.match(r"weight_quantizer\.\d+\..+_amax$", k):
+ quantizer_state_dict[k] = v
+ else:
+ quantizer_state_dict[k] = v.view(-1) if v.numel() == 1 else v
+
+ def _expert_parallel_groups(self):
+ """Return ``(ep_group, expt_tp_group, expt_dp_group)`` for fused experts."""
+ pg_collection = getattr(self, "_pg_collection", None)
+ if pg_collection is not None:
+ return pg_collection.ep, pg_collection.expt_tp, pg_collection.expt_dp
+ return (
+ mcore_parallel.get_expert_model_parallel_group(),
+ mcore_parallel.get_expert_tensor_parallel_group(),
+ mcore_parallel.get_expert_data_parallel_group(),
+ )
+
+ def sharded_state_dict(self, prefix="", sharded_offsets=(), metadata=None):
+ """Emit per-expert quantizer amax with the same global expert identity as the weights.
+
+ The base linear emits ``weight_quantizer.{local_i}._amax`` with the local index and no
+ expert offset, so every EP rank writes identical keys and ``torch_dist`` dedup keeps only
+ one rank's experts. Here we mirror Megatron ``TEGroupedLinear._sharded_state_dict_grouped``:
+ each fused expert comes with its ``global_expert_idx`` (baked into the key prefix under
+ ``singleton_local_shards``, otherwise an EP sharded-offset) so all ``num_global_experts``
+ persist and reshard to any EP. Shared, whole-linear quantizer buffers (e.g.
+ ``input_quantizer``) keep the plain replicated path.
+ """
+ metadata = ensure_metadata_has_dp_cp_group(metadata)
+ singleton_local_shards = bool((metadata or {}).get("singleton_local_shards", False))
+
+ # Weights/bias/_extra_state come from the wrapped TE grouped linear, which already
+ # assigns each expert its global identity. Skip _MegatronParallelLinear's local-index
+ # amax emission by starting from the base MCore module's sharded_state_dict.
+ sharded_state_dict = super(_MegatronParallelLinear, self).sharded_state_dict(
+ prefix, sharded_offsets, metadata
+ )
- def _process_quantizer_amax(self, k, v, quantizer_state_dict):
- # Per-expert quantizers have independent checkpoint keys. Preserve their native
- # scalar, channel, or block shape instead of flattening them through the legacy
- # single-quantizer path.
- if re.match(r"weight_quantizer\.\d+\..+_amax$", k):
+ # Collect the quantizer buffers exactly like _MegatronParallelLinear.sharded_state_dict.
+ quantizer_state_dict = {}
+ for k, v in self.state_dict(prefix="", keep_vars=True).items():
+ if "_quantizer" in k and "_amax" in k:
+ self._process_quantizer_amax(k, v, quantizer_state_dict)
+ elif k == "input_quantizer._pre_quant_scale":
+ self._process_activation_quantizer_pre_quant_scale(k, v, quantizer_state_dict)
+ elif self._parameter_to_keep_in_quantizer_state_dict(k):
quantizer_state_dict[k] = v
+ elif "quantizer" in k:
+ warn_rank_0(
+ f"Quantizer state {k} is not supported for sharded_state_dict. "
+ "Please use regular state_dict."
+ )
+
+ # Channel shard axes (per real key); _global_amax stays un-sharded along channels but
+ # still rides with the expert identity below.
+ shard_axis_dict = self._get_shard_axis_dict(quantizer_state_dict)
+ ep_group, expt_tp_group, expt_dp_group = self._expert_parallel_groups()
+
+ # Split per-expert weight_quantizer.{i}.* from shared (input/output) quantizer buffers.
+ expert_re = re.compile(r"^weight_quantizer\.(\d+)\.(.+)$")
+ per_expert_subs = [[] for _ in range(self.num_gemms)]
+ shared_state = {}
+ for k, v in quantizer_state_dict.items():
+ m = expert_re.match(k)
+ if m:
+ per_expert_subs[int(m.group(1))].append((m.group(2), v, shard_axis_dict.get(k)))
else:
- quantizer_state_dict[k] = v.view(-1) if v.numel() == 1 else v
-
- def _expert_parallel_groups(self):
- """Return ``(ep_group, expt_tp_group, expt_dp_group)`` for fused experts."""
- pg_collection = getattr(self, "_pg_collection", None)
- if pg_collection is not None:
- return pg_collection.ep, pg_collection.expt_tp, pg_collection.expt_dp
- return (
- mcore_parallel.get_expert_model_parallel_group(),
- mcore_parallel.get_expert_tensor_parallel_group(),
- mcore_parallel.get_expert_data_parallel_group(),
- )
+ shared_state[k] = v
- def sharded_state_dict(self, prefix="", sharded_offsets=(), metadata=None):
- """Emit per-expert quantizer amax with the same global expert identity as the weights.
-
- The base linear emits ``weight_quantizer.{local_i}._amax`` with the local index and no
- expert offset, so every EP rank writes identical keys and ``torch_dist`` dedup keeps only
- one rank's experts. Here we mirror Megatron ``TEGroupedLinear._sharded_state_dict_grouped``:
- each fused expert comes with its ``global_expert_idx`` (baked into the key prefix under
- ``singleton_local_shards``, otherwise an EP sharded-offset) so all ``num_global_experts``
- persist and reshard to any EP. Shared, whole-linear quantizer buffers (e.g.
- ``input_quantizer``) keep the plain replicated path.
- """
- metadata = ensure_metadata_has_dp_cp_group(metadata)
- singleton_local_shards = bool((metadata or {}).get("singleton_local_shards", False))
-
- # Weights/bias/_extra_state come from the wrapped TE grouped linear, which already
- # assigns each expert its global identity. Skip _MegatronParallelLinear's local-index
- # amax emission by starting from the base MCore module's sharded_state_dict.
- sharded_state_dict = super(_MegatronParallelLinear, self).sharded_state_dict(
- prefix, sharded_offsets, metadata
+ # Shared quantizer buffers have no expert identity in their keys or offsets. Keep the
+ # dense TP/DP defaults so replica IDs distinguish EP ranks; using expt_tp/expt_dp here
+ # would collide across EP ranks. Expert-axis sharding would require an EP-aware
+ # replica ID in addition to the expert process groups.
+ shared_axis_dict = {k: shard_axis_dict[k] for k in shared_state if k in shard_axis_dict}
+ sharded_state_dict.update(
+ make_sharded_tensors_for_checkpoint(
+ shared_state, prefix, shared_axis_dict, sharded_offsets
)
+ )
- # Collect the quantizer buffers exactly like _MegatronParallelLinear.sharded_state_dict.
- quantizer_state_dict = {}
- for k, v in self.state_dict(prefix="", keep_vars=True).items():
- if "_quantizer" in k and "_amax" in k:
- self._process_quantizer_amax(k, v, quantizer_state_dict)
- elif k == "input_quantizer._pre_quant_scale":
- self._process_activation_quantizer_pre_quant_scale(k, v, quantizer_state_dict)
- elif self._parameter_to_keep_in_quantizer_state_dict(k):
- quantizer_state_dict[k] = v
- elif "quantizer" in k:
- warn_rank_0(
- f"Quantizer state {k} is not supported for sharded_state_dict. "
- "Please use regular state_dict."
- )
-
- # Channel shard axes (per real key); _global_amax stays un-sharded along channels but
- # still rides with the expert identity below.
- shard_axis_dict = self._get_shard_axis_dict(quantizer_state_dict)
- ep_group, expt_tp_group, expt_dp_group = self._expert_parallel_groups()
-
- # Split per-expert weight_quantizer.{i}.* from shared (input/output) quantizer buffers.
- expert_re = re.compile(r"^weight_quantizer\.(\d+)\.(.+)$")
- per_expert_subs = [[] for _ in range(self.num_gemms)]
- shared_state = {}
- for k, v in quantizer_state_dict.items():
- m = expert_re.match(k)
- if m:
- per_expert_subs[int(m.group(1))].append((m.group(2), v, shard_axis_dict.get(k)))
- else:
- shared_state[k] = v
-
- # Shared quantizer buffers have no expert identity in their keys or offsets. Keep the
- # dense TP/DP defaults so replica IDs distinguish EP ranks; using expt_tp/expt_dp here
- # would collide across EP ranks. Expert-axis sharding would require an EP-aware
- # replica ID in addition to the expert process groups.
- shared_axis_dict = {k: shard_axis_dict[k] for k in shared_state if k in shard_axis_dict}
- sharded_state_dict.update(
- make_sharded_tensors_for_checkpoint(
- shared_state, prefix, shared_axis_dict, sharded_offsets
+ # Per-expert amax: assign the same global expert identity the weights use.
+ num_global_experts = get_pg_size(ep_group) * self.num_gemms
+ local_expert_indices_offset = get_pg_rank(ep_group) * self.num_gemms
+ ep_axis = len(sharded_offsets)
+ for gemm_idx, subs in enumerate(per_expert_subs):
+ if not subs:
+ continue
+ global_expert_idx = local_expert_indices_offset + gemm_idx
+ if singleton_local_shards:
+ expert_prefix = f"{global_expert_idx}.{prefix}"
+ new_sharded_offsets = sharded_offsets
+ else:
+ expert_prefix = prefix
+ new_sharded_offsets = (
+ *sharded_offsets,
+ (ep_axis, global_expert_idx, num_global_experts),
)
+ expert_state = {f"{gemm_idx}.weight_quantizer.{sub}": v for sub, v, _ in subs}
+ expert_axis = {
+ f"{gemm_idx}.weight_quantizer.{sub}": axis
+ for sub, _, axis in subs
+ if axis is not None
+ }
+ sub_sd = make_sharded_tensors_for_checkpoint(
+ expert_state,
+ "",
+ expert_axis,
+ new_sharded_offsets,
+ tp_group=expt_tp_group,
+ dp_cp_group=expt_dp_group,
)
+ # Rewrite each ShardedTensor.key to carry the global expert identity (dict keys,
+ # which map to the local buffers on restore, are left untouched).
+ replace_prefix_for_sharding(sub_sd, f"{gemm_idx}.", expert_prefix)
+ for sub, _, _ in subs:
+ sh_ten = sub_sd[f"{gemm_idx}.weight_quantizer.{sub}"]
+ sharded_state_dict[f"{prefix}weight_quantizer.{gemm_idx}.{sub}"] = sh_ten
+ return sharded_state_dict
- # Per-expert amax: assign the same global expert identity the weights use.
- num_global_experts = get_pg_size(ep_group) * self.num_gemms
- local_expert_indices_offset = get_pg_rank(ep_group) * self.num_gemms
- ep_axis = len(sharded_offsets)
- for gemm_idx, subs in enumerate(per_expert_subs):
- if not subs:
- continue
- global_expert_idx = local_expert_indices_offset + gemm_idx
- if singleton_local_shards:
- expert_prefix = f"{global_expert_idx}.{prefix}"
- new_sharded_offsets = sharded_offsets
- else:
- expert_prefix = prefix
- new_sharded_offsets = (
- *sharded_offsets,
- (ep_axis, global_expert_idx, num_global_experts),
- )
- expert_state = {f"{gemm_idx}.weight_quantizer.{sub}": v for sub, v, _ in subs}
- expert_axis = {
- f"{gemm_idx}.weight_quantizer.{sub}": axis
- for sub, _, axis in subs
- if axis is not None
- }
- sub_sd = make_sharded_tensors_for_checkpoint(
- expert_state,
- "",
- expert_axis,
- new_sharded_offsets,
- tp_group=expt_tp_group,
- dp_cp_group=expt_dp_group,
- )
- # Rewrite each ShardedTensor.key to carry the global expert identity (dict keys,
- # which map to the local buffers on restore, are left untouched).
- replace_prefix_for_sharding(sub_sd, f"{gemm_idx}.", expert_prefix)
- for sub, _, _ in subs:
- sh_ten = sub_sd[f"{gemm_idx}.weight_quantizer.{sub}"]
- sharded_state_dict[f"{prefix}weight_quantizer.{gemm_idx}.{sub}"] = sh_ten
- return sharded_state_dict
-
- @QuantModuleRegistry.register(
- {TEColumnParallelGroupedLinear: "megatron_TEColumnParallelGroupedLinear"}
- )
- class _MegatronTEGroupedColumnParallelLinear(
- _QuantMegatronTEGroupedLinear, _MegatronColumnParallelLinear
- ):
- pass
- @QuantModuleRegistry.register(
- {TERowParallelGroupedLinear: "megatron_TERowParallelGroupedLinear"}
- )
- class _MegatronTEGroupedRowParallelLinear(
- _QuantMegatronTEGroupedLinear, _MegatronRowParallelLinear
- ):
- pass
+@QuantModuleRegistry.register(
+ {TEColumnParallelGroupedLinear: "megatron_TEColumnParallelGroupedLinear"}
+)
+class _MegatronTEGroupedColumnParallelLinear(
+ _QuantMegatronTEGroupedLinear, _MegatronColumnParallelLinear
+):
+ pass
- @QuantModuleRegistry.register({megatron_moe.TEGroupedMLP: "megatron_moe_TEGroupedMLP"})
- class _MegatronTEGroupedMLP(_MegatronMLP):
- def _setup(self):
- if not hasattr(self, "parallel_state") or self.parallel_state is None:
- self.parallel_state = ParallelState(
- mcore_parallel.get_expert_data_parallel_group(),
- tensor_parallel_group=mcore_parallel.get_expert_tensor_parallel_group(),
- expert_model_parallel_group=mcore_parallel.get_expert_model_parallel_group(),
- )
- # These child linears are still native MCore modules here. Seed `_parallel_state`
- # directly so the later QuantModule conversion sees the intended parallel state.
- self.linear_fc1._parallel_state = self.parallel_state
- self.linear_fc2._parallel_state = self.parallel_state
-
- @QuantModuleRegistry.register({TEDotProductAttention: "TEDotProductAttention"})
- class _QuantTEDotProductAttention(QuantModule):
- """Quantized version of TEDotProductAttention for Megatron models with KV cache quantization.
-
- This class adds KV cache quantization support to Transformer Engine's TEDotProductAttention
- module used in Megatron-Core models. It introduces three quantizers (q_bmm_quantizer,
- k_bmm_quantizer, v_bmm_quantizer) that quantize the query, key, and value tensors after
- RoPE has been applied.
- """
- def _setup(self):
- """Initialize quantizers for Q, K, V tensors."""
- self.q_bmm_quantizer = TensorQuantizer()
- self.k_bmm_quantizer = TensorQuantizer()
- self.v_bmm_quantizer = TensorQuantizer()
+@QuantModuleRegistry.register({TERowParallelGroupedLinear: "megatron_TERowParallelGroupedLinear"})
+class _MegatronTEGroupedRowParallelLinear(
+ _QuantMegatronTEGroupedLinear, _MegatronRowParallelLinear
+):
+ pass
- # Set parallel_state for distributed sync of BMM quantizers
- try:
- data_parallel_group = get_data_parallel_group(with_context_parallel=True)
- except AssertionError:
- data_parallel_group = get_data_parallel_group()
+
+@QuantModuleRegistry.register({megatron_moe.TEGroupedMLP: "megatron_moe_TEGroupedMLP"})
+class _MegatronTEGroupedMLP(_MegatronMLP):
+ def _setup(self):
+ if not hasattr(self, "parallel_state") or self.parallel_state is None:
self.parallel_state = ParallelState(
- data_parallel_group,
- mcore_parallel.get_tensor_model_parallel_group(),
+ mcore_parallel.get_expert_data_parallel_group(),
+ tensor_parallel_group=mcore_parallel.get_expert_tensor_parallel_group(),
+ expert_model_parallel_group=mcore_parallel.get_expert_model_parallel_group(),
)
+ # These child linears are still native MCore modules here. Seed `_parallel_state`
+ # directly so the later QuantModule conversion sees the intended parallel state.
+ self.linear_fc1._parallel_state = self.parallel_state
+ self.linear_fc2._parallel_state = self.parallel_state
+
+
+@QuantModuleRegistry.register({TEDotProductAttention: "TEDotProductAttention"})
+class _QuantCoreAttention(QuantModule):
+ """Core attention (TEDotProductAttention, DSAttention) with KV cache quantization.
+
+ Adds q/k/v_bmm_quantizers that quantize the post-RoPE query, key and value tensors.
+ """
- def forward(self, query, key, value, *args, **kwargs):
- """Apply post-RoPE quantization to KV cache."""
- # Quantize Q, K, V
- query = self.q_bmm_quantizer(query)
- key = self.k_bmm_quantizer(key)
+ def _setup(self):
+ """Initialize quantizers for Q, K, V tensors."""
+ self.q_bmm_quantizer = TensorQuantizer()
+ self.k_bmm_quantizer = TensorQuantizer()
+ self.v_bmm_quantizer = TensorQuantizer()
+
+ # Set parallel_state for distributed sync of BMM quantizers
+ try:
+ data_parallel_group = get_data_parallel_group(with_context_parallel=True)
+ except AssertionError:
+ data_parallel_group = get_data_parallel_group()
+ self.parallel_state = ParallelState(
+ data_parallel_group,
+ mcore_parallel.get_tensor_model_parallel_group(),
+ )
+
+ def forward(self, query, key, value, *args, **kwargs):
+ """Apply post-RoPE quantization to KV cache."""
+ # Quantize Q, K, V
+ query = self.q_bmm_quantizer(query)
+ if value is None:
+ # Absorbed MLA (DSAttention) passes value=None: the key is the KV latent that both K
+ # and V are read from, so calibrate V on it too (output unused) to export a V scale.
+ if self.v_bmm_quantizer._if_calib:
+ self.v_bmm_quantizer(key)
+ else:
value = self.v_bmm_quantizer(value)
- return super().forward(query, key, value, *args, **kwargs)
-
- def modelopt_post_restore(self, name=""):
- """Restore quantizer states after model loading."""
- for tq in [self.q_bmm_quantizer, self.k_bmm_quantizer, self.v_bmm_quantizer]:
- # TODO: Add support for non-scalar states such as
- # Affine KVCache bias vector which is per head per channel
- if not all(v.numel() == 1 for v in tq.state_dict().values()):
- raise NotImplementedError(
- "Only scalar states are supported for KV Cache/BMM Quantizers"
- )
- # dtype and device should have been set in `megatron_replace_quant_module_hook`
- # via `_configure_attention_for_kv_cache_quant`
- assert hasattr(self, "device") and hasattr(self, "dtype")
- self.to(device=self.device, dtype=self.dtype)
-
- def sharded_state_dict(self, prefix="", sharded_offsets=(), metadata=None):
- # Currently we do not need sharded_state_dict for TEDotProductAttention since the amax are scalar values.
- # However we would need this in future to support non-scalar states such as
- # Affine KVCache Quant bias vector.
- state_dict = self.state_dict(prefix="", keep_vars=True)
- return make_sharded_tensors_for_checkpoint(state_dict, prefix, {}, sharded_offsets)
+ key = self.k_bmm_quantizer(key)
+ return super().forward(query, key, value, *args, **kwargs)
+
+ def modelopt_post_restore(self, name=""):
+ """Restore quantizer states after model loading."""
+ for tq in [self.q_bmm_quantizer, self.k_bmm_quantizer, self.v_bmm_quantizer]:
+ # TODO: Add support for non-scalar states such as
+ # Affine KVCache bias vector which is per head per channel
+ if not all(v.numel() == 1 for v in tq.state_dict().values()):
+ raise NotImplementedError(
+ "Only scalar states are supported for KV Cache/BMM Quantizers"
+ )
+ # dtype and device should have been set in `megatron_replace_quant_module_hook`
+ # via `_configure_attention_for_kv_cache_quant`
+ assert hasattr(self, "device") and hasattr(self, "dtype")
+ self.to(device=self.device, dtype=self.dtype)
+
+ def sharded_state_dict(self, prefix="", sharded_offsets=(), metadata=None):
+ # Currently we do not need sharded_state_dict for core attention since the amax are scalar values.
+ # However we would need this in future to support non-scalar states such as
+ # Affine KVCache Quant bias vector.
+ state_dict = self.state_dict(prefix="", keep_vars=True)
+ return make_sharded_tensors_for_checkpoint(state_dict, prefix, {}, sharded_offsets)
+
+
+if HAS_DSA:
+
+ @QuantModuleRegistry.register({DSAttention: "megatron_DSAttention"})
+ class _QuantDSAttention(_QuantCoreAttention):
+ """DSAttention with KV-cache quantization.
+
+ torch's state_dict() / load_state_dict() route ``_extra_state`` only through classes that
+ override these; TEDotProductAttention does, DSAttention does not, so without them the
+ quantizer state (including amax) was dropped from checkpoints. They exist for that class
+ check: at runtime the instance-level modelopt callbacks shadow them, so don't remove them.
+ """
+
+ def get_extra_state(self):
+ return quant_module_get_extra_state(self)
+
+ def set_extra_state(self, state):
+ quant_module_set_extra_state(self, state)
if HAS_GDN:
@@ -1093,6 +1117,14 @@ class _QuantGatedDeltaNet(GatedDeltaNetStateQuantMixin):
paths (``ssm_prefill`` / ``ssm_decode``) are left untouched.
"""
+ # Class-level overrides so torch routes the quantizer state through ``_extra_state``
+ # (GatedDeltaNet has none); see _QuantDSAttention.
+ def get_extra_state(self):
+ return quant_module_get_extra_state(self)
+
+ def set_extra_state(self, state):
+ quant_module_set_extra_state(self, state)
+
def _setup(self):
super()._setup()
try:
diff --git a/tests/gpu_megatron/torch/export/plugins/test_mcore_export_mappings.py b/tests/gpu_megatron/torch/export/plugins/test_mcore_export_mappings.py
index 9debbc2231d..8b048e87bcd 100644
--- a/tests/gpu_megatron/torch/export/plugins/test_mcore_export_mappings.py
+++ b/tests/gpu_megatron/torch/export/plugins/test_mcore_export_mappings.py
@@ -34,6 +34,7 @@
"Qwen3VLForConditionalGeneration": "model.language_model.layers.{}.self_attn.",
"Qwen3_5ForConditionalGeneration": "model.language_model.layers.{}.self_attn.",
"Qwen3_5MoeForConditionalGeneration": "model.language_model.layers.{}.self_attn.",
+ "GlmMoeDsaForCausalLM": "model.layers.{}.self_attn.",
}
@@ -54,6 +55,7 @@ def test_export_mapping_emits_kv_cache_scales(arch, prefix):
"Qwen3_5MoeForConditionalGeneration": "model.language_model.layers.{}.mlp.experts.{}",
"Qwen3MoeForCausalLM": "model.layers.{}.mlp.experts.{}",
"NemotronHForCausalLM": "backbone.layers.{}.mixer.experts.{}",
+ "GlmMoeDsaForCausalLM": "model.layers.{}.mlp.experts.{}",
}
diff --git a/tests/gpu_megatron/torch/export/test_unified_export_megatron.py b/tests/gpu_megatron/torch/export/test_unified_export_megatron.py
index 480be83018a..1060e1d6ea1 100644
--- a/tests/gpu_megatron/torch/export/test_unified_export_megatron.py
+++ b/tests/gpu_megatron/torch/export/test_unified_export_megatron.py
@@ -47,6 +47,7 @@
import modelopt.torch.quantization.ggml as ggml
import modelopt.torch.speculative as mtsp
from modelopt.torch.export import KV_CACHE_FP8, export_mcore_gpt_to_hf, import_mcore_gpt_from_hf
+from modelopt.torch.export.plugins.mcore_common import all_mcore_hf_export_mapping
from modelopt.torch.export.quant_format import IQ_FORMATS
from modelopt.torch.export.unified_export_megatron import GPTModelExporter
from modelopt.torch.quantization.config import QuantizerAttributeConfig
@@ -976,6 +977,7 @@ def _make_exporter_for_mtp(model_dir: Path) -> GPTModelExporter:
exporter._hf_pretrained_model_name = str(model_dir)
exporter._state_dict = {} # MTP keys are absent — they should be picked up
exporter.exclude_modules = []
+ exporter.rules = {} # a Qwen-style ``mtp.*`` checkpoint, not MTP stored as decoder layers
return exporter
@@ -1052,6 +1054,104 @@ def test_mtp_state_dict_index_file(tmp_path):
assert "mtp*" in exporter.exclude_modules
+def _test_live_decoder_mtp_export_rejected(model_dir, rank, size):
+ model = get_mcore_gpt_model(
+ tensor_model_parallel_size=size,
+ pipeline_model_parallel_size=1,
+ initialize_megatron=True,
+ num_layers=2,
+ hidden_size=64,
+ num_attention_heads=4,
+ vocab_size=128,
+ max_sequence_length=32,
+ mtp_num_layers=1,
+ ).cuda()
+ with pytest.raises(NotImplementedError, match="Megatron-built MTP"):
+ GPTModelExporter(model, str(model_dir))
+
+
+def test_live_decoder_mtp_export_rejected(dist_workers_size_1, tmp_path):
+ """GLM-5 MTP is only copied from the source; a Megatron-built one must not export misnamed."""
+ transformers.GlmMoeDsaConfig(
+ num_hidden_layers=2, architectures=["GlmMoeDsaForCausalLM"]
+ ).save_pretrained(tmp_path)
+ dist_workers_size_1.run(partial(_test_live_decoder_mtp_export_rejected, tmp_path))
+
+
+def test_mtp_state_dict_copies_decoder_mtp_layers(tmp_path):
+ """GLM-5 keeps MTP as an extra decoder layer; copy it dequantized when Megatron did not build it."""
+ model_dir = tmp_path / "fake_glm5"
+ model_dir.mkdir()
+ fp8 = torch.full((128, 128), 2.0).to(torch.float8_e4m3fn)
+ save_file(
+ {
+ "model.layers.1.input_layernorm.weight": torch.ones(8), # pruned decoder layer
+ "model.layers.2.enorm.weight": torch.full((8,), 3.0), # MTP layer of the source
+ "model.layers.2.eh_proj.weight": fp8,
+ "model.layers.2.eh_proj.weight_scale_inv": torch.full((1, 1), 0.5),
+ },
+ str(model_dir / "model.safetensors"),
+ )
+ exporter = _make_exporter_for_mtp(model_dir)
+ exporter.rules = {"mtp_in_decoder_layers": True}
+ exporter.all_mcore_mappings = all_mcore_hf_export_mapping["GlmMoeDsaForCausalLM"]
+ # Depth-pruned from 2 to 1 decoder layers: the source MTP (layer 2) lands at layer 1.
+ exporter._src_num_hidden_layers = 2
+ exporter._hf_text_config = SimpleNamespace(num_hidden_layers=1, num_nextn_predict_layers=1)
+
+ mtp_state_dict = exporter._get_mtp_state_dict()
+
+ assert sorted(mtp_state_dict) == [
+ "model.layers.1.eh_proj.weight",
+ "model.layers.1.enorm.weight",
+ ]
+ assert mtp_state_dict["model.layers.1.eh_proj.weight"].dtype == torch.bfloat16
+ torch.testing.assert_close(
+ mtp_state_dict["model.layers.1.eh_proj.weight"].float(), torch.ones(128, 128)
+ )
+ assert exporter.exclude_modules == ["model.layers.1.*"]
+
+
+def test_mtp_state_dict_copies_decoder_mtp_layers_from_hub(tmp_path, monkeypatch):
+ """A Hub-ID source downloads only the shards holding the MTP layer, then copies it."""
+ hub = tmp_path / "hub"
+ hub.mkdir()
+ shards = {
+ "model.layers.0.input_layernorm.weight": "model-00001-of-00002.safetensors",
+ "model.layers.1.enorm.weight": "model-00002-of-00002.safetensors",
+ }
+ save_file(
+ {"model.layers.0.input_layernorm.weight": torch.ones(8)},
+ str(hub / shards["model.layers.0.input_layernorm.weight"]),
+ )
+ save_file(
+ {"model.layers.1.enorm.weight": torch.full((8,), 3.0)},
+ str(hub / shards["model.layers.1.enorm.weight"]),
+ )
+ (hub / "model.safetensors.index.json").write_text(json.dumps({"weight_map": shards}))
+ requested = {}
+
+ def fake_snapshot_download(repo_id, allow_patterns):
+ requested[repo_id] = allow_patterns
+ return str(hub)
+
+ monkeypatch.setattr(uem, "hf_hub_download", lambda repo_id, filename: str(hub / filename))
+ monkeypatch.setattr(uem, "snapshot_download", fake_snapshot_download)
+ exporter = _make_exporter_for_mtp(Path("zai-org/GLM-5.2"))
+ exporter.rules = {"mtp_in_decoder_layers": True}
+ exporter.all_mcore_mappings = all_mcore_hf_export_mapping["GlmMoeDsaForCausalLM"]
+ exporter._src_num_hidden_layers = 1
+ exporter._hf_text_config = SimpleNamespace(num_hidden_layers=1, num_nextn_predict_layers=1)
+
+ mtp_state_dict = exporter._get_mtp_state_dict()
+
+ assert requested == {
+ "zai-org/GLM-5.2": ["model.safetensors.index.json", "model-00002-of-00002.safetensors"]
+ }
+ assert list(mtp_state_dict) == ["model.layers.1.enorm.weight"]
+ assert exporter.exclude_modules == ["model.layers.1.*"]
+
+
class _FakeTEGroupedMLP:
"""Minimal TEGroupedMLP stand-in exposing num_gemms, weight{i}, and state_dict()."""
@@ -1260,12 +1360,39 @@ def test_is_sidecar_writer_rank_pins_to_dp0_ep0(monkeypatch):
assert GPTModelExporter._is_sidecar_writer_rank(True) is False
-def _make_exporter_for_key_check(num_layers: int) -> GPTModelExporter:
+def _make_exporter_for_key_check(
+ num_layers: int, src_num_layers: int | None = None, num_mtp: int = 0
+) -> GPTModelExporter:
+ """``num_layers`` is the exported HF decoder depth; ``num_mtp`` MTP layers follow it."""
exporter = object.__new__(GPTModelExporter)
- exporter.model = SimpleNamespace(config=SimpleNamespace(num_layers=num_layers))
+ exporter._hf_text_config = SimpleNamespace(
+ num_hidden_layers=num_layers, num_nextn_predict_layers=num_mtp
+ )
+ exporter._src_num_hidden_layers = num_layers if src_num_layers is None else src_num_layers
+ exporter.rules = {"mtp_in_decoder_layers": num_mtp > 0}
return exporter
+@pytest.mark.parametrize(("export_mtp", "raises"), [(True, False), (False, True)])
+def test_verify_exported_keys_depth_pruned_with_decoder_mtp(tmp_path, export_mtp, raises):
+ """Pruned 3 -> 2 layers: source layer 2 is not required, and its MTP (layer 3) must land at 2."""
+ source, export = tmp_path / "src", tmp_path / "exp"
+ _write_index(
+ source,
+ [f"model.layers.{i}.input_layernorm.weight" for i in range(3)]
+ + ["model.layers.3.eh_proj.weight"],
+ )
+ exported = [f"model.layers.{i}.input_layernorm.weight" for i in range(2)]
+ _write_index(export, exported + (["model.layers.2.eh_proj.weight"] if export_mtp else []))
+ exporter = _make_exporter_for_key_check(num_layers=2, src_num_layers=3, num_mtp=1)
+
+ if raises:
+ with pytest.raises(RuntimeError, match=r"model\.layers\.2\.eh_proj\.weight"):
+ exporter._verify_exported_keys(str(export), str(source))
+ else:
+ exporter._verify_exported_keys(str(export), str(source))
+
+
def _write_index(dir_path: Path, keys) -> None:
dir_path.mkdir(parents=True, exist_ok=True)
(dir_path / "model.safetensors.index.json").write_text(
diff --git a/tests/gpu_megatron/torch/quantization/plugins/test_megatron.py b/tests/gpu_megatron/torch/quantization/plugins/test_megatron.py
index 9a9d0e61977..e2702f7cf30 100644
--- a/tests/gpu_megatron/torch/quantization/plugins/test_megatron.py
+++ b/tests/gpu_megatron/torch/quantization/plugins/test_megatron.py
@@ -60,12 +60,14 @@
)
from megatron.core.tensor_parallel.layers import ColumnParallelLinear, RowParallelLinear
from megatron.core.transformer import MegatronModule, TransformerConfig
+from megatron.core.transformer.mlp import MLP
from megatron.core.transformer.moe.experts import SequentialMLP, TEGroupedMLP
from megatron.core.transformer.moe.router import TopKRouter
import modelopt
import modelopt.torch.opt as mto
import modelopt.torch.quantization as mtq
+from modelopt.torch.opt.dynamic import DynamicModule
from modelopt.torch.opt.plugins.mcore_dist_checkpointing import (
restore_sharded_modelopt_state,
save_sharded_modelopt_state,
@@ -2076,6 +2078,117 @@ def test_csa_indexer_quant(dist_workers_size_1, tmp_path):
dist_workers_size_1.run(partial(_test_csa_indexer_quant_helper, tmp_path))
+def test_registered_megatron_quant_modules_checkpoint_quantizer_state():
+ """Quantizer state rides in ``_extra_state``, which torch saves only for classes overriding it.
+
+ ``register_modelopt_extra_state_callbacks`` binds the hooks per instance, so a registered class
+ without a class-level ``get/set_extra_state`` silently drops its amax from checkpoints (as
+ DSAttention did). Containers whose child linears hold all quantizers are exempt.
+ """
+ containers = {MLP, SequentialMLP, TEGroupedMLP}
+ base = torch.nn.Module
+ missing = []
+ for cls, quant_cls in QuantModuleRegistry._registry.items():
+ if not issubclass(cls, MegatronModule) or cls in containers:
+ continue
+ if issubclass(cls, DynamicModule):
+ continue # e.g. LoRA wrappers: the runtime class also includes the wrapped Megatron layer
+ routes = all(
+ any(getattr(c, attr) is not getattr(base, attr) for c in (quant_cls, cls))
+ for attr in ("get_extra_state", "set_extra_state")
+ )
+ if not routes:
+ missing.append(cls.__name__)
+ assert not missing, f"quantizer state would be dropped from checkpoints for: {missing}"
+
+
+def _get_tiny_dsa_gpt_model():
+ """Tiny GPT with DSA sparse attention (AbsorbedMLASelfAttention + DSAttention + indexer)."""
+ # Local: DSA is optional; test_dsa_kv_cache_quant importorskips it before calling this.
+ from megatron.core.models.gpt.experimental_attention_variant_module_specs import (
+ get_transformer_block_with_experimental_attention_variant_spec,
+ )
+ from megatron.core.transformer.transformer_config import MLATransformerConfig
+
+ config = MLATransformerConfig(
+ num_layers=2,
+ hidden_size=64,
+ num_attention_heads=4,
+ ffn_hidden_size=128,
+ q_lora_rank=32,
+ kv_lora_rank=16,
+ qk_head_dim=16,
+ qk_pos_emb_head_dim=16,
+ v_head_dim=16,
+ experimental_attention_variant="dsa",
+ dsa_indexer_n_heads=2,
+ dsa_indexer_head_dim=32,
+ dsa_indexer_topk=8,
+ normalization="RMSNorm",
+ add_bias_linear=False,
+ hidden_dropout=0.0, # deterministic forward for the checkpoint round-trip comparison
+ attention_dropout=0.0,
+ bf16=True,
+ params_dtype=torch.bfloat16,
+ )
+ spec = get_transformer_block_with_experimental_attention_variant_spec(config)
+ return GPTModel(
+ config=config,
+ transformer_layer_spec=spec,
+ vocab_size=64,
+ max_sequence_length=64,
+ position_embedding_type="rope",
+ # Tied so a checkpoint round-trip restores the output layer too.
+ share_embeddings_and_output_weights=True,
+ ).cuda()
+
+
+def _test_dsa_kv_cache_quant_helper(tmp_path, rank, size):
+ # Local: DSA is optional; test_dsa_kv_cache_quant importorskips it before calling this.
+ from megatron.core.transformer.experimental_attention_variant.dsa import DSAttention
+
+ initialize_for_megatron(tensor_model_parallel_size=1, pipeline_model_parallel_size=1, seed=SEED)
+ model, model_test = _get_tiny_dsa_gpt_model(), _get_tiny_dsa_gpt_model()
+ forward = get_forward(model)
+ # Calibrated (not constant-amax) FP8 KV cache: the case that needs a real V amax to export.
+ kv_fp8_calibrated = {
+ "quant_cfg": [
+ {"quantizer_name": "*", "enable": False},
+ {"quantizer_name": "*[kv]_bmm_quantizer", "cfg": {"num_bits": (4, 3), "axis": None}},
+ ],
+ "algorithm": "max",
+ }
+ model = mtq.quantize(model, kv_fp8_calibrated, forward)
+
+ dsa_modules = [m for m in model.modules() if isinstance(m, DSAttention)]
+ assert dsa_modules, "DSAttention was not converted for KV-cache quantization"
+ for module in dsa_modules:
+ assert module.k_bmm_quantizer.is_enabled and module.v_bmm_quantizer.is_enabled
+ # Absorbed MLA passes value=None; V is calibrated on the shared KV latent like K, so a
+ # calibrated FP8 KV cache still exports a v_scale.
+ assert module.k_bmm_quantizer.amax is not None
+ assert torch.equal(module.v_bmm_quantizer.amax, module.k_bmm_quantizer.amax)
+
+ # The KV quantizer state (incl. amax) survives a torch-dist round-trip alongside the indexer.
+ # DSA trains its indexer through a separate indexer loss, so the LM-loss backward in the helper
+ # gives it no gradient; exclude it from that check.
+ for name, param in model_test.named_parameters():
+ if ".indexer." in name:
+ param.requires_grad_(False)
+ sharded_state_dict_test_helper(tmp_path, model, model_test, forward)
+ # The helper compares every restored tensor (incl. K/V amax), but V is unused in the forward
+ # pass, so also check both quantizers came back enabled.
+ restored = [m for m in model_test.modules() if isinstance(m, DSAttention)]
+ assert len(restored) == len(dsa_modules)
+ assert all(m.k_bmm_quantizer.is_enabled and m.v_bmm_quantizer.is_enabled for m in restored)
+
+
+def test_dsa_kv_cache_quant(dist_workers_size_1, tmp_path):
+ """DSAttention gets calibrated K/V KV-cache quantizers that survive a checkpoint round-trip."""
+ pytest.importorskip("megatron.core.transformer.experimental_attention_variant.dsa")
+ dist_workers_size_1.run(partial(_test_dsa_kv_cache_quant_helper, tmp_path))
+
+
def _test_kv_cache_amax_sync_helper(config, rank, size, tensor_model_parallel_size=1):
"""Helper function for testing KV cache quantizer amax sync across distributed world."""
# Use rank in seed to produce different amax values across ranks