Skip to content

Commit fe77986

Browse files
Quantize the DSA KV cache and export GLM-5 / GLM-5.2 from Megatron-Bridge
- DSAttention gets FP8 KV-cache quantizers (absorbed-MLA value=None calibrates V on the KV latent) with its quantizer state kept in Megatron checkpoints. - GlmMoeDsaForCausalLM export mapping: DeepSeek MLA/MoE rules plus the DSA indexer. The MTP layer, which Megatron-Bridge does not build, is copied dequantized from the source checkpoint (local or Hub, shards only) and excluded from quantization; key verification follows it. - Drop the HAS_TE guard in the Megatron quantization plugin (TE is always present there). Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com> Signed-off-by: Keval Morabia <28916987+kevalmorabia97@users.noreply.github.com>
1 parent 67a68f8 commit fe77986

8 files changed

Lines changed: 609 additions & 246 deletions

File tree

‎CHANGELOG.rst‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -39,6 +39,7 @@ Changelog
3939
- Add an end-to-end W4A4 NVFP4 PTQ and QAD tutorial for Qwen3.6-35B-A3B also covering evaluation and vLLM throughput benchmarking. See `examples/megatron_bridge/tutorials/Qwen3.6-35B-A3B/README.md <https://github.com/NVIDIA/Model-Optimizer/tree/main/examples/megatron_bridge/tutorials/Qwen3.6-35B-A3B/>`_ for details.
4040
- Add ``--mlflow <tracking-uri>`` to the ``examples/megatron_bridge`` scripts that write a checkpoint -- ``prune_minitron.py``, ``quantize.py``, ``distill.py``, ``export_quantized_megatron_to_hf.py`` and ``export_distilled_megatron_to_hf.py`` (MLflow's own ``MLFLOW_TRACKING_URI`` is honoured too). Each run records the invocation, its arguments as searchable params and its log -- a distillation records its training metrics instead of the rank-0 log -- and writes ``.experiment.json`` into the checkpoint it produced, so a pruning, a quantization, the distillation that refines its checkpoint and the export that deploys it can be traced to one another; uploading the checkpoints themselves stays off unless ``--mlflow_log_checkpoints`` is passed.
4141
- Add ``--ep_size`` to ``examples/megatron_bridge/export_quantized_megatron_to_hf.py`` so large MoE models with grouped-GEMM experts can be exported with their experts sharded across GPUs. Checkpoints built with ``--no_moe_grouped_gemm`` must still be exported at ``--ep_size 1``.
42+
- Add unified HF export of GLM-5 / GLM-5.2 (``glm_moe_dsa``) checkpoints quantized with ``examples/megatron_bridge``. The MTP layer, which Megatron-Bridge does not build for these models, is copied through unquantized from the source checkpoint. DSA sparse-attention models also gain FP8 KV-cache quantization on Megatron.
4243

4344
*Misc*
4445

‎modelopt/torch/export/plugins/mcore_common.py‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,7 @@
1818
from typing import Any
1919

2020
from .mcore_deepseek import deepseek_causal_lm_export, deepseek_causal_lm_import
21+
from .mcore_glm import glm_moe_dsa_causal_lm_export
2122
from .mcore_gptoss import gptoss_causal_lm_export, gptoss_causal_lm_import
2223
from .mcore_llama import (
2324
eagle3_deep_llama_causal_lm_export,
@@ -63,6 +64,7 @@
6364
"Qwen3VLForConditionalGeneration": qwen3vl_causal_lm_export,
6465
"Qwen3_5ForConditionalGeneration": qwen3_5_vl_causal_lm_export,
6566
"Qwen3_5MoeForConditionalGeneration": qwen3_5_vl_causal_lm_export,
67+
"GlmMoeDsaForCausalLM": glm_moe_dsa_causal_lm_export,
6668
}
6769

6870
# VLM architectures whose Megatron export covers the language model only: the vision tower is copied
Lines changed: 44 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,44 @@
1+
# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
2+
# SPDX-License-Identifier: Apache-2.0
3+
#
4+
# Licensed under the Apache License, Version 2.0 (the "License");
5+
# you may not use this file except in compliance with the License.
6+
# You may obtain a copy of the License at
7+
#
8+
# http://www.apache.org/licenses/LICENSE-2.0
9+
#
10+
# Unless required by applicable law or agreed to in writing, software
11+
# distributed under the License is distributed on an "AS IS" BASIS,
12+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
# See the License for the specific language governing permissions and
14+
# limitations under the License.
15+
16+
"""Custom mappings from Megatron Core models to GLM-5.x Hugging Face models.
17+
18+
GLM-5 / GLM-5.2 (``glm_moe_dsa``) is DeepSeek-V3-style MLA with a DSA indexer. It keeps the MTP
19+
layer at HF index ``num_hidden_layers`` under the decoder's names.
20+
"""
21+
22+
from .mcore_custom import (
23+
GroupedGatedMLPSlicing,
24+
GroupedMLPSlicing,
25+
NameRemapping,
26+
SelfAttentionScaling,
27+
)
28+
from .mcore_deepseek import deepseek_causal_lm_export
29+
30+
glm_moe_dsa_causal_lm_export: dict = {
31+
**deepseek_causal_lm_export,
32+
"mtp_in_decoder_layers": True,
33+
"core_attention": SelfAttentionScaling("model.layers.{}.self_attn."),
34+
"indexer.linear_wq_b": NameRemapping("model.layers.{}.self_attn.indexer.wq_b."),
35+
"indexer.linear_wk": NameRemapping("model.layers.{}.self_attn.indexer.wk."),
36+
"indexer.k_norm": NameRemapping("model.layers.{}.self_attn.indexer.k_norm."),
37+
"indexer.linear_weights_proj": NameRemapping("model.layers.{}.self_attn.indexer.weights_proj."),
38+
"experts.linear_fc1": GroupedGatedMLPSlicing("model.layers.{}.mlp.experts.{{}}"),
39+
"experts.linear_fc2": GroupedMLPSlicing("model.layers.{}.mlp.experts.{{}}.down_proj"),
40+
"mtp.enorm": NameRemapping("model.layers.{}.enorm."),
41+
"mtp.hnorm": NameRemapping("model.layers.{}.hnorm."),
42+
"mtp.eh_proj": NameRemapping("model.layers.{}.eh_proj."),
43+
"mtp.final_layernorm": NameRemapping("model.layers.{}.shared_head.norm."),
44+
}

‎modelopt/torch/export/unified_export_megatron.py‎

Lines changed: 86 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -29,7 +29,7 @@
2929

3030
import torch
3131
import torch.distributed
32-
from huggingface_hub import get_safetensors_metadata, hf_hub_download
32+
from huggingface_hub import get_safetensors_metadata, hf_hub_download, snapshot_download
3333
from huggingface_hub.errors import EntryNotFoundError
3434
from safetensors import safe_open
3535
from safetensors.torch import save_file
@@ -175,6 +175,7 @@ def __init__(
175175
self._hf_text_config = getattr(self._hf_config, "text_config", self._hf_config)
176176

177177
# Update hf_config
178+
self._src_num_hidden_layers = self._hf_text_config.num_hidden_layers
178179
self._hf_text_config.num_hidden_layers = language_model.config.num_layers
179180
self._hf_text_config.hidden_size = language_model.config.hidden_size
180181
# MLA's kv_channels is the V head dim, not HF's head_dim (e.g. glm5_next derives it from RoPE).
@@ -203,6 +204,7 @@ def __init__(
203204
del self._hf_config.quantization_config
204205
self.all_rules = self._populate_rule_book()
205206
self.rules = self.all_rules[self.arch]
207+
self.all_mcore_mappings = all_mcore_hf_export_mapping[self.arch]
206208
# The vision tower is copied through unquantized, so deployments must not treat it as such.
207209
self.exclude_modules = [
208210
prefix.removesuffix(".") + "*" for prefix in self.vision_passthrough_prefixes or ()
@@ -550,7 +552,11 @@ def _verify_exported_keys(self, save_directory, pretrained_model_name_or_path) -
550552
# Narrow on purpose: compare module prefixes, not tensor names, since a quantized source
551553
# carries extras with no export counterpart, and only inside decoder layers, whose naming
552554
# is stable. A dropped decoder module is the case that loads fine and produces garbage.
553-
num_layers = self.model.config.num_layers
555+
# HF decoder depth of this export: Megatron may build several physical layers per HF layer.
556+
hf_depth = self._hf_text_config.num_hidden_layers
557+
num_mtp = 0
558+
if self.rules.get("mtp_in_decoder_layers", False):
559+
num_mtp = getattr(self._hf_text_config, "num_nextn_predict_layers", 0) or 0
554560
# Ancestors too: an export may expand one source module into several (Qwen3.5 packs
555561
# routed experts; the quantized export writes them per expert). Expansion is not a drop.
556562
exported_modules = set()
@@ -562,12 +568,18 @@ def _verify_exported_keys(self, save_directory, pretrained_model_name_or_path) -
562568
break
563569
exported_modules.add(prefix)
564570
missing = set()
565-
for key in source - exported:
571+
for key in source:
566572
layer = re.search(r"\.layers\.(\d+)\.", key)
567573
if layer is None:
568574
continue # see the note above: decoder layers only
569-
if int(layer.group(1)) >= num_layers:
570-
continue # depth-pruned model: the source has layers this export does not
575+
if int(layer.group(1)) >= hf_depth:
576+
mtp_id = int(layer.group(1)) - self._src_num_hidden_layers
577+
if not 0 <= mtp_id < num_mtp:
578+
continue # depth-pruned model: the source has layers this export does not
579+
# An MTP stored as extra decoder layers follows the exported decoder layers.
580+
key = f"{key[: layer.start(1)]}{hf_depth + mtp_id}{key[layer.end(1) :]}"
581+
if key in exported:
582+
continue
571583
if key.rsplit(".", 1)[0] in exported_modules:
572584
continue # module is exported; this name is a source-side quantization artifact
573585
if "rotary_emb" in key:
@@ -693,6 +705,12 @@ def _get_transformer_layer_state_dict(self, layer, layer_id, is_mtp=False):
693705
layer.self_attention.linear_kv_up_proj, layer_id, is_mtp=is_mtp
694706
)
695707
self.rules["linear_proj"](layer.self_attention.linear_proj, layer_id, is_mtp=is_mtp)
708+
core_attention = getattr(layer.self_attention, "core_attention", None)
709+
if core_attention is not None and "core_attention" in self.rules:
710+
self.rules["core_attention"](core_attention, layer_id, is_mtp=is_mtp)
711+
indexer = getattr(core_attention, "indexer", None)
712+
if indexer is not None:
713+
self._get_dsa_indexer_state_dict(indexer, layer_id, is_mtp)
696714
elif "linear_attn" in self.rules and hasattr(layer.self_attention, "in_proj"):
697715
# GatedDeltaNet (Qwen3.5 linear attention): no q/k layernorm, no core_attention.
698716
self._get_gated_delta_net_state_dict(layer, layer_id, is_mtp=is_mtp)
@@ -857,6 +875,8 @@ def _copy_mtp_state_dict_from_pretrained(self) -> dict[str, torch.Tensor]:
857875
mtp_state_dict = {}
858876
if not self._hf_pretrained_model_name:
859877
return mtp_state_dict
878+
if self.rules.get("mtp_in_decoder_layers", False):
879+
return self._copy_decoder_mtp_layers_from_pretrained()
860880

861881
mtp_exists = False
862882

@@ -910,6 +930,53 @@ def _copy_mtp_state_dict_from_pretrained(self) -> dict[str, torch.Tensor]:
910930
self.exclude_modules.append("mtp*")
911931
return mtp_state_dict
912932

933+
def _copy_decoder_mtp_layers_from_pretrained(self) -> dict[str, torch.Tensor]:
934+
"""Copy MTP layers stored as extra decoder layers (GLM-5.x) from the source, dequantized.
935+
936+
Used when Megatron did not build the MTP (e.g. Megatron-Bridge's GLM-5 bridge); the copies
937+
stay BF16 and are excluded from quantization, like the released NVFP4 checkpoints.
938+
"""
939+
num_mtp = getattr(self._hf_text_config, "num_nextn_predict_layers", 0) or 0
940+
source = self._hf_pretrained_model_name
941+
if num_mtp == 0 or source is None:
942+
return {}
943+
layers_prefix = self.all_mcore_mappings["input_layernorm"].target_name_or_prefix
944+
layers_prefix = layers_prefix.split("{}")[0] # e.g. "model.layers."
945+
src_prefixes = [
946+
f"{layers_prefix}{self._src_num_hidden_layers + i}." for i in range(num_mtp)
947+
]
948+
if not os.path.isdir(source):
949+
source = self._download_hub_shards(str(source), tuple(src_prefixes))
950+
keys = _read_checkpoint_keys(source)
951+
mtp_state_dict = {}
952+
for i in range(num_mtp):
953+
src = src_prefixes[i]
954+
dst = f"{layers_prefix}{self._hf_text_config.num_hidden_layers + i}."
955+
for key in sorted(
956+
k for k in keys if k.startswith(src) and not k.endswith("_scale_inv")
957+
):
958+
mtp_state_dict[dst + key[len(src) :]] = get_safetensor(
959+
str(source), key, dequantize=True
960+
)
961+
self.exclude_modules.append(dst + "*")
962+
if mtp_state_dict:
963+
print(f"Copied {len(mtp_state_dict)} MTP tensors from {source}")
964+
return mtp_state_dict
965+
966+
@staticmethod
967+
def _download_hub_shards(repo_id: str, key_prefixes: tuple[str, ...]) -> str:
968+
"""Download only the Hub shards holding tensors under ``key_prefixes``; return the local dir."""
969+
try:
970+
index_file = hf_hub_download(repo_id, "model.safetensors.index.json")
971+
except EntryNotFoundError: # unsharded checkpoint
972+
return snapshot_download(repo_id, allow_patterns=["model.safetensors"])
973+
with open(index_file) as f:
974+
weight_map = json.load(f)["weight_map"]
975+
shards = sorted(
976+
{shard for key, shard in weight_map.items() if key.startswith(key_prefixes)}
977+
)
978+
return snapshot_download(repo_id, allow_patterns=["model.safetensors.index.json", *shards])
979+
913980
def _get_gated_delta_net_state_dict(self, layer, layer_id, is_mtp=False):
914981
"""Export a GatedDeltaNet (Qwen3.5 linear-attention) layer's ``self_attention``."""
915982
gdn = layer.self_attention
@@ -920,6 +987,20 @@ def _get_gated_delta_net_state_dict(self, layer, layer_id, is_mtp=False):
920987
self.rules["linear_attn.out_norm"](gdn.out_norm, layer_id, is_mtp=is_mtp)
921988
self.rules["linear_attn.out_proj"](gdn.out_proj, layer_id, is_mtp=is_mtp)
922989

990+
def _get_dsa_indexer_state_dict(self, indexer, layer_id, is_mtp=False):
991+
"""Export the DSA kpool indexer of a sparse MLA layer."""
992+
if "indexer.linear_wq_b" not in self.rules:
993+
raise NotImplementedError(f"No export rule for the DSA indexer of {self.arch}.")
994+
for name in ("linear_wq_b", "linear_wk", "k_norm", "linear_weights_proj"):
995+
self.rules[f"indexer.{name}"](getattr(indexer, name), layer_id, is_mtp=is_mtp)
996+
# KPool (GLM-5.3-Flash) only; plain DSA indexers (GLM-5.2) have no compression params.
997+
for name in ("index_kpool_compress_ape", "index_kpool_compress_gate"):
998+
param = getattr(indexer, name, None)
999+
if param is not None:
1000+
self.rules[f"indexer.{name}"](
1001+
param.detach().to(self.dtype), layer_id, is_mtp=is_mtp
1002+
)
1003+
9231004
def _get_mamba_layer_state_dict(self, layer, layer_id, is_mtp=False):
9241005
if not isinstance(layer.norm, IdentityOp):
9251006
self.rules["norm"](layer.norm, layer_id, is_mtp=is_mtp)

0 commit comments

Comments
 (0)