Skip to content

refactor(model): build Qwen2 and MiMo from builder-backed model configs #6379

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
kamran-nvidia wants to merge 3 commits into
base: main
Choose a base branch
from
Open

refactor(model): build Qwen2 and MiMo from builder-backed model configs #6379

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
c08dcf4
refactor(model): build Qwen2 from a builder-backed model config
kamran-nvidia Oct 9, 2026
098c1bf
refactor(model): build MiMo from a builder-backed model config
kamran-nvidia Oct 9, 2026
96e9e4d
refactor(model): opt the migrated families into builder-backed model …
kamran-nvidia Oct 14, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
17 changes: 8 additions & 9 deletions src/megatron/bridge/models/mimo/mimo_bridge.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,10 +12,11 @@
# See the License for the specific language governing permissions and
# limitations under the License.

from typing import Mapping
from typing import Any, Mapping

import torch
from megatron.core.models.gpt.gpt_model import GPTModel
from transformers import PretrainedConfig

from megatron.bridge.models.conversion.mapping_registry import MegatronMappingRegistry
from megatron.bridge.models.conversion.model_bridge import MegatronModelBridge, WeightConversionTask
Expand All @@ -31,20 +32,18 @@
class MimoBridge(Qwen2Bridge):
"""Megatron Bridge for MiMo Causal LM."""

def provider_bridge(self, hf_pretrained):
provider = super().provider_bridge(hf_pretrained)
hf_config = hf_pretrained.config
def hf_config_to_model_config_kwargs(self, hf_config: PretrainedConfig) -> dict[str, Any]:
"""Convert a Hugging Face MiMo config to builder config kwargs."""
config_kwargs = super().hf_config_to_model_config_kwargs(hf_config)

# MiMo follows Qwen2 attention behavior and adds MTP on top.
provider.qk_layernorm = False
provider.add_qkv_bias = True
config_kwargs.update(qk_layernorm=False, add_qkv_bias=True)

num_mtp_layers = getattr(hf_config, "num_nextn_predict_layers", 0)
if num_mtp_layers > 0:
provider.mtp_num_layers = num_mtp_layers
provider.mtp_loss_scaling_factor = 0.1
config_kwargs.update(mtp_num_layers=num_mtp_layers, mtp_loss_scaling_factor=0.1)

return provider
return config_kwargs

def mapping_registry(self) -> MegatronMappingRegistry:
mapping_list = list(super().mapping_registry().mappings)
Expand Down
41 changes: 28 additions & 13 deletions src/megatron/bridge/models/qwen/qwen2_bridge.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,9 +12,11 @@
# See the License for the specific language governing permissions and
# limitations under the License.

from typing import Any

import torch
from megatron.core.models.gpt.gpt_model import GPTModel
from transformers import Qwen2ForCausalLM
from transformers import PretrainedConfig, Qwen2ForCausalLM

from megatron.bridge.models.conversion.mapping_registry import MegatronMappingRegistry
from megatron.bridge.models.conversion.model_bridge import MegatronModelBridge
Expand All @@ -23,6 +25,7 @@
GatedMLPMapping,
QKVMapping,
)
from megatron.bridge.models.gpt.model_config import BridgeGPTModelConfig


@MegatronModelBridge.register_bridge(source=Qwen2ForCausalLM, target=GPTModel, model_type="qwen2")
Expand All @@ -35,26 +38,38 @@ class Qwen2Bridge(MegatronModelBridge):
configuration translation.

Qwen2 inherits CONFIG_MAPPING from MegatronModelBridge base class.
Model-specific settings are applied in provider_bridge.
Model-specific settings are applied in hf_config_to_model_config_kwargs.

Example:
>>> from megatron.bridge import AutoBridge
>>> bridge = AutoBridge.from_hf_pretrained("Qwen/Qwen2-7B")
>>> provider = bridge.to_megatron_provider()
>>> model_config = bridge.get_model_config()
"""

def provider_bridge(self, hf_pretrained):
"""Convert HuggingFace Qwen2 config to GPTModelProvider."""
provider = super().provider_bridge(hf_pretrained)
MODEL_CONFIG_CLASS = BridgeGPTModelConfig
USE_MODEL_CONFIG_FOR_CONVERSION = True

@cuichenx cuichenx Oct 14, 2026

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Tied embeddings with PP > 1 lose the conversion tie flag

Enabling the builder path here changes where share_embeddings_and_output_weights lives: it is on the outer BridgeGPTModelConfig, while the runtime model's .config is only the nested TransformerConfig. However, build_conversion_tasks() reads that nested config, and _share_embeddings_and_output_weights() defaults the missing field to False.

For a tied Qwen2/2.5 checkpoint at PP > 1, the last pipeline stage materializes output_layer.weight. Conversion therefore keeps that parameter and requests lm_head.weight, which is absent from a deduplicated tied HF checkpoint. The post-import embedding broadcast reads the same missing flag and is also skipped. The provider path retained the flag in .config. PP=1 avoids the duplicate output parameter, so TP2-only checks do not cover this case.

Could we make the shared conversion helpers resolve the tie flag from the runtime model or outer model config, and add a tied-checkpoint PP2 import/export regression that checks the output-stage weights and synchronization? This also affects HF initialization for training through load_hf_weights().

This finding is based on the pinned builder/conversion code and source-level probes; I have not run a distributed checkpoint reproduction.

@kamran-nvidia kamran-nvidia Oct 14, 2026

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Thanks, confirmed. A tied tiny Qwen2 at PP=2 through the builder path fails on this branch with ValueError: Hugging Face checkpoint is missing mapped parameter(s): output_layer.weight -> lm_head.weight. Tied Qwen3 fails the same way on current main, since #6365 moved Qwen3 conversion to the builder.

The shared fix is in #6460: _share_embeddings_and_output_weights() now takes the unwrapped model. It keeps reading model.config when the config declares the flag, so the provider path and VLM wrappers are unchanged, and otherwise falls back to the flag Megatron Core stores on the built language model. With it, tied Qwen2 at PP=2 has a last-stage output_layer.weight equal to the HF embed_tokens, and the export reproduces all 50 HF tensors exactly with no lm_head.weight. #6460 also adds a tied PP=2 builder-path functional test (Qwen3, L0). I'll rebase this PR onto it once it merges.


provider.normalization = "RMSNorm"
provider.gated_linear_unit = True
provider.add_bias_linear = False
provider.add_qkv_bias = True
provider.hidden_dropout = 0.0
provider.autocast_dtype = torch.bfloat16
def hf_config_to_model_config_kwargs(self, hf_config: PretrainedConfig) -> dict[str, Any]:
"""Convert a Hugging Face Qwen2 config to builder config kwargs."""
config_kwargs = super().hf_config_to_model_config_kwargs(hf_config)
config_kwargs.update(
normalization="RMSNorm",
gated_linear_unit=True,
add_bias_linear=False,
add_qkv_bias=True,
hidden_dropout=0.0,
autocast_dtype=torch.bfloat16,
masked_softmax_fusion=True,
rope_scaling=False,
rope_scaling_factor=1.0,
)
# The shared mapping already selects "yarn" for YaRN-scaled checkpoints.
config_kwargs.setdefault("position_embedding_type", "rope")
return config_kwargs

return provider
def hf_config_to_provider_kwargs(self, hf_config: PretrainedConfig) -> dict[str, Any]:
"""Adapt the canonical builder mapping to the deprecated provider path."""
return self.hf_config_to_model_config_kwargs(hf_config)

def mapping_registry(self) -> MegatronMappingRegistry:
# Return MegatronMappingRegistry containing parameter mappings from Megatron to HF format
Expand Down
60 changes: 30 additions & 30 deletions src/megatron/bridge/recipes/qwen/h100/qwen2.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -37,7 +37,7 @@ def qwen2_500m_pretrain_1gpu_h100_bf16_config() -> ConfigContainer:
cfg = _pretrain_common()

# Model config (--tensor-model-parallel-size, --pipeline-model-parallel-size, etc.)
cfg.model = AutoBridge.from_hf_pretrained("Qwen/Qwen2-0.5B").to_megatron_provider(load_weights=False)
cfg.model = AutoBridge.from_hf_pretrained("Qwen/Qwen2-0.5B").get_model_config()

# Tokenizer
# Qwen2 uses NullTokenizer by default for pretraining
Expand Down Expand Up @@ -128,7 +128,7 @@ def qwen2_1p5b_pretrain_1gpu_h100_bf16_config() -> ConfigContainer:
cfg = _pretrain_common()

# Model config (--tensor-model-parallel-size, --pipeline-model-parallel-size, etc.)
cfg.model = AutoBridge.from_hf_pretrained("Qwen/Qwen2-1.5B").to_megatron_provider(load_weights=False)
cfg.model = AutoBridge.from_hf_pretrained("Qwen/Qwen2-1.5B").get_model_config()

# Tokenizer
cfg.tokenizer.tokenizer_type = "NullTokenizer"
Expand Down Expand Up @@ -217,7 +217,7 @@ def qwen2_7b_pretrain_2gpu_h100_bf16_config() -> ConfigContainer:
"""
cfg = _pretrain_common()

cfg.model = AutoBridge.from_hf_pretrained("Qwen/Qwen2-7B").to_megatron_provider(load_weights=False)
cfg.model = AutoBridge.from_hf_pretrained("Qwen/Qwen2-7B").get_model_config()

# Tokenizer
cfg.tokenizer.tokenizer_type = "NullTokenizer"
Expand Down Expand Up @@ -307,7 +307,7 @@ def qwen2_72b_pretrain_32gpu_h100_bf16_config() -> ConfigContainer:
cfg = _pretrain_common()

# Model config
cfg.model = AutoBridge.from_hf_pretrained("Qwen/Qwen2-72B").to_megatron_provider(load_weights=False)
cfg.model = AutoBridge.from_hf_pretrained("Qwen/Qwen2-72B").get_model_config()

# Tokenizer
cfg.tokenizer.tokenizer_type = "NullTokenizer"
Expand Down Expand Up @@ -402,7 +402,7 @@ def qwen25_500m_pretrain_1gpu_h100_bf16_config() -> ConfigContainer:
cfg = _pretrain_common()

# Model config (--tensor-model-parallel-size, --pipeline-model-parallel-size, etc.)
cfg.model = AutoBridge.from_hf_pretrained("Qwen/Qwen2.5-0.5B").to_megatron_provider(load_weights=False)
cfg.model = AutoBridge.from_hf_pretrained("Qwen/Qwen2.5-0.5B").get_model_config()

# Tokenizer (--tokenizer-model)
cfg.tokenizer.tokenizer_type = "NullTokenizer"
Expand Down Expand Up @@ -487,7 +487,7 @@ def qwen25_1p5b_pretrain_1gpu_h100_bf16_config() -> ConfigContainer:
cfg = _pretrain_common()

# Model config
cfg.model = AutoBridge.from_hf_pretrained("Qwen/Qwen2.5-1.5B").to_megatron_provider(load_weights=False)
cfg.model = AutoBridge.from_hf_pretrained("Qwen/Qwen2.5-1.5B").get_model_config()

# Tokenizer
cfg.tokenizer.tokenizer_type = "NullTokenizer"
Expand Down Expand Up @@ -572,7 +572,7 @@ def qwen25_7b_pretrain_2gpu_h100_bf16_config() -> ConfigContainer:
cfg = _pretrain_common()

# Model config
cfg.model = AutoBridge.from_hf_pretrained("Qwen/Qwen2.5-7B").to_megatron_provider(load_weights=False)
cfg.model = AutoBridge.from_hf_pretrained("Qwen/Qwen2.5-7B").get_model_config()

# Tokenizer
cfg.tokenizer.tokenizer_type = "NullTokenizer"
Expand Down Expand Up @@ -657,7 +657,7 @@ def qwen25_14b_pretrain_4gpu_h100_bf16_config() -> ConfigContainer:
cfg = _pretrain_common()

# Model config
cfg.model = AutoBridge.from_hf_pretrained("Qwen/Qwen2.5-14B").to_megatron_provider(load_weights=False)
cfg.model = AutoBridge.from_hf_pretrained("Qwen/Qwen2.5-14B").get_model_config()

# Tokenizer (--tokenizer-model)
cfg.tokenizer.tokenizer_type = "NullTokenizer"
Expand Down Expand Up @@ -747,7 +747,7 @@ def qwen25_32b_pretrain_16gpu_h100_bf16_config() -> ConfigContainer:
cfg = _pretrain_common()

# Model config
cfg.model = AutoBridge.from_hf_pretrained("Qwen/Qwen2.5-32B").to_megatron_provider(load_weights=False)
cfg.model = AutoBridge.from_hf_pretrained("Qwen/Qwen2.5-32B").get_model_config()

# Tokenizer
cfg.tokenizer.tokenizer_type = "NullTokenizer"
Expand Down Expand Up @@ -832,7 +832,7 @@ def qwen25_72b_pretrain_32gpu_h100_bf16_config() -> ConfigContainer:
cfg = _pretrain_common()

# Model config
cfg.model = AutoBridge.from_hf_pretrained("Qwen/Qwen2.5-72B").to_megatron_provider(load_weights=False)
cfg.model = AutoBridge.from_hf_pretrained("Qwen/Qwen2.5-72B").get_model_config()

# Tokenizer
cfg.tokenizer.tokenizer_type = "NullTokenizer"
Expand Down Expand Up @@ -927,7 +927,7 @@ def qwen2_500m_sft_1gpu_h100_bf16_config() -> ConfigContainer:
cfg = _sft_common()

# Model config
cfg.model = AutoBridge.from_hf_pretrained("Qwen/Qwen2-0.5B").to_megatron_provider(load_weights=False)
cfg.model = AutoBridge.from_hf_pretrained("Qwen/Qwen2-0.5B").get_model_config()

# Tokenizer
cfg.tokenizer.tokenizer_model = "Qwen/Qwen2-0.5B"
Expand Down Expand Up @@ -1029,7 +1029,7 @@ def qwen2_1p5b_sft_1gpu_h100_bf16_config() -> ConfigContainer:
cfg = _sft_common()

# Model config
cfg.model = AutoBridge.from_hf_pretrained("Qwen/Qwen2-1.5B").to_megatron_provider(load_weights=False)
cfg.model = AutoBridge.from_hf_pretrained("Qwen/Qwen2-1.5B").get_model_config()

# Tokenizer
cfg.tokenizer.tokenizer_model = "Qwen/Qwen2-1.5B"
Expand Down Expand Up @@ -1131,7 +1131,7 @@ def qwen2_7b_sft_2gpu_h100_bf16_config() -> ConfigContainer:
cfg = _sft_common()

# Model config
cfg.model = AutoBridge.from_hf_pretrained("Qwen/Qwen2-7B").to_megatron_provider(load_weights=False)
cfg.model = AutoBridge.from_hf_pretrained("Qwen/Qwen2-7B").get_model_config()

# Tokenizer
cfg.tokenizer.tokenizer_model = "Qwen/Qwen2-7B"
Expand Down Expand Up @@ -1233,7 +1233,7 @@ def qwen2_72b_sft_32gpu_h100_bf16_config() -> ConfigContainer:
cfg = _sft_common()

# Model config
cfg.model = AutoBridge.from_hf_pretrained("Qwen/Qwen2-72B").to_megatron_provider(load_weights=False)
cfg.model = AutoBridge.from_hf_pretrained("Qwen/Qwen2-72B").get_model_config()

# Tokenizer
cfg.tokenizer.tokenizer_model = "Qwen/Qwen2-72B"
Expand Down Expand Up @@ -1340,7 +1340,7 @@ def qwen25_500m_sft_1gpu_h100_bf16_config() -> ConfigContainer:
cfg = _sft_common()

# Model config
cfg.model = AutoBridge.from_hf_pretrained("Qwen/Qwen2.5-0.5B").to_megatron_provider(load_weights=False)
cfg.model = AutoBridge.from_hf_pretrained("Qwen/Qwen2.5-0.5B").get_model_config()

# Tokenizer
cfg.tokenizer.tokenizer_model = "Qwen/Qwen2.5-0.5B"
Expand Down Expand Up @@ -1442,7 +1442,7 @@ def qwen25_1p5b_sft_1gpu_h100_bf16_config() -> ConfigContainer:
cfg = _sft_common()

# Model config
cfg.model = AutoBridge.from_hf_pretrained("Qwen/Qwen2.5-1.5B").to_megatron_provider(load_weights=False)
cfg.model = AutoBridge.from_hf_pretrained("Qwen/Qwen2.5-1.5B").get_model_config()

# Tokenizer
cfg.tokenizer.tokenizer_model = "Qwen/Qwen2.5-1.5B"
Expand Down Expand Up @@ -1544,7 +1544,7 @@ def qwen25_7b_sft_2gpu_h100_bf16_config() -> ConfigContainer:
cfg = _sft_common()

# Model config
cfg.model = AutoBridge.from_hf_pretrained("Qwen/Qwen2.5-7B").to_megatron_provider(load_weights=False)
cfg.model = AutoBridge.from_hf_pretrained("Qwen/Qwen2.5-7B").get_model_config()

# Tokenizer
cfg.tokenizer.tokenizer_model = "Qwen/Qwen2.5-7B"
Expand Down Expand Up @@ -1646,7 +1646,7 @@ def qwen25_14b_sft_4gpu_h100_bf16_config() -> ConfigContainer:
cfg = _sft_common()

# Model config
cfg.model = AutoBridge.from_hf_pretrained("Qwen/Qwen2.5-14B").to_megatron_provider(load_weights=False)
cfg.model = AutoBridge.from_hf_pretrained("Qwen/Qwen2.5-14B").get_model_config()

# Tokenizer
cfg.tokenizer.tokenizer_model = "Qwen/Qwen2.5-14B"
Expand Down Expand Up @@ -1748,7 +1748,7 @@ def qwen25_32b_sft_16gpu_h100_bf16_config() -> ConfigContainer:
cfg = _sft_common()

# Model config
cfg.model = AutoBridge.from_hf_pretrained("Qwen/Qwen2.5-32B").to_megatron_provider(load_weights=False)
cfg.model = AutoBridge.from_hf_pretrained("Qwen/Qwen2.5-32B").get_model_config()

# Tokenizer
cfg.tokenizer.tokenizer_model = "Qwen/Qwen2.5-32B"
Expand Down Expand Up @@ -1850,7 +1850,7 @@ def qwen25_72b_sft_32gpu_h100_bf16_config() -> ConfigContainer:
cfg = _sft_common()

# Model config
cfg.model = AutoBridge.from_hf_pretrained("Qwen/Qwen2.5-72B").to_megatron_provider(load_weights=False)
cfg.model = AutoBridge.from_hf_pretrained("Qwen/Qwen2.5-72B").get_model_config()

# Tokenizer
cfg.tokenizer.tokenizer_model = "Qwen/Qwen2.5-72B"
Expand Down Expand Up @@ -1960,7 +1960,7 @@ def qwen2_500m_peft_1gpu_h100_bf16_config(peft_scheme: str | PEFT = "lora") -> C
cfg = _peft_common()

# Model config
cfg.model = AutoBridge.from_hf_pretrained("Qwen/Qwen2-0.5B").to_megatron_provider(load_weights=False)
cfg.model = AutoBridge.from_hf_pretrained("Qwen/Qwen2-0.5B").get_model_config()

# Tokenizer
cfg.tokenizer.tokenizer_model = "Qwen/Qwen2-0.5B"
Expand Down Expand Up @@ -2067,7 +2067,7 @@ def qwen2_1p5b_peft_1gpu_h100_bf16_config(peft_scheme: str | PEFT = "lora") -> C
cfg = _peft_common()

# Model config
cfg.model = AutoBridge.from_hf_pretrained("Qwen/Qwen2-1.5B").to_megatron_provider(load_weights=False)
cfg.model = AutoBridge.from_hf_pretrained("Qwen/Qwen2-1.5B").get_model_config()

# Tokenizer
cfg.tokenizer.tokenizer_model = "Qwen/Qwen2-1.5B"
Expand Down Expand Up @@ -2174,7 +2174,7 @@ def qwen2_7b_peft_1gpu_h100_bf16_config(peft_scheme: str | PEFT = "lora") -> Con
cfg = _peft_common()

# Model config
cfg.model = AutoBridge.from_hf_pretrained("Qwen/Qwen2-7B").to_megatron_provider(load_weights=False)
cfg.model = AutoBridge.from_hf_pretrained("Qwen/Qwen2-7B").get_model_config()

# Tokenizer
cfg.tokenizer.tokenizer_model = "Qwen/Qwen2-7B"
Expand Down Expand Up @@ -2281,7 +2281,7 @@ def qwen2_72b_peft_8gpu_h100_bf16_config(peft_scheme: str | PEFT = "lora") -> Co
cfg = _peft_common()

# Model config
cfg.model = AutoBridge.from_hf_pretrained("Qwen/Qwen2-72B").to_megatron_provider(load_weights=False)
cfg.model = AutoBridge.from_hf_pretrained("Qwen/Qwen2-72B").get_model_config()

# Tokenizer
cfg.tokenizer.tokenizer_model = "Qwen/Qwen2-72B"
Expand Down Expand Up @@ -2393,7 +2393,7 @@ def qwen25_500m_peft_1gpu_h100_bf16_config(peft_scheme: str | PEFT = "lora") ->
cfg = _peft_common()

# Model config
cfg.model = AutoBridge.from_hf_pretrained("Qwen/Qwen2.5-0.5B").to_megatron_provider(load_weights=False)
cfg.model = AutoBridge.from_hf_pretrained("Qwen/Qwen2.5-0.5B").get_model_config()

# Tokenizer
cfg.tokenizer.tokenizer_model = "Qwen/Qwen2.5-0.5B"
Expand Down Expand Up @@ -2500,7 +2500,7 @@ def qwen25_1p5b_peft_1gpu_h100_bf16_config(peft_scheme: str | PEFT = "lora") ->
cfg = _peft_common()

# Model config
cfg.model = AutoBridge.from_hf_pretrained("Qwen/Qwen2.5-1.5B").to_megatron_provider(load_weights=False)
cfg.model = AutoBridge.from_hf_pretrained("Qwen/Qwen2.5-1.5B").get_model_config()

# Tokenizer
cfg.tokenizer.tokenizer_model = "Qwen/Qwen2.5-1.5B"
Expand Down Expand Up @@ -2607,7 +2607,7 @@ def qwen25_7b_peft_1gpu_h100_bf16_config(peft_scheme: str | PEFT = "lora") -> Co
cfg = _peft_common()

# Model config
cfg.model = AutoBridge.from_hf_pretrained("Qwen/Qwen2.5-7B").to_megatron_provider(load_weights=False)
cfg.model = AutoBridge.from_hf_pretrained("Qwen/Qwen2.5-7B").get_model_config()

# Tokenizer
cfg.tokenizer.tokenizer_model = "Qwen/Qwen2.5-7B"
Expand Down Expand Up @@ -2714,7 +2714,7 @@ def qwen25_14b_peft_1gpu_h100_bf16_config(peft_scheme: str | PEFT = "lora") -> C
cfg = _peft_common()

# Model config
cfg.model = AutoBridge.from_hf_pretrained("Qwen/Qwen2.5-14B").to_megatron_provider(load_weights=False)
cfg.model = AutoBridge.from_hf_pretrained("Qwen/Qwen2.5-14B").get_model_config()

# Tokenizer
cfg.tokenizer.tokenizer_model = "Qwen/Qwen2.5-14B"
Expand Down Expand Up @@ -2821,7 +2821,7 @@ def qwen25_32b_peft_8gpu_h100_bf16_config(peft_scheme: str | PEFT = "lora") -> C
cfg = _peft_common()

# Model config
cfg.model = AutoBridge.from_hf_pretrained("Qwen/Qwen2.5-32B").to_megatron_provider(load_weights=False)
cfg.model = AutoBridge.from_hf_pretrained("Qwen/Qwen2.5-32B").get_model_config()

# Tokenizer
cfg.tokenizer.tokenizer_model = "Qwen/Qwen2.5-32B"
Expand Down Expand Up @@ -2928,7 +2928,7 @@ def qwen25_72b_peft_8gpu_h100_bf16_config(peft_scheme: str | PEFT = "lora") -> C
cfg = _peft_common()

# Model config
cfg.model = AutoBridge.from_hf_pretrained("Qwen/Qwen2.5-72B").to_megatron_provider(load_weights=False)
cfg.model = AutoBridge.from_hf_pretrained("Qwen/Qwen2.5-72B").get_model_config()

# Tokenizer
cfg.tokenizer.tokenizer_model = "Qwen/Qwen2.5-72B"
Expand Down
Loading
Loading