|
| 1 | +# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. |
| 2 | + |
| 3 | +import importlib |
| 4 | +from copy import deepcopy |
| 5 | +from pathlib import Path |
| 6 | +from types import SimpleNamespace |
| 7 | + |
| 8 | +import pytest |
| 9 | +import torch |
| 10 | +from transformers import AutoConfig |
| 11 | + |
| 12 | +from megatron.bridge.recipes.qwen.gb200 import qwen35 |
| 13 | + |
| 14 | + |
| 15 | +pytestmark = pytest.mark.unit |
| 16 | +_MXFP8 = qwen35.qwen35_text_35b_a3b_sft_long_context_16gpu_gb200_fp8mx_config |
| 17 | + |
| 18 | + |
| 19 | +@pytest.fixture |
| 20 | +def hf_config(monkeypatch): |
| 21 | + config = AutoConfig.for_model( |
| 22 | + "qwen3_5_moe", |
| 23 | + text_config={ |
| 24 | + "num_hidden_layers": 4, |
| 25 | + "hidden_size": 512, |
| 26 | + "num_attention_heads": 4, |
| 27 | + "num_key_value_heads": 4, |
| 28 | + "head_dim": 128, |
| 29 | + "max_position_embeddings": 262144, |
| 30 | + "num_experts": 32, |
| 31 | + "num_experts_per_tok": 2, |
| 32 | + "moe_intermediate_size": 128, |
| 33 | + "shared_expert_intermediate_size": 128, |
| 34 | + "linear_num_key_heads": 8, |
| 35 | + "linear_num_value_heads": 8, |
| 36 | + "linear_key_head_dim": 128, |
| 37 | + "linear_value_head_dim": 128, |
| 38 | + "tie_word_embeddings": True, |
| 39 | + }, |
| 40 | + tie_word_embeddings=False, |
| 41 | + ) |
| 42 | + calls = [] |
| 43 | + |
| 44 | + def load(path, **kwargs): |
| 45 | + calls.append((path, kwargs)) |
| 46 | + return config |
| 47 | + |
| 48 | + monkeypatch.setattr(qwen35.AutoConfig, "from_pretrained", load) |
| 49 | + return config, calls |
| 50 | + |
| 51 | + |
| 52 | +def test_text_long_context_validates_and_preserves_inputs(hf_config, monkeypatch): |
| 53 | + config, calls = hf_config |
| 54 | + original = deepcopy(config.to_dict()) |
| 55 | + training_module = importlib.import_module("megatron.bridge.training.config") |
| 56 | + monkeypatch.setattr(training_module, "get_world_size_safe", lambda: 16) |
| 57 | + monkeypatch.setattr( |
| 58 | + torch.cuda, "get_device_properties", lambda index: SimpleNamespace(major=10, name="NVIDIA GB200") |
| 59 | + ) |
| 60 | + |
| 61 | + cfg = _MXFP8() |
| 62 | + cfg.validate() |
| 63 | + |
| 64 | + assert config.to_dict() == original |
| 65 | + assert calls == [("Qwen/Qwen3.5-35B-A3B", {"revision": qwen35._QWEN35_35B_A3B_INSTRUCT_REVISION})] |
| 66 | + assert cfg.tokenizer.hf_tokenizer_kwargs["revision"] == calls[0][1]["revision"] |
| 67 | + assert cfg.model.share_embeddings_and_output_weights is False |
| 68 | + assert cfg.model.mtp_num_layers == 1 |
| 69 | + assert cfg.model.mtp_loss_scaling_factor == 0.1 |
| 70 | + assert cfg.model.context_parallel_size == 8 |
| 71 | + assert cfg.model.expert_model_parallel_size == 16 |
| 72 | + assert cfg.model.tensor_model_parallel_size == 1 |
| 73 | + assert cfg.model.sequence_parallel is False |
| 74 | + assert cfg.model.recompute_modules == ["gdn_norm_out", "moe"] |
| 75 | + assert cfg.dataset.hf_dataset.dataset_name == "coderforge" |
| 76 | + assert cfg.dataset.hf_dataset.load_kwargs == {"revision": qwen35._CODERFORGE_REVISION} |
| 77 | + assert cfg.dataset.dataset_root is None |
| 78 | + assert cfg.dataset.hf_validation_proportion == 0.05 |
| 79 | + assert cfg.dataset.offline_packing_specs.pad_seq_to_mult == 16 |
| 80 | + assert cfg.dataset.offline_packing_specs.packed_sequence_size == 131072 |
| 81 | + assert cfg.dataset.dataset_kwargs["pad_to_max_length"] is True |
| 82 | + assert cfg.dataset.seed == 1234 |
| 83 | + assert cfg.dataset.persistent_workers is True |
| 84 | + assert cfg.train.global_batch_size == 32 |
| 85 | + assert cfg.train.micro_batch_size == 1 |
| 86 | + assert cfg.train.train_iters == 500 |
| 87 | + assert cfg.scheduler.lr_warmup_iters == 200 |
| 88 | + assert cfg.scheduler.lr_decay_iters == 300000 |
| 89 | + assert cfg.ddp.grad_reduce_in_fp32 is True |
| 90 | + |
| 91 | + |
| 92 | +def test_verified_mxfp8_kernel_and_parameter_settings(hf_config): |
| 93 | + cfg = _MXFP8() |
| 94 | + assert cfg.mixed_precision.fp8_recipe == "mxfp8" |
| 95 | + assert cfg.mixed_precision.fp8_param_gather is True |
| 96 | + assert cfg.mixed_precision.reuse_grad_buf_for_mxfp8_param_ag is True |
| 97 | + assert cfg.checkpoint.load_main_params_from_ckpt is True |
| 98 | + assert cfg.checkpoint.load_optim is False |
| 99 | + assert cfg.checkpoint.load_rng is False |
| 100 | + assert cfg.model.moe_use_grouped_tensor is True |
| 101 | + assert cfg.model.use_transformer_engine_op_fuser is True |
| 102 | + assert cfg.model.moe_mlp_glu_interleave_size == 32 |
| 103 | + assert cfg.model.moe_single_grouped_weight is True |
| 104 | + assert cfg.model.moe_single_grouped_bias is False |
| 105 | + assert cfg.env_vars["NUM_OF_HYBRID_EP_RANKS_PER_NVLINK_DOMAIN"] == 16 |
| 106 | + assert cfg.env_vars["NVTE_CUTEDSL_FUSED_GROUPED_MLP"] == 1 |
| 107 | + assert cfg.env_vars["CUDNN_FE_GROUPED_GEMM_DYNAMIC_MNKL"] == 1 |
| 108 | + assert cfg.env_vars["NVTE_GROUPED_LINEAR_SINGLE_PARAM"] == 1 |
| 109 | + |
| 110 | + |
| 111 | +@pytest.mark.parametrize("seq_length", [0, -16, 15, 131073]) |
| 112 | +def test_invalid_sequence_length_fails_before_hub_access(hf_config, seq_length): |
| 113 | + _, calls = hf_config |
| 114 | + with pytest.raises(ValueError, match="positive and divisible by 16"): |
| 115 | + _MXFP8(seq_length) |
| 116 | + assert calls == [] |
| 117 | + |
| 118 | + |
| 119 | +def test_local_text_config_uses_matching_tokenizer(hf_config, monkeypatch): |
| 120 | + config, _ = hf_config |
| 121 | + monkeypatch.setattr(qwen35.AutoConfig, "from_pretrained", lambda *_args, **_kwargs: config.text_config) |
| 122 | + |
| 123 | + cfg = _MXFP8(1024, hf_path="local-text-checkpoint", hf_revision=None) |
| 124 | + |
| 125 | + assert cfg.tokenizer.tokenizer_model == "local-text-checkpoint" |
| 126 | + assert cfg.tokenizer.hf_tokenizer_kwargs == {"revision": None} |
| 127 | + assert cfg.model.seq_length == 1024 |
| 128 | + assert cfg.dataset.offline_packing_specs.packed_sequence_size == 1024 |
| 129 | + |
| 130 | + |
| 131 | +def test_wrong_model_type_is_rejected(hf_config): |
| 132 | + config, _ = hf_config |
| 133 | + config.text_config.model_type = "qwen3_5_text" |
| 134 | + with pytest.raises(ValueError, match="Expected a Qwen3.5 MoE"): |
| 135 | + _MXFP8() |
| 136 | + |
| 137 | + |
| 138 | +@pytest.mark.parametrize("seq_length", [16, 32768, 65536, 131072]) |
| 139 | +def test_aligned_context_lengths(hf_config, seq_length): |
| 140 | + cfg = _MXFP8(seq_length) |
| 141 | + assert cfg.model.seq_length == cfg.dataset.seq_length == seq_length |
| 142 | + assert cfg.dataset.offline_packing_specs.packed_sequence_size == seq_length |
| 143 | + |
| 144 | + |
| 145 | +def test_context_exceeding_model_limit_is_rejected(hf_config): |
| 146 | + config, _ = hf_config |
| 147 | + with pytest.raises(ValueError, match="max_position_embeddings"): |
| 148 | + _MXFP8(config.text_config.max_position_embeddings + 16) |
| 149 | + |
| 150 | + |
| 151 | +def test_public_runner_finds_unique_text_recipe(hf_config, monkeypatch): |
| 152 | + monkeypatch.syspath_prepend(str(Path(__file__).resolve().parents[3] / "scripts/training")) |
| 153 | + runner = importlib.import_module("recipe_runner") |
| 154 | + assert runner.find_library_recipe(_MXFP8.__name__) is _MXFP8 |
| 155 | + assert runner.find_benchmark_recipe(_MXFP8.__name__) is None |
| 156 | + cfg = runner.load_recipe(_MXFP8.__name__) |
| 157 | + assert cfg.model.mtp_num_layers == 1 |
| 158 | + assert cfg.dataset.hf_dataset.dataset_name == "coderforge" |
0 commit comments