From 10648a5647bb821d9f4a2597e30f1eca5eb37fe0 Mon Sep 17 00:00:00 2001 From: Chen Cui Date: Thu, 8 Oct 2026 23:37:43 -0700 Subject: [PATCH 1/5] refactor(model)!: remove retired model support Signed-off-by: Chen Cui --- .../ISSUE_TEMPLATE/model_support_request.yml | 2 +- README.md | 16 +- docs/bridge-guide.md | 2 +- docs/fern/versions/_nav_order.yml | 7 - docs/fern/versions/nightly.yml | 16 +- .../versions/nightly/pages/bridge-guide.mdx | 2 +- docs/fern/versions/nightly/pages/index.mdx | 10 +- .../pages/models/deepseek/deepseek-v2.mdx | 165 --- .../nightly/pages/models/deepseek/index.mdx | 1 - .../nightly/pages/models/gemma/gemma.mdx | 44 - .../nightly/pages/models/gemma/gemma2.mdx | 279 ----- .../nightly/pages/models/gemma/index.mdx | 2 - .../nightly/pages/models/llama/index.mdx | 1 - .../nightly/pages/models/llama/llama2.mdx | 47 - .../nightly/pages/models/mistral/index.mdx | 1 - .../pages/models/mistral/ministral3.mdx | 2 +- .../nightly/pages/models/mistral/mistral.mdx | 107 -- .../nightly/pages/models/nemotron/index.mdx | 4 +- .../pages/models/nemotron/llama-nemotron.mdx | 126 --- .../models/nemotron/nemotron-nano-v2-vl.mdx | 199 ---- .../pages/models/nemotron/nemotronh.mdx | 141 +-- .../nightly/pages/nemo2-migration-guide.mdx | 7 +- docs/models/deepseek/deepseek-v2.md | 166 --- docs/models/deepseek/index.md | 2 - docs/models/gemma/gemma.md | 44 - docs/models/gemma/gemma2.md | 279 ----- docs/models/gemma/index.md | 4 - docs/models/llama/index.md | 2 - docs/models/llama/llama2.md | 47 - docs/models/mistral/index.md | 2 - docs/models/mistral/ministral3.md | 2 +- docs/models/mistral/mistral.md | 109 -- docs/models/nemotron/index.md | 6 +- docs/models/nemotron/llama-nemotron.md | 126 --- docs/models/nemotron/nemotron-nano-v2-vl.md | 200 ---- docs/models/nemotron/nemotronh.md | 141 +-- docs/nemo2-migration-guide.md | 7 +- .../hf_to_megatron_generate_nemotron_vlm.py | 494 -------- .../llava/convert_hf_llama_to_megatron.py | 2 +- .../llava/verify_llama_conversion.py | 2 +- examples/models/mistral/mistral/README.md | 34 - examples/models/mistral/mistral/conversion.sh | 49 - examples/models/mistral/mistral/inference.sh | 55 - .../hf_to_megatron_generate_nemotron_omni.py | 2 +- .../nemotron_nano_v2_vl_override_example.yaml | 57 - .../conf/nemotron_nano_v2_vl_video.yaml | 58 - .../finetune_nemotron_nano_v2_vl.py | 203 ---- scripts/training/recipe_metadata.py | 1 - .../nemo-mbridge-mlm-bridge-training/SKILL.md | 1 - .../SKILL.md | 3 +- .../references/recipe-index.md | 6 - .../bridge/data/collators/registry.py | 4 - .../bridge/data/megatron_mimo/dataset.py | 2 +- .../bridge/data/megatron_mimo/hf_provider.py | 2 +- .../bridge/data/megatron_mimo/loaders.py | 2 +- .../data/megatron_mimo/mock_provider.py | 4 +- src/megatron/bridge/models/__init__.py | 18 - src/megatron/bridge/models/_deprecation.py | 54 +- .../bridge/models/common/te_layers.py | 2 +- .../bridge/models/conversion/auto_bridge.py | 18 +- .../bridge/models/deepseek/__init__.py | 2 - .../models/deepseek/deepseek_v2_bridge.py | 122 -- .../models/exaone/exaone4/exaone4_bridge.py | 14 +- .../models/exaone/exaone4/exaone4_provider.py | 2 +- src/megatron/bridge/models/gemma/__init__.py | 10 - .../bridge/models/gemma/gemma2_bridge.py | 135 --- .../bridge/models/gemma/gemma2_provider.py | 638 ----------- .../bridge/models/gemma/gemma_bridge.py | 121 -- .../bridge/models/gemma/gemma_provider.py | 80 -- src/megatron/bridge/models/gemma/modules.py | 18 - .../bridge/models/hf_pretrained/README.md | 2 +- .../bridge/models/hf_pretrained/causal_lm.py | 4 +- .../bridge/models/llama/llama_bridge.py | 2 + .../bridge/models/llama_nemotron/__init__.py | 24 - .../llama_nemotron/llama_nemotron_bridge.py | 181 --- .../llama_nemotron/llama_nemotron_provider.py | 80 -- .../bridge/models/mistral/mistral_bridge.py | 4 +- .../bridge/models/mistral/mistral_provider.py | 10 +- src/megatron/bridge/models/mla_provider.py | 4 +- .../bridge/models/nemotron/__init__.py | 20 - .../bridge/models/nemotron/nemotron_bridge.py | 112 -- .../modeling_nemotron_omni_llava.py | 4 +- .../nemotron_omni/nemotron_omni_bridge.py | 10 + .../bridge/models/nemotron_vl/__init__.py | 4 - .../models/nemotron_vl/data/__init__.py | 15 - .../models/nemotron_vl/data/collate_fn.py | 204 ---- .../nemotron_vl/modeling_nemotron_vl.py | 4 +- .../models/nemotron_vl/nemotron_vl_bridge.py | 44 +- .../nemotron_vl/nemotron_vl_provider.py | 188 ---- .../models/nemotronh/nemotron_h_bridge.py | 2 + src/megatron/bridge/recipes/__init__.py | 2 - .../bridge/recipes/deepseek/__init__.py | 11 +- .../bridge/recipes/deepseek/deepseek_v2.py | 30 - .../bridge/recipes/deepseek/h100/__init__.py | 3 - .../recipes/deepseek/h100/deepseek_v2.py | 272 ----- src/megatron/bridge/recipes/gemma/__init__.py | 22 - src/megatron/bridge/recipes/gemma/gemma2.py | 61 - .../bridge/recipes/gemma/h100/__init__.py | 10 - .../bridge/recipes/gemma/h100/gemma2.py | 1000 ----------------- src/megatron/bridge/recipes/llama/__init__.py | 6 - .../bridge/recipes/llama/h100/__init__.py | 2 - .../bridge/recipes/llama/h100/llama2.py | 130 --- src/megatron/bridge/recipes/llama/llama2.py | 26 - .../bridge/recipes/nemotron_vl/__init__.py | 24 - .../recipes/nemotron_vl/h100/__init__.py | 21 - .../nemotron_vl/h100/nemotron_nano_v2_vl.py | 288 ----- .../nemotron_vl/nemotron_nano_v2_vl.py | 35 - .../bridge/recipes/nemotronh/__init__.py | 16 - .../bridge/recipes/nemotronh/h100/__init__.py | 7 - .../nemotronh/h100/nemotron_nano_v2.py | 891 --------------- .../recipes/nemotronh/nemotron_nano_v2.py | 46 - src/megatron/bridge/training/config.py | 2 +- .../bridge/training/utils/flop_utils.py | 2 +- .../unit_tests/data/collators/test_imports.py | 1 - .../data/energon/test_hf_task_encoder.py | 76 -- .../models/autobridge_registration_check.py | 7 +- .../deepseek/test_deepseek_attention.py | 10 +- .../models/gemma/test_gemma2_te_attention.py | 473 -------- tests/unit_tests/models/gemma/test_modules.py | 33 +- .../models/llama/test_llama_bridge.py | 2 +- .../test_nemotron_omni_conversion.py | 13 +- .../nemotron_vl/test_nemotron_vl_provider.py | 62 - .../test_autobridge_registration_matrix.py | 7 +- .../test_logit_dtype_wrapper_propagation.py | 30 - .../models/test_model_deprecation.py | 145 ++- .../unit_tests/recipes/test_llama_recipes.py | 2 +- .../recipes/test_nemotronh_recipes.py | 70 -- tests/unit_tests/training/test_setup.py | 36 - tests/unit_tests/training/test_tokenizer.py | 2 +- .../training/utils/test_flop_utils.py | 4 +- 130 files changed, 268 insertions(+), 9063 deletions(-) delete mode 100644 docs/fern/versions/nightly/pages/models/deepseek/deepseek-v2.mdx delete mode 100644 docs/fern/versions/nightly/pages/models/gemma/gemma.mdx delete mode 100644 docs/fern/versions/nightly/pages/models/gemma/gemma2.mdx delete mode 100644 docs/fern/versions/nightly/pages/models/llama/llama2.mdx delete mode 100644 docs/fern/versions/nightly/pages/models/mistral/mistral.mdx delete mode 100644 docs/fern/versions/nightly/pages/models/nemotron/llama-nemotron.mdx delete mode 100644 docs/fern/versions/nightly/pages/models/nemotron/nemotron-nano-v2-vl.mdx delete mode 100644 docs/models/deepseek/deepseek-v2.md delete mode 100644 docs/models/gemma/gemma.md delete mode 100644 docs/models/gemma/gemma2.md delete mode 100644 docs/models/llama/llama2.md delete mode 100644 docs/models/mistral/mistral.md delete mode 100644 docs/models/nemotron/llama-nemotron.md delete mode 100644 docs/models/nemotron/nemotron-nano-v2-vl.md delete mode 100644 examples/conversion/hf_to_megatron_generate_nemotron_vlm.py delete mode 100644 examples/models/mistral/mistral/README.md delete mode 100755 examples/models/mistral/mistral/conversion.sh delete mode 100755 examples/models/mistral/mistral/inference.sh delete mode 100644 examples/models/nemotron/nemotron_vl/conf/nemotron_nano_v2_vl_override_example.yaml delete mode 100644 examples/models/nemotron/nemotron_vl/conf/nemotron_nano_v2_vl_video.yaml delete mode 100644 examples/models/nemotron/nemotron_vl/finetune_nemotron_nano_v2_vl.py delete mode 100644 src/megatron/bridge/models/deepseek/deepseek_v2_bridge.py delete mode 100644 src/megatron/bridge/models/gemma/gemma2_bridge.py delete mode 100644 src/megatron/bridge/models/gemma/gemma2_provider.py delete mode 100644 src/megatron/bridge/models/gemma/gemma_bridge.py delete mode 100644 src/megatron/bridge/models/gemma/gemma_provider.py delete mode 100644 src/megatron/bridge/models/llama_nemotron/__init__.py delete mode 100644 src/megatron/bridge/models/llama_nemotron/llama_nemotron_bridge.py delete mode 100644 src/megatron/bridge/models/llama_nemotron/llama_nemotron_provider.py delete mode 100644 src/megatron/bridge/models/nemotron/__init__.py delete mode 100644 src/megatron/bridge/models/nemotron/nemotron_bridge.py delete mode 100644 src/megatron/bridge/models/nemotron_vl/data/__init__.py delete mode 100644 src/megatron/bridge/models/nemotron_vl/data/collate_fn.py delete mode 100644 src/megatron/bridge/models/nemotron_vl/nemotron_vl_provider.py delete mode 100644 src/megatron/bridge/recipes/deepseek/deepseek_v2.py delete mode 100644 src/megatron/bridge/recipes/deepseek/h100/deepseek_v2.py delete mode 100644 src/megatron/bridge/recipes/gemma/gemma2.py delete mode 100644 src/megatron/bridge/recipes/gemma/h100/gemma2.py delete mode 100644 src/megatron/bridge/recipes/llama/h100/llama2.py delete mode 100644 src/megatron/bridge/recipes/llama/llama2.py delete mode 100644 src/megatron/bridge/recipes/nemotron_vl/__init__.py delete mode 100644 src/megatron/bridge/recipes/nemotron_vl/h100/__init__.py delete mode 100644 src/megatron/bridge/recipes/nemotron_vl/h100/nemotron_nano_v2_vl.py delete mode 100644 src/megatron/bridge/recipes/nemotron_vl/nemotron_nano_v2_vl.py delete mode 100644 src/megatron/bridge/recipes/nemotronh/h100/nemotron_nano_v2.py delete mode 100644 src/megatron/bridge/recipes/nemotronh/nemotron_nano_v2.py delete mode 100644 tests/unit_tests/models/gemma/test_gemma2_te_attention.py delete mode 100644 tests/unit_tests/models/nemotron_vl/test_nemotron_vl_provider.py diff --git a/.github/ISSUE_TEMPLATE/model_support_request.yml b/.github/ISSUE_TEMPLATE/model_support_request.yml index c83d289223..7ea355aae6 100644 --- a/.github/ISSUE_TEMPLATE/model_support_request.yml +++ b/.github/ISSUE_TEMPLATE/model_support_request.yml @@ -27,7 +27,7 @@ body: description: Pick the closest fit for the requested model. options: - Dense decoder-only (e.g. Llama, Qwen, Gemma, Mistral, Nemotron) - - MoE decoder-only (e.g. DeepSeek V2/V3, OLMoE, Qwen3-MoE, MiniMax-M2) + - MoE decoder-only (e.g. DeepSeek V3, OLMoE, Qwen3-MoE, MiniMax-M2) - VLM / multimodal (e.g. Qwen-VL, Gemma-VL, Nemotron-VL) - SSM / hybrid (e.g. Mamba, Nemotron-H) - Encoder-decoder (e.g. T5) diff --git a/README.md b/README.md index a03980cd15..0af2e5e920 100644 --- a/README.md +++ b/README.md @@ -221,34 +221,30 @@ For a deeper dive into conversion design and advanced usage, see the [models REA Megatron Bridge provides out-of-the-box bridges and training recipes for a wide range of models, built on top of base model architectures from [Megatron Core](https://github.com/NVIDIA/Megatron-LM/tree/main/megatron/core). Refer to the [models directory](https://github.com/NVIDIA-NeMo/Megatron-Bridge/tree/main/src/megatron/bridge/models) for the full list of model bridges. -> **Deprecation notice:** Gemma 1 (2B/7B), Gemma 2 (2B/9B/27B), Llama 2, -> Mistral 7B, Mistral Small 3 24B, Nemotron H v1 (4B/8B/47B/56B), and the -> legacy `NemotronForCausalLM` bridge previously documented for Nemotron-4 340B -> are no longer actively maintained or tested against current upstream -> checkpoints. They will be removed in Megatron Bridge 0.7.0. +> **Deprecation notice:** Nemotron H v1 (4B/8B/47B/56B) is no longer actively maintained and will be removed in Megatron Bridge 0.7.0. | Family | Supported variants | |----------------|--------------------| | [**BAGEL**](docs/models/bagel/bagel.md) | BAGEL-7B-MoT (requires the tested MCore `dev` revision and `bagel` extra) | | [**Bailing**](docs/models/bailing/index.md) | Ling 2.0 / Ling MoE V2 (Bailing) | | [**BERT**](docs/models/bert/index.md) | Megatron-Style BERT (`MegatronBertForMaskedLM`) | -| [**DeepSeek**](docs/models/deepseek/index.md) | DeepSeek V2 / V2 Lite (deprecated), DeepSeek V3, DeepSeek V4 / V4 Flash | +| [**DeepSeek**](docs/models/deepseek/index.md) | DeepSeek V3, DeepSeek V4 / V4 Flash | | [**Diffusion**](https://github.com/NVIDIA-NeMo/Megatron-Bridge/tree/main/src/megatron/bridge/diffusion/models) | FLUX, LLaDA 1.5, Nemotron-Labs Diffusion, WAN | | **Ernie** | [Ernie 4.5 MoE](https://github.com/NVIDIA-NeMo/Megatron-Bridge/tree/main/src/megatron/bridge/models/ernie), [Ernie 4.5 VL MoE](https://github.com/NVIDIA-NeMo/Megatron-Bridge/tree/main/src/megatron/bridge/models/ernie_vl) | | [**EXAONE**](docs/models/exaone/exaone.md) | EXAONE 4.0, EXAONE 4.5 VL, K-EXAONE MoE | | [**Falcon**](docs/models/falcon/index.md) | Falcon H1 | -| [**Gemma**](docs/models/gemma/index.md) | Gemma / Gemma 2 (deprecated), Gemma 3, Gemma 3-VL, Gemma 4 (26B-A4B MoE / 31B dense), Gemma 4-VL (26B-A4B MoE) | +| [**Gemma**](docs/models/gemma/index.md) | Gemma 3, Gemma 3-VL, Gemma 4 (26B-A4B MoE / 31B dense), Gemma 4-VL (26B-A4B MoE) | | [**GLM**](docs/models/glm/index.md) | GLM-4.5 / GLM-4.7 / GLM-4.7-Flash, GLM-4.5V, GLM-5 / GLM-5.1 / GLM-5.2 / [GLM-5.3](docs/models/glm/glm5-2.md) | | [**GPT-OSS**](docs/models/gpt_oss/index.md) | GPT-oss | | [**HY V3**](https://huggingface.co/tencent/Hy3-preview-Base) | Hy3 preview-Base (HF β†’ Megatron checkpoint conversion) | | [**Kimi**](docs/models/kimi/index.md) | Kimi K2, Kimi-K2.5-VL, Kimi K3 | -| [**Llama**](docs/models/llama/index.md) | Llama 2 (deprecated), Llama 3 / 3.1 / 3.2 / 3.3 | +| [**Llama**](docs/models/llama/index.md) | Llama 3 / 3.1 / 3.2 / 3.3 | | [**MiniMax**](docs/models/minimax/index.md) | MiniMax-M2 / M2.5 / M2.7, MiniMax-M3 | -| [**Mistral**](docs/models/mistral/index.md) | Mistral 7B / Small 3 24B (deprecated), Ministral 3 (3B/8B/14B) | +| [**Mistral**](docs/models/mistral/index.md) | Ministral 3 (3B/8B/14B) | | [**Xiaomi-MiMo**](docs/models/mimo/index.md) | Xiaomi-MiMo, MiMo-V2-Flash | | [**Moonlight**](docs/models/moonlight/index.md) | Moonlight | | [**Muse Glimmer**](docs/models/muse_glimmer/muse-glimmer.md) | Muse Glimmer 30B | -| [**Nemotron**](docs/models/nemotron/index.md) | Nemotron H v1 (deprecated), Nemotron Nano v2 (deprecated), Nemotron-3 Nano, Nemotron-3 Super, Nemotron-3 Ultra, Llama Nemotron (deprecated), Nemotron Nano v2 VL (deprecated), Nemotron-3 Nano Omni | +| [**Nemotron**](docs/models/nemotron/index.md) | Nemotron H v1 (deprecated), Nemotron-3 Nano, Nemotron-3 Super, Nemotron-3 Ultra, Nemotron-3 Nano Omni | | [**OLMoE**](docs/models/olmoe/index.md) | OLMoE | | [**Qwen**](docs/models/qwen/index.md) | Qwen2 / Qwen2.5, Qwen3, Qwen3-MoE, Qwen3 Next, Qwen3.5 (dense/MoE), Qwen2.5-VL, Qwen3-VL, Qwen3.5-VL, Qwen3.6-VL, Qwen2 Audio, Qwen2.5-Omni, Qwen3-Omni, Qwen3-ASR | | [**Sarvam**](docs/models/sarvam/index.md) | Sarvam | diff --git a/docs/bridge-guide.md b/docs/bridge-guide.md index 0c42cdab31..3ef27efc9c 100644 --- a/docs/bridge-guide.md +++ b/docs/bridge-guide.md @@ -63,7 +63,7 @@ from megatron.bridge import AutoBridge # Load with specific settings bridge = AutoBridge.from_hf_pretrained( - "meta-llama/Llama-2-7b-hf", + "meta-llama/Meta-Llama-3-8B", torch_dtype=torch.float16, device_map="auto", trust_remote_code=True, diff --git a/docs/fern/versions/_nav_order.yml b/docs/fern/versions/_nav_order.yml index 7e45e569f2..0abfe0192b 100644 --- a/docs/fern/versions/_nav_order.yml +++ b/docs/fern/versions/_nav_order.yml @@ -8,20 +8,15 @@ "./versions/nightly/pages/modelopt/quantization.mdx": 1 "./versions/nightly/pages/models/README.mdx": 1 "./versions/nightly/pages/models/bailing/ling-2.mdx": 1 -"./versions/nightly/pages/models/deepseek/deepseek-v2.mdx": 1 "./versions/nightly/pages/models/falcon/falcon-h1.mdx": 1 -"./versions/nightly/pages/models/gemma/gemma.mdx": 1 "./versions/nightly/pages/models/glm/glm45.mdx": 1 "./versions/nightly/pages/models/gpt_oss/gpt-oss-20b.mdx": 1 "./versions/nightly/pages/models/gpt_oss/gpt-oss-120b.mdx": 2 "./versions/nightly/pages/models/kimi/kimi-k2.mdx": 1 -"./versions/nightly/pages/models/llama/llama2.mdx": 1 "./versions/nightly/pages/models/mimo/mimo.mdx": 1 "./versions/nightly/pages/models/minimax/minimax-m2.mdx": 1 "./versions/nightly/pages/models/minimax/minimax-m3.mdx": 2 -"./versions/nightly/pages/models/mistral/mistral.mdx": 1 "./versions/nightly/pages/models/moonlight/moonlight.mdx": 1 -"./versions/nightly/pages/models/nemotron/llama-nemotron.mdx": 1 "./versions/nightly/pages/models/olmoe/olmoe.mdx": 1 "./versions/nightly/pages/models/qwen/qwen3-235b-a22b.mdx": 1 "./versions/nightly/pages/models/qwen/qwen3-30b-a3b.mdx": 2 @@ -37,7 +32,6 @@ "./versions/nightly/pages/modelopt/distillation.mdx": 2 "./versions/nightly/pages/models/bailing/index.mdx": 2 "./versions/nightly/pages/models/deepseek/deepseek-v3.mdx": 2 -"./versions/nightly/pages/models/gemma/gemma2.mdx": 2 "./versions/nightly/pages/models/glm/glm-45v.mdx": 2 "./versions/nightly/pages/models/kimi/kimi-k25-vl.mdx": 2 "./versions/nightly/pages/models/llama/llama3.mdx": 2 @@ -76,7 +70,6 @@ "./versions/nightly/pages/models/gemma/gemma4-vl.mdx": 5 "./versions/nightly/pages/models/gemma/index.mdx": 5 "./versions/nightly/pages/models/nemotron/nemotron3.5-lightning.mdx": 6 -"./versions/nightly/pages/models/nemotron/nemotron-nano-v2-vl.mdx": 7 "./versions/nightly/pages/models/qwen/qwen3-vl.mdx": 7 "./versions/nightly/pages/releases/known-issues.mdx": 5 "./versions/nightly/pages/training/optimizer-scheduler.mdx": 5 diff --git a/docs/fern/versions/nightly.yml b/docs/fern/versions/nightly.yml index d8d46934d2..f7b50d0278 100644 --- a/docs/fern/versions/nightly.yml +++ b/docs/fern/versions/nightly.yml @@ -46,8 +46,6 @@ navigation: path: ./nightly/pages/models/bailing/ling-2.mdx section: Bailing - contents: - - page: DeepSeek V2 - path: ./nightly/pages/models/deepseek/deepseek-v2.mdx - page: DeepSeek V3 path: ./nightly/pages/models/deepseek/deepseek-v3.mdx - page: DeepSeek V4 @@ -58,10 +56,6 @@ navigation: path: ./nightly/pages/models/falcon/falcon-h1.mdx section: Falcon - contents: - - page: Gemma - path: ./nightly/pages/models/gemma/gemma.mdx - - page: Gemma 2 - path: ./nightly/pages/models/gemma/gemma2.mdx - page: Gemma 3 path: ./nightly/pages/models/gemma/gemma3.mdx - page: Gemma 3 VL (Vision-Language) @@ -96,8 +90,6 @@ navigation: path: ./nightly/pages/models/kimi/kimi-k3.mdx section: Kimi - contents: - - page: Llama 2 - path: ./nightly/pages/models/llama/llama2.mdx - page: Llama 3 path: ./nightly/pages/models/llama/llama3.mdx section: Llama @@ -108,8 +100,6 @@ navigation: path: ./nightly/pages/models/minimax/minimax-m3.mdx section: MiniMax - contents: - - page: Mistral - path: ./nightly/pages/models/mistral/mistral.mdx - page: Ministral 3 path: ./nightly/pages/models/mistral/ministral3.mdx section: Mistral @@ -122,9 +112,7 @@ navigation: path: ./nightly/pages/models/moonlight/moonlight.mdx section: Moonlight - contents: - - page: Llama Nemotron - path: ./nightly/pages/models/nemotron/llama-nemotron.mdx - - page: Nemotron H and Nemotron Nano v2 + - page: Nemotron H path: ./nightly/pages/models/nemotron/nemotronh.mdx - page: Nemotron 3 Nano 30B-A3B path: ./nightly/pages/models/nemotron/nemotron3-nano.mdx @@ -140,8 +128,6 @@ navigation: path: ./nightly/pages/models/nemotron/nemotron3.5-super-vl-text-only.mdx - page: Nemotron 3 Ultra path: ./nightly/pages/models/nemotron/nemotron3-ultra.mdx - - page: Nemotron Nano V2 VL - path: ./nightly/pages/models/nemotron/nemotron-nano-v2-vl.mdx - page: Nemotron-3 Nano Omni path: ./nightly/pages/models/nemotron/nemotron-3-omni.mdx section: Nemotron diff --git a/docs/fern/versions/nightly/pages/bridge-guide.mdx b/docs/fern/versions/nightly/pages/bridge-guide.mdx index 180a996fc9..8ceb08a5cd 100644 --- a/docs/fern/versions/nightly/pages/bridge-guide.mdx +++ b/docs/fern/versions/nightly/pages/bridge-guide.mdx @@ -63,7 +63,7 @@ from megatron.bridge import AutoBridge # Load with specific settings bridge = AutoBridge.from_hf_pretrained( - "meta-llama/Llama-2-7b-hf", + "meta-llama/Meta-Llama-3-8B", torch_dtype=torch.float16, device_map="auto", trust_remote_code=True, diff --git a/docs/fern/versions/nightly/pages/index.mdx b/docs/fern/versions/nightly/pages/index.mdx index 450eec4263..3e93c20520 100644 --- a/docs/fern/versions/nightly/pages/index.mdx +++ b/docs/fern/versions/nightly/pages/index.mdx @@ -206,18 +206,18 @@ Megatron Bridge provides out-of-the-box bridges and training recipes for a wide | Family | Supported variants | |----------------|--------------------| | [**Bailing**](docs/models/bailing/index.md) | Ling 2.0 (Bailing) | -| [**DeepSeek**](docs/models/deepseek/index.md) | DeepSeek V2 / V2 Lite, DeepSeek V3, DeepSeek V4 | +| [**DeepSeek**](docs/models/deepseek/index.md) | DeepSeek V3, DeepSeek V4 | | [**Falcon**](docs/models/falcon/index.md) | Falcon H1 | -| [**Gemma**](docs/models/gemma/index.md) | Gemma / Gemma 2, Gemma 3, Gemma 3-VL, Gemma 4-VL (26B-A4B MoE) | +| [**Gemma**](docs/models/gemma/index.md) | Gemma 3, Gemma 3-VL, Gemma 4-VL (26B-A4B MoE) | | [**GLM**](docs/models/glm/index.md) | GLM-4.5 / 4.7 / 4.7-Flash, GLM-4.5V, GLM-5 / 5.1 | | [**GPT-OSS**](docs/models/gpt_oss/index.md) | GPT-oss | | [**Kimi**](docs/models/kimi/index.md) | Kimi K2, Kimi-K2.5-VL | -| [**Llama**](docs/models/llama/index.md) | Llama 2, Llama 3 / 3.1 / 3.2 / 3.3 | +| [**Llama**](docs/models/llama/index.md) | Llama 3 / 3.1 / 3.2 / 3.3 | | [**MiniMax**](docs/models/minimax/index.md) | MiniMax-M2 / M2.5 / M2.7 | -| [**Mistral**](docs/models/mistral/index.md) | Mistral, Ministral 3 (3B/8B/14B) | +| [**Mistral**](docs/models/mistral/index.md) | Ministral 3 (3B/8B/14B) | | [**Xiaomi-MiMo**](docs/models/mimo/index.md) | Xiaomi-MiMo | | [**Moonlight**](docs/models/moonlight/index.md) | Moonlight | -| [**Nemotron**](docs/models/nemotron/index.md) | Nemotron H, Nemotron Nano v2, Nemotron-3 Nano, Nemotron-3 Super, Llama Nemotron, Nemotron Nano v2 VL, Nemotron-3 Nano Omni | +| [**Nemotron**](docs/models/nemotron/index.md) | Nemotron H, Nemotron-3 Nano, Nemotron-3 Super, Nemotron-3 Nano Omni | | [**OLMoE**](docs/models/olmoe/index.md) | OLMoE | | [**Qwen**](docs/models/qwen/index.md) | Qwen2 / Qwen2.5, Qwen3, Qwen3-MoE, Qwen3 Next, Qwen2.5-VL, Qwen3-VL, Qwen3.5-VL, Qwen3.6-VL, Qwen2 Audio, Qwen2.5-Omni, Qwen3-Omni, Qwen3-ASR | | [**Sarvam**](docs/models/sarvam/index.md) | Sarvam | diff --git a/docs/fern/versions/nightly/pages/models/deepseek/deepseek-v2.mdx b/docs/fern/versions/nightly/pages/models/deepseek/deepseek-v2.mdx deleted file mode 100644 index b100d0896c..0000000000 --- a/docs/fern/versions/nightly/pages/models/deepseek/deepseek-v2.mdx +++ /dev/null @@ -1,165 +0,0 @@ -# DeepSeek V2 - -> **Deprecation notice:** DeepSeek V2 and DeepSeek V2 Lite support is no -> longer actively maintained or tested against current upstream checkpoints and -> will be removed in Megatron Bridge 0.7.0. - -[DeepSeek-V2](https://github.com/deepseek-ai/DeepSeek-V2) is a Mixture-of-Experts (MoE) language model that uses innovative Multi-head Latent Attention (MLA) for efficient inference and DeepSeekMoE architecture for economical training and inference. The model achieves performance comparable to GPT-4 while using significantly fewer activated parameters. More information is available in the companion paper ["DeepSeek-V2: A Strong, Economical, and Efficient Mixture-of-Experts Language Model"](https://arxiv.org/abs/2405.04434). - -DeepSeek V2 models are supported via the Bridge system with auto-detected configuration and weight mapping. - -## Available Models - -Megatron Bridge supports the following DeepSeek V2 model variants: - -- **DeepSeek-V2**: 236B parameters (21B activated per token) -- **DeepSeek-V2-Lite**: 16B parameters (2.4B activated per token) - -Both models support pretraining with expert parallelism for efficient MoE training. - -## Model Architecture Features - -- **Multi-head Latent Attention (MLA)**: Novel attention mechanism that reduces KV cache requirements -- **DeepSeekMoE**: Efficient MoE architecture with routed and shared experts -- **Expert Parallelism**: Distributes experts across GPUs for scalable training -- **RoPE Embeddings**: Rotary position embeddings for position encoding -- **128K Context Length**: Native support for long sequences (DeepSeek-V2) -- **Pre-normalization**: RMSNorm before each transformer sub-layer - -## Conversion with πŸ€— Hugging Face - -### Load HF β†’ Megatron - -```python -from megatron.bridge import AutoBridge - -# Example: DeepSeek-V2-Lite -bridge = AutoBridge.from_hf_pretrained("deepseek-ai/DeepSeek-V2-Lite", trust_remote_code=True) -provider = bridge.to_megatron_provider() - -# Optionally configure parallelism before instantiating the model -provider.tensor_model_parallel_size = 1 -provider.pipeline_model_parallel_size = 1 -provider.expert_model_parallel_size = 8 - -model = provider.provide_distributed_model(wrap_with_ddp=False) -``` - -### Import Checkpoint from HF - -```bash -./scripts/conversion/convert.sh import \ - --hf-model deepseek-ai/DeepSeek-V2-Lite \ - --megatron-path /checkpoints/deepseek_v2_lite_megatron \ - --trust-remote-code -``` - -### Export Megatron β†’ HF - -```python -from megatron.bridge import AutoBridge - -# Load the bridge from HF model ID -bridge = AutoBridge.from_hf_pretrained("deepseek-ai/DeepSeek-V2-Lite", trust_remote_code=True) - -# Export a trained Megatron checkpoint to HF format -bridge.export_ckpt( - megatron_path="/results/deepseek_v2_lite/checkpoints/iter_0000500", - hf_path="/exports/deepseek_v2_lite_hf", -) -``` - -### Run Inference on Converted Checkpoint - -```bash -uv run python examples/conversion/hf_to_megatron_generate_text.py \ - --hf_model_path deepseek-ai/DeepSeek-V2-Lite \ - --megatron_model_path /checkpoints/deepseek_v2_lite_megatron \ - --prompt "What is artificial intelligence?" \ - --max_new_tokens 100 \ - --ep 8 \ - --trust-remote-code -``` - -For more details, see [examples/conversion/hf_to_megatron_generate_text.py](https://github.com/NVIDIA-NeMo/Megatron-Bridge/blob/main/examples/conversion/hf_to_megatron_generate_text.py) - -## Recipes - -See: [bridge.recipes.deepseek.deepseek_v2](../../apidocs/bridge/bridge.recipes.deepseek.deepseek_v2.md) - -### Available Recipes - -- **Pretrain recipes**: - - `deepseek_v2_lite_pretrain_config`: Pre-training for DeepSeek-V2-Lite (16B parameters, 2.4B activated per token) - - `deepseek_v2_pretrain_config`: Pre-training for DeepSeek-V2 (236B parameters, 21B activated per token) - -### Parallelism Configurations - -| Model | TP | PP | EP | Total GPUs | Use Case | -|-------|----|----|----|-----------:|----------| -| **DeepSeek-V2-Lite** | 1 | 1 | 8 | 8 | Pre-training (single node) | -| **DeepSeek-V2** | 1 | 4 | 32 | 128 | Pre-training (16 nodes) | - -**Key Features**: -- **Expert Parallelism**: EP=8 (V2-Lite) or EP=32 (V2) for efficient MoE training -- **Selective Recomputation**: Enabled by default for memory optimization -- **Sequence Length**: Default 4096, V2 supports up to 128K tokens - -### Pre-training Examples - -#### DeepSeek-V2-Lite (16B) - -```python -from megatron.bridge.recipes.deepseek import deepseek_v2_lite_pretrain_config - -config = deepseek_v2_lite_pretrain_config() -config.dataset.blend = [["/path/to/dataset_text_document"], None] -config.checkpoint.save = "/results/deepseek_v2_lite/checkpoints" -config.checkpoint.load = "/results/deepseek_v2_lite/checkpoints" -config.logger.tensorboard_dir = "/results/deepseek_v2_lite/tb_logs" -config.train.train_iters = 500_000 -config.train.global_batch_size = 512 -config.model.seq_length = 4096 -config.dataset.seq_length = 4096 -# Uses TP=1, PP=1, EP=8 (8 GPUs) automatically -``` - -#### DeepSeek-V2 (236B) - -```python -from megatron.bridge.recipes.deepseek import deepseek_v2_pretrain_config - -config = deepseek_v2_pretrain_config() -config.dataset.blend = [["/path/to/dataset_text_document"], None] -config.checkpoint.save = "/results/deepseek_v2/checkpoints" -config.checkpoint.load = "/results/deepseek_v2/checkpoints" -config.logger.tensorboard_dir = "/results/deepseek_v2/tb_logs" -config.train.train_iters = 500_000 -config.train.global_batch_size = 512 -config.model.seq_length = 4096 -config.dataset.seq_length = 4096 -# Uses TP=1, PP=4, EP=32 (128 GPUs) automatically -``` - -### Finetuning Recipes - -Finetuning recipes for DeepSeek V2 models are not currently available. - -## Hugging Face Model Cards & References - -### Hugging Face Model Cards -- DeepSeek-V2: https://huggingface.co/deepseek-ai/DeepSeek-V2 -- DeepSeek-V2-Lite: https://huggingface.co/deepseek-ai/DeepSeek-V2-Lite -- DeepSeek-V2-Chat: https://huggingface.co/deepseek-ai/DeepSeek-V2-Chat -- DeepSeek-V2-Lite-Chat: https://huggingface.co/deepseek-ai/DeepSeek-V2-Lite-Chat - -### Technical Papers -- DeepSeek-V2: A Strong, Economical, and Efficient Mixture-of-Experts Language Model: [arXiv:2405.04434](https://arxiv.org/abs/2405.04434) - -### Additional Resources -- GitHub Repository: https://github.com/deepseek-ai/DeepSeek-V2 - -## Related Docs -- Recipe usage: [Recipe usage](../../recipe-usage.md) -- Customizing the training recipe configuration: [Configuration overview](../../training/config-container-overview.md) -- Training entry points: [Entry points](../../training/entry-points.md) diff --git a/docs/fern/versions/nightly/pages/models/deepseek/index.mdx b/docs/fern/versions/nightly/pages/models/deepseek/index.mdx index 56fab12f87..a1b03f9472 100644 --- a/docs/fern/versions/nightly/pages/models/deepseek/index.mdx +++ b/docs/fern/versions/nightly/pages/models/deepseek/index.mdx @@ -4,6 +4,5 @@ DeepSeek model documentation is organized by model variant. | Variant | Guide | |---------|-------| -| DeepSeek V2 | [deepseek-v2.md](deepseek-v2.md) | | DeepSeek V3 | [deepseek-v3.md](deepseek-v3.md) | | DeepSeek V4 | [deepseek-v4.md](deepseek-v4.md) | diff --git a/docs/fern/versions/nightly/pages/models/gemma/gemma.mdx b/docs/fern/versions/nightly/pages/models/gemma/gemma.mdx deleted file mode 100644 index 6260beacf9..0000000000 --- a/docs/fern/versions/nightly/pages/models/gemma/gemma.mdx +++ /dev/null @@ -1,44 +0,0 @@ -# Gemma - -> **Deprecation notice:** Gemma 1 support (2B and 7B) is no longer actively -> maintained or tested against current upstream checkpoints and will be removed -> in Megatron Bridge 0.7.0. - -[Gemma](https://huggingface.co/collections/google/gemma-release-65d5efbccdbb8c4202ec078b) is Google's original lightweight open model family. Megatron Bridge supports Gemma causal language models through the `GemmaBridge` implementation for the Hugging Face `GemmaForCausalLM` architecture. - -## Supported Variants - -Megatron Bridge supports Hugging Face Gemma checkpoints that use the `gemma` model type, including: - -- Gemma 2B: https://huggingface.co/google/gemma-2b -- Gemma 7B: https://huggingface.co/google/gemma-7b -- Gemma release collection: https://huggingface.co/collections/google/gemma-release-65d5efbccdbb8c4202ec078b - -## Architecture Notes - -- RMSNorm with zero-centered gamma. -- GeGLU-style gated MLPs. -- RoPE positional embeddings and flash attention backend. -- Shared input/output embedding weights. - -## Examples - -Gemma uses the common conversion and generation entry points: - -```bash -./scripts/conversion/convert.sh import \ - --hf-model google/gemma-2b \ - --megatron-path /checkpoints/gemma_2b_megatron -``` - -```bash -uv run python examples/conversion/hf_to_megatron_generate_text.py \ - --hf_model_path google/gemma-2b \ - --megatron_model_path /checkpoints/gemma_2b_megatron \ - --prompt "What is artificial intelligence?" -``` - -## Related Implementation - -- Bridge implementation: [`src/megatron/bridge/models/gemma/gemma_bridge.py`](https://github.com/NVIDIA-NeMo/Megatron-Bridge/blob/main/src/megatron/bridge/models/gemma/gemma_bridge.py) -- Conversion examples: [`examples/conversion`](https://github.com/NVIDIA-NeMo/Megatron-Bridge/tree/main/examples/conversion) diff --git a/docs/fern/versions/nightly/pages/models/gemma/gemma2.mdx b/docs/fern/versions/nightly/pages/models/gemma/gemma2.mdx deleted file mode 100644 index 17992ed0a2..0000000000 --- a/docs/fern/versions/nightly/pages/models/gemma/gemma2.mdx +++ /dev/null @@ -1,279 +0,0 @@ -# Gemma 2 - -> **Deprecation notice:** Gemma 2 support (2B, 9B, and 27B) is no longer -> actively maintained or tested against current upstream checkpoints and will -> be removed in Megatron Bridge 0.7.0. - -[Google's Gemma 2](https://huggingface.co/collections/google/gemma-2-release-667d6600fd5220e7b967f315) is a family of lightweight, open models built on the same research and technology used to create Gemini models. The Gemma 2 architecture builds on the transformer decoder framework with enhancements including pre-normalization with RMSNorm, GeGLU activations, Rotary Positional Embeddings (RoPE), attention logit softcapping, and sliding window attention. - -Gemma 2 models are designed for a wide range of text generation tasks and are available in multiple sizes to suit different computational budgets. - -Gemma family models are supported via the Bridge system with auto-detected configuration and weight mapping. - -## Available Models - -### Text-Only Models -- **Gemma 2 2B** (`google/gemma-2-2b`): Compact 2B parameter model optimized for efficiency - - 26 layers, 2304 hidden size - - 8 attention heads, 4 query groups (GQA) - - Sequence length: 8,192 tokens - - Ideal for single-GPU deployment - -- **Gemma 2 9B** (`google/gemma-2-9b`): Medium-scale 9B parameter model balancing performance and efficiency - - 42 layers, 3584 hidden size - - 16 attention heads, 8 query groups (GQA) - - Sequence length: 8,192 tokens - - Recommended: 4-8 GPUs with tensor parallelism - -- **Gemma 2 27B** (`google/gemma-2-27b`): Large-scale 27B parameter model for maximum performance - - 46 layers, 4608 hidden size - - 32 attention heads, 16 query groups (GQA) - - Sequence length: 8,192 tokens - - Recommended: 8-16 GPUs with tensor and pipeline parallelism - -All models support a sequence length of 8,192 tokens and use sliding window attention for efficient context processing. - -## Model Architecture Features - -Gemma 2 introduces several architectural innovations: - -- **Sliding Window Attention**: Local attention with a 4,096 token window for efficient long-context processing -- **Attention Logit Softcapping**: Caps attention logits at 50.0 to prevent extreme values -- **Final Logit Softcapping**: Caps final output logits at 30.0 for stable generation -- **GeGLU Activation**: Uses gated linear units with GELU activation for improved performance -- **RMSNorm**: Layer normalization without mean centering for faster computation -- **Rotary Embeddings**: RoPE with base frequency 10,000 -- **Query Pre-Attention Scaling**: Scales queries by 224 before attention computation - -## Conversion with πŸ€— Hugging Face - -### Load HF β†’ Megatron -```python -from megatron.bridge import AutoBridge - -# Example: Gemma 2 9B -bridge = AutoBridge.from_hf_pretrained("google/gemma-2-9b") -provider = bridge.to_megatron_provider() - -# Configure parallelism before instantiating the model -provider.tensor_model_parallel_size = 8 -provider.pipeline_model_parallel_size = 1 - -provider.finalize() -model = provider.provide_distributed_model(wrap_with_ddp=False) -``` - -### Import HF β†’ Megatron -To import the HF model to your desired Megatron path: -```bash -# Gemma 2 2B -./scripts/conversion/convert.sh import \ ---hf-model google/gemma-2-2b \ ---megatron-path /models/gemma-2-2b - -# Gemma 2 9B -./scripts/conversion/convert.sh import \ ---hf-model google/gemma-2-9b \ ---megatron-path /models/gemma-2-9b - -# Gemma 2 27B -./scripts/conversion/convert.sh import \ ---hf-model google/gemma-2-27b \ ---megatron-path /models/gemma-2-27b -``` - -### Export Megatron β†’ HF -```bash -# Gemma 2 9B example -./scripts/conversion/convert.sh export \ ---hf-model google/gemma-2-9b \ ---megatron-path /results/gemma2_9b/checkpoints/iter_00001000 \ ---hf-path ./gemma2-9b-hf-export -``` - -### Run Inference on Converted Checkpoint - -```bash -uv run python examples/conversion/hf_to_megatron_generate_text.py \ ---hf_model_path google/gemma-2-9b \ ---megatron_model_path /models/gemma-2-9b \ ---prompt "What is artificial intelligence?" \ ---max_new_tokens 100 -``` - -Note: -- `--megatron_model_path` is optional. If not specified, the script will convert the model and then run forward. - -## Pretrain and Finetune Recipes - -- See: [bridge.recipes.gemma](../../apidocs/bridge/bridge.recipes.gemma.md) -- Available recipes: - - **Pretraining:** - - `gemma2_2b_pretrain_config`: Pre-training configuration for Gemma 2 2B - - `gemma2_9b_pretrain_config`: Pre-training configuration for Gemma 2 9B - - `gemma2_27b_pretrain_config`: Pre-training configuration for Gemma 2 27B - - **SFT:** - - `gemma2_2b_sft_config`: Full SFT for Gemma 2 2B - - `gemma2_9b_sft_config`: Full SFT for Gemma 2 9B - - `gemma2_27b_sft_config`: Full SFT for Gemma 2 27B - - **PEFT** (LoRA, DoRA): - - `gemma2_2b_peft_config`: PEFT for Gemma 2 2B - - `gemma2_9b_peft_config`: PEFT for Gemma 2 9B - - `gemma2_27b_peft_config`: PEFT for Gemma 2 27B - -Before training, ensure the following environment variables are set: -1. `SAVE_DIR`: checkpoint and log saving directory -2. `HF_TOKEN`: to download models from HF Hub (if required) -3. `HF_HOME`: (optional) to avoid re-downloading models and datasets -4. `WANDB_API_KEY`: (optional) to enable WandB logging - -### Pretraining - -#### Gemma 2 2B -```python -from megatron.bridge.recipes.gemma import gemma2_2b_pretrain_config - -# Create and customize a pretraining configuration. -config = gemma2_2b_pretrain_config() -config.dataset.blend = [["path/to/data"], None] -config.train.train_iters = 100000 -config.train.global_batch_size = 32 -``` - -#### Gemma 2 9B -```python -from megatron.bridge.recipes.gemma import gemma2_9b_pretrain_config - -config = gemma2_9b_pretrain_config() -config.dataset.blend = [["path/to/data"], None] -config.train.train_iters = 100000 -config.train.global_batch_size = 32 -``` - -#### Gemma 2 27B -```python -from megatron.bridge.recipes.gemma import gemma2_27b_pretrain_config - -config = gemma2_27b_pretrain_config() -config.dataset.blend = [["path/to/data"], None] -config.train.train_iters = 100000 -config.train.global_batch_size = 32 -``` - -### Full Finetuning - -#### Gemma 2 2B -```bash -uv run python -m torch.distributed.run --nproc-per-node=8 scripts/training/run_recipe.py \ ---recipe gemma2_2b_sft_config \ ---mode sft \ -checkpoint.pretrained_checkpoint=/models/gemma-2-2b \ -train.global_batch_size=64 \ -train.train_iters=1000 \ -checkpoint.save=$SAVE_DIR/gemma2_2b_finetune -``` - -Or programmatically: -```python -from megatron.bridge.recipes.gemma import gemma2_2b_sft_config - -config = gemma2_2b_sft_config() -config.checkpoint.pretrained_checkpoint = "/models/gemma-2-2b" -config.train.train_iters = 1000 -config.train.global_batch_size = 64 -``` - -#### Gemma 2 9B -```bash -uv run python -m torch.distributed.run --nproc-per-node=8 scripts/training/run_recipe.py \ ---recipe gemma2_9b_sft_config \ ---mode sft \ -checkpoint.pretrained_checkpoint=/models/gemma-2-9b \ -train.global_batch_size=64 \ -train.train_iters=1000 \ -checkpoint.save=$SAVE_DIR/gemma2_9b_finetune -``` - -#### Gemma 2 27B -```bash -uv run python -m torch.distributed.run --nproc-per-node=16 scripts/training/run_recipe.py \ ---recipe gemma2_27b_sft_config \ ---mode sft \ -checkpoint.pretrained_checkpoint=/models/gemma-2-27b \ -train.global_batch_size=64 \ -train.train_iters=1000 \ -checkpoint.save=$SAVE_DIR/gemma2_27b_finetune -``` - -### Parameter-Efficient Finetuning (PEFT) with LoRA - -#### Gemma 2 2B -```bash -uv run python -m torch.distributed.run --nproc-per-node=8 scripts/training/run_recipe.py \ ---recipe gemma2_2b_peft_config \ ---mode lora \ -checkpoint.pretrained_checkpoint=/models/gemma-2-2b \ -train.global_batch_size=128 \ -checkpoint.save=$SAVE_DIR/gemma2_2b_lora -``` - -PEFT options: -- Use `--mode lora` for LoRA or `--mode dora` for DoRA. Use `gemma2_*_sft_config` with `--mode sft` for full finetuning. - -Or programmatically: -```python -from megatron.bridge.recipes.gemma import gemma2_2b_peft_config - -# LoRA finetuning -config = gemma2_2b_peft_config(peft_scheme="lora") # or "dora" -config.checkpoint.pretrained_checkpoint = "/models/gemma-2-2b" -config.train.train_iters = 1000 -config.train.global_batch_size = 128 -``` - -#### Gemma 2 9B LoRA -```python -from megatron.bridge.recipes.gemma import gemma2_9b_peft_config - -config = gemma2_9b_peft_config(peft_scheme="lora") -config.checkpoint.pretrained_checkpoint = "/models/gemma-2-9b" -config.train.train_iters = 1000 -config.train.global_batch_size = 128 -``` - -#### Gemma 2 27B LoRA -```python -from megatron.bridge.recipes.gemma import gemma2_27b_peft_config - -config = gemma2_27b_peft_config(peft_scheme="lora") -config.checkpoint.pretrained_checkpoint = "/models/gemma-2-27b" -config.train.train_iters = 1000 -config.train.global_batch_size = 128 -``` - -### Recommended Configurations - -| Model | Mode | TP | PP | Global Batch Size | Learning Rate | -|-------|------|----|----|-------------------|---------------| -| Gemma 2 2B | Full SFT | 1 | 1 | 64-128 | 5e-6 | -| Gemma 2 2B | LoRA/DoRA | 1 | 1 | 128-256 | 1e-4 | -| Gemma 2 9B | Full SFT | 4 | 1 | 64-128 | 5e-6 | -| Gemma 2 9B | LoRA/DoRA | 1 | 1 | 128-256 | 1e-4 | -| Gemma 2 27B | Full SFT | 8 | 2 | 64-128 | 5e-6 | -| Gemma 2 27B | LoRA/DoRA | 4 | 1 | 128-256 | 1e-4 | - -## Examples -- Checkpoint import/export: [scripts/conversion/convert.sh](https://github.com/NVIDIA-NeMo/Megatron-Bridge/blob/main/scripts/conversion/convert.sh) -- Generate text (HFβ†’Megatron): [examples/conversion/hf_to_megatron_generate_text.py](https://github.com/NVIDIA-NeMo/Megatron-Bridge/blob/main/examples/conversion/hf_to_megatron_generate_text.py) - -## Hugging Face Model Cards - -- Gemma 2 2B: https://huggingface.co/google/gemma-2-2b -- Gemma 2 9B: https://huggingface.co/google/gemma-2-9b -- Gemma 2 27B: https://huggingface.co/google/gemma-2-27b -- Gemma 2 Collection: https://huggingface.co/collections/google/gemma-2-release-667d6600fd5220e7b967f315 - -## Related Docs -- Recipe usage: [Recipe usage](../../recipe-usage.md) -- Customizing the training recipe configuration: [Configuration overview](../../training/config-container-overview.md) -- Training entry points: [Entry points](../../training/entry-points.md) diff --git a/docs/fern/versions/nightly/pages/models/gemma/index.mdx b/docs/fern/versions/nightly/pages/models/gemma/index.mdx index 0b6775da1a..04f5fbe32d 100644 --- a/docs/fern/versions/nightly/pages/models/gemma/index.mdx +++ b/docs/fern/versions/nightly/pages/models/gemma/index.mdx @@ -4,8 +4,6 @@ Gemma model documentation is organized by model variant. | Variant | Guide | |---------|-------| -| Gemma | [gemma.md](gemma.md) | -| Gemma 2 | [gemma2.md](gemma2.md) | | Gemma 3 | [gemma3.md](gemma3.md) | | Gemma 3 VL | [gemma3-vl.md](gemma3-vl.md) | | Gemma 4 VL | [gemma4-vl.md](gemma4-vl.md) | diff --git a/docs/fern/versions/nightly/pages/models/llama/index.mdx b/docs/fern/versions/nightly/pages/models/llama/index.mdx index b56276469d..02390176ce 100644 --- a/docs/fern/versions/nightly/pages/models/llama/index.mdx +++ b/docs/fern/versions/nightly/pages/models/llama/index.mdx @@ -4,7 +4,6 @@ Llama model documentation is organized by model variant. | Variant | Guide | |---------|-------| -| Llama 2 | [llama2.md](llama2.md) | | Llama 3 | [llama3.md](llama3.md) | ## Recipes diff --git a/docs/fern/versions/nightly/pages/models/llama/llama2.mdx b/docs/fern/versions/nightly/pages/models/llama/llama2.mdx deleted file mode 100644 index e5071fbdbe..0000000000 --- a/docs/fern/versions/nightly/pages/models/llama/llama2.mdx +++ /dev/null @@ -1,47 +0,0 @@ -# Llama 2 - -> **Deprecation notice:** Llama 2 support is no longer actively maintained or -> tested against current upstream checkpoints and will be removed in Megatron -> Bridge 0.7.0. The shared Llama bridge remains supported for Llama 3 variants. - -[Llama 2](https://huggingface.co/meta-llama/Llama-2-7b-hf) is Meta's decoder-only transformer model family. Megatron Bridge supports Llama 2 through the shared Llama bridge for the Hugging Face `LlamaForCausalLM` architecture. - -## Supported Variants - -The repository includes a pretraining recipe for: - -- Llama-2-7B: https://huggingface.co/meta-llama/Llama-2-7b-hf - -The same bridge can auto-detect compatible Llama 2 causal language model checkpoints. - -## Architecture Notes - -- RMSNorm with SwiGLU feed-forward layers. -- RoPE positional embeddings. -- Grouped Query Attention support follows the Hugging Face configuration. -- Uses the same `LlamaBridge` implementation as later Llama 3 variants. - -## Examples - -Llama 2 uses the common conversion and generation entry points: - -```bash -./scripts/conversion/convert.sh import \ - --hf-model meta-llama/Llama-2-7b-hf \ - --megatron-path /checkpoints/llama2_7b_megatron -``` - -```bash -uv run python examples/conversion/hf_to_megatron_generate_text.py \ - --hf_model_path meta-llama/Llama-2-7b-hf \ - --megatron_model_path /checkpoints/llama2_7b_megatron \ - --prompt "What is artificial intelligence?" -``` - -## Recipes - -- Llama 2 recipe: [`src/megatron/bridge/recipes/llama/llama2.py`](https://github.com/NVIDIA-NeMo/Megatron-Bridge/blob/main/src/megatron/bridge/recipes/llama/llama2.py) - -## Related Implementation - -- Bridge implementation: [`src/megatron/bridge/models/llama/llama_bridge.py`](https://github.com/NVIDIA-NeMo/Megatron-Bridge/blob/main/src/megatron/bridge/models/llama/llama_bridge.py) diff --git a/docs/fern/versions/nightly/pages/models/mistral/index.mdx b/docs/fern/versions/nightly/pages/models/mistral/index.mdx index 9ec3925c5e..0385dc7fc6 100644 --- a/docs/fern/versions/nightly/pages/models/mistral/index.mdx +++ b/docs/fern/versions/nightly/pages/models/mistral/index.mdx @@ -4,5 +4,4 @@ Mistral model documentation is organized by model variant, including Ministral. | Variant | Guide | |---------|-------| -| Mistral | [mistral.md](mistral.md) | | Ministral 3 | [ministral3.md](ministral3.md) | diff --git a/docs/fern/versions/nightly/pages/models/mistral/ministral3.mdx b/docs/fern/versions/nightly/pages/models/mistral/ministral3.mdx index 3c2d9a4f89..134cc18df6 100644 --- a/docs/fern/versions/nightly/pages/models/mistral/ministral3.mdx +++ b/docs/fern/versions/nightly/pages/models/mistral/ministral3.mdx @@ -64,7 +64,7 @@ For checkpoint conversion, inference, finetuning recipes, and step-by-step train - Ministral 3 14B Instruct: https://huggingface.co/mistralai/Ministral-3-14B-Instruct-2512 ## Related Docs -- Related Mistral variant: [Mistral](mistral.md) +- Mistral family: [Overview](index.md) - Recipe usage: [Recipe usage](../../recipe-usage.md) - Customizing the training recipe configuration: [Configuration overview](../../training/config-container-overview.md) - Training entry points: [Entry points](../../training/entry-points.md) diff --git a/docs/fern/versions/nightly/pages/models/mistral/mistral.mdx b/docs/fern/versions/nightly/pages/models/mistral/mistral.mdx deleted file mode 100644 index 83b059ef79..0000000000 --- a/docs/fern/versions/nightly/pages/models/mistral/mistral.mdx +++ /dev/null @@ -1,107 +0,0 @@ -# Mistral - -> **Deprecation notice:** Mistral 7B and Mistral Small 3 24B support is no -> longer actively maintained or tested against current upstream checkpoints and -> will be removed in Megatron Bridge 0.7.0. Ministral 3 is not deprecated. - -[Mistral AI](https://mistral.ai/) develops frontier large language models with a focus on efficiency and performance. The Mistral family includes both dense and Mixture-of-Experts architectures, featuring innovations like sliding window attention and efficient context handling. - -Mistral models are supported via the Bridge system with auto-detected configuration and weight mapping. - -## Available Models - -Megatron Bridge supports the following Mistral model variants: - -- **Mistral Small 3 (24B)**: 24B parameters with 128K context length -- **Mistral 7B**: 7B parameters, efficient baseline model -- **Mistral 7B Instruct**: Instruction-tuned variant - -Additional Mistral models (including MoE variants like Mixtral) may be supported through the standard conversion pipeline. - -## Model Architecture Features - -- **Sliding Window Attention**: Efficient attention mechanism for long sequences -- **Grouped Query Attention (GQA)**: Memory-efficient attention mechanism -- **Rotary Positional Embeddings (RoPE)**: Relative position encoding -- **SwiGLU Activation**: Gated linear units in the feedforward network -- **Extended Context**: Support for sequences up to 128K tokens (Mistral Small 3) -- **YaRN RoPE Scaling**: Advanced rope scaling for extended context lengths - -## Conversion with πŸ€— Hugging Face - -### Load HF β†’ Megatron - -```python -from megatron.bridge import AutoBridge - -# Example: Mistral Small 3 24B -bridge = AutoBridge.from_hf_pretrained("mistralai/Mistral-Small-24B-Base-2501") -provider = bridge.to_megatron_provider() - -# Optionally configure parallelism before instantiating the model -provider.tensor_model_parallel_size = 2 -provider.pipeline_model_parallel_size = 1 - -model = provider.provide_distributed_model(wrap_with_ddp=False) -``` - -### Import Checkpoint from HF - -```bash -./scripts/conversion/convert.sh import \ - --hf-model mistralai/Mistral-Small-24B-Base-2501 \ - --megatron-path /checkpoints/mistral_small_24b_megatron -``` - -### Export Megatron β†’ HF - -```python -from megatron.bridge import AutoBridge - -# Load the bridge from HF model ID -bridge = AutoBridge.from_hf_pretrained("mistralai/Mistral-Small-24B-Base-2501") - -# Export a trained Megatron checkpoint to HF format -bridge.export_ckpt( - megatron_path="/results/mistral_small_24b/checkpoints/iter_0000500", - hf_path="/exports/mistral_small_24b_hf", -) -``` - -### Run Inference on Converted Checkpoint - -```bash -uv run python examples/conversion/hf_to_megatron_generate_text.py \ - --hf_model_path mistralai/Mistral-Small-24B-Base-2501 \ - --megatron_model_path /checkpoints/mistral_small_24b_megatron \ - --prompt "What is artificial intelligence?" \ - --max_new_tokens 100 \ - --tp 2 -``` - -For more details, see [examples/conversion/hf_to_megatron_generate_text.py](https://github.com/NVIDIA-NeMo/Megatron-Bridge/blob/main/examples/conversion/hf_to_megatron_generate_text.py) - -## Recipes - -Training recipes for Mistral models are not currently available. The Bridge supports checkpoint conversion for inference and deployment use cases. - -## Hugging Face Model Cards & References - -### Hugging Face Model Cards -- Mistral Small 3 (24B): https://huggingface.co/mistralai/Mistral-Small-24B-Base-2501 -- Mistral Small 3 (24B) Instruct: https://huggingface.co/mistralai/Mistral-Small-24B-Instruct-2501 -- Mistral 7B v0.1: https://huggingface.co/mistralai/Mistral-7B-v0.1 -- Mistral 7B Instruct v0.2: https://huggingface.co/mistralai/Mistral-7B-Instruct-v0.2 - -### Technical Papers -- Mistral 7B: https://arxiv.org/abs/2310.06825 - -### Additional Resources -- Mistral AI Website: https://mistral.ai/ -- Mistral Documentation: https://docs.mistral.ai/ - -## Related Docs -- Related Mistral variant: [Ministral 3](ministral3.md) -- Recipe usage: [Recipe usage](../../recipe-usage.md) -- Customizing the training recipe configuration: [Configuration overview](../../training/config-container-overview.md) -- Training entry points: [Entry points](../../training/entry-points.md) diff --git a/docs/fern/versions/nightly/pages/models/nemotron/index.mdx b/docs/fern/versions/nightly/pages/models/nemotron/index.mdx index 9d2924111d..913e3436ee 100644 --- a/docs/fern/versions/nightly/pages/models/nemotron/index.mdx +++ b/docs/fern/versions/nightly/pages/models/nemotron/index.mdx @@ -4,8 +4,7 @@ Nemotron model documentation is organized by model variant. | Variant | Guide | |---------|-------| -| Llama Nemotron | [llama-nemotron.md](llama-nemotron.md) | -| Nemotron H and Nemotron Nano v2 | [nemotronh.md](nemotronh.md) | +| Nemotron H | [nemotronh.md](nemotronh.md) | | Nemotron-3 Nano 30B-A3B | [nemotron3-nano.md](nemotron3-nano.md) | | Nemotron-3 Nano 4B | [nemotron3-nano-4b.md](nemotron3-nano-4b.md) | | Nemotron-3 Super | [nemotron3-super.md](nemotron3-super.md) | @@ -13,7 +12,6 @@ Nemotron model documentation is organized by model variant. | Nemotron-3.5 Super VL | [nemotron3.5-super-vl.md](nemotron3.5-super-vl.md) | | Nemotron-3.5 Super Text-only | [nemotron3.5-super-vl-text-only.md](nemotron3.5-super-vl-text-only.md) | | Nemotron-3 Ultra | [nemotron3-ultra.md](nemotron3-ultra.md) | -| Nemotron Nano V2 VL | [nemotron-nano-v2-vl.md](nemotron-nano-v2-vl.md) | | Nemotron-3 Nano Omni | [nemotron-3-omni.md](nemotron-3-omni.md) | ## Examples diff --git a/docs/fern/versions/nightly/pages/models/nemotron/llama-nemotron.mdx b/docs/fern/versions/nightly/pages/models/nemotron/llama-nemotron.mdx deleted file mode 100644 index 5763d8a23b..0000000000 --- a/docs/fern/versions/nightly/pages/models/nemotron/llama-nemotron.mdx +++ /dev/null @@ -1,126 +0,0 @@ -# Llama Nemotron - -> **Deprecation notice:** Llama Nemotron support (Super 49B, Ultra 253B, 70B, -> Nano 8B, and Nano 4B) is no longer actively maintained or tested against -> current upstream checkpoints and will be removed in Megatron Bridge 0.7.0. - -[Llama Nemotron](https://huggingface.co/collections/nvidia/llama-nemotron) is NVIDIA's family of large language models derived from Meta's Llama architecture, post-trained for enhanced reasoning, human chat preferences, and agentic tasks such as RAG and tool calling. The models feature neural architecture search (NAS) optimizations for improved efficiency and accuracy trade-offs. - -Llama Nemotron models are supported via the Bridge system with auto-detected configuration and weight mapping. - -## Available Models - -Megatron Bridge supports the following Llama Nemotron model variants: - -- **Llama-3.3-Nemotron-Super-49B**: 49B parameters (NAS-optimized from 70B) -- **Llama-3.1-Nemotron-Ultra-253B**: 253B parameters (large-scale reasoning model) -- **Llama-3.1-Nemotron-70B**: 70B parameters (standard size) -- **Llama-3.1-Nemotron-Nano-8B**: 8B parameters (efficient variant) -- **Llama-3.1-Nemotron-Nano-4B**: 4B parameters (ultra-compact variant) - -All models are ready for commercial use and support context lengths up to 128K tokens. - -## Model Architecture Features - -- **Neural Architecture Search (NAS)**: Novel approach to reduce memory footprint while maintaining accuracy -- **Heterogeneous Blocks**: Non-standard and non-repetitive layer configurations for efficiency - - Skip attention in some blocks - - Variable FFN expansion/compression ratios between blocks -- **Multi-Phase Post-Training**: - - Supervised fine-tuning for Math, Code, Science, and Tool Calling - - Reward-aware Preference Optimization (RPO) for chat - - Reinforcement Learning with Verifiable Rewards (RLVR) for reasoning - - Iterative Direct Preference Optimization (DPO) for tool calling -- **Extended Context**: Native support for sequences up to 128K tokens -- **Commercial Ready**: Fully licensed for commercial deployment - -## Conversion with πŸ€— Hugging Face - -### Load HF β†’ Megatron - -```python -from megatron.bridge import AutoBridge - -# Example: Llama-3.3-Nemotron-Super-49B -bridge = AutoBridge.from_hf_pretrained( - "nvidia/Llama-3_3-Nemotron-Super-49B-v1_5", - trust_remote_code=True -) -provider = bridge.to_megatron_provider() - -# Optionally configure parallelism before instantiating the model -provider.tensor_model_parallel_size = 2 -provider.pipeline_model_parallel_size = 1 - -model = provider.provide_distributed_model(wrap_with_ddp=False) -``` - -**Note**: Heterogeneous Llama-Nemotron models (Super/Ultra) require `trust_remote_code=True` as they use custom `DeciLMForCausalLM` architecture. Homogeneous models (Nano/70B) use standard Llama architecture and don't require this flag. - -### Import Checkpoint from HF - -```bash -./scripts/conversion/convert.sh import \ - --hf-model nvidia/Llama-3_3-Nemotron-Super-49B-v1_5 \ - --megatron-path /checkpoints/llama_nemotron_super_49b_megatron \ - --trust-remote-code -``` - -### Export Megatron β†’ HF - -```python -from megatron.bridge import AutoBridge - -# Load the bridge from HF model ID -bridge = AutoBridge.from_hf_pretrained( - "nvidia/Llama-3_3-Nemotron-Super-49B-v1_5", - trust_remote_code=True -) - -# Export a trained Megatron checkpoint to HF format -bridge.export_ckpt( - megatron_path="/results/llama_nemotron_super_49b/checkpoints/iter_0000500", - hf_path="/exports/llama_nemotron_super_49b_hf", -) -``` - -### Run Inference on Converted Checkpoint - -```bash -uv run python examples/conversion/hf_to_megatron_generate_text.py \ - --hf_model_path nvidia/Llama-3_3-Nemotron-Super-49B-v1_5 \ - --megatron_model_path /checkpoints/llama_nemotron_super_49b_megatron \ - --prompt "What is artificial intelligence?" \ - --max_new_tokens 100 \ - --tp 2 \ - --trust-remote-code -``` - -For more details, see [examples/conversion/hf_to_megatron_generate_text.py](https://github.com/NVIDIA-NeMo/Megatron-Bridge/blob/main/examples/conversion/hf_to_megatron_generate_text.py) - -## Recipes - -Training recipes for Llama Nemotron models are not currently available. - -## Hugging Face Model Cards & References - -### Hugging Face Model Cards -- Llama Nemotron Collection: https://huggingface.co/collections/nvidia/llama-nemotron -- Llama-3.3-Nemotron-Super-49B-v1.5: https://huggingface.co/nvidia/Llama-3_3-Nemotron-Super-49B-v1_5 -- Llama-3.1-Nemotron-Ultra-253B-v1: https://huggingface.co/nvidia/Llama-3_1-Nemotron-Ultra-253B-v1 -- Llama-3.1-Nemotron-Nano-8B-v1: https://huggingface.co/nvidia/Llama-3.1-Nemotron-Nano-8B-v1 -- Llama-3.1-Nemotron-Nano-4B-v1.1: https://huggingface.co/nvidia/Llama-3.1-Nemotron-Nano-4B-v1.1 - -### Technical Papers -- Llama-Nemotron: Efficient Reasoning Models: [arXiv:2505.00949](https://arxiv.org/abs/2505.00949) -- Puzzle: Distillation-Based NAS for Inference-Optimized LLMs: [arXiv:2411.19146](https://arxiv.org/abs/2411.19146) -- Reward-aware Preference Optimization: [arXiv:2502.00203](https://arxiv.org/abs/2502.00203) - -### Additional Resources -- NVIDIA Build Platform: https://build.nvidia.com/ -- Llama Nemotron Post-Training Dataset: https://huggingface.co/nvidia/Llama-Nemotron-Post-Training-Dataset - -## Related Docs -- Recipe usage: [Recipe usage](../../recipe-usage.md) -- Customizing the training recipe configuration: [Configuration overview](../../training/config-container-overview.md) -- Training entry points: [Entry points](../../training/entry-points.md) diff --git a/docs/fern/versions/nightly/pages/models/nemotron/nemotron-nano-v2-vl.mdx b/docs/fern/versions/nightly/pages/models/nemotron/nemotron-nano-v2-vl.mdx deleted file mode 100644 index 4537d74691..0000000000 --- a/docs/fern/versions/nightly/pages/models/nemotron/nemotron-nano-v2-vl.mdx +++ /dev/null @@ -1,199 +0,0 @@ -# Nemotron Nano V2 VL - -> **Deprecation notice:** Nemotron Nano v2 VL 12B support is no longer actively -> maintained or tested against current upstream checkpoints and will be removed -> in Megatron Bridge 0.7.0. - -NVIDIA Nemotron Nano v2 VL is an open 12B multimodal reasoning model for document intelligence and video understanding. -It enables [AI assistants](https://www.nvidia.com/en-us/use-cases/ai-assistants) to extract, interpret, and act on -information across text, images, tables, and videos. This makes the model valuable for agents focused on data analysis, -document processing and visual understanding in applications like generating reports, curating videos, and dense -captioning for media asset management, and retrieval-augmented search. - -NeMo Megatron Bridge supports finetuning this model (including LoRA finetuning) on single-image, multi-image, and video -datasets. -The finetuned model can be converted back to the πŸ€— Hugging Face format for downstream evaluation. - - -Please use the custom container `nvcr.io/nvidia/nemo:25.09.nemotron_nano_v2_vl` when working with this model. - -Run all commands from `/opt/Megatron-Bridge` (e.g. `docker run -w /opt/Megatron-Bridge ...`) - - - - -We use the following environment variables throughout this page -- `HF_MODEL_PATH=nvidia/NVIDIA-Nemotron-Nano-12B-v2-VL-BF16` -- `MEGATRON_MODEL_PATH=/models/NVIDIA-Nemotron-Nano-12B-v2-VL-BF16` (feel free to set your own path) - -Unless explicitly stated, any megatron model path in the commands below should NOT contain the iteration number -`iter_xxxxxx`. For more details on checkpointing, please see -[here](../../training/checkpointing.md#checkpoint-contents) - - - -## Conversion with πŸ€— Hugging Face - -### Import HF β†’ Megatron -To import the HF model to your desired `$MEGATRON_MODEL_PATH`, run the following command. -```bash -./scripts/conversion/convert.sh import \ ---hf-model $HF_MODEL_PATH \ ---megatron-path $MEGATRON_MODEL_PATH \ ---trust-remote-code -``` - -### Export Megatron β†’ HF -You can export a trained model with the following command. -```bash -./scripts/conversion/convert.sh export \ ---hf-model $HF_MODEL_PATH \ ---megatron-path \ ---hf-path \ ---not-strict -``` - -Note: it is normal to see a warning that `vision_model.radio_model.input_conditioner.norm_mean` and `vision_model.radio_model.input_conditioner.norm_std` from source are not in the exported checkpoint. These two weights are not needed in the checkpoint. - -### Run In-Framework Inference on Converted Checkpoint -You can run a quick sanity check on the converted checkpoint with the following command. -```bash -uv run python examples/conversion/hf_to_megatron_generate_vlm.py \ ---hf_model_path $HF_MODEL_PATH \ ---megatron_model_path $MEGATRON_MODEL_PATH \ ---image_path \ ---prompt "Describe this image." \ ---max_new_tokens 100 \ ---use_llava_model -``` - -Note: -- `--megatron_model_path` is optional. If not specified, the script will convert the model and then run forward. If - specified, the script will just load the megatron model -- `--max_new_tokens` controls the number of tokens to generate. -- For inference with multiple images, pass in a comma-separated list, e.g. - `--image_path="/path/to/example1.jpeg,/path/to/example2.jpeg"`. - Use a suitable prompt, e.g. `--prompt="Describe the two images in detail."`. -- For inference with video, pass in video path instead, e.g. `--video_path="/path/to/demo.mp4"`. Use a suitable prompt, - e.g. `--prompt="Describe what you see."`. - -## Finetuning Recipes -Before training, ensure the following environment variables are set. -1. `SAVE_DIR`: to specify a checkpoint and log saving directory, used in the commands below. -2. `HF_TOKEN`: to download models from HF Hub. -3. `HF_HOME`: (optional) to avoid re-downloading models and datasets every time. -4. `WANDB_API_KEY`: (optional) to enable WandB logging. - -### Full Finetuning -Example usage for full parameter finetuning using the -[Raven dataset](https://huggingface.co/datasets/HuggingFaceM4/the_cauldron/viewer/raven): - -```bash -uv run python -m torch.distributed.run --nproc-per-node=8 examples/models/nemotron/nemotron_vl/finetune_nemotron_nano_v2_vl.py \ ---hf-model-path $HF_MODEL_PATH \ ---pretrained-checkpoint \ -dataset.source.dataset_name=raven \ -logger.wandb_project= \ -logger.wandb_save_dir=$SAVE_DIR \ -checkpoint.save=$SAVE_DIR/ -``` - -Note: -- The config file `examples/models/nemotron/nemotron_vl/conf/nemotron_nano_v2_vl_override_example.yaml` contains a list of arguments - that can be overridden in the command. For example, you can set `train.global_batch_size=<batch size>` in the command. -- To change to another built-in dataset, set its `dataset.source.dataset_name` preset. To replace the recipe preset with a custom native source from the CLI, set `dataset.source.dataset_name=null` plus `dataset.source.path_or_dataset=`; custom schemas also require an explicit adapter. See the dataset section below. -- After training, you can run inference with `hf_to_megatron_generate_vlm.py` by supplying the trained megatron checkpoint. - You can also export the trained checkpoint to Hugging Face format. -- This full finetuning recipe requires at least 4xH100 (80G) GPUs. - -### Parameter-Efficient Finetuning (PEFT) -Parameter-efficient finetuning (PEFT) using LoRA is supported. -LoRA can be independently applied to the vision model, vision projection, and language model. We support two commonly used -settings out of the box in the example script: -1. Apply LoRA to the language model, and fully finetune the vision model and projection (used when the visual - distribution is substantially different from pretrained.) - -```bash -uv run python -m torch.distributed.run --nproc-per-node=8 examples/models/nemotron/nemotron_vl/finetune_nemotron_nano_v2_vl.py \ ---hf-model-path $HF_MODEL_PATH \ ---pretrained-checkpoint $MEGATRON_MODEL_PATH \ ---lora-on-language-model \ -dataset.source.dataset_name=raven \ -logger.wandb_project= \ -logger.wandb_save_dir=$SAVE_DIR \ -checkpoint.save=$SAVE_DIR/ \ -model.freeze_language_model=True \ -model.freeze_vision_model=False \ -model.freeze_vision_projection=False -``` - -2. Apply LoRA to all linear layers in attention and MLP modules of the vision model, vision projection, and the language model. - -```bash -uv run python -m torch.distributed.run --nproc-per-node=8 examples/models/nemotron/nemotron_vl/finetune_nemotron_nano_v2_vl.py \ ---hf-model-path $HF_MODEL_PATH \ ---pretrained-checkpoint $MEGATRON_MODEL_PATH \ ---lora-on-language-model \ ---lora-on-vision-model \ -dataset.source.dataset_name=raven \ -logger.wandb_project= \ -logger.wandb_save_dir=$SAVE_DIR \ -checkpoint.save=$SAVE_DIR/ \ -model.freeze_language_model=True \ -model.freeze_vision_model=True \ -model.freeze_vision_projection=True -``` - -These LoRA finetuning recipe requires at least 2xH100 (80G) GPUs. - -A LoRA checkpoint only contains the learnable adapter weights. In order to convert the LoRA checkpoint to Hugging Face -format for downstream evaluation, it is necessary to merge the LoRA adapters back to the base model. - -```bash -uv run python examples/peft/merge_lora.py \ ---hf-model-path $HF_MODEL_PATH \ ---lora-checkpoint /iter_N \ ---output -``` -The output is a merged Hugging Face checkpoint for downstream evaluation. - -## Example Datasets - -Megatron Bridge supports various vision-language dataset examples which can be used to finetune Nemotron Nano V2 VL: -| Dataset | Dataset preset | Description | -|---------|------------|-------------| -| [cord-v2](https://huggingface.co/datasets/naver-clova-ix/cord-v2) | `cord_v2` | OCR receipts: Single-image-text dataset for receipt understanding, outputs xml-like annotated text. | -| [MedPix-VQA](https://huggingface.co/datasets/mmoukouba/MedPix-VQA) | `medpix` | Medical VQA: Single-image question-answer dataset covering clinical medical images and free-form answers. | -| [The Cauldron (Raven subset)](https://huggingface.co/datasets/HuggingFaceM4/the_cauldron) | `raven` | Visual reasoning: Multi-image, vision reasoning dataset for analogical reasoning in different visual layouts. | -| [LLaVA-Video-178K (0_30_s_nextqa subset)](https://huggingface.co/datasets/lmms-lab/LLaVA-Video-178K) | `llava_video_178k` | Video understanding: video question-answer dataset covering every-day scenarios. | - -`cord-v2` is a tiny dataset and is included for demonstration only. This dataset is not recommended for PEFT tuning for this model since the XML output format interacts with the special tokens, leading to unexpected results. - -Note on video training example: -- We provide a video config yaml file instead of the default config yaml file that overwrites a few commands. Please - pass in `--config-file "examples/models/nemotron/nemotron_vl/conf/nemotron_nano_v2_vl_video.yaml"`. -- The LLaVA video dataset requires manual download beforehand. Please place the downloaded and extracted video files - in a folder `VIDEO_ROOT` and pass it to the adapter with `dataset.source.adapter_kwargs={"video_root_path":$VIDEO_ROOT}`. - In the nextqa subset example, `VIDEO_ROOT` should look like - ``` - $VIDEO_ROOT/ - β”œβ”€β”€ NextQA/ - β”‚ └── NExTVideo/ - β”‚ └── 0000/ - β”‚ └── 2440175990.mp4 - β”‚ └── 0001/ - β”‚ └── ... - └── ... - ``` - -Full video training example command: -```bash -uv run python -m torch.distributed.run --nproc-per-node=8 examples/models/nemotron/nemotron_vl/finetune_nemotron_nano_v2_vl.py \ ---hf-model-path $HF_MODEL_PATH \ ---pretrained-checkpoint $MEGATRON_MODEL_PATH \ ---config-file "examples/models/nemotron/nemotron_vl/conf/nemotron_nano_v2_vl_video.yaml" \ -logger.wandb_project= \ -logger.wandb_save_dir=$SAVE_DIR \ -checkpoint.save=$SAVE_DIR/ \ -dataset.source.adapter_kwargs={"video_root_path":$VIDEO_ROOT} -``` diff --git a/docs/fern/versions/nightly/pages/models/nemotron/nemotronh.mdx b/docs/fern/versions/nightly/pages/models/nemotron/nemotronh.mdx index 645b87615a..5cfd1dcd41 100644 --- a/docs/fern/versions/nightly/pages/models/nemotron/nemotronh.mdx +++ b/docs/fern/versions/nightly/pages/models/nemotron/nemotronh.mdx @@ -1,13 +1,10 @@ -# Nemotron H and Nemotron Nano v2 +# Nemotron H -> **Deprecation notice:** Nemotron H v1 support (4B, 8B, 47B, and 56B) and -> Nemotron Nano v2 support (9B and 12B) are no longer actively maintained or -> tested against current upstream checkpoints and will be removed in Megatron -> Bridge 0.7.0. Nemotron 3 is not deprecated. +> **Deprecation notice:** Nemotron H v1 support (4B, 8B, 47B, and 56B) is no longer actively maintained or tested against current upstream checkpoints and will be removed in Megatron Bridge 0.7.0. Nemotron 3 is not deprecated. -[Nemotron H](https://huggingface.co/collections/nvidia/nemotron-h) and [Nemotron Nano v2](https://huggingface.co/collections/nvidia/nvidia-nemotron-v2) are families of **hybrid SSM-Attention models** from **NVIDIA** that combine Mamba (State Space Model) layers with traditional attention layers. These models achieve strong performance while maintaining computational efficiency through their hybrid architecture. +[Nemotron H](https://huggingface.co/collections/nvidia/nemotron-h) is a family of hybrid SSM-Attention models combining Mamba and attention layers. -The Nemotron H family includes models from 4B to 56B parameters with 8K context length, while Nemotron Nano v2 models (9B and 12B) are optimized for edge deployment with extended 128K context support. +The Nemotron H family includes models from 4B to 56B parameters with 8K context length. ## Model Families @@ -17,12 +14,6 @@ The Nemotron H family includes models from 4B to 56B parameters with 8K context - **47B**: 98 layers, 8192 hidden size, 8K context - **56B**: 118 layers, 8192 hidden size, 8K context -### Nemotron Nano v2 -- **9B**: 56 layers, 4480 hidden size, 128K context -- **12B**: 62 layers, 5120 hidden size, 128K context - -All models are supported via the Bridge system with specialized configurations for hybrid SSM-Attention architecture. - ## Model Architecture ### Common Features Across All Models @@ -82,32 +73,6 @@ All models are supported via the Bridge system with specialized configurations f - **Mamba state dim**: 256 - **Context Length**: 8K tokens -### Nemotron Nano 9B v2 Specifications -- **Parameters**: 9B -- **Layers**: 56 (Hybrid pattern: `M-M-M-MM-M-M-M*-M-M-M*-M-M-M-M*-M-M-M-M*-M-MM-M-M-M-M-M-`) -- **Hidden size**: 4480 -- **FFN hidden size**: 15680 -- **Attention heads**: 40 query heads, 8 key-value groups -- **KV channels**: 128 -- **Mamba heads**: 128 -- **Mamba head dim**: 80 -- **Mamba state dim**: 128 -- **Context Length**: 128K tokens -- **Vocab size**: 131,072 - -### Nemotron Nano 12B v2 Specifications -- **Parameters**: 12B -- **Layers**: 62 (Hybrid pattern: `M-M-M-M*-M-M-M-M*-M-M-M-M*-M-M-M-M*-M-M-M-M*-M-M-M-M*-M-M-M-M-`) -- **Hidden size**: 5120 -- **FFN hidden size**: 20480 -- **Attention heads**: 40 query heads, 8 key-value groups -- **KV channels**: 128 -- **Mamba heads**: 128 -- **Mamba head dim**: 80 -- **Mamba state dim**: 128 -- **Context Length**: 128K tokens -- **Vocab size**: 131,072 - ## Key Features ### Hybrid SSM-Attention Architecture @@ -122,10 +87,6 @@ All models are supported via the Bridge system with specialized configurations f - **Multi-Query Attention**: Efficient attention with shared key-value heads - **Selective State Space**: Mamba-2 architecture with selective gating -### Extended Context (Nano v2) -- **128K Context Window**: Nemotron Nano v2 models support up to 128K tokens -- **Efficient Long-Range Modeling**: Hybrid architecture optimized for long sequences - ## Conversion with πŸ€— Hugging Face ### Load HF β†’ Megatron @@ -153,30 +114,6 @@ model = provider.provide_distributed_model(wrap_with_ddp=False) # bridge = AutoBridge.from_hf_pretrained("nvidia/Nemotron-H-56B-Base-8K", trust_remote_code=True) ``` -#### Nemotron Nano v2 Models -```python -from megatron.bridge import AutoBridge - -# Example: Nemotron Nano 9B v2 -bridge = AutoBridge.from_hf_pretrained("nvidia/NVIDIA-Nemotron-Nano-9B-v2-Base", trust_remote_code=True) -provider = bridge.to_megatron_provider() - -# Configure parallelism -provider.tensor_model_parallel_size = 2 -provider.pipeline_model_parallel_size = 1 -provider.context_parallel_size = 1 -provider.sequence_parallel = True - -provider.finalize() -model = provider.provide_distributed_model(wrap_with_ddp=False) - -# For instruct variant: -# bridge = AutoBridge.from_hf_pretrained("nvidia/NVIDIA-Nemotron-Nano-9B-v2", trust_remote_code=True) - -# For 12B model: -# bridge = AutoBridge.from_hf_pretrained("nvidia/NVIDIA-Nemotron-Nano-12B-v2-Base", trust_remote_code=True) -``` - ### Export Megatron β†’ HF ```python # Convert from a Megatron checkpoint directory to HF format @@ -275,42 +212,6 @@ cfg = nemotronh_56b_peft_config( ) ``` -### Nemotron Nano 9B v2 Finetuning - -```python -from megatron.bridge.recipes.nemotronh import nemotron_nano_9b_v2_peft_config - -# LoRA finetuning -cfg = nemotron_nano_9b_v2_peft_config( - tokenizer_path="nvidia/NVIDIA-Nemotron-Nano-9B-v2-Base", - name="nano_9b_v2_lora", - pretrained_checkpoint="path/to/nano/9b/v2/checkpoint", - peft_scheme="lora", - train_iters=1000, - global_batch_size=128, - seq_length=2048, # Can use up to 128K - finetune_lr=1e-4, -) -``` - -### Nemotron Nano 12B v2 Finetuning - -```python -from megatron.bridge.recipes.nemotronh import nemotron_nano_12b_v2_peft_config - -# LoRA finetuning -cfg = nemotron_nano_12b_v2_peft_config( - tokenizer_path="nvidia/NVIDIA-Nemotron-Nano-12B-v2-Base", - name="nano_12b_v2_lora", - pretrained_checkpoint="path/to/nano/12b/v2/checkpoint", - peft_scheme="lora", - train_iters=1000, - global_batch_size=128, - seq_length=2048, # Can use up to 128K - finetune_lr=1e-4, -) -``` - ## Default Configurations ### Nemotron H Models @@ -356,44 +257,16 @@ cfg = nemotron_nano_12b_v2_peft_config( - Sequence Parallel: True - Precision: FP8 hybrid -### Nemotron Nano v2 Models - -#### 9B - LoRA (1 node, 8 GPUs) -- TP=2, PP=1, CP=1, LR=1e-4 -- Sequence Parallel: True -- Precision: BF16 mixed -- Context: Up to 128K tokens - -#### 9B - Full SFT (1 node, 8 GPUs) -- TP=2, PP=1, CP=1, LR=1e-4 -- Sequence Parallel: True -- Precision: BF16 mixed - -#### 12B - LoRA (2 nodes, 16 GPUs) -- TP=4, PP=1, CP=1, LR=1e-4 -- Sequence Parallel: True -- Precision: FP8 hybrid (recommended) -- Context: Up to 128K tokens - -#### 12B - Full SFT (2 nodes, 16 GPUs) -- TP=4, PP=1, CP=1, LR=1e-4 -- Sequence Parallel: True -- Precision: FP8 hybrid - ## API Reference ### Nemotron H - Nemotron H recipes: [bridge.recipes.nemotronh](../../apidocs/bridge/bridge.recipes.nemotronh.md) -### Nemotron Nano v2 -- Nemotron Nano v2 recipes: [bridge.recipes.nemotronh.nemotron_nano_v2](../../apidocs/bridge/bridge.recipes.nemotronh.md) - ## Performance Optimizations ### Memory Efficiency - **Selective Recomputation**: Reduces activation memory for larger models - **Sequence Parallelism**: Distributes sequence dimension across GPUs (enabled for 8B+) -- **Context Parallelism**: Support for ultra-long sequences (Nano v2) - **Manual GC**: Aggressive garbage collection for stable memory usage - **Precision-aware optimizer**: BF16/FP8 gradients with FP32 master weights @@ -426,12 +299,6 @@ Nemotron H models support several PP configurations with pre-defined layouts: - **47B Base**: [nvidia/Nemotron-H-47B-Base-8K](https://huggingface.co/nvidia/Nemotron-H-47B-Base-8K) - **56B Base**: [nvidia/Nemotron-H-56B-Base-8K](https://huggingface.co/nvidia/Nemotron-H-56B-Base-8K) -### Nemotron Nano v2 Models -- **9B Base**: [nvidia/NVIDIA-Nemotron-Nano-9B-v2-Base](https://huggingface.co/nvidia/NVIDIA-Nemotron-Nano-9B-v2-Base) -- **9B Instruct**: [nvidia/NVIDIA-Nemotron-Nano-9B-v2](https://huggingface.co/nvidia/NVIDIA-Nemotron-Nano-9B-v2) -- **12B Base**: [nvidia/NVIDIA-Nemotron-Nano-12B-v2-Base](https://huggingface.co/nvidia/NVIDIA-Nemotron-Nano-12B-v2-Base) -- **12B Instruct**: [nvidia/NVIDIA-Nemotron-Nano-12B-v2](https://huggingface.co/nvidia/NVIDIA-Nemotron-Nano-12B-v2) - ## Technical Resources ### Research Papers diff --git a/docs/fern/versions/nightly/pages/nemo2-migration-guide.mdx b/docs/fern/versions/nightly/pages/nemo2-migration-guide.mdx index 7fd15d42b7..7c834ae58f 100644 --- a/docs/fern/versions/nightly/pages/nemo2-migration-guide.mdx +++ b/docs/fern/versions/nightly/pages/nemo2-migration-guide.mdx @@ -21,16 +21,15 @@ Megatron Bridge derives model provider settings from Hugging Face configs throug | `llm.Llama3Config8B` | `AutoBridge.from_hf_pretrained("meta-llama/Meta-Llama-3-8B").to_megatron_provider()` | | `llm.Llama31Config70B` | `AutoBridge.from_hf_pretrained("meta-llama/Llama-3.1-70B").to_megatron_provider()` | | `llm.Qwen2Config7B` | `AutoBridge.from_hf_pretrained("Qwen/Qwen2-7B").to_megatron_provider()` | -| `llm.DeepseekV2Config` | `bridge.models.DeepseekV2ModelProvider` | ### Supported Model Families Megatron Bridge supports the following model families through bridge mappings and provider classes: - **Base Models**: `GPTModelProvider`, `T5ModelProvider`, `MambaModelProvider` -- **Llama**: Llama2, Llama3, Llama3.1, Llama3.2, CodeLlama, Llama4 +- **Llama**: Llama3, Llama3.1, Llama3.2, CodeLlama, Llama4 - **Qwen**: Qwen2, Qwen2.5, Qwen3, Qwen3MoE, Qwen2.5VL -- **DeepSeek**: DeepSeek, DeepSeekV2, DeepSeekV2Lite, DeepSeekV3, Moonlight -- **Nemotron**: Nemotron3, Nemotron4, NemotronH, NemotronNano +- **DeepSeek**: DeepSeek V3, DeepSeek V4 +- **Nemotron**: Nemotron 3, Nemotron 3.5, Nemotron H (deprecated) - **NVIDIA Mamba**: Mamba variants and hybrid models For a complete list of bridge mappings, providers, and their parameters, see `bridge.models`. diff --git a/docs/models/deepseek/deepseek-v2.md b/docs/models/deepseek/deepseek-v2.md deleted file mode 100644 index 2814bdeb33..0000000000 --- a/docs/models/deepseek/deepseek-v2.md +++ /dev/null @@ -1,166 +0,0 @@ -# DeepSeek V2 - -> **Deprecation notice:** DeepSeek V2 and DeepSeek V2 Lite support is no -> longer actively maintained or tested against current upstream checkpoints and -> will be removed in Megatron Bridge 0.7.0. - -[DeepSeek-V2](https://github.com/deepseek-ai/DeepSeek-V2) is a Mixture-of-Experts (MoE) language model that uses innovative Multi-head Latent Attention (MLA) for efficient inference and DeepSeekMoE architecture for economical training and inference. The model achieves performance comparable to GPT-4 while using significantly fewer activated parameters. More information is available in the companion paper ["DeepSeek-V2: A Strong, Economical, and Efficient Mixture-of-Experts Language Model"](https://arxiv.org/abs/2405.04434). - -DeepSeek V2 models are supported via the Bridge system with auto-detected configuration and weight mapping. - -## Available Models - -Megatron Bridge supports the following DeepSeek V2 model variants: - -- **DeepSeek-V2**: 236B parameters (21B activated per token) -- **DeepSeek-V2-Lite**: 16B parameters (2.4B activated per token) - -Both models support pretraining with expert parallelism for efficient MoE training. - -## Model Architecture Features - -- **Multi-head Latent Attention (MLA)**: Novel attention mechanism that reduces KV cache requirements -- **DeepSeekMoE**: Efficient MoE architecture with routed and shared experts -- **Expert Parallelism**: Distributes experts across GPUs for scalable training -- **RoPE Embeddings**: Rotary position embeddings for position encoding -- **128K Context Length**: Native support for long sequences (DeepSeek-V2) -- **Pre-normalization**: RMSNorm before each transformer sub-layer - -## Conversion with πŸ€— Hugging Face - -### Load HF β†’ Megatron - -```python -from megatron.bridge import AutoBridge - -# Example: DeepSeek-V2-Lite -bridge = AutoBridge.from_hf_pretrained("deepseek-ai/DeepSeek-V2-Lite", trust_remote_code=True) -provider = bridge.to_megatron_provider() - -# Optionally configure parallelism before instantiating the model -provider.tensor_model_parallel_size = 1 -provider.pipeline_model_parallel_size = 1 -provider.expert_model_parallel_size = 8 - -model = provider.provide_distributed_model(wrap_with_ddp=False) -``` - -### Import Checkpoint from HF - -```bash -./scripts/conversion/convert.sh import \ - --hf-model deepseek-ai/DeepSeek-V2-Lite \ - --megatron-path /checkpoints/deepseek_v2_lite_megatron \ - --trust-remote-code -``` - -### Export Megatron β†’ HF - -```python -from megatron.bridge import AutoBridge - -# Load the bridge from HF model ID -bridge = AutoBridge.from_hf_pretrained("deepseek-ai/DeepSeek-V2-Lite", trust_remote_code=True) - -# Export a trained Megatron checkpoint to HF format -bridge.export_ckpt( - megatron_path="/results/deepseek_v2_lite/checkpoints/iter_0000500", - hf_path="/exports/deepseek_v2_lite_hf", -) -``` - -### Run Inference on Converted Checkpoint - -```bash -uv run python examples/conversion/hf_to_megatron_generate_text.py \ - --hf_model_path deepseek-ai/DeepSeek-V2-Lite \ - --megatron_model_path /checkpoints/deepseek_v2_lite_megatron \ - --prompt "What is artificial intelligence?" \ - --max_new_tokens 100 \ - --ep 8 \ - --trust-remote-code -``` - -For more details, see [examples/conversion/hf_to_megatron_generate_text.py](https://github.com/NVIDIA-NeMo/Megatron-Bridge/blob/main/examples/conversion/hf_to_megatron_generate_text.py) - -## Recipes - -See: [bridge.recipes.deepseek.deepseek_v2](../../apidocs/bridge/bridge.recipes.deepseek.deepseek_v2.md) - -### Available Recipes - -- **Pretrain recipes**: - - `deepseek_v2_lite_pretrain_config`: Pre-training for DeepSeek-V2-Lite (16B parameters, 2.4B activated per token) - - `deepseek_v2_pretrain_config`: Pre-training for DeepSeek-V2 (236B parameters, 21B activated per token) - -### Parallelism Configurations - -| Model | TP | PP | EP | Total GPUs | Use Case | -|-------|----|----|----|-----------:|----------| -| **DeepSeek-V2-Lite** | 1 | 1 | 8 | 8 | Pre-training (single node) | -| **DeepSeek-V2** | 1 | 4 | 32 | 128 | Pre-training (16 nodes) | - -**Key Features**: -- **Expert Parallelism**: EP=8 (V2-Lite) or EP=32 (V2) for efficient MoE training -- **Selective Recomputation**: Enabled by default for memory optimization -- **Sequence Length**: Default 4096, V2 supports up to 128K tokens - -### Pre-training Examples - -#### DeepSeek-V2-Lite (16B) - -```python -from megatron.bridge.recipes.deepseek import deepseek_v2_lite_pretrain_config - -config = deepseek_v2_lite_pretrain_config() -config.dataset.blend = [["/path/to/dataset_text_document"], None] -config.checkpoint.save = "/results/deepseek_v2_lite/checkpoints" -config.checkpoint.load = "/results/deepseek_v2_lite/checkpoints" -config.logger.tensorboard_dir = "/results/deepseek_v2_lite/tb_logs" -config.train.train_iters = 500_000 -config.train.global_batch_size = 512 -config.model.seq_length = 4096 -config.dataset.seq_length = 4096 -# Uses TP=1, PP=1, EP=8 (8 GPUs) automatically -``` - -#### DeepSeek-V2 (236B) - -```python -from megatron.bridge.recipes.deepseek import deepseek_v2_pretrain_config - -config = deepseek_v2_pretrain_config() -config.dataset.blend = [["/path/to/dataset_text_document"], None] -config.checkpoint.save = "/results/deepseek_v2/checkpoints" -config.checkpoint.load = "/results/deepseek_v2/checkpoints" -config.logger.tensorboard_dir = "/results/deepseek_v2/tb_logs" -config.train.train_iters = 500_000 -config.train.global_batch_size = 512 -config.model.seq_length = 4096 -config.dataset.seq_length = 4096 -# Uses TP=1, PP=4, EP=32 (128 GPUs) automatically -``` - -### Finetuning Recipes - -Finetuning recipes for DeepSeek V2 models are not currently available. - - -## Hugging Face Model Cards & References - -### Hugging Face Model Cards -- DeepSeek-V2: https://huggingface.co/deepseek-ai/DeepSeek-V2 -- DeepSeek-V2-Lite: https://huggingface.co/deepseek-ai/DeepSeek-V2-Lite -- DeepSeek-V2-Chat: https://huggingface.co/deepseek-ai/DeepSeek-V2-Chat -- DeepSeek-V2-Lite-Chat: https://huggingface.co/deepseek-ai/DeepSeek-V2-Lite-Chat - -### Technical Papers -- DeepSeek-V2: A Strong, Economical, and Efficient Mixture-of-Experts Language Model: [arXiv:2405.04434](https://arxiv.org/abs/2405.04434) - -### Additional Resources -- GitHub Repository: https://github.com/deepseek-ai/DeepSeek-V2 - -## Related Docs -- Recipe usage: [Recipe usage](../../recipe-usage.md) -- Customizing the training recipe configuration: [Configuration overview](../../training/config-container-overview.md) -- Training entry points: [Entry points](../../training/entry-points.md) diff --git a/docs/models/deepseek/index.md b/docs/models/deepseek/index.md index c1ccd6f922..c7b5091e5d 100644 --- a/docs/models/deepseek/index.md +++ b/docs/models/deepseek/index.md @@ -5,13 +5,11 @@ DeepSeek model documentation is organized by model variant. ```{toctree} :hidden: -deepseek-v2.md deepseek-v3.md deepseek-v4.md ``` | Variant | Guide | |---------|-------| -| DeepSeek V2 (deprecated) | [deepseek-v2.md](deepseek-v2.md) | | DeepSeek V3 | [deepseek-v3.md](deepseek-v3.md) | | DeepSeek V4 | [deepseek-v4.md](deepseek-v4.md) | diff --git a/docs/models/gemma/gemma.md b/docs/models/gemma/gemma.md deleted file mode 100644 index 6260beacf9..0000000000 --- a/docs/models/gemma/gemma.md +++ /dev/null @@ -1,44 +0,0 @@ -# Gemma - -> **Deprecation notice:** Gemma 1 support (2B and 7B) is no longer actively -> maintained or tested against current upstream checkpoints and will be removed -> in Megatron Bridge 0.7.0. - -[Gemma](https://huggingface.co/collections/google/gemma-release-65d5efbccdbb8c4202ec078b) is Google's original lightweight open model family. Megatron Bridge supports Gemma causal language models through the `GemmaBridge` implementation for the Hugging Face `GemmaForCausalLM` architecture. - -## Supported Variants - -Megatron Bridge supports Hugging Face Gemma checkpoints that use the `gemma` model type, including: - -- Gemma 2B: https://huggingface.co/google/gemma-2b -- Gemma 7B: https://huggingface.co/google/gemma-7b -- Gemma release collection: https://huggingface.co/collections/google/gemma-release-65d5efbccdbb8c4202ec078b - -## Architecture Notes - -- RMSNorm with zero-centered gamma. -- GeGLU-style gated MLPs. -- RoPE positional embeddings and flash attention backend. -- Shared input/output embedding weights. - -## Examples - -Gemma uses the common conversion and generation entry points: - -```bash -./scripts/conversion/convert.sh import \ - --hf-model google/gemma-2b \ - --megatron-path /checkpoints/gemma_2b_megatron -``` - -```bash -uv run python examples/conversion/hf_to_megatron_generate_text.py \ - --hf_model_path google/gemma-2b \ - --megatron_model_path /checkpoints/gemma_2b_megatron \ - --prompt "What is artificial intelligence?" -``` - -## Related Implementation - -- Bridge implementation: [`src/megatron/bridge/models/gemma/gemma_bridge.py`](https://github.com/NVIDIA-NeMo/Megatron-Bridge/blob/main/src/megatron/bridge/models/gemma/gemma_bridge.py) -- Conversion examples: [`examples/conversion`](https://github.com/NVIDIA-NeMo/Megatron-Bridge/tree/main/examples/conversion) diff --git a/docs/models/gemma/gemma2.md b/docs/models/gemma/gemma2.md deleted file mode 100644 index 17992ed0a2..0000000000 --- a/docs/models/gemma/gemma2.md +++ /dev/null @@ -1,279 +0,0 @@ -# Gemma 2 - -> **Deprecation notice:** Gemma 2 support (2B, 9B, and 27B) is no longer -> actively maintained or tested against current upstream checkpoints and will -> be removed in Megatron Bridge 0.7.0. - -[Google's Gemma 2](https://huggingface.co/collections/google/gemma-2-release-667d6600fd5220e7b967f315) is a family of lightweight, open models built on the same research and technology used to create Gemini models. The Gemma 2 architecture builds on the transformer decoder framework with enhancements including pre-normalization with RMSNorm, GeGLU activations, Rotary Positional Embeddings (RoPE), attention logit softcapping, and sliding window attention. - -Gemma 2 models are designed for a wide range of text generation tasks and are available in multiple sizes to suit different computational budgets. - -Gemma family models are supported via the Bridge system with auto-detected configuration and weight mapping. - -## Available Models - -### Text-Only Models -- **Gemma 2 2B** (`google/gemma-2-2b`): Compact 2B parameter model optimized for efficiency - - 26 layers, 2304 hidden size - - 8 attention heads, 4 query groups (GQA) - - Sequence length: 8,192 tokens - - Ideal for single-GPU deployment - -- **Gemma 2 9B** (`google/gemma-2-9b`): Medium-scale 9B parameter model balancing performance and efficiency - - 42 layers, 3584 hidden size - - 16 attention heads, 8 query groups (GQA) - - Sequence length: 8,192 tokens - - Recommended: 4-8 GPUs with tensor parallelism - -- **Gemma 2 27B** (`google/gemma-2-27b`): Large-scale 27B parameter model for maximum performance - - 46 layers, 4608 hidden size - - 32 attention heads, 16 query groups (GQA) - - Sequence length: 8,192 tokens - - Recommended: 8-16 GPUs with tensor and pipeline parallelism - -All models support a sequence length of 8,192 tokens and use sliding window attention for efficient context processing. - -## Model Architecture Features - -Gemma 2 introduces several architectural innovations: - -- **Sliding Window Attention**: Local attention with a 4,096 token window for efficient long-context processing -- **Attention Logit Softcapping**: Caps attention logits at 50.0 to prevent extreme values -- **Final Logit Softcapping**: Caps final output logits at 30.0 for stable generation -- **GeGLU Activation**: Uses gated linear units with GELU activation for improved performance -- **RMSNorm**: Layer normalization without mean centering for faster computation -- **Rotary Embeddings**: RoPE with base frequency 10,000 -- **Query Pre-Attention Scaling**: Scales queries by 224 before attention computation - -## Conversion with πŸ€— Hugging Face - -### Load HF β†’ Megatron -```python -from megatron.bridge import AutoBridge - -# Example: Gemma 2 9B -bridge = AutoBridge.from_hf_pretrained("google/gemma-2-9b") -provider = bridge.to_megatron_provider() - -# Configure parallelism before instantiating the model -provider.tensor_model_parallel_size = 8 -provider.pipeline_model_parallel_size = 1 - -provider.finalize() -model = provider.provide_distributed_model(wrap_with_ddp=False) -``` - -### Import HF β†’ Megatron -To import the HF model to your desired Megatron path: -```bash -# Gemma 2 2B -./scripts/conversion/convert.sh import \ ---hf-model google/gemma-2-2b \ ---megatron-path /models/gemma-2-2b - -# Gemma 2 9B -./scripts/conversion/convert.sh import \ ---hf-model google/gemma-2-9b \ ---megatron-path /models/gemma-2-9b - -# Gemma 2 27B -./scripts/conversion/convert.sh import \ ---hf-model google/gemma-2-27b \ ---megatron-path /models/gemma-2-27b -``` - -### Export Megatron β†’ HF -```bash -# Gemma 2 9B example -./scripts/conversion/convert.sh export \ ---hf-model google/gemma-2-9b \ ---megatron-path /results/gemma2_9b/checkpoints/iter_00001000 \ ---hf-path ./gemma2-9b-hf-export -``` - -### Run Inference on Converted Checkpoint - -```bash -uv run python examples/conversion/hf_to_megatron_generate_text.py \ ---hf_model_path google/gemma-2-9b \ ---megatron_model_path /models/gemma-2-9b \ ---prompt "What is artificial intelligence?" \ ---max_new_tokens 100 -``` - -Note: -- `--megatron_model_path` is optional. If not specified, the script will convert the model and then run forward. - -## Pretrain and Finetune Recipes - -- See: [bridge.recipes.gemma](../../apidocs/bridge/bridge.recipes.gemma.md) -- Available recipes: - - **Pretraining:** - - `gemma2_2b_pretrain_config`: Pre-training configuration for Gemma 2 2B - - `gemma2_9b_pretrain_config`: Pre-training configuration for Gemma 2 9B - - `gemma2_27b_pretrain_config`: Pre-training configuration for Gemma 2 27B - - **SFT:** - - `gemma2_2b_sft_config`: Full SFT for Gemma 2 2B - - `gemma2_9b_sft_config`: Full SFT for Gemma 2 9B - - `gemma2_27b_sft_config`: Full SFT for Gemma 2 27B - - **PEFT** (LoRA, DoRA): - - `gemma2_2b_peft_config`: PEFT for Gemma 2 2B - - `gemma2_9b_peft_config`: PEFT for Gemma 2 9B - - `gemma2_27b_peft_config`: PEFT for Gemma 2 27B - -Before training, ensure the following environment variables are set: -1. `SAVE_DIR`: checkpoint and log saving directory -2. `HF_TOKEN`: to download models from HF Hub (if required) -3. `HF_HOME`: (optional) to avoid re-downloading models and datasets -4. `WANDB_API_KEY`: (optional) to enable WandB logging - -### Pretraining - -#### Gemma 2 2B -```python -from megatron.bridge.recipes.gemma import gemma2_2b_pretrain_config - -# Create and customize a pretraining configuration. -config = gemma2_2b_pretrain_config() -config.dataset.blend = [["path/to/data"], None] -config.train.train_iters = 100000 -config.train.global_batch_size = 32 -``` - -#### Gemma 2 9B -```python -from megatron.bridge.recipes.gemma import gemma2_9b_pretrain_config - -config = gemma2_9b_pretrain_config() -config.dataset.blend = [["path/to/data"], None] -config.train.train_iters = 100000 -config.train.global_batch_size = 32 -``` - -#### Gemma 2 27B -```python -from megatron.bridge.recipes.gemma import gemma2_27b_pretrain_config - -config = gemma2_27b_pretrain_config() -config.dataset.blend = [["path/to/data"], None] -config.train.train_iters = 100000 -config.train.global_batch_size = 32 -``` - -### Full Finetuning - -#### Gemma 2 2B -```bash -uv run python -m torch.distributed.run --nproc-per-node=8 scripts/training/run_recipe.py \ ---recipe gemma2_2b_sft_config \ ---mode sft \ -checkpoint.pretrained_checkpoint=/models/gemma-2-2b \ -train.global_batch_size=64 \ -train.train_iters=1000 \ -checkpoint.save=$SAVE_DIR/gemma2_2b_finetune -``` - -Or programmatically: -```python -from megatron.bridge.recipes.gemma import gemma2_2b_sft_config - -config = gemma2_2b_sft_config() -config.checkpoint.pretrained_checkpoint = "/models/gemma-2-2b" -config.train.train_iters = 1000 -config.train.global_batch_size = 64 -``` - -#### Gemma 2 9B -```bash -uv run python -m torch.distributed.run --nproc-per-node=8 scripts/training/run_recipe.py \ ---recipe gemma2_9b_sft_config \ ---mode sft \ -checkpoint.pretrained_checkpoint=/models/gemma-2-9b \ -train.global_batch_size=64 \ -train.train_iters=1000 \ -checkpoint.save=$SAVE_DIR/gemma2_9b_finetune -``` - -#### Gemma 2 27B -```bash -uv run python -m torch.distributed.run --nproc-per-node=16 scripts/training/run_recipe.py \ ---recipe gemma2_27b_sft_config \ ---mode sft \ -checkpoint.pretrained_checkpoint=/models/gemma-2-27b \ -train.global_batch_size=64 \ -train.train_iters=1000 \ -checkpoint.save=$SAVE_DIR/gemma2_27b_finetune -``` - -### Parameter-Efficient Finetuning (PEFT) with LoRA - -#### Gemma 2 2B -```bash -uv run python -m torch.distributed.run --nproc-per-node=8 scripts/training/run_recipe.py \ ---recipe gemma2_2b_peft_config \ ---mode lora \ -checkpoint.pretrained_checkpoint=/models/gemma-2-2b \ -train.global_batch_size=128 \ -checkpoint.save=$SAVE_DIR/gemma2_2b_lora -``` - -PEFT options: -- Use `--mode lora` for LoRA or `--mode dora` for DoRA. Use `gemma2_*_sft_config` with `--mode sft` for full finetuning. - -Or programmatically: -```python -from megatron.bridge.recipes.gemma import gemma2_2b_peft_config - -# LoRA finetuning -config = gemma2_2b_peft_config(peft_scheme="lora") # or "dora" -config.checkpoint.pretrained_checkpoint = "/models/gemma-2-2b" -config.train.train_iters = 1000 -config.train.global_batch_size = 128 -``` - -#### Gemma 2 9B LoRA -```python -from megatron.bridge.recipes.gemma import gemma2_9b_peft_config - -config = gemma2_9b_peft_config(peft_scheme="lora") -config.checkpoint.pretrained_checkpoint = "/models/gemma-2-9b" -config.train.train_iters = 1000 -config.train.global_batch_size = 128 -``` - -#### Gemma 2 27B LoRA -```python -from megatron.bridge.recipes.gemma import gemma2_27b_peft_config - -config = gemma2_27b_peft_config(peft_scheme="lora") -config.checkpoint.pretrained_checkpoint = "/models/gemma-2-27b" -config.train.train_iters = 1000 -config.train.global_batch_size = 128 -``` - -### Recommended Configurations - -| Model | Mode | TP | PP | Global Batch Size | Learning Rate | -|-------|------|----|----|-------------------|---------------| -| Gemma 2 2B | Full SFT | 1 | 1 | 64-128 | 5e-6 | -| Gemma 2 2B | LoRA/DoRA | 1 | 1 | 128-256 | 1e-4 | -| Gemma 2 9B | Full SFT | 4 | 1 | 64-128 | 5e-6 | -| Gemma 2 9B | LoRA/DoRA | 1 | 1 | 128-256 | 1e-4 | -| Gemma 2 27B | Full SFT | 8 | 2 | 64-128 | 5e-6 | -| Gemma 2 27B | LoRA/DoRA | 4 | 1 | 128-256 | 1e-4 | - -## Examples -- Checkpoint import/export: [scripts/conversion/convert.sh](https://github.com/NVIDIA-NeMo/Megatron-Bridge/blob/main/scripts/conversion/convert.sh) -- Generate text (HFβ†’Megatron): [examples/conversion/hf_to_megatron_generate_text.py](https://github.com/NVIDIA-NeMo/Megatron-Bridge/blob/main/examples/conversion/hf_to_megatron_generate_text.py) - -## Hugging Face Model Cards - -- Gemma 2 2B: https://huggingface.co/google/gemma-2-2b -- Gemma 2 9B: https://huggingface.co/google/gemma-2-9b -- Gemma 2 27B: https://huggingface.co/google/gemma-2-27b -- Gemma 2 Collection: https://huggingface.co/collections/google/gemma-2-release-667d6600fd5220e7b967f315 - -## Related Docs -- Recipe usage: [Recipe usage](../../recipe-usage.md) -- Customizing the training recipe configuration: [Configuration overview](../../training/config-container-overview.md) -- Training entry points: [Entry points](../../training/entry-points.md) diff --git a/docs/models/gemma/index.md b/docs/models/gemma/index.md index 84394d88bf..226ec5b4f1 100644 --- a/docs/models/gemma/index.md +++ b/docs/models/gemma/index.md @@ -5,8 +5,6 @@ Gemma model documentation is organized by model variant. ```{toctree} :hidden: -gemma.md -gemma2.md gemma3.md gemma3-vl.md gemma4-vl.md @@ -14,8 +12,6 @@ gemma4-vl.md | Variant | Guide | |---------|-------| -| Gemma | [gemma.md](gemma.md) | -| Gemma 2 | [gemma2.md](gemma2.md) | | Gemma 3 | [gemma3.md](gemma3.md) | | Gemma 3 VL | [gemma3-vl.md](gemma3-vl.md) | | Gemma 4 VL | [gemma4-vl.md](gemma4-vl.md) | diff --git a/docs/models/llama/index.md b/docs/models/llama/index.md index 26d0150817..e55350b141 100644 --- a/docs/models/llama/index.md +++ b/docs/models/llama/index.md @@ -5,13 +5,11 @@ Llama model documentation is organized by model variant. ```{toctree} :hidden: -llama2.md llama3.md ``` | Variant | Guide | |---------|-------| -| Llama 2 | [llama2.md](llama2.md) | | Llama 3 | [llama3.md](llama3.md) | ## Recipes diff --git a/docs/models/llama/llama2.md b/docs/models/llama/llama2.md deleted file mode 100644 index e5071fbdbe..0000000000 --- a/docs/models/llama/llama2.md +++ /dev/null @@ -1,47 +0,0 @@ -# Llama 2 - -> **Deprecation notice:** Llama 2 support is no longer actively maintained or -> tested against current upstream checkpoints and will be removed in Megatron -> Bridge 0.7.0. The shared Llama bridge remains supported for Llama 3 variants. - -[Llama 2](https://huggingface.co/meta-llama/Llama-2-7b-hf) is Meta's decoder-only transformer model family. Megatron Bridge supports Llama 2 through the shared Llama bridge for the Hugging Face `LlamaForCausalLM` architecture. - -## Supported Variants - -The repository includes a pretraining recipe for: - -- Llama-2-7B: https://huggingface.co/meta-llama/Llama-2-7b-hf - -The same bridge can auto-detect compatible Llama 2 causal language model checkpoints. - -## Architecture Notes - -- RMSNorm with SwiGLU feed-forward layers. -- RoPE positional embeddings. -- Grouped Query Attention support follows the Hugging Face configuration. -- Uses the same `LlamaBridge` implementation as later Llama 3 variants. - -## Examples - -Llama 2 uses the common conversion and generation entry points: - -```bash -./scripts/conversion/convert.sh import \ - --hf-model meta-llama/Llama-2-7b-hf \ - --megatron-path /checkpoints/llama2_7b_megatron -``` - -```bash -uv run python examples/conversion/hf_to_megatron_generate_text.py \ - --hf_model_path meta-llama/Llama-2-7b-hf \ - --megatron_model_path /checkpoints/llama2_7b_megatron \ - --prompt "What is artificial intelligence?" -``` - -## Recipes - -- Llama 2 recipe: [`src/megatron/bridge/recipes/llama/llama2.py`](https://github.com/NVIDIA-NeMo/Megatron-Bridge/blob/main/src/megatron/bridge/recipes/llama/llama2.py) - -## Related Implementation - -- Bridge implementation: [`src/megatron/bridge/models/llama/llama_bridge.py`](https://github.com/NVIDIA-NeMo/Megatron-Bridge/blob/main/src/megatron/bridge/models/llama/llama_bridge.py) diff --git a/docs/models/mistral/index.md b/docs/models/mistral/index.md index dfaf9cb3f0..4bdff75318 100644 --- a/docs/models/mistral/index.md +++ b/docs/models/mistral/index.md @@ -5,11 +5,9 @@ Mistral model documentation is organized by model variant, including Ministral. ```{toctree} :hidden: -mistral.md ministral3.md ``` | Variant | Guide | |---------|-------| -| Mistral | [mistral.md](mistral.md) | | Ministral 3 | [ministral3.md](ministral3.md) | diff --git a/docs/models/mistral/ministral3.md b/docs/models/mistral/ministral3.md index b806103314..fbe3297b99 100644 --- a/docs/models/mistral/ministral3.md +++ b/docs/models/mistral/ministral3.md @@ -65,7 +65,7 @@ For checkpoint conversion, inference, finetuning recipes, and step-by-step train - Ministral 3 14B Instruct: https://huggingface.co/mistralai/Ministral-3-14B-Instruct-2512 ## Related Docs -- Related Mistral variant: [Mistral](mistral.md) +- Mistral family: [Overview](index.md) - Recipe usage: [Recipe usage](../../recipe-usage.md) - Customizing the training recipe configuration: [Configuration overview](../../training/config-container-overview.md) - Training entry points: [Entry points](../../training/entry-points.md) diff --git a/docs/models/mistral/mistral.md b/docs/models/mistral/mistral.md deleted file mode 100644 index 8b191e1ca8..0000000000 --- a/docs/models/mistral/mistral.md +++ /dev/null @@ -1,109 +0,0 @@ -# Mistral - -> **Deprecation notice:** Mistral 7B and Mistral Small 3 24B support is no -> longer actively maintained or tested against current upstream checkpoints and -> will be removed in Megatron Bridge 0.7.0. Ministral 3 is not deprecated. - -[Mistral AI](https://mistral.ai/) develops frontier large language models with a focus on efficiency and performance. The Mistral family includes both dense and Mixture-of-Experts architectures, featuring innovations like sliding window attention and efficient context handling. - -Mistral models are supported via the Bridge system with auto-detected configuration and weight mapping. - -## Available Models - -Megatron Bridge supports the following Mistral model variants: - -- **Mistral Small 3 (24B)**: 24B parameters with 128K context length -- **Mistral 7B**: 7B parameters, efficient baseline model -- **Mistral 7B Instruct**: Instruction-tuned variant - -Additional Mistral models (including MoE variants like Mixtral) may be supported through the standard conversion pipeline. - -## Model Architecture Features - -- **Sliding Window Attention**: Efficient attention mechanism for long sequences -- **Grouped Query Attention (GQA)**: Memory-efficient attention mechanism -- **Rotary Positional Embeddings (RoPE)**: Relative position encoding -- **SwiGLU Activation**: Gated linear units in the feedforward network -- **Extended Context**: Support for sequences up to 128K tokens (Mistral Small 3) -- **YaRN RoPE Scaling**: Advanced rope scaling for extended context lengths - -## Conversion with πŸ€— Hugging Face - -### Load HF β†’ Megatron - -```python -from megatron.bridge import AutoBridge - -# Example: Mistral 7B -bridge = AutoBridge.from_hf_pretrained("mistralai/Mistral-7B-v0.1") -provider = bridge.to_megatron_provider() - -# Optionally configure parallelism before instantiating the model -provider.tensor_model_parallel_size = 2 -provider.pipeline_model_parallel_size = 1 - -model = provider.provide_distributed_model(wrap_with_ddp=False) -``` - -### Import Checkpoint from HF - -```bash -./scripts/conversion/convert.sh import \ - --hf-model mistralai/Mistral-7B-v0.1 \ - --megatron-path /checkpoints/mistral_7b_megatron -``` - -### Export Megatron β†’ HF - -```python -from megatron.bridge import AutoBridge - -# Load the bridge from HF model ID -bridge = AutoBridge.from_hf_pretrained("mistralai/Mistral-7B-v0.1") - -# Export a trained Megatron checkpoint to HF format -bridge.export_ckpt( - megatron_path="/results/mistral_7b/checkpoints/iter_0000500", - hf_path="/exports/mistral_7b_hf", -) -``` - -### Run Inference on Converted Checkpoint - -```bash -uv run python examples/conversion/hf_to_megatron_generate_text.py \ - --hf_model_path mistralai/Mistral-7B-v0.1 \ - --megatron_model_path /checkpoints/mistral_7b_megatron \ - --prompt "What is artificial intelligence?" \ - --max_new_tokens 100 \ - --tp 2 -``` - -For the complete import, export, round-trip, and three-source generation -workflow, see -[`examples/models/mistral/mistral`](https://github.com/NVIDIA-NeMo/Megatron-Bridge/blob/6a706b1c3afab4d21a6f5dd88aa6a75296d33fe2/examples/models/mistral/mistral/README.md). - -## Recipes - -Training recipes for Mistral models are not currently available. The Bridge supports checkpoint conversion for inference and deployment use cases. - -## Hugging Face Model Cards & References - -### Hugging Face Model Cards -- Mistral Small 3 (24B): https://huggingface.co/mistralai/Mistral-Small-24B-Base-2501 -- Mistral Small 3 (24B) Instruct: https://huggingface.co/mistralai/Mistral-Small-24B-Instruct-2501 -- Mistral 7B v0.1: https://huggingface.co/mistralai/Mistral-7B-v0.1 -- Mistral 7B Instruct v0.2: https://huggingface.co/mistralai/Mistral-7B-Instruct-v0.2 - -### Technical Papers -- Mistral 7B: https://arxiv.org/abs/2310.06825 - -### Additional Resources -- Mistral AI Website: https://mistral.ai/ -- Mistral Documentation: https://docs.mistral.ai/ - -## Related Docs -- Related Mistral variant: [Ministral 3](ministral3.md) -- Recipe usage: [Recipe usage](../../recipe-usage.md) -- Customizing the training recipe configuration: [Configuration overview](../../training/config-container-overview.md) -- Training entry points: [Entry points](../../training/entry-points.md) diff --git a/docs/models/nemotron/index.md b/docs/models/nemotron/index.md index dc11355e4f..f0d10b2984 100644 --- a/docs/models/nemotron/index.md +++ b/docs/models/nemotron/index.md @@ -5,7 +5,6 @@ Nemotron model documentation is organized by model variant. ```{toctree} :hidden: -llama-nemotron.md nemotronh.md nemotron3-nano.md nemotron3-nano-4b.md @@ -14,14 +13,12 @@ nemotron3.5-lightning.md nemotron3.5-super-vl.md nemotron3.5-super-vl-text-only.md nemotron3-ultra.md -nemotron-nano-v2-vl.md nemotron-3-omni.md ``` | Variant | Guide | |---------|-------| -| Llama Nemotron (deprecated) | [llama-nemotron.md](llama-nemotron.md) | -| Nemotron H and Nemotron Nano v2 (deprecated) | [nemotronh.md](nemotronh.md) | +| Nemotron H (deprecated) | [nemotronh.md](nemotronh.md) | | Nemotron-3 Nano 30B-A3B | [nemotron3-nano.md](nemotron3-nano.md) | | Nemotron-3 Nano 4B | [nemotron3-nano-4b.md](nemotron3-nano-4b.md) | | Nemotron-3 Super | [nemotron3-super.md](nemotron3-super.md) | @@ -29,7 +26,6 @@ nemotron-3-omni.md | Nemotron-3.5 Super VL | [nemotron3.5-super-vl.md](nemotron3.5-super-vl.md) | | Nemotron-3.5 Super Text-only | [nemotron3.5-super-vl-text-only.md](nemotron3.5-super-vl-text-only.md) | | Nemotron-3 Ultra | [nemotron3-ultra.md](nemotron3-ultra.md) | -| Nemotron Nano V2 VL (deprecated) | [nemotron-nano-v2-vl.md](nemotron-nano-v2-vl.md) | | Nemotron-3 Nano Omni | [nemotron-3-omni.md](nemotron-3-omni.md) | ## Examples diff --git a/docs/models/nemotron/llama-nemotron.md b/docs/models/nemotron/llama-nemotron.md deleted file mode 100644 index 5763d8a23b..0000000000 --- a/docs/models/nemotron/llama-nemotron.md +++ /dev/null @@ -1,126 +0,0 @@ -# Llama Nemotron - -> **Deprecation notice:** Llama Nemotron support (Super 49B, Ultra 253B, 70B, -> Nano 8B, and Nano 4B) is no longer actively maintained or tested against -> current upstream checkpoints and will be removed in Megatron Bridge 0.7.0. - -[Llama Nemotron](https://huggingface.co/collections/nvidia/llama-nemotron) is NVIDIA's family of large language models derived from Meta's Llama architecture, post-trained for enhanced reasoning, human chat preferences, and agentic tasks such as RAG and tool calling. The models feature neural architecture search (NAS) optimizations for improved efficiency and accuracy trade-offs. - -Llama Nemotron models are supported via the Bridge system with auto-detected configuration and weight mapping. - -## Available Models - -Megatron Bridge supports the following Llama Nemotron model variants: - -- **Llama-3.3-Nemotron-Super-49B**: 49B parameters (NAS-optimized from 70B) -- **Llama-3.1-Nemotron-Ultra-253B**: 253B parameters (large-scale reasoning model) -- **Llama-3.1-Nemotron-70B**: 70B parameters (standard size) -- **Llama-3.1-Nemotron-Nano-8B**: 8B parameters (efficient variant) -- **Llama-3.1-Nemotron-Nano-4B**: 4B parameters (ultra-compact variant) - -All models are ready for commercial use and support context lengths up to 128K tokens. - -## Model Architecture Features - -- **Neural Architecture Search (NAS)**: Novel approach to reduce memory footprint while maintaining accuracy -- **Heterogeneous Blocks**: Non-standard and non-repetitive layer configurations for efficiency - - Skip attention in some blocks - - Variable FFN expansion/compression ratios between blocks -- **Multi-Phase Post-Training**: - - Supervised fine-tuning for Math, Code, Science, and Tool Calling - - Reward-aware Preference Optimization (RPO) for chat - - Reinforcement Learning with Verifiable Rewards (RLVR) for reasoning - - Iterative Direct Preference Optimization (DPO) for tool calling -- **Extended Context**: Native support for sequences up to 128K tokens -- **Commercial Ready**: Fully licensed for commercial deployment - -## Conversion with πŸ€— Hugging Face - -### Load HF β†’ Megatron - -```python -from megatron.bridge import AutoBridge - -# Example: Llama-3.3-Nemotron-Super-49B -bridge = AutoBridge.from_hf_pretrained( - "nvidia/Llama-3_3-Nemotron-Super-49B-v1_5", - trust_remote_code=True -) -provider = bridge.to_megatron_provider() - -# Optionally configure parallelism before instantiating the model -provider.tensor_model_parallel_size = 2 -provider.pipeline_model_parallel_size = 1 - -model = provider.provide_distributed_model(wrap_with_ddp=False) -``` - -**Note**: Heterogeneous Llama-Nemotron models (Super/Ultra) require `trust_remote_code=True` as they use custom `DeciLMForCausalLM` architecture. Homogeneous models (Nano/70B) use standard Llama architecture and don't require this flag. - -### Import Checkpoint from HF - -```bash -./scripts/conversion/convert.sh import \ - --hf-model nvidia/Llama-3_3-Nemotron-Super-49B-v1_5 \ - --megatron-path /checkpoints/llama_nemotron_super_49b_megatron \ - --trust-remote-code -``` - -### Export Megatron β†’ HF - -```python -from megatron.bridge import AutoBridge - -# Load the bridge from HF model ID -bridge = AutoBridge.from_hf_pretrained( - "nvidia/Llama-3_3-Nemotron-Super-49B-v1_5", - trust_remote_code=True -) - -# Export a trained Megatron checkpoint to HF format -bridge.export_ckpt( - megatron_path="/results/llama_nemotron_super_49b/checkpoints/iter_0000500", - hf_path="/exports/llama_nemotron_super_49b_hf", -) -``` - -### Run Inference on Converted Checkpoint - -```bash -uv run python examples/conversion/hf_to_megatron_generate_text.py \ - --hf_model_path nvidia/Llama-3_3-Nemotron-Super-49B-v1_5 \ - --megatron_model_path /checkpoints/llama_nemotron_super_49b_megatron \ - --prompt "What is artificial intelligence?" \ - --max_new_tokens 100 \ - --tp 2 \ - --trust-remote-code -``` - -For more details, see [examples/conversion/hf_to_megatron_generate_text.py](https://github.com/NVIDIA-NeMo/Megatron-Bridge/blob/main/examples/conversion/hf_to_megatron_generate_text.py) - -## Recipes - -Training recipes for Llama Nemotron models are not currently available. - -## Hugging Face Model Cards & References - -### Hugging Face Model Cards -- Llama Nemotron Collection: https://huggingface.co/collections/nvidia/llama-nemotron -- Llama-3.3-Nemotron-Super-49B-v1.5: https://huggingface.co/nvidia/Llama-3_3-Nemotron-Super-49B-v1_5 -- Llama-3.1-Nemotron-Ultra-253B-v1: https://huggingface.co/nvidia/Llama-3_1-Nemotron-Ultra-253B-v1 -- Llama-3.1-Nemotron-Nano-8B-v1: https://huggingface.co/nvidia/Llama-3.1-Nemotron-Nano-8B-v1 -- Llama-3.1-Nemotron-Nano-4B-v1.1: https://huggingface.co/nvidia/Llama-3.1-Nemotron-Nano-4B-v1.1 - -### Technical Papers -- Llama-Nemotron: Efficient Reasoning Models: [arXiv:2505.00949](https://arxiv.org/abs/2505.00949) -- Puzzle: Distillation-Based NAS for Inference-Optimized LLMs: [arXiv:2411.19146](https://arxiv.org/abs/2411.19146) -- Reward-aware Preference Optimization: [arXiv:2502.00203](https://arxiv.org/abs/2502.00203) - -### Additional Resources -- NVIDIA Build Platform: https://build.nvidia.com/ -- Llama Nemotron Post-Training Dataset: https://huggingface.co/nvidia/Llama-Nemotron-Post-Training-Dataset - -## Related Docs -- Recipe usage: [Recipe usage](../../recipe-usage.md) -- Customizing the training recipe configuration: [Configuration overview](../../training/config-container-overview.md) -- Training entry points: [Entry points](../../training/entry-points.md) diff --git a/docs/models/nemotron/nemotron-nano-v2-vl.md b/docs/models/nemotron/nemotron-nano-v2-vl.md deleted file mode 100644 index 090cb7b235..0000000000 --- a/docs/models/nemotron/nemotron-nano-v2-vl.md +++ /dev/null @@ -1,200 +0,0 @@ -# Nemotron Nano V2 VL - -> **Deprecation notice:** Nemotron Nano v2 VL 12B support is no longer actively -> maintained or tested against current upstream checkpoints and will be removed -> in Megatron Bridge 0.7.0. - -NVIDIA Nemotron Nano v2 VL is an open 12B multimodal reasoning model for document intelligence and video understanding. -It enables [AI assistants](https://www.nvidia.com/en-us/use-cases/ai-assistants) to extract, interpret, and act on -information across text, images, tables, and videos. This makes the model valuable for agents focused on data analysis, -document processing and visual understanding in applications like generating reports, curating videos, and dense -captioning for media asset management, and retrieval-augmented search. - -NeMo Megatron Bridge supports finetuning this model (including LoRA finetuning) on single-image, multi-image, and video -datasets. -The finetuned model can be converted back to the πŸ€— Hugging Face format for downstream evaluation. - -```{important} -Please use the custom container `nvcr.io/nvidia/nemo:25.09.nemotron_nano_v2_vl` when working with this model. - -Run all commands from `/opt/Megatron-Bridge` (e.g. `docker run -w /opt/Megatron-Bridge ...`) -``` - -```{tip} -We use the following environment variables throughout this page -- `HF_MODEL_PATH=nvidia/NVIDIA-Nemotron-Nano-12B-v2-VL-BF16` -- `MEGATRON_MODEL_PATH=/models/NVIDIA-Nemotron-Nano-12B-v2-VL-BF16` (feel free to set your own path) - -Unless explicitly stated, any megatron model path in the commands below should NOT contain the iteration number -`iter_xxxxxx`. For more details on checkpointing, please see -[here](../../training/checkpointing.md#checkpoint-contents) -``` - -## Conversion with πŸ€— Hugging Face - -### Import HF β†’ Megatron -To import the HF model to your desired `$MEGATRON_MODEL_PATH`, run the following command. -```bash -./scripts/conversion/convert.sh import \ ---hf-model $HF_MODEL_PATH \ ---megatron-path $MEGATRON_MODEL_PATH \ ---trust-remote-code -``` - -### Export Megatron β†’ HF -You can export a trained model with the following command. -```bash -./scripts/conversion/convert.sh export \ ---hf-model $HF_MODEL_PATH \ ---megatron-path \ ---hf-path \ ---not-strict -``` - -Note: it is normal to see a warning that `vision_model.radio_model.input_conditioner.norm_mean` and `vision_model.radio_model.input_conditioner.norm_std` from source are not in the exported checkpoint. These two weights are not needed in the checkpoint. - - -### Run In-Framework Inference on Converted Checkpoint -You can run a quick sanity check on the converted checkpoint with the following command. -```bash -uv run python examples/conversion/hf_to_megatron_generate_vlm.py \ ---hf_model_path $HF_MODEL_PATH \ ---megatron_model_path $MEGATRON_MODEL_PATH \ ---image_path \ ---prompt "Describe this image." \ ---max_new_tokens 100 \ ---use_llava_model -``` - -Note: -- `--megatron_model_path` is optional. If not specified, the script will convert the model and then run forward. If - specified, the script will just load the megatron model -- `--max_new_tokens` controls the number of tokens to generate. -- For inference with multiple images, pass in a comma-separated list, e.g. - `--image_path="/path/to/example1.jpeg,/path/to/example2.jpeg"`. - Use a suitable prompt, e.g. `--prompt="Describe the two images in detail."`. -- For inference with video, pass in video path instead, e.g. `--video_path="/path/to/demo.mp4"`. Use a suitable prompt, - e.g. `--prompt="Describe what you see."`. - - -## Finetuning Recipes -Before training, ensure the following environment variables are set. -1. `SAVE_DIR`: to specify a checkpoint and log saving directory, used in the commands below. -2. `HF_TOKEN`: to download models from HF Hub. -3. `HF_HOME`: (optional) to avoid re-downloading models and datasets every time. -4. `WANDB_API_KEY`: (optional) to enable WandB logging. - -### Full Finetuning -Example usage for full parameter finetuning using the -[Raven dataset](https://huggingface.co/datasets/HuggingFaceM4/the_cauldron/viewer/raven): - -```bash -uv run python -m torch.distributed.run --nproc-per-node=8 examples/models/nemotron/nemotron_vl/finetune_nemotron_nano_v2_vl.py \ ---hf-model-path $HF_MODEL_PATH \ ---pretrained-checkpoint \ -dataset.source.dataset_name=raven \ -logger.wandb_project= \ -logger.wandb_save_dir=$SAVE_DIR \ -checkpoint.save=$SAVE_DIR/ -``` - -Note: -- The config file `examples/models/nemotron/nemotron_vl/conf/nemotron_nano_v2_vl_override_example.yaml` contains a list of arguments - that can be overridden in the command. For example, you can set `train.global_batch_size=` in the command. -- To change to another built-in dataset, set its `dataset.source.dataset_name` preset. To replace the recipe preset with a custom native source from the CLI, set `dataset.source.dataset_name=null` plus `dataset.source.path_or_dataset=`; custom schemas also require an explicit adapter. See the dataset section below. -- After training, you can run inference with `hf_to_megatron_generate_vlm.py` by supplying the trained megatron checkpoint. - You can also export the trained checkpoint to Hugging Face format. -- This full finetuning recipe requires at least 4xH100 (80G) GPUs. - -### Parameter-Efficient Finetuning (PEFT) -Parameter-efficient finetuning (PEFT) using LoRA is supported. -LoRA can be independently applied to the vision model, vision projection, and language model. We support two commonly used -settings out of the box in the example script: -1. Apply LoRA to the language model, and fully finetune the vision model and projection (used when the visual - distribution is substantially different from pretrained.) - -```bash -uv run python -m torch.distributed.run --nproc-per-node=8 examples/models/nemotron/nemotron_vl/finetune_nemotron_nano_v2_vl.py \ ---hf-model-path $HF_MODEL_PATH \ ---pretrained-checkpoint $MEGATRON_MODEL_PATH \ ---lora-on-language-model \ -dataset.source.dataset_name=raven \ -logger.wandb_project= \ -logger.wandb_save_dir=$SAVE_DIR \ -checkpoint.save=$SAVE_DIR/ \ -model.freeze_language_model=True \ -model.freeze_vision_model=False \ -model.freeze_vision_projection=False -``` - -2. Apply LoRA to all linear layers in attention and MLP modules of the vision model, vision projection, and the language model. - -```bash -uv run python -m torch.distributed.run --nproc-per-node=8 examples/models/nemotron/nemotron_vl/finetune_nemotron_nano_v2_vl.py \ ---hf-model-path $HF_MODEL_PATH \ ---pretrained-checkpoint $MEGATRON_MODEL_PATH \ ---lora-on-language-model \ ---lora-on-vision-model \ -dataset.source.dataset_name=raven \ -logger.wandb_project= \ -logger.wandb_save_dir=$SAVE_DIR \ -checkpoint.save=$SAVE_DIR/ \ -model.freeze_language_model=True \ -model.freeze_vision_model=True \ -model.freeze_vision_projection=True -``` - -These LoRA finetuning recipe requires at least 2xH100 (80G) GPUs. - -A LoRA checkpoint only contains the learnable adapter weights. In order to convert the LoRA checkpoint to Hugging Face -format for downstream evaluation, it is necessary to merge the LoRA adapters back to the base model. - -```bash -uv run python examples/peft/merge_lora.py \ ---hf-model-path $HF_MODEL_PATH \ ---lora-checkpoint /iter_N \ ---output -``` -The output is a merged Hugging Face checkpoint for downstream evaluation. - - -## Example Datasets - -Megatron Bridge supports various vision-language dataset examples which can be used to finetune Nemotron Nano V2 VL: -| Dataset | Dataset preset | Description | -|---------|------------|-------------| -| [cord-v2](https://huggingface.co/datasets/naver-clova-ix/cord-v2) | `cord_v2` | OCR receipts: Single-image-text dataset for receipt understanding, outputs xml-like annotated text. | -| [MedPix-VQA](https://huggingface.co/datasets/mmoukouba/MedPix-VQA) | `medpix` | Medical VQA: Single-image question-answer dataset covering clinical medical images and free-form answers. | -| [The Cauldron (Raven subset)](https://huggingface.co/datasets/HuggingFaceM4/the_cauldron) | `raven` | Visual reasoning: Multi-image, vision reasoning dataset for analogical reasoning in different visual layouts. | -| [LLaVA-Video-178K (0_30_s_nextqa subset)](https://huggingface.co/datasets/lmms-lab/LLaVA-Video-178K) | `llava_video_178k` | Video understanding: video question-answer dataset covering every-day scenarios. | - -`cord-v2` is a tiny dataset and is included for demonstration only. This dataset is not recommended for PEFT tuning for this model since the XML output format interacts with the special tokens, leading to unexpected results. - -Note on video training example: -- We provide a video config yaml file instead of the default config yaml file that overwrites a few commands. Please - pass in `--config-file "examples/models/nemotron/nemotron_vl/conf/nemotron_nano_v2_vl_video.yaml"`. -- The LLaVA video dataset requires manual download beforehand. Please place the downloaded and extracted video files - in a folder `VIDEO_ROOT` and pass it to the adapter with `dataset.source.adapter_kwargs={"video_root_path":$VIDEO_ROOT}`. - In the nextqa subset example, `VIDEO_ROOT` should look like - ``` - $VIDEO_ROOT/ - β”œβ”€β”€ NextQA/ - β”‚ └── NExTVideo/ - β”‚ └── 0000/ - β”‚ └── 2440175990.mp4 - β”‚ └── 0001/ - β”‚ └── ... - └── ... - ``` - -Full video training example command: -```bash -uv run python -m torch.distributed.run --nproc-per-node=8 examples/models/nemotron/nemotron_vl/finetune_nemotron_nano_v2_vl.py \ ---hf-model-path $HF_MODEL_PATH \ ---pretrained-checkpoint $MEGATRON_MODEL_PATH \ ---config-file "examples/models/nemotron/nemotron_vl/conf/nemotron_nano_v2_vl_video.yaml" \ -logger.wandb_project= \ -logger.wandb_save_dir=$SAVE_DIR \ -checkpoint.save=$SAVE_DIR/ \ -dataset.source.adapter_kwargs={"video_root_path":$VIDEO_ROOT} -``` diff --git a/docs/models/nemotron/nemotronh.md b/docs/models/nemotron/nemotronh.md index 645b87615a..5cfd1dcd41 100644 --- a/docs/models/nemotron/nemotronh.md +++ b/docs/models/nemotron/nemotronh.md @@ -1,13 +1,10 @@ -# Nemotron H and Nemotron Nano v2 +# Nemotron H -> **Deprecation notice:** Nemotron H v1 support (4B, 8B, 47B, and 56B) and -> Nemotron Nano v2 support (9B and 12B) are no longer actively maintained or -> tested against current upstream checkpoints and will be removed in Megatron -> Bridge 0.7.0. Nemotron 3 is not deprecated. +> **Deprecation notice:** Nemotron H v1 support (4B, 8B, 47B, and 56B) is no longer actively maintained or tested against current upstream checkpoints and will be removed in Megatron Bridge 0.7.0. Nemotron 3 is not deprecated. -[Nemotron H](https://huggingface.co/collections/nvidia/nemotron-h) and [Nemotron Nano v2](https://huggingface.co/collections/nvidia/nvidia-nemotron-v2) are families of **hybrid SSM-Attention models** from **NVIDIA** that combine Mamba (State Space Model) layers with traditional attention layers. These models achieve strong performance while maintaining computational efficiency through their hybrid architecture. +[Nemotron H](https://huggingface.co/collections/nvidia/nemotron-h) is a family of hybrid SSM-Attention models combining Mamba and attention layers. -The Nemotron H family includes models from 4B to 56B parameters with 8K context length, while Nemotron Nano v2 models (9B and 12B) are optimized for edge deployment with extended 128K context support. +The Nemotron H family includes models from 4B to 56B parameters with 8K context length. ## Model Families @@ -17,12 +14,6 @@ The Nemotron H family includes models from 4B to 56B parameters with 8K context - **47B**: 98 layers, 8192 hidden size, 8K context - **56B**: 118 layers, 8192 hidden size, 8K context -### Nemotron Nano v2 -- **9B**: 56 layers, 4480 hidden size, 128K context -- **12B**: 62 layers, 5120 hidden size, 128K context - -All models are supported via the Bridge system with specialized configurations for hybrid SSM-Attention architecture. - ## Model Architecture ### Common Features Across All Models @@ -82,32 +73,6 @@ All models are supported via the Bridge system with specialized configurations f - **Mamba state dim**: 256 - **Context Length**: 8K tokens -### Nemotron Nano 9B v2 Specifications -- **Parameters**: 9B -- **Layers**: 56 (Hybrid pattern: `M-M-M-MM-M-M-M*-M-M-M*-M-M-M-M*-M-M-M-M*-M-MM-M-M-M-M-M-`) -- **Hidden size**: 4480 -- **FFN hidden size**: 15680 -- **Attention heads**: 40 query heads, 8 key-value groups -- **KV channels**: 128 -- **Mamba heads**: 128 -- **Mamba head dim**: 80 -- **Mamba state dim**: 128 -- **Context Length**: 128K tokens -- **Vocab size**: 131,072 - -### Nemotron Nano 12B v2 Specifications -- **Parameters**: 12B -- **Layers**: 62 (Hybrid pattern: `M-M-M-M*-M-M-M-M*-M-M-M-M*-M-M-M-M*-M-M-M-M*-M-M-M-M*-M-M-M-M-`) -- **Hidden size**: 5120 -- **FFN hidden size**: 20480 -- **Attention heads**: 40 query heads, 8 key-value groups -- **KV channels**: 128 -- **Mamba heads**: 128 -- **Mamba head dim**: 80 -- **Mamba state dim**: 128 -- **Context Length**: 128K tokens -- **Vocab size**: 131,072 - ## Key Features ### Hybrid SSM-Attention Architecture @@ -122,10 +87,6 @@ All models are supported via the Bridge system with specialized configurations f - **Multi-Query Attention**: Efficient attention with shared key-value heads - **Selective State Space**: Mamba-2 architecture with selective gating -### Extended Context (Nano v2) -- **128K Context Window**: Nemotron Nano v2 models support up to 128K tokens -- **Efficient Long-Range Modeling**: Hybrid architecture optimized for long sequences - ## Conversion with πŸ€— Hugging Face ### Load HF β†’ Megatron @@ -153,30 +114,6 @@ model = provider.provide_distributed_model(wrap_with_ddp=False) # bridge = AutoBridge.from_hf_pretrained("nvidia/Nemotron-H-56B-Base-8K", trust_remote_code=True) ``` -#### Nemotron Nano v2 Models -```python -from megatron.bridge import AutoBridge - -# Example: Nemotron Nano 9B v2 -bridge = AutoBridge.from_hf_pretrained("nvidia/NVIDIA-Nemotron-Nano-9B-v2-Base", trust_remote_code=True) -provider = bridge.to_megatron_provider() - -# Configure parallelism -provider.tensor_model_parallel_size = 2 -provider.pipeline_model_parallel_size = 1 -provider.context_parallel_size = 1 -provider.sequence_parallel = True - -provider.finalize() -model = provider.provide_distributed_model(wrap_with_ddp=False) - -# For instruct variant: -# bridge = AutoBridge.from_hf_pretrained("nvidia/NVIDIA-Nemotron-Nano-9B-v2", trust_remote_code=True) - -# For 12B model: -# bridge = AutoBridge.from_hf_pretrained("nvidia/NVIDIA-Nemotron-Nano-12B-v2-Base", trust_remote_code=True) -``` - ### Export Megatron β†’ HF ```python # Convert from a Megatron checkpoint directory to HF format @@ -275,42 +212,6 @@ cfg = nemotronh_56b_peft_config( ) ``` -### Nemotron Nano 9B v2 Finetuning - -```python -from megatron.bridge.recipes.nemotronh import nemotron_nano_9b_v2_peft_config - -# LoRA finetuning -cfg = nemotron_nano_9b_v2_peft_config( - tokenizer_path="nvidia/NVIDIA-Nemotron-Nano-9B-v2-Base", - name="nano_9b_v2_lora", - pretrained_checkpoint="path/to/nano/9b/v2/checkpoint", - peft_scheme="lora", - train_iters=1000, - global_batch_size=128, - seq_length=2048, # Can use up to 128K - finetune_lr=1e-4, -) -``` - -### Nemotron Nano 12B v2 Finetuning - -```python -from megatron.bridge.recipes.nemotronh import nemotron_nano_12b_v2_peft_config - -# LoRA finetuning -cfg = nemotron_nano_12b_v2_peft_config( - tokenizer_path="nvidia/NVIDIA-Nemotron-Nano-12B-v2-Base", - name="nano_12b_v2_lora", - pretrained_checkpoint="path/to/nano/12b/v2/checkpoint", - peft_scheme="lora", - train_iters=1000, - global_batch_size=128, - seq_length=2048, # Can use up to 128K - finetune_lr=1e-4, -) -``` - ## Default Configurations ### Nemotron H Models @@ -356,44 +257,16 @@ cfg = nemotron_nano_12b_v2_peft_config( - Sequence Parallel: True - Precision: FP8 hybrid -### Nemotron Nano v2 Models - -#### 9B - LoRA (1 node, 8 GPUs) -- TP=2, PP=1, CP=1, LR=1e-4 -- Sequence Parallel: True -- Precision: BF16 mixed -- Context: Up to 128K tokens - -#### 9B - Full SFT (1 node, 8 GPUs) -- TP=2, PP=1, CP=1, LR=1e-4 -- Sequence Parallel: True -- Precision: BF16 mixed - -#### 12B - LoRA (2 nodes, 16 GPUs) -- TP=4, PP=1, CP=1, LR=1e-4 -- Sequence Parallel: True -- Precision: FP8 hybrid (recommended) -- Context: Up to 128K tokens - -#### 12B - Full SFT (2 nodes, 16 GPUs) -- TP=4, PP=1, CP=1, LR=1e-4 -- Sequence Parallel: True -- Precision: FP8 hybrid - ## API Reference ### Nemotron H - Nemotron H recipes: [bridge.recipes.nemotronh](../../apidocs/bridge/bridge.recipes.nemotronh.md) -### Nemotron Nano v2 -- Nemotron Nano v2 recipes: [bridge.recipes.nemotronh.nemotron_nano_v2](../../apidocs/bridge/bridge.recipes.nemotronh.md) - ## Performance Optimizations ### Memory Efficiency - **Selective Recomputation**: Reduces activation memory for larger models - **Sequence Parallelism**: Distributes sequence dimension across GPUs (enabled for 8B+) -- **Context Parallelism**: Support for ultra-long sequences (Nano v2) - **Manual GC**: Aggressive garbage collection for stable memory usage - **Precision-aware optimizer**: BF16/FP8 gradients with FP32 master weights @@ -426,12 +299,6 @@ Nemotron H models support several PP configurations with pre-defined layouts: - **47B Base**: [nvidia/Nemotron-H-47B-Base-8K](https://huggingface.co/nvidia/Nemotron-H-47B-Base-8K) - **56B Base**: [nvidia/Nemotron-H-56B-Base-8K](https://huggingface.co/nvidia/Nemotron-H-56B-Base-8K) -### Nemotron Nano v2 Models -- **9B Base**: [nvidia/NVIDIA-Nemotron-Nano-9B-v2-Base](https://huggingface.co/nvidia/NVIDIA-Nemotron-Nano-9B-v2-Base) -- **9B Instruct**: [nvidia/NVIDIA-Nemotron-Nano-9B-v2](https://huggingface.co/nvidia/NVIDIA-Nemotron-Nano-9B-v2) -- **12B Base**: [nvidia/NVIDIA-Nemotron-Nano-12B-v2-Base](https://huggingface.co/nvidia/NVIDIA-Nemotron-Nano-12B-v2-Base) -- **12B Instruct**: [nvidia/NVIDIA-Nemotron-Nano-12B-v2](https://huggingface.co/nvidia/NVIDIA-Nemotron-Nano-12B-v2) - ## Technical Resources ### Research Papers diff --git a/docs/nemo2-migration-guide.md b/docs/nemo2-migration-guide.md index dfd0f9ff49..687c542fc1 100644 --- a/docs/nemo2-migration-guide.md +++ b/docs/nemo2-migration-guide.md @@ -22,16 +22,15 @@ Megatron Bridge derives model provider settings from Hugging Face configs throug | `llm.Llama3Config8B` | `AutoBridge.from_hf_pretrained("meta-llama/Meta-Llama-3-8B").to_megatron_provider()` | | `llm.Llama31Config70B` | `AutoBridge.from_hf_pretrained("meta-llama/Llama-3.1-70B").to_megatron_provider()` | | `llm.Qwen2Config7B` | `AutoBridge.from_hf_pretrained("Qwen/Qwen2-7B").to_megatron_provider()` | -| `llm.DeepseekV2Config` | {py:class}`bridge.models.DeepseekV2ModelProvider` | ### Supported Model Families Megatron Bridge supports the following model families through bridge mappings and provider classes: - **Base Models**: `GPTModelProvider`, `T5ModelProvider`, `HybridModelProvider` -- **Llama**: Llama2, Llama3, Llama3.1, Llama3.2, CodeLlama, Llama4 +- **Llama**: Llama3, Llama3.1, Llama3.2, CodeLlama, Llama4 - **Qwen**: Qwen2, Qwen2.5, Qwen3, Qwen3MoE, Qwen2.5VL -- **DeepSeek**: DeepSeek, DeepSeekV2, DeepSeekV2Lite, DeepSeekV3, Moonlight -- **Nemotron**: Nemotron3, Nemotron4, NemotronH, NemotronNano +- **DeepSeek**: DeepSeek V3, DeepSeek V4 +- **Nemotron**: Nemotron 3, Nemotron 3.5, Nemotron H (deprecated) - **NVIDIA Mamba**: Mamba variants and hybrid models For a complete list of bridge mappings, providers, and their parameters, see {py:mod}`bridge.models`. diff --git a/examples/conversion/hf_to_megatron_generate_nemotron_vlm.py b/examples/conversion/hf_to_megatron_generate_nemotron_vlm.py deleted file mode 100644 index 942a7c1700..0000000000 --- a/examples/conversion/hf_to_megatron_generate_nemotron_vlm.py +++ /dev/null @@ -1,494 +0,0 @@ -# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -""" -Examples with Nemotron Nano V2 VL: - # Single image using Megatron checkpoint: - uv run python examples/conversion/hf_to_megatron_generate_nemotron_vlm.py \ - --image_path="https://huggingface.co/nvidia/NVIDIA-Nemotron-Nano-12B-v2-VL-BF16/resolve/main/images/table.png" \ - --prompt="Describe this image." \ - --max_new_tokens 300 - - # Multiple images: - uv run python examples/conversion/hf_to_megatron_generate_nemotron_vlm.py \ - --image_path="https://huggingface.co/nvidia/NVIDIA-Nemotron-Nano-12B-v2-VL-BF16/resolve/main/images/example1a.jpeg,https://huggingface.co/nvidia/NVIDIA-Nemotron-Nano-12B-v2-VL-BF16/resolve/main/images/example1b.jpeg" \ - --prompt="Describe the two images in detail." \ - --max_new_tokens 300 - - # Video description: - uv run python examples/conversion/hf_to_megatron_generate_nemotron_vlm.py \ - --video_path="https://huggingface.co/nvidia/NVIDIA-Nemotron-Nano-12B-v2-VL-BF16/resolve/main/images/demo.mp4" \ - --prompt="Describe what you see." \ - --max_new_tokens 300 -""" - -import argparse -import io -from typing import Optional - -import torch -import torch.distributed as dist -from megatron.core import parallel_state -from megatron.core.pipeline_parallel.schedules import get_forward_backward_func -from PIL import Image -from qwen_vl_utils import process_vision_info -from transformers import AutoProcessor, AutoTokenizer - -from megatron.bridge import AutoBridge -from megatron.bridge.models.nemotron_vl.nemotron_vl_utils import adjust_image_tokens -from megatron.bridge.utils.common_utils import get_last_rank, print_rank_0 -from megatron.bridge.utils.safe_url import is_safe_public_http_url, safe_url_open - - -class SingleBatchIterator: - """Iterator that yields a single batch of data for text generation. - Required by the forward_backward_func function. - - This class creates an iterator that yields exactly one batch containing - input tokens, position IDs, attention mask, and optional vision inputs, - then raises StopIteration. Used for single-step inference in the forward pass. - """ - - def __init__(self, input_ids, position_ids, attention_mask, **kwargs): - self.batch = dict( - tokens=input_ids, - position_ids=position_ids, - attention_mask=attention_mask, - ) - - # Add vision inputs if provided - if kwargs.get("images", None) is not None: - self.batch["images"] = kwargs.get("images", None) - elif kwargs.get("pixel_values", None) is not None: - self.batch["pixel_values"] = kwargs.get("pixel_values", None) - - self._yielded = False - - def __iter__(self): - return self - - def __next__(self): - if self._yielded: - raise StopIteration - self._yielded = True - return self.batch - - -def vlm_forward_step(data_iterator, model, **kwargs) -> torch.Tensor: - """Forward step function for vision-language generation. - Required by the forward_backward_func function. - - Extracts a batch from the data iterator and runs the model forward pass - with the provided input tokens, position IDs, attention mask, and vision inputs. - - Args: - data_iterator: Iterator providing batches of input data - model: The Megatron model to run forward pass on - **kwargs: Additional keyword arguments (unused) - - Returns: - Tuple of (model_output, loss_function) - """ - batch = next(data_iterator) - forward_args = { - "input_ids": batch["tokens"], - "position_ids": batch["position_ids"], - "attention_mask": batch.get("attention_mask", None), - } - - if "images" in batch: - forward_args["images"] = batch["images"] - elif "pixel_values" in batch: - forward_args["pixel_values"] = batch["pixel_values"] - - def loss_func(x, **kwargs): - return x - - output = model(**forward_args) - if isinstance(output, tuple): - output = output[0] - return output, loss_func - - -def load_image(image_path: str) -> Image.Image: - """Load an image from URL or file path. - - Args: - image_path: URL or local file path to the image - - Returns: - PIL Image object - """ - if image_path.startswith(("http://", "https://")): - is_safe, reason = is_safe_public_http_url(image_path) - if not is_safe: - raise ValueError(f"Refusing to fetch image URL ({reason}): {image_path}") - with safe_url_open(image_path) as resp: - return Image.open(io.BytesIO(resp.read())) - return Image.open(image_path) - - -def process_image_inputs(processor, image_path: Optional[str], prompt: str, system_prompt: Optional[str] = None): - """Process image inputs for vision-language model. - - Args: - processor: AutoProcessor for the VL model - image_path: Path or URL to the image (optional) - prompt: Text prompt - - Returns: - Tuple of (input_ids, pixel_values, num_patches) - """ - if image_path: - if "," in image_path: - image_paths = image_path.split(",") - content = [] - for i, path in enumerate(image_paths): - prefix = "\n" if i > 0 else "" - content.append({"type": "text", "text": f"{prefix}Image-{i + 1}: "}) - content.append({"type": "image", "image": path}) - content.append({"type": "text", "text": "\n" + prompt}) - else: - content = [ - {"type": "image", "image": image_path}, - {"type": "text", "text": prompt}, - ] - # Create messages with image and text - messages = [ - { - "role": "user", - "content": content, - } - ] - if system_prompt: - messages.insert(0, {"role": "system", "content": system_prompt}) - - # Process vision info - image_inputs, video_inputs = process_vision_info(messages) - - # Apply chat template - text = processor.apply_chat_template(messages, tokenize=False, add_generation_prompt=True) - - # Process inputs - inputs = processor( - text=[text], - images=image_inputs, - videos=video_inputs, - padding=processor.tokenizer.pad_token is not None, - return_tensors="pt", - ) - return inputs.input_ids, inputs.pixel_values, inputs.num_patches - else: - # Text-only processing - inputs = processor(text=[prompt], return_tensors="pt") - return inputs.input_ids, None, 0 - - -def process_video_inputs(processor, video_path: Optional[str], prompt: str, system_prompt: Optional[str] = None): - """Process video inputs for vision-language model.""" - from megatron.bridge.models.nemotron_vl.nemotron_vl_utils import ( - maybe_path_or_url_to_data_urls, - pil_image_from_base64, - ) - - video_fps = -1 - video_nframe = 10 - video_nframe_max = -1 - - # Get frames and metadata - image_urls, metadata = maybe_path_or_url_to_data_urls( - video_path, - fps=max(0, int(video_fps)), - nframe=max(0, int(video_nframe)), - nframe_max=int(video_nframe_max), - ) - frames = [pil_image_from_base64(image_url) for image_url in image_urls] - - print(f"Video Metadata: {metadata}") - - messages = [ - { - "role": "user", - "content": [ - { - "type": "video", - "video": f"file://{video_path}", - }, - { - "type": "text", - "text": "\n" + prompt, - }, - ], - } - ] - if system_prompt: - messages.insert(0, {"role": "system", "content": system_prompt}) - prompt = processor.apply_chat_template(messages, tokenize=False, add_generation_prompt=True) - - # Process with FPS metadata - if metadata: - inputs = processor( - text=[prompt], - videos=frames, - videos_kwargs={"video_metadata": metadata}, - return_tensors="pt", - ) - else: - inputs = processor( - text=[prompt], - videos=frames, - return_tensors="pt", - ) - return inputs.input_ids, inputs.pixel_values_videos, inputs.num_patches - - -def main(args) -> None: - """Main function for vision-language generation from HuggingFace VL models. - - Loads a VL model either from HuggingFace (with optional conversion to Megatron) - or directly from a Megatron checkpoint, then performs greedy generation - using the provided prompt and optional image input. - - Args: - args: Parsed command line arguments containing model paths, prompt, - image path, parallelism settings, and generation parameters - """ - # pylint: disable=C0115,C0116 - tp = args.tp - pp = args.pp - ep = args.ep - etp = args.etp - - # Choose loading method based on arguments - if args.megatron_model_path: - # Load from Megatron checkpoint - print_rank_0(f"Loading Megatron model from: {args.megatron_model_path}") - - # We still need HF config for tokenizer, but we'll load the model from Megatron checkpoint - # Create bridge from HF config only (no weights) - bridge = AutoBridge.from_hf_pretrained(args.hf_model_path, trust_remote_code=True) - - # Initialize model parallel before loading - model_provider = bridge.to_megatron_provider(load_weights=False) - model_provider.tensor_model_parallel_size = tp - model_provider.pipeline_model_parallel_size = pp - model_provider.expert_model_parallel_size = ep - model_provider.expert_tensor_parallel_size = etp - model_provider.pipeline_dtype = torch.bfloat16 - model_provider.initialize_model_parallel(seed=0) - - # Load the Megatron model directly - model = bridge.load_megatron_model( - args.megatron_model_path, - mp_overrides={ - "tensor_model_parallel_size": tp, - "pipeline_model_parallel_size": pp, - "expert_model_parallel_size": ep, - "expert_tensor_parallel_size": etp, - "pipeline_dtype": torch.bfloat16, - }, - wrap_with_ddp=False, - ) - - else: - # Load from HuggingFace and convert to Megatron - print_rank_0(f"Loading HuggingFace model from: {args.hf_model_path}") - bridge = AutoBridge.from_hf_pretrained(args.hf_model_path, trust_remote_code=True) - model_provider = bridge.to_megatron_provider(load_weights=True) - model_provider.tensor_model_parallel_size = tp - model_provider.pipeline_model_parallel_size = pp - model_provider.expert_model_parallel_size = ep - model_provider.expert_tensor_parallel_size = etp - model_provider.pipeline_dtype = torch.bfloat16 - model_provider.initialize_model_parallel(seed=0) - model_provider.finalize() - model = model_provider.provide_distributed_model(wrap_with_ddp=False) - - model = [m.cuda() for m in model] - for m in model: - m.eval() - - # Initialize tokenizer and processor - tokenizer = AutoTokenizer.from_pretrained(args.hf_model_path, trust_remote_code=True) - processor = AutoProcessor.from_pretrained(args.hf_model_path, trust_remote_code=True) - img_start_token_id = tokenizer.convert_tokens_to_ids("") - img_end_token_id = tokenizer.convert_tokens_to_ids("") - - if tokenizer.pad_token is None: - tokenizer.pad_token = tokenizer.eos_token - if args.video_path: - input_ids, pixel_values, num_patches = process_video_inputs( - processor, args.video_path, args.prompt, args.system_prompt - ) - else: - input_ids, pixel_values, num_patches = process_image_inputs( - processor, args.image_path, args.prompt, args.system_prompt - ) - - if pixel_values is None: - images = torch.empty((0, 3, 512, 512), dtype=torch.bfloat16) - else: - images = pixel_values.bfloat16() - input_ids = adjust_image_tokens(input_ids, num_patches, img_start_token_id, img_end_token_id) - if args.video_path: - video_token_id = tokenizer.convert_tokens_to_ids("