diff --git a/docs/_static/model-verification/catalog-v1.json b/docs/_static/model-verification/catalog-v1.json index 6f82bbae6e..20cde5af5d 100644 --- a/docs/_static/model-verification/catalog-v1.json +++ b/docs/_static/model-verification/catalog-v1.json @@ -912,7 +912,7 @@ } ], "commands": [ - "./scripts/training/train.sh --nodes 8 --gpus-per-node 4 --recipe deepseek_v4_flash_sft_openmath_thinking_packed_gb200_config --mode sft --step-func dsv4_step --pretrained_checkpoint work/models/deepseek-v4-flash-megatron --save_dir work/model-verification/dsv4-flash/sft-ref --save_interval 100 --max_steps 100 --seq_length 1024 --pipeline_model_parallel_size 4 --context_parallel_size 1 --expert_model_parallel_size 8 'model.pipeline_model_parallel_layout=Et*11|t*11|t*11|t*10mL' scheduler.lr_warmup_iters=10 scheduler.lr_decay_iters=100 validation.eval_interval=0 validation.eval_iters=0 rng.seed=5678 ddp.check_for_nan_in_grad=true ddp.check_for_large_grads=true rerun_state_machine.check_for_nan_in_loss=true checkpoint.exit_on_missing_checkpoint=false checkpoint.load=null checkpoint.load_optim=false checkpoint.load_rng=false checkpoint.save_optim=false ddp.overlap_grad_reduce=false logger.log_interval=1 logger.log_throughput=true logger.tensorboard_dir=null dist.distributed_timeout_minutes=180" + "./scripts/training/train.sh --nodes 8 --gpus-per-node 4 --recipe deepseek_v4_flash_sft_openmath_thinking_packed_gb200_config --mode sft --step-func dsv4_step --pretrained_checkpoint work/model-verification/dsv4-flash/import-gpu/iter_0000000 --save_dir work/model-verification/dsv4-flash/sft-ref --save_interval 100 --max_steps 100 --seq_length 1024 --pipeline_model_parallel_size 4 --context_parallel_size 1 --expert_model_parallel_size 8 'model.pipeline_model_parallel_layout=Et*11|t*11|t*11|t*10mL' scheduler.lr_warmup_iters=10 scheduler.lr_decay_iters=100 validation.eval_interval=0 validation.eval_iters=0 rng.seed=5678 ddp.check_for_nan_in_grad=true ddp.check_for_large_grads=true rerun_state_machine.check_for_nan_in_loss=true checkpoint.exit_on_missing_checkpoint=false checkpoint.load=null checkpoint.load_optim=false checkpoint.load_rng=false checkpoint.save_optim=false ddp.overlap_grad_reduce=false logger.log_interval=1 logger.log_throughput=true logger.tensorboard_dir=null dist.distributed_timeout_minutes=180" ], "dimensions": {}, "enabled_features": { @@ -984,7 +984,7 @@ } ], "commands": [ - "./scripts/training/train.sh --nodes 16 --gpus-per-node 4 --recipe deepseek_v4_flash_sft_openmath_thinking_packed_config --step-func dsv4_step --pretrained_checkpoint work/models/deepseek-v4-flash-megatron --save_dir work/model-verification/dsv4-flash/sft-long-context-ref --save_interval 100 --max_steps 100 model.pipeline_model_parallel_size=4 'model.pipeline_model_parallel_layout=Et*11|t*11|t*11|t*10mL' model.context_parallel_size=2 model.cp_partition_mode=contiguous model.expert_model_parallel_size=8 model.moe_grouped_gemm=false model.recompute_granularity=full model.recompute_method=uniform model.recompute_num_layers=1 dataset.seq_length=1024 scheduler.lr_warmup_iters=10 scheduler.lr_decay_iters=100 validation.eval_interval=0 validation.eval_iters=0 rng.seed=5678 ddp.check_for_nan_in_grad=true ddp.check_for_large_grads=true rerun_state_machine.check_for_nan_in_loss=true checkpoint.exit_on_missing_checkpoint=false checkpoint.load=null ddp.overlap_grad_reduce=false logger.log_interval=1 logger.log_throughput=true logger.tensorboard_dir=null dist.distributed_timeout_minutes=180" + "./scripts/training/train.sh --nodes 16 --gpus-per-node 4 --recipe deepseek_v4_flash_sft_openmath_thinking_packed_config --step-func dsv4_step --pretrained_checkpoint work/model-verification/dsv4-flash/import-gpu/iter_0000000 --save_dir work/model-verification/dsv4-flash/sft-long-context-ref --save_interval 100 --max_steps 100 model.pipeline_model_parallel_size=4 'model.pipeline_model_parallel_layout=Et*11|t*11|t*11|t*10mL' model.context_parallel_size=2 model.cp_partition_mode=contiguous model.expert_model_parallel_size=8 model.moe_grouped_gemm=false model.recompute_granularity=full model.recompute_method=uniform model.recompute_num_layers=1 dataset.seq_length=1024 scheduler.lr_warmup_iters=10 scheduler.lr_decay_iters=100 validation.eval_interval=0 validation.eval_iters=0 rng.seed=5678 ddp.check_for_nan_in_grad=true ddp.check_for_large_grads=true rerun_state_machine.check_for_nan_in_loss=true checkpoint.exit_on_missing_checkpoint=false checkpoint.load=null ddp.overlap_grad_reduce=false logger.log_interval=1 logger.log_throughput=true logger.tensorboard_dir=null dist.distributed_timeout_minutes=180" ], "dimensions": {}, "enabled_features": { @@ -1023,7 +1023,7 @@ } ], "commands": [ - "./scripts/training/train.sh --nodes 8 --gpus-per-node 4 --recipe deepseek_v4_flash_peft_openmath_thinking_packed_gb200_config --mode lora --step-func dsv4_step --pretrained_checkpoint work/models/deepseek-v4-flash-megatron --save_dir work/model-verification/dsv4-flash/peft-ref --save_interval 100 --max_steps 100 --seq_length 1024 --pipeline_model_parallel_size 4 --context_parallel_size 1 --expert_model_parallel_size 8 'model.pipeline_model_parallel_layout=Et*11|t*11|t*11|t*10mL' scheduler.lr_warmup_iters=10 scheduler.lr_decay_iters=100 validation.eval_interval=0 validation.eval_iters=0 rng.seed=5678 ddp.check_for_nan_in_grad=true ddp.check_for_large_grads=true rerun_state_machine.check_for_nan_in_loss=true checkpoint.exit_on_missing_checkpoint=false checkpoint.load=null checkpoint.load_optim=false checkpoint.load_rng=false checkpoint.save_optim=false ddp.overlap_grad_reduce=false logger.log_interval=1 logger.log_throughput=true logger.tensorboard_dir=null dist.distributed_timeout_minutes=180" + "./scripts/training/train.sh --nodes 8 --gpus-per-node 4 --recipe deepseek_v4_flash_peft_openmath_thinking_packed_gb200_config --mode lora --step-func dsv4_step --pretrained_checkpoint work/model-verification/dsv4-flash/import-gpu/iter_0000000 --save_dir work/model-verification/dsv4-flash/peft-ref --save_interval 100 --max_steps 100 --seq_length 1024 --pipeline_model_parallel_size 4 --context_parallel_size 1 --expert_model_parallel_size 8 'model.pipeline_model_parallel_layout=Et*11|t*11|t*11|t*10mL' scheduler.lr_warmup_iters=10 scheduler.lr_decay_iters=100 validation.eval_interval=0 validation.eval_iters=0 rng.seed=5678 ddp.check_for_nan_in_grad=true ddp.check_for_large_grads=true rerun_state_machine.check_for_nan_in_loss=true checkpoint.exit_on_missing_checkpoint=false checkpoint.load=null checkpoint.load_optim=false checkpoint.load_rng=false checkpoint.save_optim=false ddp.overlap_grad_reduce=false logger.log_interval=1 logger.log_throughput=true logger.tensorboard_dir=null dist.distributed_timeout_minutes=180" ], "dimensions": {}, "enabled_features": { @@ -9911,7 +9911,7 @@ } ], "commands": [ - "./scripts/training/train.sh --nodes 2 --gpus-per-node 4 --recipe qwen35_vl_35b_a3b_sft_8gpu_gb200_bf16_functional_config --dataset medpix --pretrained_checkpoint work/model-verification/qwen3.6-35b-a3b/gb200-imported/iter_0000000 --max_steps 100 --warmup_iters 10 'dataset.source.load_kwargs={revision:69eb219d477ab8699296da1fd6b89beb768042d3}' 'dataset.source.adapter_kwargs={resized_height:448,resized_width:448}' dataset.hf_processor_path=Qwen/Qwen3.6-35B-A3B 'dataset.hf_processor_kwargs={revision:995ad96eacd98c81ed38be0c5b274b04031597b0}' scheduler.lr_decay_iters=100 model.hf_model_id=Qwen/Qwen3.6-35B-A3B model.bos_token_id=248044 logger.save_config_filepath=work/model-verification/qwen3.6-35b-a3b/gb200-sft/resolved-config.yaml" + "./scripts/training/train.sh --nodes 2 --gpus-per-node 4 --recipe qwen35_vl_35b_a3b_sft_8gpu_gb200_bf16_functional_config --dataset medpix --pretrained_checkpoint work/model-verification/qwen3.6-35b-a3b/imported-megatron/iter_0000000 --max_steps 100 --warmup_iters 10 'dataset.source.load_kwargs={revision:69eb219d477ab8699296da1fd6b89beb768042d3}' 'dataset.source.adapter_kwargs={resized_height:448,resized_width:448}' dataset.hf_processor_path=Qwen/Qwen3.6-35B-A3B 'dataset.hf_processor_kwargs={revision:995ad96eacd98c81ed38be0c5b274b04031597b0}' scheduler.lr_decay_iters=100 model.hf_model_id=Qwen/Qwen3.6-35B-A3B model.bos_token_id=248044 logger.save_config_filepath=work/model-verification/qwen3.6-35b-a3b/gb200-sft/resolved-config.yaml" ], "dimensions": {}, "enabled_features": { @@ -10054,7 +10054,7 @@ } ], "commands": [ - "./scripts/training/train.sh --nodes 2 --gpus-per-node 4 --recipe qwen35_vl_35b_a3b_peft_8gpu_gb200_bf16_functional_config --dataset medpix --pretrained_checkpoint work/model-verification/qwen3.6-35b-a3b/gb200-imported/iter_0000000 --max_steps 100 --warmup_iters 10 'dataset.source.load_kwargs={revision:69eb219d477ab8699296da1fd6b89beb768042d3}' 'dataset.source.adapter_kwargs={resized_height:448,resized_width:448}' dataset.hf_processor_path=Qwen/Qwen3.6-35B-A3B 'dataset.hf_processor_kwargs={revision:995ad96eacd98c81ed38be0c5b274b04031597b0}' scheduler.lr_decay_iters=100 model.hf_model_id=Qwen/Qwen3.6-35B-A3B model.bos_token_id=248044 logger.save_config_filepath=work/model-verification/qwen3.6-35b-a3b/gb200-peft/resolved-config.yaml" + "./scripts/training/train.sh --nodes 2 --gpus-per-node 4 --recipe qwen35_vl_35b_a3b_peft_8gpu_gb200_bf16_functional_config --dataset medpix --pretrained_checkpoint work/model-verification/qwen3.6-35b-a3b/imported-megatron/iter_0000000 --max_steps 100 --warmup_iters 10 'dataset.source.load_kwargs={revision:69eb219d477ab8699296da1fd6b89beb768042d3}' 'dataset.source.adapter_kwargs={resized_height:448,resized_width:448}' dataset.hf_processor_path=Qwen/Qwen3.6-35B-A3B 'dataset.hf_processor_kwargs={revision:995ad96eacd98c81ed38be0c5b274b04031597b0}' scheduler.lr_decay_iters=100 model.hf_model_id=Qwen/Qwen3.6-35B-A3B model.bos_token_id=248044 logger.save_config_filepath=work/model-verification/qwen3.6-35b-a3b/gb200-peft/resolved-config.yaml" ], "dimensions": {}, "enabled_features": { diff --git a/docs/fern/versions/nightly/pages/models/deepseek/deepseek-v4.mdx b/docs/fern/versions/nightly/pages/models/deepseek/deepseek-v4.mdx index a395e3dc94..e271def87e 100644 --- a/docs/fern/versions/nightly/pages/models/deepseek/deepseek-v4.mdx +++ b/docs/fern/versions/nightly/pages/models/deepseek/deepseek-v4.mdx @@ -315,7 +315,7 @@ Choose a workflow, precision, and exact recorded combination. The command and ex Command - 
./scripts/training/train.sh --nodes 8 --gpus-per-node 4 --recipe deepseek_v4_flash_sft_openmath_thinking_packed_gb200_config --mode sft --step-func dsv4_step --pretrained_checkpoint work/models/deepseek-v4-flash-megatron --save_dir work/model-verification/dsv4-flash/sft-ref --save_interval 100 --max_steps 100 --seq_length 1024 --pipeline_model_parallel_size 4 --context_parallel_size 1 --expert_model_parallel_size 8 'model.pipeline_model_parallel_layout=Et*11|t*11|t*11|t*10mL' scheduler.lr_warmup_iters=10 scheduler.lr_decay_iters=100 validation.eval_interval=0 validation.eval_iters=0 rng.seed=5678 ddp.check_for_nan_in_grad=true ddp.check_for_large_grads=true rerun_state_machine.check_for_nan_in_loss=true checkpoint.exit_on_missing_checkpoint=false checkpoint.load=null checkpoint.load_optim=false checkpoint.load_rng=false checkpoint.save_optim=false ddp.overlap_grad_reduce=false logger.log_interval=1 logger.log_throughput=true logger.tensorboard_dir=null dist.distributed_timeout_minutes=180
+ 
./scripts/training/train.sh --nodes 8 --gpus-per-node 4 --recipe deepseek_v4_flash_sft_openmath_thinking_packed_gb200_config --mode sft --step-func dsv4_step --pretrained_checkpoint work/model-verification/dsv4-flash/import-gpu/iter_0000000 --save_dir work/model-verification/dsv4-flash/sft-ref --save_interval 100 --max_steps 100 --seq_length 1024 --pipeline_model_parallel_size 4 --context_parallel_size 1 --expert_model_parallel_size 8 'model.pipeline_model_parallel_layout=Et*11|t*11|t*11|t*10mL' scheduler.lr_warmup_iters=10 scheduler.lr_decay_iters=100 validation.eval_interval=0 validation.eval_iters=0 rng.seed=5678 ddp.check_for_nan_in_grad=true ddp.check_for_large_grads=true rerun_state_machine.check_for_nan_in_loss=true checkpoint.exit_on_missing_checkpoint=false checkpoint.load=null checkpoint.load_optim=false checkpoint.load_rng=false checkpoint.save_optim=false ddp.overlap_grad_reduce=false logger.log_interval=1 logger.log_throughput=true logger.tensorboard_dir=null dist.distributed_timeout_minutes=180
@@ -365,7 +365,7 @@ Choose a workflow, precision, and exact recorded combination. The command and ex Command - 
./scripts/training/train.sh --nodes 16 --gpus-per-node 4 --recipe deepseek_v4_flash_sft_openmath_thinking_packed_config --step-func dsv4_step --pretrained_checkpoint work/models/deepseek-v4-flash-megatron --save_dir work/model-verification/dsv4-flash/sft-long-context-ref --save_interval 100 --max_steps 100 model.pipeline_model_parallel_size=4 'model.pipeline_model_parallel_layout=Et*11|t*11|t*11|t*10mL' model.context_parallel_size=2 model.cp_partition_mode=contiguous model.expert_model_parallel_size=8 model.moe_grouped_gemm=false model.recompute_granularity=full model.recompute_method=uniform model.recompute_num_layers=1 dataset.seq_length=1024 scheduler.lr_warmup_iters=10 scheduler.lr_decay_iters=100 validation.eval_interval=0 validation.eval_iters=0 rng.seed=5678 ddp.check_for_nan_in_grad=true ddp.check_for_large_grads=true rerun_state_machine.check_for_nan_in_loss=true checkpoint.exit_on_missing_checkpoint=false checkpoint.load=null ddp.overlap_grad_reduce=false logger.log_interval=1 logger.log_throughput=true logger.tensorboard_dir=null dist.distributed_timeout_minutes=180
+ 
./scripts/training/train.sh --nodes 16 --gpus-per-node 4 --recipe deepseek_v4_flash_sft_openmath_thinking_packed_config --step-func dsv4_step --pretrained_checkpoint work/model-verification/dsv4-flash/import-gpu/iter_0000000 --save_dir work/model-verification/dsv4-flash/sft-long-context-ref --save_interval 100 --max_steps 100 model.pipeline_model_parallel_size=4 'model.pipeline_model_parallel_layout=Et*11|t*11|t*11|t*10mL' model.context_parallel_size=2 model.cp_partition_mode=contiguous model.expert_model_parallel_size=8 model.moe_grouped_gemm=false model.recompute_granularity=full model.recompute_method=uniform model.recompute_num_layers=1 dataset.seq_length=1024 scheduler.lr_warmup_iters=10 scheduler.lr_decay_iters=100 validation.eval_interval=0 validation.eval_iters=0 rng.seed=5678 ddp.check_for_nan_in_grad=true ddp.check_for_large_grads=true rerun_state_machine.check_for_nan_in_loss=true checkpoint.exit_on_missing_checkpoint=false checkpoint.load=null ddp.overlap_grad_reduce=false logger.log_interval=1 logger.log_throughput=true logger.tensorboard_dir=null dist.distributed_timeout_minutes=180
@@ -423,7 +423,7 @@ Choose a workflow, precision, and exact recorded combination. The command and ex Command - 
./scripts/training/train.sh --nodes 8 --gpus-per-node 4 --recipe deepseek_v4_flash_peft_openmath_thinking_packed_gb200_config --mode lora --step-func dsv4_step --pretrained_checkpoint work/models/deepseek-v4-flash-megatron --save_dir work/model-verification/dsv4-flash/peft-ref --save_interval 100 --max_steps 100 --seq_length 1024 --pipeline_model_parallel_size 4 --context_parallel_size 1 --expert_model_parallel_size 8 'model.pipeline_model_parallel_layout=Et*11|t*11|t*11|t*10mL' scheduler.lr_warmup_iters=10 scheduler.lr_decay_iters=100 validation.eval_interval=0 validation.eval_iters=0 rng.seed=5678 ddp.check_for_nan_in_grad=true ddp.check_for_large_grads=true rerun_state_machine.check_for_nan_in_loss=true checkpoint.exit_on_missing_checkpoint=false checkpoint.load=null checkpoint.load_optim=false checkpoint.load_rng=false checkpoint.save_optim=false ddp.overlap_grad_reduce=false logger.log_interval=1 logger.log_throughput=true logger.tensorboard_dir=null dist.distributed_timeout_minutes=180
+ 
./scripts/training/train.sh --nodes 8 --gpus-per-node 4 --recipe deepseek_v4_flash_peft_openmath_thinking_packed_gb200_config --mode lora --step-func dsv4_step --pretrained_checkpoint work/model-verification/dsv4-flash/import-gpu/iter_0000000 --save_dir work/model-verification/dsv4-flash/peft-ref --save_interval 100 --max_steps 100 --seq_length 1024 --pipeline_model_parallel_size 4 --context_parallel_size 1 --expert_model_parallel_size 8 'model.pipeline_model_parallel_layout=Et*11|t*11|t*11|t*10mL' scheduler.lr_warmup_iters=10 scheduler.lr_decay_iters=100 validation.eval_interval=0 validation.eval_iters=0 rng.seed=5678 ddp.check_for_nan_in_grad=true ddp.check_for_large_grads=true rerun_state_machine.check_for_nan_in_loss=true checkpoint.exit_on_missing_checkpoint=false checkpoint.load=null checkpoint.load_optim=false checkpoint.load_rng=false checkpoint.save_optim=false ddp.overlap_grad_reduce=false logger.log_interval=1 logger.log_throughput=true logger.tensorboard_dir=null dist.distributed_timeout_minutes=180
diff --git a/docs/fern/versions/nightly/pages/models/qwen/qwen3.6-35b-a3b.mdx b/docs/fern/versions/nightly/pages/models/qwen/qwen3.6-35b-a3b.mdx index 3f47c21fd8..3304c9a1f6 100644 --- a/docs/fern/versions/nightly/pages/models/qwen/qwen3.6-35b-a3b.mdx +++ b/docs/fern/versions/nightly/pages/models/qwen/qwen3.6-35b-a3b.mdx @@ -365,7 +365,7 @@ Choose a workflow, precision, and exact recorded combination. The command and ex Command - 
./scripts/training/train.sh --nodes 2 --gpus-per-node 4 --recipe qwen35_vl_35b_a3b_sft_8gpu_gb200_bf16_functional_config --dataset medpix --pretrained_checkpoint work/model-verification/qwen3.6-35b-a3b/gb200-imported/iter_0000000 --max_steps 100 --warmup_iters 10 'dataset.source.load_kwargs={revision:69eb219d477ab8699296da1fd6b89beb768042d3}' 'dataset.source.adapter_kwargs={resized_height:448,resized_width:448}' dataset.hf_processor_path=Qwen/Qwen3.6-35B-A3B 'dataset.hf_processor_kwargs={revision:995ad96eacd98c81ed38be0c5b274b04031597b0}' scheduler.lr_decay_iters=100 model.hf_model_id=Qwen/Qwen3.6-35B-A3B model.bos_token_id=248044 logger.save_config_filepath=work/model-verification/qwen3.6-35b-a3b/gb200-sft/resolved-config.yaml
+ 
./scripts/training/train.sh --nodes 2 --gpus-per-node 4 --recipe qwen35_vl_35b_a3b_sft_8gpu_gb200_bf16_functional_config --dataset medpix --pretrained_checkpoint work/model-verification/qwen3.6-35b-a3b/imported-megatron/iter_0000000 --max_steps 100 --warmup_iters 10 'dataset.source.load_kwargs={revision:69eb219d477ab8699296da1fd6b89beb768042d3}' 'dataset.source.adapter_kwargs={resized_height:448,resized_width:448}' dataset.hf_processor_path=Qwen/Qwen3.6-35B-A3B 'dataset.hf_processor_kwargs={revision:995ad96eacd98c81ed38be0c5b274b04031597b0}' scheduler.lr_decay_iters=100 model.hf_model_id=Qwen/Qwen3.6-35B-A3B model.bos_token_id=248044 logger.save_config_filepath=work/model-verification/qwen3.6-35b-a3b/gb200-sft/resolved-config.yaml
@@ -515,7 +515,7 @@ Choose a workflow, precision, and exact recorded combination. The command and ex Command - 
./scripts/training/train.sh --nodes 2 --gpus-per-node 4 --recipe qwen35_vl_35b_a3b_peft_8gpu_gb200_bf16_functional_config --dataset medpix --pretrained_checkpoint work/model-verification/qwen3.6-35b-a3b/gb200-imported/iter_0000000 --max_steps 100 --warmup_iters 10 'dataset.source.load_kwargs={revision:69eb219d477ab8699296da1fd6b89beb768042d3}' 'dataset.source.adapter_kwargs={resized_height:448,resized_width:448}' dataset.hf_processor_path=Qwen/Qwen3.6-35B-A3B 'dataset.hf_processor_kwargs={revision:995ad96eacd98c81ed38be0c5b274b04031597b0}' scheduler.lr_decay_iters=100 model.hf_model_id=Qwen/Qwen3.6-35B-A3B model.bos_token_id=248044 logger.save_config_filepath=work/model-verification/qwen3.6-35b-a3b/gb200-peft/resolved-config.yaml
+ 
./scripts/training/train.sh --nodes 2 --gpus-per-node 4 --recipe qwen35_vl_35b_a3b_peft_8gpu_gb200_bf16_functional_config --dataset medpix --pretrained_checkpoint work/model-verification/qwen3.6-35b-a3b/imported-megatron/iter_0000000 --max_steps 100 --warmup_iters 10 'dataset.source.load_kwargs={revision:69eb219d477ab8699296da1fd6b89beb768042d3}' 'dataset.source.adapter_kwargs={resized_height:448,resized_width:448}' dataset.hf_processor_path=Qwen/Qwen3.6-35B-A3B 'dataset.hf_processor_kwargs={revision:995ad96eacd98c81ed38be0c5b274b04031597b0}' scheduler.lr_decay_iters=100 model.hf_model_id=Qwen/Qwen3.6-35B-A3B model.bos_token_id=248044 logger.save_config_filepath=work/model-verification/qwen3.6-35b-a3b/gb200-peft/resolved-config.yaml
diff --git a/docs/models/deepseek/deepseek-v4.md b/docs/models/deepseek/deepseek-v4.md index 67a4a677e2..8b618818b3 100644 --- a/docs/models/deepseek/deepseek-v4.md +++ b/docs/models/deepseek/deepseek-v4.md @@ -320,7 +320,7 @@ Choose a workflow, precision, and exact recorded combination. The command and ex Command - 
./scripts/training/train.sh --nodes 8 --gpus-per-node 4 --recipe deepseek_v4_flash_sft_openmath_thinking_packed_gb200_config --mode sft --step-func dsv4_step --pretrained_checkpoint work/models/deepseek-v4-flash-megatron --save_dir work/model-verification/dsv4-flash/sft-ref --save_interval 100 --max_steps 100 --seq_length 1024 --pipeline_model_parallel_size 4 --context_parallel_size 1 --expert_model_parallel_size 8 'model.pipeline_model_parallel_layout=Et*11|t*11|t*11|t*10mL' scheduler.lr_warmup_iters=10 scheduler.lr_decay_iters=100 validation.eval_interval=0 validation.eval_iters=0 rng.seed=5678 ddp.check_for_nan_in_grad=true ddp.check_for_large_grads=true rerun_state_machine.check_for_nan_in_loss=true checkpoint.exit_on_missing_checkpoint=false checkpoint.load=null checkpoint.load_optim=false checkpoint.load_rng=false checkpoint.save_optim=false ddp.overlap_grad_reduce=false logger.log_interval=1 logger.log_throughput=true logger.tensorboard_dir=null dist.distributed_timeout_minutes=180
+ 
./scripts/training/train.sh --nodes 8 --gpus-per-node 4 --recipe deepseek_v4_flash_sft_openmath_thinking_packed_gb200_config --mode sft --step-func dsv4_step --pretrained_checkpoint work/model-verification/dsv4-flash/import-gpu/iter_0000000 --save_dir work/model-verification/dsv4-flash/sft-ref --save_interval 100 --max_steps 100 --seq_length 1024 --pipeline_model_parallel_size 4 --context_parallel_size 1 --expert_model_parallel_size 8 'model.pipeline_model_parallel_layout=Et*11|t*11|t*11|t*10mL' scheduler.lr_warmup_iters=10 scheduler.lr_decay_iters=100 validation.eval_interval=0 validation.eval_iters=0 rng.seed=5678 ddp.check_for_nan_in_grad=true ddp.check_for_large_grads=true rerun_state_machine.check_for_nan_in_loss=true checkpoint.exit_on_missing_checkpoint=false checkpoint.load=null checkpoint.load_optim=false checkpoint.load_rng=false checkpoint.save_optim=false ddp.overlap_grad_reduce=false logger.log_interval=1 logger.log_throughput=true logger.tensorboard_dir=null dist.distributed_timeout_minutes=180
@@ -371,7 +371,7 @@ Choose a workflow, precision, and exact recorded combination. The command and ex Command - 
./scripts/training/train.sh --nodes 16 --gpus-per-node 4 --recipe deepseek_v4_flash_sft_openmath_thinking_packed_config --step-func dsv4_step --pretrained_checkpoint work/models/deepseek-v4-flash-megatron --save_dir work/model-verification/dsv4-flash/sft-long-context-ref --save_interval 100 --max_steps 100 model.pipeline_model_parallel_size=4 'model.pipeline_model_parallel_layout=Et*11|t*11|t*11|t*10mL' model.context_parallel_size=2 model.cp_partition_mode=contiguous model.expert_model_parallel_size=8 model.moe_grouped_gemm=false model.recompute_granularity=full model.recompute_method=uniform model.recompute_num_layers=1 dataset.seq_length=1024 scheduler.lr_warmup_iters=10 scheduler.lr_decay_iters=100 validation.eval_interval=0 validation.eval_iters=0 rng.seed=5678 ddp.check_for_nan_in_grad=true ddp.check_for_large_grads=true rerun_state_machine.check_for_nan_in_loss=true checkpoint.exit_on_missing_checkpoint=false checkpoint.load=null ddp.overlap_grad_reduce=false logger.log_interval=1 logger.log_throughput=true logger.tensorboard_dir=null dist.distributed_timeout_minutes=180
+ 
./scripts/training/train.sh --nodes 16 --gpus-per-node 4 --recipe deepseek_v4_flash_sft_openmath_thinking_packed_config --step-func dsv4_step --pretrained_checkpoint work/model-verification/dsv4-flash/import-gpu/iter_0000000 --save_dir work/model-verification/dsv4-flash/sft-long-context-ref --save_interval 100 --max_steps 100 model.pipeline_model_parallel_size=4 'model.pipeline_model_parallel_layout=Et*11|t*11|t*11|t*10mL' model.context_parallel_size=2 model.cp_partition_mode=contiguous model.expert_model_parallel_size=8 model.moe_grouped_gemm=false model.recompute_granularity=full model.recompute_method=uniform model.recompute_num_layers=1 dataset.seq_length=1024 scheduler.lr_warmup_iters=10 scheduler.lr_decay_iters=100 validation.eval_interval=0 validation.eval_iters=0 rng.seed=5678 ddp.check_for_nan_in_grad=true ddp.check_for_large_grads=true rerun_state_machine.check_for_nan_in_loss=true checkpoint.exit_on_missing_checkpoint=false checkpoint.load=null ddp.overlap_grad_reduce=false logger.log_interval=1 logger.log_throughput=true logger.tensorboard_dir=null dist.distributed_timeout_minutes=180
@@ -430,7 +430,7 @@ Choose a workflow, precision, and exact recorded combination. The command and ex Command - 
./scripts/training/train.sh --nodes 8 --gpus-per-node 4 --recipe deepseek_v4_flash_peft_openmath_thinking_packed_gb200_config --mode lora --step-func dsv4_step --pretrained_checkpoint work/models/deepseek-v4-flash-megatron --save_dir work/model-verification/dsv4-flash/peft-ref --save_interval 100 --max_steps 100 --seq_length 1024 --pipeline_model_parallel_size 4 --context_parallel_size 1 --expert_model_parallel_size 8 'model.pipeline_model_parallel_layout=Et*11|t*11|t*11|t*10mL' scheduler.lr_warmup_iters=10 scheduler.lr_decay_iters=100 validation.eval_interval=0 validation.eval_iters=0 rng.seed=5678 ddp.check_for_nan_in_grad=true ddp.check_for_large_grads=true rerun_state_machine.check_for_nan_in_loss=true checkpoint.exit_on_missing_checkpoint=false checkpoint.load=null checkpoint.load_optim=false checkpoint.load_rng=false checkpoint.save_optim=false ddp.overlap_grad_reduce=false logger.log_interval=1 logger.log_throughput=true logger.tensorboard_dir=null dist.distributed_timeout_minutes=180
+ 
./scripts/training/train.sh --nodes 8 --gpus-per-node 4 --recipe deepseek_v4_flash_peft_openmath_thinking_packed_gb200_config --mode lora --step-func dsv4_step --pretrained_checkpoint work/model-verification/dsv4-flash/import-gpu/iter_0000000 --save_dir work/model-verification/dsv4-flash/peft-ref --save_interval 100 --max_steps 100 --seq_length 1024 --pipeline_model_parallel_size 4 --context_parallel_size 1 --expert_model_parallel_size 8 'model.pipeline_model_parallel_layout=Et*11|t*11|t*11|t*10mL' scheduler.lr_warmup_iters=10 scheduler.lr_decay_iters=100 validation.eval_interval=0 validation.eval_iters=0 rng.seed=5678 ddp.check_for_nan_in_grad=true ddp.check_for_large_grads=true rerun_state_machine.check_for_nan_in_loss=true checkpoint.exit_on_missing_checkpoint=false checkpoint.load=null checkpoint.load_optim=false checkpoint.load_rng=false checkpoint.save_optim=false ddp.overlap_grad_reduce=false logger.log_interval=1 logger.log_throughput=true logger.tensorboard_dir=null dist.distributed_timeout_minutes=180
diff --git a/docs/models/qwen/qwen3.6-35b-a3b.md b/docs/models/qwen/qwen3.6-35b-a3b.md index f1f0c4560d..7529e55f5b 100644 --- a/docs/models/qwen/qwen3.6-35b-a3b.md +++ b/docs/models/qwen/qwen3.6-35b-a3b.md @@ -371,7 +371,7 @@ Choose a workflow, precision, and exact recorded combination. The command and ex Command - 
./scripts/training/train.sh --nodes 2 --gpus-per-node 4 --recipe qwen35_vl_35b_a3b_sft_8gpu_gb200_bf16_functional_config --dataset medpix --pretrained_checkpoint work/model-verification/qwen3.6-35b-a3b/gb200-imported/iter_0000000 --max_steps 100 --warmup_iters 10 'dataset.source.load_kwargs={revision:69eb219d477ab8699296da1fd6b89beb768042d3}' 'dataset.source.adapter_kwargs={resized_height:448,resized_width:448}' dataset.hf_processor_path=Qwen/Qwen3.6-35B-A3B 'dataset.hf_processor_kwargs={revision:995ad96eacd98c81ed38be0c5b274b04031597b0}' scheduler.lr_decay_iters=100 model.hf_model_id=Qwen/Qwen3.6-35B-A3B model.bos_token_id=248044 logger.save_config_filepath=work/model-verification/qwen3.6-35b-a3b/gb200-sft/resolved-config.yaml
+ 
./scripts/training/train.sh --nodes 2 --gpus-per-node 4 --recipe qwen35_vl_35b_a3b_sft_8gpu_gb200_bf16_functional_config --dataset medpix --pretrained_checkpoint work/model-verification/qwen3.6-35b-a3b/imported-megatron/iter_0000000 --max_steps 100 --warmup_iters 10 'dataset.source.load_kwargs={revision:69eb219d477ab8699296da1fd6b89beb768042d3}' 'dataset.source.adapter_kwargs={resized_height:448,resized_width:448}' dataset.hf_processor_path=Qwen/Qwen3.6-35B-A3B 'dataset.hf_processor_kwargs={revision:995ad96eacd98c81ed38be0c5b274b04031597b0}' scheduler.lr_decay_iters=100 model.hf_model_id=Qwen/Qwen3.6-35B-A3B model.bos_token_id=248044 logger.save_config_filepath=work/model-verification/qwen3.6-35b-a3b/gb200-sft/resolved-config.yaml
@@ -524,7 +524,7 @@ Choose a workflow, precision, and exact recorded combination. The command and ex Command - 
./scripts/training/train.sh --nodes 2 --gpus-per-node 4 --recipe qwen35_vl_35b_a3b_peft_8gpu_gb200_bf16_functional_config --dataset medpix --pretrained_checkpoint work/model-verification/qwen3.6-35b-a3b/gb200-imported/iter_0000000 --max_steps 100 --warmup_iters 10 'dataset.source.load_kwargs={revision:69eb219d477ab8699296da1fd6b89beb768042d3}' 'dataset.source.adapter_kwargs={resized_height:448,resized_width:448}' dataset.hf_processor_path=Qwen/Qwen3.6-35B-A3B 'dataset.hf_processor_kwargs={revision:995ad96eacd98c81ed38be0c5b274b04031597b0}' scheduler.lr_decay_iters=100 model.hf_model_id=Qwen/Qwen3.6-35B-A3B model.bos_token_id=248044 logger.save_config_filepath=work/model-verification/qwen3.6-35b-a3b/gb200-peft/resolved-config.yaml
+ 
./scripts/training/train.sh --nodes 2 --gpus-per-node 4 --recipe qwen35_vl_35b_a3b_peft_8gpu_gb200_bf16_functional_config --dataset medpix --pretrained_checkpoint work/model-verification/qwen3.6-35b-a3b/imported-megatron/iter_0000000 --max_steps 100 --warmup_iters 10 'dataset.source.load_kwargs={revision:69eb219d477ab8699296da1fd6b89beb768042d3}' 'dataset.source.adapter_kwargs={resized_height:448,resized_width:448}' dataset.hf_processor_path=Qwen/Qwen3.6-35B-A3B 'dataset.hf_processor_kwargs={revision:995ad96eacd98c81ed38be0c5b274b04031597b0}' scheduler.lr_decay_iters=100 model.hf_model_id=Qwen/Qwen3.6-35B-A3B model.bos_token_id=248044 logger.save_config_filepath=work/model-verification/qwen3.6-35b-a3b/gb200-peft/resolved-config.yaml
diff --git a/examples/model_verification_cards/deepseek-v4-flash/card.yaml b/examples/model_verification_cards/deepseek-v4-flash/card.yaml index ae2850a864..d7da408c3c 100644 --- a/examples/model_verification_cards/deepseek-v4-flash/card.yaml +++ b/examples/model_verification_cards/deepseek-v4-flash/card.yaml @@ -238,7 +238,7 @@ items: ./scripts/training/train.sh --nodes 8 --gpus-per-node 4 --recipe deepseek_v4_flash_sft_openmath_thinking_packed_gb200_config --mode sft --step-func dsv4_step - --pretrained_checkpoint work/models/deepseek-v4-flash-megatron + --pretrained_checkpoint work/model-verification/dsv4-flash/import-gpu/iter_0000000 --save_dir work/model-verification/dsv4-flash/sft-ref --save_interval 100 --max_steps 100 --seq_length 1024 @@ -326,7 +326,7 @@ items: ./scripts/training/train.sh --nodes 16 --gpus-per-node 4 --recipe deepseek_v4_flash_sft_openmath_thinking_packed_config --step-func dsv4_step - --pretrained_checkpoint work/models/deepseek-v4-flash-megatron + --pretrained_checkpoint work/model-verification/dsv4-flash/import-gpu/iter_0000000 --save_dir work/model-verification/dsv4-flash/sft-long-context-ref --save_interval 100 --max_steps 100 model.pipeline_model_parallel_size=4 @@ -368,7 +368,7 @@ items: ./scripts/training/train.sh --nodes 8 --gpus-per-node 4 --recipe deepseek_v4_flash_peft_openmath_thinking_packed_gb200_config --mode lora --step-func dsv4_step - --pretrained_checkpoint work/models/deepseek-v4-flash-megatron + --pretrained_checkpoint work/model-verification/dsv4-flash/import-gpu/iter_0000000 --save_dir work/model-verification/dsv4-flash/peft-ref --save_interval 100 --max_steps 100 --seq_length 1024 diff --git a/examples/model_verification_cards/qwen3.6-35b-a3b/card.yaml b/examples/model_verification_cards/qwen3.6-35b-a3b/card.yaml index 376246f486..e2fc40b9cb 100644 --- a/examples/model_verification_cards/qwen3.6-35b-a3b/card.yaml +++ b/examples/model_verification_cards/qwen3.6-35b-a3b/card.yaml @@ -264,7 +264,7 @@ items: --recipe qwen35_vl_35b_a3b_sft_8gpu_gb200_bf16_functional_config --dataset medpix --pretrained_checkpoint - work/model-verification/qwen3.6-35b-a3b/gb200-imported/iter_0000000 + work/model-verification/qwen3.6-35b-a3b/imported-megatron/iter_0000000 --max_steps 100 --warmup_iters 10 'dataset.source.load_kwargs={revision:69eb219d477ab8699296da1fd6b89beb768042d3}' 'dataset.source.adapter_kwargs={resized_height:448,resized_width:448}' @@ -431,7 +431,7 @@ items: --recipe qwen35_vl_35b_a3b_peft_8gpu_gb200_bf16_functional_config --dataset medpix --pretrained_checkpoint - work/model-verification/qwen3.6-35b-a3b/gb200-imported/iter_0000000 + work/model-verification/qwen3.6-35b-a3b/imported-megatron/iter_0000000 --max_steps 100 --warmup_iters 10 'dataset.source.load_kwargs={revision:69eb219d477ab8699296da1fd6b89beb768042d3}' 'dataset.source.adapter_kwargs={resized_height:448,resized_width:448}'