Skip to content

Commit 206507d

Browse files
sraman-rgbsraman-rgb
authored
Merge branch 'main' into sraman/dataloader-prefetch-factor
2 parents d7bbb48 + 2856c2e commit 206507d

44 files changed

Lines changed: 1635 additions & 447 deletions

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎.github/workflows/cicd-main.yml‎

Lines changed: 41 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1384,3 +1384,44 @@ jobs:
13841384
path: |
13851385
.coverage
13861386
include-hidden-files: true
1387+
1388+
notify-nightly-failure:
1389+
name: Notify scheduled CI failure
1390+
runs-on: ubuntu-latest
1391+
environment: main
1392+
needs:
1393+
- ephemeral-runner-routing
1394+
- pre-flight
1395+
- configure
1396+
- lint-check
1397+
- cicd-wait-in-queue
1398+
- cicd-compute-build-matrix
1399+
- cicd-container-build
1400+
- cicd-import-check
1401+
- cicd-unit-tests-core
1402+
- cicd-unit-tests-diffusion
1403+
- generate-test-matrix
1404+
- cicd-functional-tests-l0
1405+
- cicd-functional-tests-l1
1406+
- cicd-functional-tests-l2
1407+
- cicd-functional-tests-flaky
1408+
- generate-gb200-test-matrix
1409+
- cicd-functional-tests-gb200-l0
1410+
- cicd-functional-tests-gb200-l1
1411+
- cicd-functional-tests-gb200-l2
1412+
- cicd-functional-tests-gb200-flaky
1413+
- Nemo_CICD_Test
1414+
- Coverage
1415+
permissions: {}
1416+
# Keep setup failures visible even when downstream jobs are skipped.
1417+
if: >-
1418+
${{ always() && !cancelled() && failure() &&
1419+
github.repository == 'NVIDIA-NeMo/Megatron-Bridge' &&
1420+
github.event_name == 'schedule' &&
1421+
github.ref_name == github.event.repository.default_branch }}
1422+
steps:
1423+
- name: Notify CI alerts channel
1424+
uses: NVIDIA-NeMo/FW-CI-templates/.github/actions/notify-ci-failure@631c404d00a9e60afc591cd071d35d2e18f82fc6
1425+
with:
1426+
needs-json: ${{ toJSON(needs) }}
1427+
webhook: ${{ secrets.SLACK_WEBHOOK }}

‎docs/_static/model-verification/catalog-v1.json‎

Lines changed: 46 additions & 46 deletions
Original file line numberDiff line numberDiff line change
@@ -1122,7 +1122,7 @@
11221122
"workflow": "checkpoint_resume"
11231123
},
11241124
{
1125-
"bridge_commit": "3ecb452af619bef88083e10a7738362df8f5d0bf",
1125+
"bridge_commit": "b53e66c47b9704dd16ce399865bea4f4f004e53b",
11261126
"command_topologies": [
11271127
{
11281128
"gpus_per_node": "4",
@@ -1137,15 +1137,15 @@
11371137
"dimensions": {},
11381138
"enabled_features": {},
11391139
"entry_id": "deepseek-v4-flash-pretrain-performance-gb200",
1140-
"expected_result": "On 128 GB200, PP1/EP64/CP1/TP1/DP2 with GBS/MBS 2048/1, FP8-MX, HybridEP dispatch, and full-iteration CUDA graphs completes 50 steps with finite losses, no skipped or NaN iterations, and all five metrics recorded. This benchmark uses forced routing, a static expert-rank capacity, paged stash, and a different batch/objective contract, so its metrics must not be compared with natural-routing library pretraining as convergence evidence.\n",
1140+
"expected_result": "On 128 GB200, PP1/EP32/CP1/TP1/DP4 with GBS/MBS 2048/1, FP8-MX, HybridEP dispatch, and full-iteration CUDA graphs completes 50 steps with finite losses, no skipped or NaN iterations, and all five metrics recorded. This benchmark uses forced routing, a static expert-rank capacity, paged stash, and a different batch/objective contract, so its metrics must not be compared with natural-routing library pretraining as convergence evidence.\n",
11411141
"hardware": "GB200",
1142-
"last_verified": "2026-08-06",
1142+
"last_verified": "2026-10-14",
11431143
"metrics": {
1144-
"final_loss": 3.62683,
1145-
"initial_loss": 13.5875,
1146-
"last_10_steps_model_tflops_per_gpu_avg": 731.4,
1147-
"last_10_steps_step_time_ms_avg": 8134.6,
1148-
"last_10_steps_tokens_per_second_per_gpu_avg": 8056.45
1144+
"final_loss": 4.25929,
1145+
"initial_loss": 12.59972,
1146+
"last_10_steps_model_tflops_per_gpu_avg": 731.83,
1147+
"last_10_steps_step_time_ms_avg": 8146.44,
1148+
"last_10_steps_tokens_per_second_per_gpu_avg": 8044.741
11491149
},
11501150
"precision": "fp8_mx",
11511151
"source_pointer": "items.pretrain_performance.GB200",
@@ -11820,7 +11820,7 @@
1182011820
"workflow": "pretrain"
1182111821
},
1182211822
{
11823-
"bridge_commit": "49c103c714f3d62d7609f27a46e42dc14e95afa6",
11823+
"bridge_commit": "e2f2496cbb89e3c881fd54b1568d2689d91df764",
1182411824
"command_topologies": [
1182511825
{
1182611826
"gpus_per_node": "4",
@@ -11837,15 +11837,15 @@
1183711837
"sequence_packing": "offline"
1183811838
},
1183911839
"entry_id": "glm5-sft-gb200",
11840-
"expected_result": "Completed 100 BF16 Hybrid full-SFT steps with pinned Tulu 3 train[:10000], 8K packing, GBS/MBS 8/1 and CP4 on 192 GB200 GPUs, with MTP1, HybridEP and natural routing. The exact runtime passed full-shape causal top-k reference checks with explicit cuDNN query offsets before training. All LM/MTP losses and gradients were finite with zero skipped or NaN iterations. Saved the complete final model checkpoint and post-setup configuration. Strictly reloaded the step-100 checkpoint with raise_all and completed a further training step. Metrics cover the original 100 steps only.",
11840+
"expected_result": "Completed 100 BF16 Hybrid full-SFT steps with pinned Tulu 3 train[:10000], 8K packing, GBS/MBS 8/1, TP1/PP6/EP32 and CP4 on 192 GB200 GPUs, with MTP1, HybridEP and natural routing, warm-started from the pinned Hugging Face checkpoint. All LM/MTP losses and gradients were finite with zero skipped or NaN iterations, and the step-100 checkpoint and post-setup configuration were saved.",
1184111841
"hardware": "GB200",
11842-
"last_verified": "2026-10-07",
11842+
"last_verified": "2026-10-15",
1184311843
"metrics": {
11844-
"final_loss": 0.3063886,
11845-
"initial_loss": 1.456939,
11846-
"last_10_steps_model_tflops_per_gpu_avg": 14.18,
11847-
"last_10_steps_step_time_ms_avg": 6468.4,
11848-
"last_10_steps_tokens_per_second_per_gpu_avg": 52.769361
11844+
"final_loss": 0.6933057,
11845+
"initial_loss": 1.473073,
11846+
"last_10_steps_model_tflops_per_gpu_avg": 14.74,
11847+
"last_10_steps_step_time_ms_avg": 6157.27,
11848+
"last_10_steps_tokens_per_second_per_gpu_avg": 55.435824
1184911849
},
1185011850
"precision": "bf16",
1185111851
"source_pointer": "items.sft.GB200",
@@ -11854,7 +11854,7 @@
1185411854
"workflow": "sft"
1185511855
},
1185611856
{
11857-
"bridge_commit": "49c103c714f3d62d7609f27a46e42dc14e95afa6",
11857+
"bridge_commit": "e2f2496cbb89e3c881fd54b1568d2689d91df764",
1185811858
"command_topologies": [
1185911859
{
1186011860
"expert_parallel": "8",
@@ -11878,9 +11878,9 @@
1187811878
"dimensions": {},
1187911879
"enabled_features": {},
1188011880
"entry_id": "glm5-sft-export-inference-gb200",
11881-
"expected_result": "Exported the verified corrected packed-CP SFT step-100 checkpoint to BF16 Hugging Face format on 48 GB200 GPUs, including all 59870 tensors and the 791 MTP tensors. All 16 native Transformers TP ranks strictly reloaded the export on 16 GB200 GPUs without CPU or disk offload. Greedy inference generated 16 tokens with a maximum of 16; completion: \"Deterministic tests are important because they provide consistent and reliable results, allowing for easier\"",
11881+
"expected_result": "Exported the GLM-5 SFT step-100 checkpoint to BF16 Hugging Face format on 48 GB200 GPUs, including all 59870 tensors and the 791 MTP tensors. All 16 native Transformers TP ranks strictly reloaded the export on 16 GB200 GPUs. Greedy inference generated 16 tokens with a maximum of 16; completion: \"Deterministic tests matter because they provide consistent and reliable results, allowing for easier identification\"",
1188211882
"hardware": "GB200",
11883-
"last_verified": "2026-10-07",
11883+
"last_verified": "2026-10-15",
1188411884
"metrics": {},
1188511885
"precision": "bf16",
1188611886
"source_pointer": "items.sft_export_inference.GB200",
@@ -11924,7 +11924,7 @@
1192411924
"workflow": "sft_long_context"
1192511925
},
1192611926
{
11927-
"bridge_commit": "a860b62ff21c9bdceb7f285d97d41acc790fe4d8",
11927+
"bridge_commit": "e2f2496cbb89e3c881fd54b1568d2689d91df764",
1192811928
"command_topologies": [
1192911929
{
1193011930
"gpus_per_node": "4",
@@ -11941,15 +11941,15 @@
1194111941
"sequence_packing": "offline"
1194211942
},
1194311943
"entry_id": "glm5-peft-gb200",
11944-
"expected_result": "Completed 100 BF16 Hybrid LoRA steps on 192 GB200 GPUs with pinned Tulu 3 train[:10000], 2K offline packing, GBS/MBS 32/1, MTP1 and natural routing. All LM/MTP losses and gradients were finite with zero skipped or NaN iterations. Saved adapter, optimizer, scheduler and RNG state, strictly reloaded the step-100 adapters with raise_all, and completed step 101 at zero LR. All 790 saved adapter tensors matched the reloaded checkpoint exactly; only the five configured MLA projection targets were adapted. Metrics cover training steps 1-100 only.",
11944+
"expected_result": "Completed 100 BF16 Hybrid LoRA steps on 192 GB200 GPUs with pinned Tulu 3 train[:10000], 2K offline packing, GBS/MBS 32/1, TP1/PP6/EP32, MTP1, all-to-all expert dispatch and natural routing, warm-started from the pinned Hugging Face checkpoint. All LM/MTP losses and gradients were finite with zero skipped or NaN iterations, and the step-100 adapter checkpoint with optimizer and RNG state was saved.",
1194511945
"hardware": "GB200",
11946-
"last_verified": "2026-10-07",
11946+
"last_verified": "2026-10-15",
1194711947
"metrics": {
11948-
"final_loss": 0.941183,
11949-
"initial_loss": 1.544981,
11950-
"last_10_steps_model_tflops_per_gpu_avg": 11.71,
11951-
"last_10_steps_step_time_ms_avg": 7883.89,
11952-
"last_10_steps_tokens_per_second_per_gpu_avg": 43.29504
11948+
"final_loss": 0.9403105,
11949+
"initial_loss": 1.545165,
11950+
"last_10_steps_model_tflops_per_gpu_avg": 15.72,
11951+
"last_10_steps_step_time_ms_avg": 5760.54,
11952+
"last_10_steps_tokens_per_second_per_gpu_avg": 59.253704
1195311953
},
1195411954
"precision": "bf16",
1195511955
"source_pointer": "items.peft.GB200",
@@ -12208,7 +12208,7 @@
1220812208
"workflow": "pretrain"
1220912209
},
1221012210
{
12211-
"bridge_commit": "49c103c714f3d62d7609f27a46e42dc14e95afa6",
12211+
"bridge_commit": "e2f2496cbb89e3c881fd54b1568d2689d91df764",
1221212212
"command_topologies": [
1221312213
{
1221412214
"gpus_per_node": "4",
@@ -12225,15 +12225,15 @@
1222512225
"sequence_packing": "offline"
1222612226
},
1222712227
"entry_id": "glm5-2-sft-gb200",
12228-
"expected_result": "Completed 100 BF16 Hybrid full-SFT steps with pinned Tulu 3 train[:10000], 8K packing, GBS/MBS 8/1 and CP4 on 192 GB200 GPUs, with MTP1, HybridEP and natural routing. The exact runtime passed full-shape causal top-k reference checks with explicit cuDNN query offsets before training. All LM/MTP losses and gradients were finite with zero skipped or NaN iterations. Saved the complete final model checkpoint and post-setup configuration. Strictly reloaded the step-100 checkpoint with raise_all and completed a further training step. Metrics cover the original 100 steps only.",
12228+
"expected_result": "Completed 100 BF16 Hybrid full-SFT steps with pinned Tulu 3 train[:10000], 8K packing, GBS/MBS 8/1, TP1/PP6/EP32 and CP4 on 192 GB200 GPUs, with MTP1, HybridEP and natural routing, warm-started from the pinned Hugging Face checkpoint. All LM/MTP losses were finite with zero skipped or NaN iterations, and the step-100 checkpoint and post-setup configuration were saved.",
1222912229
"hardware": "GB200",
12230-
"last_verified": "2026-10-07",
12230+
"last_verified": "2026-10-15",
1223112231
"metrics": {
12232-
"final_loss": 0.7940679,
12233-
"initial_loss": 1.913098,
12234-
"last_10_steps_model_tflops_per_gpu_avg": 17.14,
12235-
"last_10_steps_step_time_ms_avg": 6738.85,
12236-
"last_10_steps_tokens_per_second_per_gpu_avg": 50.65157
12232+
"final_loss": 1.063773,
12233+
"initial_loss": 1.913887,
12234+
"last_10_steps_model_tflops_per_gpu_avg": 17.89,
12235+
"last_10_steps_step_time_ms_avg": 7959.68,
12236+
"last_10_steps_tokens_per_second_per_gpu_avg": 42.882796
1223712237
},
1223812238
"precision": "bf16",
1223912239
"source_pointer": "items.sft.GB200",
@@ -12242,7 +12242,7 @@
1224212242
"workflow": "sft"
1224312243
},
1224412244
{
12245-
"bridge_commit": "49c103c714f3d62d7609f27a46e42dc14e95afa6",
12245+
"bridge_commit": "e2f2496cbb89e3c881fd54b1568d2689d91df764",
1224612246
"command_topologies": [
1224712247
{
1224812248
"expert_parallel": "8",
@@ -12266,9 +12266,9 @@
1226612266
"dimensions": {},
1226712267
"enabled_features": {},
1226812268
"entry_id": "glm5-2-sft-export-inference-gb200",
12269-
"expected_result": "Exported the verified corrected packed-CP SFT step-100 checkpoint to BF16 Hugging Face format on 48 GB200 GPUs, including all 59585 tensors and the 791 MTP tensors. All 16 native Transformers TP ranks strictly reloaded the export on 16 GB200 GPUs without CPU or disk offload. Greedy inference generated 16 tokens with a maximum of 16; completion: \"Deterministic tests matter because they provide consistent and repeatable results, allowing for reliable\"",
12269+
"expected_result": "Exported the GLM-5.2 SFT step-100 checkpoint to BF16 Hugging Face format on 48 GB200 GPUs, including all 59585 tensors and the 791 MTP tensors. All 16 native Transformers TP ranks strictly reloaded the export on 16 GB200 GPUs. Greedy inference generated 16 tokens with a maximum of 16; completion: \"Deterministic tests matter because they provide consistent and reliable results, allowing for accurate identification\"",
1227012270
"hardware": "GB200",
12271-
"last_verified": "2026-10-07",
12271+
"last_verified": "2026-10-15",
1227212272
"metrics": {},
1227312273
"precision": "bf16",
1227412274
"source_pointer": "items.sft_export_inference.GB200",
@@ -12312,7 +12312,7 @@
1231212312
"workflow": "sft_long_context"
1231312313
},
1231412314
{
12315-
"bridge_commit": "0b59ee513ff26c259461c87c085f491a8fa3ef4d",
12315+
"bridge_commit": "e2f2496cbb89e3c881fd54b1568d2689d91df764",
1231612316
"command_topologies": [
1231712317
{
1231812318
"gpus_per_node": "4",
@@ -12329,15 +12329,15 @@
1232912329
"sequence_packing": "offline"
1233012330
},
1233112331
"entry_id": "glm5-2-peft-gb200",
12332-
"expected_result": "Completed 100 BF16 Hybrid LoRA steps on 192 GB200 GPUs with pinned Tulu 3 train[:10000], 2K offline packing, GBS/MBS 32/1, MTP1 and natural routing. All LM/MTP losses and gradients were finite with zero skipped or NaN iterations. Saved adapter, optimizer, scheduler and RNG state, strictly reloaded the step-100 adapters with raise_all, and completed step 101 at zero LR. All 790 saved adapter tensors matched the reloaded checkpoint exactly; only the five configured MLA projection targets were adapted. Metrics cover training steps 1-100 only.",
12332+
"expected_result": "Completed 100 BF16 Hybrid LoRA steps on 192 GB200 GPUs with pinned Tulu 3 train[:10000], 2K offline packing, GBS/MBS 32/1, TP1/PP6/EP32, MTP1, all-to-all expert dispatch and natural routing, warm-started from the pinned Hugging Face checkpoint. All LM/MTP losses were finite with zero skipped or NaN iterations, and the step-100 adapter checkpoint with optimizer and RNG state was saved.",
1233312333
"hardware": "GB200",
12334-
"last_verified": "2026-10-07",
12334+
"last_verified": "2026-10-15",
1233512335
"metrics": {
12336-
"final_loss": 0.8791012,
12337-
"initial_loss": 1.903576,
12338-
"last_10_steps_model_tflops_per_gpu_avg": 23.03,
12339-
"last_10_steps_step_time_ms_avg": 4322.97,
12340-
"last_10_steps_tokens_per_second_per_gpu_avg": 78.958062
12336+
"final_loss": 0.8795374,
12337+
"initial_loss": 1.903709,
12338+
"last_10_steps_model_tflops_per_gpu_avg": 21.57,
12339+
"last_10_steps_step_time_ms_avg": 5695.94,
12340+
"last_10_steps_tokens_per_second_per_gpu_avg": 59.925725
1234112341
},
1234212342
"precision": "bf16",
1234312343
"source_pointer": "items.peft.GB200",

‎docs/fern/versions/nightly/pages/models/deepseek/deepseek-v4.mdx‎

Lines changed: 7 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -439,30 +439,30 @@ Choose a workflow, precision, and exact recorded combination. The command and ex
439439
<dl class="verification-model-detail-meta">
440440
<div><dt>Hardware</dt><dd>GB200</dd></div>
441441
<div><dt>Precision</dt><dd>FP8 MX</dd></div>
442-
<div><dt>Last verified</dt><dd>2026-08-06</dd></div>
442+
<div><dt>Last verified</dt><dd>2026-10-14</dd></div>
443443
</dl>
444444
<section class="verification-recorded-metrics">
445445
<h5>Recorded metrics</h5>
446446
<dl class="verification-metric-list">
447447
<div>
448448
<dt>Initial loss</dt>
449-
<dd>13.5875</dd>
449+
<dd>12.59972</dd>
450450
</div>
451451
<div>
452452
<dt>Final loss</dt>
453-
<dd>3.62683</dd>
453+
<dd>4.25929</dd>
454454
</div>
455455
<div>
456456
<dt>Step time · last 10 avg</dt>
457-
<dd>8,134.600 ms</dd>
457+
<dd>8,146.440 ms</dd>
458458
</div>
459459
<div>
460460
<dt>Model throughput · last 10 avg</dt>
461-
<dd>731.400 TFLOP/s/GPU</dd>
461+
<dd>731.830 TFLOP/s/GPU</dd>
462462
</div>
463463
<div>
464464
<dt>Token throughput · last 10 avg</dt>
465-
<dd>8,056.450 tokens/s/GPU</dd>
465+
<dd>8,044.741 tokens/s/GPU</dd>
466466
</div>
467467
</dl>
468468
</section>
@@ -478,7 +478,7 @@ Choose a workflow, precision, and exact recorded combination. The command and ex
478478
</section>
479479
<section class="verification-expected-result">
480480
<h5>Expected result</h5>
481-
<p>On 128 GB200, PP1/EP64/CP1/TP1/DP2 with GBS/MBS 2048/1, FP8-MX, HybridEP dispatch, and full-iteration CUDA graphs completes 50 steps with finite losses, no skipped or NaN iterations, and all five metrics recorded. This benchmark uses forced routing, a static expert-rank capacity, paged stash, and a different batch/objective contract, so its metrics must not be compared with natural-routing library pretraining as convergence evidence.</p>
481+
<p>On 128 GB200, PP1/EP32/CP1/TP1/DP4 with GBS/MBS 2048/1, FP8-MX, HybridEP dispatch, and full-iteration CUDA graphs completes 50 steps with finite losses, no skipped or NaN iterations, and all five metrics recorded. This benchmark uses forced routing, a static expert-rank capacity, paged stash, and a different batch/objective contract, so its metrics must not be compared with natural-routing library pretraining as convergence evidence.</p>
482482
</section>
483483
</article>
484484
<article id="deepseek-v4-flash-pretrain-performance-gb300" class="verification-model-detail" data-entry-detail="deepseek-v4-flash-pretrain-performance-gb300" tabindex="-1">

0 commit comments

Comments
 (0)