Merge branch 'main' into super-v3-pr

Author: liding-nv

.dev.commit

@@ -1 +1 @@
-9374a4d328641a2fbbad4fd41cf038b539dcf1e6
+c72c4599012297cfbd1d57e006b544478b6bbf78

.github/actions/test-template/action.yml

@@ -52,6 +52,10 @@ inputs:
   test-data-path:
     description: "Test data path"
     required: true
+  script_dir:
+    description: "Subdirectory within launch_scripts/ for functional tests (active or flaky)"
+    required: false
+    default: "active"
   runner:
     description: "Runner to use for test"
     required: true
@@ -197,7 +201,7 @@ runs:
             echo "📝 Stored MCore commit SHA: ${{ github.event.inputs.mcore_commit }}"
           fi
 
-          timeout $(( ${{ inputs.timeout }} * 60 ))s bash tests/${{ inputs.is_unit_test == 'true' && 'unit_tests' || 'functional_tests' }}/${{ inputs.script }}.sh || EXIT_CODE=$?
+          timeout $(( ${{ inputs.timeout }} * 60 ))s bash tests/${{ inputs.is_unit_test == 'true' && 'unit_tests' || format('functional_tests/launch_scripts/{0}', inputs.script_dir) }}/${{ inputs.script }}.sh || EXIT_CODE=$?
 
           if [[ "$EXIT_CODE" -eq 124 ]]; then
             echo "::error:: Test timed out after ${{ inputs.timeout }} minutes. Please reach out to the PIC."

.github/workflows/cicd-main.yml

@@ -295,7 +295,7 @@ jobs:
 
   check-labels:
     runs-on: ubuntu-latest
-    needs: [pre-flight]
+    needs: [pre-flight, cicd-container-build]
     outputs:
       needs_more_tests: ${{ steps.check.outputs.needs_more_tests }}
     steps:
@@ -438,22 +438,51 @@ jobs:
           test-data-path: ${{ needs.pre-flight.outputs.test_data_path }}
           runner: ${{ needs.pre-flight.outputs.runner_prefix }}-gpu-x2
 
+  generate-test-matrix:
+    needs: [pre-flight, cicd-container-build]
+    runs-on: ubuntu-latest
+    outputs:
+      matrix_l0: ${{ steps.scan.outputs.matrix_l0 }}
+      matrix_l1: ${{ steps.scan.outputs.matrix_l1 }}
+      matrix_l2: ${{ steps.scan.outputs.matrix_l2 }}
+      matrix_flaky: ${{ steps.scan.outputs.matrix_flaky }}
+    steps:
+      - uses: actions/checkout@v4
+      - id: scan
+        shell: bash
+        run: |
+          get_timeout() {
+            local f="$1"
+            local t
+            t=$(grep -m1 '^# CI_TIMEOUT=' "$f" | cut -d= -f2)
+            echo "${t:-30}"
+          }
+          for tier in L0 L1 L2; do
+            entries=""
+            for f in tests/functional_tests/launch_scripts/active/${tier}_*.sh; do
+              name=$(basename "$f" .sh)
+              timeout=$(get_timeout "$f")
+              entries="${entries}{\"script\":\"${name}\",\"timeout\":${timeout}},"
+            done
+            matrix="{\"include\":[${entries%,}]}"
+            echo "matrix_${tier,,}=${matrix}" | tee -a "$GITHUB_OUTPUT"
+          done
+          entries=""
+          for f in tests/functional_tests/launch_scripts/flaky/L*.sh; do
+            [ -f "$f" ] || continue
+            name=$(basename "$f" .sh)
+            timeout=$(get_timeout "$f")
+            entries="${entries}{\"script\":\"${name}\",\"timeout\":${timeout}},"
+          done
+          echo "matrix_flaky={\"include\":[${entries%,}]}" | tee -a "$GITHUB_OUTPUT"
+
   # L0: runs on every PR, main push, and schedule
   cicd-functional-tests-l0:
     strategy:
       fail-fast: false
       max-parallel: 16
-      matrix:
-        include:
-          - script: L0_Launch_training
-            timeout: 50
-          - script: L0_Launch_converter
-          - script: L0_Launch_data
-          - script: L0_Launch_models_deepseek
-          - script: L0_Launch_models_qwen3
-          - script: L0_Launch_recipes_llama_1b
-          - script: L0_Launch_utils
-    needs: [pre-flight, cicd-unit-tests-core, cicd-unit-tests-diffusion]
+      matrix: ${{ fromJSON(needs.generate-test-matrix.outputs.matrix_l0) }}
+    needs: [pre-flight, generate-test-matrix, cicd-unit-tests-core, cicd-unit-tests-diffusion]
     runs-on: ${{ needs.pre-flight.outputs.runner_prefix }}-gpu-x2
     if: |
       (
@@ -519,30 +548,8 @@ jobs:
     strategy:
       fail-fast: false
       max-parallel: 16
-      matrix:
-        include:
-          - script: L1_Launch_models_gemma1
-          - script: L1_Launch_models_gemma2
-          - script: L1_Launch_models_gemma3
-          - script: L1_Launch_models_glm
-          - script: L1_Launch_models_gpt_oss
-          - script: L1_Launch_models_ministral3
-          - script: L1_Launch_models_mistral
-          - script: L1_Launch_models_nemotron
-          - script: L1_Launch_models_nemotronh
-          - script: L1_Launch_models_olmoe
-          - script: L1_Launch_models_qwen2
-          - script: L1_Launch_models_qwen25_vl
-          - script: L1_Launch_models_qwen35_vl
-          - script: L1_Launch_models_qwen3_vl
-          - script: L1_Launch_post_training_quantization
-          # - script: L1_Launch_quantization_export
-          - script: L1_Launch_recipes_gpt_oss
-          - script: L1_Launch_recipes_llama_3b
-          - script: L1_Launch_recipes_ministral3
-          - script: L1_Launch_recipes_nemotronh
-          - script: L1_Launch_recipes_qwen
-    needs: [pre-flight, cicd-unit-tests-core, cicd-unit-tests-diffusion, check-labels]
+      matrix: ${{ fromJSON(needs.generate-test-matrix.outputs.matrix_l1) }}
+    needs: [pre-flight, generate-test-matrix, cicd-unit-tests-core, cicd-unit-tests-diffusion, check-labels]
     runs-on: ${{ needs.pre-flight.outputs.runner_prefix }}-gpu-x2
     if: |
       (
@@ -609,29 +616,8 @@ jobs:
     strategy:
       fail-fast: false
       max-parallel: 16
-      matrix:
-        include:
-          - script: L2_Launch_ckpts_mbridge_to_mlm_llama32_1b
-          - script: L2_Launch_ckpts_mlm_to_mbridge_llama32_1b
-          - script: L2_Launch_ckpts_mbridge_to_mlm_qwen3_4b
-          - script: L2_Launch_ckpts_mlm_to_mbridge_qwen3_4b
-          # NemotronH 4B ckpt jobs skipped until hybrid_layer_pattern rename lands
-          # - script: L2_Launch_ckpts_mbridge_to_mlm_nemotronh_4b
-          # - script: L2_Launch_ckpts_mlm_to_mbridge_nemotronh_4b
-          - script: L2_Launch_models_gemma_vl
-          - script: L2_Launch_models_llama_nemotron
-          - script: L2_Launch_models_nemotron_vl
-          - script: L2_Launch_models_nemotronh_quantization
-          - script: L2_Launch_models_qwen_quantization
-          - script: L2_Launch_models_qwen_vl_quantization
-          - script: L2_Launch_quantization_aware_training
-          - script: L2_Launch_recipes_gemma_vl
-          - script: L2_Launch_recipes_llama_cuda_graphs
-          - script: L2_Launch_recipes_llama_distill
-          - script: L2_Launch_recipes_qwen_vl
-          - script: L2_Launch_models_flux
-          - script: L2_Launch_models_wan
-    needs: [pre-flight, cicd-unit-tests-core, cicd-unit-tests-diffusion]
+      matrix: ${{ fromJSON(needs.generate-test-matrix.outputs.matrix_l2) }}
+    needs: [pre-flight, generate-test-matrix, cicd-unit-tests-core, cicd-unit-tests-diffusion]
     runs-on: ${{ needs.pre-flight.outputs.runner_prefix }}-gpu-x2
     if: |
       (
@@ -693,6 +679,66 @@ jobs:
           test-data-path: ${{ needs.pre-flight.outputs.test_data_path }}
           runner: ${{ needs.pre-flight.outputs.runner_prefix }}-gpu-x2
 
+  cicd-functional-tests-flaky:
+    if: github.event_name == 'workflow_dispatch'
+    strategy:
+      fail-fast: false
+      max-parallel: 16
+      matrix: ${{ fromJSON(needs.generate-test-matrix.outputs.matrix_flaky) }}
+    needs: [pre-flight, generate-test-matrix, cicd-unit-tests-core, cicd-unit-tests-diffusion]
+    runs-on: ${{ needs.pre-flight.outputs.runner_prefix }}-gpu-x2
+    name: ${{ matrix.script }}
+    environment: nemo-ci
+    env:
+      HF_HOME: /home/TestData/HF_HOME
+      TRANSFORMERS_OFFLINE: "1"
+      HF_HUB_OFFLINE: "1"
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+        with:
+          submodules: recursive
+
+      - name: Update MCore submodule (if triggered from MCore)
+        if: ${{ github.event.inputs.mcore_ref != '' }}
+        run: |
+          echo "🔄 Updating MCore submodule for functional tests..."
+          cd 3rdparty/Megatron-LM
+          git fetch ${{ github.event.inputs.mcore_repo || 'origin' }} ${{ github.event.inputs.mcore_ref }}
+          git checkout ${{ github.event.inputs.mcore_ref }}
+
+          ACTUAL_COMMIT=$(git rev-parse HEAD)
+          EXPECTED_COMMIT="${{ github.event.inputs.mcore_ref }}"
+
+          echo "🧪 FUNCTIONAL TESTS - MCore verification:"
+          echo "Expected: ${EXPECTED_COMMIT}"
+          echo "Actual:   ${ACTUAL_COMMIT}"
+
+          if [ "${ACTUAL_COMMIT}" != "${EXPECTED_COMMIT}" ]; then
+            echo "❌ ERROR: MCore commit mismatch!"
+            exit 1
+          fi
+
+          echo "✅ MCore commit verified"
+          echo "📦 Container: ${{ env.container-registry }}/megatron-bridge:${{ github.sha }}"
+          git log -1 --oneline
+
+      - name: main
+        uses: ./.github/actions/test-template
+        with:
+          script_dir: flaky
+          script: ${{ matrix.script }}
+          timeout: ${{ fromJSON(matrix.timeout || '30') }}
+          is_unit_test: "false"
+          has-azure-credentials: "true"
+          azure-client-id: ${{ secrets.AZURE_CLIENT_ID }}
+          azure-tenant-id: ${{ secrets.AZURE_TENANT_ID }}
+          azure-subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }}
+          PAT: ${{ secrets.PAT }}
+          container-image: ${{ env.container-registry }}/megatron-bridge:${{ github.sha }}
+          test-data-path: ${{ needs.pre-flight.outputs.test_data_path }}
+          runner: ${{ needs.pre-flight.outputs.runner_prefix }}-gpu-x2
+
   Nemo_CICD_Test:
     needs:
       - pre-flight