Merge origin/main into cgomes/pao_with_bf16_params

Resolve the conflict in DistributedOptimizer.__init__ by keeping both the fp8_param_gather resolution assert and main's full-iteration CUDA graph grad copy setup.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Signed-off-by: CarlosGomes98 <cgomes@nvidia.com>

Author: CarlosGomes98

.github/copy-pr-bot.yaml

@@ -1,4 +1,4 @@
 enabled: true
 auto_sync_draft: false
 auto_sync_ready: true
-trustees_override: ["AAnoosheh", "ArEsKay3", "Autumn1998", "BestJuly", "BoxiangW", "CarlosGomes98", "ChenhanYu", "Connor-XY", "DanialTaheri", "FDecaYed", "HaochenYuan", "ISEEKYAN", "JF-D", "JRD971000", "Leili", "Mellonta", "Phlip79", "QiZhangNV", "RPrenger", "ShriyaRishab", "WanZzzzzz", "Wohox", "XiaodaNV", "YangFei1990", "ZhiyuLi-Nvidia", "abhinav-khattar", "adistomar", "ahmadki", "alokpathy", "ananthsub", "anlthms", "aroshanghias-nvd", "ashehper", "asolergi-nv", "athitten", "balasaajay", "buptzyb", "chtruong814", "cjld", "cspades", "cuichenx", "deepakn94", "desh2608", "dimapihtar", "dingqingy-nv", "duncanriach", "ehosseiniasl", "erhoo82", "ericharper", "fanshiqing", "faradawn", "fitsumreda", "freewym", "frsun-nvda", "gautham-kollu", "gdengk", "goelarushi", "guihong-nv", "guyueh1", "hexinw-nvidia", "huvunvidia", "hxbai", "ilml", "jalbericiola", "janEbert", "jaredcasper", "jiaji-huang", "jiemingz", "jingqiny-99", "jkamalu", "jon-barker", "jstjohn", "jupiterepoch", "kajalj22", "kamran-nvidia", "kevalmorabia97", "kevjshih", "kingformatty", "ko3n1g", "ksivaman", "kunlunl", "kvareddy", "kwangjunahn", "kwyss-nvidia", "lauradang", "layalir", "lhb8125", "liding-nv", "lmcafee-nvidia", "maanug-nv", "macandro96", "mathemakitten", "matthieule", "mchrzanowski", "mehraakash", "minitu", "mkhona-nvidia", "nanz-nv", "nathan-nvidia", "niyunsheng", "ntajbakhsh", "nvcsathe", "nvegesna-netizen", "parthmannan", "philipcmonk", "prajwal1210", "pthombre", "rapatel", "rhewett-nv", "rogerwaleffe", "sajadn", "sancha", "santhnm2", "sbak5", "shanmugamr1992", "sharathts", "sheliang-nv", "shengf-nv", "shifangx", "shjwudp", "sidsingh-nvidia", "skyw", "sraman-rgb", "sudhakarsingh27", "svcnvidia-nemo-ci", "tdene", "theothermike", "tomlifu", "trintamaki", "tylerpoon", "vasunvidia", "wanyingw", "wdykas", "wujingyue", "xiaoyao0115", "xuantengh", "xuwchen", "yaox12", "yaoyu-33", "yashaswikarnati", "yeyu-nvidia", "yobibyte", "youngeunkwon0405", "yqwangustc", "yueshen2016", "yuzhongw-nvidia", "zhehuaichen", "zhongbozhu"]
+trustees_override: ["AAnoosheh", "ArEsKay3", "Autumn1998", "BestJuly", "BoxiangW", "CarlosGomes98", "ChenhanYu", "Connor-XY", "DanialTaheri", "FDecaYed", "HaochenYuan", "ISEEKYAN", "JF-D", "JRD971000", "Leili", "Mellonta", "Phlip79", "QiZhangNV", "RPrenger", "ShriyaRishab", "WanZzzzzz", "Wohox", "XiaodaNV", "YangFei1990", "YigongQin", "ZhiyuLi-Nvidia", "abhinav-khattar", "adistomar", "ahmadki", "alokpathy", "ananthsub", "anlthms", "aroshanghias-nvd", "ashehper", "asolergi-nv", "athitten", "balasaajay", "buptzyb", "chtruong814", "cjld", "cspades", "cuichenx", "deepakn94", "desh2608", "dimapihtar", "dingqingy-nv", "duncanriach", "ehosseiniasl", "erhoo82", "ericharper", "fanshiqing", "faradawn", "fitsumreda", "freewym", "frsun-nvda", "gautham-kollu", "gdengk", "goelarushi", "guihong-nv", "guyueh1", "hexinw-nvidia", "huvunvidia", "hxbai", "ilml", "jalbericiola", "janEbert", "jaredcasper", "jiaji-huang", "jiemingz", "jingqiny-99", "jkamalu", "jon-barker", "jstjohn", "jupiterepoch", "kajalj22", "kamran-nvidia", "kevalmorabia97", "kevjshih", "kingformatty", "ko3n1g", "ksivaman", "kunlunl", "kvareddy", "kwangjunahn", "kwyss-nvidia", "lauradang", "layalir", "lhb8125", "liding-nv", "lmcafee-nvidia", "maanug-nv", "macandro96", "mathemakitten", "matthieule", "mchrzanowski", "mehraakash", "minitu", "mkhona-nvidia", "nanz-nv", "nathan-nvidia", "niyunsheng", "ntajbakhsh", "nvcsathe", "nvegesna-netizen", "parthmannan", "philipcmonk", "prajwal1210", "pthombre", "rapatel", "rhewett-nv", "rogerwaleffe", "sajadn", "sancha", "santhnm2", "santiagoakle", "sbak5", "shanmugamr1992", "sharathts", "sheliang-nv", "shengf-nv", "shifangx", "shjwudp", "sidsingh-nvidia", "skyw", "sraman-rgb", "sudhakarsingh27", "svcnvidia-nemo-ci", "tdene", "theothermike", "tomlifu", "trintamaki", "tylerpoon", "vasunvidia", "wanyingw", "wdykas", "wujingyue", "xiaoyao0115", "xrennvidia", "xuantengh", "xuwchen", "yaox12", "yaoyu-33", "yashaswikarnati", "yeyu-nvidia", "yobibyte", "youngeunkwon0405", "yqwangustc", "yueshen2016", "yuzhongw-nvidia", "zhehuaichen", "zhongbozhu"]

.github/oncall_schedule.json

@@ -1,8 +1,4 @@
 [
-    {
-        "user": "guihong-nv",
-        "date": "2026-10-07"
-    },
     {
         "user": "ilml",
         "date": "2026-10-14"
@@ -46,5 +42,9 @@
     {
         "user": "guihong-nv",
         "date": "2026-12-23"
+    },
+    {
+        "user": "ilml",
+        "date": "2026-12-30"
     }
 ]

.github/workflows/_build_ci_container.yml

@@ -30,6 +30,11 @@ on:
         required: false
         default: nvidia-ci-aws-gpu-x8
         type: string
+      selected-runner-gcp-h100:
+        description: Runner label for an additional Singapore GCP H100 build
+        required: false
+        default: nvidia-ci-gcp-cpu-h100
+        type: string
       selected-runner-gb200:
         description: Runner label for the GB200 build
         required: false
@@ -41,6 +46,22 @@ on:
         # Keep registry defaults aligned with the image registries in cicd-main.yml.
         default: 766267172432.dkr.ecr.us-east-1.amazonaws.com
         type: string
+      registry-gcp-h100:
+        description: Registry for H100 images and caches built on Singapore GCP runners
+        required: false
+        # GAR paths include the repository unlike ECR registry roots.
+        default: asia-southeast1-docker.pkg.dev/nv-nemoautomati-20260505210812/megatron-lm
+        type: string
+      use-gcp-h100:
+        description: Build and publish the H100 image on the Singapore GCP CPU runner
+        required: false
+        default: false
+        type: boolean
+      include-gcp-h100:
+        description: Also build and publish the Singapore GCP H100 image alongside the selected H100 build
+        required: false
+        default: false
+        type: boolean
       registry-gb200:
         description: Registry for GB200 images and caches
         required: false
@@ -83,19 +104,35 @@ jobs:
           IS_MAINTAINER: ${{ inputs.is-maintainer }}
           ENABLE_GB_TESTING: ${{ vars.ENABLE_GB200_TESTING }}
           SELECTED_RUNNER: ${{ inputs.selected-runner }}
+          SELECTED_RUNNER_GCP_H100: ${{ inputs.selected-runner-gcp-h100 }}
           SELECTED_RUNNER_GB_GPU: ${{ inputs.selected-runner-gb200 }}
           REGISTRY_AWS: ${{ inputs.registry-aws }}
+          REGISTRY_GCP_H100: ${{ inputs.registry-gcp-h100 }}
+          USE_GCP_H100: ${{ inputs.use-gcp-h100 }}
+          INCLUDE_GCP_H100: ${{ inputs.include-gcp-h100 }}
           REGISTRY_GB_GPU: ${{ inputs.registry-gb200 }}
         run: |
-          AWS_H100=$(jq -nc --arg registry "$REGISTRY_AWS" --arg runner "$SELECTED_RUNNER" \
-            '{"cloud": "aws-h100", "registry": $registry, "runner": $runner}')
+          if [ "$IS_MAINTAINER" = "true" ] && [ "$USE_GCP_H100" = "true" ]; then
+            GCP_H100=$(jq -nc --arg registry "$REGISTRY_GCP_H100" --arg runner "$SELECTED_RUNNER" \
+              '{"cloud": "gcp-h100", "registry": $registry, "runner": $runner}')
+            H100_MATRIX=$(jq -nc --argjson gcp "$GCP_H100" '[$gcp]')
+          else
+            AWS_H100=$(jq -nc --arg registry "$REGISTRY_AWS" --arg runner "$SELECTED_RUNNER" \
+              '{"cloud": "aws-h100", "registry": $registry, "runner": $runner}')
+            H100_MATRIX=$(jq -nc --argjson aws "$AWS_H100" '[$aws]')
+            if [ "$IS_MAINTAINER" = "true" ] && [ "$INCLUDE_GCP_H100" = "true" ]; then
+              GCP_H100=$(jq -nc --arg registry "$REGISTRY_GCP_H100" --arg runner "$SELECTED_RUNNER_GCP_H100" \
+                '{"cloud": "gcp-h100", "registry": $registry, "runner": $runner}')
+              H100_MATRIX=$(jq -nc --argjson h100 "$H100_MATRIX" --argjson gcp "$GCP_H100" '($h100 + [$gcp])')
+            fi
+          fi
           if [ "$IS_MAINTAINER" = "true" ] && [ "$ENABLE_GB_TESTING" = "true" ]; then
             GB_GPU=$(jq -nc --arg registry "$REGISTRY_GB_GPU" --arg runner "$SELECTED_RUNNER_GB_GPU" \
               '{"cloud": "gb-gpu", "registry": $registry, "runner": $runner}')
-            MATRIX=$(jq -nc --argjson aws "$AWS_H100" --argjson gb_gpu "$GB_GPU" \
-              '{"include": [$aws, $gb_gpu]}')
+            MATRIX=$(jq -nc --argjson h100 "$H100_MATRIX" --argjson gb_gpu "$GB_GPU" \
+              '{"include": ($h100 + [$gb_gpu])}')
           else
-            MATRIX=$(jq -nc --argjson aws "$AWS_H100" '{"include": [$aws]}')
+            MATRIX=$(jq -nc --argjson h100 "$H100_MATRIX" '{"include": $h100}')
           fi
           echo "matrix=$MATRIX" | tee -a "$GITHUB_OUTPUT"
 
@@ -314,8 +351,19 @@ jobs:
           echo "target-build-hash=$TARGET_BUILD_HASH_KEY" | tee -a "$GITHUB_OUTPUT"
           echo "pr-build-hash=$PR_BUILD_HASH_KEY" | tee -a "$GITHUB_OUTPUT"
 
+      - name: Configure GAR Docker authentication
+        if: inputs.is-maintainer && matrix.cloud == 'gcp-h100' && startsWith(matrix.registry, 'asia-southeast1-docker.pkg.dev/')
+        shell: bash
+        env:
+          REGISTRY: ${{ matrix.registry }}
+        run: |
+          GAR_HOST="${REGISTRY%%/*}"
+          command -v docker-credential-gcr
+          docker-credential-gcr configure-docker --registries="$GAR_HOST"
+          docker-credential-gcr get <<< "https://$GAR_HOST" >/dev/null
+
       - name: Login to ECR
-        if: inputs.is-maintainer && (matrix.cloud == 'aws-h100' || matrix.cloud == 'gb-gpu')
+        if: inputs.is-maintainer && (matrix.cloud == 'gb-gpu' || (matrix.cloud == 'aws-h100' && !startsWith(matrix.registry, 'asia-southeast1-docker.pkg.dev/')))
         uses: aws-actions/amazon-ecr-login@03f1aad4c6c7ffd436567f42f9384779290529bd # v2.1.7
         env:
           AWS_REGION: ${{ matrix.cloud == 'gb-gpu' && 'us-east-2' || 'us-east-1' }}

.github/workflows/cicd-approve-test-queue.yml

@@ -114,7 +114,7 @@ jobs:
               "X-GitHub-Api-Version": "2022-11-28",
           }
 
-          def make_request(endpoint, method="GET", data=None, max_retries=5):
+          def make_request(endpoint, method="GET", data=None, max_retries=5, return_response=False):
               """Make a request to the GitHub API with retry on transient errors."""
               url = f"{API_BASE}/{endpoint}"
               for attempt in range(max_retries):
@@ -134,7 +134,7 @@ jobs:
                           time.sleep(delay)
                           continue
                       response.raise_for_status()
-                      return response.json()
+                      return response if return_response else response.json()
                   except (requests.exceptions.ConnectionError, requests.exceptions.Timeout) as e:
                       delay = 2 ** attempt
                       print(f"Transient error on {endpoint}: {e}, retrying in {delay}s (attempt {attempt + 1}/{max_retries})")
@@ -147,6 +147,24 @@ jobs:
               print(f"Max retries ({max_retries}) exceeded for {endpoint}")
               return None
 
+          def get_workflow_runs(status):
+              """Fetch every page before filtering runs or making approval decisions."""
+              runs = {}
+              page = 1
+              while True:
+                  response = make_request(
+                      f"actions/runs?status={status}&per_page=100&page={page}",
+                      return_response=True,
+                  )
+                  if response is None:
+                      return None
+                  # Runs can move between pages while the queue is changing.
+                  for run in response.json()["workflow_runs"]:
+                      runs[run["id"]] = run
+                  if "next" not in response.links:
+                      return {"workflow_runs": list(runs.values())}
+                  page += 1
+
           def is_internal_contributor(pr_info):
               """Return True for internal service accounts or NVIDIA organization members."""
               login = pr_info.get("user", {}).get("login", "")
@@ -225,12 +243,12 @@ jobs:
 
           # Get current running and queued workflows
           print("Fetching workflow runs...")
-          queued_resp = make_request("actions/runs?status=queued")
+          queued_resp = get_workflow_runs("queued")
           if queued_resp is None:
               print("Failed to fetch queued workflow runs after retries, exiting")
               exit(1)
           queued_workflow_runs = queued_resp.get("workflow_runs", [])
-          in_progress_resp = make_request("actions/runs?status=in_progress")
+          in_progress_resp = get_workflow_runs("in_progress")
           if in_progress_resp is None:
               print("Failed to fetch in-progress workflow runs after retries, exiting")
               exit(1)
@@ -272,7 +290,7 @@ jobs:
 
           # Get waiting CI workflows for test environment
           print("Fetching deployments...")
-          waiting_resp = make_request("actions/runs?status=waiting")
+          waiting_resp = get_workflow_runs("waiting")
           if waiting_resp is None:
               print("Failed to fetch waiting workflow runs after retries, exiting")
               exit(1)