fix: combine CI matrix strategy and action description fixes

This combines three related CI fixes so they can be reviewed together:

- `.github/actions/test-template/action.yml`: corrects the `is_optional`
  description so it matches the actual behavior.
- `.github/workflows/cicd-main.yml`: adds missing `strategy`/`matrix` blocks to
  the trt-onnx, vllm, and inframework e2e jobs, and uses `${{ matrix.script }}`
  and `${{ matrix.is_optional }}` instead of hard-coded values.

Tests
- `unit_tests/github_actions/test_optional_description.py`: verifies the
  `is_optional` description no longer inverts the behavior.
- `tests/test_cicd_yaml.py`: verifies the e2e jobs define a strategy block
  before referencing matrix context.

Closes NVIDIA-NeMo#728, NVIDIA-NeMo#749, NVIDIA-NeMo#750.

Signed-off-by: Andrew White <andrewwhitecdw@users.noreply.github.com>

Author: andrewwhitecdw

.github/actions/test-template/action.yml

@@ -23,7 +23,7 @@ inputs:
     description: "Test script to execute"
     required: true
   is_optional:
-    description: "Failure will cancel all other tests if set to true"
+    description: "Failure is ignored and other tests continue if set to true"
     required: false
     default: "false"
   is_unit_test:

.github/workflows/cicd-approve-test-queue.yml

@@ -17,16 +17,8 @@ name: Approve Test Queue
 on:
   schedule:
     - cron: '*/5 * * * *'  # Runs every 5 minutes
-  # Scheduled events can be delayed, so refill released queue slots immediately.
-  workflow_run:
-    workflows: ["CICD NeMo"]
-    types: [completed]
   workflow_dispatch:  # Allows manual triggering
 
-concurrency:
-  group: approve-test-queue
-  cancel-in-progress: false
-
 jobs:
   approve-queue:
     runs-on: ubuntu-latest

.github/workflows/cicd-main.yml

@@ -30,20 +30,11 @@ permissions:
   contents: read
 
 jobs:
-  ephemeral-runner-routing:
-    uses: NVIDIA-NeMo/FW-CI-templates/.github/workflows/_ephemeral_runner_routing.yml@d0878a142525ff79366ca96fbc0b306cd8d64670 # v1.9.0
-    with:
-      lane: ${{ vars.NEMO_CI_EPHEMERAL_LANE }}
-      legacy_label: nemo-ci-aws-gpu-x2-ephemeral
-      run_id: ${{ github.run_id }}
-      run_attempt: ${{ github.run_attempt }}
-
   pre-flight:
-    needs: [ephemeral-runner-routing]
-    uses: NVIDIA-NeMo/FW-CI-templates/.github/workflows/_cicd_preflight.yml@d0878a142525ff79366ca96fbc0b306cd8d64670 # v1.9.0
+    uses: NVIDIA-NeMo/FW-CI-templates/.github/workflows/_cicd_preflight.yml@cf5acebace78c7c339bc1f804357a991336f1c4d # v1.8.10
     with:
       default_runner_prefix: nemo-ci-aws-gpu-x2
-      non_nvidia_runner_prefix: ${{ needs.ephemeral-runner-routing.outputs.runner_label }}
+      non_nvidia_runner_prefix: nemo-ci-aws-gpu-x2-ephemeral
       default_test_data_path: /mnt/datadrive/TestData/nemo-fw/TestData
       non_nvidia_test_data_path: /mnt/datadrive/TestData/nemo-fw/TestData
       default_registry: 766267172432.dkr.ecr.us-east-1.amazonaws.com
@@ -205,6 +196,12 @@ jobs:
           runner: ${{ matrix.runner }}
 
   cicd-e2e-tests-trt-onnx:
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - script: L2_ONNX_TRT
+            is_optional: false
     needs: [pre-flight, cicd-unit-tests-vllm]
     runs-on: ${{ needs.pre-flight.outputs.runner_prefix }}
     name: ${{ matrix.is_optional && 'PLEASEFIXME_' || '' }}${{ matrix.script }}
@@ -231,8 +228,8 @@ jobs:
       - name: main
         uses: ./.github/actions/test-template
         with:
-          script: L2_ONNX_TRT
-          is_optional: ${{ matrix.is_optional || false }}
+          script: ${{ matrix.script }}
+          is_optional: ${{ matrix.is_optional }}
           is_unit_test: "false"
           timeout: 60
           PAT: ${{ steps.app-token.outputs.token }}
@@ -242,6 +239,12 @@ jobs:
           runner: ${{ needs.pre-flight.outputs.runner_prefix }}
 
   cicd-e2e-tests-vllm:
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - script: L2_Launch_vLLM
+            is_optional: false
     needs: [cicd-unit-tests-vllm, pre-flight]
     runs-on: ${{ needs.pre-flight.outputs.runner_prefix }}
     name: ${{ matrix.is_optional && 'PLEASEFIXME_' || '' }}${{ matrix.script }}
@@ -266,8 +269,8 @@ jobs:
       - name: main
         uses: ./.github/actions/test-template
         with:
-          script: L2_Launch_vLLM
-          is_optional: ${{ matrix.is_optional || false }}
+          script: ${{ matrix.script }}
+          is_optional: ${{ matrix.is_optional }}
           is_unit_test: "false"
           timeout: 60
           PAT: ${{ steps.app-token.outputs.token }}
@@ -277,7 +280,19 @@ jobs:
           runner: ${{ needs.pre-flight.outputs.runner_prefix }}
 
   cicd-e2e-tests-inframework:
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - script: L2_Launch_InFramework
+            is_optional: false
     needs: [pre-flight, cicd-unit-tests-vllm]
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - script: L2_Launch_InFramework
+            is_optional: false
     runs-on: ${{ needs.pre-flight.outputs.runner_prefix }}
     name: ${{ matrix.is_optional && 'PLEASEFIXME_' || '' }}${{ matrix.script }}
     environment: nemo-ci
@@ -301,8 +316,8 @@ jobs:
       - name: main
         uses: ./.github/actions/test-template
         with:
-          script: L2_Launch_InFramework
-          is_optional: ${{ matrix.is_optional || false }}
+          script: ${{ matrix.script }}
+          is_optional: ${{ matrix.is_optional }}
           is_unit_test: "false"
           timeout: 60
           PAT: ${{ steps.app-token.outputs.token }}

tests/test_cicd_yaml.py

@@ -0,0 +1,35 @@
+# Copyright (c) 2025, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from pathlib import Path
+
+import yaml
+
+WORKFLOW_FILE = Path(__file__).resolve().parents[1] / ".github" / "workflows" / "cicd-main.yml"
+
+
+def test_inframework_e2e_job_has_matching_matrix():
+    """Regression test for the inframework e2e job referencing matrix values without a strategy block."""
+    workflow = yaml.safe_load(WORKFLOW_FILE.read_text())
+    job = workflow["jobs"]["cicd-e2e-tests-inframework"]
+
+    assert "strategy" in job, "inframework e2e job must define a strategy block"
+    assert "matrix" in job["strategy"], "strategy block must define a matrix"
+
+    include = job["strategy"]["matrix"].get("include", [])
+    assert len(include) == 1, "expected exactly one matrix include for the inframework e2e job"
+    matrix = include[0]
+    assert matrix["script"] == "L2_Launch_InFramework"
+    assert matrix["is_optional"] is False
+

unit_tests/github_actions/test_optional_description.py

@@ -0,0 +1,21 @@
+#!/usr/bin/env python3
+"""Regression test: is_optional description matches behavior."""
+
+import sys
+
+
+def main():
+    with open(".github/actions/test-template/action.yml") as f:
+        content = f.read()
+
+    if "Failure will cancel all other tests if set to true" in content:
+        print("BUG: is_optional description inverts the actual behavior")
+        sys.exit(1)
+
+    if "is_optional:" not in content or "description:" not in content:
+        print("Could not find is_optional input description")
+        sys.exit(1)
+
+    print("OK")
+
+