ci: submit Phoenix GPU test jobs to a partition list for backfill (#1832)

Author: sbryngelson

.github/scripts/submit-slurm-job.sh

@@ -126,13 +126,29 @@ elif [ "$device" = "gpu" ]; then
     # Determine GPU partition
     gpu_partition="batch"
     if [ "$gpu_partition_dynamic" = "true" ]; then
-        # Use pre-selected bench partition if available, otherwise query sinfo
-        if [ -n "${BENCH_GPU_PARTITION:-}" ]; then
-            gpu_partition="$BENCH_GPU_PARTITION"
-            echo "Using pre-selected bench partition: $gpu_partition (PR/master consistency)"
+        if [ "$job_type" = "bench" ]; then
+            # Benchmarks compare PR against master, so both jobs must land on the
+            # SAME GPU type or the comparison is meaningless. That rules out a
+            # partition list (SLURM could place PR and master on different
+            # hardware); instead a single partition is picked and pinned across
+            # both jobs via BENCH_GPU_PARTITION. See run_parallel_benchmarks.sh.
+            if [ -n "${BENCH_GPU_PARTITION:-}" ]; then
+                gpu_partition="$BENCH_GPU_PARTITION"
+                echo "Using pre-selected bench partition: $gpu_partition (PR/master consistency)"
+            else
+                source "${SCRIPT_DIR}/select-gpu-partition.sh"
+                gpu_partition="$SELECTED_GPU_PARTITION"
+            fi
         else
-            source "${SCRIPT_DIR}/select-gpu-partition.sh"
-            gpu_partition="$SELECTED_GPU_PARTITION"
+            # Tests (and build+test) don't compare across hardware, so submit to a
+            # partition LIST and let SLURM start on whichever frees first instead
+            # of pinning one partition and queueing behind it. This restores the
+            # multi-partition backfill that #1299 dropped when it unified test and
+            # bench onto the single-partition bench selector. gpu-l40s (bad
+            # hardware) and gpu-rtx6000 (too slow for the test time limit) are
+            # intentionally omitted.
+            gpu_partition="gpu-h200,gpu-h100,gpu-a100,gpu-v100"
+            echo "Using GPU partition list for test job: $gpu_partition"
         fi
     fi