diff --git a/.github/scripts/submit-slurm-job.sh b/.github/scripts/submit-slurm-job.sh index 72f558226..28f476a64 100755 --- a/.github/scripts/submit-slurm-job.sh +++ b/.github/scripts/submit-slurm-job.sh @@ -126,13 +126,29 @@ elif [ "$device" = "gpu" ]; then # Determine GPU partition gpu_partition="batch" if [ "$gpu_partition_dynamic" = "true" ]; then - # Use pre-selected bench partition if available, otherwise query sinfo - if [ -n "${BENCH_GPU_PARTITION:-}" ]; then - gpu_partition="$BENCH_GPU_PARTITION" - echo "Using pre-selected bench partition: $gpu_partition (PR/master consistency)" + if [ "$job_type" = "bench" ]; then + # Benchmarks compare PR against master, so both jobs must land on the + # SAME GPU type or the comparison is meaningless. That rules out a + # partition list (SLURM could place PR and master on different + # hardware); instead a single partition is picked and pinned across + # both jobs via BENCH_GPU_PARTITION. See run_parallel_benchmarks.sh. + if [ -n "${BENCH_GPU_PARTITION:-}" ]; then + gpu_partition="$BENCH_GPU_PARTITION" + echo "Using pre-selected bench partition: $gpu_partition (PR/master consistency)" + else + source "${SCRIPT_DIR}/select-gpu-partition.sh" + gpu_partition="$SELECTED_GPU_PARTITION" + fi else - source "${SCRIPT_DIR}/select-gpu-partition.sh" - gpu_partition="$SELECTED_GPU_PARTITION" + # Tests (and build+test) don't compare across hardware, so submit to a + # partition LIST and let SLURM start on whichever frees first instead + # of pinning one partition and queueing behind it. This restores the + # multi-partition backfill that #1299 dropped when it unified test and + # bench onto the single-partition bench selector. gpu-l40s (bad + # hardware) and gpu-rtx6000 (too slow for the test time limit) are + # intentionally omitted. + gpu_partition="gpu-h200,gpu-h100,gpu-a100,gpu-v100" + echo "Using GPU partition list for test job: $gpu_partition" fi fi