Skip to content
Merged
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
28 changes: 22 additions & 6 deletions .github/scripts/submit-slurm-job.sh
Original file line number Diff line number Diff line change
Expand Up @@ -126,13 +126,29 @@ elif [ "$device" = "gpu" ]; then
# Determine GPU partition
gpu_partition="batch"
if [ "$gpu_partition_dynamic" = "true" ]; then
# Use pre-selected bench partition if available, otherwise query sinfo
if [ -n "${BENCH_GPU_PARTITION:-}" ]; then
gpu_partition="$BENCH_GPU_PARTITION"
echo "Using pre-selected bench partition: $gpu_partition (PR/master consistency)"
if [ "$job_type" = "bench" ]; then
# Benchmarks compare PR against master, so both jobs must land on the
# SAME GPU type or the comparison is meaningless. That rules out a
# partition list (SLURM could place PR and master on different
# hardware); instead a single partition is picked and pinned across
# both jobs via BENCH_GPU_PARTITION. See run_parallel_benchmarks.sh.
if [ -n "${BENCH_GPU_PARTITION:-}" ]; then
gpu_partition="$BENCH_GPU_PARTITION"
echo "Using pre-selected bench partition: $gpu_partition (PR/master consistency)"
else
source "${SCRIPT_DIR}/select-gpu-partition.sh"
gpu_partition="$SELECTED_GPU_PARTITION"
fi
else
source "${SCRIPT_DIR}/select-gpu-partition.sh"
gpu_partition="$SELECTED_GPU_PARTITION"
# Tests (and build+test) don't compare across hardware, so submit to a
# partition LIST and let SLURM start on whichever frees first instead
# of pinning one partition and queueing behind it. This restores the
# multi-partition backfill that #1299 dropped when it unified test and
# bench onto the single-partition bench selector. gpu-l40s (bad
# hardware) and gpu-rtx6000 (too slow for the test time limit) are
# intentionally omitted.
gpu_partition="gpu-h200,gpu-h100,gpu-a100,gpu-v100"
echo "Using GPU partition list for test job: $gpu_partition"
Comment on lines +150 to +151
fi
fi

Expand Down
Loading