Merge remote-tracking branch 'upstream/master' into feature/mie-gruneisen-jwl

Author: sbryngelson

.github/scripts/select-gpu-partition.sh

@@ -4,24 +4,38 @@
 #
 # Priority order prefers partitions most likely to have availability.
 # V100 is last due to slower performance near the test time limit.
-# Falls back to gpu-l40s if no partition meets the idle node threshold.
+# Falls back to gpu-a100 if no partition meets the idle node threshold.
+#
+# gpu-l40s is out of rotation: it has been failing jobs for weeks, and it was
+# also the partition CI kept selecting and then starving on.
+#
+# Only fully idle nodes count. A "mix" node is partially allocated and may have
+# no free GPU, so counting it overstates availability: run 33553417354 picked
+# gpu-l40s on "1 idle/mix nodes", then sat in the queue until the 3-5.5h job
+# timeout without ever starting.
+#
+# The match is anchored at both ends. sinfo's %t suffixes a state to flag it --
+# "*" not responding, "$" reserved for maintenance, "~" powered down -- and this
+# cluster does emit them (drain*, down*, alloc$, drain$ are all live right now).
+# A bare "^idle" would count idle* and idle$ as available and starve the job on
+# nodes that cannot take it.
 # RTX 6000 nodes are excluded (too slow for the test suite time limit).
 #
 # Optional: set GPU_PARTITION_MIN_NODES before sourcing to require a minimum
-# number of idle/mix nodes (e.g. GPU_PARTITION_MIN_NODES=2 for parallel bench jobs).
+# number of idle nodes (e.g. GPU_PARTITION_MIN_NODES=2 for parallel bench jobs).
 #
 # Usage: source .github/scripts/select-gpu-partition.sh
 
-_GPU_PARTITION_PRIORITY="gpu-l40s gpu-h200 gpu-h100 gpu-a100 gpu-v100"
-_GPU_PARTITION_FALLBACK="gpu-l40s"
+_GPU_PARTITION_PRIORITY="gpu-h200 gpu-h100 gpu-a100 gpu-v100"
+_GPU_PARTITION_FALLBACK="gpu-a100"
 _GPU_PARTITION_MIN_NODES="${GPU_PARTITION_MIN_NODES:-1}"
 
 SELECTED_GPU_PARTITION=""
 for _part in $_GPU_PARTITION_PRIORITY; do
-    _idle=$(sinfo -p "$_part" --noheader -o "%t" 2>/dev/null | grep -cE "^(idle|mix)" || true)
+    _idle=$(sinfo -p "$_part" --noheader -o "%t" 2>/dev/null | grep -cE "^idle$" || true)
     if [ "${_idle:-0}" -ge "$_GPU_PARTITION_MIN_NODES" ]; then
         SELECTED_GPU_PARTITION="$_part"
-        echo "Selected GPU partition: $SELECTED_GPU_PARTITION ($_idle idle/mix nodes)"
+        echo "Selected GPU partition: $SELECTED_GPU_PARTITION ($_idle idle nodes)"
         break
     fi
 done

.github/scripts/submit-slurm-job.sh

@@ -150,6 +150,12 @@ elif [ "$device" = "gpu" ]; then
             sbatch_device_opts="\
 #SBATCH -n 8
 #SBATCH -p g1"
+            # Seed, same as phoenix above: the preflight adds nodes to this at
+            # run time. frontier10202 produced all 183 GPU memory-access faults
+            # in run 33553417354 (43 distinct tests) while the same lanes passed
+            # on eight other g1 nodes with none. Its faults are intermittent --
+            # 379 of 382 tests still passed there -- so syscheck can clear it.
+            node_exclude="frontier10202"
             ;;
     esac
 else