Merge remote-tracking branch 'upstream/master' into fix/precheck-under-git-hooks

Author: sbryngelson

.github/scripts/select-gpu-partition.sh

@@ -4,24 +4,38 @@
 #
 # Priority order prefers partitions most likely to have availability.
 # V100 is last due to slower performance near the test time limit.
-# Falls back to gpu-l40s if no partition meets the idle node threshold.
+# Falls back to gpu-a100 if no partition meets the idle node threshold.
+#
+# gpu-l40s is out of rotation: it has been failing jobs for weeks, and it was
+# also the partition CI kept selecting and then starving on.
+#
+# Only fully idle nodes count. A "mix" node is partially allocated and may have
+# no free GPU, so counting it overstates availability: run 33553417354 picked
+# gpu-l40s on "1 idle/mix nodes", then sat in the queue until the 3-5.5h job
+# timeout without ever starting.
+#
+# The match is anchored at both ends. sinfo's %t suffixes a state to flag it --
+# "*" not responding, "$" reserved for maintenance, "~" powered down -- and this
+# cluster does emit them (drain*, down*, alloc$, drain$ are all live right now).
+# A bare "^idle" would count idle* and idle$ as available and starve the job on
+# nodes that cannot take it.
 # RTX 6000 nodes are excluded (too slow for the test suite time limit).
 #
 # Optional: set GPU_PARTITION_MIN_NODES before sourcing to require a minimum
-# number of idle/mix nodes (e.g. GPU_PARTITION_MIN_NODES=2 for parallel bench jobs).
+# number of idle nodes (e.g. GPU_PARTITION_MIN_NODES=2 for parallel bench jobs).
 #
 # Usage: source .github/scripts/select-gpu-partition.sh
 
-_GPU_PARTITION_PRIORITY="gpu-l40s gpu-h200 gpu-h100 gpu-a100 gpu-v100"
-_GPU_PARTITION_FALLBACK="gpu-l40s"
+_GPU_PARTITION_PRIORITY="gpu-h200 gpu-h100 gpu-a100 gpu-v100"
+_GPU_PARTITION_FALLBACK="gpu-a100"
 _GPU_PARTITION_MIN_NODES="${GPU_PARTITION_MIN_NODES:-1}"
 
 SELECTED_GPU_PARTITION=""
 for _part in $_GPU_PARTITION_PRIORITY; do
-    _idle=$(sinfo -p "$_part" --noheader -o "%t" 2>/dev/null | grep -cE "^(idle|mix)" || true)
+    _idle=$(sinfo -p "$_part" --noheader -o "%t" 2>/dev/null | grep -cE "^idle$" || true)
     if [ "${_idle:-0}" -ge "$_GPU_PARTITION_MIN_NODES" ]; then
         SELECTED_GPU_PARTITION="$_part"
-        echo "Selected GPU partition: $SELECTED_GPU_PARTITION ($_idle idle/mix nodes)"
+        echo "Selected GPU partition: $SELECTED_GPU_PARTITION ($_idle idle nodes)"
         break
     fi
 done

.github/scripts/submit-slurm-job.sh

@@ -150,6 +150,12 @@ elif [ "$device" = "gpu" ]; then
             sbatch_device_opts="\
 #SBATCH -n 8
 #SBATCH -p g1"
+            # Seed, same as phoenix above: the preflight adds nodes to this at
+            # run time. frontier10202 produced all 183 GPU memory-access faults
+            # in run 33553417354 (43 distinct tests) while the same lanes passed
+            # on eight other g1 nodes with none. Its faults are intermittent --
+            # 379 of 382 tests still passed there -- so syscheck can clear it.
+            node_exclude="frontier10202"
             ;;
     esac
 else

examples/1D_advection_convergence/case.py

@@ -95,10 +95,8 @@
             "patch_icpp(1)%alpha(2)": "0.5 - 0.2 * sin(2.0 * pi * x / lx)",
             "fluid_pp(1)%gamma": 1.0 / (gamma - 1.0),
             "fluid_pp(1)%eos": "ideal_gas",
-            "fluid_pp(1)%pi_inf": 0.0,
             "fluid_pp(2)%gamma": 1.0 / (gamma - 1.0),
             "fluid_pp(2)%eos": "ideal_gas",
-            "fluid_pp(2)%pi_inf": 0.0,
             **scheme_params,
         }
     )

examples/1D_brio_wu/case.py

@@ -74,7 +74,6 @@
             # Fluids Physical Parameters
             "fluid_pp(1)%gamma": 1.0e00 / (2.0e00 - 1.0e00),
             "fluid_pp(1)%eos": "ideal_gas",
-            "fluid_pp(1)%pi_inf": 0.0,
         }
     )
 )

examples/1D_brio_wu_hlld/case.py

@@ -75,7 +75,6 @@
             # Fluids Physical Parameters
             "fluid_pp(1)%gamma": 1.0e00 / (2.0e00 - 1.0e00),
             "fluid_pp(1)%eos": "ideal_gas",
-            "fluid_pp(1)%pi_inf": 0.0,
         }
     )
 )

examples/1D_brio_wu_rmhd/case.py

@@ -76,7 +76,6 @@
             # Fluids Physical Parameters
             "fluid_pp(1)%gamma": 1.0e00 / (2.0e00 - 1.0e00),
             "fluid_pp(1)%eos": "ideal_gas",
-            "fluid_pp(1)%pi_inf": 0.0,
         }
     )
 )

examples/1D_convergence/case.py

@@ -74,10 +74,8 @@
             # Fluids Physical Parameters
             "fluid_pp(1)%gamma": 1.0e00 / (1.4 - 1.0e00),
             "fluid_pp(1)%eos": "ideal_gas",
-            "fluid_pp(1)%pi_inf": 0.0,
             "fluid_pp(2)%gamma": 1.0e00 / (1.4 - 1.0e00),
             "fluid_pp(2)%eos": "ideal_gas",
-            "fluid_pp(2)%pi_inf": 0.0,
         }
     )
 )

examples/1D_dai_woodward/case.py

@@ -83,7 +83,6 @@
             # Fluids Physical Parameters
             "fluid_pp(1)%gamma": 1.0e00 / ((5.0 / 3.0) - 1.0e00),
             "fluid_pp(1)%eos": "ideal_gas",
-            "fluid_pp(1)%pi_inf": 0.0,
         },
     )
 )

examples/1D_dai_woodward_hlld/case.py

@@ -84,7 +84,6 @@
             # Fluids Physical Parameters
             "fluid_pp(1)%gamma": 1.0e00 / ((5.0 / 3.0) - 1.0e00),
             "fluid_pp(1)%eos": "ideal_gas",
-            "fluid_pp(1)%pi_inf": 0.0,
         },
     )
 )

examples/1D_euler_convergence/case.py

@@ -100,7 +100,6 @@
             "patch_icpp(1)%alpha(1)": 1.0,
             "fluid_pp(1)%gamma": 1.0 / (gamma - 1.0),
             "fluid_pp(1)%eos": "ideal_gas",
-            "fluid_pp(1)%pi_inf": 0.0,
             **scheme_params,
         }
     )