Merge remote-tracking branch 'upstream/master' into pr-1792

Author: sbryngelson

.github/scripts/check_coverage_map_health.py

@@ -1,11 +1,10 @@
 """Fail loudly if the committed coverage map is stale or under-covers. Used by coverage-health.yml."""
 import datetime
-import subprocess
 import sys
 from pathlib import Path
 
 sys.path.insert(0, str(Path(__file__).resolve().parents[2] / "toolchain"))
-from mfc.test.coverage import COVERAGE_MAP_PATH, load_map, map_health  # noqa: E402
+from mfc.test.coverage import COVERAGE_MAP_PATH, _git, load_map, map_health  # noqa: E402
 from mfc.test.cases import list_cases  # noqa: E402  (returns the current test list)
 
 MAX_AGE_DAYS = 10
@@ -40,7 +39,7 @@ def verified_sha(cwd=None):
     caller must read that as undeterminable and fall back to the wall-clock age rule, not
     as a failure -- an absent ref is not evidence of a broken refresh.
     """
-    rev = subprocess.run(["git", "rev-parse", "--verify", "--quiet", f"{VERIFIED_REF}^{{commit}}"], capture_output=True, text=True, check=False, cwd=cwd)
+    rev = _git(["rev-parse", "--verify", "--quiet", f"{VERIFIED_REF}^{{commit}}"], cwd)
     return rev.stdout.strip() or None
 
 
@@ -53,10 +52,10 @@ def verified_after_last_change(git_sha, cwd=None):
     """
     if not git_sha:
         return None
-    last = subprocess.run(["git", "log", "-1", "--format=%H", "--", *COVERAGE_RELEVANT_PATHS], capture_output=True, text=True, check=False, cwd=cwd)
+    last = _git(["log", "-1", "--format=%H", "--", *COVERAGE_RELEVANT_PATHS], cwd)
     if last.returncode != 0 or not last.stdout.strip():
         return None  # shallow clone or no such commit -> fall back to the age rule
-    ancestor = subprocess.run(["git", "merge-base", "--is-ancestor", last.stdout.strip(), git_sha], capture_output=True, check=False, cwd=cwd)
+    ancestor = _git(["merge-base", "--is-ancestor", last.stdout.strip(), git_sha], cwd)
     return {0: True, 1: False}.get(ancestor.returncode)  # anything else -> None (unknown sha, shallow history)
 
 

.github/scripts/select-gpu-partition.sh

@@ -4,24 +4,38 @@
 #
 # Priority order prefers partitions most likely to have availability.
 # V100 is last due to slower performance near the test time limit.
-# Falls back to gpu-l40s if no partition meets the idle node threshold.
+# Falls back to gpu-a100 if no partition meets the idle node threshold.
+#
+# gpu-l40s is out of rotation: it has been failing jobs for weeks, and it was
+# also the partition CI kept selecting and then starving on.
+#
+# Only fully idle nodes count. A "mix" node is partially allocated and may have
+# no free GPU, so counting it overstates availability: run 33553417354 picked
+# gpu-l40s on "1 idle/mix nodes", then sat in the queue until the 3-5.5h job
+# timeout without ever starting.
+#
+# The match is anchored at both ends. sinfo's %t suffixes a state to flag it --
+# "*" not responding, "$" reserved for maintenance, "~" powered down -- and this
+# cluster does emit them (drain*, down*, alloc$, drain$ are all live right now).
+# A bare "^idle" would count idle* and idle$ as available and starve the job on
+# nodes that cannot take it.
 # RTX 6000 nodes are excluded (too slow for the test suite time limit).
 #
 # Optional: set GPU_PARTITION_MIN_NODES before sourcing to require a minimum
-# number of idle/mix nodes (e.g. GPU_PARTITION_MIN_NODES=2 for parallel bench jobs).
+# number of idle nodes (e.g. GPU_PARTITION_MIN_NODES=2 for parallel bench jobs).
 #
 # Usage: source .github/scripts/select-gpu-partition.sh
 
-_GPU_PARTITION_PRIORITY="gpu-l40s gpu-h200 gpu-h100 gpu-a100 gpu-v100"
-_GPU_PARTITION_FALLBACK="gpu-l40s"
+_GPU_PARTITION_PRIORITY="gpu-h200 gpu-h100 gpu-a100 gpu-v100"
+_GPU_PARTITION_FALLBACK="gpu-a100"
 _GPU_PARTITION_MIN_NODES="${GPU_PARTITION_MIN_NODES:-1}"
 
 SELECTED_GPU_PARTITION=""
 for _part in $_GPU_PARTITION_PRIORITY; do
-    _idle=$(sinfo -p "$_part" --noheader -o "%t" 2>/dev/null | grep -cE "^(idle|mix)" || true)
+    _idle=$(sinfo -p "$_part" --noheader -o "%t" 2>/dev/null | grep -cE "^idle$" || true)
     if [ "${_idle:-0}" -ge "$_GPU_PARTITION_MIN_NODES" ]; then
         SELECTED_GPU_PARTITION="$_part"
-        echo "Selected GPU partition: $SELECTED_GPU_PARTITION ($_idle idle/mix nodes)"
+        echo "Selected GPU partition: $SELECTED_GPU_PARTITION ($_idle idle nodes)"
         break
     fi
 done

.github/scripts/submit-slurm-job.sh

@@ -150,6 +150,12 @@ elif [ "$device" = "gpu" ]; then
             sbatch_device_opts="\
 #SBATCH -n 8
 #SBATCH -p g1"
+            # Seed, same as phoenix above: the preflight adds nodes to this at
+            # run time. frontier10202 produced all 183 GPU memory-access faults
+            # in run 33553417354 (43 distinct tests) while the same lanes passed
+            # on eight other g1 nodes with none. Its faults are intermittent --
+            # 379 of 382 tests still passed there -- so syscheck can clear it.
+            node_exclude="frontier10202"
             ;;
     esac
 else

examples/1D_advection_convergence/case.py

@@ -95,10 +95,8 @@
             "patch_icpp(1)%alpha(2)": "0.5 - 0.2 * sin(2.0 * pi * x / lx)",
             "fluid_pp(1)%gamma": 1.0 / (gamma - 1.0),
             "fluid_pp(1)%eos": "ideal_gas",
-            "fluid_pp(1)%pi_inf": 0.0,
             "fluid_pp(2)%gamma": 1.0 / (gamma - 1.0),
             "fluid_pp(2)%eos": "ideal_gas",
-            "fluid_pp(2)%pi_inf": 0.0,
             **scheme_params,
         }
     )

examples/1D_brio_wu/case.py

@@ -74,7 +74,6 @@
             # Fluids Physical Parameters
             "fluid_pp(1)%gamma": 1.0e00 / (2.0e00 - 1.0e00),
             "fluid_pp(1)%eos": "ideal_gas",
-            "fluid_pp(1)%pi_inf": 0.0,
         }
     )
 )

examples/1D_brio_wu_hlld/case.py

@@ -75,7 +75,6 @@
             # Fluids Physical Parameters
             "fluid_pp(1)%gamma": 1.0e00 / (2.0e00 - 1.0e00),
             "fluid_pp(1)%eos": "ideal_gas",
-            "fluid_pp(1)%pi_inf": 0.0,
         }
     )
 )

examples/1D_brio_wu_rmhd/case.py

@@ -76,7 +76,6 @@
             # Fluids Physical Parameters
             "fluid_pp(1)%gamma": 1.0e00 / (2.0e00 - 1.0e00),
             "fluid_pp(1)%eos": "ideal_gas",
-            "fluid_pp(1)%pi_inf": 0.0,
         }
     )
 )

examples/1D_convergence/case.py

@@ -74,10 +74,8 @@
             # Fluids Physical Parameters
             "fluid_pp(1)%gamma": 1.0e00 / (1.4 - 1.0e00),
             "fluid_pp(1)%eos": "ideal_gas",
-            "fluid_pp(1)%pi_inf": 0.0,
             "fluid_pp(2)%gamma": 1.0e00 / (1.4 - 1.0e00),
             "fluid_pp(2)%eos": "ideal_gas",
-            "fluid_pp(2)%pi_inf": 0.0,
         }
     )
 )

examples/1D_dai_woodward/case.py

@@ -83,7 +83,6 @@
             # Fluids Physical Parameters
             "fluid_pp(1)%gamma": 1.0e00 / ((5.0 / 3.0) - 1.0e00),
             "fluid_pp(1)%eos": "ideal_gas",
-            "fluid_pp(1)%pi_inf": 0.0,
         },
     )
 )

examples/1D_dai_woodward_hlld/case.py

@@ -84,7 +84,6 @@
             # Fluids Physical Parameters
             "fluid_pp(1)%gamma": 1.0e00 / ((5.0 / 3.0) - 1.0e00),
             "fluid_pp(1)%eos": "ideal_gas",
-            "fluid_pp(1)%pi_inf": 0.0,
         },
     )
 )