ci(frontier_amd): build base and chemistry simulation variants concurrently (2 h walltime overrun) (#1662)

Author: sbryngelson

.github/scripts/submit-slurm-job.sh

@@ -3,22 +3,25 @@
 # Submits a script as a SLURM batch job, then monitors it until completion.
 # Rerun-safe: cancels stale jobs from previous runs before resubmission.
 #
-# Usage: submit-slurm-job.sh <script.sh> <cpu|gpu> <none|acc|omp> <cluster> [shard]
+# Usage: submit-slurm-job.sh <script.sh> <cpu|gpu> <none|acc|omp> <cluster> [shard] [variant]
 
 set -euo pipefail
 
 # Ignore SIGHUP to survive login node session drops
 trap '' HUP
 
 usage() {
-    echo "Usage: $0 <script.sh> <cpu|gpu> <none|acc|omp> <cluster> [shard]"
+    echo "Usage: $0 <script.sh> <cpu|gpu> <none|acc|omp> <cluster> [shard] [variant]"
 }
 
 script_path="${1:-}"
 device="${2:-}"
 interface="${3:-}"
 cluster="${4:-}"
 shard="${5:-}"
+# Build variant ("base"/"chem"), letting one build job be split into concurrent
+# per-variant jobs. Empty = build everything in this job (default).
+variant="${6:-}"
 
 if [ -z "$script_path" ] || [ -z "$device" ] || [ -z "$interface" ] || [ -z "$cluster" ]; then
     usage
@@ -136,7 +139,12 @@ shard_suffix=""
 if [ -n "$shard" ]; then
     shard_suffix="-$(echo "$shard" | sed 's|/|-of-|')"
 fi
-job_slug="$(basename "$script_path" | sed 's/\.sh$//' | sed 's/[^a-zA-Z0-9]/-/g')-${device}-${interface}${shard_suffix}"
+variant_suffix=""
+if [ -n "$variant" ]; then
+    # Sanitized like the rest of the slug: these become file names in the workspace.
+    variant_suffix="-$(echo "$variant" | sed 's/[^a-zA-Z0-9]/-/g')"
+fi
+job_slug="$(basename "$script_path" | sed 's/\.sh$//' | sed 's/[^a-zA-Z0-9]/-/g')-${device}-${interface}${shard_suffix}${variant_suffix}"
 output_file="$job_slug.out"
 id_file="${job_slug}.slurm_job_id"
 
@@ -187,6 +195,7 @@ job_slug="$job_slug"
 job_device="$device"
 job_interface="$interface"
 job_shard="$shard"
+job_variant="$variant"
 job_cluster="$cluster"
 export GITHUB_EVENT_NAME="$GITHUB_EVENT_NAME"
 

.github/workflows/common/build.sh

@@ -39,5 +39,20 @@ if [ "$job_cluster" = "phoenix" ]; then
     validate_cmd='syscheck_bin=$(find build/install -name syscheck -type f 2>/dev/null | head -1); [ -z "$syscheck_bin" ] || "$syscheck_bin" > /dev/null 2>&1'
 fi
 
+# --- Variant selection ---
+# The suite needs two simulation binaries: the base one and a chemistry one (the
+# mechanism is compiled in). On amdflang each device link is ~1 hour, so building
+# both serially exceeds the 2 h walltime. $job_variant lets the caller split them
+# into concurrent jobs that write to disjoint staging directories:
+#   base -> plain targets, what every non-chemistry test runs
+#   chem -> the chemistry variant only
+# Unset builds everything in this job (default for every other cluster).
+case "${job_variant:-}" in
+    base) build_cmd=(./mfc.sh build -j 8 $build_opts) ;;
+    chem) build_cmd=(./mfc.sh test -v --dry-run -a -j 8 -o Chemistry $build_opts) ;;
+    "")   build_cmd=(./mfc.sh test -v --dry-run -a -j 8 $build_opts) ;;
+    *)    echo "ERROR: unknown job_variant '$job_variant'"; exit 1 ;;
+esac
+
 RETRY_VALIDATE_CMD="$validate_cmd" \
-    retry_build ./mfc.sh test -v --dry-run -a -j 8 $build_opts || exit 1
+    retry_build "${build_cmd[@]}" || exit 1

.github/workflows/test.yml

@@ -431,8 +431,25 @@ jobs:
         run:  bash .github/workflows/${{ matrix.cluster }}/build.sh ${{ matrix.device }} ${{ matrix.interface }}
 
       - name: Build
+        if:   ${{ !(matrix.cluster == 'frontier_amd' && matrix.device == 'gpu') }}
         run:  bash .github/scripts/submit-slurm-job.sh .github/workflows/common/build.sh ${{ matrix.device }} ${{ matrix.interface }} ${{ matrix.cluster }} ${{ matrix.shard }}
 
+      - name: Build (concurrent variants)
+        if:   matrix.cluster == 'frontier_amd' && matrix.device == 'gpu'
+        # The suite needs a base and a chemistry simulation binary, and on amdflang
+        # each device LTO link is ~1 h -- building both serially overruns the 2 h
+        # walltime. They touch disjoint staging directories, so submit one SLURM job
+        # per variant and wait on both (mirrors the case-optimization pre-build).
+        run:  |
+          pids=""
+          for v in base chem; do
+            bash .github/scripts/submit-slurm-job.sh .github/workflows/common/build.sh ${{ matrix.device }} ${{ matrix.interface }} ${{ matrix.cluster }} "${{ matrix.shard }}" "$v" &
+            pids="$pids $!"
+          done
+          rc=0
+          for p in $pids; do wait "$p" || rc=1; done
+          exit $rc
+
       - name: Test
         run:  bash .github/scripts/submit-slurm-job.sh .github/workflows/common/test.sh ${{ matrix.device }} ${{ matrix.interface }} ${{ matrix.cluster }} ${{ matrix.shard }}
 
@@ -460,7 +477,9 @@ jobs:
       - name: Print Logs
         if:   always()
         run:  |
-          for f in ${{ steps.log.outputs.build_slug }}.out ${{ steps.log.outputs.test_slug }}.out; do
+          # The build slug matches the plain log and, where the build is split per
+          # variant, the build-...-base/-chem ones.
+          for f in ${{ steps.log.outputs.build_slug }}*.out ${{ steps.log.outputs.test_slug }}.out; do
             [ -f "$f" ] && echo "=== $f ===" && cat "$f"
           done
 
@@ -470,7 +489,7 @@ jobs:
         with:
           name: logs-${{ strategy.job-index }}-${{ steps.log.outputs.test_slug }}
           path: |
-            ${{ steps.log.outputs.build_slug }}.out
+            ${{ steps.log.outputs.build_slug }}*.out
             ${{ steps.log.outputs.test_slug }}.out
 
   case-optimization: