Refactor NVHPC setup in GitHub Actions workflow

Updated NVHPC configuration to simplify CUDA handling and improve disk management during CI runs.

Author: sbryngelson

.github/workflows/test.yml

@@ -179,61 +179,59 @@ jobs:
 
           # NVHPC compiler matrix: cpu (build+test), gpu (build-only, acc then omp)
           # Every release from 23.11 through 26.3 (current)
-          # Using single-CUDA tags instead of cuda_multi to fit GitHub runner disk.
-          #   nvhpc  -> highest bundled CUDA 12.x (from release notes)
-          - { nvhpc: '23.11', target: cpu, cuda: '12.3' }
-          - { nvhpc: '23.11', target: gpu, cuda: '12.3' }
-          - { nvhpc: '24.1',  target: cpu, cuda: '12.3' }
-          - { nvhpc: '24.1',  target: gpu, cuda: '12.3' }
-          - { nvhpc: '24.3',  target: cpu, cuda: '12.3' }
-          - { nvhpc: '24.3',  target: gpu, cuda: '12.3' }
-          - { nvhpc: '24.5',  target: cpu, cuda: '12.4' }
-          - { nvhpc: '24.5',  target: gpu, cuda: '12.4' }
-          - { nvhpc: '24.7',  target: cpu, cuda: '12.5' }
-          - { nvhpc: '24.7',  target: gpu, cuda: '12.5' }
-          - { nvhpc: '24.9',  target: cpu, cuda: '12.6' }
-          - { nvhpc: '24.9',  target: gpu, cuda: '12.6' }
-          - { nvhpc: '24.11', target: cpu, cuda: '12.6' }
-          - { nvhpc: '24.11', target: gpu, cuda: '12.6' }
-          - { nvhpc: '25.1',  target: cpu, cuda: '12.6' }
-          - { nvhpc: '25.1',  target: gpu, cuda: '12.6' }
-          - { nvhpc: '25.3',  target: cpu, cuda: '12.8' }
-          - { nvhpc: '25.3',  target: gpu, cuda: '12.8' }
-          - { nvhpc: '25.5',  target: cpu, cuda: '12.8' }
-          - { nvhpc: '25.5',  target: gpu, cuda: '12.8' }
-          - { nvhpc: '25.7',  target: cpu, cuda: '12.8' }
-          - { nvhpc: '25.7',  target: gpu, cuda: '12.8' }
-          - { nvhpc: '25.9',  target: cpu, cuda: '12.9' }
-          - { nvhpc: '25.9',  target: gpu, cuda: '12.9' }
-          - { nvhpc: '25.11', target: cpu, cuda: '12.9' }
-          - { nvhpc: '25.11', target: gpu, cuda: '12.9' }
-          - { nvhpc: '26.1',  target: cpu, cuda: '12.9' }
-          - { nvhpc: '26.1',  target: gpu, cuda: '12.9' }
-          - { nvhpc: '26.3',  target: cpu, cuda: '12.9' }
-          - { nvhpc: '26.3',  target: gpu, cuda: '12.9' }
+          - { nvhpc: '23.11', target: cpu }
+          - { nvhpc: '23.11', target: gpu }
+          - { nvhpc: '24.1',  target: cpu }
+          - { nvhpc: '24.1',  target: gpu }
+          - { nvhpc: '24.3',  target: cpu }
+          - { nvhpc: '24.3',  target: gpu }
+          - { nvhpc: '24.5',  target: cpu }
+          - { nvhpc: '24.5',  target: gpu }
+          - { nvhpc: '24.7',  target: cpu }
+          - { nvhpc: '24.7',  target: gpu }
+          - { nvhpc: '24.9',  target: cpu }
+          - { nvhpc: '24.9',  target: gpu }
+          - { nvhpc: '24.11', target: cpu }
+          - { nvhpc: '24.11', target: gpu }
+          - { nvhpc: '25.1',  target: cpu }
+          - { nvhpc: '25.1',  target: gpu }
+          - { nvhpc: '25.3',  target: cpu }
+          - { nvhpc: '25.3',  target: gpu }
+          - { nvhpc: '25.5',  target: cpu }
+          - { nvhpc: '25.5',  target: gpu }
+          - { nvhpc: '25.7',  target: cpu }
+          - { nvhpc: '25.7',  target: gpu }
+          - { nvhpc: '25.9',  target: cpu }
+          - { nvhpc: '25.9',  target: gpu }
+          - { nvhpc: '25.11', target: cpu }
+          - { nvhpc: '25.11', target: gpu }
+          - { nvhpc: '26.1',  target: cpu }
+          - { nvhpc: '26.1',  target: gpu }
+          - { nvhpc: '26.3',  target: cpu }
+          - { nvhpc: '26.3',  target: gpu }
 
       fail-fast: false
     continue-on-error: true
     runs-on: ${{ matrix.nvhpc && 'ubuntu-22.04' || format('{0}-latest', matrix.os) }}
-    container:
-      image: ${{ matrix.nvhpc && format('nvcr.io/nvidia/nvhpc:{0}-devel-cuda{1}-ubuntu22.04', matrix.nvhpc, matrix.cuda) || '' }}
-      options: ${{ matrix.nvhpc && '--security-opt seccomp=unconfined' || '' }}
     env:
-      CC:  ${{ matrix.nvhpc && 'nvc' || '' }}
-      CXX: ${{ matrix.nvhpc && 'nvc++' || '' }}
-      FC:  ${{ matrix.nvhpc && 'nvfortran' || '' }}
-      OMPI_ALLOW_RUN_AS_ROOT:         ${{ matrix.nvhpc && '1' || '' }}
-      OMPI_ALLOW_RUN_AS_ROOT_CONFIRM: ${{ matrix.nvhpc && '1' || '' }}
-      PMIX_MCA_gds:                   ${{ matrix.nvhpc && 'hash' || '' }}
-      OMPI_MCA_hwloc_base_binding_policy: ${{ matrix.nvhpc && 'none' || '' }}
-      FFLAGS:  ${{ matrix.nvhpc && '-tp=px -Kieee -noswitcherror' || '' }}
-      CFLAGS:  ${{ matrix.nvhpc && '-tp=px' || '' }}
-      CXXFLAGS: ${{ matrix.nvhpc && '-tp=px' || '' }}
+      # Image tag for NVHPC jobs; empty for non-NVHPC jobs.
+      NVHPC_IMAGE: ${{ matrix.nvhpc && format('nvcr.io/nvidia/nvhpc:{0}-devel-cuda_multi-ubuntu22.04', matrix.nvhpc) || '' }}
 
     steps:
-      - name: Git safe directory
+      # ── NVHPC: free disk before pulling the ~25-30 GB cuda_multi image ──
+      - name: Free disk space
         if:   matrix.nvhpc
-        run:  git config --global --add safe.directory /__w/MFC/MFC
+        run: |
+          echo "=== Disk before cleanup ==="
+          df -h /
+          sudo rm -rf /usr/share/dotnet /usr/local/lib/android \
+            /opt/ghc /usr/local/share/boost /opt/hostedtoolcache \
+            /usr/local/graalvm /usr/local/.ghcup \
+            /usr/local/share/chromium /usr/local/lib/node_modules
+          sudo docker image prune -af
+          sudo apt-get clean
+          echo "=== Disk after cleanup ==="
+          df -h /
 
       - name: Clone
         uses: actions/checkout@v4
@@ -276,6 +274,62 @@ jobs:
             echo "Coverage cache: none available — full test suite will run"
           fi
 
+      # ── NVHPC: pull image and start a long-lived container ──────────────
+      # Replaces the container: directive so we can free disk space first.
+      # Uses "docker run -d ... sleep infinity" + "docker exec" to preserve
+      # installed packages and env vars across steps.
+      - name: Pull NVHPC container
+        if:   matrix.nvhpc
+        run:  docker pull "$NVHPC_IMAGE"
+
+      - name: Start NVHPC container
+        if:   matrix.nvhpc
+        run: |
+          docker run -d --name nvhpc \
+            --security-opt seccomp=unconfined \
+            -v "${{ github.workspace }}:/workspace" \
+            -w /workspace \
+            -e CC=nvc \
+            -e CXX=nvc++ \
+            -e FC=nvfortran \
+            -e OMPI_ALLOW_RUN_AS_ROOT=1 \
+            -e OMPI_ALLOW_RUN_AS_ROOT_CONFIRM=1 \
+            -e PMIX_MCA_gds=hash \
+            -e OMPI_MCA_hwloc_base_binding_policy=none \
+            -e "FFLAGS=-tp=px -Kieee -noswitcherror" \
+            -e CFLAGS=-tp=px \
+            -e CXXFLAGS=-tp=px \
+            "$NVHPC_IMAGE" sleep infinity
+
+      - name: Setup NVHPC
+        if:   matrix.nvhpc
+        run: |
+          docker exec nvhpc bash -c '
+            set -e
+            apt-get update -y
+            apt-get install -y cmake python3 python3-venv python3-pip \
+                               libfftw3-dev libhdf5-dev hdf5-tools git
+
+            # Set up NVHPC HPC-X MPI runtime paths
+            HPCX_DIR=$(dirname "$(find /opt/nvidia/hpc_sdk -path "*/hpcx/hpcx-*/ompi/bin/mpirun" | head -1)")/../..
+            MPI_LIB=$(mpifort --showme:link | grep -oP "(?<=-L)\S+" | head -1)
+
+            # Persist env vars for subsequent docker exec calls
+            cat > /etc/nvhpc-env.sh <<EOF
+          export LD_LIBRARY_PATH=${MPI_LIB}:${HPCX_DIR}/ucx/lib:${HPCX_DIR}/ucc/lib:\$LD_LIBRARY_PATH
+          export OMPI_MCA_rmaps_base_oversubscribe=1
+          EOF
+
+            # Debug: confirm compiler flags are set
+            echo "=== NVHPC Environment ==="
+            echo "FFLAGS=$FFLAGS"
+            echo "CFLAGS=$CFLAGS"
+            echo "CXXFLAGS=$CXXFLAGS"
+            nvfortran --version
+            cat /proc/cpuinfo | grep "model name" | head -1
+          '
+
+      # ── Standard (non-NVHPC) setup ─────────────────────────────────────
       - name: Setup MacOS
         if:   matrix.os == 'macos' && !matrix.nvhpc
         run:  |
@@ -315,30 +369,7 @@ jobs:
           echo "MPICC=mpiicx" >> $GITHUB_ENV
           echo "MPICXX=mpiicpx" >> $GITHUB_ENV
 
-      # --- NVHPC container setup ---
-      - name: Setup NVHPC
-        if:   matrix.nvhpc
-        run: |
-          apt-get update -y
-          apt-get install -y cmake python3 python3-venv python3-pip \
-                             libfftw3-dev libhdf5-dev hdf5-tools git
-          # Set up NVHPC HPC-X MPI runtime paths
-          HPCX_DIR=$(dirname "$(find /opt/nvidia/hpc_sdk -path "*/hpcx/hpcx-*/ompi/bin/mpirun" | head -1)")/../..
-          MPI_LIB=$(mpifort --showme:link | grep -oP '(?<=-L)\S+' | head -1)
-          echo "LD_LIBRARY_PATH=${MPI_LIB}:${HPCX_DIR}/ucx/lib:${HPCX_DIR}/ucc/lib:${LD_LIBRARY_PATH}" >> $GITHUB_ENV
-          # Container MPI fixes: PMIx shared-memory, hwloc binding
-          echo "PMIX_MCA_gds=hash" >> $GITHUB_ENV
-          echo "OMPI_MCA_hwloc_base_binding_policy=none" >> $GITHUB_ENV
-          echo "OMPI_MCA_rmaps_base_oversubscribe=1" >> $GITHUB_ENV
-          # Debug: confirm compiler flags are set
-          echo "=== NVHPC Environment ==="
-          echo "FFLAGS=$FFLAGS"
-          echo "CFLAGS=$CFLAGS"
-          echo "CXXFLAGS=$CXXFLAGS"
-          nvfortran --version
-          cat /proc/cpuinfo | grep "model name" | head -1
-
-      # --- Standard build + test ---
+      # ── Standard build + test ───────────────────────────────────────────
       - name: Build
         if:   '!matrix.nvhpc'
         run:  |
@@ -356,22 +387,37 @@ jobs:
           TEST_PCT: ${{ matrix.debug == 'reldebug' && '-% 20' || '' }}
           ONLY_CHANGES: ${{ github.event_name == 'pull_request' && '--only-changes' || '' }}
 
-      # --- NVHPC build + test ---
+      # ── NVHPC build + test (via docker exec into long-lived container) ──
       - name: Build (NVHPC)
         if:   matrix.nvhpc && matrix.target == 'cpu'
-        run:  /bin/bash mfc.sh test -v --dry-run -j $(nproc) --test-all
+        run: |
+          docker exec nvhpc bash -c '
+            source /etc/nvhpc-env.sh
+            /bin/bash mfc.sh test -v --dry-run -j $(nproc) --test-all
+          '
 
       - name: Build (NVHPC GPU)
         if:   matrix.nvhpc && matrix.target == 'gpu'
-        run:  |
-          /bin/bash mfc.sh test -v --dry-run -j 2 --test-all --gpu acc
-          /bin/bash mfc.sh test -v --dry-run -j 2 --test-all --gpu mp
+        run: |
+          docker exec nvhpc bash -c '
+            source /etc/nvhpc-env.sh
+            /bin/bash mfc.sh test -v --dry-run -j 2 --test-all --gpu acc
+            /bin/bash mfc.sh test -v --dry-run -j 2 --test-all --gpu mp
+          '
 
       - name: Test (NVHPC)
         if:   matrix.nvhpc && matrix.target == 'cpu'
-        run:  |
-          ulimit -s unlimited || ulimit -s 65536 || true
-          /bin/bash mfc.sh test -v --max-attempts 3 -j $(nproc) --test-all
+        run: |
+          docker exec nvhpc bash -c '
+            source /etc/nvhpc-env.sh
+            ulimit -s unlimited || ulimit -s 65536 || true
+            /bin/bash mfc.sh test -v --max-attempts 3 -j $(nproc) --test-all
+          '
+
+      # ── Cleanup ─────────────────────────────────────────────────────────
+      - name: Stop NVHPC container
+        if:   always() && matrix.nvhpc
+        run:  docker rm -f nvhpc || true
 
   self:
     name: "${{ matrix.cluster_name }} (${{ matrix.device }}${{ matrix.interface != 'none' && format('-{0}', matrix.interface) || '' }}${{ matrix.shard != '' && format(' [{0}]', matrix.shard) || '' }})"