File-Per-Process IB markers #8303
Workflow file for this run
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: 'Test Suite' | |
| on: | |
| push: | |
| branches: [master] | |
| pull_request: | |
| types: [opened, synchronize, reopened, ready_for_review] | |
| workflow_dispatch: | |
| concurrency: | |
| # PRs: group by branch (new push cancels old). Push to master: unique per SHA (never cancelled). | |
| group: ${{ github.workflow }}-${{ github.event_name == 'push' && github.sha || github.ref }} | |
| cancel-in-progress: ${{ github.event_name != 'push' }} | |
| # Least-privilege default: no job in this workflow writes to the repo. | |
| permissions: | |
| contents: read | |
| jobs: | |
| lint-gate: | |
| name: Lint Gate | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Clone | |
| uses: actions/checkout@v5 | |
| - name: Setup Python | |
| uses: actions/setup-python@v6 | |
| with: | |
| python-version: '3.12' | |
| - name: Initialize MFC | |
| run: ./mfc.sh init | |
| - name: Check Formatting | |
| run: | | |
| ./mfc.sh format -j "$(nproc)" | |
| git diff --exit-code || (echo "::error::Code is not formatted. Run './mfc.sh format' locally." && exit 1) | |
| - name: Spell Check | |
| run: ./mfc.sh spelling | |
| - name: Lint Toolchain | |
| run: ./mfc.sh lint | |
| - name: Lint Source | |
| run: python3 toolchain/mfc/lint_source.py | |
| - name: Lint Docs | |
| run: python3 toolchain/mfc/lint_docs.py | |
| - name: Lint Parameter Docs | |
| run: python3 toolchain/mfc/lint_param_docs.py | |
| file-changes: | |
| name: Detect File Changes | |
| # Job-level permissions replace the workflow default outright rather than | |
| # merging with it, so contents must be restated here. paths-filter reads the | |
| # PR's changed-file list via pulls.listFiles; this is the only job that needs | |
| # it, so it is granted here instead of workflow-wide. | |
| permissions: | |
| contents: read | |
| pull-requests: read | |
| runs-on: 'ubuntu-latest' | |
| outputs: | |
| checkall: ${{ steps.changes.outputs.checkall }} | |
| changed_files: ${{ steps.changes.outputs.checkall_files }} | |
| steps: | |
| - name: Clone | |
| uses: actions/checkout@v5 | |
| - name: Detect Changes | |
| uses: dorny/paths-filter@v4 | |
| id: changes | |
| with: | |
| filters: ".github/file-filter.yml" | |
| list-files: shell | |
| github: | |
| name: ${{ matrix.nvhpc && format('NVHPC {0} ({1})', matrix.nvhpc, matrix.target) || format('Github ({0}, {1}, {2}, {3})', matrix.os, matrix.mpi, matrix.debug, matrix.intel && 'intel' || 'GNU') }} | |
| needs: [lint-gate, file-changes] | |
| if: >- | |
| !cancelled() && | |
| needs.lint-gate.result == 'success' && | |
| needs.file-changes.result == 'success' && | |
| needs.file-changes.outputs.checkall == 'true' | |
| strategy: | |
| matrix: | |
| os: ['ubuntu', 'macos'] | |
| mpi: ['mpi'] | |
| precision: [''] | |
| debug: ['reldebug', 'no-debug'] | |
| intel: [true, false] | |
| nvhpc: [''] | |
| target: [''] | |
| exclude: | |
| - os: macos | |
| intel: true | |
| include: | |
| - os: ubuntu | |
| mpi: mpi | |
| precision: single | |
| debug: no-debug | |
| intel: false | |
| # NVHPC compiler matrix: cpu (build+test), gpu (build-only, acc then omp) | |
| # Every release from 23.11 through 26.3 (current) | |
| - { nvhpc: '23.11', target: cpu } | |
| - { nvhpc: '23.11', target: gpu } | |
| - { nvhpc: '24.1', target: cpu } | |
| - { nvhpc: '24.1', target: gpu } | |
| - { nvhpc: '24.3', target: cpu } | |
| - { nvhpc: '24.3', target: gpu } | |
| - { nvhpc: '24.5', target: cpu } | |
| - { nvhpc: '24.5', target: gpu } | |
| - { nvhpc: '24.7', target: cpu } | |
| - { nvhpc: '24.7', target: gpu } | |
| - { nvhpc: '24.9', target: cpu } | |
| - { nvhpc: '24.9', target: gpu } | |
| - { nvhpc: '24.11', target: cpu } | |
| - { nvhpc: '24.11', target: gpu } | |
| - { nvhpc: '25.1', target: cpu } | |
| - { nvhpc: '25.1', target: gpu } | |
| - { nvhpc: '25.3', target: cpu } | |
| - { nvhpc: '25.3', target: gpu } | |
| - { nvhpc: '25.5', target: cpu } | |
| - { nvhpc: '25.5', target: gpu } | |
| - { nvhpc: '25.7', target: cpu } | |
| - { nvhpc: '25.7', target: gpu } | |
| - { nvhpc: '25.9', target: cpu } | |
| - { nvhpc: '25.9', target: gpu } | |
| - { nvhpc: '25.11', target: cpu } | |
| - { nvhpc: '25.11', target: gpu } | |
| - { nvhpc: '26.1', target: cpu } | |
| - { nvhpc: '26.1', target: gpu } | |
| - { nvhpc: '26.3', target: cpu } | |
| - { nvhpc: '26.3', target: gpu } | |
| fail-fast: false | |
| continue-on-error: true | |
| runs-on: ${{ matrix.nvhpc && 'ubuntu-22.04' || format('{0}-latest', matrix.os) }} | |
| env: | |
| # Image tag for NVHPC jobs; empty for non-NVHPC jobs. | |
| NVHPC_IMAGE: ${{ matrix.nvhpc && format('nvcr.io/nvidia/nvhpc:{0}-devel-cuda_multi-ubuntu22.04', matrix.nvhpc) || '' }} | |
| steps: | |
| # ── NVHPC: free disk before pulling the ~25-30 GB cuda_multi image ── | |
| - name: Free disk space | |
| if: matrix.nvhpc | |
| run: | | |
| echo "=== Disk before cleanup ===" | |
| df -h / | |
| sudo rm -rf /usr/share/dotnet /usr/local/lib/android \ | |
| /opt/ghc /usr/local/share/boost /opt/hostedtoolcache \ | |
| /usr/local/graalvm /usr/local/.ghcup \ | |
| /usr/local/share/chromium /usr/local/lib/node_modules | |
| sudo docker image prune -af | |
| sudo apt-get clean | |
| echo "=== Disk after cleanup ===" | |
| df -h / | |
| - name: Clone | |
| uses: actions/checkout@v5 | |
| # ── NVHPC: pull image and start a long-lived container ────────────── | |
| # Replaces the container: directive so we can free disk space first. | |
| # Uses "docker run -d ... sleep infinity" + "docker exec" to preserve | |
| # installed packages and env vars across steps. | |
| # Retry the pull: nvcr.io intermittently times out ("context deadline | |
| # exceeded") under load, and ~30 matrix jobs hit it at once. Pulls | |
| # resume completed layers, so retries are cheap. | |
| - name: Pull NVHPC container | |
| if: matrix.nvhpc | |
| run: | | |
| for attempt in 1 2 3 4 5; do | |
| docker pull "$NVHPC_IMAGE" && exit 0 | |
| echo "docker pull failed (attempt $attempt/5); retrying in $((attempt * 30))s..." | |
| sleep $((attempt * 30)) | |
| done | |
| echo "::error::Failed to pull $NVHPC_IMAGE after 5 attempts" | |
| exit 1 | |
| - name: Start NVHPC container | |
| if: matrix.nvhpc | |
| run: | | |
| docker run -d --name nvhpc \ | |
| --security-opt seccomp=unconfined \ | |
| -v "${{ github.workspace }}:/workspace" \ | |
| -w /workspace \ | |
| -e CC=nvc \ | |
| -e CXX=nvc++ \ | |
| -e FC=nvfortran \ | |
| -e OMPI_ALLOW_RUN_AS_ROOT=1 \ | |
| -e OMPI_ALLOW_RUN_AS_ROOT_CONFIRM=1 \ | |
| -e PMIX_MCA_gds=hash \ | |
| -e OMPI_MCA_hwloc_base_binding_policy=none \ | |
| -e "FFLAGS=-tp=px -Kieee -noswitcherror" \ | |
| -e CFLAGS=-tp=px \ | |
| -e CXXFLAGS=-tp=px \ | |
| -e "MFC_NVHPC_TEST_FLAGS=--test-all --no-mpi" \ | |
| "$NVHPC_IMAGE" sleep infinity | |
| - name: Setup NVHPC | |
| if: matrix.nvhpc | |
| run: | | |
| docker exec nvhpc bash -c ' | |
| set -e | |
| apt-get update -y | |
| apt-get install -y cmake python3 python3-venv python3-pip \ | |
| libfftw3-dev libhdf5-dev hdf5-tools git | |
| # The repo is bind-mounted from the host so git sees a different | |
| # owner. Mark it safe to suppress "dubious ownership" errors that | |
| # otherwise spam 80 000+ lines into the CI log. | |
| git config --global --add safe.directory /workspace | |
| # Set up NVHPC HPC-X MPI runtime paths | |
| HPCX_DIR=$(dirname "$(find /opt/nvidia/hpc_sdk -path "*/hpcx/hpcx-*/ompi/bin/mpirun" | head -1)")/../.. | |
| MPI_LIB=$(mpifort --showme:link | grep -oP "(?<=-L)\S+" | head -1) | |
| # Persist env vars for subsequent docker exec calls | |
| cat > /etc/nvhpc-env.sh <<EOF | |
| export LD_LIBRARY_PATH=${MPI_LIB}:${HPCX_DIR}/ucx/lib:${HPCX_DIR}/ucc/lib:\$LD_LIBRARY_PATH | |
| export OMPI_MCA_rmaps_base_oversubscribe=1 | |
| EOF | |
| # Debug: confirm compiler flags are set | |
| echo "=== NVHPC Environment ===" | |
| echo "FFLAGS=$FFLAGS" | |
| echo "CFLAGS=$CFLAGS" | |
| echo "CXXFLAGS=$CXXFLAGS" | |
| nvfortran --version | |
| cat /proc/cpuinfo | grep "model name" | head -1 | |
| ' | |
| # ── Standard (non-NVHPC) setup ───────────────────────────────────── | |
| - name: Setup MacOS | |
| if: matrix.os == 'macos' && !matrix.nvhpc | |
| run: | | |
| brew update | |
| brew upgrade || true | |
| brew install coreutils python fftw hdf5 gcc@15 boost open-mpi lapack | |
| echo "FC=gfortran-15" >> $GITHUB_ENV | |
| echo "BOOST_INCLUDE=/opt/homebrew/include/" >> $GITHUB_ENV | |
| - name: Setup Ubuntu | |
| if: matrix.os == 'ubuntu' && matrix.intel == false && !matrix.nvhpc | |
| run: | | |
| sudo apt update -y | |
| sudo apt install -y cmake gcc g++ python3 python3-dev hdf5-tools \ | |
| libfftw3-dev libhdf5-dev openmpi-bin libopenmpi-dev \ | |
| libblas-dev liblapack-dev | |
| - name: Setup Ubuntu (Intel) | |
| if: matrix.os == 'ubuntu' && matrix.intel == true | |
| run: | | |
| wget https://apt.repos.intel.com/intel-gpg-keys/GPG-PUB-KEY-INTEL-SW-PRODUCTS.PUB | |
| sudo apt-key add GPG-PUB-KEY-INTEL-SW-PRODUCTS.PUB | |
| sudo add-apt-repository "deb https://apt.repos.intel.com/oneapi all main" | |
| sudo apt-get update | |
| sudo apt-get install -y intel-oneapi-compiler-fortran intel-oneapi-compiler-dpcpp-cpp intel-oneapi-mpi intel-oneapi-mpi-devel | |
| # Export only new/changed env vars from setvars.sh. | |
| # `printenv >> $GITHUB_ENV` dumps all vars including shell internals | |
| # with special characters that corrupt GITHUB_ENV parsing. | |
| printenv | sort > /tmp/env_before | |
| source /opt/intel/oneapi/setvars.sh | |
| printenv | sort > /tmp/env_after | |
| diff /tmp/env_before /tmp/env_after | grep '^>' | sed 's/^> //' >> $GITHUB_ENV | |
| echo "FC=ifx" >> $GITHUB_ENV | |
| echo "CC=icx" >> $GITHUB_ENV | |
| echo "CXX=icpx" >> $GITHUB_ENV | |
| echo "MPIFC=mpiifx" >> $GITHUB_ENV | |
| echo "MPICC=mpiicx" >> $GITHUB_ENV | |
| echo "MPICXX=mpiicpx" >> $GITHUB_ENV | |
| # ── Standard build + test ─────────────────────────────────────────── | |
| - name: Build | |
| if: '!matrix.nvhpc' | |
| run: | | |
| /bin/bash mfc.sh test -v --dry-run -j $(nproc) --${{ matrix.debug }} --${{ matrix.mpi }} $PRECISION $TEST_ALL | |
| env: | |
| TEST_ALL: ${{ matrix.mpi == 'mpi' && '--test-all' || '' }} | |
| PRECISION: ${{ matrix.precision != '' && format('--{0}', matrix.precision) || '' }} | |
| - name: Test | |
| if: '!matrix.nvhpc' | |
| run: | | |
| # Coverage-based test selection ENFORCED on PRs: runs only the tests whose | |
| # recorded coverage overlaps the PR's changed files (conservative ladder in | |
| # coverage.py — non-.fpp changes and .fpp files no test covers fall back to | |
| # run-all). Pushes to master run the full suite (SELECT empty) as a backstop, | |
| # and the NVHPC jobs below still run --test-all as a pre-merge full check. | |
| SELECT=() | |
| [ "${{ github.event_name }}" = "pull_request" ] && SELECT=(--select-enforce --changed-files "$CHANGED_FILES") | |
| /bin/bash mfc.sh test -v --max-attempts 3 -j $(nproc) "${SELECT[@]}" $TEST_ALL $TEST_PCT $PRECISION | |
| env: | |
| TEST_ALL: ${{ matrix.mpi == 'mpi' && '--test-all' || '' }} | |
| TEST_PCT: ${{ matrix.debug == 'reldebug' && '-% 20' || '' }} | |
| PRECISION: ${{ matrix.precision != '' && format('--{0}', matrix.precision) || '' }} | |
| CHANGED_FILES: ${{ needs.file-changes.outputs.changed_files }} | |
| # ── NVHPC build + test (via docker exec into long-lived container) ── | |
| # MFC_NVHPC_TEST_FLAGS carries --no-mpi: post_process segfaults under | |
| # mpirun in these containers, and every failure the lanes have produced | |
| # has been an MPI test (see PR #1822). The harness skips ppn>1 cases | |
| # without MPI, so the rest of --test-all -- the compile coverage these | |
| # lanes exist for -- is unaffected, as is MPI coverage elsewhere. Set in | |
| # one place so the build and test steps cannot disagree: a no-MPI build | |
| # tested with MPI would run ppn>1 cases against a binary that has none. | |
| - name: Build (NVHPC) | |
| if: matrix.nvhpc && matrix.target == 'cpu' | |
| run: | | |
| docker exec nvhpc bash -c ' | |
| source /etc/nvhpc-env.sh | |
| /bin/bash mfc.sh test -v --dry-run -j $(nproc) $MFC_NVHPC_TEST_FLAGS | |
| ' | |
| - name: Build (NVHPC GPU) | |
| if: matrix.nvhpc && matrix.target == 'gpu' | |
| run: | | |
| docker exec nvhpc bash -c ' | |
| source /etc/nvhpc-env.sh | |
| /bin/bash mfc.sh test -v --dry-run -j 2 --test-all --gpu acc | |
| /bin/bash mfc.sh test -v --dry-run -j 2 --test-all --gpu mp | |
| ' | |
| # Case-optimized build guard. | |
| # | |
| # --case-optimization hard-codes case parameters into the generated | |
| # sources, which sharply increases cross-file inlining pressure in the | |
| # -Mextract/-Minline IPO pass (cmake/MFCTargets.cmake). That pass is | |
| # where NVHPC fort2 has historically hit internal compiler errors, and | |
| # the jobs above cannot see them: they never pass --case-optimization, | |
| # and the self-hosted case-opt jobs run a single pinned NVHPC. NVHPC | |
| # 25.5 shipped an ICE that was invisible to CI for exactly that reason. | |
| # | |
| # One 3D case is enough: the failure is at compile time, so nothing is | |
| # run. 3D specifically, because m_sim_helpers.fpp guards its 3D block | |
| # with "#:if not MFC_CASE_OPTIMIZATION or num_dims > 2" -- a | |
| # case-optimized 2D build elides that code entirely. OpenACC only, | |
| # since the two-pass IPO is disabled for OpenMP offload. | |
| - name: Build (NVHPC GPU, case-optimized) | |
| if: matrix.nvhpc && matrix.target == 'gpu' | |
| run: | | |
| docker exec nvhpc bash -c ' | |
| source /etc/nvhpc-env.sh | |
| /bin/bash mfc.sh build -v -j 2 --gpu acc -t simulation \ | |
| -i examples/3D_sphbubcollapse/case.py --case-optimization | |
| ' | |
| - name: Test (NVHPC) | |
| if: matrix.nvhpc && matrix.target == 'cpu' | |
| run: | | |
| docker exec nvhpc bash -c ' | |
| source /etc/nvhpc-env.sh | |
| ulimit -s unlimited || ulimit -s 65536 || true | |
| /bin/bash mfc.sh test -v --max-attempts 3 -j $(nproc) $MFC_NVHPC_TEST_FLAGS | |
| ' | |
| # ── Cleanup ───────────────────────────────────────────────────────── | |
| - name: Stop NVHPC container | |
| if: always() && matrix.nvhpc | |
| run: docker rm -f nvhpc || true | |
| self: | |
| name: "${{ matrix.cluster_name }} (${{ matrix.device }}${{ matrix.interface != 'none' && format('-{0}', matrix.interface) || '' }}${{ matrix.shard != '' && format(' [{0}]', matrix.shard) || '' }})" | |
| needs: [lint-gate, file-changes] | |
| if: >- | |
| !cancelled() && | |
| needs.lint-gate.result == 'success' && | |
| needs.file-changes.result == 'success' && | |
| github.repository == 'MFlowCode/MFC' && | |
| needs.file-changes.outputs.checkall == 'true' && | |
| github.event.pull_request.draft != true | |
| # Frontier CCE compiler is periodically broken by toolchain updates (e.g. | |
| # cpe/25.03 introduced an IPA SIGSEGV in CCE 19.0.0). Allow Frontier to | |
| # fail without blocking PR merges; Phoenix remains a hard gate. | |
| continue-on-error: ${{ matrix.runner == 'frontier' }} | |
| timeout-minutes: 480 | |
| strategy: | |
| matrix: | |
| include: | |
| # Phoenix (GT) — build+test combined in SLURM job | |
| - runner: 'gt' | |
| cluster: 'phoenix' | |
| cluster_name: 'Georgia Tech | Phoenix (NVHPC)' | |
| device: 'gpu' | |
| interface: 'acc' | |
| - runner: 'gt' | |
| cluster: 'phoenix' | |
| cluster_name: 'Georgia Tech | Phoenix (NVHPC)' | |
| device: 'gpu' | |
| interface: 'omp' | |
| - runner: 'gt' | |
| cluster: 'phoenix' | |
| cluster_name: 'Georgia Tech | Phoenix (GNU)' | |
| device: 'cpu' | |
| interface: 'none' | |
| # Frontier (ORNL) — CCE | |
| - runner: 'frontier' | |
| cluster: 'frontier' | |
| cluster_name: 'Oak Ridge | Frontier (CCE)' | |
| device: 'gpu' | |
| interface: 'acc' | |
| shard: '1/2' | |
| - runner: 'frontier' | |
| cluster: 'frontier' | |
| cluster_name: 'Oak Ridge | Frontier (CCE)' | |
| device: 'gpu' | |
| interface: 'acc' | |
| shard: '2/2' | |
| - runner: 'frontier' | |
| cluster: 'frontier' | |
| cluster_name: 'Oak Ridge | Frontier (CCE)' | |
| device: 'gpu' | |
| interface: 'omp' | |
| shard: '1/2' | |
| - runner: 'frontier' | |
| cluster: 'frontier' | |
| cluster_name: 'Oak Ridge | Frontier (CCE)' | |
| device: 'gpu' | |
| interface: 'omp' | |
| shard: '2/2' | |
| - runner: 'frontier' | |
| cluster: 'frontier' | |
| cluster_name: 'Oak Ridge | Frontier (CCE)' | |
| device: 'cpu' | |
| interface: 'none' | |
| # Frontier AMD — build on login node, GPU tests sharded for batch partition | |
| - runner: 'frontier' | |
| cluster: 'frontier_amd' | |
| cluster_name: 'Oak Ridge | Frontier (AMD)' | |
| device: 'gpu' | |
| interface: 'omp' | |
| shard: '1/2' | |
| - runner: 'frontier' | |
| cluster: 'frontier_amd' | |
| cluster_name: 'Oak Ridge | Frontier (AMD)' | |
| device: 'gpu' | |
| interface: 'omp' | |
| shard: '2/2' | |
| - runner: 'frontier' | |
| cluster: 'frontier_amd' | |
| cluster_name: 'Oak Ridge | Frontier (AMD)' | |
| device: 'cpu' | |
| interface: 'none' | |
| shard: '1/2' | |
| - runner: 'frontier' | |
| cluster: 'frontier_amd' | |
| cluster_name: 'Oak Ridge | Frontier (AMD)' | |
| device: 'cpu' | |
| interface: 'none' | |
| shard: '2/2' | |
| runs-on: | |
| group: phoenix | |
| labels: ${{ matrix.runner }} | |
| env: | |
| NODE_OPTIONS: ${{ matrix.cluster == 'phoenix' && '--max-old-space-size=2048' || '' }} | |
| steps: | |
| - name: Clone | |
| uses: actions/checkout@v5 | |
| with: | |
| # clean: false preserves .slurm_job_id files across reruns so | |
| # submit-slurm-job.sh can detect and cancel stale SLURM jobs on retry. | |
| clean: false | |
| - name: Clean stale output files | |
| run: rm -f *.out | |
| - name: Fetch Dependencies | |
| if: matrix.cluster != 'phoenix' | |
| timeout-minutes: 120 | |
| run: bash .github/workflows/${{ matrix.cluster }}/build.sh ${{ matrix.device }} ${{ matrix.interface }} | |
| - name: Build | |
| # Phoenix builds+tests in one allocation (see "Build & Test"); others build separately. | |
| if: ${{ matrix.cluster != 'phoenix' && !(matrix.cluster == 'frontier_amd' && matrix.device == 'gpu') }} | |
| run: bash .github/scripts/submit-slurm-job.sh .github/workflows/common/build.sh ${{ matrix.device }} ${{ matrix.interface }} ${{ matrix.cluster }} ${{ matrix.shard }} | |
| - name: Build (concurrent variants) | |
| if: matrix.cluster == 'frontier_amd' && matrix.device == 'gpu' | |
| # The suite needs a base and a chemistry simulation binary, and on amdflang | |
| # each device LTO link is ~1 h -- building both serially overruns the 2 h | |
| # walltime. They touch disjoint staging directories, so submit one SLURM job | |
| # per variant and wait on both (mirrors the case-optimization pre-build). | |
| run: | | |
| pids="" | |
| for v in base chem; do | |
| bash .github/scripts/submit-slurm-job.sh .github/workflows/common/build.sh ${{ matrix.device }} ${{ matrix.interface }} ${{ matrix.cluster }} "${{ matrix.shard }}" "$v" & | |
| pids="$pids $!" | |
| done | |
| rc=0 | |
| for p in $pids; do wait "$p" || rc=1; done | |
| exit $rc | |
| - name: Build & Test | |
| # Phoenix: build+test in one allocation so the 'embers' queue wait is paid once. | |
| if: matrix.cluster == 'phoenix' | |
| run: bash .github/scripts/submit-slurm-job.sh .github/workflows/common/build-and-test.sh ${{ matrix.device }} ${{ matrix.interface }} ${{ matrix.cluster }} ${{ matrix.shard }} | |
| - name: Test | |
| if: matrix.cluster != 'phoenix' | |
| run: bash .github/scripts/submit-slurm-job.sh .github/workflows/common/test.sh ${{ matrix.device }} ${{ matrix.interface }} ${{ matrix.cluster }} ${{ matrix.shard }} | |
| - name: Cancel SLURM Jobs | |
| if: cancelled() | |
| run: | | |
| find . -name "*.slurm_job_id" | while read -r f; do | |
| job_id=$(cat "$f") | |
| echo "Cancelling SLURM job $job_id" | |
| scancel "$job_id" 2>/dev/null || true | |
| done | |
| - name: Compute Log Slug | |
| if: always() | |
| id: log | |
| run: | | |
| SHARD_SUFFIX="" | |
| SHARD="${{ matrix.shard }}" | |
| if [ -n "$SHARD" ]; then | |
| SHARD_SUFFIX="-$(echo "$SHARD" | sed 's|/|-of-|')" | |
| fi | |
| echo "build_slug=build-${{ matrix.device }}-${{ matrix.interface }}${SHARD_SUFFIX}" >> "$GITHUB_OUTPUT" | |
| echo "test_slug=test-${{ matrix.device }}-${{ matrix.interface }}${SHARD_SUFFIX}" >> "$GITHUB_OUTPUT" | |
| echo "combined_slug=build-and-test-${{ matrix.device }}-${{ matrix.interface }}${SHARD_SUFFIX}" >> "$GITHUB_OUTPUT" | |
| - name: Print Logs | |
| if: always() | |
| run: | | |
| # build_slug matches the plain log (and per-variant base/chem builds); | |
| # combined_slug matches Phoenix's single build-and-test allocation. | |
| # Use `if` (not `[ -f ] && ...`) so a missing file — e.g. test_slug on | |
| # Phoenix's combined jobs — doesn't make the loop exit non-zero under -e. | |
| for f in ${{ steps.log.outputs.build_slug }}*.out ${{ steps.log.outputs.combined_slug }}.out ${{ steps.log.outputs.test_slug }}.out; do | |
| if [ -f "$f" ]; then echo "=== $f ==="; cat "$f"; fi | |
| done | |
| - name: Archive Logs | |
| uses: actions/upload-artifact@v6 | |
| if: matrix.cluster != 'phoenix' | |
| with: | |
| name: logs-${{ strategy.job-index }}-${{ steps.log.outputs.test_slug }} | |
| # build-*.log is the teed compiler output. Some CCE and amdflang | |
| # failures emit no diagnostic into the step log at all, so this file is | |
| # the only evidence left of why the build died. | |
| path: | | |
| ${{ steps.log.outputs.build_slug }}*.out | |
| ${{ steps.log.outputs.test_slug }}.out | |
| build-*.log | |
| case-optimization: | |
| name: "Case Opt | ${{ matrix.cluster_name }} (${{ matrix.device }}-${{ matrix.interface }})" | |
| if: github.repository == 'MFlowCode/MFC' && needs.file-changes.outputs.checkall == 'true' && github.event.pull_request.draft != true | |
| needs: [lint-gate, file-changes] | |
| # Frontier is non-blocking for the same reason as the self job above. | |
| continue-on-error: ${{ matrix.runner == 'frontier' }} | |
| timeout-minutes: 480 | |
| strategy: | |
| matrix: | |
| include: | |
| - runner: 'gt' | |
| cluster: 'phoenix' | |
| cluster_name: 'Georgia Tech | Phoenix (NVHPC)' | |
| device: 'gpu' | |
| interface: 'acc' | |
| - runner: 'gt' | |
| cluster: 'phoenix' | |
| cluster_name: 'Georgia Tech | Phoenix (NVHPC)' | |
| device: 'gpu' | |
| interface: 'omp' | |
| - runner: 'frontier' | |
| cluster: 'frontier' | |
| cluster_name: 'Oak Ridge | Frontier (CCE)' | |
| device: 'gpu' | |
| interface: 'acc' | |
| - runner: 'frontier' | |
| cluster: 'frontier' | |
| cluster_name: 'Oak Ridge | Frontier (CCE)' | |
| device: 'gpu' | |
| interface: 'omp' | |
| - runner: 'frontier' | |
| cluster: 'frontier_amd' | |
| cluster_name: 'Oak Ridge | Frontier (AMD)' | |
| device: 'gpu' | |
| interface: 'omp' | |
| runs-on: | |
| group: phoenix | |
| labels: ${{ matrix.runner }} | |
| steps: | |
| - name: Clone | |
| uses: actions/checkout@v5 | |
| with: | |
| clean: false | |
| - name: Clean stale output files | |
| run: rm -f *.out | |
| - name: Fetch Dependencies | |
| if: matrix.cluster != 'phoenix' | |
| run: bash .github/workflows/${{ matrix.cluster }}/build.sh ${{ matrix.device }} ${{ matrix.interface }} | |
| - name: Pre-Build (SLURM) | |
| if: matrix.cluster == 'phoenix' | |
| run: bash .github/scripts/submit-slurm-job.sh .github/scripts/prebuild-case-optimization.sh cpu ${{ matrix.interface }} ${{ matrix.cluster }} | |
| - name: Pre-Build (SLURM) | |
| if: matrix.cluster == 'frontier_amd' | |
| # AMD flang case-opt builds are ~30 min/case, so one serial pre-build | |
| # job exceeds its walltime. Split the case list across three concurrent | |
| # SLURM jobs (heavy shard ~2 cases). The shards share this workspace and | |
| # skip their in-job staging clean, so clean once here first. | |
| run: | | |
| find build/staging -maxdepth 1 -regex '.*/\(gpu-acc\|gpu-mp\|cpu\)-.*' -type d -exec rm -rf {} + 2>/dev/null || true | |
| find build/install -maxdepth 1 -regex '.*/\(gpu-acc\|gpu-mp\|cpu\)-.*' -type d -exec rm -rf {} + 2>/dev/null || true | |
| rm -f build/.prebuild-shared-targets-done build/.prebuild-shared-targets-failed | |
| pids="" | |
| for s in 1/3 2/3 3/3; do | |
| bash .github/scripts/submit-slurm-job.sh .github/scripts/prebuild-case-optimization.sh gpu ${{ matrix.interface }} ${{ matrix.cluster }} "$s" & | |
| pids="$pids $!" | |
| done | |
| rc=0 | |
| for p in $pids; do wait "$p" || rc=1; done | |
| exit $rc | |
| - name: Build & Run Case-Optimization Tests | |
| if: matrix.cluster != 'phoenix' && matrix.cluster != 'frontier_amd' | |
| run: bash .github/scripts/submit-slurm-job.sh .github/scripts/run_case_optimization.sh ${{ matrix.device }} ${{ matrix.interface }} ${{ matrix.cluster }} | |
| - name: Run Case-Optimization Tests | |
| if: matrix.cluster == 'phoenix' | |
| run: bash .github/scripts/submit-slurm-job.sh .github/scripts/run_case_optimization.sh ${{ matrix.device }} ${{ matrix.interface }} ${{ matrix.cluster }} | |
| - name: Run Case-Optimization Tests | |
| if: matrix.cluster == 'frontier_amd' | |
| # Mirror the pre-build sharding so each run job executes only the cases | |
| # it has binaries for, concurrently within the batch walltime. | |
| run: | | |
| pids="" | |
| for s in 1/3 2/3 3/3; do | |
| bash .github/scripts/submit-slurm-job.sh .github/scripts/run_case_optimization.sh ${{ matrix.device }} ${{ matrix.interface }} ${{ matrix.cluster }} "$s" & | |
| pids="$pids $!" | |
| done | |
| rc=0 | |
| for p in $pids; do wait "$p" || rc=1; done | |
| exit $rc | |
| - name: Cancel SLURM Jobs | |
| if: cancelled() | |
| run: | | |
| find . -name "*.slurm_job_id" | while read -r f; do | |
| job_id=$(cat "$f") | |
| echo "Cancelling SLURM job $job_id" | |
| scancel "$job_id" 2>/dev/null || true | |
| done | |
| - name: Print Logs | |
| if: always() | |
| run: | | |
| for f in prebuild-case-optimization-*.out run-case-optimization-*.out; do | |
| [ -f "$f" ] && echo "=== $f ===" && cat "$f" | |
| done | |
| - name: Archive Logs | |
| uses: actions/upload-artifact@v6 | |
| if: always() | |
| with: | |
| name: case-opt-${{ strategy.job-index }}-${{ matrix.cluster }}-${{ matrix.interface }} | |
| path: | | |
| prebuild-case-optimization-*.out | |
| run-case-optimization-*.out |