Skip to content

Commit 5ef74e3

Browse files
sbryngelsonsbryngelson
authored
Merge branch 'master' into nvhpc
2 parents 23a0bd9 + ace3632 commit 5ef74e3

104 files changed

Lines changed: 13993 additions & 20874 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎.ffmt.toml‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,18 @@
1+
# MFC Fortran formatting configuration
2+
# These are the defaults — this file makes them explicit.
3+
4+
indent-width = 4
5+
keyword-case = "lower"
6+
normalize-keywords = true
7+
indent-fypp = true
8+
9+
[whitespace]
10+
relational = true
11+
logical = true
12+
plusminus = true
13+
multdiv = false
14+
power = false
15+
assignment = true
16+
declaration = true
17+
comma = true
18+
slice-colon = false

‎.github/workflows/test.yml‎

Lines changed: 68 additions & 42 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,8 @@ on:
66
pull_request:
77
types: [opened, synchronize, reopened, ready_for_review]
88
workflow_dispatch:
9+
schedule:
10+
- cron: '0 6 * * 1' # Weekly Monday 6 AM UTC: refresh coverage cache before 7-day expiry
911

1012
concurrency:
1113
# PRs: group by branch (new push cancels old). Push to master: unique per SHA (never cancelled).
@@ -39,21 +41,15 @@ jobs:
3941
- name: Lint Toolchain
4042
run: ./mfc.sh lint
4143

42-
- name: Lint Source - No Raw Directives
43-
run: |
44-
! grep -iR '!\$acc\|!\$omp' --exclude="parallel_macros.fpp" --exclude="acc_macros.fpp" --exclude="omp_macros.fpp" --exclude="shared_parallel_macros.fpp" --exclude="syscheck.fpp" ./src/*
45-
46-
- name: Lint Source - No Double Precision Intrinsics
47-
run: |
48-
! grep -iR 'double_precision\|dsqrt\|dexp\|dlog\|dble\|dabs\|double\ precision\|real(8)\|real(4)\|dprod\|dmin\|dmax\|dfloat\|dreal\|dcos\|dsin\|dtan\|dsign\|dtanh\|dsinh\|dcosh\|d0' --exclude-dir=syscheck --exclude="*nvtx*" --exclude="*precision_select*" ./src/*
49-
50-
- name: Lint Source - No Junk Code
51-
run: |
52-
! grep -iR -e '\.\.\.' -e '\-\-\-' -e '===' ./src/*
44+
- name: Lint Source
45+
run: python3 toolchain/mfc/lint_source.py
5346

5447
- name: Lint Docs
5548
run: python3 toolchain/mfc/lint_docs.py
5649

50+
- name: Lint Parameter Docs
51+
run: python3 toolchain/mfc/lint_param_docs.py
52+
5753
file-changes:
5854
name: Detect File Changes
5955
runs-on: 'ubuntu-latest'
@@ -117,14 +113,13 @@ jobs:
117113
(github.event_name == 'push' &&
118114
(needs.file-changes.outputs.cases_py == 'true' ||
119115
needs.file-changes.outputs.dep_changed == 'true')) ||
120-
github.event_name == 'workflow_dispatch'
116+
github.event_name == 'workflow_dispatch' ||
117+
github.event_name == 'schedule'
121118
)
122119
timeout-minutes: 240
123120
runs-on:
124121
group: phoenix
125122
labels: gt
126-
permissions:
127-
contents: write # Required for Commit Cache to Master on push events
128123
steps:
129124
- name: Clone
130125
uses: actions/checkout@v4
@@ -140,36 +135,18 @@ jobs:
140135
run: cat rebuild-cache-cpu-none.out
141136

142137
- name: Upload Cache Artifact
143-
if: github.event_name == 'pull_request'
144138
uses: actions/upload-artifact@v4
145139
with:
146140
name: coverage-cache
147141
path: toolchain/mfc/test/test_coverage_cache.json.gz
148142
retention-days: 1
149143

150-
- name: Commit Cache to Master
151-
if: (github.event_name == 'push' || github.event_name == 'workflow_dispatch') && github.ref == 'refs/heads/master'
152-
env:
153-
CACHE_PUSH_TOKEN: ${{ secrets.CACHE_PUSH_TOKEN }}
154-
run: |
155-
git config user.name "github-actions[bot]"
156-
git config user.email "github-actions[bot]@users.noreply.github.com"
157-
git add toolchain/mfc/test/test_coverage_cache.json.gz
158-
if git diff --cached --quiet; then
159-
echo "Coverage cache unchanged."
160-
else
161-
git commit --no-verify -m "Regenerate gcov coverage cache [skip ci]"
162-
# Rebase onto latest master in case it advanced during the build
163-
# (which can take 20-240 minutes). Only test_coverage_cache.json.gz
164-
# is changed, so rebase conflicts are essentially impossible.
165-
git fetch origin master
166-
git rebase origin/master || { git rebase --abort; echo "Rebase conflict — cache will be regenerated on the next trigger."; exit 0; }
167-
# Push using admin token to bypass branch protection rulesets.
168-
# The default GITHUB_TOKEN lacks the Repository Admin role needed
169-
# to push directly to master. Use credential helper to avoid
170-
# leaking the token in git error messages.
171-
git -c "http.https://github.com/.extraheader=Authorization: basic $(echo -n "x-access-token:${CACHE_PUSH_TOKEN}" | base64)" push origin HEAD:refs/heads/master
172-
fi
144+
- name: Save Coverage Cache
145+
uses: actions/cache/save@v4
146+
with:
147+
path: toolchain/mfc/test/test_coverage_cache.json.gz
148+
key: coverage-cache-${{ github.event.pull_request.number || 'master' }}-${{ hashFiles('toolchain/mfc/test/cases.py') }}-${{ github.sha }}
149+
continue-on-error: true
173150

174151
github:
175152
name: ${{ matrix.nvhpc && format('NVHPC {0} ({1})', matrix.nvhpc, matrix.target) || format('Github ({0}, {1}, {2}, intel={3})', matrix.os, matrix.mpi, matrix.debug, matrix.intel) }}
@@ -253,19 +230,43 @@ jobs:
253230
git fetch --deepen=200
254231
continue-on-error: true
255232

256-
- name: Download Coverage Cache
233+
- name: Download Coverage Cache (from rebuild in this run)
234+
id: cache-artifact
257235
uses: actions/download-artifact@v4
258236
with:
259237
name: coverage-cache
260238
path: toolchain/mfc/test
261239
continue-on-error: true
262240

263-
# --- Standard (non-NVHPC) setup ---
241+
- name: Restore Coverage Cache (from previous run)
242+
if: steps.cache-artifact.outcome != 'success'
243+
id: cache-restore
244+
uses: actions/cache/restore@v4
245+
with:
246+
path: toolchain/mfc/test/test_coverage_cache.json.gz
247+
key: coverage-cache-${{ github.event.pull_request.number || 'master' }}-${{ hashFiles('toolchain/mfc/test/cases.py') }}-${{ github.sha }}
248+
restore-keys: |
249+
coverage-cache-${{ github.event.pull_request.number || 'master' }}-
250+
coverage-cache-master-
251+
continue-on-error: true
252+
253+
- name: Coverage Cache Status
254+
run: |
255+
if [ "${{ steps.cache-artifact.outcome }}" = "success" ]; then
256+
echo "Coverage cache: loaded from rebuild artifact (this run)"
257+
elif [ "${{ steps.cache-restore.outputs.cache-hit }}" = "true" ]; then
258+
echo "Coverage cache: restored from actions/cache (previous run)"
259+
elif [ -f toolchain/mfc/test/test_coverage_cache.json.gz ]; then
260+
echo "Coverage cache: using committed fallback in repo"
261+
else
262+
echo "Coverage cache: none available — full test suite will run"
263+
fi
264+
264265
- name: Setup MacOS
265266
if: matrix.os == 'macos' && !matrix.nvhpc
266267
run: |
267268
brew update
268-
brew upgrade
269+
brew upgrade || true
269270
brew install coreutils python fftw hdf5 gcc@15 boost open-mpi lapack
270271
echo "FC=gfortran-15" >> $GITHUB_ENV
271272
echo "BOOST_INCLUDE=/opt/homebrew/include/" >> $GITHUB_ENV
@@ -431,13 +432,38 @@ jobs:
431432
- name: Clean stale output files
432433
run: rm -f *.out
433434

434-
- name: Download Coverage Cache
435+
- name: Download Coverage Cache (from rebuild in this run)
436+
id: cache-artifact
435437
uses: actions/download-artifact@v4
436438
with:
437439
name: coverage-cache
438440
path: toolchain/mfc/test
439441
continue-on-error: true
440442

443+
- name: Restore Coverage Cache (from previous run)
444+
if: steps.cache-artifact.outcome != 'success'
445+
id: cache-restore
446+
uses: actions/cache/restore@v4
447+
with:
448+
path: toolchain/mfc/test/test_coverage_cache.json.gz
449+
key: coverage-cache-${{ github.event.pull_request.number || 'master' }}-${{ hashFiles('toolchain/mfc/test/cases.py') }}-${{ github.sha }}
450+
restore-keys: |
451+
coverage-cache-${{ github.event.pull_request.number || 'master' }}-
452+
coverage-cache-master-
453+
continue-on-error: true
454+
455+
- name: Coverage Cache Status
456+
run: |
457+
if [ "${{ steps.cache-artifact.outcome }}" = "success" ]; then
458+
echo "Coverage cache: loaded from rebuild artifact (this run)"
459+
elif [ "${{ steps.cache-restore.outputs.cache-hit }}" = "true" ]; then
460+
echo "Coverage cache: restored from actions/cache (previous run)"
461+
elif [ -f toolchain/mfc/test/test_coverage_cache.json.gz ]; then
462+
echo "Coverage cache: using committed fallback in repo"
463+
else
464+
echo "Coverage cache: none available — full test suite will run"
465+
fi
466+
441467
- name: Build (login node)
442468
if: matrix.cluster != 'phoenix'
443469
timeout-minutes: 60

‎.gitignore‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -113,3 +113,4 @@ benchmarks/*.png
113113
cce_*/
114114
cce_*.log
115115
run_cce_*.sh
116+
.ffmt_cache/

‎CLAUDE.md‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -99,7 +99,7 @@ IMPORTANT: Follow this loop for ALL code changes. Do not skip steps.
9999
2. **Plan** — For multi-file changes, outline your approach before implementing.
100100
3. **Implement** — Make small, focused changes. One logical change per commit.
101101
4. **Format** — Run `./mfc.sh format -j 8` to auto-format.
102-
5. **Verify** — Run `./mfc.sh precheck -j 8` (same 5 checks as CI lint gate).
102+
5. **Verify** — Run `./mfc.sh precheck -j 8` (same 6 checks as CI lint gate).
103103
6. **Build** — Run `./mfc.sh build -j 8` to verify compilation.
104104
7. **Test** — Run relevant tests: `./mfc.sh test --only <feature> -j 8`.
105105
For changes to `src/common/`, test ALL three targets: `./mfc.sh test -j 8`.

‎docs/documentation/case.md‎

Lines changed: 38 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -104,6 +104,11 @@ Definition of the parameters is described in the following subsections.
104104
| ---: | :----: | :--- |
105105
| `run_time_info` | Logical | Output run-time information |
106106
| `rdma_mpi` | Logical | (GPUs) Enable RDMA for MPI communication. |
107+
| `case_dir` | String | Case directory path |
108+
| `old_grid` | Logical | Use grid from previous simulation |
109+
| `old_ic` | Logical | Use initial conditions from previous simulation |
110+
| `t_step_old` | Integer | Time step to restart from |
111+
| `n_start_old` | Integer | Starting index from previous simulation |
107112

108113
- `run_time_info` generates a text file that includes run-time information including the CFL number(s) at each time-step.
109114
- `rdma_mpi` optimizes data transfers between GPUs using Remote Direct Memory Access (RDMA).
@@ -124,6 +129,8 @@ feature, detecting GPU pointers and performing RDMA accordingly.
124129
| `m` | Integer | Number of grid cells in the $x$-coordinate direction |
125130
| `n` | Integer | Number of grid cells in the $y$-coordinate direction |
126131
| `p` | Integer | Number of grid cells in the $z$-coordinate direction |
132+
| `pref` | Real | Reference pressure |
133+
| `rhoref` | Real | Reference density |
127134

128135
The parameters define the boundaries of the spatial and temporal domains, and their discretization that are used in simulation.
129136

@@ -304,6 +311,7 @@ This is enabled by adding ``'elliptic_smoothing': "T",`` and ``'elliptic_smoothi
304311

305312
| Parameter | Type | Description |
306313
| ---: | :----: | :--- |
314+
| `num_ibs` | Integer | Number of immersed boundary patches |
307315
| `geometry` | Integer | Geometry configuration of the patch.|
308316
| `x[y,z]_centroid` | Real | Centroid of the applied geometry in the [x,y,z]-direction. |
309317
| `length_x[y,z]` | Real | Length, if applicable, in the [x,y,z]-direction. |
@@ -341,7 +349,7 @@ Definitions for currently implemented immersed boundary patch types are listed i
341349

342350
- `c`, `t`, `p`, and `m` specify the parameters for a NACA airfoil.
343351
`m` is the maximum camber, `p` is the location of maximum camber, `c` is the coord length, and `t` is the thickness.
344-
Additional details on this specification can be found in [The Naca Airfoil Series](https://web.stanford.edu/~cantwell/AA200_Course_Material/The%20NACA%20airfoil%20series.pdf)
352+
Additional details on this specification can be found in [NACA airfoil](https://en.wikipedia.org/wiki/NACA_airfoil).
345353

346354
- `slip` applies a slip boundary to the surface of the patch if true and a no-slip boundary condition to the surface if false.
347355

@@ -425,6 +433,7 @@ See @ref equations "Equations" for the mathematical models these parameters cont
425433
| `mp_weno` | Logical | Monotonicity preserving WENO |
426434
| `muscl_order` | Integer | MUSCL order [1,2] |
427435
| `muscl_lim` | Integer | MUSCL Slope Limiter: [1] minmod; [2] monotonized central; [3] Van Albada; [4] Van Leer; [5] SUPERBEE |
436+
| `flux_lim` | Integer | Flux limiter for post-process: [1] minmod; [2] MUSCL; [3] OSPRE; [4] SUPERBEE |
428437
| `int_comp` | Logical | THINC Interface Compression |
429438
| `ic_eps` | Real | Interface compression threshold (default: 1e-4) |
430439
| `ic_beta` | Real | Interface compression sharpness parameter (default: 1.6) |
@@ -441,13 +450,15 @@ See @ref equations "Equations" for the mathematical models these parameters cont
441450
| `t_step_print` | Integer | Frequency to print the current step number to standard output (default 1) |
442451
| `cfl_adap_dt` | Logical | CFL based adaptive time-stepping |
443452
| `cfl_const_dt` | Logical | CFL based non-adaptive time-stepping |
453+
| `cfl_dt` | Logical | Enable CFL-based time stepping |
444454
| `cfl_target` | Real | Specified CFL value |
445455
| `n_start` | Integer | Save file from which to start simulation |
446456
| `t_save` | Real | Time duration between data output |
447457
| `t_stop` | Real | Simulation stop time |
448458
| `surface_tension` | Logical | Activate surface tension |
449459
| `viscous` | Logical | Activate viscosity |
450460
| `hypoelasticity` | Logical | Activate hypoelasticity* |
461+
| `pre_stress` | Logical | Enable pre-stress initialization for hypoelasticity |
451462
| `igr` | Logical | Enable solution via information geometric regularization (IGR) \cite Cao24 |
452463
| `igr_order` | Integer | Order of reconstruction for IGR [3,5] |
453464
| `alf_factor` | Real | Alpha factor for IGR entropic pressure (default 10) |
@@ -623,6 +634,13 @@ To restart the simulation from $k$-th time step, see @ref running "Restarting Ca
623634
| `schlieren_wrt` | Logical | Add the numerical schlieren to the database|
624635
| `qm_wrt` | Logical | Add the Q-criterion to the database|
625636
| `liutex_wrt` | Logical | Add the Liutex to the database|
637+
| `cf_wrt` | Logical | Write color function field |
638+
| `chem_wrt_T` | Logical | Write temperature field for chemistry output |
639+
| `fft_wrt` | Logical | Enable FFT output |
640+
| `sim_data` | Logical | Write interface and energy data files (post_process) |
641+
| `integral_wrt` | Logical | Write integral data |
642+
| `num_integrals` | Integer | Number of integral regions |
643+
| `down_sample` | Logical | Enable output downsampling |
626644
| `fd_order` | Integer | Order of finite differences for computing the vorticity and the numerical Schlieren function [1,2,4] |
627645
| `schlieren_alpha(i)` | Real | Intensity of the numerical Schlieren computed via `alpha(i)` |
628646
| `probe_wrt` | Logical | Write the flow chosen probes data files for each time step |
@@ -771,6 +789,12 @@ Details of the transducer acoustic source model can be found in \cite Maeda17.
771789
| `bubbles_euler` | Logical | Ensemble-averaged bubble modeling |
772790
| `bubbles_lagrange` | Logical | Volume-averaged bubble modeling |
773791
| `bubble_model` | Integer | [1] Gilmore; [2] Keller--Miksis; [3] Rayleigh-Plesset |
792+
| `Ca` | Real | Cavitation number |
793+
| `Web` | Real | Weber number |
794+
| `Re_inv` | Real | Inverse Reynolds number |
795+
| `pref` | Real | Reference pressure for bubble models |
796+
| `rhoref` | Real | Reference density for bubble models |
797+
| `fluid_rho` | Real | Reference fluid density |
774798
| `bub_pp%%R0ref`*†‡ | Real | Reference bubble radius |
775799
| `bub_pp%%p0ref`*†‡ | Real | Reference pressure |
776800
| `bub_pp%%rho0ref`*†‡| Real | Reference density |
@@ -893,6 +917,9 @@ When ``polytropic = 'F'``, the gas compression is modeled as non-polytropic due
893917
| `mixlayer_vel_profile` | Logical | Set the mean streamwise velocity to hyperbolic tangent profile |
894918
| `mixlayer_vel_coef` | Real | Coefficient for the hyperbolic tangent profile of a mixing layer |
895919
| `mixlayer_perturb` | Logical | Perturb the initial velocity field using a spectrum-based synthetic turbulence generation method |
920+
| `mixlayer_perturb_k0` | Real | Base wavenumber for mixing layer perturbation |
921+
| `mixlayer_perturb_nk` | Integer | Number of perturbation modes for mixing layer |
922+
| `simplex_perturb` | Logical | Enable simplex noise perturbation of initial conditions |
896923

897924
The table lists velocity field parameters.
898925
The parameters are optionally used to define initial velocity profiles and perturbations.
@@ -1030,6 +1057,16 @@ When ``cyl_coord = 'T'`` is set in 2D the following constraints must be met:
10301057
- `cantera_file` specifies the chemical mechanism file. If the file is part of the standard Cantera library, only the filename is required. Otherwise, the file must be located in the same directory as your `case.py` file
10311058

10321059

1060+
### 18. GPU Performance (NVIDIA UVM)
1061+
1062+
| Parameter | Type | Description |
1063+
| ---: | :---: | :--- |
1064+
| `nv_uvm_out_of_core` | Logical | Enable NVIDIA Unified Virtual Memory out-of-core execution |
1065+
| `nv_uvm_pref_gpu` | Logical | Prefer GPU placement for NVIDIA UVM allocations |
1066+
| `nv_uvm_igr_temps_on_gpu` | Integer | Store IGR temporaries on GPU with UVM |
1067+
1068+
- These parameters are for NVIDIA Grace-Hopper and similar architectures with hardware-managed unified memory. They allow MFC to run problems larger than GPU memory by paging data between host and device.
1069+
10331070
## Enumerations
10341071

10351072
### Boundary conditions {#boundary-conditions}

‎docs/index.html‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -55,7 +55,7 @@
5555
// Aerodynamics
5656
{ name: "Flow over an airfoil (vorticity)", image: "res/simulations/g.png", computer: "Delta", computerUrl: "https://www.ncsa.illinois.edu/research/project-highlights/delta/", accelerators: "128 A100s", walltime: "19h", source: "https://www.youtube.com/watch?v=FvAgnBW59cY" },
5757
{ name: "Pitching airfoil (3D)", image: "res/simulations/m.png", computer: "Phoenix", computerUrl: "https://www.pace.gatech.edu/", accelerators: "1 A100", walltime: "5h", source: "https://www.youtube.com/watch?v=2XH-9MumDHU" },
58-
{ name: "Flow over a corgi (pressure)", image: "res/simulations/u.png", computer: "HiPerGator", computerUrl: "https://www.rc.ufl.edu/about/hipergator/", accelerators: "4 B200s", walltime: "1h", source: "https://www.youtube.com/watch?v=IFNK3psPf3g" },
58+
{ name: "Mach 0.3 flow over a corgi (2M STL)", image: "res/simulations/u.png", computer: "HiPerGator", computerUrl: "https://www.rc.ufl.edu/about/hipergator/", accelerators: "2 GPUs", walltime: "80s", source: "https://www.youtube.com/watch?v=O8dSRqHLp_o" },
5959
// Shock-droplet
6060
{ name: "Shedding water droplet", image: "res/simulations/a.png", computer: "Summit", computerUrl: "https://www.olcf.ornl.gov/summit/", accelerators: "960 V100s", walltime: "4h", source: "https://www.youtube.com/watch?v=Gjj-qZkXcrg" },
6161
// Biomedical & acoustics

‎docs/res/simulations/u.png‎

1010 KB
Loading

0 commit comments

Comments
 (0)