Cuda sn solver (#63)

* fix lattice perimeter definitions

* fix max outflow MPI

* fix output of varAbsorption

* fixed rms bug with mpi exec

* fix linspace vulnerability

* fixed MPI flag

* recomputation of scalar flux missing after heun averaging

* more guardrails around greenprobing

* made the probing blocks move with the center of the capsule

* update restart sim time

* fixed validation tests

* stage 1 cuda fvm solver

* cuda solver validation ok

* lattice runs, but has some artifacts near boundary

* update installers

* add singularity cuda installer

* update reame

Author: ScSteffen

CMakeLists.txt

@@ -9,12 +9,24 @@ option( BUILD_UNITY "enables unity build for faster compile times" ON )
 option( BUILD_CODE_COV "enables compiler option required for code coverage analysis" OFF )
 option( BUILD_ML "enables build with tensorflow backend access" OFF )
 option( BUILD_MPI "enables build with MPI access" OFF )
+option( BUILD_CUDA_HPC "enables CUDA backend for SN HPC solver (single GPU)" OFF )
 #################################################
 
 
 ### COMPILER ####################################
 set( CMAKE_CXX_STANDARD 17 )
 set( CMAKE_CXX_STANDARD_REQUIRED ON )
+if( BUILD_CUDA_HPC )
+    enable_language( CUDA )
+    if( CMAKE_VERSION VERSION_LESS 3.18 )
+        # CMake 3.16 does not ship CUDA17 mappings by default.
+        set( CMAKE_CUDA17_STANDARD_COMPILE_OPTION "--std=c++17" )
+        set( CMAKE_CUDA17_EXTENSION_COMPILE_OPTION "--std=c++17" )
+    endif()
+    set( CMAKE_CUDA_STANDARD 17 )
+    set( CMAKE_CUDA_STANDARD_REQUIRED ON )
+    set( CMAKE_CUDA_EXTENSIONS OFF )
+endif()
 set( KITRT_RELEASE_OPTIONS -march=native -w )
 set( KITRT_RELWITHDEBINFO_OPTIONS -march=native -pg -no-pie )
 set( KITRT_DEBUG_OPTIONS -Wall -Wextra -Wpedantic )
@@ -30,6 +42,21 @@ set(CMAKE_MODULE_PATH ${PROJECT_SOURCE_DIR}/cmake ${CMAKE_MODULE_PATH})
 
 find_package( OpenMP REQUIRED )
 
+if( BUILD_CUDA_HPC )
+    add_compile_definitions( KITRT_ENABLE_CUDA_HPC )
+    if( CMAKE_VERSION VERSION_GREATER_EQUAL 3.17 )
+        find_package( CUDAToolkit QUIET )
+    endif()
+    if( DEFINED CMAKE_CUDA_TOOLKIT_INCLUDE_DIRECTORIES )
+        include_directories( ${CMAKE_CUDA_TOOLKIT_INCLUDE_DIRECTORIES} )
+    elseif( EXISTS "/usr/local/cuda/include" )
+        include_directories( /usr/local/cuda/include )
+    endif()
+    message( STATUS "CUDA HPC solver: enabled" )
+else()
+    message( STATUS "CUDA HPC solver: disabled" )
+endif()
+
 message(STATUS "MPI build flag: ${BUILD_MPI}")
 if( BUILD_MPI )
     add_definitions(-DIMPORT_MPI)
@@ -72,6 +99,20 @@ else()
     set( CORE_LIBRARIES ${LAPACK_LIBRARIES} ${BLAS_LIBRARIES}  ${VTK_LIBRARIES} OpenMP::OpenMP_CXX -lstdc++fs )
 endif()
 
+if( BUILD_CUDA_HPC )
+    if( TARGET CUDA::cudart )
+        list( APPEND CORE_LIBRARIES CUDA::cudart )
+    else()
+        find_library( CUDART_LIBRARY
+                      NAMES cudart
+                      HINTS ${CMAKE_CUDA_IMPLICIT_LINK_DIRECTORIES} /usr/local/cuda/lib64 /usr/lib/x86_64-linux-gnu )
+        if( NOT CUDART_LIBRARY )
+            message( FATAL_ERROR "Could not find CUDA runtime library (cudart)." )
+        endif()
+        list( APPEND CORE_LIBRARIES ${CUDART_LIBRARY} )
+    endif()
+endif()
+
 
 #################################################
 
@@ -89,6 +130,12 @@ add_compile_definitions( GIT_HASH="${GIT_HASH}" )
 
 ### BUILD KIT-RT ################################
 file( GLOB_RECURSE SRCS RELATIVE ${CMAKE_SOURCE_DIR} "src/*.cpp" "include/*.h" )
+if( BUILD_CUDA_HPC )
+    list( APPEND SRCS "src/solvers/snsolver_hpc.cu" "include/solvers/snsolver_hpc_cuda.hpp" )
+    # Force nvcc to parse this translation unit as C++17 even if transitive
+    # imported target features inject an older fallback standard.
+    set_source_files_properties( src/solvers/snsolver_hpc.cu PROPERTIES COMPILE_FLAGS "--std=c++17" )
+endif()
 set( EXCLUDE_DIR "/gui/" )
 foreach( TMP_PATH ${SRCS} )
     string( FIND ${TMP_PATH} ${EXCLUDE_DIR} EXCLUDE_DIR_FOUND )
@@ -110,9 +157,15 @@ if( BUILD_ML )
     target_link_libraries( ${CMAKE_PROJECT_NAME} ${CORE_LIBRARIES} ${TENSORFLOW_LIB})
 endif()
 target_link_libraries( ${CMAKE_PROJECT_NAME} ${CORE_LIBRARIES} )
-target_compile_options( ${CMAKE_PROJECT_NAME} PUBLIC "$<$<CONFIG:DEBUG>:${KITRT_DEBUG_OPTIONS}>" )
-target_compile_options( ${CMAKE_PROJECT_NAME} PUBLIC "$<$<CONFIG:RELWITHDEBINFO>:${KITRT_RELWITHDEBINFO_OPTIONS}>" )
-target_compile_options( ${CMAKE_PROJECT_NAME} PUBLIC "$<$<CONFIG:RELEASE>:${KITRT_RELEASE_OPTIONS}>" )
+if( BUILD_CUDA_HPC )
+    set_target_properties(
+        ${CMAKE_PROJECT_NAME}
+        PROPERTIES CUDA_STANDARD 17 CUDA_STANDARD_REQUIRED ON CUDA_EXTENSIONS OFF
+    )
+endif()
+target_compile_options( ${CMAKE_PROJECT_NAME} PUBLIC "$<$<AND:$<COMPILE_LANGUAGE:CXX>,$<CONFIG:DEBUG>>:${KITRT_DEBUG_OPTIONS}>" )
+target_compile_options( ${CMAKE_PROJECT_NAME} PUBLIC "$<$<AND:$<COMPILE_LANGUAGE:CXX>,$<CONFIG:RELWITHDEBINFO>>:${KITRT_RELWITHDEBINFO_OPTIONS}>" )
+target_compile_options( ${CMAKE_PROJECT_NAME} PUBLIC "$<$<AND:$<COMPILE_LANGUAGE:CXX>,$<CONFIG:RELEASE>>:${KITRT_RELEASE_OPTIONS}>" )
 #################################################
 
 ### BUILD UNIT TESTS ############################
@@ -145,20 +198,26 @@ if( BUILD_TESTING )
     target_compile_definitions( unit_tests PUBLIC BUILD_TESTING )
     target_compile_definitions( unit_tests PUBLIC TESTS_PATH="${CMAKE_SOURCE_DIR}/tests/" )
     target_link_libraries( unit_tests Catch ${CORE_LIBRARIES} )
-    target_compile_options( unit_tests PUBLIC "$<$<CONFIG:DEBUG>:${KITRT_DEBUG_OPTIONS}>" )
+    if( BUILD_CUDA_HPC )
+        set_target_properties(
+            unit_tests
+            PROPERTIES CUDA_STANDARD 17 CUDA_STANDARD_REQUIRED ON CUDA_EXTENSIONS OFF
+        )
+    endif()
+    target_compile_options( unit_tests PUBLIC "$<$<AND:$<COMPILE_LANGUAGE:CXX>,$<CONFIG:DEBUG>>:${KITRT_DEBUG_OPTIONS}>" )
     if( BUILD_CODE_COV )
         if( CMAKE_COMPILER_IS_GNUCXX )
             set( CODE_COVERAGE_OPTIONS --coverage -g -O0 -w )
-            target_compile_options( unit_tests PUBLIC "$<$<CONFIG:DEBUG>:${CODE_COVERAGE_OPTIONS}>" )
+            target_compile_options( unit_tests PUBLIC "$<$<AND:$<COMPILE_LANGUAGE:CXX>,$<CONFIG:DEBUG>>:${CODE_COVERAGE_OPTIONS}>" )
             target_link_libraries( unit_tests Catch gcov )
         else()
             message( FATAL_ERROR "Code coverage is currently only supported for gcc!" )
         endif()
     else()
         target_link_libraries( unit_tests Catch )
     endif()
-    target_compile_options( unit_tests PUBLIC "$<$<CONFIG:RELWITHDEBINFO>:${KITRT_RELWITHDEBINFO_OPTIONS}>" )
-    target_compile_options( unit_tests PUBLIC "$<$<CONFIG:RELEASE>:${KITRT_RELEASE_OPTIONS}>" )
+    target_compile_options( unit_tests PUBLIC "$<$<AND:$<COMPILE_LANGUAGE:CXX>,$<CONFIG:RELWITHDEBINFO>>:${KITRT_RELWITHDEBINFO_OPTIONS}>" )
+    target_compile_options( unit_tests PUBLIC "$<$<AND:$<COMPILE_LANGUAGE:CXX>,$<CONFIG:RELEASE>>:${KITRT_RELEASE_OPTIONS}>" )
     catch_discover_tests( unit_tests )
 endif()
 #################################################
@@ -179,9 +238,9 @@ if( BUILD_GUI )
     set( GUI_LIBRARIES Qt5::Core Qt5::Widgets Qt5::Gui Qt5::OpenGL ${VTK_LIBRARIES} )
     target_link_libraries( ${CMAKE_PROJECT_NAME}_gui ${CORE_LIBRARIES} ${GUI_LIBRARIES} )
     set_target_properties( ${CMAKE_PROJECT_NAME}_gui PROPERTIES RUNTIME_OUTPUT_DIRECTORY "${CMAKE_SOURCE_DIR}/bin" )
-    target_compile_options( ${CMAKE_PROJECT_NAME}_gui PUBLIC "$<$<CONFIG:DEBUG>:${KITRT_DEBUG_OPTIONS}>" )
-    target_compile_options( ${CMAKE_PROJECT_NAME}_gui PUBLIC "$<$<CONFIG:RELWITHDEBINFO>:${KITRT_RELWITHDEBINFO_OPTIONS}>" )
-    target_compile_options( ${CMAKE_PROJECT_NAME}_gui PUBLIC "$<$<CONFIG:RELEASE>:${KITRT_RELEASE_OPTIONS}>" )
+    target_compile_options( ${CMAKE_PROJECT_NAME}_gui PUBLIC "$<$<AND:$<COMPILE_LANGUAGE:CXX>,$<CONFIG:DEBUG>>:${KITRT_DEBUG_OPTIONS}>" )
+    target_compile_options( ${CMAKE_PROJECT_NAME}_gui PUBLIC "$<$<AND:$<COMPILE_LANGUAGE:CXX>,$<CONFIG:RELWITHDEBINFO>>:${KITRT_RELWITHDEBINFO_OPTIONS}>" )
+    target_compile_options( ${CMAKE_PROJECT_NAME}_gui PUBLIC "$<$<AND:$<COMPILE_LANGUAGE:CXX>,$<CONFIG:RELEASE>>:${KITRT_RELEASE_OPTIONS}>" )
 endif()
 #################################################
 

README.md

@@ -16,7 +16,7 @@ KiT-RT is an open-source, multi-fidelity **C++ PDE solver for radiative transpor
 
 * **Modular, HPC-ready** architecture. Supports hybrid **MPI/OpenMP distributed parallelism**.
 * **Containerized** for portable deployment across HPC systems (Docker & Singularity).
-* **Python-wrapped** via [CharmKIT](https://github.com/KiT-RT/CharmKIT)
+* **Python-wrapped** via [charm_kit](https://github.com/KiT-RT/charm_kit)
 * Downstream applications:
   - Data generation for scientific **foundation models**.
   - high-resolution **reference solutions** for AI-based surrogate modeling.
@@ -66,79 +66,144 @@ Applications include:
 
 
 
-## Installation
+## Installation and Run (By Parallelization)
 
-### Plain cpp setup
+One-time setup:
 ```bash
-# Clone repository
 git clone https://github.com/KiT-RT/kitrt_code.git
 cd kitrt_code
 git submodule update --init --recursive
+```
 
-# Build with CMake
-mkdir build && cd build
-cmake -DCMAKE_BUILD_TYPE=Release ../
-make -j
+Then run all commands from the repository root.
 
-```
-### Docker setup 
-A preconfigured docker container can also be used to run the code.
-By running
+### 1. CPU (OpenMP only)
 
+#### 1a) Plain installation (no container)
 ```bash
-docker run --rm -ti -v $(pwd):/home kitrt/test:latest
+mkdir -p build_omp
+cd build_omp
+cmake -DCMAKE_BUILD_TYPE=Release -DBUILD_MPI=OFF -DBUILD_CUDA_HPC=OFF -DBUILD_ML=OFF ..
+make -j
+cd ..
+./build_omp/KiT-RT tests/input/validation_tests/SN_solver/checkerboard_SN.cfg
 ```
 
-Bash scripts are provided in the folder tools/CI to get started with the docker environments. To start an interactive docker environment, execute
+#### 1b) Docker installation
 ```bash
-docker run -i -t --rm -v $(pwd)/../..:/mnt kitrt/test:latest /bin/bash
+docker run --rm -it -v $(pwd):/mnt -w /mnt kitrt/test:latest /bin/bash
+mkdir -p build_docker_omp
+cd build_docker_omp
+cmake -DCMAKE_BUILD_TYPE=Release -DBUILD_MPI=OFF -DBUILD_CUDA_HPC=OFF -DBUILD_ML=OFF ..
+make -j
+cd ..
+./build_docker_omp/KiT-RT tests/input/validation_tests/SN_solver/checkerboard_SN.cfg
 ```
 
-### Singularity setup 
-Create the singularity container
+#### 1c) Singularity installation
 ```bash
-mkdir build_singularity
 cd tools/singularity
-sudo sh build_container.sh
-chmod +x install_kitrt_singularity.sh
-singularity exec kit_rt.sif ./install_kitrt_singularity.sh
+sudo singularity build kit_rt.sif kit_rt.def
+cd ../..
+mkdir -p build_singularity_omp
+cd build_singularity_omp
+singularity exec ../tools/singularity/kit_rt.sif \
+  cmake -DCMAKE_BUILD_TYPE=Release -DBUILD_MPI=OFF -DBUILD_CUDA_HPC=OFF -DBUILD_ML=OFF ..
+singularity exec ../tools/singularity/kit_rt.sif make -j
+cd ..
+singularity exec tools/singularity/kit_rt.sif \
+  ./build_singularity_omp/KiT-RT tests/input/validation_tests/SN_solver/checkerboard_SN.cfg
+```
+
+### 2. CPU (OpenMP + MPI)
+
+#### 2a) Plain installation (no container)
+```bash
+mkdir -p build_mpi
+cd build_mpi
+cmake -DCMAKE_BUILD_TYPE=Release -DBUILD_MPI=ON -DBUILD_CUDA_HPC=OFF -DBUILD_ML=OFF ..
+make -j
+cd ..
+mpirun -np 4 ./build_mpi/KiT-RT tests/input/validation_tests/SN_solver_hpc/lattice_hpc_200_cpu_order2.cfg
 ```
-Run the singularity container
+
+#### 2b) Singularity installation
 ```bash
-singularity shell --bind $(pwd)/../..:/mnt kit_rt.sif
+cd tools/singularity
+sudo singularity build kit_rt_MPI.sif kit_rt_MPI.def
+cd ../..
+mkdir -p build_singularity_mpi
+cd build_singularity_mpi
+singularity exec ../tools/singularity/kit_rt_MPI.sif \
+  cmake -DCMAKE_BUILD_TYPE=Release -DBUILD_MPI=ON -DBUILD_CUDA_HPC=OFF -DBUILD_ML=OFF ..
+singularity exec ../tools/singularity/kit_rt_MPI.sif make -j
+cd ..
+singularity exec tools/singularity/kit_rt_MPI.sif \
+  mpirun -np 4 ./build_singularity_mpi/KiT-RT tests/input/validation_tests/SN_solver_hpc/lattice_hpc_200_cpu_order2.cfg
 ```
 
-## Running simulations
-Within any of the above setups, navigate to the example folder and execute KiT-RT
+### 3. CPU + single GPU (OpenMP + CUDA)
+
+#### 3a) Singularity installation
 ```bash
-cd examples
-../<build_folder_name>/KiT-RT configs/lattice_SN.cfg
+cd tools/singularity
+sudo singularity build kit_rt_MPI_cuda.sif kit_rt_MPI_cuda.def
+cd ../..
+mkdir -p build_singularity_cuda
+cd build_singularity_cuda
+singularity exec --nv ../tools/singularity/kit_rt_MPI_cuda.sif \
+  cmake -DCMAKE_BUILD_TYPE=Release -DBUILD_MPI=OFF -DBUILD_CUDA_HPC=ON -DBUILD_ML=OFF ..
+singularity exec --nv ../tools/singularity/kit_rt_MPI_cuda.sif make -j
+cd ..
+singularity exec --nv tools/singularity/kit_rt_MPI_cuda.sif \
+  ./build_singularity_cuda/KiT-RT tests/input/validation_tests/SN_solver_hpc/lattice_hpc_200_cuda_order2.cfg
 ```
 
-## Tensorflow backend
-If you choose to enable the integrated machine learning tools via the BUILD_ML option, you need to install the tensorflow C-API:
+When compiled with `-DBUILD_CUDA_HPC=ON`, HPC runs use the CUDA backend if a GPU is visible, and fall back to CPU if no GPU is detected.
+
+### 4. Build with TensorFlow backend (CPU + OpenMP only)
+
 ```bash
 FILENAME=libtensorflow-cpu-linux-x86_64-2.7.0.tar.gz
 wget -q --no-check-certificate https://storage.googleapis.com/tensorflow/libtensorflow/${FILENAME}
-tar -C /usr/local -xzf ${FILENAME}
-ldconfig /usr/local/lib
+sudo tar -C /usr/local -xzf ${FILENAME}
+sudo ldconfig /usr/local/lib
+rm ${FILENAME}
+mkdir -p build_ml
+cd build_ml
+cmake -DCMAKE_BUILD_TYPE=Release -DBUILD_MPI=OFF -DBUILD_CUDA_HPC=OFF -DBUILD_ML=ON ..
+make -j
+cd ..
+./build_ml/KiT-RT tests/input/validation_tests/MN_solver/checkerboard_MN_neural.cfg
 ```
-and for a gpu based version (you need supported hardware and gpu drivers, see here ):
+
+### 5. Debug build
+
 ```bash
-FILENAME=libtensorflow-gpu-linux-x86_64-2.7.0.tar.gz
-wget -q --no-check-certificate https://storage.googleapis.com/tensorflow/libtensorflow/${FILENAME}
-tar -C /usr/local -xzf ${FILENAME}
-ldconfig /usr/local/lib
+mkdir -p build_debug
+cd build_debug
+cmake -DCMAKE_BUILD_TYPE=Debug -DBUILD_MPI=OFF -DBUILD_CUDA_HPC=OFF -DBUILD_ML=OFF ..
+make -j
+cd ..
+./build_debug/KiT-RT tests/input/validation_tests/SN_solver/checkerboard_SN.cfg
 ```
-Or use the docker container
+
+### 6. Test + coverage build
+
 ```bash
-docker run --rm -ti -v $(pwd):/home kitrt/test_ml:latest
+mkdir -p build_coverage
+cd build_coverage
+cmake -DCMAKE_BUILD_TYPE=Debug -DBUILD_TESTING=ON -DBUILD_CODE_COV=ON -DBUILD_UNITY=OFF ..
+make -j
+./unit_tests
+ctest --output-on-failure
+gcovr -r .. --html-details coverage.html
 ```
 
 
 ## Python API
 
-The Python interface is provided via [CharmKIT](https://github.com/KiT-RT/CharmKIT), allowing seamless integration into AI and outer-loop (UQ, Optimization) workflows. 
+The Python interface is provided via [charm_kit](https://github.com/KiT-RT/charm_kit), allowing seamless integration into AI and outer-loop (UQ, Optimization) workflows. 
 Check the corresponding readme for further info
 
 

include/solvers/snsolver_hpc.hpp

@@ -80,7 +80,8 @@ class SNSolverHPC
     std::vector<double> _flux; /*!< @brief dim = _nCells x _nSys */
 
     // Output related members
-    std::vector<double> _scalarFlux; /*!< @brief dim = _nCells  */
+    std::vector<double> _scalarFlux;         /*!< @brief dim = _nCells  */
+    std::vector<double> _scalarFluxPrevIter; /*!< @brief scalar flux snapshot before current solver iteration */
 
     // Lattice QOIS
     unsigned _nOutputMoments;