Merge pull request #6330 from aritorto/useLevelCartIdxMapp

(Partial) Support for parallel runs with CpGrid with LGRs

Author: blattms

compareECLFiles.cmake

@@ -408,6 +408,13 @@ add_test_runSimulator(CASENAME spe1case1_carfin
 		      DIR lgr
 		      TEST_ARGS --parsing-strictness=low --enable-ecl-output=false --enable-vtk-output=true)
 
+add_test_runSimulator(CASENAME spe1case1_carfin_parallel
+                      FILENAME SPE1CASE1_CARFIN
+   	              SIMULATOR flow
+		      DIR lgr
+		      PROCS 4
+		      TEST_ARGS --parsing-strictness=low --enable-ecl-output=false --enable-vtk-output=true)
+
 # Tests that are run based on simulator results, but not necessarily direct comparison to reference results
 add_test_runSimulator(CASENAME tuning_xxxmbe
                       FILENAME 01_TUNING_XXXMBE

opm/simulators/flow/CollectDataOnIORank_impl.hpp

@@ -851,8 +851,8 @@ CollectDataOnIORank(const Grid& grid, const EquilGrid* equilGrid,
     : toIORankComm_(grid.comm())
     , globalInterRegFlows_(InterRegFlowMap::createMapFromNames(toVector(fipRegionsInterregFlow)))
 {
-    // index maps only have to be build when reordering is needed
-    if (!needsReordering && !isParallel())
+    // Build index maps only when reordering is needed; skip in parallel runs for CpGrid with LGRs
+    if ((!needsReordering && !isParallel()) || (isParallel() && (grid.maxLevel()>0)))
         return;
 
     const CollectiveCommunication& comm = grid.comm();

opm/simulators/flow/CpGridVanguard.hpp

@@ -263,19 +263,26 @@ class CpGridVanguard : public FlowBaseVanguard<TypeTag>
      */
     void addLgrs()
     {
-        // Check if input file contains Lgrs.
-        //
-        // If there are lgrs, create the grid with them, and update the leaf grid view.
-        if (const auto& lgrs = this->eclState().getLgrs(); (lgrs.size() > 0) && (this->grid_->comm().size() == 1)) {
+        // Check if input file contains Lgrs. Add them, if any.
+        // In a parallel run, this adds the LGRs on the distributed simulation grid.
+        if (const auto& lgrs = this->eclState().getLgrs(); lgrs.size() > 0) {
             OpmLog::info("\nAdding LGRs to the grid and updating its leaf grid view");
             this->addLgrsUpdateLeafView(lgrs, lgrs.size(), *this->grid_);
 
             this->updateGridView_();
             this->updateCellDepths_();
             this->updateCellThickness_();
-        }
-        else if (this->grid_->comm().size() > 1) {
-            OpmLog::warning("Adding LGRs in parallel run is not supported yet.\n");
+
+            if (this->grid_->comm().size()>1) {
+                // Add LGRs and update the leaf grid view in the global (undistributed) simulation grid.
+                // Purpose: To enable synchronization of cell ids in 'serial mode',
+                //          we rely on the "parent-to-children" cell id mapping.
+                OpmLog::info("\nAdding LGRs to the global view and updating its leaf grid view");
+                this->grid_->switchToGlobalView();
+                this->addLgrsUpdateLeafView(lgrs, lgrs.size(), *this->grid_);
+                this->grid_->switchToDistributedView();
+                this->grid_->syncDistributedGlobalCellIds();
+            }
         }
     }
 

opm/simulators/flow/GenericCpGridVanguard.cpp

@@ -33,6 +33,7 @@
 #include <opm/common/utility/ActiveGridCells.hpp>
 
 #include <opm/grid/cpgrid/GridHelpers.hpp>
+#include <opm/grid/cpgrid/LevelCartesianIndexMapper.hpp>
 
 #include <opm/input/eclipse/Schedule/Schedule.hpp>
 #include <opm/input/eclipse/Schedule/Well/Well.hpp>
@@ -306,7 +307,13 @@ distributeFieldProps_(EclipseState& eclState1)
     {
         // Reset Cartesian index mapper for automatic creation of field
         // properties
-        parallelEclState->resetCartesianMapper(this->cartesianIndexMapper_.get());
+        // Note: the previous cartesianIndexMapper_ has been replaces by levelCartesianIndexMapper_
+        // to support also the case of a distributed level zero grid in a CpGrid with LGRs.
+        // This change allows access to the Cartesian indices of the distributed level zero grid,
+        // where the field properties are given - for now.
+        // In case of supporting LGR field properties, this need to be adapted, to access instead
+        // each local/level Cartesian index set.
+        parallelEclState->resetCartesianMapper(this->levelCartesianIndexMapper_.get());
         parallelEclState->switchToDistributedProps();
     }
     else {
@@ -500,6 +507,7 @@ void GenericCpGridVanguard<ElementMapper,GridView,Scalar>::doCreateGrids_(Eclips
     }
 
     cartesianIndexMapper_ = std::make_unique<CartesianIndexMapper>(*grid_);
+    levelCartesianIndexMapper_ = std::make_unique<LevelCartesianIndexMapper>(*grid_);
 
 #if HAVE_MPI
     if (this->grid_->comm().size() > 1) {
@@ -640,7 +648,7 @@ template<class ElementMapper, class GridView, class Scalar>
 const LevelCartesianIndexMapper<Dune::CpGrid>
 GenericCpGridVanguard<ElementMapper,GridView,Scalar>::levelCartesianIndexMapper() const
 {
-    return LevelCartesianIndexMapper(*grid_);
+    return *levelCartesianIndexMapper_;
 }
 
 template<class ElementMapper, class GridView, class Scalar>

opm/simulators/flow/GenericCpGridVanguard.hpp

@@ -233,6 +233,7 @@ class GenericCpGridVanguard {
     std::unique_ptr<Dune::CpGrid> equilGrid_;
     std::unique_ptr<CartesianIndexMapper> cartesianIndexMapper_;
     std::unique_ptr<CartesianIndexMapper> equilCartesianIndexMapper_;
+    std::unique_ptr<LevelCartesianIndexMapper> levelCartesianIndexMapper_;
 
     int mpiRank;
     std::vector<int> cell_part_{};

opm/simulators/flow/OutputBlackoilModule.hpp

@@ -673,6 +673,11 @@ class OutputBlackOilModule : public GenericOutputBlackoilModule<GetPropType<Type
 
     void createLocalRegion_(std::vector<int>& region)
     {
+        // For CpGrid with LGRs, where level zero grid has been distributed,
+        // resize region is needed, since in this case the total amount of
+        // element - per process - in level zero grid and leaf grid do not
+        // coincide, in general.
+        region.resize(simulator_.gridView().size(0));
         std::size_t elemIdx = 0;
         for (const auto& elem : elements(simulator_.gridView())) {
             if (elem.partitionType() != Dune::InteriorEntity) {

opm/simulators/utils/ParallelEclipseState.hpp

@@ -102,9 +102,14 @@ class ParallelFieldPropsManager : public FieldPropsManager {
     template<class T>
     void resetCartesianMapper(const T* mapper)
     {
-        m_activeSize = std::bind(&T::compressedSize, mapper);
-        m_local2Global = std::bind(&T::cartesianIndex, mapper,
-                                   std::placeholders::_1);
+        // Note: mapper would usually be a CartesianIndexMapper. However, to support also
+        // the case of a distributed level zero grid in a CpGrid with LGRs, mapper will be
+        // Opm::LevelCartesianIndexMapper. This change allows access to the Cartesian indices
+        // of the distributed level zero grid, where the field properties are given - for now.
+        // In case of supporting LGR field properties, this need to be adapted, to access instead
+        // each local/level Cartesian index set.
+        m_activeSize = [mapper]() { return mapper->compressedSize(/*level = */ 0); };
+        m_local2Global = [mapper](int localIdx) { return mapper->cartesianIndex(localIdx, /* level = */ 0); };
     }
 
     bool tran_active(const std::string& keyword) const override;