Make using The_Comms_Arena() for particle communication opt-in (#4286)

For mesh data it is better to use a small, separate Arena for
communication. But, we found that for particle communication this
approach uses too much memory (see the ImpactX Issue here:
ECP-WarpX/impactx#769)

This restores the old default behavior (prior to PR #4175), but allows
users to opt-in to using The_Comms_Arena if desired.

The proposed changes:
- [ ] fix a bug or incorrect behavior in AMReX
- [ ] add new capabilities to AMReX
- [ ] changes answers in the test suite to more than roundoff level
- [ ] are likely to significantly affect the results of downstream AMReX
users
- [ ] include documentation in the code and/or rst files, if appropriate

Src/Particle/AMReX_NeighborParticlesGPUImpl.H

@@ -256,7 +256,7 @@ updateNeighborsGPU (bool boundary_neighbors_only)
 
     clearNeighbors();
 
-    if (ParallelDescriptor::UseGpuAwareMpi()) {
+    if (this->use_comms_arena) {
         snd_buffer.setArena(The_Comms_Arena());
         rcv_buffer.setArena(The_Comms_Arena());
     }

Src/Particle/AMReX_ParticleContainerBase.H

@@ -256,6 +256,7 @@ public:
     static AMREX_EXPORT bool do_tiling;
     static AMREX_EXPORT IntVect tile_size;
     static AMREX_EXPORT bool memEfficientSort;
+    static AMREX_EXPORT bool use_comms_arena;
     mutable AmrParticleLocator<DenseBins<Box> > m_particle_locator;
 
 protected:

Src/Particle/AMReX_ParticleContainerBase.cpp

@@ -10,6 +10,7 @@ using namespace amrex;
 bool    ParticleContainerBase::do_tiling = false;
 IntVect ParticleContainerBase::tile_size { AMREX_D_DECL(1024000,8,8) };
 bool    ParticleContainerBase::memEfficientSort = true;
+bool    ParticleContainerBase::use_comms_arena = false;
 
 void ParticleContainerBase::Define (const Geometry            & geom,
                                     const DistributionMapping & dmap,

Src/Particle/AMReX_ParticleContainerI.H

@@ -63,6 +63,7 @@ ParticleContainer_impl<ParticleType, NArrayReal, NArrayInt, Allocator, CellAssig
         pp.query("use_prepost", usePrePost);
         pp.query("do_unlink", doUnlink);
         pp.queryAdd("do_mem_efficient_sort", memEfficientSort);
+        pp.queryAdd("use_comms_arena", use_comms_arena);
 
         // add default names for SoA Real and Int compile-time arguments
         for (int i=0; i<NArrayReal; ++i)
@@ -1392,7 +1393,7 @@ ParticleContainer_impl<ParticleType, NArrayReal, NArrayInt, Allocator, CellAssig
     amrex::PODVector<char, PolymorphicArenaAllocator<char> > snd_buffer;
     amrex::PODVector<char, PolymorphicArenaAllocator<char> > rcv_buffer;
 
-    if (ParallelDescriptor::UseGpuAwareMpi()) {
+    if (use_comms_arena) {
         snd_buffer.setArena(The_Comms_Arena());
         rcv_buffer.setArena(The_Comms_Arena());
     }